From 8027310320a3729a43dd4dddf41da9553078f8f1 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Sun, 10 Mar 2024 14:37:39 -0700 Subject: [PATCH] Implement JournalIntegritySaveDataFileSystemDriver --- .../FsSystem/Buffers/BufferManagerUtility.cs | 12 +- .../JournalIntegritySaveDataFileSystem.cs | 22 +- ...ournalIntegritySaveDataFileSystemDriver.cs | 269 ++++++++++++++++-- ...yFileSystemWithRetryingBufferAllocation.cs | 2 +- .../Save/SaveDataInternalStorageFileSystem.cs | 2 +- src/LibHac/FsSystem/UnionStorage.cs | 8 +- src/LibHac/LibHac.csproj | 5 + 7 files changed, 274 insertions(+), 46 deletions(-) diff --git a/src/LibHac/FsSystem/Buffers/BufferManagerUtility.cs b/src/LibHac/FsSystem/Buffers/BufferManagerUtility.cs index c83f3aa0..e6d9eba1 100644 --- a/src/LibHac/FsSystem/Buffers/BufferManagerUtility.cs +++ b/src/LibHac/FsSystem/Buffers/BufferManagerUtility.cs @@ -1,6 +1,7 @@ using System; using System.Runtime.CompilerServices; using System.Threading; +using JetBrains.Annotations; using LibHac.Diag; using LibHac.Fs; using Buffer = LibHac.Mem.Buffer; @@ -19,8 +20,7 @@ public struct ScopedBufferManagerContextRegistration : IDisposable { private BufferManagerContext _oldContext; - // ReSharper disable once UnusedParameter.Local - public ScopedBufferManagerContextRegistration(int unused = default) + public ScopedBufferManagerContextRegistration() { _oldContext = BufferManagerUtility.GetBufferManagerContext(); } @@ -41,15 +41,15 @@ internal static class BufferManagerUtility public delegate bool IsValidBufferFunction(in Buffer buffer); - public static Result DoContinuouslyUntilBufferIsAllocated(Func function, Func onFailure, - [CallerMemberName] string callerName = "") + public static Result DoContinuouslyUntilBufferIsAllocated([InstantHandle] Func function, + [InstantHandle] Func onFailure, [CallerMemberName] string callerName = "") { const int bufferAllocationRetryLogCountMax = 10; const int bufferAllocationRetryLogInterval = 100; Result result; - for (int count = 1; ; count++) + for (int count = 1;; count++) { result = function(); if (!ResultFs.BufferAllocationFailed.Includes(result)) @@ -71,7 +71,7 @@ internal static class BufferManagerUtility return result; } - public static Result DoContinuouslyUntilBufferIsAllocated(Func function, + public static Result DoContinuouslyUntilBufferIsAllocated([InstantHandle] Func function, [CallerMemberName] string callerName = "") { return DoContinuouslyUntilBufferIsAllocated(function, static () => Result.Success, callerName); diff --git a/src/LibHac/FsSystem/Save/JournalIntegritySaveDataFileSystem.cs b/src/LibHac/FsSystem/Save/JournalIntegritySaveDataFileSystem.cs index d9ca704e..e3cde5f8 100644 --- a/src/LibHac/FsSystem/Save/JournalIntegritySaveDataFileSystem.cs +++ b/src/LibHac/FsSystem/Save/JournalIntegritySaveDataFileSystem.cs @@ -125,9 +125,15 @@ file static class Anonymous public class JournalIntegritySaveDataFileSystem : IFileSystem { - public struct ExtraData { } + public struct ExtraData + { + public int Dummy; + } - public struct FileSystemLayoutHeader { } + public struct FileSystemLayoutHeader + { + public int Dummy; + } public struct MasterHeader { @@ -136,7 +142,9 @@ public class JournalIntegritySaveDataFileSystem : IFileSystem public long Counter; } - public struct CommitData2 { } + public struct CommitData2 + { + } } public class ControlAreaHolder : IDisposable @@ -435,10 +443,12 @@ public class JournalIntegritySaveDataFileSystem : IFileSystem throw new NotImplementedException(); } - public static Result QuerySize(out long outSizeTotal, long sizeBlock, int countAvailableBlock, - int countJournalBlock, int countExpandMax, uint minimumVersion) + public static Result QuerySize(out long outSizeTotal, long sizeBlock, uint countAvailableBlock, + uint countJournalBlock, int countExpandMax, uint minimumVersion) { - throw new NotImplementedException(); + // Todo: Implement + outSizeTotal = 0; + return Result.Success; } private static Result Sign(Span outBuffer, IMacGenerator macGenerator, diff --git a/src/LibHac/FsSystem/Save/JournalIntegritySaveDataFileSystemDriver.cs b/src/LibHac/FsSystem/Save/JournalIntegritySaveDataFileSystemDriver.cs index 7c56d967..b36885cb 100644 --- a/src/LibHac/FsSystem/Save/JournalIntegritySaveDataFileSystemDriver.cs +++ b/src/LibHac/FsSystem/Save/JournalIntegritySaveDataFileSystemDriver.cs @@ -1,12 +1,17 @@ -// ReSharper disable UnusedMember.Local UnusedType.Local -#pragma warning disable CS0169 // Field is never used -using System; +using LibHac.Common; +using LibHac.Diag; using LibHac.Fs; +using LibHac.FsSystem.Buffers; using LibHac.Os; using LibHac.Util; namespace LibHac.FsSystem.Save; +/// +/// Provides access to a . Abstracts some operations such as expansion, +/// and retries operations if buffer allocation fails. +/// +/// Based on nnSdk 17.5.0 (FS 17.0.0) public class JournalIntegritySaveDataFileSystemDriver : ProxyFileSystemWithRetryingBufferAllocation, IInternalStorageFileSystem { private ValueSubStorage _baseStorage; @@ -17,26 +22,70 @@ public class JournalIntegritySaveDataFileSystemDriver : ProxyFileSystemWithRetry public JournalIntegritySaveDataFileSystemDriver() { - throw new NotImplementedException(); + _baseStorage = new ValueSubStorage(); + _mutex = new SdkRecursiveMutex(); + _fileSystem = new JournalIntegritySaveDataFileSystem(); } public override void Dispose() { - throw new NotImplementedException(); + FinalizeObject(); + _fileSystem.Dispose(); + _baseStorage.Dispose(); + base.Dispose(); } - public static Result QueryDataBlockCount(out uint outCountDataBlock, long blockSize, long availableSize, + public static Result QueryDataBlockCount(out uint outCountDataBlock, long blockSize, long totalSize, long journalSize, int countExpandMax, uint version) { - throw new NotImplementedException(); + outCountDataBlock = 0; + + uint lowerBound = 0; + uint upperBound = (uint)(totalSize / blockSize); + uint journalBlockCount = (uint)(journalSize / blockSize); + + int iterationsRemaining = 32; + + // Do a binary search to find the maximum number of data blocks you can have without going over the total + // save image size limit. + uint midpoint; + do + { + if (upperBound == lowerBound) + break; + if (iterationsRemaining-- == 0) + break; + + midpoint = (upperBound + lowerBound) / 2; + Result res = QueryTotalSize(out long sizeQuery, blockSize, midpoint, journalBlockCount, countExpandMax, version); + if (res.IsFailure()) return res.Miss(); + + if (sizeQuery > totalSize) + upperBound = midpoint; + else + lowerBound = midpoint; + } while (upperBound - lowerBound > 1 || midpoint != lowerBound); + + { + Result res = QueryTotalSize(out long sizeQuery, blockSize, lowerBound, journalBlockCount, countExpandMax, version); + if (res.IsFailure()) return res.Miss(); + + if (sizeQuery > totalSize) + { + return ResultFs.UsableSpaceNotEnough.Log(); + } + + outCountDataBlock = lowerBound; + } + + return Result.Success; } public static Result QueryTotalSize(out long outSizeTotal, long blockSize, uint countAvailableBlock, uint countJournalBlock, int countExpandMax, uint version) { - // Todo: Implement - outSizeTotal = 0; - return Result.Success; + return JournalIntegritySaveDataFileSystem.QuerySize(out outSizeTotal, blockSize, countAvailableBlock, + countJournalBlock, countExpandMax, version).Ret(); } public static Result Format( @@ -52,12 +101,33 @@ public class JournalIntegritySaveDataFileSystemDriver : ProxyFileSystemWithRetry RandomDataGenerator encryptionKeyGenerator, uint version) { - throw new NotImplementedException(); + // Create a local copy of the SubStorage so it can be captured for use in a local function or lambda + using var tempSaveFileStorage = new ValueSubStorage(in saveFileStorage); + + var locker = new SdkRecursiveMutex(); + + var bufferManagerSet = new FileSystemBufferManagerSet(); + for (int i = 0; i < bufferManagerSet.Buffers.Length; i++) + { + bufferManagerSet.Buffers[i] = bufferManager; + } + + return BufferManagerUtility.DoContinuouslyUntilBufferIsAllocated(FormatImpl).Ret(); + + Result FormatImpl() + { + using var scopedRegistration = new ScopedBufferManagerContextRegistration(); + + return JournalIntegritySaveDataFileSystem.Format(in tempSaveFileStorage, blockSize, blockCount, + journalBlockCount, countExpandMax, bufferManagerSet, bufferManager, locker, macGenerator, + hashGeneratorFactorySelector, hashSalt, encryptionKeyGenerator, version).Ret(); + } } public static long QueryExpandLogSize(long blockSize, uint availableBlockCount, uint journalBlockCount) { - throw new NotImplementedException(); + long dataSize = ((long)journalBlockCount + availableBlockCount) * blockSize; + return BitUtil.DivideUp(dataSize, 0x2000000) * 0x10000 + 0x100000; } public static Result OperateExpand( @@ -71,7 +141,49 @@ public class JournalIntegritySaveDataFileSystemDriver : ProxyFileSystemWithRetry IHash256GeneratorFactorySelector hashGeneratorFactorySelector, uint minimumVersion) { - throw new NotImplementedException(); + using var tempBaseStorage = new ValueSubStorage(in baseStorage); + using var tempLogStorage = new ValueSubStorage(in logStorage); + + var locker = new SdkRecursiveMutex(); + + return BufferManagerUtility.DoContinuouslyUntilBufferIsAllocated(OperateImpl).Ret(); + + Result OperateImpl() + { + using var scopedRegistration = new ScopedBufferManagerContextRegistration(); + using var unionStorage = new UnionStorage(); + + Result res = tempLogStorage.GetSize(out long logStorageSize); + if (res.IsFailure()) return res.Miss(); + + using var storageBufferingLog = new BufferedStorage(); + res = storageBufferingLog.Initialize(in tempLogStorage, bufferManager, (int)blockSize, bufferCount: 2); + if (res.IsFailure()) return res.Miss(); + + using var bufferedLogStorage = new ValueSubStorage(storageBufferingLog, 0, logStorageSize); + + res = UnionStorage.Format(in bufferedLogStorage, blockSize); + if (res.IsFailure()) return res.Miss(); + + res = unionStorage.Initialize(in tempBaseStorage, in bufferedLogStorage, blockSize); + if (res.IsFailure()) return res.Miss(); + + res = unionStorage.GetSize(out long sizeUnionStorage); + if (res.IsFailure()) return res.Miss(); + + using var tempUnionStorage = new ValueSubStorage(unionStorage, 0, sizeUnionStorage); + res = JournalIntegritySaveDataFileSystem.Expand(in tempUnionStorage, availableBlockCount, journalBlockCount, + bufferManager, locker, macGenerator, hashGeneratorFactorySelector, minimumVersion); + if (res.IsFailure()) return res.Miss(); + + res = unionStorage.Freeze(); + if (res.IsFailure()) return res.Miss(); + + res = unionStorage.Flush(); + if (res.IsFailure()) return res.Miss(); + + return Result.Success; + } } public static Result CommitExpand( @@ -80,61 +192,150 @@ public class JournalIntegritySaveDataFileSystemDriver : ProxyFileSystemWithRetry long blockSize, IBufferManager bufferManager) { - throw new NotImplementedException(); + using var tempBaseStorage = new ValueSubStorage(in baseStorage); + using var tempLogStorage = new ValueSubStorage(in logStorage); + + return BufferManagerUtility.DoContinuouslyUntilBufferIsAllocated(CommitImpl).Ret(); + + Result CommitImpl() + { + using var scopedRegistration = new ScopedBufferManagerContextRegistration(); + using var unionStorage = new UnionStorage(); + + Result res = tempBaseStorage.GetSize(out long baseStorageSize); + if (res.IsFailure()) return res.Miss(); + + using var storageBufferingBase = new BufferedStorage(); + res = storageBufferingBase.Initialize(in tempBaseStorage, bufferManager, (int)blockSize, bufferCount: 16); + if (res.IsFailure()) return res.Miss(); + + res = tempLogStorage.GetSize(out long logStorageSize); + if (res.IsFailure()) return res.Miss(); + + using var storageBufferingLog = new BufferedStorage(); + res = storageBufferingBase.Initialize(in tempLogStorage, bufferManager, (int)blockSize, bufferCount: 2); + if (res.IsFailure()) return res.Miss(); + + using var bufferedBaseStorage = new ValueSubStorage(storageBufferingBase, 0, baseStorageSize); + using var bufferedLogStorage = new ValueSubStorage(storageBufferingLog, 0, logStorageSize); + + res = unionStorage.Initialize(in bufferedBaseStorage, in bufferedLogStorage, blockSize); + if (res.IsFailure()) return res.Miss(); + + res = unionStorage.Commit(); + if (res.IsFailure()) return res.Miss(); + + res = unionStorage.Flush(); + if (res.IsFailure()) return res.Miss(); + + return Result.Success; + } } public static Result ReadExtraData(out JournalIntegritySaveDataFileSystem.ExtraData outData, in ValueSubStorage saveImageStorage, IBufferManager bufferManager, IMacGenerator macGenerator, IHash256GeneratorFactorySelector hashGeneratorFactorySelector, uint minimumVersion) { - throw new NotImplementedException(); + UnsafeHelpers.SkipParamInit(out outData); + + using var tempSaveImageStorage = new ValueSubStorage(in saveImageStorage); + JournalIntegritySaveDataFileSystem.ExtraData extraData = default; + + Result res = BufferManagerUtility.DoContinuouslyUntilBufferIsAllocated(ReadExtraDataImpl).Ret(); + if (res.IsFailure()) return res.Miss(); + + outData = extraData; + return Result.Success; + + Result ReadExtraDataImpl() + { + using var scopedRegistration = new ScopedBufferManagerContextRegistration(); + + return JournalIntegritySaveDataFileSystem.ReadExtraData(out extraData, in tempSaveImageStorage, + bufferManager, macGenerator, hashGeneratorFactorySelector, minimumVersion).Ret(); + } } public Result Initialize(in ValueSubStorage baseStorage, IBufferManager bufferManager, IMacGenerator macGenerator, IHash256GeneratorFactorySelector hashGeneratorFactorySelector, uint minimumVersion) { - throw new NotImplementedException(); + Assert.SdkRequiresNotNull(bufferManager); + + _baseStorage.Set(in baseStorage); + _bufferManager = bufferManager; + _bufferManagerSet = new FileSystemBufferManagerSet(); + + for (int i = 0; i < _bufferManagerSet.Buffers.Length; i++) + { + _bufferManagerSet.Buffers[i] = bufferManager; + } + + using var scopedRegistration = new ScopedBufferManagerContextRegistration(); + BufferManagerUtility.EnableBlockingBufferManagerAllocation(); + + Result res = _fileSystem.Initialize(in _baseStorage, _bufferManagerSet, _bufferManager, _mutex, macGenerator, + hashGeneratorFactorySelector, minimumVersion); + if (res.IsFailure()) return res.Miss(); + + base.Initialize(_fileSystem); + return Result.Success; } - public void FinalizeObject() + public new void FinalizeObject() { - throw new NotImplementedException(); + using var scopedRegistration = new ScopedBufferManagerContextRegistration(); + BufferManagerUtility.EnableBlockingBufferManagerAllocation(); + + _fileSystem.FinalizeObject(); + base.FinalizeObject(); } public Result WriteExtraData(in JournalIntegritySaveDataFileSystem.ExtraData extraData) { - throw new NotImplementedException(); + return _fileSystem.WriteExtraData(in extraData).Ret(); } - public Result ReadExtraData(out JournalIntegritySaveDataFileSystem.ExtraData outExtraData) + public void ReadExtraData(out JournalIntegritySaveDataFileSystem.ExtraData outExtraData) { - throw new NotImplementedException(); + _fileSystem.ReadExtraData(out outExtraData); } public Result RollbackOnlyModified() { - throw new NotImplementedException(); + return BufferManagerUtility.DoContinuouslyUntilBufferIsAllocated(RollbackOnlyModifiedImpl).Ret(); + + Result RollbackOnlyModifiedImpl() + { + using var scopedRegistration = new ScopedBufferManagerContextRegistration(); + + return _fileSystem.RollbackOnlyModified().Ret(); + } } public static Result UpdateMac(in ValueSubStorage saveDataStorage, IMacGenerator macGenerator, IHash256GeneratorFactorySelector hashGeneratorFactorySelector, uint minimumVersion) { - throw new NotImplementedException(); + return JournalIntegritySaveDataFileSystem + .UpdateMac(in saveDataStorage, macGenerator, hashGeneratorFactorySelector, minimumVersion).Ret(); } public long GetCounterForBundledCommit() { - throw new NotImplementedException(); + return _fileSystem.GetCounterForBundledCommit(); } public Result CommitFileSystem() { - throw new NotImplementedException(); + return Commit().Ret(); } public void ExtractParameters(out JournalIntegritySaveDataParameters outParams) { - throw new NotImplementedException(); + outParams = default; + + _fileSystem.ExtractParameters(out outParams.BlockSize, out outParams.CountDataBlock, + out outParams.CountJournalBlock, out outParams.CountExpandMax, out outParams.Version, + out outParams.IsMetaSetVerificationEnabled); } public static JournalIntegritySaveDataParameters SetUpSaveDataParameters(long blockSize, long dataSize, long journalSize) @@ -151,11 +352,23 @@ public class JournalIntegritySaveDataFileSystemDriver : ProxyFileSystemWithRetry public Result AcceptVisitor(IInternalStorageFileSystemVisitor visitor) { - throw new NotImplementedException(); + Assert.SdkRequiresNotNull(visitor); + + return BufferManagerUtility.DoContinuouslyUntilBufferIsAllocated(AcceptVisitorImpl).Ret(); + + Result AcceptVisitorImpl() + { + using var scopedRegistration = new ScopedBufferManagerContextRegistration(); + BufferManagerUtility.EnableBlockingBufferManagerAllocation(); + + return _fileSystem.AcceptVisitor(visitor).Ret(); + } } public Result UpdateMac(IMacGenerator macGenerator) { - throw new NotImplementedException(); + Assert.SdkRequiresNotNull(macGenerator); + + return _fileSystem.UpdateMacAndCommit(macGenerator).Ret(); } } \ No newline at end of file diff --git a/src/LibHac/FsSystem/Save/ProxyFileSystemWithRetryingBufferAllocation.cs b/src/LibHac/FsSystem/Save/ProxyFileSystemWithRetryingBufferAllocation.cs index ef4cde50..583fef63 100644 --- a/src/LibHac/FsSystem/Save/ProxyFileSystemWithRetryingBufferAllocation.cs +++ b/src/LibHac/FsSystem/Save/ProxyFileSystemWithRetryingBufferAllocation.cs @@ -107,7 +107,7 @@ public class ProxyFileSystemWithRetryingBufferAllocation : IFileSystem throw new NotImplementedException(); } - public void Finalize(IFileSystem fileSystem) + public void FinalizeObject() { throw new NotImplementedException(); } diff --git a/src/LibHac/FsSystem/Save/SaveDataInternalStorageFileSystem.cs b/src/LibHac/FsSystem/Save/SaveDataInternalStorageFileSystem.cs index 55bbff36..9ccd08b1 100644 --- a/src/LibHac/FsSystem/Save/SaveDataInternalStorageFileSystem.cs +++ b/src/LibHac/FsSystem/Save/SaveDataInternalStorageFileSystem.cs @@ -24,7 +24,7 @@ public interface IInternalStorageFileSystem { Result AcceptVisitor(IInternalStorageFileSystemVisitor visitor); Result WriteExtraData(in JournalIntegritySaveDataFileSystem.ExtraData extraData); - Result ReadExtraData(out JournalIntegritySaveDataFileSystem.ExtraData outExtraData); + void ReadExtraData(out JournalIntegritySaveDataFileSystem.ExtraData outExtraData); Result CommitFileSystem(); Result UpdateMac(IMacGenerator macGenerator); } diff --git a/src/LibHac/FsSystem/UnionStorage.cs b/src/LibHac/FsSystem/UnionStorage.cs index 485dd2b5..c9e9e64d 100644 --- a/src/LibHac/FsSystem/UnionStorage.cs +++ b/src/LibHac/FsSystem/UnionStorage.cs @@ -77,7 +77,7 @@ public class UnionStorage : IStorage header[0] = blockSize; header[1] = Sentinel; - return storage.Write(0, SpanHelpers.AsReadOnlyByteSpan(in header)); + return storage.Write(0, SpanHelpers.AsReadOnlyByteSpan(in header)).Ret(); } public Result Initialize(ref readonly ValueSubStorage baseStorage, ref readonly ValueSubStorage logStorage, @@ -157,7 +157,7 @@ public class UnionStorage : IStorage if (res.IsFailure()) return res.Miss(); } - return _baseStorage.Flush(); + return _baseStorage.Flush().Ret(); } public override Result Read(long offset, Span destination) @@ -292,7 +292,7 @@ public class UnionStorage : IStorage { Assert.SdkRequiresNotNull(_buffer); - return _baseStorage.GetSize(out size); + return _baseStorage.GetSize(out size).Ret(); } public override Result SetSize(long size) @@ -317,7 +317,7 @@ public class UnionStorage : IStorage } } - return _baseStorage.OperateRange(outBuffer, operationId, offset, size, inBuffer); + return _baseStorage.OperateRange(outBuffer, operationId, offset, size, inBuffer).Ret(); } private Result FindLog(out bool logFound, out long outLogOffset, long offsetOriginal) diff --git a/src/LibHac/LibHac.csproj b/src/LibHac/LibHac.csproj index c153c866..a60af391 100644 --- a/src/LibHac/LibHac.csproj +++ b/src/LibHac/LibHac.csproj @@ -43,4 +43,9 @@ + + + + + \ No newline at end of file