From 492716af745c05ba43a312977abd46836f4d1b4a Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Sun, 21 Apr 2024 22:59:53 -0700 Subject: [PATCH] Add DeepRetryStorage --- src/LibHac/FsSrv/Impl/DeepRetryStorage.cs | 183 +++++++++++++++++++ src/LibHac/FsSrv/NcaFileSystemService.cs | 162 +++++++++++----- src/LibHac/FsSrv/NcaFileSystemServiceImpl.cs | 2 +- src/LibHac/FsSystem/AsynchronousAccess.cs | 69 +++++++ 4 files changed, 369 insertions(+), 47 deletions(-) create mode 100644 src/LibHac/FsSrv/Impl/DeepRetryStorage.cs diff --git a/src/LibHac/FsSrv/Impl/DeepRetryStorage.cs b/src/LibHac/FsSrv/Impl/DeepRetryStorage.cs new file mode 100644 index 00000000..2b29eebd --- /dev/null +++ b/src/LibHac/FsSrv/Impl/DeepRetryStorage.cs @@ -0,0 +1,183 @@ +using System; +using LibHac.Common; +using LibHac.Crypto; +using LibHac.Diag; +using LibHac.Fs; +using LibHac.FsSystem; +using LibHac.Ncm; +using LibHac.Os; + +namespace LibHac.FsSrv.Impl; + +public class DeepRetryStorage : IStorage +{ + private AsynchronousAccessStorage _asyncStorage; + private SharedRef _accessSplitter; + private SharedRef _parent; + private UniqueRef _mountCountLock; + private DataStorageContext _dataStorageContext; + private bool _deepRetryEnabled; + private ReaderWriterLock _readWriteLock; + + // LibHac addition + private FileSystemServer _fsServer; + + private struct DataStorageContext + { + private Hash _digest; + private ulong _programId; + private StorageId _storageId; + private bool _isValid; + + public DataStorageContext() + { + _digest = default; + _programId = 0; + _storageId = StorageId.None; + _isValid = false; + } + + public DataStorageContext(in Hash digest, ulong programId, StorageId storageId) + { + _digest = digest; + _programId = programId; + _storageId = storageId; + _isValid = true; + } + + public readonly bool IsValid() => _isValid; + public readonly Hash GetDigest() => _digest; + public readonly ulong GetProgramIdValue() => _programId; + public readonly StorageId GetStorageId() => _storageId; + } + + public DeepRetryStorage(ref readonly SharedRef baseStorage, + ref readonly SharedRef accessSplitter, + ref readonly SharedRef parent, + ref UniqueRef mountCountSemaphore, + bool deepRetryEnabled, FileSystemServer fsServer) + { + // Missing: Getting the thread pool via GetRegisteredThreadPool() + _asyncStorage = new AsynchronousAccessStorage(in baseStorage, accessSplitter.Get); + _accessSplitter = SharedRef.CreateCopy(in accessSplitter); + _parent = SharedRef.CreateCopy(in parent); + _mountCountLock = UniqueRef.Create(ref mountCountSemaphore); + _dataStorageContext = new DataStorageContext(); + _deepRetryEnabled = deepRetryEnabled; + _readWriteLock = new ReaderWriterLock(fsServer.Hos.Os); + + _fsServer = fsServer; + } + + public DeepRetryStorage(ref readonly SharedRef baseStorage, + ref readonly SharedRef accessSplitter, + ref readonly SharedRef parent, + ref UniqueRef mountCountSemaphore, + in Hash hash, ulong programId, StorageId storageId, FileSystemServer fsServer) + { + // Missing: Getting the thread pool via GetRegisteredThreadPool() + _asyncStorage = new AsynchronousAccessStorage(in baseStorage, accessSplitter.Get); + _accessSplitter = SharedRef.CreateCopy(in accessSplitter); + _parent = SharedRef.CreateCopy(in parent); + _mountCountLock = UniqueRef.Create(ref mountCountSemaphore); + _dataStorageContext = new DataStorageContext(in hash, programId, storageId); + _deepRetryEnabled = true; + _readWriteLock = new ReaderWriterLock(fsServer.Hos.Os); + + _fsServer = fsServer; + } + + public override void Dispose() + { + _readWriteLock.Dispose(); + _mountCountLock.Destroy(); + _parent.Destroy(); + _accessSplitter.Destroy(); + _asyncStorage.Dispose(); + + base.Dispose(); + } + + public override Result Read(long offset, Span destination) + { + throw new NotImplementedException(); + } + + public override Result Write(long offset, ReadOnlySpan source) + { + throw new NotImplementedException(); + } + + public override Result Flush() + { + throw new NotImplementedException(); + } + + public override Result SetSize(long size) + { + throw new NotImplementedException(); + } + + public override Result GetSize(out long size) + { + throw new NotImplementedException(); + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, ReadOnlySpan inBuffer) + { + throw new NotImplementedException(); + } + + private Result InvalidateCacheOnStorage(bool remount) + { + Abort.DoAbortUnless(_deepRetryEnabled); + + using var scopedWriterLock = new UniqueLock(_readWriteLock); + + if (!remount || !_dataStorageContext.IsValid()) + { + _asyncStorage.OperateRange(default, OperationId.InvalidateCache, 0, long.MaxValue, default).IgnoreResult(); + return Result.Success; + } + + Assert.SdkNotNull(_parent); + + using var remountStorage = new SharedRef(); + using var remountStorageAccessSplitter = new SharedRef(); + + const int maxRetryCount = 2; + + Result retryResult = Result.Success; + Hash digest = default; + for (int i = 0; i < maxRetryCount; i++) + { + retryResult = _parent.Get.OpenDataStorageCore(ref remountStorage.Ref, ref remountStorageAccessSplitter.Ref, + out digest, _dataStorageContext.GetProgramIdValue(), _dataStorageContext.GetStorageId()); + + if (!ResultFs.DataCorrupted.Includes(retryResult)) + break; + + // Todo: Log + // _fsServer.Hos.Diag.Impl.LogImpl() + } + + if (retryResult.IsFailure()) return retryResult.Miss(); + + // Compare the digest of the remounted NCA header to the one we originally mounted + Hash originalDigest = _dataStorageContext.GetDigest(); + if (!CryptoUtil.IsSameBytes(originalDigest.Value, digest.Value, digest.Value.Length)) + { + return ResultFs.NcaDigestInconsistent.Log(); + } + + _accessSplitter.SetByMove(ref remountStorageAccessSplitter.Ref); + _asyncStorage.SetBaseStorage(in remountStorage, _accessSplitter.Get); + + return Result.Success; + } + + private void AcquireReaderLockForCacheInvalidation(ref SharedLock outReaderLock) + { + outReaderLock.Reset(_readWriteLock); + } +} \ No newline at end of file diff --git a/src/LibHac/FsSrv/NcaFileSystemService.cs b/src/LibHac/FsSrv/NcaFileSystemService.cs index 400203bd..88f5a64a 100644 --- a/src/LibHac/FsSrv/NcaFileSystemService.cs +++ b/src/LibHac/FsSrv/NcaFileSystemService.cs @@ -19,6 +19,12 @@ using Utility = LibHac.FsSrv.Impl.Utility; namespace LibHac.FsSrv; +/// +/// Handles NCA-related calls for . +/// +/// FS will have one instance of this class for every connected process. +/// The FS permissions of the calling process are checked on every function call. +///
Based on nnSdk 17.5.0 (FS 17.0.0) internal class NcaFileSystemService : IRomFileSystemAccessFailureManager { private const int AocSemaphoreCount = 128; @@ -90,27 +96,26 @@ internal class NcaFileSystemService : IRomFileSystemAccessFailureManager Result res = GetProgramInfo(out ProgramInfo callerProgramInfo); if (res.IsFailure()) return res.Miss(); - if (fsType != FileSystemProxyType.Manual) + switch (fsType) { - switch (fsType) - { - case FileSystemProxyType.Logo: - case FileSystemProxyType.Control: - case FileSystemProxyType.Meta: - case FileSystemProxyType.Data: - case FileSystemProxyType.Package: - return ResultFs.NotImplemented.Log(); - default: - return ResultFs.InvalidArgument.Log(); - } + case FileSystemProxyType.Manual: + Accessibility accessibility = + callerProgramInfo.AccessControl.GetAccessibilityFor(AccessibilityType.MountContentManual); + + if (!accessibility.CanRead) + return ResultFs.PermissionDenied.Log(); + + break; + case FileSystemProxyType.Logo: + case FileSystemProxyType.Control: + case FileSystemProxyType.Meta: + case FileSystemProxyType.Data: + case FileSystemProxyType.Package: + return ResultFs.NotImplemented.Log(); + default: + return ResultFs.InvalidArgument.Log(); } - Accessibility accessibility = - callerProgramInfo.AccessControl.GetAccessibilityFor(AccessibilityType.MountContentManual); - - if (!accessibility.CanRead) - return ResultFs.PermissionDenied.Log(); - // Get the program info for the owner of the file system being opened res = GetProgramInfoByProgramId(out ProgramInfo ownerProgramInfo, programId.Value); if (res.IsFailure()) return res.Miss(); @@ -123,7 +128,7 @@ internal class NcaFileSystemService : IRomFileSystemAccessFailureManager // The file system might have a patch version with no original version, so continue if not found if (originalResult.IsFailure() && !ResultLr.HtmlDocumentNotFound.Includes(originalResult)) - return originalResult; + return originalResult.Miss(); // Try to find the path to the patch file system using var patchPath = new Path(); @@ -136,17 +141,17 @@ internal class NcaFileSystemService : IRomFileSystemAccessFailureManager { // There must either be an original version or patch version of the file system being opened if (originalResult.IsFailure()) - return originalResult; + return originalResult.Miss(); // There is an original version and no patch version. Open the original directly - res = _serviceImpl.OpenFileSystem(ref fileSystem.Ref, in originalPath, default, fsType, programId.Value, - isDirectory); + res = _serviceImpl.OpenFileSystem(ref fileSystem.Ref, in originalPath, contentAttributes, fsType, + programId.Value, isDirectory); if (res.IsFailure()) return res.Miss(); } else { if (patchResult.IsFailure()) - return patchResult; + return patchResult.Miss(); ref readonly Path originalNcaPath = ref originalResult.IsSuccess() ? ref originalPath @@ -214,7 +219,7 @@ internal class NcaFileSystemService : IRomFileSystemAccessFailureManager public void IncrementRomFsDeepRetryStartCount() { - _serviceImpl.IncrementRomFsRemountForDataCorruptionCount(); + _serviceImpl.IncrementRomFsDeepRetryStartCount(); } public void IncrementRomFsRemountForDataCorruptionCount() @@ -234,7 +239,7 @@ internal class NcaFileSystemService : IRomFileSystemAccessFailureManager public void IncrementRomFsUnrecoverableByGameCardAccessFailedCount() { - _serviceImpl.IncrementRomFsRecoveredByInvalidateCacheCount(); + _serviceImpl.IncrementRomFsUnrecoverableByGameCardAccessFailedCount(); } private Result OpenDataStorageCore(ref SharedRef outStorage, @@ -249,13 +254,13 @@ internal class NcaFileSystemService : IRomFileSystemAccessFailureManager throw new NotImplementedException(); } - public Result OpenDataStorageByProgramId(ref SharedRef outStorage, ProgramId programId) + public Result OpenDataStorageByPath(ref SharedRef outStorage, in FspPath path, + ContentAttributes attributes, FileSystemProxyType fspType) { throw new NotImplementedException(); } - public Result OpenDataStorageByPath(ref SharedRef outStorage, in FspPath path, - ContentAttributes attributes, FileSystemProxyType fspType) + public Result OpenDataStorageByProgramId(ref SharedRef outStorage, ProgramId programId) { throw new NotImplementedException(); } @@ -316,17 +321,17 @@ internal class NcaFileSystemService : IRomFileSystemAccessFailureManager res = pathNormalized.InitializeWithReplaceUnc(path.Str); if (res.IsFailure()) return res.Miss(); - var pathFlags = new PathFlags(); - pathFlags.AllowWindowsPath(); - pathFlags.AllowMountName(); - res = pathNormalized.Normalize(pathFlags); + var flags = new PathFlags(); + flags.AllowMountName(); + flags.AllowWindowsPath(); + res = pathNormalized.Normalize(flags); if (res.IsFailure()) return res.Miss(); bool isDirectory = PathUtility.IsDirectoryPath(in path); using var fileSystem = new SharedRef(); - res = _serviceImpl.OpenFileSystem(ref fileSystem.Ref, in pathNormalized, default, fsType, canMountSystemDataPrivate, - id, isDirectory); + res = _serviceImpl.OpenFileSystem(ref fileSystem.Ref, in pathNormalized, attributes, fsType, + canMountSystemDataPrivate, id, isDirectory); if (res.IsFailure()) return res.Miss(); // Add all the wrappers for the file system @@ -399,7 +404,45 @@ internal class NcaFileSystemService : IRomFileSystemAccessFailureManager public Result OpenDataStorageWithProgramIndex(ref SharedRef outStorage, byte programIndex) { - throw new NotImplementedException(); + Result res = GetProgramInfo(out ProgramInfo programInfo); + if (res.IsFailure()) return res.Miss(); + + // Get the program ID of the program with the specified index if in a multi-program application + res = _serviceImpl.ResolveRomReferenceProgramId(out ProgramId targetProgramId, programInfo.ProgramId, programIndex); + if (res.IsFailure()) return res.Miss(); + + StorageLayoutType storageFlag = _serviceImpl.GetStorageFlag(targetProgramId.Value); + using var scopedContext = new ScopedStorageLayoutTypeSetter(storageFlag); + + using var romMountCountSemaphore = new UniqueRef(); + res = TryAcquireRomMountCountSemaphore(ref romMountCountSemaphore.Ref); + if (res.IsFailure()) return res.Miss(); + + using var storage = new SharedRef(); + using var storageAccessSplitter = new SharedRef(); + res = OpenDataStorageCore(ref storage.Ref, ref storageAccessSplitter.Ref, out Hash digest, targetProgramId.Value, programInfo.StorageId); + if (res.IsFailure()) return res.Miss(); + + if (programInfo.ProgramId != targetProgramId && !programInfo.AccessControl.HasContentOwnerId(targetProgramId.Value)) + return ResultFs.PermissionDenied.Log(); + + using var romDivisionSizeUnitCountSemaphore = new UniqueRef(); + res = TryAcquireRomDivisionSizeUnitCountSemaphore(ref romDivisionSizeUnitCountSemaphore.Ref, + ref romMountCountSemaphore.Ref, storage.Get); + if (res.IsFailure()) return res.Miss(); + + using SharedRef accessFailureManager = SharedRef.Create(in _selfReference); + + using var retryStorage = new SharedRef(new DeepRetryStorage(in storage, in storageAccessSplitter, + in accessFailureManager, ref romDivisionSizeUnitCountSemaphore.Ref, in digest, targetProgramId.Value, + programInfo.StorageId, _serviceImpl.FsServer)); + + using var typeSetStorage = new SharedRef(new StorageLayoutTypeSetStorage(ref retryStorage.Ref, storageFlag)); + using var storageAdapter = new SharedRef(new StorageInterfaceAdapter(ref typeSetStorage.Ref)); + + outStorage.SetByMove(ref storageAdapter.Ref); + + return Result.Success; } public Result GetRightsId(out RightsId outRightsId, ProgramId programId, StorageId storageId) @@ -433,7 +476,6 @@ internal class NcaFileSystemService : IRomFileSystemAccessFailureManager public Result GetRightsIdAndKeyGenerationByPath(out RightsId outRightsId, out byte outKeyGeneration, ref readonly FspPath path, ContentAttributes attributes) { - const ulong checkThroughProgramId = ulong.MaxValue; UnsafeHelpers.SkipParamInit(out outRightsId, out outKeyGeneration); using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.All); @@ -448,16 +490,18 @@ internal class NcaFileSystemService : IRomFileSystemAccessFailureManager res = pathNormalized.Initialize(path.Str); if (res.IsFailure()) return res.Miss(); - var pathFlags = new PathFlags(); - pathFlags.AllowWindowsPath(); - pathFlags.AllowMountName(); - res = pathNormalized.Normalize(pathFlags); + var flags = new PathFlags(); + flags.AllowMountName(); + flags.AllowWindowsPath(); + res = pathNormalized.Normalize(flags); if (res.IsFailure()) return res.Miss(); if (PathUtility.IsDirectoryPath(in path)) return ResultFs.TargetNotFound.Log(); - res = _serviceImpl.GetRightsId(out RightsId rightsId, out byte keyGeneration, in pathNormalized, default, + const ulong checkThroughProgramId = ulong.MaxValue; + + res = _serviceImpl.GetRightsId(out RightsId rightsId, out byte keyGeneration, in pathNormalized, attributes, new ProgramId(checkThroughProgramId)); if (res.IsFailure()) return res.Miss(); @@ -469,7 +513,34 @@ internal class NcaFileSystemService : IRomFileSystemAccessFailureManager public Result GetProgramId(out ProgramId outProgramId, ref readonly FspPath path, ContentAttributes attributes) { - throw new NotImplementedException(); + UnsafeHelpers.SkipParamInit(out outProgramId); + + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.All); + + Result res = GetProgramInfo(out ProgramInfo programInfo); + if (res.IsFailure()) return res.Miss(); + + if (!programInfo.AccessControl.CanCall(OperationType.GetProgramId)) + return ResultFs.PermissionDenied.Log(); + + using var pathNormalized = new Path(); + res = pathNormalized.Initialize(path.Str); + if (res.IsFailure()) return res.Miss(); + + var flags = new PathFlags(); + flags.AllowMountName(); + flags.AllowWindowsPath(); + res = pathNormalized.Normalize(flags); + if (res.IsFailure()) return res.Miss(); + + if (PathUtility.IsDirectoryPath(in path)) + return ResultFs.TargetNotFound.Log(); + + res = _serviceImpl.GetProgramId(out ProgramId programId, in pathNormalized, attributes); + if (res.IsFailure()) return res.Miss(); + + outProgramId = programId; + return Result.Success; } // ReSharper disable once OutParameterValueIsAlwaysDiscarded.Local @@ -489,13 +560,12 @@ internal class NcaFileSystemService : IRomFileSystemAccessFailureManager isHostFs = Utility.IsHostFsMountName(programPath.GetString()); using var fileSystem = new SharedRef(); - res = _serviceImpl.OpenDataFileSystem(ref fileSystem.Ref, in programPath, contentAttributes, FileSystemProxyType.Rom, - programId, isDirectory); + res = _serviceImpl.OpenDataFileSystem(ref fileSystem.Ref, in programPath, contentAttributes, + FileSystemProxyType.Rom, programId, isDirectory); if (res.IsFailure()) return res.Miss(); // Add all the file system wrappers - using var typeSetFileSystem = - new SharedRef(new StorageLayoutTypeSetFileSystem(ref fileSystem.Ref, storageFlag)); + using var typeSetFileSystem = new SharedRef(new StorageLayoutTypeSetFileSystem(ref fileSystem.Ref, storageFlag)); outFileSystem.SetByMove(ref typeSetFileSystem.Ref); diff --git a/src/LibHac/FsSrv/NcaFileSystemServiceImpl.cs b/src/LibHac/FsSrv/NcaFileSystemServiceImpl.cs index 8af41156..66d865ab 100644 --- a/src/LibHac/FsSrv/NcaFileSystemServiceImpl.cs +++ b/src/LibHac/FsSrv/NcaFileSystemServiceImpl.cs @@ -238,7 +238,7 @@ public class NcaFileSystemServiceImpl : IDisposable throw new NotImplementedException(); } - public Result GetProgramId(out ProgramId outProgramId, ref readonly Path path, ContentAttributes attributes, ProgramId programId) + public Result GetProgramId(out ProgramId outProgramId, ref readonly Path path, ContentAttributes attributes) { throw new NotImplementedException(); } diff --git a/src/LibHac/FsSystem/AsynchronousAccess.cs b/src/LibHac/FsSystem/AsynchronousAccess.cs index 45527b88..af007268 100644 --- a/src/LibHac/FsSystem/AsynchronousAccess.cs +++ b/src/LibHac/FsSystem/AsynchronousAccess.cs @@ -87,4 +87,73 @@ public class DefaultAsynchronousAccessSplitter : IAsynchronousAccessSplitter count = BitUtil.DivideUp(endOffset - alignedStartOffset, accessSize); return Result.Success; } +} + +public class AsynchronousAccessStorage : IStorage +{ + private SharedRef _baseStorage; + // private ThreadPool _threadPool; + private IAsynchronousAccessSplitter _baseStorageAccessSplitter; + + public AsynchronousAccessStorage(ref readonly SharedRef baseStorage) : this(in baseStorage, + IAsynchronousAccessSplitter.GetDefaultAsynchronousAccessSplitter()) + { + } + + public AsynchronousAccessStorage(ref readonly SharedRef baseStorage, IAsynchronousAccessSplitter baseStorageAccessSplitter) + { + _baseStorage = SharedRef.CreateCopy(in baseStorage); + _baseStorageAccessSplitter = baseStorageAccessSplitter; + + Assert.SdkRequiresNotNull(in _baseStorage); + Assert.SdkRequiresNotNull(_baseStorageAccessSplitter); + } + + public override void Dispose() + { + _baseStorage.Destroy(); + base.Dispose(); + } + + public void SetBaseStorage(ref readonly SharedRef baseStorage, IAsynchronousAccessSplitter baseStorageAccessSplitter) + { + _baseStorage.SetByCopy(in baseStorage); + _baseStorageAccessSplitter = baseStorageAccessSplitter; + } + + // Todo: Implement + public override Result Read(long offset, Span destination) + { + return _baseStorage.Get.Read(offset, destination).Ret(); + } + + private Result ReadImpl(long offset, Span destination) + { + throw new NotImplementedException(); + } + + public override Result Write(long offset, ReadOnlySpan source) + { + return _baseStorage.Get.Write(offset, source).Ret(); + } + + public override Result GetSize(out long size) + { + return _baseStorage.Get.GetSize(out size).Ret(); + } + + public override Result SetSize(long size) + { + return _baseStorage.Get.SetSize(size).Ret(); + } + + public override Result Flush() + { + return _baseStorage.Get.Flush().Ret(); + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, ReadOnlySpan inBuffer) + { + return _baseStorage.Get.OperateRange(outBuffer, operationId, offset, size, inBuffer).Ret(); + } } \ No newline at end of file