Add BufferedStorage with some supporting classes

build/CodeGen/results.csv

@@ -109,6 +109,7 @@ Module,DescriptionStart,DescriptionEnd,Name,Summary
 2,3383,,AllocationFailureInAesXtsFileE,In Initialize
 2,3394,,AllocationFailureInEncryptedFileSystemCreatorA,In Create allocating AesXtsFileSystem
 2,3407,,AllocationFailureInFileSystemInterfaceAdapter, In OpenFile or OpenDirectory
+2,3411,,AllocationFailureInBufferedStorageA, In Initialize allocating Cache array
 2,3420,,AllocationFailureInNew,
 2,3421,,AllocationFailureInCreateShared,
 2,3422,,AllocationFailureInMakeUnique,

src/LibHac/Common/Ref.cs

@@ -0,0 +1,72 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+namespace LibHac.Common
+{
+    /// <summary>
+    /// A <see langword="struct"/> that can store a reference to a value of a specified type.
+    /// </summary>
+    /// <typeparam name="T">The type of value to reference.</typeparam>
+    public readonly ref struct Ref<T>
+    {
+        /// <summary>
+        /// The 1-length <see cref="Span{T}"/> instance used to track the target <typeparamref name="T"/> value.
+        /// </summary>
+        private readonly Span<T> _span;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="Ref{T}"/> struct.
+        /// </summary>
+        /// <param name="value">The reference to the target <typeparamref name="T"/> value.</param>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public Ref(ref T value)
+        {
+            _span = MemoryMarshal.CreateSpan(ref value, 1);
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="Ref{T}"/> struct.
+        /// </summary>
+        /// <param name="pointer">The pointer to the target value.</param>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public unsafe Ref(void* pointer)
+            : this(ref Unsafe.AsRef<T>(pointer))
+        {
+        }
+
+        /// <summary>
+        /// Gets the <typeparamref name="T"/> reference represented by the current <see cref="Ref{T}"/> instance.
+        /// </summary>
+        public ref T Value
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get => ref MemoryMarshal.GetReference(_span);
+        }
+
+        /// <summary>
+        /// Returns a value that indicates whether the current <see cref="Ref{T}"/> is <see langword="null"/>.
+        /// </summary>
+        /// <returns><see langword="true"/> if the held reference is <see langword="null"/>;
+        /// otherwise <see langword="false"/>.</returns>
+        public bool IsNull
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get => Unsafe.IsNullRef(ref Value);
+        }
+
+        /// <summary>
+        /// Implicitly gets the <typeparamref name="T"/> value from a given <see cref="Ref{T}"/> instance.
+        /// </summary>
+        /// <param name="reference">The input <see cref="Ref{T}"/> instance.</param>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static implicit operator T(Ref<T> reference)
+        {
+            return reference.Value;
+        }
+    }
+}

src/LibHac/Diag/Assert.cs

@@ -20,6 +20,20 @@ namespace LibHac.Diag
             throw new LibHacException($"Assertion failed: {message}");
         }
 
+        [Conditional("DEBUG")]
+        public static void False([DoesNotReturnIf(true)] bool condition, string message = null)
+        {
+            if (!condition)
+                return;
+
+            if (string.IsNullOrWhiteSpace(message))
+            {
+                throw new LibHacException("Assertion failed.");
+            }
+
+            throw new LibHacException($"Assertion failed: {message}");
+        }
+
         [Conditional("DEBUG")]
         public static void Null<T>([NotNull] T item) where T : class
         {

src/LibHac/Fs/Buffers/IBufferManager.cs

@@ -6,6 +6,14 @@ using CacheHandle = System.Int64;
 namespace LibHac.Fs
 {
     // ReSharper disable once InconsistentNaming
+    /// <summary>
+    /// Handles buffer allocation, deallocation, and caching.<br/>
+    /// An allocated buffer may be placed in the cache using <see cref="RegisterCache"/>.
+    /// Caching a buffer saves the buffer for later retrieval, but tells the buffer manager that it can deallocate the
+    /// buffer if the memory is needed elsewhere. Any cached buffer may be evicted from the cache if there is no free
+    /// space for a requested allocation or if the cache is full when caching a new buffer.
+    /// A cached buffer can be retrieved using <see cref="AcquireCache"/>.
+    /// </summary>
     public abstract class IBufferManager : IDisposable
     {
         public readonly struct BufferAttribute
@@ -23,18 +31,80 @@ namespace LibHac.Fs
         public Buffer AllocateBuffer(int size, BufferAttribute attribute) =>
             DoAllocateBuffer(size, attribute);
 
+        /// <summary>
+        /// Allocates a new buffer with an attribute of level 0.
+        /// </summary>
+        /// <param name="size">The minimum size of the buffer to allocate</param>
+        /// <returns>The allocated <see cref="Buffer"/> if successful. Otherwise a null <see cref="Buffer"/>.</returns>
+        public Buffer AllocateBuffer(int size) => DoAllocateBuffer(size, new BufferAttribute());
+
+        /// <summary>
+        /// Deallocates the provided <see cref="Buffer"/>.
+        /// </summary>
+        /// <param name="buffer">The Buffer to deallocate.</param>
         public void DeallocateBuffer(Buffer buffer) => DoDeallocateBuffer(buffer);
 
+        /// <summary>
+        /// Adds a <see cref="Buffer"/> to the cache.
+        /// The buffer must have been allocated from this <see cref="IBufferManager"/>.<br/>
+        /// The buffer must not be used after adding it to the cache.
+        /// </summary>
+        /// <param name="buffer">The buffer to cache.</param>
+        /// <param name="attribute">The buffer attribute.</param>
+        /// <returns>A handle that can be used to retrieve the buffer at a later time.</returns>
         public CacheHandle RegisterCache(Buffer buffer, BufferAttribute attribute) =>
             DoRegisterCache(buffer, attribute);
 
+        /// <summary>
+        /// Attempts to acquire a cached <see cref="Buffer"/>.
+        /// If the buffer was evicted from the cache, a null buffer is returned.
+        /// </summary>
+        /// <param name="handle">The handle received when registering the buffer.</param>
+        /// <returns>The requested <see cref="Buffer"/> if it's still in the cache;
+        /// otherwise a null <see cref="Buffer"/></returns>
         public Buffer AcquireCache(CacheHandle handle) => DoAcquireCache(handle);
+
+        /// <summary>
+        /// Gets the total size of the <see cref="IBufferManager"/>'s heap.
+        /// </summary>
+        /// <returns>The total size of the heap.</returns>
         public int GetTotalSize() => DoGetTotalSize();
+
+        /// <summary>
+        /// Gets the amount of free space in the heap that is not currently allocated or cached.
+        /// </summary>
+        /// <returns>The amount of free space.</returns>
         public int GetFreeSize() => DoGetFreeSize();
+
+        /// <summary>
+        /// Gets the amount of space that can be used for new allocations.
+        /// This includes free space and space used by cached buffers.
+        /// </summary>
+        /// <returns>The amount of allocatable space.</returns>
         public int GetTotalAllocatableSize() => DoGetTotalAllocatableSize();
+
+        /// <summary>
+        /// Gets the largest amount of free space there's been at one time since the peak was last cleared.
+        /// </summary>
+        /// <returns>The peak amount of free space.</returns>
         public int GetFreeSizePeak() => DoGetFreeSizePeak();
+
+        /// <summary>
+        /// Gets the largest amount of allocatable space there's been at one time since the peak was last cleared.
+        /// </summary>
+        /// <returns>The peak amount of allocatable space.</returns>
         public int GetTotalAllocatableSizePeak() => DoGetTotalAllocatableSizePeak();
+
+        /// <summary>
+        /// Gets the number of times an allocation or cache registration needed to be retried after deallocating
+        /// a cache entry because of insufficient heap space or cache space.
+        /// </summary>
+        /// <returns>The number of retries.</returns>
         public int GetRetriedCount() => DoGetRetriedCount();
+
+        /// <summary>
+        /// Resets the free and allocatable peak sizes, setting the peak sizes to the actual current sizes.
+        /// </summary>
         public void ClearPeak() => DoClearPeak();
 
         protected abstract Buffer DoAllocateBuffer(int size, BufferAttribute attribute);

src/LibHac/Fs/ResultFs.cs

@@ -136,6 +136,8 @@ namespace LibHac.Fs
             public static Result.Base AllocationFailureInEncryptedFileSystemCreatorA => new Result.Base(ModuleFs, 3394);
             /// <summary> In OpenFile or OpenDirectory<br/>Error code: 2002-3407; Inner value: 0x1a9e02</summary>
             public static Result.Base AllocationFailureInFileSystemInterfaceAdapter => new Result.Base(ModuleFs, 3407);
+            /// <summary> In Initialize allocating Cache array<br/>Error code: 2002-3411; Inner value: 0x1aa602</summary>
+            public static Result.Base AllocationFailureInBufferedStorageA => new Result.Base(ModuleFs, 3411);
             /// <summary>Error code: 2002-3420; Inner value: 0x1ab802</summary>
             public static Result.Base AllocationFailureInNew => new Result.Base(ModuleFs, 3420);
             /// <summary>Error code: 2002-3421; Inner value: 0x1aba02</summary>

src/LibHac/FsSystem/Buffers/BufferManagerUtility.cs

@@ -0,0 +1,143 @@
+using System;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using LibHac.Diag;
+using LibHac.Fs;
+using Buffer = LibHac.Fs.Buffer;
+
+namespace LibHac.FsSystem.Buffers
+{
+    public struct BufferManagerContext
+    {
+        private bool _needsBlocking;
+
+        public bool IsNeedBlocking() => _needsBlocking;
+        public void SetNeedBlocking(bool needsBlocking) => _needsBlocking = needsBlocking;
+    }
+
+    public struct ScopedBufferManagerContextRegistration : IDisposable
+    {
+        private BufferManagerContext _oldContext;
+
+        // ReSharper disable once UnusedParameter.Local
+        public ScopedBufferManagerContextRegistration(int unused = default)
+        {
+            _oldContext = BufferManagerUtility.GetBufferManagerContext();
+        }
+
+        public void Dispose()
+        {
+            BufferManagerUtility.RegisterBufferManagerContext(in _oldContext);
+        }
+    }
+
+    internal static class BufferManagerUtility
+    {
+        // Todo: Use TimeSpan
+        private const int RetryWait = 10;
+
+        [ThreadStatic]
+        private static BufferManagerContext _context;
+
+        public delegate bool IsValidBufferFunction(in Buffer buffer);
+
+        public static Result DoContinuouslyUntilBufferIsAllocated(Func<Result> function, Func<Result> onFailure,
+            [CallerMemberName] string callerName = "")
+        {
+            const int bufferAllocationRetryLogCountMax = 10;
+            const int bufferAllocationRetryLogInterval = 100;
+
+            Result result;
+
+            for (int count = 1; ; count++)
+            {
+                result = function();
+                if (!ResultFs.BufferAllocationFailed.Includes(result))
+                    break;
+
+                // Failed to allocate. Wait and try again.
+                if (1 <= count && count <= bufferAllocationRetryLogCountMax ||
+                    count % bufferAllocationRetryLogInterval == 0)
+                {
+                    // Todo: Log allocation failure
+                }
+
+                Result rc = onFailure();
+                if (rc.IsFailure()) return rc;
+
+                Thread.Sleep(RetryWait);
+            }
+
+            return result;
+        }
+
+        public static Result DoContinuouslyUntilBufferIsAllocated(Func<Result> function,
+            [CallerMemberName] string callerName = "")
+        {
+            return DoContinuouslyUntilBufferIsAllocated(function, static () => Result.Success, callerName);
+        }
+
+        public static void RegisterBufferManagerContext(in BufferManagerContext context)
+        {
+            _context = context;
+        }
+
+        public static ref BufferManagerContext GetBufferManagerContext() => ref _context;
+
+        public static void EnableBlockingBufferManagerAllocation()
+        {
+            ref BufferManagerContext context = ref GetBufferManagerContext();
+            context.SetNeedBlocking(true);
+        }
+
+        public static Result AllocateBufferUsingBufferManagerContext(out Buffer outBuffer, IBufferManager bufferManager,
+            int size, IBufferManager.BufferAttribute attribute, IsValidBufferFunction isValidBuffer,
+            [CallerMemberName] string callerName = "")
+        {
+            Assert.NotNull(bufferManager);
+            Assert.NotNull(callerName);
+
+            // Clear the output.
+            outBuffer = default;
+            Buffer tempBuffer = default;
+
+            // Get the context.
+            ref BufferManagerContext context = ref GetBufferManagerContext();
+
+            Result AllocateBufferImpl()
+            {
+                Buffer buffer = bufferManager.AllocateBuffer(size, attribute);
+
+                if (!isValidBuffer(in buffer))
+                {
+                    if (!buffer.IsNull)
+                    {
+                        bufferManager.DeallocateBuffer(buffer);
+                    }
+
+                    return ResultFs.BufferAllocationFailed.Log();
+                }
+
+                tempBuffer = buffer;
+                return Result.Success;
+            }
+
+            if (!context.IsNeedBlocking())
+            {
+                // If we don't need to block, just allocate the buffer.
+                Result rc = AllocateBufferImpl();
+                if (rc.IsFailure()) return rc;
+            }
+            else
+            {
+                // Otherwise, try to allocate repeatedly.
+                Result rc = DoContinuouslyUntilBufferIsAllocated(AllocateBufferImpl);
+                if (rc.IsFailure()) return rc;
+            }
+
+            Assert.True(!tempBuffer.IsNull);
+            outBuffer = tempBuffer;
+            return Result.Success;
+        }
+    }
+}

src/LibHac/FsSystem/Buffers/FileSystemBuddyHeap.cs

@@ -199,13 +199,15 @@ namespace LibHac.FsSystem
         public Result Initialize(UIntPtr address, nuint size, nuint blockSize, int orderMax, void* workBuffer,
             nuint workBufferSize)
         {
-            // Note: Buffer size assert is done before adjusting for alignment
             Assert.True(workBufferSize >= QueryWorkBufferSize(orderMax));
 
             uint pageListAlignment = (uint)Unsafe.SizeOf<nint>();
             var alignedWork = (void*)Alignment.AlignUpPow2((ulong)workBuffer, pageListAlignment);
             ExternalFreeLists = (PageList*)alignedWork;
 
+            // Note: The original code does not have a buffer size assert after adjusting for alignment.
+            Assert.True(workBufferSize - ((nuint)alignedWork - (nuint)workBuffer) >= QueryWorkBufferSize(orderMax));
+
             return Initialize(address, size, blockSize, orderMax);
         }
 
@@ -264,7 +266,7 @@ namespace LibHac.FsSystem
             // Allocate remaining space to smaller orders as possible.
             {
                 nuint remaining = HeapSize - (maxPageCount - 1) * maxPageSize;
-                nuint curAddress = (nuint)HeapStart - (maxPageCount - 1) * maxPageSize;
+                nuint curAddress = HeapStart - (maxPageCount - 1) * maxPageSize;
                 Assert.True(Alignment.IsAlignedPow2(remaining, (uint)BlockSize));
 
                 do
@@ -572,7 +574,6 @@ namespace LibHac.FsSystem
         private MemoryHandle PinnedHeapMemoryHandle { get; set; }
         private Memory<byte> HeapBuffer { get; set; }
         private MemoryHandle PinnedWorkMemoryHandle { get; set; }
-        private Memory<byte> WorkBuffer { get; set; }
 
         public Result Initialize(Memory<byte> heapBuffer, int blockSize, Memory<byte> workBuffer)
         {
@@ -583,7 +584,6 @@ namespace LibHac.FsSystem
         public Result Initialize(Memory<byte> heapBuffer, int blockSize, int orderMax, Memory<byte> workBuffer)
         {
             PinnedWorkMemoryHandle = workBuffer.Pin();
-            WorkBuffer = workBuffer;
 
             PinnedHeapMemoryHandle = heapBuffer.Pin();
             HeapBuffer = heapBuffer;
@@ -591,8 +591,8 @@ namespace LibHac.FsSystem
             var heapAddress = (UIntPtr)PinnedHeapMemoryHandle.Pointer;
             var heapSize = (nuint)heapBuffer.Length;
 
-            void* workAddress = PinnedHeapMemoryHandle.Pointer;
-            var workSize = (nuint)heapBuffer.Length;
+            void* workAddress = PinnedWorkMemoryHandle.Pointer;
+            var workSize = (nuint)workBuffer.Length;
 
             return Initialize(heapAddress, heapSize, (nuint)blockSize, orderMax, workAddress, workSize);
         }

src/LibHac/FsSystem/Buffers/FileSystemBufferManager.cs

@@ -90,7 +90,7 @@ namespace LibHac.FsSystem
                 // Validate pre-conditions.
                 Assert.True(Entries == null);
 
-                // Note: We don't have the option of using an external Entry buffer like the original
+                // Note: We don't have the option of using an external Entry buffer like the original C++ code
                 // because Entry includes managed references so we can't cast a byte* to Entry* without pinning.
                 // If we don't have an external buffer, try to allocate an internal one.
 
@@ -215,6 +215,7 @@ namespace LibHac.FsSystem
                     if (CanUnregister(this, ref Entries[i]))
                     {
                         entry = ref Entries[i];
+                        break;
                     }
                 }
 
@@ -278,7 +279,7 @@ namespace LibHac.FsSystem
                     entry = ref Entries[EntryCount];
                     entry.Initialize(PublishCacheHandle(), buffer, attr);
                     EntryCount++;
-                    Assert.True(EntryCount == 1 || Entries[EntryCount - 1].GetHandle() < entry.GetHandle());
+                    Assert.True(EntryCount == 1 || Entries[EntryCount - 2].GetHandle() < entry.GetHandle());
                 }
 
                 return ref entry;
@@ -292,7 +293,7 @@ namespace LibHac.FsSystem
 
                 // Ensure the entry is valid.
                 Span<Entry> entryBuffer = Entries;
-                Assert.True(Unsafe.IsAddressGreaterThan(ref entry, ref MemoryMarshal.GetReference(entryBuffer)));
+                Assert.True(!Unsafe.IsAddressLessThan(ref entry, ref MemoryMarshal.GetReference(entryBuffer)));
                 Assert.True(Unsafe.IsAddressLessThan(ref entry,
                     ref Unsafe.Add(ref MemoryMarshal.GetReference(entryBuffer), entryBuffer.Length)));
 
@@ -347,8 +348,9 @@ namespace LibHac.FsSystem
 
         public Result Initialize(int maxCacheCount, Memory<byte> heapBuffer, int blockSize, Memory<byte> workBuffer)
         {
-            // Note: We can't use an external buffer for the cache handle table,
+            // Note: We can't use an external buffer for the cache handle table since it contains managed pointers,
             // so pass the work buffer directly to the buddy heap.
+
             Result rc = CacheTable.Initialize(maxCacheCount);
             if (rc.IsFailure()) return rc;
 

src/LibHac/FsSystem/PooledBuffer.cs

@@ -0,0 +1,117 @@
+using System;
+using System.Buffers;
+using LibHac.Diag;
+
+namespace LibHac.FsSystem
+{
+    // Implement the PooledBuffer interface using .NET ArrayPools
+    public struct PooledBuffer : IDisposable
+    {
+        // It's faster to create new smaller arrays than rent them
+        private const int RentThresholdBytes = 512;
+
+        private const int HeapBlockSize = 1024 * 4;
+
+        // Keep the max sizes that FS uses.
+        // A heap block is 4KB.An order is a power of two. 
+        // This gives blocks of the order 512KB, 4MB.
+        private const int HeapOrderMax = 7;
+        private const int HeapOrderMaxForLarge = HeapOrderMax + 3;
+
+        private const int HeapAllocatableSizeMax = HeapBlockSize * (1 << HeapOrderMax);
+        private const int HeapAllocatableSizeMaxForLarge = HeapBlockSize * (1 << HeapOrderMaxForLarge);
+
+        private byte[] Array { get; set; }
+        private int Length { get; set; }
+
+        public PooledBuffer(int idealSize, int requiredSize)
+        {
+            Array = null;
+            Length = default;
+            Allocate(idealSize, requiredSize);
+        }
+
+        public Span<byte> GetBuffer()
+        {
+            Assert.NotNull(Array);
+            return Array.AsSpan(0, Length);
+        }
+
+        public int GetSize()
+        {
+            Assert.NotNull(Array);
+            return Length;
+        }
+
+        public static int GetAllocatableSizeMax() => GetAllocatableSizeMaxCore(false);
+        public static int GetAllocatableParticularlyLargeSizeMax => GetAllocatableSizeMaxCore(true);
+
+        private static int GetAllocatableSizeMaxCore(bool enableLargeCapacity)
+        {
+            return enableLargeCapacity ? HeapAllocatableSizeMaxForLarge : HeapAllocatableSizeMax;
+        }
+
+        public void Allocate(int idealSize, int requiredSize) => AllocateCore(idealSize, requiredSize, false);
+        public void AllocateParticularlyLarge(int idealSize, int requiredSize) => AllocateCore(idealSize, requiredSize, true);
+
+        private void AllocateCore(int idealSize, int requiredSize, bool enableLargeCapacity)
+        {
+            Assert.Null(Array);
+
+            // Check that we can allocate this size.
+            Assert.True(requiredSize <= GetAllocatableSizeMaxCore(enableLargeCapacity));
+
+            int targetSize = Math.Min(Math.Max(idealSize, requiredSize),
+                GetAllocatableSizeMaxCore(enableLargeCapacity));
+
+            if (targetSize >= RentThresholdBytes)
+            {
+                Array = ArrayPool<byte>.Shared.Rent(targetSize);
+            }
+            else
+            {
+                Array = new byte[targetSize];
+            }
+
+            Length = Array.Length;
+        }
+
+        public void Deallocate()
+        {
+            // Shrink the buffer to empty.
+            Shrink(0);
+            Assert.Null(Array);
+        }
+
+        public void Shrink(int idealSize)
+        {
+            Assert.True(idealSize <= GetAllocatableSizeMaxCore(true));
+
+            // Check if we actually need to shrink.
+            if (Length > idealSize)
+            {
+                Assert.NotNull(Array);
+
+                // Pretend we shrank the buffer.
+                Length = idealSize;
+
+                // Shrinking to zero means that we have no buffer.
+                if (Length == 0)
+                {
+                    // Return the array if we rented it.
+                    if (Array?.Length >= RentThresholdBytes)
+                    {
+                        ArrayPool<byte>.Shared.Return(Array);
+                    }
+
+                    Array = null;
+                }
+            }
+        }
+
+        public void Dispose()
+        {
+            Deallocate();
+        }
+    }
+}

src/LibHac/Util/Alignment.cs

@@ -25,7 +25,6 @@ namespace LibHac.Util
             return (value & ~invMask);
         }
 
-        [MethodImpl(MethodImplOptions.NoInlining)]
         public static bool IsAlignedPow2(ulong value, uint alignment)
         {
             Assert.True(BitUtil.IsPowerOfTwo(alignment));
@@ -34,11 +33,6 @@ namespace LibHac.Util
             return (value & invMask) == 0;
         }
 
-        public static bool IsAlignedPow2<T>(Span<T> buffer, uint alignment)
-        {
-            return IsAlignedPow2(buffer, alignment);
-        }
-
         public static bool IsAlignedPow2<T>(ReadOnlySpan<T> buffer, uint alignment)
         {
             return IsAlignedPow2(ref MemoryMarshal.GetReference(buffer), alignment);

tests/LibHac.Tests/Fs/StorageTester.cs

@@ -0,0 +1,265 @@
+using System;
+using System.IO;
+using System.Linq;
+using LibHac.Fs;
+
+namespace LibHac.Tests.Fs
+{
+    public class StorageTester
+    {
+        private Random _random;
+        private byte[][] _backingArrays;
+        private byte[][] _buffers;
+        private int _size;
+
+        private int[] _frequentAccessOffsets;
+        private int _lastAccessEnd;
+        private int _totalAccessCount;
+        private Configuration _config;
+
+        public class Configuration
+        {
+            public Entry[] Entries { get; set; }
+            public int[] SizeClassProbs { get; set; }
+            public int[] SizeClassMaxSizes { get; set; }
+            public int[] TaskProbs { get; set; }
+            public int[] AccessTypeProbs { get; set; }
+            public ulong RngSeed { get; set; }
+            public int FrequentAccessBlockCount { get; set; }
+        }
+
+        public StorageTester(Configuration config)
+        {
+            Entry[] entries = config.Entries;
+
+            if (entries.Length < 2)
+            {
+                throw new ArgumentException("At least 2 storage entries must be provided", nameof(config.Entries));
+            }
+
+            if (entries.Select(x => x.BackingArray.Length).Distinct().Count() != 1)
+            {
+                throw new ArgumentException("All storages must have the same size.", nameof(config.Entries));
+            }
+
+            if (entries[0].BackingArray.Length == 0)
+            {
+                throw new ArgumentException("The storage size must be greater than 0.", nameof(config.Entries));
+            }
+
+            _config = config;
+            _random = new Random(config.RngSeed);
+
+            _backingArrays = entries.Select(x => x.BackingArray).ToArray();
+
+            _buffers = new byte[entries.Length][];
+            for (int i = 0; i < entries.Length; i++)
+            {
+                _buffers[i] = new byte[config.SizeClassMaxSizes[^1]];
+            }
+
+            _size = entries[0].BackingArray.Length;
+            _lastAccessEnd = 0;
+
+            _frequentAccessOffsets = new int[config.FrequentAccessBlockCount];
+            for (int i = 0; i < _frequentAccessOffsets.Length; i++)
+            {
+                _frequentAccessOffsets[i] = ChooseOffset(AccessType.Random);
+            }
+        }
+
+        //public StorageTester(ulong rngSeed, int frequentAccessBlockCount, params Entry[] entries)
+        //{
+        //    if (entries.Length < 2)
+        //    {
+        //        throw new ArgumentException("At least 2 storage entries must be provided", nameof(entries));
+        //    }
+
+        //    if (entries.Select(x => x.BackingArray.Length).Distinct().Count() != 1)
+        //    {
+        //        throw new ArgumentException("All storages must have the same size.", nameof(entries));
+        //    }
+
+        //    if (entries[0].BackingArray.Length == 0)
+        //    {
+        //        throw new ArgumentException("The storage size must be greater than 0.", nameof(entries));
+        //    }
+
+        //    _random = new Random(rngSeed);
+
+        //    _entries = entries;
+        //    _backingArrays = entries.Select(x => x.BackingArray).ToArray();
+
+        //    _buffers = new byte[entries.Length][];
+        //    for (int i = 0; i < entries.Length; i++)
+        //    {
+        //        _buffers[i] = new byte[SizeClassMaxSizes[^1]];
+        //    }
+
+        //    _size = _entries[0].BackingArray.Length;
+        //    _lastAccessEnd = 0;
+
+        //    _frequentAccessOffsets = new int[frequentAccessBlockCount];
+        //    for (int i = 0; i < _frequentAccessOffsets.Length; i++)
+        //    {
+        //        _frequentAccessOffsets[i] = ChooseOffset(AccessType.Random);
+        //    }
+        //}
+
+        public void Run(long accessCount)
+        {
+            long endCount = _totalAccessCount + accessCount;
+
+            while (_totalAccessCount < endCount)
+            {
+                Task task = ChooseTask();
+                switch (task)
+                {
+                    case Task.Read:
+                        RunRead();
+                        break;
+                    case Task.Write:
+                        RunWrite();
+                        break;
+                    case Task.Flush:
+                        RunFlush();
+                        break;
+                }
+
+                _totalAccessCount++;
+            }
+        }
+
+        private void RunRead()
+        {
+            int sizeClass = ChooseSizeClass();
+            AccessType accessType = ChooseAccessType();
+            int offset = ChooseOffset(accessType);
+            int size = ChooseSize(offset, sizeClass);
+
+            for (int i = 0; i < _config.Entries.Length; i++)
+            {
+                Entry entry = _config.Entries[i];
+                entry.Storage.Read(offset, _buffers[i].AsSpan(0, size)).ThrowIfFailure();
+            }
+
+            if (!CompareBuffers(_buffers, size))
+            {
+                throw new InvalidDataException($"Read: Offset {offset}; Size {size}");
+            }
+        }
+
+        private void RunWrite()
+        {
+            int sizeClass = ChooseSizeClass();
+            AccessType accessType = ChooseAccessType();
+            int offset = ChooseOffset(accessType);
+            int size = ChooseSize(offset, sizeClass);
+
+            Span<byte> buffer = _buffers[0].AsSpan(0, size);
+            _random.NextBytes(buffer);
+
+            for (int i = 0; i < _config.Entries.Length; i++)
+            {
+                Entry entry = _config.Entries[i];
+                entry.Storage.Write(offset, buffer).ThrowIfFailure();
+            }
+        }
+
+        private void RunFlush()
+        {
+            foreach (Entry entry in _config.Entries)
+            {
+                entry.Storage.Flush().ThrowIfFailure();
+            }
+
+            if (!CompareBuffers(_backingArrays, _size))
+            {
+                throw new InvalidDataException("Flush");
+            }
+        }
+
+        private Task ChooseTask() => (Task)ChooseProb(_config.TaskProbs);
+        private int ChooseSizeClass() => ChooseProb(_config.SizeClassProbs);
+        private AccessType ChooseAccessType() => (AccessType)ChooseProb(_config.AccessTypeProbs);
+
+        private int ChooseOffset(AccessType type) => type switch
+        {
+            AccessType.Random => _random.Next(0, _size),
+            AccessType.Sequential => _lastAccessEnd == _size ? 0 : _lastAccessEnd,
+            AccessType.FrequentBlock => _frequentAccessOffsets[_random.Next(0, _frequentAccessOffsets.Length)],
+            _ => 0
+        };
+
+        private int ChooseSize(int offset, int sizeClass)
+        {
+            int availableSize = Math.Max(0, _size - offset);
+            int randSize = _random.Next(0, _config.SizeClassMaxSizes[sizeClass]);
+            return Math.Min(availableSize, randSize);
+        }
+
+        private int ChooseProb(int[] weights)
+        {
+            int total = 0;
+            foreach (int weight in weights)
+            {
+                total += weight;
+            }
+
+            int rand = _random.Next(0, total);
+            int currentThreshold = 0;
+
+            for (int i = 0; i < weights.Length; i++)
+            {
+                currentThreshold += weights[i];
+
+                if (rand < currentThreshold)
+                    return i;
+            }
+
+            return 0;
+        }
+
+        private bool CompareBuffers(byte[][] buffers, int size)
+        {
+            Span<byte> baseBuffer = buffers[0].AsSpan(0, size);
+
+            for (int i = 1; i < buffers.Length; i++)
+            {
+                Span<byte> testBuffer = buffers[i].AsSpan(0, size);
+                if (!baseBuffer.SequenceEqual(testBuffer))
+                {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        public readonly struct Entry
+        {
+            public readonly IStorage Storage;
+            public readonly byte[] BackingArray;
+
+            public Entry(IStorage storage, byte[] backingArray)
+            {
+                Storage = storage;
+                BackingArray = backingArray;
+            }
+        }
+
+        private enum Task
+        {
+            Read = 0,
+            Write = 1,
+            Flush = 2
+        }
+
+        private enum AccessType
+        {
+            Random = 0,
+            Sequential = 1,
+            FrequentBlock = 2
+        }
+    }
+}

tests/LibHac.Tests/FsSystem/BufferedStorageTests.cs

@@ -0,0 +1,229 @@
+using System;
+using System.Collections.Generic;
+using LibHac.Fs;
+using LibHac.FsSystem;
+using LibHac.FsSystem.Save;
+using LibHac.Tests.Fs;
+using Xunit;
+
+namespace LibHac.Tests.FsSystem
+{
+    public class BufferedStorageTests
+    {
+        [Fact]
+        public void Write_SingleBlock_CanReadBack()
+        {
+            byte[] buffer = new byte[0x18000];
+            byte[] workBuffer = new byte[0x18000];
+            var bufferManager = new FileSystemBufferManager();
+            Assert.Success(bufferManager.Initialize(5, buffer, 0x4000, workBuffer));
+
+            byte[] storageBuffer = new byte[0x80000];
+            var baseStorage = new SubStorage(new MemoryStorage(storageBuffer), 0, storageBuffer.Length);
+
+            var bufferedStorage = new BufferedStorage();
+            Assert.Success(bufferedStorage.Initialize(baseStorage, bufferManager, 0x4000, 4));
+
+            byte[] writeBuffer = new byte[0x400];
+            byte[] readBuffer = new byte[0x400];
+
+            writeBuffer.AsSpan().Fill(0xAA);
+            Assert.Success(bufferedStorage.Write(0x10000, writeBuffer));
+            Assert.Success(bufferedStorage.Read(0x10000, readBuffer));
+
+            Assert.Equal(writeBuffer, readBuffer);
+        }
+
+        public class AccessTestConfig
+        {
+            public int[] SizeClassProbs { get; set; }
+            public int[] SizeClassMaxSizes { get; set; }
+            public int[] TaskProbs { get; set; }
+            public int[] AccessTypeProbs { get; set; }
+            public ulong RngSeed { get; set; }
+            public int FrequentAccessBlockCount { get; set; }
+            public int BlockSize { get; set; }
+            public int StorageCacheCount { get; set; }
+            public bool EnableBulkRead { get; set; }
+            public int StorageSize { get; set; }
+            public int HeapSize { get; set; }
+            public int HeapBlockSize { get; set; }
+            public int BufferManagerCacheCount { get; set; }
+        }
+
+
+        public static AccessTestConfig[] AccessTestConfigs =
+        {
+            new()
+            {
+                SizeClassProbs = new[] {50, 50, 5},
+                SizeClassMaxSizes = new[] {0x4000, 0x80000, 0x800000}, // 4 KB, 512 KB, 8 MB
+                TaskProbs = new[] {50, 50, 1}, // Read, Write, Flush
+                AccessTypeProbs = new[] {10, 10, 5}, // Random, Sequential, Frequent block
+                RngSeed = 35467,
+                FrequentAccessBlockCount = 6,
+                BlockSize = 0x4000,
+                StorageCacheCount = 40,
+                EnableBulkRead = true,
+                StorageSize = 0x1000000,
+                HeapSize = 0x180000,
+                HeapBlockSize = 0x4000,
+                BufferManagerCacheCount = 50
+            },
+            new()
+            {
+                SizeClassProbs = new[] {50, 50, 5},
+                SizeClassMaxSizes = new[] {0x4000, 0x80000, 0x800000}, // 4 KB, 512 KB, 8 MB
+                TaskProbs = new[] {50, 50, 1}, // Read, Write, Flush
+                AccessTypeProbs = new[] {10, 10, 5}, // Random, Sequential, Frequent block
+                RngSeed = 6548433,
+                FrequentAccessBlockCount = 6,
+                BlockSize = 0x4000,
+                StorageCacheCount = 40,
+                EnableBulkRead = false,
+                StorageSize = 0x1000000,
+                HeapSize = 0x180000,
+                HeapBlockSize = 0x4000,
+                BufferManagerCacheCount = 50
+            },
+            new()
+            {
+                SizeClassProbs = new[] {50, 50, 0},
+                SizeClassMaxSizes = new[] {0x4000, 0x80000, 0x800000}, // 4 KB, 512 KB, 8 MB
+                TaskProbs = new[] {50, 0, 0},
+                AccessTypeProbs = new[] {10, 10, 5}, // Random, Sequential, Frequent block
+                RngSeed = 756478,
+                FrequentAccessBlockCount = 16,
+                BlockSize = 0x4000,
+                StorageCacheCount = 8,
+                EnableBulkRead = true,
+                StorageSize = 0x1000000,
+                HeapSize = 0xE00000,
+                HeapBlockSize = 0x4000,
+                BufferManagerCacheCount = 0x400
+            },
+            new()
+            {
+                SizeClassProbs = new[] {50, 50, 0},
+                SizeClassMaxSizes = new[] {0x4000, 0x80000, 0x800000}, // 4 KB, 512 KB, 8 MB
+                TaskProbs = new[] {50, 0, 0},
+                AccessTypeProbs = new[] {0, 0, 5}, // Random, Sequential, Frequent block
+                RngSeed = 38197549,
+                FrequentAccessBlockCount = 16,
+                BlockSize = 0x4000,
+                StorageCacheCount = 16,
+                EnableBulkRead = false,
+                StorageSize = 0x1000000,
+                HeapSize = 0xE00000,
+                HeapBlockSize = 0x4000,
+                BufferManagerCacheCount = 0x400
+            },
+            new()
+            {
+                SizeClassProbs = new[] {50, 50, 0},
+                SizeClassMaxSizes = new[] {0x4000, 0x80000, 0x800000}, // 4 KB, 512 KB, 8 MB
+                TaskProbs = new[] {50, 50, 1}, // Read, Write, Flush
+                AccessTypeProbs = new[] {10, 10, 5}, // Random, Sequential, Frequent block
+                RngSeed = 567365,
+                FrequentAccessBlockCount = 6,
+                BlockSize = 0x4000,
+                StorageCacheCount = 8,
+                EnableBulkRead = false,
+                StorageSize = 0x100000,
+                HeapSize = 0x180000,
+                HeapBlockSize = 0x4000,
+                BufferManagerCacheCount = 50
+            },
+            new()
+            {
+                SizeClassProbs = new[] {50, 50, 0},
+                SizeClassMaxSizes = new[] {0x4000, 0x80000, 0x800000}, // 4 KB, 512 KB, 8 MB
+                TaskProbs = new[] {50, 50, 1}, // Read, Write, Flush
+                AccessTypeProbs = new[] {10, 10, 5}, // Random, Sequential, Frequent block
+                RngSeed = 949365,
+                FrequentAccessBlockCount = 6,
+                BlockSize = 0x4000,
+                StorageCacheCount = 8,
+                EnableBulkRead = false,
+                StorageSize = 0x100000,
+                HeapSize = 0x180000,
+                HeapBlockSize = 0x4000,
+                BufferManagerCacheCount = 50
+            },
+            new()
+            {
+                SizeClassProbs = new[] {50, 50, 10},
+                SizeClassMaxSizes = new[] {0x4000, 0x80000, 0x800000}, // 4 KB, 512 KB, 8 MB
+                TaskProbs = new[] {50, 50, 1}, // Read, Write, Flush
+                AccessTypeProbs = new[] {10, 10, 5}, // Random, Sequential, Frequent block
+                RngSeed = 670670,
+                FrequentAccessBlockCount = 16,
+                BlockSize = 0x4000,
+                StorageCacheCount = 8,
+                EnableBulkRead = true,
+                StorageSize = 0x1000000,
+                HeapSize = 0xE00000,
+                HeapBlockSize = 0x4000,
+                BufferManagerCacheCount = 0x400
+            }
+        };
+
+        private static TheoryData<T> CreateTheoryData<T>(IEnumerable<T> items)
+        {
+            var output = new TheoryData<T>();
+
+            foreach (T item in items)
+            {
+                output.Add(item);
+            }
+
+            return output;
+        }
+
+        public static TheoryData<AccessTestConfig> AccessTestTheoryData = CreateTheoryData(AccessTestConfigs);
+
+        [Theory]
+        [MemberData(nameof(AccessTestTheoryData))]
+        public void ReadWrite_AccessCorrectnessTestAgainstMemoryStorage(AccessTestConfig config)
+        {
+            int orderMax = FileSystemBuddyHeap.QueryOrderMax((nuint)config.HeapSize, (nuint)config.HeapBlockSize);
+            int workBufferSize = (int)FileSystemBuddyHeap.QueryWorkBufferSize(orderMax);
+            byte[] workBuffer = GC.AllocateArray<byte>(workBufferSize, true);
+            byte[] heapBuffer = new byte[config.HeapSize];
+
+            var bufferManager = new FileSystemBufferManager();
+            Assert.Success(bufferManager.Initialize(config.BufferManagerCacheCount, heapBuffer, config.HeapBlockSize, workBuffer));
+
+            byte[] memoryStorageArray = new byte[config.StorageSize];
+            byte[] bufferedStorageArray = new byte[config.StorageSize];
+
+            var memoryStorage = new MemoryStorage(memoryStorageArray);
+            var baseBufferedStorage = new SubStorage(new MemoryStorage(bufferedStorageArray), 0, bufferedStorageArray.Length);
+
+            var bufferedStorage = new BufferedStorage();
+            Assert.Success(bufferedStorage.Initialize(baseBufferedStorage, bufferManager, config.BlockSize, config.StorageCacheCount));
+
+            if (config.EnableBulkRead)
+            {
+                bufferedStorage.EnableBulkRead();
+            }
+
+            var memoryStorageEntry = new StorageTester.Entry(memoryStorage, memoryStorageArray);
+            var bufferedStorageEntry = new StorageTester.Entry(bufferedStorage, bufferedStorageArray);
+
+            var testerConfig = new StorageTester.Configuration()
+            {
+                Entries = new[] { memoryStorageEntry, bufferedStorageEntry },
+                SizeClassProbs = config.SizeClassProbs,
+                SizeClassMaxSizes = config.SizeClassMaxSizes,
+                TaskProbs = config.TaskProbs,
+                AccessTypeProbs = config.AccessTypeProbs,
+                RngSeed = config.RngSeed,
+                FrequentAccessBlockCount = config.FrequentAccessBlockCount
+            };
+
+            var tester = new StorageTester(testerConfig);
+            tester.Run(0x100);
+        }
+    }
+}

tests/LibHac.Tests/FsSystem/FileSystemBufferManagerTests.cs

@@ -0,0 +1,89 @@
+using LibHac.Fs;
+using LibHac.FsSystem;
+using Xunit;
+
+namespace LibHac.Tests.FsSystem
+{
+    public class FileSystemBufferManagerTests
+    {
+        private FileSystemBufferManager CreateManager(int size, int blockSize = 0x4000, int maxCacheCount = 16)
+        {
+            int orderMax = FileSystemBuddyHeap.QueryOrderMax((nuint)size, (nuint)blockSize);
+            nuint workBufferSize = FileSystemBuddyHeap.QueryWorkBufferSize(orderMax);
+            byte[] workBuffer = new byte[workBufferSize];
+            byte[] heapBuffer = new byte[size];
+
+            var bufferManager = new FileSystemBufferManager();
+            Assert.Success(bufferManager.Initialize(maxCacheCount, heapBuffer, blockSize, workBuffer));
+            return bufferManager;
+        }
+
+        [Fact]
+        public void AllocateBuffer_NoFreeSpace_ReturnsNull()
+        {
+            FileSystemBufferManager manager = CreateManager(0x20000);
+            Buffer buffer1 = manager.AllocateBuffer(0x10000);
+            Buffer buffer2 = manager.AllocateBuffer(0x10000);
+            Buffer buffer3 = manager.AllocateBuffer(0x4000);
+
+            Assert.True(!buffer1.IsNull);
+            Assert.True(!buffer2.IsNull);
+            Assert.True(buffer3.IsNull);
+        }
+
+        [Fact]
+        public void AcquireCache_EntryNotEvicted_ReturnsEntry()
+        {
+            FileSystemBufferManager manager = CreateManager(0x20000);
+            Buffer buffer1 = manager.AllocateBuffer(0x10000);
+
+            long handle = manager.RegisterCache(buffer1, new IBufferManager.BufferAttribute());
+
+            manager.AllocateBuffer(0x10000);
+            Buffer buffer3 = manager.AcquireCache(handle);
+
+            Assert.Equal(buffer1, buffer3);
+        }
+
+        [Fact]
+        public void AcquireCache_EntryEvicted_ReturnsNull()
+        {
+            FileSystemBufferManager manager = CreateManager(0x20000);
+            Buffer buffer1 = manager.AllocateBuffer(0x10000);
+
+            long handle = manager.RegisterCache(buffer1, new IBufferManager.BufferAttribute());
+
+            manager.AllocateBuffer(0x20000);
+            Buffer buffer3 = manager.AcquireCache(handle);
+
+            Assert.True(buffer3.IsNull);
+        }
+
+        [Fact]
+        public void AcquireCache_MultipleEntriesEvicted_OldestAreEvicted()
+        {
+            FileSystemBufferManager manager = CreateManager(0x20000);
+            Buffer buffer1 = manager.AllocateBuffer(0x8000);
+            Buffer buffer2 = manager.AllocateBuffer(0x8000);
+            Buffer buffer3 = manager.AllocateBuffer(0x8000);
+            Buffer buffer4 = manager.AllocateBuffer(0x8000);
+
+            long handle1 = manager.RegisterCache(buffer1, new IBufferManager.BufferAttribute());
+            long handle2 = manager.RegisterCache(buffer2, new IBufferManager.BufferAttribute());
+            long handle3 = manager.RegisterCache(buffer3, new IBufferManager.BufferAttribute());
+            long handle4 = manager.RegisterCache(buffer4, new IBufferManager.BufferAttribute());
+
+            manager.AllocateBuffer(0x10000);
+
+            Buffer buffer1B = manager.AcquireCache(handle1);
+            Buffer buffer2B = manager.AcquireCache(handle2);
+            Buffer buffer3B = manager.AcquireCache(handle3);
+            Buffer buffer4B = manager.AcquireCache(handle4);
+
+            Assert.True(buffer1B.IsNull);
+            Assert.True(buffer2B.IsNull);
+            Assert.Equal(buffer3, buffer3B);
+            Assert.Equal(buffer4, buffer4B);
+        }
+    }
+}

tests/LibHac.Tests/LibHac.Tests.csproj

@@ -2,6 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>net5.0</TargetFramework>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>

tests/LibHac.Tests/Random.cs

@@ -0,0 +1,65 @@
+using System;
+using System.Numerics;
+using System.Runtime.InteropServices;
+
+namespace LibHac.Tests
+{
+    public struct Random
+    {
+        private ulong _state1;
+        private ulong _state2;
+
+        public Random(ulong seed)
+        {
+            ulong x = seed;
+            ulong z = x + 0x9e3779b97f4a7c15;
+            z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9;
+            z = (z ^ (z >> 27)) * 0x94d049bb133111eb;
+            x = z ^ (z >> 31);
+            z = (x += 0x9e3779b97f4a7c15);
+            z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9;
+            z = (z ^ (z >> 27)) * 0x94d049bb133111eb;
+            _state1 = z ^ (z >> 31);
+            _state2 = x;
+        }
+
+        ulong Next()
+        {
+            ulong s0 = _state1;
+            ulong s1 = _state2;
+            ulong result = BitOperations.RotateLeft(s0 + s1, 17) + s0;
+
+            s1 ^= s0;
+            _state1 = BitOperations.RotateLeft(s0, 49) ^ s1 ^ (s1 << 21);
+            _state2 = BitOperations.RotateLeft(s1, 28);
+
+            return result;
+        }
+
+        public int Next(int minValue, int maxValue)
+        {
+            if (minValue > maxValue)
+            {
+                throw new ArgumentOutOfRangeException(nameof(minValue));
+            }
+
+            long range = (long)maxValue - minValue;
+            return (int)((uint)Next() * (1.0 / uint.MaxValue) * range) + minValue;
+        }
+
+        public void NextBytes(Span<byte> buffer)
+        {
+            Span<ulong> bufferUlong = MemoryMarshal.Cast<byte, ulong>(buffer);
+
+            for (int i = 0; i < bufferUlong.Length; i++)
+            {
+                bufferUlong[i] = Next();
+            }
+
+            for (int i = bufferUlong.Length * sizeof(ulong); i < buffer.Length; i++)
+            {
+                buffer[i] = (byte)Next();
+            }
+        }
+    }
+}