Add an initial functional RomFS builder

src/LibHac/HashHelpers.cs

@@ -0,0 +1,95 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+
+using System;
+using System.Diagnostics;
+
+namespace LibHac
+{
+    internal static class HashHelpers
+    {
+        public const int HashCollisionThreshold = 100;
+
+        // This is the maximum prime smaller than Array.MaxArrayLength
+        public const int MaxPrimeArrayLength = 0x7FEFFFFD;
+
+        public const int HashPrime = 101;
+
+        // Table of prime numbers to use as hash table sizes. 
+        // A typical resize algorithm would pick the smallest prime number in this array
+        // that is larger than twice the previous capacity. 
+        // Suppose our Hashtable currently has capacity x and enough elements are added 
+        // such that a resize needs to occur. Resizing first computes 2x then finds the 
+        // first prime in the table greater than 2x, i.e. if primes are ordered 
+        // p_1, p_2, ..., p_i, ..., it finds p_n such that p_n-1 < 2x < p_n. 
+        // Doubling is important for preserving the asymptotic complexity of the 
+        // hashtable operations such as add.  Having a prime guarantees that double 
+        // hashing does not lead to infinite loops.  IE, your hash function will be 
+        // h1(key) + i*h2(key), 0 <= i < size.  h2 and the size must be relatively prime.
+        // We prefer the low computation costs of higher prime numbers over the increased
+        // memory allocation of a fixed prime number i.e. when right sizing a HashSet.
+        public static readonly int[] Primes = {
+            3, 7, 11, 17, 23, 29, 37, 47, 59, 71, 89, 107, 131, 163, 197, 239, 293, 353, 431, 521, 631, 761, 919,
+            1103, 1327, 1597, 1931, 2333, 2801, 3371, 4049, 4861, 5839, 7013, 8419, 10103, 12143, 14591,
+            17519, 21023, 25229, 30293, 36353, 43627, 52361, 62851, 75431, 90523, 108631, 130363, 156437,
+            187751, 225307, 270371, 324449, 389357, 467237, 560689, 672827, 807403, 968897, 1162687, 1395263,
+            1674319, 2009191, 2411033, 2893249, 3471899, 4166287, 4999559, 5999471, 7199369 };
+
+        public static bool IsPrime(int candidate)
+        {
+            if ((candidate & 1) != 0)
+            {
+                int limit = (int)Math.Sqrt(candidate);
+                for (int divisor = 3; divisor <= limit; divisor += 2)
+                {
+                    if ((candidate % divisor) == 0)
+                        return false;
+                }
+                return true;
+            }
+            return (candidate == 2);
+        }
+
+        public static int GetPrime(int min)
+        {
+            if (min < 0)
+                throw new ArgumentException(nameof(min));
+
+            // RomFS dictionaries choose from all possible primes
+
+            //for (int i = 0; i < Primes.Length; i++)
+            //{
+            //    int prime = Primes[i];
+            //    if (prime >= min)
+            //        return prime;
+            //}
+
+            //outside of our predefined table. 
+            //compute the hard way. 
+            for (int i = (min | 1); i < int.MaxValue; i += 2)
+            {
+                if (IsPrime(i) && ((i - 1) % HashPrime != 0))
+                    return i;
+            }
+            return min;
+        }
+
+        // Returns size of hashtable to grow to.
+        public static int ExpandPrime(int oldSize)
+        {
+            int newSize = 2 * oldSize;
+
+            // Allow the hashtables to grow to maximum possible size (~2G elements) before encountering capacity overflow.
+            // Note that this check works even when _items.Length overflowed thanks to the (uint) cast
+            if ((uint)newSize > MaxPrimeArrayLength && MaxPrimeArrayLength > oldSize)
+            {
+                Debug.Assert(MaxPrimeArrayLength == GetPrime(MaxPrimeArrayLength), "Invalid MaxPrimeArrayLength");
+                return MaxPrimeArrayLength;
+            }
+
+            return GetPrime(newSize);
+        }
+    }
+}

src/LibHac/IO/PathTools.cs

@@ -97,6 +97,19 @@ namespace LibHac.IO
             return path.Substring(0, i);
         }
 
+        public static ReadOnlySpan<byte> GetParentDirectory(ReadOnlySpan<byte> path)
+        {
+            int i = path.Length - 1;
+
+            // A trailing separator should be ignored
+            if (path.Length > 0 && path[i] == '/') i--;
+
+            while (i >= 0 && path[i] != '/') i--;
+
+            if (i < 1) return new ReadOnlySpan<byte>(new[] { (byte)'/' });
+            return path.Slice(0, i);
+        }
+
         public static bool IsNormalized(ReadOnlySpan<char> path)
         {
             var state = NormalizeState.Initial;

src/LibHac/IO/RomFs/HierarchicalRomFileTable.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Runtime.InteropServices;
 using System.Text;
 
 namespace LibHac.IO.RomFs
@@ -25,9 +26,37 @@ namespace LibHac.IO.RomFs
             DirectoryTable = new RomFsDictionary<DirectoryRomEntry>(DirHashTableStorage, DirEntryTableStorage);
         }
 
+        public HierarchicalRomFileTable(int directoryCapacity, int fileCapacity)
+        {
+            FileTable = new RomFsDictionary<FileRomEntry>(fileCapacity);
+            DirectoryTable = new RomFsDictionary<DirectoryRomEntry>(directoryCapacity);
+
+            CreateRootDirectory();
+        }
+
+        public byte[] GetDirectoryBuckets()
+        {
+            return MemoryMarshal.Cast<int, byte>(DirectoryTable.GetBucketData()).ToArray();
+        }
+
+        public byte[] GetDirectoryEntries()
+        {
+            return DirectoryTable.GetEntryData().ToArray();
+        }
+
+        public byte[] GetFileBuckets()
+        {
+            return MemoryMarshal.Cast<int, byte>(FileTable.GetBucketData()).ToArray();
+        }
+
+        public byte[] GetFileEntries()
+        {
+            return FileTable.GetEntryData().ToArray();
+        }
+
         public bool OpenFile(string path, out RomFileInfo fileInfo)
         {
-            FindFileRecursive(GetUtf8Bytes(path), out RomEntryKey key);
+            FindFileRecursive(GetUtf8Bytes(path), out RomEntryKey key, out _);
 
             if (FileTable.TryGetValue(ref key, out RomKeyValuePair<FileRomEntry> keyValuePair))
             {
@@ -53,7 +82,7 @@ namespace LibHac.IO.RomFs
 
         public bool OpenDirectory(string path, out FindPosition position)
         {
-            FindDirectoryRecursive(GetUtf8Bytes(path), out RomEntryKey key);
+            FindDirectoryRecursive(GetUtf8Bytes(path), out RomEntryKey key, out _);
 
             if (DirectoryTable.TryGetValue(ref key, out RomKeyValuePair<DirectoryRomEntry> keyValuePair))
             {
@@ -110,21 +139,147 @@ namespace LibHac.IO.RomFs
             return false;
         }
 
-        private void FindFileRecursive(ReadOnlySpan<byte> path, out RomEntryKey key)
+        public void CreateRootDirectory()
         {
-            var parser = new PathParser(path);
+            var key = new RomEntryKey(ReadOnlySpan<byte>.Empty, 0);
-            FindParentDirectoryRecursive(ref parser, out RomKeyValuePair<DirectoryRomEntry> keyValuePair);
+            var entry = new DirectoryRomEntry();
+            entry.NextSibling = -1;
+            entry.Pos.NextDirectory = -1;
+            entry.Pos.NextFile = -1;
 
-            key = keyValuePair.Key;
+            DirectoryTable.Insert(ref key, ref entry);
         }
 
-        private void FindDirectoryRecursive(ReadOnlySpan<byte> path, out RomEntryKey key)
+        public void CreateFile(string path, ref RomFileInfo fileInfo)
+        {
+            path = PathTools.Normalize(path);
+            ReadOnlySpan<byte> pathBytes = GetUtf8Bytes(path);
+
+            ReadOnlySpan<byte> parentPath = PathTools.GetParentDirectory(pathBytes);
+            CreateDirectoryRecursiveInternal(parentPath);
+
+            FindFileRecursive(pathBytes, out RomEntryKey key, out RomKeyValuePair<DirectoryRomEntry> parentEntry);
+
+            if (EntryExists(ref key))
+            {
+                throw new ArgumentException("Path already exists.");
+            }
+
+            var entry = new FileRomEntry();
+            entry.NextSibling = -1;
+            entry.Info = fileInfo;
+
+            int offset = FileTable.Insert(ref key, ref entry);
+
+            if (parentEntry.Value.Pos.NextFile == -1)
+            {
+                parentEntry.Value.Pos.NextFile = offset;
+
+                DirectoryTable.TrySetValue(ref parentEntry.Key, ref parentEntry.Value);
+                return;
+            }
+
+            int nextOffset = parentEntry.Value.Pos.NextFile;
+
+            while (FileTable.TryGetValue(nextOffset, out RomKeyValuePair<FileRomEntry> chainEntry))
+            {
+                if (chainEntry.Value.NextSibling == -1)
+                {
+                    chainEntry.Value.NextSibling = offset;
+                    FileTable.TrySetValue(ref chainEntry.Key, ref chainEntry.Value);
+
+                    return;
+                }
+
+                nextOffset = chainEntry.Value.NextSibling;
+            }
+        }
+
+        public void CreateDirectoryRecursive(string path)
+        {
+            path = PathTools.Normalize(path);
+
+            CreateDirectoryRecursiveInternal(GetUtf8Bytes(path));
+        }
+
+        private void CreateDirectoryRecursiveInternal(ReadOnlySpan<byte> path)
+        {
+            for (int i = 1; i < path.Length; i++)
+            {
+                if (path[i] == '/')
+                {
+                    ReadOnlySpan<byte> subPath = path.Slice(0, i);
+                    CreateDirectoryInternal(subPath);
+                }
+            }
+
+            CreateDirectoryInternal(path);
+        }
+
+        public void CreateDirectory(string path)
+        {
+            path = PathTools.Normalize(path);
+
+            CreateDirectoryInternal(GetUtf8Bytes(path));
+        }
+
+        private void CreateDirectoryInternal(ReadOnlySpan<byte> path)
+        {
+            FindDirectoryRecursive(path, out RomEntryKey key, out RomKeyValuePair<DirectoryRomEntry> parentEntry);
+
+            if (EntryExists(ref key))
+            {
+                return;
+                // throw new ArgumentException("Path already exists.");
+            }
+
+            var entry = new DirectoryRomEntry();
+            entry.NextSibling = -1;
+            entry.Pos.NextDirectory = -1;
+            entry.Pos.NextFile = -1;
+
+            int offset = DirectoryTable.Insert(ref key, ref entry);
+
+            if (parentEntry.Value.Pos.NextDirectory == -1)
+            {
+                parentEntry.Value.Pos.NextDirectory = offset;
+
+                DirectoryTable.TrySetValue(ref parentEntry.Key, ref parentEntry.Value);
+                return;
+            }
+
+            int nextOffset = parentEntry.Value.Pos.NextDirectory;
+
+            while (nextOffset != -1)
+            {
+                DirectoryTable.TryGetValue(nextOffset, out RomKeyValuePair<DirectoryRomEntry> chainEntry);
+                if (chainEntry.Value.NextSibling == -1)
+                {
+                    chainEntry.Value.NextSibling = offset;
+                    DirectoryTable.TrySetValue(ref chainEntry.Key, ref chainEntry.Value);
+
+                    return;
+                }
+
+                nextOffset = chainEntry.Value.NextSibling;
+            }
+        }
+
+        private void FindFileRecursive(ReadOnlySpan<byte> path, out RomEntryKey key, out RomKeyValuePair<DirectoryRomEntry> parentEntry)
         {
             var parser = new PathParser(path);
-            FindParentDirectoryRecursive(ref parser, out RomKeyValuePair<DirectoryRomEntry> keyValuePair);
+            FindParentDirectoryRecursive(ref parser, out parentEntry);
+
+            key = new RomEntryKey(parser.GetCurrent(), parentEntry.Offset);
+        }
+
+        private void FindDirectoryRecursive(ReadOnlySpan<byte> path, out RomEntryKey key, out RomKeyValuePair<DirectoryRomEntry> parentEntry)
+        {
+            var parser = new PathParser(path);
+            FindParentDirectoryRecursive(ref parser, out parentEntry);
 
             ReadOnlySpan<byte> name = parser.GetCurrent();
-            int parentOffset = name.Length == 0 ? 0 : keyValuePair.Offset;
+            int parentOffset = name.Length == 0 ? 0 : parentEntry.Offset;
 
             key = new RomEntryKey(name, parentOffset);
         }
@@ -139,6 +294,19 @@ namespace LibHac.IO.RomFs
                 DirectoryTable.TryGetValue(ref key, out keyValuePair);
                 key.Parent = keyValuePair.Offset;
             }
+
+            // The above loop won't run for top-level directories, so 
+            // manually return the root directory for them
+            if (key.Parent == 0)
+            {
+                DirectoryTable.TryGetValue(0, out keyValuePair);
+            }
+        }
+
+        private bool EntryExists(ref RomEntryKey key)
+        {
+            return DirectoryTable.ContainsKey(ref key) ||
+                   FileTable.ContainsKey(ref key);
         }
     }
 }

src/LibHac/IO/RomFs/RomFsBuilder.cs

@@ -0,0 +1,77 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+namespace LibHac.IO.RomFs
+{
+    public class RomFsBuilder
+    {
+        private const int FileAlignment = 0x10;
+        private const int HeaderSize = 0x50;
+        private const int HeaderWithPaddingSize = 0x200;
+
+        public List<IStorage> Sources { get; } = new List<IStorage>();
+        public HierarchicalRomFileTable FileTable { get; }
+
+        public RomFsBuilder(IFileSystem input)
+        {
+            DirectoryEntry[] entries = input.EnumerateEntries().ToArray();
+            int fileCount = entries.Count(x => x.Type == DirectoryEntryType.File);
+            int dirCount = entries.Count(x => x.Type == DirectoryEntryType.Directory);
+
+            FileTable = new HierarchicalRomFileTable(dirCount, fileCount);
+
+            long offset = 0;
+
+            foreach (DirectoryEntry file in entries.Where(x => x.Type == DirectoryEntryType.File).OrderBy(x => x.FullPath, StringComparer.Ordinal))
+            {
+                var fileInfo = new RomFileInfo();
+                fileInfo.Offset = offset;
+                fileInfo.Length = file.Size;
+
+                IStorage fileStorage = input.OpenFile(file.FullPath, OpenMode.Read).AsStorage();
+                Sources.Add(fileStorage);
+
+                long newOffset = offset + file.Size;
+                offset = Util.AlignUp(newOffset, FileAlignment);
+
+                var padding = new NullStorage(offset - newOffset);
+                Sources.Add(padding);
+
+                FileTable.CreateFile(file.FullPath, ref fileInfo);
+            }
+        }
+
+        public IStorage Build()
+        {
+            var header = new byte[HeaderWithPaddingSize];
+            var headerWriter = new BinaryWriter(new MemoryStream(header));
+
+            var sources = new List<IStorage>();
+            sources.Add(new MemoryStorage(header));
+            sources.AddRange(Sources);
+
+            long fileLength = sources.Sum(x => x.Length);
+
+            headerWriter.Write((long)HeaderSize);
+
+            AddTable(FileTable.GetDirectoryBuckets());
+            AddTable(FileTable.GetDirectoryEntries());
+            AddTable(FileTable.GetFileBuckets());
+            AddTable(FileTable.GetFileEntries());
+
+            headerWriter.Write((long)HeaderWithPaddingSize);
+
+            return new ConcatenationStorage(sources, true);
+
+            void AddTable(byte[] table)
+            {
+                sources.Add(new MemoryStorage(table));
+                headerWriter.Write(fileLength);
+                headerWriter.Write((long)table.Length);
+                fileLength += table.Length;
+            }
+        }
+    }
+}

src/LibHac/IO/RomFs/RomFsDictionary.cs

@@ -6,18 +6,37 @@ namespace LibHac.IO.RomFs
 {
     internal class RomFsDictionary<T> where T : unmanaged
     {
-        private int[] BucketTable { get; }
+        private int _length;
-        private byte[] EntryTable { get; }
+        private int _capacity;
+
+        private int[] Buckets { get; set; }
+        private byte[] Entries { get; set; }
 
         // Hack around not being able to get the size of generic structures
         private readonly int _sizeOfEntry = 12 + Marshal.SizeOf<T>();
 
         public RomFsDictionary(IStorage bucketStorage, IStorage entryStorage)
         {
-            BucketTable = bucketStorage.ToArray<int>();
+            Buckets = bucketStorage.ToArray<int>();
-            EntryTable = entryStorage.ToArray();
+            Entries = entryStorage.ToArray();
+
+            _length = Entries.Length;
+            _capacity = Entries.Length;
         }
 
+        public RomFsDictionary(int capacity)
+        {
+            int size = HashHelpers.GetPrime(capacity);
+
+            Buckets = new int[size];
+            Buckets.AsSpan().Fill(-1);
+            Entries = new byte[(_sizeOfEntry + 0x10) * size]; // Estimate 0x10 bytes per name
+            _capacity = Entries.Length;
+        }
+
+        public ReadOnlySpan<int> GetBucketData() => Buckets.AsSpan();
+        public ReadOnlySpan<byte> GetEntryData() => Entries.AsSpan(0, _length);
+
         public bool TryGetValue(ref RomEntryKey key, out RomKeyValuePair<T> value)
         {
             int i = FindEntry(ref key);
@@ -36,7 +55,7 @@ namespace LibHac.IO.RomFs
 
         public bool TryGetValue(int offset, out RomKeyValuePair<T> value)
         {
-            if (offset < 0 || offset + _sizeOfEntry >= EntryTable.Length)
+            if (offset < 0 || offset + _sizeOfEntry >= Entries.Length)
             {
                 value = default;
                 return false;
@@ -46,14 +65,68 @@ namespace LibHac.IO.RomFs
 
             GetEntryInternal(offset, out RomFsEntry<T> entry, out value.Key.Name);
             value.Value = entry.Value;
+            value.Key.Parent = entry.Parent;
             return true;
         }
 
+        public bool TrySetValue(ref RomEntryKey key, ref T value)
+        {
+            int i = FindEntry(ref key);
+            if (i < 0) return false;
+
+            GetEntryInternal(i, out RomFsEntry<T> entry);
+            entry.Value = value;
+            SetEntryInternal(i, ref entry);
+
+            return true;
+        }
+
+        public bool ContainsKey(ref RomEntryKey key) => FindEntry(ref key) >= 0;
+
+        public int Insert(ref RomEntryKey key, ref T value)
+        {
+            if (ContainsKey(ref key))
+            {
+                throw new ArgumentException("Key already exists in dictionary.");
+            }
+
+            uint hashCode = key.GetRomHashCode();
+
+            int bucket = (int)(hashCode % Buckets.Length);
+            int newOffset = FindOffsetForInsert(ref key);
+
+            var entry = new RomFsEntry<T>();
+            entry.Next = Buckets[bucket];
+            entry.Parent = key.Parent;
+            entry.KeyLength = key.Name.Length;
+            entry.Value = value;
+
+            SetEntryInternal(newOffset, ref entry, ref key.Name);
+
+            Buckets[bucket] = newOffset;
+            return newOffset;
+        }
+
+        private int FindOffsetForInsert(ref RomEntryKey key)
+        {
+            int bytesNeeded = Util.AlignUp(_sizeOfEntry + key.Name.Length, 4);
+
+            if (_length + bytesNeeded > _capacity)
+            {
+                EnsureEntryTableCapacity(_length + bytesNeeded);
+            }
+
+            int offset = _length;
+            _length += bytesNeeded;
+
+            return offset;
+        }
+
         private int FindEntry(ref RomEntryKey key)
         {
             uint hashCode = key.GetRomHashCode();
-            int index = (int)(hashCode % BucketTable.Length);
+            int index = (int)(hashCode % Buckets.Length);
-            int i = BucketTable[index];
+            int i = Buckets[index];
 
             while (i != -1)
             {
@@ -72,7 +145,7 @@ namespace LibHac.IO.RomFs
 
         private void GetEntryInternal(int offset, out RomFsEntry<T> outEntry)
         {
-            outEntry = MemoryMarshal.Read<RomFsEntry<T>>(EntryTable.AsSpan(offset));
+            outEntry = MemoryMarshal.Read<RomFsEntry<T>>(Entries.AsSpan(offset));
         }
 
         private void GetEntryInternal(int offset, out RomFsEntry<T> outEntry, out ReadOnlySpan<byte> entryName)
@@ -84,7 +157,45 @@ namespace LibHac.IO.RomFs
                 throw new InvalidDataException("Rom entry name is too long.");
             }
 
-            entryName = EntryTable.AsSpan(offset + _sizeOfEntry, outEntry.KeyLength);
+            entryName = Entries.AsSpan(offset + _sizeOfEntry, outEntry.KeyLength);
+        }
+
+        private void SetEntryInternal(int offset, ref RomFsEntry<T> entry)
+        {
+            MemoryMarshal.Write(Entries.AsSpan(offset), ref entry);
+        }
+
+        private void SetEntryInternal(int offset, ref RomFsEntry<T> entry, ref ReadOnlySpan<byte> entryName)
+        {
+            MemoryMarshal.Write(Entries.AsSpan(offset), ref entry);
+
+            entryName.CopyTo(Entries.AsSpan(offset + _sizeOfEntry, entry.KeyLength));
+        }
+
+        private void EnsureEntryTableCapacity(int value)
+        {
+            if (value < 0) throw new ArgumentOutOfRangeException(nameof(value));
+            if (value <= _capacity) return;
+
+            long newCapacity = Math.Max(value, 256);
+            newCapacity = Math.Max(newCapacity, _capacity * 2);
+
+            SetCapacity((int)Math.Min(newCapacity, int.MaxValue));
+        }
+
+        private void SetCapacity(int value)
+        {
+            if (value < _length)
+                throw new ArgumentOutOfRangeException(nameof(value), "Capacity is smaller than the current length.");
+
+            if (value != _capacity)
+            {
+                var newBuffer = new byte[value];
+                Buffer.BlockCopy(Entries, 0, newBuffer, 0, _length);
+
+                Entries = newBuffer;
+                _capacity = value;
+            }
         }
     }
 }