Improve HierarchicalRomFileTable performance

src/LibHac/HashHelpers.cs

@@ -52,6 +52,23 @@ namespace LibHac
             return (candidate == 2);
         }
 
+        public static int GetHashTableEntryCount(int entries)
+        {
+            uint count = (uint) entries;
+            if (entries < 3)
+                count = 3;
+            else if (count < 19)
+                count |= 1;
+            else
+            {
+                while (count % 2 == 0 || count % 3 == 0 || count % 5 == 0 || count % 7 == 0 || count % 11 == 0 || count % 13 == 0 || count % 17 == 0)
+                {
+                    count++;
+                }
+            }
+            return (int) count;
+        }
+
         public static int GetPrime(int min)
         {
             if (min < 0)
@@ -68,6 +85,7 @@ namespace LibHac
 
             //outside of our predefined table. 
             //compute the hard way. 
+
             for (int i = (min | 1); i < int.MaxValue; i += 2)
             {
                 if (IsPrime(i) && ((i - 1) % HashPrime != 0))

src/LibHac/IO/PathTools.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Diagnostics;
 using System.Runtime.CompilerServices;
 
 namespace LibHac.IO
@@ -99,14 +100,16 @@ namespace LibHac.IO
 
         public static ReadOnlySpan<byte> GetParentDirectory(ReadOnlySpan<byte> path)
         {
+            Debug.Assert(IsNormalized(path));
+
             int i = path.Length - 1;
 
             // A trailing separator should be ignored
-            if (path.Length > 0 && path[i] == '/') i--;
+            if (path[i] == '/') i--;
 
-            while (i >= 0 && path[i] != '/') i--;
+            while (i >= 1 && path[i] != '/') i--;
 
-            if (i < 1) return new ReadOnlySpan<byte>(new[] { (byte)'/' });
+            i = Math.Max(i, 1);
             return path.Slice(0, i);
         }
 

src/LibHac/IO/RomFs/HierarchicalRomFileTable.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Text;
 
@@ -56,7 +57,7 @@ namespace LibHac.IO.RomFs
 
         public bool OpenFile(string path, out RomFileInfo fileInfo)
         {
-            FindFileRecursive(GetUtf8Bytes(path), out RomEntryKey key, out _);
+            FindFileRecursive(GetUtf8Bytes(path), out RomEntryKey key);
 
             if (FileTable.TryGetValue(ref key, out RomKeyValuePair<FileRomEntry> keyValuePair))
             {
@@ -82,7 +83,7 @@ namespace LibHac.IO.RomFs
 
         public bool OpenDirectory(string path, out FindPosition position)
         {
-            FindDirectoryRecursive(GetUtf8Bytes(path), out RomEntryKey key, out _);
+            FindDirectoryRecursive(GetUtf8Bytes(path), out RomEntryKey key);
 
             if (DirectoryTable.TryGetValue(ref key, out RomKeyValuePair<DirectoryRomEntry> keyValuePair))
             {
@@ -113,33 +114,48 @@ namespace LibHac.IO.RomFs
 
         public bool FindNextFile(ref FindPosition position, out RomFileInfo info, out string name)
         {
-            if (FileTable.TryGetValue(position.NextFile, out RomKeyValuePair<FileRomEntry> keyValuePair))
+            if (position.NextFile == -1)
             {
-                position.NextFile = keyValuePair.Value.NextSibling;
+                info = default;
-                info = keyValuePair.Value.Info;
+                name = default;
-                name = Encoding.UTF8.GetString(keyValuePair.Key.Name.ToArray());
+                return false;
-                return true;
             }
 
-            info = default;
+            ref FileRomEntry entry = ref FileTable.GetValueReference(position.NextFile, out Span<byte> nameBytes);
-            name = default;
+            position.NextFile = entry.NextSibling;
-            return false;
+            info = entry.Info;
+
+            name = GetUtf8String(nameBytes);
+
+            return true;
         }
 
         public bool FindNextDirectory(ref FindPosition position, out string name)
         {
-            if (DirectoryTable.TryGetValue(position.NextDirectory, out RomKeyValuePair<DirectoryRomEntry> keyValuePair))
+            if (position.NextDirectory == -1)
             {
-                position.NextDirectory = keyValuePair.Value.NextSibling;
+                name = default;
-                name = Encoding.UTF8.GetString(keyValuePair.Key.Name.ToArray());
+                return false;
-                return true;
             }
 
-            name = default;
+            ref DirectoryRomEntry entry = ref DirectoryTable.GetValueReference(position.NextDirectory, out Span<byte> nameBytes);
-            return false;
+            position.NextDirectory = entry.NextSibling;
+            name = GetUtf8String(nameBytes);
+
+            return true;
         }
 
-        public void CreateRootDirectory()
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static string GetUtf8String(ReadOnlySpan<byte> value)
+        {
+#if NETFRAMEWORK
+            return Encoding.UTF8.GetString(value.ToArray());
+#else
+            return Encoding.UTF8.GetString(value);
+#endif
+        }
+
+        private void CreateRootDirectory()
         {
             var key = new RomEntryKey(ReadOnlySpan<byte>.Empty, 0);
             var entry = new DirectoryRomEntry();
@@ -155,158 +171,144 @@ namespace LibHac.IO.RomFs
             path = PathTools.Normalize(path);
             ReadOnlySpan<byte> pathBytes = GetUtf8Bytes(path);
 
-            ReadOnlySpan<byte> parentPath = PathTools.GetParentDirectory(pathBytes);
+            CreateFileRecursiveInternal(pathBytes, ref fileInfo);
-            CreateDirectoryRecursiveInternal(parentPath);
-
-            FindFileRecursive(pathBytes, out RomEntryKey key, out RomKeyValuePair<DirectoryRomEntry> parentEntry);
-
-            if (EntryExists(ref key))
-            {
-                throw new ArgumentException("Path already exists.");
-            }
-
-            var entry = new FileRomEntry();
-            entry.NextSibling = -1;
-            entry.Info = fileInfo;
-
-            int offset = FileTable.Insert(ref key, ref entry);
-
-            if (parentEntry.Value.Pos.NextFile == -1)
-            {
-                parentEntry.Value.Pos.NextFile = offset;
-
-                DirectoryTable.TrySetValue(ref parentEntry.Key, ref parentEntry.Value);
-                return;
-            }
-
-            int nextOffset = parentEntry.Value.Pos.NextFile;
-
-            while (FileTable.TryGetValue(nextOffset, out RomKeyValuePair<FileRomEntry> chainEntry))
-            {
-                if (chainEntry.Value.NextSibling == -1)
-                {
-                    chainEntry.Value.NextSibling = offset;
-                    FileTable.TrySetValue(ref chainEntry.Key, ref chainEntry.Value);
-
-                    return;
-                }
-
-                nextOffset = chainEntry.Value.NextSibling;
-            }
-        }
-
-        public void CreateDirectoryRecursive(string path)
-        {
-            path = PathTools.Normalize(path);
-
-            CreateDirectoryRecursiveInternal(GetUtf8Bytes(path));
-        }
-
-        private void CreateDirectoryRecursiveInternal(ReadOnlySpan<byte> path)
-        {
-            for (int i = 1; i < path.Length; i++)
-            {
-                if (path[i] == '/')
-                {
-                    ReadOnlySpan<byte> subPath = path.Slice(0, i);
-                    CreateDirectoryInternal(subPath);
-                }
-            }
-
-            CreateDirectoryInternal(path);
         }
 
         public void CreateDirectory(string path)
         {
             path = PathTools.Normalize(path);
 
-            CreateDirectoryInternal(GetUtf8Bytes(path));
+            CreateDirectoryRecursive(GetUtf8Bytes(path));
         }
 
-        private void CreateDirectoryInternal(ReadOnlySpan<byte> path)
+        private void CreateDirectoryRecursive(ReadOnlySpan<byte> path)
         {
-            FindDirectoryRecursive(path, out RomEntryKey key, out RomKeyValuePair<DirectoryRomEntry> parentEntry);
+            var parser = new PathParser(path);
+            var key = new RomEntryKey();
 
-            if (EntryExists(ref key))
+            int prevOffset = 0;
+
+            while (parser.TryGetNext(out key.Name))
             {
-                return;
+                int offset = DirectoryTable.GetOffsetFromKey(ref key);
-                // throw new ArgumentException("Path already exists.");
+                if (offset < 0)
-            }
-
-            var entry = new DirectoryRomEntry();
-            entry.NextSibling = -1;
-            entry.Pos.NextDirectory = -1;
-            entry.Pos.NextFile = -1;
-
-            int offset = DirectoryTable.Insert(ref key, ref entry);
-
-            if (parentEntry.Value.Pos.NextDirectory == -1)
-            {
-                parentEntry.Value.Pos.NextDirectory = offset;
-
-                DirectoryTable.TrySetValue(ref parentEntry.Key, ref parentEntry.Value);
-                return;
-            }
-
-            int nextOffset = parentEntry.Value.Pos.NextDirectory;
-
-            while (nextOffset != -1)
-            {
-                DirectoryTable.TryGetValue(nextOffset, out RomKeyValuePair<DirectoryRomEntry> chainEntry);
-                if (chainEntry.Value.NextSibling == -1)
                 {
-                    chainEntry.Value.NextSibling = offset;
+                    ref DirectoryRomEntry entry = ref DirectoryTable.Insert(ref key, out offset, out _);
-                    DirectoryTable.TrySetValue(ref chainEntry.Key, ref chainEntry.Value);
+                    entry.NextSibling = -1;
+                    entry.Pos.NextDirectory = -1;
+                    entry.Pos.NextFile = -1;
 
-                    return;
+                    ref DirectoryRomEntry parent = ref DirectoryTable.GetValueReference(prevOffset);
+
+                    if (parent.Pos.NextDirectory == -1)
+                    {
+                        parent.Pos.NextDirectory = offset;
+                    }
+                    else
+                    {
+                        ref DirectoryRomEntry chain = ref DirectoryTable.GetValueReference(parent.Pos.NextDirectory);
+
+                        while (chain.NextSibling != -1)
+                        {
+                            chain = ref DirectoryTable.GetValueReference(chain.NextSibling);
+                        }
+
+                        chain.NextSibling = offset;
+                    }
                 }
 
-                nextOffset = chainEntry.Value.NextSibling;
+                prevOffset = offset;
+                key.Parent = offset;
             }
         }
 
-        private void FindFileRecursive(ReadOnlySpan<byte> path, out RomEntryKey key, out RomKeyValuePair<DirectoryRomEntry> parentEntry)
+        private void CreateFileRecursiveInternal(ReadOnlySpan<byte> path, ref RomFileInfo fileInfo)
         {
             var parser = new PathParser(path);
-            FindParentDirectoryRecursive(ref parser, out parentEntry);
+            var key = new RomEntryKey();
 
-            key = new RomEntryKey(parser.GetCurrent(), parentEntry.Offset);
+            parser.TryGetNext(out key.Name);
+            int prevOffset = 0;
+
+            while (!parser.IsFinished())
+            {
+                int offset = DirectoryTable.GetOffsetFromKey(ref key);
+                if (offset < 0)
+                {
+                    ref DirectoryRomEntry entry = ref DirectoryTable.Insert(ref key, out offset, out _);
+                    entry.NextSibling = -1;
+                    entry.Pos.NextDirectory = -1;
+                    entry.Pos.NextFile = -1;
+
+                    ref DirectoryRomEntry parent = ref DirectoryTable.GetValueReference(prevOffset);
+
+                    if (parent.Pos.NextDirectory == -1)
+                    {
+                        parent.Pos.NextDirectory = offset;
+                    }
+                    else
+                    {
+                        ref DirectoryRomEntry chain = ref DirectoryTable.GetValueReference(parent.Pos.NextDirectory);
+
+                        while (chain.NextSibling != -1)
+                        {
+                            chain = ref DirectoryTable.GetValueReference(chain.NextSibling);
+                        }
+
+                        chain.NextSibling = offset;
+                    }
+                }
+
+                prevOffset = offset;
+                key.Parent = offset;
+                parser.TryGetNext(out key.Name);
+            }
+
+            {
+                ref FileRomEntry entry = ref FileTable.Insert(ref key, out int offset, out _);
+                entry.NextSibling = -1;
+                entry.Info = fileInfo;
+
+                ref DirectoryRomEntry parent = ref DirectoryTable.GetValueReference(prevOffset);
+
+                if (parent.Pos.NextFile == -1)
+                {
+                    parent.Pos.NextFile = offset;
+                }
+                else
+                {
+                    ref FileRomEntry chain = ref FileTable.GetValueReference(parent.Pos.NextFile);
+
+                    while (chain.NextSibling != -1)
+                    {
+                        chain = ref FileTable.GetValueReference(chain.NextSibling);
+                    }
+
+                    chain.NextSibling = offset;
+                }
+            }
         }
 
-        private void FindDirectoryRecursive(ReadOnlySpan<byte> path, out RomEntryKey key, out RomKeyValuePair<DirectoryRomEntry> parentEntry)
+        private void FindFileRecursive(ReadOnlySpan<byte> path, out RomEntryKey key)
         {
             var parser = new PathParser(path);
-            FindParentDirectoryRecursive(ref parser, out parentEntry);
+            key = default;
-
-            ReadOnlySpan<byte> name = parser.GetCurrent();
-            int parentOffset = name.Length == 0 ? 0 : parentEntry.Offset;
-
-            key = new RomEntryKey(name, parentOffset);
-        }
-
-        private void FindParentDirectoryRecursive(ref PathParser parser, out RomKeyValuePair<DirectoryRomEntry> keyValuePair)
-        {
-            keyValuePair = default;
-            RomEntryKey key = default;
 
             while (parser.TryGetNext(out key.Name) && !parser.IsFinished())
             {
-                DirectoryTable.TryGetValue(ref key, out keyValuePair);
+                key.Parent = DirectoryTable.GetOffsetFromKey(ref key);
-                key.Parent = keyValuePair.Offset;
-            }
-
-            // The above loop won't run for top-level directories, so 
-            // manually return the root directory for them
-            if (key.Parent == 0)
-            {
-                DirectoryTable.TryGetValue(0, out keyValuePair);
             }
         }
 
-        private bool EntryExists(ref RomEntryKey key)
+        private void FindDirectoryRecursive(ReadOnlySpan<byte> path, out RomEntryKey key)
         {
-            return DirectoryTable.ContainsKey(ref key) ||
+            var parser = new PathParser(path);
-                   FileTable.ContainsKey(ref key);
+            key = default;
+
+            while (parser.TryGetNext(out key.Name) && !parser.IsFinished())
+            {
+                key.Parent = DirectoryTable.GetOffsetFromKey(ref key);
+            }
         }
 
         [StructLayout(LayoutKind.Sequential, Pack = 4)]

src/LibHac/IO/RomFs/RomFsDictionary.cs

@@ -29,7 +29,7 @@ namespace LibHac.IO.RomFs
 
         public RomFsDictionary(int capacity)
         {
-            int size = HashHelpers.GetPrime(capacity);
+            int size = HashHelpers.GetHashTableEntryCount(capacity);
 
             Buckets = new int[size];
             Buckets.AsSpan().Fill(-1);
@@ -85,6 +85,13 @@ namespace LibHac.IO.RomFs
             return true;
         }
 
+        public ref T GetValue(int offset, out Span<byte> name)
+        {
+            ref RomFsEntry entry = ref GetEntryReference(offset, out name);
+
+            return ref entry.Value;
+        }
+
         public bool ContainsKey(ref RomEntryKey key) => FindEntry(ref key) >= 0;
 
         public int Insert(ref RomEntryKey key, ref T value)
@@ -97,7 +104,7 @@ namespace LibHac.IO.RomFs
             uint hashCode = key.GetRomHashCode();
 
             int bucket = (int)(hashCode % Buckets.Length);
-            int newOffset = FindOffsetForInsert(ref key);
+            int newOffset = FindOffsetForInsert(key.Name.Length);
 
             ref RomFsEntry entry = ref GetEntryReference(newOffset, out Span<byte> name, key.Name.Length);
 
@@ -113,9 +120,31 @@ namespace LibHac.IO.RomFs
             return newOffset;
         }
 
-        private int FindOffsetForInsert(ref RomEntryKey key)
+        public ref T Insert(ref RomEntryKey key, out int offset, out Span<byte> name)
         {
-            int bytesNeeded = Util.AlignUp(_sizeOfEntry + key.Name.Length, 4);
+            uint hashCode = key.GetRomHashCode();
+
+            int bucket = (int)(hashCode % Buckets.Length);
+            int newOffset = FindOffsetForInsert(key.Name.Length);
+
+            ref RomFsEntry entry = ref GetEntryReference(newOffset, out name, key.Name.Length);
+
+            entry.KeyLength = key.Name.Length;
+
+            entry.Next = Buckets[bucket];
+            entry.Parent = key.Parent;
+            key.Name.CopyTo(name);
+
+            Buckets[bucket] = newOffset;
+            _count++;
+
+            offset = newOffset;
+            return ref entry.Value;
+        }
+
+        private int FindOffsetForInsert(int nameLength)
+        {
+            int bytesNeeded = Util.AlignUp(_sizeOfEntry + nameLength, 4);
 
             if (_length + bytesNeeded > _capacity)
             {
@@ -149,6 +178,27 @@ namespace LibHac.IO.RomFs
             return i;
         }
 
+        public int GetOffsetFromKey(ref RomEntryKey key)
+        {
+            uint hashCode = key.GetRomHashCode();
+            int index = (int)(hashCode % Buckets.Length);
+            int i = Buckets[index];
+
+            while (i != -1)
+            {
+                ref RomFsEntry entry = ref GetEntryReference(i, out Span<byte> name);
+
+                if (key.Parent == entry.Parent && key.Name.SequenceEqual(name))
+                {
+                    break;
+                }
+
+                i = entry.Next;
+            }
+
+            return i;
+        }
+
         private void EnsureCapacityBytes(int value)
         {
             if (value < 0) throw new ArgumentOutOfRangeException(nameof(value));
@@ -216,6 +266,20 @@ namespace LibHac.IO.RomFs
             Buckets = newBuckets;
         }
 
+        public ref T GetValueReference(int offset)
+        {
+            ref RomFsEntry entry = ref MemoryMarshal.Cast<byte, RomFsEntry>(Entries.AsSpan(offset))[0];
+            return ref entry.Value;
+        }
+
+        public ref T GetValueReference(int offset, out Span<byte> name)
+        {
+            ref RomFsEntry entry = ref MemoryMarshal.Cast<byte, RomFsEntry>(Entries.AsSpan(offset))[0];
+
+            name = Entries.AsSpan(offset + _sizeOfEntry, entry.KeyLength);
+            return ref entry.Value;
+        }
+
         private ref RomFsEntry GetEntryReference(int offset)
         {
             return ref MemoryMarshal.Cast<byte, RomFsEntry>(Entries.AsSpan(offset))[0];