diff --git a/src/LibHac/HashHelpers.cs b/src/LibHac/HashHelpers.cs new file mode 100644 index 00000000..b1412c38 --- /dev/null +++ b/src/LibHac/HashHelpers.cs @@ -0,0 +1,95 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + + +using System; +using System.Diagnostics; + +namespace LibHac +{ + internal static class HashHelpers + { + public const int HashCollisionThreshold = 100; + + // This is the maximum prime smaller than Array.MaxArrayLength + public const int MaxPrimeArrayLength = 0x7FEFFFFD; + + public const int HashPrime = 101; + + // Table of prime numbers to use as hash table sizes. + // A typical resize algorithm would pick the smallest prime number in this array + // that is larger than twice the previous capacity. + // Suppose our Hashtable currently has capacity x and enough elements are added + // such that a resize needs to occur. Resizing first computes 2x then finds the + // first prime in the table greater than 2x, i.e. if primes are ordered + // p_1, p_2, ..., p_i, ..., it finds p_n such that p_n-1 < 2x < p_n. + // Doubling is important for preserving the asymptotic complexity of the + // hashtable operations such as add. Having a prime guarantees that double + // hashing does not lead to infinite loops. IE, your hash function will be + // h1(key) + i*h2(key), 0 <= i < size. h2 and the size must be relatively prime. + // We prefer the low computation costs of higher prime numbers over the increased + // memory allocation of a fixed prime number i.e. when right sizing a HashSet. + public static readonly int[] Primes = { + 3, 7, 11, 17, 23, 29, 37, 47, 59, 71, 89, 107, 131, 163, 197, 239, 293, 353, 431, 521, 631, 761, 919, + 1103, 1327, 1597, 1931, 2333, 2801, 3371, 4049, 4861, 5839, 7013, 8419, 10103, 12143, 14591, + 17519, 21023, 25229, 30293, 36353, 43627, 52361, 62851, 75431, 90523, 108631, 130363, 156437, + 187751, 225307, 270371, 324449, 389357, 467237, 560689, 672827, 807403, 968897, 1162687, 1395263, + 1674319, 2009191, 2411033, 2893249, 3471899, 4166287, 4999559, 5999471, 7199369 }; + + public static bool IsPrime(int candidate) + { + if ((candidate & 1) != 0) + { + int limit = (int)Math.Sqrt(candidate); + for (int divisor = 3; divisor <= limit; divisor += 2) + { + if ((candidate % divisor) == 0) + return false; + } + return true; + } + return (candidate == 2); + } + + public static int GetPrime(int min) + { + if (min < 0) + throw new ArgumentException(nameof(min)); + + // RomFS dictionaries choose from all possible primes + + //for (int i = 0; i < Primes.Length; i++) + //{ + // int prime = Primes[i]; + // if (prime >= min) + // return prime; + //} + + //outside of our predefined table. + //compute the hard way. + for (int i = (min | 1); i < int.MaxValue; i += 2) + { + if (IsPrime(i) && ((i - 1) % HashPrime != 0)) + return i; + } + return min; + } + + // Returns size of hashtable to grow to. + public static int ExpandPrime(int oldSize) + { + int newSize = 2 * oldSize; + + // Allow the hashtables to grow to maximum possible size (~2G elements) before encountering capacity overflow. + // Note that this check works even when _items.Length overflowed thanks to the (uint) cast + if ((uint)newSize > MaxPrimeArrayLength && MaxPrimeArrayLength > oldSize) + { + Debug.Assert(MaxPrimeArrayLength == GetPrime(MaxPrimeArrayLength), "Invalid MaxPrimeArrayLength"); + return MaxPrimeArrayLength; + } + + return GetPrime(newSize); + } + } +} \ No newline at end of file diff --git a/src/LibHac/IO/PathTools.cs b/src/LibHac/IO/PathTools.cs index f2c603ab..68499cd8 100644 --- a/src/LibHac/IO/PathTools.cs +++ b/src/LibHac/IO/PathTools.cs @@ -97,6 +97,19 @@ namespace LibHac.IO return path.Substring(0, i); } + public static ReadOnlySpan GetParentDirectory(ReadOnlySpan path) + { + int i = path.Length - 1; + + // A trailing separator should be ignored + if (path.Length > 0 && path[i] == '/') i--; + + while (i >= 0 && path[i] != '/') i--; + + if (i < 1) return new ReadOnlySpan(new[] { (byte)'/' }); + return path.Slice(0, i); + } + public static bool IsNormalized(ReadOnlySpan path) { var state = NormalizeState.Initial; diff --git a/src/LibHac/IO/RomFs/HierarchicalRomFileTable.cs b/src/LibHac/IO/RomFs/HierarchicalRomFileTable.cs index 1eb71cec..d5a50065 100644 --- a/src/LibHac/IO/RomFs/HierarchicalRomFileTable.cs +++ b/src/LibHac/IO/RomFs/HierarchicalRomFileTable.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.InteropServices; using System.Text; namespace LibHac.IO.RomFs @@ -25,9 +26,37 @@ namespace LibHac.IO.RomFs DirectoryTable = new RomFsDictionary(DirHashTableStorage, DirEntryTableStorage); } + public HierarchicalRomFileTable(int directoryCapacity, int fileCapacity) + { + FileTable = new RomFsDictionary(fileCapacity); + DirectoryTable = new RomFsDictionary(directoryCapacity); + + CreateRootDirectory(); + } + + public byte[] GetDirectoryBuckets() + { + return MemoryMarshal.Cast(DirectoryTable.GetBucketData()).ToArray(); + } + + public byte[] GetDirectoryEntries() + { + return DirectoryTable.GetEntryData().ToArray(); + } + + public byte[] GetFileBuckets() + { + return MemoryMarshal.Cast(FileTable.GetBucketData()).ToArray(); + } + + public byte[] GetFileEntries() + { + return FileTable.GetEntryData().ToArray(); + } + public bool OpenFile(string path, out RomFileInfo fileInfo) { - FindFileRecursive(GetUtf8Bytes(path), out RomEntryKey key); + FindFileRecursive(GetUtf8Bytes(path), out RomEntryKey key, out _); if (FileTable.TryGetValue(ref key, out RomKeyValuePair keyValuePair)) { @@ -53,7 +82,7 @@ namespace LibHac.IO.RomFs public bool OpenDirectory(string path, out FindPosition position) { - FindDirectoryRecursive(GetUtf8Bytes(path), out RomEntryKey key); + FindDirectoryRecursive(GetUtf8Bytes(path), out RomEntryKey key, out _); if (DirectoryTable.TryGetValue(ref key, out RomKeyValuePair keyValuePair)) { @@ -110,21 +139,147 @@ namespace LibHac.IO.RomFs return false; } - private void FindFileRecursive(ReadOnlySpan path, out RomEntryKey key) + public void CreateRootDirectory() { - var parser = new PathParser(path); - FindParentDirectoryRecursive(ref parser, out RomKeyValuePair keyValuePair); + var key = new RomEntryKey(ReadOnlySpan.Empty, 0); + var entry = new DirectoryRomEntry(); + entry.NextSibling = -1; + entry.Pos.NextDirectory = -1; + entry.Pos.NextFile = -1; - key = keyValuePair.Key; + DirectoryTable.Insert(ref key, ref entry); } - private void FindDirectoryRecursive(ReadOnlySpan path, out RomEntryKey key) + public void CreateFile(string path, ref RomFileInfo fileInfo) + { + path = PathTools.Normalize(path); + ReadOnlySpan pathBytes = GetUtf8Bytes(path); + + ReadOnlySpan parentPath = PathTools.GetParentDirectory(pathBytes); + CreateDirectoryRecursiveInternal(parentPath); + + FindFileRecursive(pathBytes, out RomEntryKey key, out RomKeyValuePair parentEntry); + + if (EntryExists(ref key)) + { + throw new ArgumentException("Path already exists."); + } + + var entry = new FileRomEntry(); + entry.NextSibling = -1; + entry.Info = fileInfo; + + int offset = FileTable.Insert(ref key, ref entry); + + if (parentEntry.Value.Pos.NextFile == -1) + { + parentEntry.Value.Pos.NextFile = offset; + + DirectoryTable.TrySetValue(ref parentEntry.Key, ref parentEntry.Value); + return; + } + + int nextOffset = parentEntry.Value.Pos.NextFile; + + while (FileTable.TryGetValue(nextOffset, out RomKeyValuePair chainEntry)) + { + if (chainEntry.Value.NextSibling == -1) + { + chainEntry.Value.NextSibling = offset; + FileTable.TrySetValue(ref chainEntry.Key, ref chainEntry.Value); + + return; + } + + nextOffset = chainEntry.Value.NextSibling; + } + } + + public void CreateDirectoryRecursive(string path) + { + path = PathTools.Normalize(path); + + CreateDirectoryRecursiveInternal(GetUtf8Bytes(path)); + } + + private void CreateDirectoryRecursiveInternal(ReadOnlySpan path) + { + for (int i = 1; i < path.Length; i++) + { + if (path[i] == '/') + { + ReadOnlySpan subPath = path.Slice(0, i); + CreateDirectoryInternal(subPath); + } + } + + CreateDirectoryInternal(path); + } + + public void CreateDirectory(string path) + { + path = PathTools.Normalize(path); + + CreateDirectoryInternal(GetUtf8Bytes(path)); + } + + private void CreateDirectoryInternal(ReadOnlySpan path) + { + FindDirectoryRecursive(path, out RomEntryKey key, out RomKeyValuePair parentEntry); + + if (EntryExists(ref key)) + { + return; + // throw new ArgumentException("Path already exists."); + } + + var entry = new DirectoryRomEntry(); + entry.NextSibling = -1; + entry.Pos.NextDirectory = -1; + entry.Pos.NextFile = -1; + + int offset = DirectoryTable.Insert(ref key, ref entry); + + if (parentEntry.Value.Pos.NextDirectory == -1) + { + parentEntry.Value.Pos.NextDirectory = offset; + + DirectoryTable.TrySetValue(ref parentEntry.Key, ref parentEntry.Value); + return; + } + + int nextOffset = parentEntry.Value.Pos.NextDirectory; + + while (nextOffset != -1) + { + DirectoryTable.TryGetValue(nextOffset, out RomKeyValuePair chainEntry); + if (chainEntry.Value.NextSibling == -1) + { + chainEntry.Value.NextSibling = offset; + DirectoryTable.TrySetValue(ref chainEntry.Key, ref chainEntry.Value); + + return; + } + + nextOffset = chainEntry.Value.NextSibling; + } + } + + private void FindFileRecursive(ReadOnlySpan path, out RomEntryKey key, out RomKeyValuePair parentEntry) { var parser = new PathParser(path); - FindParentDirectoryRecursive(ref parser, out RomKeyValuePair keyValuePair); + FindParentDirectoryRecursive(ref parser, out parentEntry); + + key = new RomEntryKey(parser.GetCurrent(), parentEntry.Offset); + } + + private void FindDirectoryRecursive(ReadOnlySpan path, out RomEntryKey key, out RomKeyValuePair parentEntry) + { + var parser = new PathParser(path); + FindParentDirectoryRecursive(ref parser, out parentEntry); ReadOnlySpan name = parser.GetCurrent(); - int parentOffset = name.Length == 0 ? 0 : keyValuePair.Offset; + int parentOffset = name.Length == 0 ? 0 : parentEntry.Offset; key = new RomEntryKey(name, parentOffset); } @@ -139,6 +294,19 @@ namespace LibHac.IO.RomFs DirectoryTable.TryGetValue(ref key, out keyValuePair); key.Parent = keyValuePair.Offset; } + + // The above loop won't run for top-level directories, so + // manually return the root directory for them + if (key.Parent == 0) + { + DirectoryTable.TryGetValue(0, out keyValuePair); + } + } + + private bool EntryExists(ref RomEntryKey key) + { + return DirectoryTable.ContainsKey(ref key) || + FileTable.ContainsKey(ref key); } } } diff --git a/src/LibHac/IO/RomFs/RomFsBuilder.cs b/src/LibHac/IO/RomFs/RomFsBuilder.cs new file mode 100644 index 00000000..aa6f671e --- /dev/null +++ b/src/LibHac/IO/RomFs/RomFsBuilder.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace LibHac.IO.RomFs +{ + public class RomFsBuilder + { + private const int FileAlignment = 0x10; + private const int HeaderSize = 0x50; + private const int HeaderWithPaddingSize = 0x200; + + public List Sources { get; } = new List(); + public HierarchicalRomFileTable FileTable { get; } + + public RomFsBuilder(IFileSystem input) + { + DirectoryEntry[] entries = input.EnumerateEntries().ToArray(); + int fileCount = entries.Count(x => x.Type == DirectoryEntryType.File); + int dirCount = entries.Count(x => x.Type == DirectoryEntryType.Directory); + + FileTable = new HierarchicalRomFileTable(dirCount, fileCount); + + long offset = 0; + + foreach (DirectoryEntry file in entries.Where(x => x.Type == DirectoryEntryType.File).OrderBy(x => x.FullPath, StringComparer.Ordinal)) + { + var fileInfo = new RomFileInfo(); + fileInfo.Offset = offset; + fileInfo.Length = file.Size; + + IStorage fileStorage = input.OpenFile(file.FullPath, OpenMode.Read).AsStorage(); + Sources.Add(fileStorage); + + long newOffset = offset + file.Size; + offset = Util.AlignUp(newOffset, FileAlignment); + + var padding = new NullStorage(offset - newOffset); + Sources.Add(padding); + + FileTable.CreateFile(file.FullPath, ref fileInfo); + } + } + + public IStorage Build() + { + var header = new byte[HeaderWithPaddingSize]; + var headerWriter = new BinaryWriter(new MemoryStream(header)); + + var sources = new List(); + sources.Add(new MemoryStorage(header)); + sources.AddRange(Sources); + + long fileLength = sources.Sum(x => x.Length); + + headerWriter.Write((long)HeaderSize); + + AddTable(FileTable.GetDirectoryBuckets()); + AddTable(FileTable.GetDirectoryEntries()); + AddTable(FileTable.GetFileBuckets()); + AddTable(FileTable.GetFileEntries()); + + headerWriter.Write((long)HeaderWithPaddingSize); + + return new ConcatenationStorage(sources, true); + + void AddTable(byte[] table) + { + sources.Add(new MemoryStorage(table)); + headerWriter.Write(fileLength); + headerWriter.Write((long)table.Length); + fileLength += table.Length; + } + } + } +} diff --git a/src/LibHac/IO/RomFs/RomFsDictionary.cs b/src/LibHac/IO/RomFs/RomFsDictionary.cs index 4b9b3096..b30ac1ad 100644 --- a/src/LibHac/IO/RomFs/RomFsDictionary.cs +++ b/src/LibHac/IO/RomFs/RomFsDictionary.cs @@ -6,18 +6,37 @@ namespace LibHac.IO.RomFs { internal class RomFsDictionary where T : unmanaged { - private int[] BucketTable { get; } - private byte[] EntryTable { get; } + private int _length; + private int _capacity; + + private int[] Buckets { get; set; } + private byte[] Entries { get; set; } // Hack around not being able to get the size of generic structures private readonly int _sizeOfEntry = 12 + Marshal.SizeOf(); public RomFsDictionary(IStorage bucketStorage, IStorage entryStorage) { - BucketTable = bucketStorage.ToArray(); - EntryTable = entryStorage.ToArray(); + Buckets = bucketStorage.ToArray(); + Entries = entryStorage.ToArray(); + + _length = Entries.Length; + _capacity = Entries.Length; } + public RomFsDictionary(int capacity) + { + int size = HashHelpers.GetPrime(capacity); + + Buckets = new int[size]; + Buckets.AsSpan().Fill(-1); + Entries = new byte[(_sizeOfEntry + 0x10) * size]; // Estimate 0x10 bytes per name + _capacity = Entries.Length; + } + + public ReadOnlySpan GetBucketData() => Buckets.AsSpan(); + public ReadOnlySpan GetEntryData() => Entries.AsSpan(0, _length); + public bool TryGetValue(ref RomEntryKey key, out RomKeyValuePair value) { int i = FindEntry(ref key); @@ -36,7 +55,7 @@ namespace LibHac.IO.RomFs public bool TryGetValue(int offset, out RomKeyValuePair value) { - if (offset < 0 || offset + _sizeOfEntry >= EntryTable.Length) + if (offset < 0 || offset + _sizeOfEntry >= Entries.Length) { value = default; return false; @@ -46,14 +65,68 @@ namespace LibHac.IO.RomFs GetEntryInternal(offset, out RomFsEntry entry, out value.Key.Name); value.Value = entry.Value; + value.Key.Parent = entry.Parent; return true; } + public bool TrySetValue(ref RomEntryKey key, ref T value) + { + int i = FindEntry(ref key); + if (i < 0) return false; + + GetEntryInternal(i, out RomFsEntry entry); + entry.Value = value; + SetEntryInternal(i, ref entry); + + return true; + } + + public bool ContainsKey(ref RomEntryKey key) => FindEntry(ref key) >= 0; + + public int Insert(ref RomEntryKey key, ref T value) + { + if (ContainsKey(ref key)) + { + throw new ArgumentException("Key already exists in dictionary."); + } + + uint hashCode = key.GetRomHashCode(); + + int bucket = (int)(hashCode % Buckets.Length); + int newOffset = FindOffsetForInsert(ref key); + + var entry = new RomFsEntry(); + entry.Next = Buckets[bucket]; + entry.Parent = key.Parent; + entry.KeyLength = key.Name.Length; + entry.Value = value; + + SetEntryInternal(newOffset, ref entry, ref key.Name); + + Buckets[bucket] = newOffset; + return newOffset; + } + + private int FindOffsetForInsert(ref RomEntryKey key) + { + int bytesNeeded = Util.AlignUp(_sizeOfEntry + key.Name.Length, 4); + + if (_length + bytesNeeded > _capacity) + { + EnsureEntryTableCapacity(_length + bytesNeeded); + } + + int offset = _length; + _length += bytesNeeded; + + return offset; + } + private int FindEntry(ref RomEntryKey key) { uint hashCode = key.GetRomHashCode(); - int index = (int)(hashCode % BucketTable.Length); - int i = BucketTable[index]; + int index = (int)(hashCode % Buckets.Length); + int i = Buckets[index]; while (i != -1) { @@ -72,7 +145,7 @@ namespace LibHac.IO.RomFs private void GetEntryInternal(int offset, out RomFsEntry outEntry) { - outEntry = MemoryMarshal.Read>(EntryTable.AsSpan(offset)); + outEntry = MemoryMarshal.Read>(Entries.AsSpan(offset)); } private void GetEntryInternal(int offset, out RomFsEntry outEntry, out ReadOnlySpan entryName) @@ -84,7 +157,45 @@ namespace LibHac.IO.RomFs throw new InvalidDataException("Rom entry name is too long."); } - entryName = EntryTable.AsSpan(offset + _sizeOfEntry, outEntry.KeyLength); + entryName = Entries.AsSpan(offset + _sizeOfEntry, outEntry.KeyLength); + } + + private void SetEntryInternal(int offset, ref RomFsEntry entry) + { + MemoryMarshal.Write(Entries.AsSpan(offset), ref entry); + } + + private void SetEntryInternal(int offset, ref RomFsEntry entry, ref ReadOnlySpan entryName) + { + MemoryMarshal.Write(Entries.AsSpan(offset), ref entry); + + entryName.CopyTo(Entries.AsSpan(offset + _sizeOfEntry, entry.KeyLength)); + } + + private void EnsureEntryTableCapacity(int value) + { + if (value < 0) throw new ArgumentOutOfRangeException(nameof(value)); + if (value <= _capacity) return; + + long newCapacity = Math.Max(value, 256); + newCapacity = Math.Max(newCapacity, _capacity * 2); + + SetCapacity((int)Math.Min(newCapacity, int.MaxValue)); + } + + private void SetCapacity(int value) + { + if (value < _length) + throw new ArgumentOutOfRangeException(nameof(value), "Capacity is smaller than the current length."); + + if (value != _capacity) + { + var newBuffer = new byte[value]; + Buffer.BlockCopy(Entries, 0, newBuffer, 0, _length); + + Entries = newBuffer; + _capacity = value; + } } } }