mirrors/LibHac
1
0
Fork 0
mirror of https://github.com/Thealexbarney/LibHac.git synced 2024-11-14 10:49:41 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add an initial functional RomFS builder

Browse source
This commit is contained in:
Alex Barney 2019-01-31 21:10:38 -06:00
parent 19cf003160
commit eeb6ebf0a7
5 changed files with 482 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

95
src/LibHac/HashHelpers.cs Normal file
View file

@ -0,0 +1,95 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Diagnostics;
namespace LibHac
{
internal static class HashHelpers
{
public const int HashCollisionThreshold = 100;
// This is the maximum prime smaller than Array.MaxArrayLength
public const int MaxPrimeArrayLength = 0x7FEFFFFD;
public const int HashPrime = 101;
// Table of prime numbers to use as hash table sizes.
// A typical resize algorithm would pick the smallest prime number in this array
// that is larger than twice the previous capacity.
// Suppose our Hashtable currently has capacity x and enough elements are added
// such that a resize needs to occur. Resizing first computes 2x then finds the
// first prime in the table greater than 2x, i.e. if primes are ordered
// p_1, p_2, ..., p_i, ..., it finds p_n such that p_n-1 < 2x < p_n.
// Doubling is important for preserving the asymptotic complexity of the
// hashtable operations such as add. Having a prime guarantees that double
// hashing does not lead to infinite loops. IE, your hash function will be
// h1(key) + i*h2(key), 0 <= i < size. h2 and the size must be relatively prime.
// We prefer the low computation costs of higher prime numbers over the increased
// memory allocation of a fixed prime number i.e. when right sizing a HashSet.
public static readonly int[] Primes = {
3, 7, 11, 17, 23, 29, 37, 47, 59, 71, 89, 107, 131, 163, 197, 239, 293, 353, 431, 521, 631, 761, 919,
1103, 1327, 1597, 1931, 2333, 2801, 3371, 4049, 4861, 5839, 7013, 8419, 10103, 12143, 14591,
17519, 21023, 25229, 30293, 36353, 43627, 52361, 62851, 75431, 90523, 108631, 130363, 156437,
187751, 225307, 270371, 324449, 389357, 467237, 560689, 672827, 807403, 968897, 1162687, 1395263,
1674319, 2009191, 2411033, 2893249, 3471899, 4166287, 4999559, 5999471, 7199369 };
public static bool IsPrime(int candidate)
{
if ((candidate & 1) != 0)
{
int limit = (int)Math.Sqrt(candidate);
for (int divisor = 3; divisor <= limit; divisor += 2)
{
if ((candidate % divisor) == 0)
return false;
}
return true;
}
return (candidate == 2);
}
public static int GetPrime(int min)
{
if (min < 0)
throw new ArgumentException(nameof(min));
// RomFS dictionaries choose from all possible primes
//for (int i = 0; i < Primes.Length; i++)
//{
// int prime = Primes[i];
// if (prime >= min)
// return prime;
//}
//outside of our predefined table.
//compute the hard way.
for (int i = (min | 1); i < int.MaxValue; i += 2)
{
if (IsPrime(i) && ((i - 1) % HashPrime != 0))
return i;
}
return min;
}
// Returns size of hashtable to grow to.
public static int ExpandPrime(int oldSize)
{
int newSize = 2 * oldSize;
// Allow the hashtables to grow to maximum possible size (~2G elements) before encountering capacity overflow.
// Note that this check works even when _items.Length overflowed thanks to the (uint) cast
if ((uint)newSize > MaxPrimeArrayLength && MaxPrimeArrayLength > oldSize)
{
Debug.Assert(MaxPrimeArrayLength == GetPrime(MaxPrimeArrayLength), "Invalid MaxPrimeArrayLength");
return MaxPrimeArrayLength;
}
return GetPrime(newSize);
}
}
}

13
src/LibHac/IO/PathTools.cs
View file

@ -97,6 +97,19 @@ namespace LibHac.IO
return path.Substring(0, i);
}
public static ReadOnlySpan<byte> GetParentDirectory(ReadOnlySpan<byte> path)
{
int i = path.Length - 1;
// A trailing separator should be ignored
if (path.Length > 0 && path[i] == '/') i--;
while (i >= 0 && path[i] != '/') i--;
if (i < 1) return new ReadOnlySpan<byte>(new[] { (byte)'/' });
return path.Slice(0, i);
}
public static bool IsNormalized(ReadOnlySpan<char> path)
{
var state = NormalizeState.Initial;

186
src/LibHac/IO/RomFs/HierarchicalRomFileTable.cs
View file

@ -1,4 +1,5 @@
using System;
using System.Runtime.InteropServices;
using System.Text;
namespace LibHac.IO.RomFs
@ -25,9 +26,37 @@ namespace LibHac.IO.RomFs
DirectoryTable = new RomFsDictionary<DirectoryRomEntry>(DirHashTableStorage, DirEntryTableStorage);
}
public HierarchicalRomFileTable(int directoryCapacity, int fileCapacity)
{
FileTable = new RomFsDictionary<FileRomEntry>(fileCapacity);
DirectoryTable = new RomFsDictionary<DirectoryRomEntry>(directoryCapacity);
CreateRootDirectory();
}
public byte[] GetDirectoryBuckets()
{
return MemoryMarshal.Cast<int, byte>(DirectoryTable.GetBucketData()).ToArray();
}
public byte[] GetDirectoryEntries()
{
return DirectoryTable.GetEntryData().ToArray();
}
public byte[] GetFileBuckets()
{
return MemoryMarshal.Cast<int, byte>(FileTable.GetBucketData()).ToArray();
}
public byte[] GetFileEntries()
{
return FileTable.GetEntryData().ToArray();
}
public bool OpenFile(string path, out RomFileInfo fileInfo)
{
FindFileRecursive(GetUtf8Bytes(path), out RomEntryKey key);
FindFileRecursive(GetUtf8Bytes(path), out RomEntryKey key, out _);
if (FileTable.TryGetValue(ref key, out RomKeyValuePair<FileRomEntry> keyValuePair))
{
@ -53,7 +82,7 @@ namespace LibHac.IO.RomFs
public bool OpenDirectory(string path, out FindPosition position)
{
FindDirectoryRecursive(GetUtf8Bytes(path), out RomEntryKey key);
FindDirectoryRecursive(GetUtf8Bytes(path), out RomEntryKey key, out _);
if (DirectoryTable.TryGetValue(ref key, out RomKeyValuePair<DirectoryRomEntry> keyValuePair))
{
@ -110,21 +139,147 @@ namespace LibHac.IO.RomFs
return false;
}
private void FindFileRecursive(ReadOnlySpan<byte> path, out RomEntryKey key)
public void CreateRootDirectory()
{
var parser = new PathParser(path);
FindParentDirectoryRecursive(ref parser, out RomKeyValuePair<DirectoryRomEntry> keyValuePair);
var key = new RomEntryKey(ReadOnlySpan<byte>.Empty, 0);
var entry = new DirectoryRomEntry();
entry.NextSibling = -1;
entry.Pos.NextDirectory = -1;
entry.Pos.NextFile = -1;
key = keyValuePair.Key;
DirectoryTable.Insert(ref key, ref entry);
}
private void FindDirectoryRecursive(ReadOnlySpan<byte> path, out RomEntryKey key)
public void CreateFile(string path, ref RomFileInfo fileInfo)
{
path = PathTools.Normalize(path);
ReadOnlySpan<byte> pathBytes = GetUtf8Bytes(path);
ReadOnlySpan<byte> parentPath = PathTools.GetParentDirectory(pathBytes);
CreateDirectoryRecursiveInternal(parentPath);
FindFileRecursive(pathBytes, out RomEntryKey key, out RomKeyValuePair<DirectoryRomEntry> parentEntry);
if (EntryExists(ref key))
{
throw new ArgumentException("Path already exists.");
}
var entry = new FileRomEntry();
entry.NextSibling = -1;
entry.Info = fileInfo;
int offset = FileTable.Insert(ref key, ref entry);
if (parentEntry.Value.Pos.NextFile == -1)
{
parentEntry.Value.Pos.NextFile = offset;
DirectoryTable.TrySetValue(ref parentEntry.Key, ref parentEntry.Value);
return;
}
int nextOffset = parentEntry.Value.Pos.NextFile;
while (FileTable.TryGetValue(nextOffset, out RomKeyValuePair<FileRomEntry> chainEntry))
{
if (chainEntry.Value.NextSibling == -1)
{
chainEntry.Value.NextSibling = offset;
FileTable.TrySetValue(ref chainEntry.Key, ref chainEntry.Value);
return;
}
nextOffset = chainEntry.Value.NextSibling;
}
}
public void CreateDirectoryRecursive(string path)
{
path = PathTools.Normalize(path);
CreateDirectoryRecursiveInternal(GetUtf8Bytes(path));
}
private void CreateDirectoryRecursiveInternal(ReadOnlySpan<byte> path)
{
for (int i = 1; i < path.Length; i++)
{
if (path[i] == '/')
{
ReadOnlySpan<byte> subPath = path.Slice(0, i);
CreateDirectoryInternal(subPath);
}
}
CreateDirectoryInternal(path);
}
public void CreateDirectory(string path)
{
path = PathTools.Normalize(path);
CreateDirectoryInternal(GetUtf8Bytes(path));
}
private void CreateDirectoryInternal(ReadOnlySpan<byte> path)
{
FindDirectoryRecursive(path, out RomEntryKey key, out RomKeyValuePair<DirectoryRomEntry> parentEntry);
if (EntryExists(ref key))
{
return;
// throw new ArgumentException("Path already exists.");
}
var entry = new DirectoryRomEntry();
entry.NextSibling = -1;
entry.Pos.NextDirectory = -1;
entry.Pos.NextFile = -1;
int offset = DirectoryTable.Insert(ref key, ref entry);
if (parentEntry.Value.Pos.NextDirectory == -1)
{
parentEntry.Value.Pos.NextDirectory = offset;
DirectoryTable.TrySetValue(ref parentEntry.Key, ref parentEntry.Value);
return;
}
int nextOffset = parentEntry.Value.Pos.NextDirectory;
while (nextOffset != -1)
{
DirectoryTable.TryGetValue(nextOffset, out RomKeyValuePair<DirectoryRomEntry> chainEntry);
if (chainEntry.Value.NextSibling == -1)
{
chainEntry.Value.NextSibling = offset;
DirectoryTable.TrySetValue(ref chainEntry.Key, ref chainEntry.Value);
return;
}
nextOffset = chainEntry.Value.NextSibling;
}
}
private void FindFileRecursive(ReadOnlySpan<byte> path, out RomEntryKey key, out RomKeyValuePair<DirectoryRomEntry> parentEntry)
{
var parser = new PathParser(path);
FindParentDirectoryRecursive(ref parser, out RomKeyValuePair<DirectoryRomEntry> keyValuePair);
FindParentDirectoryRecursive(ref parser, out parentEntry);
key = new RomEntryKey(parser.GetCurrent(), parentEntry.Offset);
}
private void FindDirectoryRecursive(ReadOnlySpan<byte> path, out RomEntryKey key, out RomKeyValuePair<DirectoryRomEntry> parentEntry)
{
var parser = new PathParser(path);
FindParentDirectoryRecursive(ref parser, out parentEntry);
ReadOnlySpan<byte> name = parser.GetCurrent();
int parentOffset = name.Length == 0 ? 0 : keyValuePair.Offset;
int parentOffset = name.Length == 0 ? 0 : parentEntry.Offset;
key = new RomEntryKey(name, parentOffset);
}
@ -139,6 +294,19 @@ namespace LibHac.IO.RomFs
DirectoryTable.TryGetValue(ref key, out keyValuePair);
key.Parent = keyValuePair.Offset;
}
// The above loop won't run for top-level directories, so
// manually return the root directory for them
if (key.Parent == 0)
{
DirectoryTable.TryGetValue(0, out keyValuePair);
}
}
private bool EntryExists(ref RomEntryKey key)
{
return DirectoryTable.ContainsKey(ref key) ||
FileTable.ContainsKey(ref key);
}
}
}

77
src/LibHac/IO/RomFs/RomFsBuilder.cs Normal file
View file

@ -0,0 +1,77 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace LibHac.IO.RomFs
{
public class RomFsBuilder
{
private const int FileAlignment = 0x10;
private const int HeaderSize = 0x50;
private const int HeaderWithPaddingSize = 0x200;
public List<IStorage> Sources { get; } = new List<IStorage>();
public HierarchicalRomFileTable FileTable { get; }
public RomFsBuilder(IFileSystem input)
{
DirectoryEntry[] entries = input.EnumerateEntries().ToArray();
int fileCount = entries.Count(x => x.Type == DirectoryEntryType.File);
int dirCount = entries.Count(x => x.Type == DirectoryEntryType.Directory);
FileTable = new HierarchicalRomFileTable(dirCount, fileCount);
long offset = 0;
foreach (DirectoryEntry file in entries.Where(x => x.Type == DirectoryEntryType.File).OrderBy(x => x.FullPath, StringComparer.Ordinal))
{
var fileInfo = new RomFileInfo();
fileInfo.Offset = offset;
fileInfo.Length = file.Size;
IStorage fileStorage = input.OpenFile(file.FullPath, OpenMode.Read).AsStorage();
Sources.Add(fileStorage);
long newOffset = offset + file.Size;
offset = Util.AlignUp(newOffset, FileAlignment);
var padding = new NullStorage(offset - newOffset);
Sources.Add(padding);
FileTable.CreateFile(file.FullPath, ref fileInfo);
}
}
public IStorage Build()
{
var header = new byte[HeaderWithPaddingSize];
var headerWriter = new BinaryWriter(new MemoryStream(header));
var sources = new List<IStorage>();
sources.Add(new MemoryStorage(header));
sources.AddRange(Sources);
long fileLength = sources.Sum(x => x.Length);
headerWriter.Write((long)HeaderSize);
AddTable(FileTable.GetDirectoryBuckets());
AddTable(FileTable.GetDirectoryEntries());
AddTable(FileTable.GetFileBuckets());
AddTable(FileTable.GetFileEntries());
headerWriter.Write((long)HeaderWithPaddingSize);
return new ConcatenationStorage(sources, true);
void AddTable(byte[] table)
{
sources.Add(new MemoryStorage(table));
headerWriter.Write(fileLength);
headerWriter.Write((long)table.Length);
fileLength += table.Length;
}
}
}
}

129
src/LibHac/IO/RomFs/RomFsDictionary.cs
View file

@ -6,18 +6,37 @@ namespace LibHac.IO.RomFs
{
internal class RomFsDictionary<T> where T : unmanaged
{
private int[] BucketTable { get; }
private byte[] EntryTable { get; }
private int _length;
private int _capacity;
private int[] Buckets { get; set; }
private byte[] Entries { get; set; }
// Hack around not being able to get the size of generic structures
private readonly int _sizeOfEntry = 12 + Marshal.SizeOf<T>();
public RomFsDictionary(IStorage bucketStorage, IStorage entryStorage)
{
BucketTable = bucketStorage.ToArray<int>();
EntryTable = entryStorage.ToArray();
Buckets = bucketStorage.ToArray<int>();
Entries = entryStorage.ToArray();
_length = Entries.Length;
_capacity = Entries.Length;
}
public RomFsDictionary(int capacity)
{
int size = HashHelpers.GetPrime(capacity);
Buckets = new int[size];
Buckets.AsSpan().Fill(-1);
Entries = new byte[(_sizeOfEntry + 0x10) * size]; // Estimate 0x10 bytes per name
_capacity = Entries.Length;
}
public ReadOnlySpan<int> GetBucketData() => Buckets.AsSpan();
public ReadOnlySpan<byte> GetEntryData() => Entries.AsSpan(0, _length);
public bool TryGetValue(ref RomEntryKey key, out RomKeyValuePair<T> value)
{
int i = FindEntry(ref key);
@ -36,7 +55,7 @@ namespace LibHac.IO.RomFs
public bool TryGetValue(int offset, out RomKeyValuePair<T> value)
{
if (offset < 0 || offset + _sizeOfEntry >= EntryTable.Length)
if (offset < 0 || offset + _sizeOfEntry >= Entries.Length)
{
value = default;
return false;
@ -46,14 +65,68 @@ namespace LibHac.IO.RomFs
GetEntryInternal(offset, out RomFsEntry<T> entry, out value.Key.Name);
value.Value = entry.Value;
value.Key.Parent = entry.Parent;
return true;
}
public bool TrySetValue(ref RomEntryKey key, ref T value)
{
int i = FindEntry(ref key);
if (i < 0) return false;
GetEntryInternal(i, out RomFsEntry<T> entry);
entry.Value = value;
SetEntryInternal(i, ref entry);
return true;
}
public bool ContainsKey(ref RomEntryKey key) => FindEntry(ref key) >= 0;
public int Insert(ref RomEntryKey key, ref T value)
{
if (ContainsKey(ref key))
{
throw new ArgumentException("Key already exists in dictionary.");
}
uint hashCode = key.GetRomHashCode();
int bucket = (int)(hashCode % Buckets.Length);
int newOffset = FindOffsetForInsert(ref key);
var entry = new RomFsEntry<T>();
entry.Next = Buckets[bucket];
entry.Parent = key.Parent;
entry.KeyLength = key.Name.Length;
entry.Value = value;
SetEntryInternal(newOffset, ref entry, ref key.Name);
Buckets[bucket] = newOffset;
return newOffset;
}
private int FindOffsetForInsert(ref RomEntryKey key)
{
int bytesNeeded = Util.AlignUp(_sizeOfEntry + key.Name.Length, 4);
if (_length + bytesNeeded > _capacity)
{
EnsureEntryTableCapacity(_length + bytesNeeded);
}
int offset = _length;
_length += bytesNeeded;
return offset;
}
private int FindEntry(ref RomEntryKey key)
{
uint hashCode = key.GetRomHashCode();
int index = (int)(hashCode % BucketTable.Length);
int i = BucketTable[index];
int index = (int)(hashCode % Buckets.Length);
int i = Buckets[index];
while (i != -1)
{
@ -72,7 +145,7 @@ namespace LibHac.IO.RomFs
private void GetEntryInternal(int offset, out RomFsEntry<T> outEntry)
{
outEntry = MemoryMarshal.Read<RomFsEntry<T>>(EntryTable.AsSpan(offset));
outEntry = MemoryMarshal.Read<RomFsEntry<T>>(Entries.AsSpan(offset));
}
private void GetEntryInternal(int offset, out RomFsEntry<T> outEntry, out ReadOnlySpan<byte> entryName)
@ -84,7 +157,45 @@ namespace LibHac.IO.RomFs
throw new InvalidDataException("Rom entry name is too long.");
}
entryName = EntryTable.AsSpan(offset + _sizeOfEntry, outEntry.KeyLength);
entryName = Entries.AsSpan(offset + _sizeOfEntry, outEntry.KeyLength);
}
private void SetEntryInternal(int offset, ref RomFsEntry<T> entry)
{
MemoryMarshal.Write(Entries.AsSpan(offset), ref entry);
}
private void SetEntryInternal(int offset, ref RomFsEntry<T> entry, ref ReadOnlySpan<byte> entryName)
{
MemoryMarshal.Write(Entries.AsSpan(offset), ref entry);
entryName.CopyTo(Entries.AsSpan(offset + _sizeOfEntry, entry.KeyLength));
}
private void EnsureEntryTableCapacity(int value)
{
if (value < 0) throw new ArgumentOutOfRangeException(nameof(value));
if (value <= _capacity) return;
long newCapacity = Math.Max(value, 256);
newCapacity = Math.Max(newCapacity, _capacity * 2);
SetCapacity((int)Math.Min(newCapacity, int.MaxValue));
}
private void SetCapacity(int value)
{
if (value < _length)
throw new ArgumentOutOfRangeException(nameof(value), "Capacity is smaller than the current length.");
if (value != _capacity)
{
var newBuffer = new byte[value];
Buffer.BlockCopy(Entries, 0, newBuffer, 0, _length);
Entries = newBuffer;
_capacity = value;
}
}
}
}