From 34e926f2a4bb45d17b0d36437b63d71628b24cae Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Wed, 30 Jan 2019 15:31:53 -0600 Subject: [PATCH] Don't use IStorage for reading the romfs file table. It's at least 2-3x faster reading it all from a byte array --- src/LibHac/IO/PathParser.cs | 8 +- src/LibHac/IO/PathTools.cs | 39 +++++- .../IO/RomFs/HierarchicalRomFileTable.cs | 125 +++++------------- src/LibHac/IO/RomFs/RomFsDictionary.cs | 78 +++-------- src/LibHac/IO/RomFs/RomFsEntries.cs | 73 ++++++++++ src/LibHac/IO/StorageExtensions.cs | 12 ++ 6 files changed, 182 insertions(+), 153 deletions(-) create mode 100644 src/LibHac/IO/RomFs/RomFsEntries.cs diff --git a/src/LibHac/IO/PathParser.cs b/src/LibHac/IO/PathParser.cs index 1a997c71..2fdb4e3f 100644 --- a/src/LibHac/IO/PathParser.cs +++ b/src/LibHac/IO/PathParser.cs @@ -5,12 +5,12 @@ namespace LibHac.IO { public ref struct PathParser { - private ReadOnlySpan _path; + private ReadOnlySpan _path; private int _offset; private int _length; private bool _finished; - public PathParser(ReadOnlySpan path) + public PathParser(ReadOnlySpan path) { Debug.Assert(PathTools.IsNormalized(path)); @@ -25,7 +25,7 @@ namespace LibHac.IO _finished = false; } - public bool TryGetNext(out ReadOnlySpan name) + public bool TryGetNext(out ReadOnlySpan name) { bool success = MoveNext(); name = GetCurrent(); @@ -50,7 +50,7 @@ namespace LibHac.IO return true; } - public ReadOnlySpan GetCurrent() + public ReadOnlySpan GetCurrent() { return _path.Slice(_offset, _length); } diff --git a/src/LibHac/IO/PathTools.cs b/src/LibHac/IO/PathTools.cs index e47b2830..f2c603ab 100644 --- a/src/LibHac/IO/PathTools.cs +++ b/src/LibHac/IO/PathTools.cs @@ -7,10 +7,16 @@ namespace LibHac.IO { public static readonly char DirectorySeparator = '/'; + public static string Normalize(string inPath) + { + if (IsNormalized(inPath.AsSpan())) return inPath; + return NormalizeInternal(inPath); + } + // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. - public static string Normalize(string inPath) + public static string NormalizeInternal(string inPath) { // Relative paths aren't a thing for IFileSystem, so assume all paths are absolute // and add a '/' to the beginning of the path if it doesn't already begin with one @@ -111,7 +117,36 @@ namespace LibHac.IO case NormalizeState.Dot when c == '/': return false; case NormalizeState.Dot when c == '.': state = NormalizeState.DoubleDot; break; case NormalizeState.Dot: state = NormalizeState.Normal; break; - + + case NormalizeState.DoubleDot when c == '/': return false; + case NormalizeState.DoubleDot: state = NormalizeState.Normal; break; + } + } + + return state == NormalizeState.Normal || state == NormalizeState.Delimiter; + } + + public static bool IsNormalized(ReadOnlySpan path) + { + var state = NormalizeState.Initial; + + foreach (byte c in path) + { + switch (state) + { + case NormalizeState.Initial when c == '/': state = NormalizeState.Delimiter; break; + case NormalizeState.Initial: return false; + + case NormalizeState.Normal when c == '/': state = NormalizeState.Delimiter; break; + + case NormalizeState.Delimiter when c == '/': return false; + case NormalizeState.Delimiter when c == '.': state = NormalizeState.Dot; break; + case NormalizeState.Delimiter: state = NormalizeState.Normal; break; + + case NormalizeState.Dot when c == '/': return false; + case NormalizeState.Dot when c == '.': state = NormalizeState.DoubleDot; break; + case NormalizeState.Dot: state = NormalizeState.Normal; break; + case NormalizeState.DoubleDot when c == '/': return false; case NormalizeState.DoubleDot: state = NormalizeState.Normal; break; } diff --git a/src/LibHac/IO/RomFs/HierarchicalRomFileTable.cs b/src/LibHac/IO/RomFs/HierarchicalRomFileTable.cs index deb239e2..1eb71cec 100644 --- a/src/LibHac/IO/RomFs/HierarchicalRomFileTable.cs +++ b/src/LibHac/IO/RomFs/HierarchicalRomFileTable.cs @@ -1,5 +1,5 @@ using System; -using System.Runtime.InteropServices; +using System.Text; namespace LibHac.IO.RomFs { @@ -27,11 +27,11 @@ namespace LibHac.IO.RomFs public bool OpenFile(string path, out RomFileInfo fileInfo) { - FindFileRecursive(path.AsSpan(), out RomEntryKey key); + FindFileRecursive(GetUtf8Bytes(path), out RomEntryKey key); - if (FileTable.TryGetValue(ref key, out FileRomEntry entry, out int _)) + if (FileTable.TryGetValue(ref key, out RomKeyValuePair keyValuePair)) { - fileInfo = entry.Info; + fileInfo = keyValuePair.Value.Info; return true; } @@ -41,9 +41,9 @@ namespace LibHac.IO.RomFs public bool OpenFile(int offset, out RomFileInfo fileInfo) { - if (FileTable.TryGetValue(offset, out FileRomEntry entry)) + if (FileTable.TryGetValue(offset, out RomKeyValuePair keyValuePair)) { - fileInfo = entry.Info; + fileInfo = keyValuePair.Value.Info; return true; } @@ -53,11 +53,11 @@ namespace LibHac.IO.RomFs public bool OpenDirectory(string path, out FindPosition position) { - FindDirectoryRecursive(path.AsSpan(), out RomEntryKey key); + FindDirectoryRecursive(GetUtf8Bytes(path), out RomEntryKey key); - if (DirectoryTable.TryGetValue(ref key, out DirectoryRomEntry entry, out int _)) + if (DirectoryTable.TryGetValue(ref key, out RomKeyValuePair keyValuePair)) { - position = entry.Pos; + position = keyValuePair.Value.Pos; return true; } @@ -67,9 +67,9 @@ namespace LibHac.IO.RomFs public bool OpenDirectory(int offset, out FindPosition position) { - if (DirectoryTable.TryGetValue(offset, out DirectoryRomEntry entry)) + if (DirectoryTable.TryGetValue(offset, out RomKeyValuePair keyValuePair)) { - position = entry.Pos; + position = keyValuePair.Value.Pos; return true; } @@ -77,121 +77,68 @@ namespace LibHac.IO.RomFs return false; } + private static ReadOnlySpan GetUtf8Bytes(string value) + { + return Encoding.UTF8.GetBytes(value).AsSpan(); + } + public bool FindNextFile(ref FindPosition position, out RomFileInfo info, out string name) { - if (FileTable.TryGetValue(position.NextFile, out FileRomEntry entry, out name)) + if (FileTable.TryGetValue(position.NextFile, out RomKeyValuePair keyValuePair)) { - position.NextFile = entry.NextSibling; - info = entry.Info; + position.NextFile = keyValuePair.Value.NextSibling; + info = keyValuePair.Value.Info; + name = Encoding.UTF8.GetString(keyValuePair.Key.Name.ToArray()); return true; } info = default; + name = default; return false; } public bool FindNextDirectory(ref FindPosition position, out string name) { - if (DirectoryTable.TryGetValue(position.NextDirectory, out DirectoryRomEntry entry, out name)) + if (DirectoryTable.TryGetValue(position.NextDirectory, out RomKeyValuePair keyValuePair)) { - position.NextDirectory = entry.NextSibling; + position.NextDirectory = keyValuePair.Value.NextSibling; + name = Encoding.UTF8.GetString(keyValuePair.Key.Name.ToArray()); return true; } + name = default; return false; } - private void FindFileRecursive(ReadOnlySpan path, out RomEntryKey key) + private void FindFileRecursive(ReadOnlySpan path, out RomEntryKey key) { var parser = new PathParser(path); - FindParentDirectoryRecursive(ref parser, out DirectoryRomEntry _, out int parentOffset); + FindParentDirectoryRecursive(ref parser, out RomKeyValuePair keyValuePair); - key = new RomEntryKey(parser.GetCurrent(), parentOffset); + key = keyValuePair.Key; } - private void FindDirectoryRecursive(ReadOnlySpan path, out RomEntryKey key) + private void FindDirectoryRecursive(ReadOnlySpan path, out RomEntryKey key) { var parser = new PathParser(path); - FindParentDirectoryRecursive(ref parser, out DirectoryRomEntry _, out int parentOffset); + FindParentDirectoryRecursive(ref parser, out RomKeyValuePair keyValuePair); - ReadOnlySpan name = parser.GetCurrent(); - if (name.Length == 0) parentOffset = 0; + ReadOnlySpan name = parser.GetCurrent(); + int parentOffset = name.Length == 0 ? 0 : keyValuePair.Offset; key = new RomEntryKey(name, parentOffset); } - private void FindParentDirectoryRecursive(ref PathParser parser, out DirectoryRomEntry parentEntry, out int parentOffset) + private void FindParentDirectoryRecursive(ref PathParser parser, out RomKeyValuePair keyValuePair) { - parentEntry = default; - parentOffset = default; + keyValuePair = default; RomEntryKey key = default; while (parser.TryGetNext(out key.Name) && !parser.IsFinished()) { - DirectoryTable.TryGetValue(ref key, out parentEntry, out parentOffset); - key.Parent = parentOffset; + DirectoryTable.TryGetValue(ref key, out keyValuePair); + key.Parent = keyValuePair.Offset; } } } - - internal ref struct RomEntryKey - { - public ReadOnlySpan Name; - public int Parent; - - public RomEntryKey(ReadOnlySpan name, int parent) - { - Name = name; - Parent = parent; - } - - public uint GetRomHashCode() - { - uint hash = 123456789 ^ (uint)Parent; - - foreach (char c in Name) - { - hash = c ^ ((hash << 27) | (hash >> 5)); - } - - return hash; - } - } - - [StructLayout(LayoutKind.Sequential)] - public struct RomFsEntry where T : unmanaged - { - public int Parent; - public T Value; - public int Next; - public int KeyLength; - } - - [StructLayout(LayoutKind.Sequential, Pack = 1)] - public struct FileRomEntry - { - public int NextSibling; - public RomFileInfo Info; - } - - [StructLayout(LayoutKind.Sequential, Pack = 1)] - public struct RomFileInfo - { - public long Offset; - public long Length; - } - - [StructLayout(LayoutKind.Sequential)] - public struct DirectoryRomEntry - { - public int NextSibling; - public FindPosition Pos; - } - - [StructLayout(LayoutKind.Sequential)] - public struct FindPosition - { - public int NextDirectory; - public int NextFile; - } } diff --git a/src/LibHac/IO/RomFs/RomFsDictionary.cs b/src/LibHac/IO/RomFs/RomFsDictionary.cs index 2c05416d..4b9b3096 100644 --- a/src/LibHac/IO/RomFs/RomFsDictionary.cs +++ b/src/LibHac/IO/RomFs/RomFsDictionary.cs @@ -1,36 +1,32 @@ using System; -using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; -using System.Text; namespace LibHac.IO.RomFs { internal class RomFsDictionary where T : unmanaged { - private int HashBucketCount { get; } - private IStorage BucketStorage { get; } - private IStorage EntryStorage { get; } + private int[] BucketTable { get; } + private byte[] EntryTable { get; } // Hack around not being able to get the size of generic structures private readonly int _sizeOfEntry = 12 + Marshal.SizeOf(); public RomFsDictionary(IStorage bucketStorage, IStorage entryStorage) { - BucketStorage = bucketStorage; - EntryStorage = entryStorage; - HashBucketCount = (int)(bucketStorage.Length / 4); + BucketTable = bucketStorage.ToArray(); + EntryTable = entryStorage.ToArray(); } - public bool TryGetValue(ref RomEntryKey key, out T value, out int offset) + public bool TryGetValue(ref RomEntryKey key, out RomKeyValuePair value) { int i = FindEntry(ref key); - offset = i; if (i >= 0) { GetEntryInternal(i, out RomFsEntry entry); - value = entry.Value; + + value = new RomKeyValuePair { Key = key, Value = entry.Value, Offset = i }; return true; } @@ -38,43 +34,32 @@ namespace LibHac.IO.RomFs return false; } - public bool TryGetValue(int offset, out T value, out string entryName) + public bool TryGetValue(int offset, out RomKeyValuePair value) { - if (offset < 0 || offset + _sizeOfEntry >= EntryStorage.Length) - { - value = default; - entryName = default; - return false; - } - - GetEntryInternal(offset, out RomFsEntry entry, out entryName); - value = entry.Value; - return true; - } - - public bool TryGetValue(int offset, out T value) - { - if (offset < 0 || offset + _sizeOfEntry >= EntryStorage.Length) + if (offset < 0 || offset + _sizeOfEntry >= EntryTable.Length) { value = default; return false; } - GetEntryInternal(offset, out RomFsEntry entry); - value = entry.Value; + value = new RomKeyValuePair(); + + GetEntryInternal(offset, out RomFsEntry entry, out value.Key.Name); + value.Value = entry.Value; return true; } private int FindEntry(ref RomEntryKey key) { uint hashCode = key.GetRomHashCode(); - int i = GetBucket((int)(hashCode % HashBucketCount)); + int index = (int)(hashCode % BucketTable.Length); + int i = BucketTable[index]; while (i != -1) { - GetEntryInternal(i, out RomFsEntry entry); + GetEntryInternal(i, out RomFsEntry entry, out ReadOnlySpan name); - if (IsEqual(ref key, ref entry, i)) + if (key.Parent == entry.Parent && key.Name.SequenceEqual(name)) { break; } @@ -85,24 +70,12 @@ namespace LibHac.IO.RomFs return i; } - private bool IsEqual(ref RomEntryKey key, ref RomFsEntry entry, int entryOffset) - { - if (key.Parent != entry.Parent) return false; - if (key.Name.Length != entry.KeyLength) return false; - - GetEntryInternal(entryOffset, out RomFsEntry _, out string name); - - return key.Name.Equals(name.AsSpan(), StringComparison.Ordinal); - } - private void GetEntryInternal(int offset, out RomFsEntry outEntry) { - Span b = stackalloc byte[_sizeOfEntry]; - EntryStorage.Read(b, offset); - outEntry = MemoryMarshal.Read>(b); + outEntry = MemoryMarshal.Read>(EntryTable.AsSpan(offset)); } - private void GetEntryInternal(int offset, out RomFsEntry outEntry, out string entryName) + private void GetEntryInternal(int offset, out RomFsEntry outEntry, out ReadOnlySpan entryName) { GetEntryInternal(offset, out outEntry); @@ -111,18 +84,7 @@ namespace LibHac.IO.RomFs throw new InvalidDataException("Rom entry name is too long."); } - var buf = new byte[outEntry.KeyLength]; - EntryStorage.Read(buf, offset + _sizeOfEntry); - entryName = Encoding.ASCII.GetString(buf); - } - - private int GetBucket(int index) - { - Debug.Assert(index < HashBucketCount); - - Span buf = stackalloc byte[4]; - BucketStorage.Read(buf, index * 4); - return MemoryMarshal.Read(buf); + entryName = EntryTable.AsSpan(offset + _sizeOfEntry, outEntry.KeyLength); } } } diff --git a/src/LibHac/IO/RomFs/RomFsEntries.cs b/src/LibHac/IO/RomFs/RomFsEntries.cs new file mode 100644 index 00000000..06c00a8a --- /dev/null +++ b/src/LibHac/IO/RomFs/RomFsEntries.cs @@ -0,0 +1,73 @@ +using System; +using System.Runtime.InteropServices; + +namespace LibHac.IO.RomFs +{ + internal ref struct RomEntryKey + { + public ReadOnlySpan Name; + public int Parent; + + public RomEntryKey(ReadOnlySpan name, int parent) + { + Name = name; + Parent = parent; + } + + public uint GetRomHashCode() + { + uint hash = 123456789 ^ (uint)Parent; + + foreach (byte c in Name) + { + hash = c ^ ((hash << 27) | (hash >> 5)); + } + + return hash; + } + } + + internal ref struct RomKeyValuePair where T : unmanaged + { + public RomEntryKey Key; + public int Offset; + public T Value; + } + + [StructLayout(LayoutKind.Sequential, Pack = 1)] + public struct RomFsEntry where T : unmanaged + { + public int Parent; + public T Value; + public int Next; + public int KeyLength; + } + + [StructLayout(LayoutKind.Sequential, Pack = 1)] + public struct FileRomEntry + { + public int NextSibling; + public RomFileInfo Info; + } + + [StructLayout(LayoutKind.Sequential)] + public struct RomFileInfo + { + public long Offset; + public long Length; + } + + [StructLayout(LayoutKind.Sequential, Pack = 1)] + public struct DirectoryRomEntry + { + public int NextSibling; + public FindPosition Pos; + } + + [StructLayout(LayoutKind.Sequential)] + public struct FindPosition + { + public int NextDirectory; + public int NextFile; + } +} diff --git a/src/LibHac/IO/StorageExtensions.cs b/src/LibHac/IO/StorageExtensions.cs index 26c91b60..5cd2bde3 100644 --- a/src/LibHac/IO/StorageExtensions.cs +++ b/src/LibHac/IO/StorageExtensions.cs @@ -1,6 +1,7 @@ using System; using System.Buffers; using System.IO; +using System.Runtime.InteropServices; namespace LibHac.IO { @@ -111,6 +112,17 @@ namespace LibHac.IO return arr; } + public static T[] ToArray(this IStorage storage) where T : unmanaged + { + if (storage == null) return new T[0]; + + var arr = new T[storage.Length / Marshal.SizeOf()]; + Span dest = MemoryMarshal.Cast(arr.AsSpan()); + + storage.Read(dest, 0); + return arr; + } + public static void CopyToStream(this IStorage input, Stream output, long length, IProgressReport progress = null) { const int bufferSize = 0x8000;