mirrors/LibHac
1
0
Fork 0
mirror of https://github.com/Thealexbarney/LibHac.git synced 2024-11-14 10:49:41 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Don't use IStorage for reading the romfs file table.

Browse source 
It's at least 2-3x faster reading it all from a byte array
This commit is contained in:
Alex Barney 2019-01-30 15:31:53 -06:00
parent 7004b22958
commit 34e926f2a4
6 changed files with 182 additions and 153 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/LibHac/IO/PathParser.cs
View file

@ -5,12 +5,12 @@ namespace LibHac.IO
{
public ref struct PathParser
{
private ReadOnlySpan<char> _path;
private ReadOnlySpan<byte> _path;
private int _offset;
private int _length;
private bool _finished;
public PathParser(ReadOnlySpan<char> path)
public PathParser(ReadOnlySpan<byte> path)
{
Debug.Assert(PathTools.IsNormalized(path));
@ -25,7 +25,7 @@ namespace LibHac.IO
_finished = false;
}
public bool TryGetNext(out ReadOnlySpan<char> name)
public bool TryGetNext(out ReadOnlySpan<byte> name)
{
bool success = MoveNext();
name = GetCurrent();
@ -50,7 +50,7 @@ namespace LibHac.IO
return true;
}
public ReadOnlySpan<char> GetCurrent()
public ReadOnlySpan<byte> GetCurrent()
{
return _path.Slice(_offset, _length);
}

37
src/LibHac/IO/PathTools.cs
View file

@ -7,10 +7,16 @@ namespace LibHac.IO
{
public static readonly char DirectorySeparator = '/';
public static string Normalize(string inPath)
{
if (IsNormalized(inPath.AsSpan())) return inPath;
return NormalizeInternal(inPath);
}
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
public static string Normalize(string inPath)
public static string NormalizeInternal(string inPath)
{
// Relative paths aren't a thing for IFileSystem, so assume all paths are absolute
// and add a '/' to the beginning of the path if it doesn't already begin with one
@ -120,6 +126,35 @@ namespace LibHac.IO
return state == NormalizeState.Normal || state == NormalizeState.Delimiter;
}
public static bool IsNormalized(ReadOnlySpan<byte> path)
{
var state = NormalizeState.Initial;
foreach (byte c in path)
{
switch (state)
{
case NormalizeState.Initial when c == '/': state = NormalizeState.Delimiter; break;
case NormalizeState.Initial: return false;
case NormalizeState.Normal when c == '/': state = NormalizeState.Delimiter; break;
case NormalizeState.Delimiter when c == '/': return false;
case NormalizeState.Delimiter when c == '.': state = NormalizeState.Dot; break;
case NormalizeState.Delimiter: state = NormalizeState.Normal; break;
case NormalizeState.Dot when c == '/': return false;
case NormalizeState.Dot when c == '.': state = NormalizeState.DoubleDot; break;
case NormalizeState.Dot: state = NormalizeState.Normal; break;
case NormalizeState.DoubleDot when c == '/': return false;
case NormalizeState.DoubleDot: state = NormalizeState.Normal; break;
}
}
return state == NormalizeState.Normal || state == NormalizeState.Delimiter;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static bool IsDirectorySeparator(char c)
{

125
src/LibHac/IO/RomFs/HierarchicalRomFileTable.cs
View file

@ -1,5 +1,5 @@
using System;
using System.Runtime.InteropServices;
using System.Text;
namespace LibHac.IO.RomFs
{
@ -27,11 +27,11 @@ namespace LibHac.IO.RomFs
public bool OpenFile(string path, out RomFileInfo fileInfo)
{
FindFileRecursive(path.AsSpan(), out RomEntryKey key);
FindFileRecursive(GetUtf8Bytes(path), out RomEntryKey key);
if (FileTable.TryGetValue(ref key, out FileRomEntry entry, out int _))
if (FileTable.TryGetValue(ref key, out RomKeyValuePair<FileRomEntry> keyValuePair))
{
fileInfo = entry.Info;
fileInfo = keyValuePair.Value.Info;
return true;
}
@ -41,9 +41,9 @@ namespace LibHac.IO.RomFs
public bool OpenFile(int offset, out RomFileInfo fileInfo)
{
if (FileTable.TryGetValue(offset, out FileRomEntry entry))
if (FileTable.TryGetValue(offset, out RomKeyValuePair<FileRomEntry> keyValuePair))
{
fileInfo = entry.Info;
fileInfo = keyValuePair.Value.Info;
return true;
}
@ -53,11 +53,11 @@ namespace LibHac.IO.RomFs
public bool OpenDirectory(string path, out FindPosition position)
{
FindDirectoryRecursive(path.AsSpan(), out RomEntryKey key);
FindDirectoryRecursive(GetUtf8Bytes(path), out RomEntryKey key);
if (DirectoryTable.TryGetValue(ref key, out DirectoryRomEntry entry, out int _))
if (DirectoryTable.TryGetValue(ref key, out RomKeyValuePair<DirectoryRomEntry> keyValuePair))
{
position = entry.Pos;
position = keyValuePair.Value.Pos;
return true;
}
@ -67,9 +67,9 @@ namespace LibHac.IO.RomFs
public bool OpenDirectory(int offset, out FindPosition position)
{
if (DirectoryTable.TryGetValue(offset, out DirectoryRomEntry entry))
if (DirectoryTable.TryGetValue(offset, out RomKeyValuePair<DirectoryRomEntry> keyValuePair))
{
position = entry.Pos;
position = keyValuePair.Value.Pos;
return true;
}
@ -77,121 +77,68 @@ namespace LibHac.IO.RomFs
return false;
}
private static ReadOnlySpan<byte> GetUtf8Bytes(string value)
{
return Encoding.UTF8.GetBytes(value).AsSpan();
}
public bool FindNextFile(ref FindPosition position, out RomFileInfo info, out string name)
{
if (FileTable.TryGetValue(position.NextFile, out FileRomEntry entry, out name))
if (FileTable.TryGetValue(position.NextFile, out RomKeyValuePair<FileRomEntry> keyValuePair))
{
position.NextFile = entry.NextSibling;
info = entry.Info;
position.NextFile = keyValuePair.Value.NextSibling;
info = keyValuePair.Value.Info;
name = Encoding.UTF8.GetString(keyValuePair.Key.Name.ToArray());
return true;
}
info = default;
name = default;
return false;
}
public bool FindNextDirectory(ref FindPosition position, out string name)
{
if (DirectoryTable.TryGetValue(position.NextDirectory, out DirectoryRomEntry entry, out name))
if (DirectoryTable.TryGetValue(position.NextDirectory, out RomKeyValuePair<DirectoryRomEntry> keyValuePair))
{
position.NextDirectory = entry.NextSibling;
position.NextDirectory = keyValuePair.Value.NextSibling;
name = Encoding.UTF8.GetString(keyValuePair.Key.Name.ToArray());
return true;
}
name = default;
return false;
}
private void FindFileRecursive(ReadOnlySpan<char> path, out RomEntryKey key)
private void FindFileRecursive(ReadOnlySpan<byte> path, out RomEntryKey key)
{
var parser = new PathParser(path);
FindParentDirectoryRecursive(ref parser, out DirectoryRomEntry _, out int parentOffset);
FindParentDirectoryRecursive(ref parser, out RomKeyValuePair<DirectoryRomEntry> keyValuePair);
key = new RomEntryKey(parser.GetCurrent(), parentOffset);
key = keyValuePair.Key;
}
private void FindDirectoryRecursive(ReadOnlySpan<char> path, out RomEntryKey key)
private void FindDirectoryRecursive(ReadOnlySpan<byte> path, out RomEntryKey key)
{
var parser = new PathParser(path);
FindParentDirectoryRecursive(ref parser, out DirectoryRomEntry _, out int parentOffset);
FindParentDirectoryRecursive(ref parser, out RomKeyValuePair<DirectoryRomEntry> keyValuePair);
ReadOnlySpan<char> name = parser.GetCurrent();
if (name.Length == 0) parentOffset = 0;
ReadOnlySpan<byte> name = parser.GetCurrent();
int parentOffset = name.Length == 0 ? 0 : keyValuePair.Offset;
key = new RomEntryKey(name, parentOffset);
}
private void FindParentDirectoryRecursive(ref PathParser parser, out DirectoryRomEntry parentEntry, out int parentOffset)
private void FindParentDirectoryRecursive(ref PathParser parser, out RomKeyValuePair<DirectoryRomEntry> keyValuePair)
{
parentEntry = default;
parentOffset = default;
keyValuePair = default;
RomEntryKey key = default;
while (parser.TryGetNext(out key.Name) && !parser.IsFinished())
{
DirectoryTable.TryGetValue(ref key, out parentEntry, out parentOffset);
key.Parent = parentOffset;
DirectoryTable.TryGetValue(ref key, out keyValuePair);
key.Parent = keyValuePair.Offset;
}
}
}
internal ref struct RomEntryKey
{
public ReadOnlySpan<char> Name;
public int Parent;
public RomEntryKey(ReadOnlySpan<char> name, int parent)
{
Name = name;
Parent = parent;
}
public uint GetRomHashCode()
{
uint hash = 123456789 ^ (uint)Parent;
foreach (char c in Name)
{
hash = c ^ ((hash << 27) | (hash >> 5));
}
return hash;
}
}
[StructLayout(LayoutKind.Sequential)]
public struct RomFsEntry<T> where T : unmanaged
{
public int Parent;
public T Value;
public int Next;
public int KeyLength;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public struct FileRomEntry
{
public int NextSibling;
public RomFileInfo Info;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public struct RomFileInfo
{
public long Offset;
public long Length;
}
[StructLayout(LayoutKind.Sequential)]
public struct DirectoryRomEntry
{
public int NextSibling;
public FindPosition Pos;
}
[StructLayout(LayoutKind.Sequential)]
public struct FindPosition
{
public int NextDirectory;
public int NextFile;
}
}

78
src/LibHac/IO/RomFs/RomFsDictionary.cs
View file

@ -1,36 +1,32 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
using System.Text;
namespace LibHac.IO.RomFs
{
internal class RomFsDictionary<T> where T : unmanaged
{
private int HashBucketCount { get; }
private IStorage BucketStorage { get; }
private IStorage EntryStorage { get; }
private int[] BucketTable { get; }
private byte[] EntryTable { get; }
// Hack around not being able to get the size of generic structures
private readonly int _sizeOfEntry = 12 + Marshal.SizeOf<T>();
public RomFsDictionary(IStorage bucketStorage, IStorage entryStorage)
{
BucketStorage = bucketStorage;
EntryStorage = entryStorage;
HashBucketCount = (int)(bucketStorage.Length / 4);
BucketTable = bucketStorage.ToArray<int>();
EntryTable = entryStorage.ToArray();
}
public bool TryGetValue(ref RomEntryKey key, out T value, out int offset)
public bool TryGetValue(ref RomEntryKey key, out RomKeyValuePair<T> value)
{
int i = FindEntry(ref key);
offset = i;
if (i >= 0)
{
GetEntryInternal(i, out RomFsEntry<T> entry);
value = entry.Value;
value = new RomKeyValuePair<T> { Key = key, Value = entry.Value, Offset = i };
return true;
}
@ -38,43 +34,32 @@ namespace LibHac.IO.RomFs
return false;
}
public bool TryGetValue(int offset, out T value, out string entryName)
public bool TryGetValue(int offset, out RomKeyValuePair<T> value)
{
if (offset < 0 || offset + _sizeOfEntry >= EntryStorage.Length)
{
value = default;
entryName = default;
return false;
}
GetEntryInternal(offset, out RomFsEntry<T> entry, out entryName);
value = entry.Value;
return true;
}
public bool TryGetValue(int offset, out T value)
{
if (offset < 0 || offset + _sizeOfEntry >= EntryStorage.Length)
if (offset < 0 || offset + _sizeOfEntry >= EntryTable.Length)
{
value = default;
return false;
}
GetEntryInternal(offset, out RomFsEntry<T> entry);
value = entry.Value;
value = new RomKeyValuePair<T>();
GetEntryInternal(offset, out RomFsEntry<T> entry, out value.Key.Name);
value.Value = entry.Value;
return true;
}
private int FindEntry(ref RomEntryKey key)
{
uint hashCode = key.GetRomHashCode();
int i = GetBucket((int)(hashCode % HashBucketCount));
int index = (int)(hashCode % BucketTable.Length);
int i = BucketTable[index];
while (i != -1)
{
GetEntryInternal(i, out RomFsEntry<T> entry);
GetEntryInternal(i, out RomFsEntry<T> entry, out ReadOnlySpan<byte> name);
if (IsEqual(ref key, ref entry, i))
if (key.Parent == entry.Parent && key.Name.SequenceEqual(name))
{
break;
}
@ -85,24 +70,12 @@ namespace LibHac.IO.RomFs
return i;
}
private bool IsEqual(ref RomEntryKey key, ref RomFsEntry<T> entry, int entryOffset)
{
if (key.Parent != entry.Parent) return false;
if (key.Name.Length != entry.KeyLength) return false;
GetEntryInternal(entryOffset, out RomFsEntry<T> _, out string name);
return key.Name.Equals(name.AsSpan(), StringComparison.Ordinal);
}
private void GetEntryInternal(int offset, out RomFsEntry<T> outEntry)
{
Span<byte> b = stackalloc byte[_sizeOfEntry];
EntryStorage.Read(b, offset);
outEntry = MemoryMarshal.Read<RomFsEntry<T>>(b);
outEntry = MemoryMarshal.Read<RomFsEntry<T>>(EntryTable.AsSpan(offset));
}
private void GetEntryInternal(int offset, out RomFsEntry<T> outEntry, out string entryName)
private void GetEntryInternal(int offset, out RomFsEntry<T> outEntry, out ReadOnlySpan<byte> entryName)
{
GetEntryInternal(offset, out outEntry);
@ -111,18 +84,7 @@ namespace LibHac.IO.RomFs
throw new InvalidDataException("Rom entry name is too long.");
}
var buf = new byte[outEntry.KeyLength];
EntryStorage.Read(buf, offset + _sizeOfEntry);
entryName = Encoding.ASCII.GetString(buf);
}
private int GetBucket(int index)
{
Debug.Assert(index < HashBucketCount);
Span<byte> buf = stackalloc byte[4];
BucketStorage.Read(buf, index * 4);
return MemoryMarshal.Read<int>(buf);
entryName = EntryTable.AsSpan(offset + _sizeOfEntry, outEntry.KeyLength);
}
}
}

73
src/LibHac/IO/RomFs/RomFsEntries.cs Normal file
View file

@ -0,0 +1,73 @@
using System;
using System.Runtime.InteropServices;
namespace LibHac.IO.RomFs
{
internal ref struct RomEntryKey
{
public ReadOnlySpan<byte> Name;
public int Parent;
public RomEntryKey(ReadOnlySpan<byte> name, int parent)
{
Name = name;
Parent = parent;
}
public uint GetRomHashCode()
{
uint hash = 123456789 ^ (uint)Parent;
foreach (byte c in Name)
{
hash = c ^ ((hash << 27) | (hash >> 5));
}
return hash;
}
}
internal ref struct RomKeyValuePair<T> where T : unmanaged
{
public RomEntryKey Key;
public int Offset;
public T Value;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public struct RomFsEntry<T> where T : unmanaged
{
public int Parent;
public T Value;
public int Next;
public int KeyLength;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public struct FileRomEntry
{
public int NextSibling;
public RomFileInfo Info;
}
[StructLayout(LayoutKind.Sequential)]
public struct RomFileInfo
{
public long Offset;
public long Length;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public struct DirectoryRomEntry
{
public int NextSibling;
public FindPosition Pos;
}
[StructLayout(LayoutKind.Sequential)]
public struct FindPosition
{
public int NextDirectory;
public int NextFile;
}
}

12
src/LibHac/IO/StorageExtensions.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Buffers;
using System.IO;
using System.Runtime.InteropServices;
namespace LibHac.IO
{
@ -111,6 +112,17 @@ namespace LibHac.IO
return arr;
}
public static T[] ToArray<T>(this IStorage storage) where T : unmanaged
{
if (storage == null) return new T[0];
var arr = new T[storage.Length / Marshal.SizeOf<T>()];
Span<byte> dest = MemoryMarshal.Cast<T, byte>(arr.AsSpan());
storage.Read(dest, 0);
return arr;
}
public static void CopyToStream(this IStorage input, Stream output, long length, IProgressReport progress = null)
{
const int bufferSize = 0x8000;