From dc8aad1e71beef407d6d201c107161e64e31ebf2 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 3 Jun 2019 20:24:38 -0500 Subject: [PATCH] Add IMKV database reader and writer (#61) * Add Result struct * Add IMKV database reading * Add imkvdb writing * Add get and set to kvdb * Add Freeze method to IExportable * Add generic kvdb value * Add ContentMetaKey for use with kvdb --- src/LibHac/Fs/SaveDataStruct.cs | 95 +++++++++++++++++++++++++++++ src/LibHac/Fs/SaveIndexerStruct.cs | 40 ++++++++++++ src/LibHac/Fs/UserId.cs | 66 ++++++++++++++++++++ src/LibHac/Kvdb/GenericValue.cs | 31 ++++++++++ src/LibHac/Kvdb/IExportable.cs | 16 +++++ src/LibHac/Kvdb/ImkvdbHeader.cs | 24 ++++++++ src/LibHac/Kvdb/ImkvdbReader.cs | 79 ++++++++++++++++++++++++ src/LibHac/Kvdb/ImkvdbWriter.cs | 61 ++++++++++++++++++ src/LibHac/Kvdb/KeyValueDatabase.cs | 93 ++++++++++++++++++++++++++++ src/LibHac/Kvdb/ResultsKvdb.cs | 13 ++++ src/LibHac/Ncm/ContentMetaKey.cs | 84 +++++++++++++++++++++++++ src/LibHac/Result.cs | 28 +++++++++ 12 files changed, 630 insertions(+) create mode 100644 src/LibHac/Fs/SaveDataStruct.cs create mode 100644 src/LibHac/Fs/SaveIndexerStruct.cs create mode 100644 src/LibHac/Fs/UserId.cs create mode 100644 src/LibHac/Kvdb/GenericValue.cs create mode 100644 src/LibHac/Kvdb/IExportable.cs create mode 100644 src/LibHac/Kvdb/ImkvdbHeader.cs create mode 100644 src/LibHac/Kvdb/ImkvdbReader.cs create mode 100644 src/LibHac/Kvdb/ImkvdbWriter.cs create mode 100644 src/LibHac/Kvdb/KeyValueDatabase.cs create mode 100644 src/LibHac/Kvdb/ResultsKvdb.cs create mode 100644 src/LibHac/Ncm/ContentMetaKey.cs create mode 100644 src/LibHac/Result.cs diff --git a/src/LibHac/Fs/SaveDataStruct.cs b/src/LibHac/Fs/SaveDataStruct.cs new file mode 100644 index 00000000..a0a71f49 --- /dev/null +++ b/src/LibHac/Fs/SaveDataStruct.cs @@ -0,0 +1,95 @@ +using System; +using System.Buffers.Binary; +using LibHac.Fs.Save; +using LibHac.Kvdb; + +namespace LibHac.Fs +{ + public class SaveDataStruct : IComparable, IComparable, IEquatable, IExportable + { + public ulong TitleId { get; private set; } + public UserId UserId { get; private set; } + public ulong SaveId { get; private set; } + public SaveDataType Type { get; private set; } + public byte Rank { get; private set; } + public short Index { get; private set; } + + public int ExportSize => 0x40; + private bool _isFrozen; + + public void ToBytes(Span output) + { + if (output.Length < ExportSize) throw new InvalidOperationException("Output buffer is too small."); + + BinaryPrimitives.WriteUInt64LittleEndian(output, TitleId); + UserId.ToBytes(output.Slice(8)); + BinaryPrimitives.WriteUInt64LittleEndian(output.Slice(0x18), SaveId); + output[0x20] = (byte)Type; + output[0x21] = Rank; + BinaryPrimitives.WriteInt16LittleEndian(output.Slice(0x22), Index); + } + + public void FromBytes(ReadOnlySpan input) + { + if (_isFrozen) throw new InvalidOperationException("Unable to modify frozen object."); + if (input.Length < ExportSize) throw new InvalidOperationException("Input data is too short."); + + TitleId = BinaryPrimitives.ReadUInt64LittleEndian(input); + UserId = new UserId(input.Slice(8)); + SaveId = BinaryPrimitives.ReadUInt64LittleEndian(input.Slice(0x18)); + Type = (SaveDataType)input[0x20]; + Rank = input[0x21]; + Index = BinaryPrimitives.ReadInt16LittleEndian(input.Slice(0x22)); + } + + public void Freeze() => _isFrozen = true; + + public bool Equals(SaveDataStruct other) + { + return other != null && TitleId == other.TitleId && UserId.Equals(other.UserId) && SaveId == other.SaveId && + Type == other.Type && Rank == other.Rank && Index == other.Index; + } + + public override bool Equals(object obj) + { + return obj is SaveDataStruct other && Equals(other); + } + + public override int GetHashCode() + { + unchecked + { + // ReSharper disable NonReadonlyMemberInGetHashCode + int hashCode = TitleId.GetHashCode(); + hashCode = (hashCode * 397) ^ UserId.GetHashCode(); + hashCode = (hashCode * 397) ^ SaveId.GetHashCode(); + hashCode = (hashCode * 397) ^ (int)Type; + hashCode = (hashCode * 397) ^ Rank.GetHashCode(); + hashCode = (hashCode * 397) ^ Index.GetHashCode(); + return hashCode; + // ReSharper restore NonReadonlyMemberInGetHashCode + } + } + + public int CompareTo(SaveDataStruct other) + { + int titleIdComparison = TitleId.CompareTo(other.TitleId); + if (titleIdComparison != 0) return titleIdComparison; + int typeComparison = Type.CompareTo(other.Type); + if (typeComparison != 0) return typeComparison; + int userIdComparison = UserId.CompareTo(other.UserId); + if (userIdComparison != 0) return userIdComparison; + int saveIdComparison = SaveId.CompareTo(other.SaveId); + if (saveIdComparison != 0) return saveIdComparison; + int rankComparison = Rank.CompareTo(other.Rank); + if (rankComparison != 0) return rankComparison; + return Index.CompareTo(other.Index); + } + + public int CompareTo(object obj) + { + if (obj is null) return 1; + return obj is SaveDataStruct other ? CompareTo(other) : throw new ArgumentException($"Object must be of type {nameof(SaveDataStruct)}"); + } + } +} diff --git a/src/LibHac/Fs/SaveIndexerStruct.cs b/src/LibHac/Fs/SaveIndexerStruct.cs new file mode 100644 index 00000000..fedada8f --- /dev/null +++ b/src/LibHac/Fs/SaveIndexerStruct.cs @@ -0,0 +1,40 @@ +using System; +using System.Buffers.Binary; +using LibHac.Kvdb; + +namespace LibHac.Fs +{ + public class SaveIndexerStruct : IExportable + { + public ulong SaveId { get; private set; } + public ulong Size { get; private set; } + public byte SpaceId { get; private set; } + public byte Field19 { get; private set; } + + public int ExportSize => 0x40; + private bool _isFrozen; + + public void ToBytes(Span output) + { + if(output.Length < ExportSize) throw new InvalidOperationException("Output buffer is too small."); + + BinaryPrimitives.WriteUInt64LittleEndian(output, SaveId); + BinaryPrimitives.WriteUInt64LittleEndian(output.Slice(8), Size); + output[0x18] = SpaceId; + output[0x19] = Field19; + } + + public void FromBytes(ReadOnlySpan input) + { + if(_isFrozen) throw new InvalidOperationException("Unable to modify frozen object."); + if (input.Length < ExportSize) throw new InvalidOperationException("Input data is too short."); + + SaveId = BinaryPrimitives.ReadUInt64LittleEndian(input); + Size = BinaryPrimitives.ReadUInt64LittleEndian(input.Slice(8)); + SpaceId = input[0x18]; + Field19 = input[0x19]; + } + + public void Freeze() => _isFrozen = true; + } +} diff --git a/src/LibHac/Fs/UserId.cs b/src/LibHac/Fs/UserId.cs new file mode 100644 index 00000000..3045249b --- /dev/null +++ b/src/LibHac/Fs/UserId.cs @@ -0,0 +1,66 @@ +using System; +using System.Runtime.InteropServices; + +namespace LibHac.Fs +{ + public struct UserId : IEquatable, IComparable, IComparable + { + public readonly ulong High; + public readonly ulong Low; + + public UserId(ulong high, ulong low) + { + High = high; + Low = low; + } + + public UserId(ReadOnlySpan uid) + { + ReadOnlySpan longs = MemoryMarshal.Cast(uid); + + High = longs[0]; + Low = longs[1]; + } + + public bool Equals(UserId other) + { + return High == other.High && Low == other.Low; + } + + public override bool Equals(object obj) + { + return obj is UserId other && Equals(other); + } + + public override int GetHashCode() + { + unchecked + { + return (High.GetHashCode() * 397) ^ Low.GetHashCode(); + } + } + + public int CompareTo(UserId other) + { + // ReSharper disable ImpureMethodCallOnReadonlyValueField + int highComparison = High.CompareTo(other.High); + if (highComparison != 0) return highComparison; + return Low.CompareTo(other.Low); + // ReSharper restore ImpureMethodCallOnReadonlyValueField + } + + public int CompareTo(object obj) + { + if (ReferenceEquals(null, obj)) return 1; + return obj is UserId other ? CompareTo(other) : throw new ArgumentException($"Object must be of type {nameof(UserId)}"); + } + + public void ToBytes(Span output) + { + Span longs = MemoryMarshal.Cast(output); + + longs[0] = High; + longs[1] = Low; + } + } +} diff --git a/src/LibHac/Kvdb/GenericValue.cs b/src/LibHac/Kvdb/GenericValue.cs new file mode 100644 index 00000000..1d599384 --- /dev/null +++ b/src/LibHac/Kvdb/GenericValue.cs @@ -0,0 +1,31 @@ +using System; + +namespace LibHac.Kvdb +{ + /// + /// A class for handling any value used by + /// + public class GenericValue : IExportable + { + private bool _isFrozen; + private byte[] _value; + + public int ExportSize => _value?.Length ?? 0; + + public void ToBytes(Span output) + { + if (output.Length < ExportSize) throw new InvalidOperationException("Output buffer is too small."); + + _value.CopyTo(output); + } + + public void FromBytes(ReadOnlySpan input) + { + if (_isFrozen) throw new InvalidOperationException("Unable to modify frozen object."); + + _value = input.ToArray(); + } + + public void Freeze() => _isFrozen = true; + } +} diff --git a/src/LibHac/Kvdb/IExportable.cs b/src/LibHac/Kvdb/IExportable.cs new file mode 100644 index 00000000..f72e3c18 --- /dev/null +++ b/src/LibHac/Kvdb/IExportable.cs @@ -0,0 +1,16 @@ +using System; + +namespace LibHac.Kvdb +{ + public interface IExportable + { + int ExportSize { get; } + void ToBytes(Span output); + void FromBytes(ReadOnlySpan input); + + /// + /// Prevent further modification of this object. + /// + void Freeze(); + } +} diff --git a/src/LibHac/Kvdb/ImkvdbHeader.cs b/src/LibHac/Kvdb/ImkvdbHeader.cs new file mode 100644 index 00000000..41f3c392 --- /dev/null +++ b/src/LibHac/Kvdb/ImkvdbHeader.cs @@ -0,0 +1,24 @@ +using System.Runtime.InteropServices; + +namespace LibHac.Kvdb +{ + [StructLayout(LayoutKind.Sequential, Size = 0xC)] + internal struct ImkvdbHeader + { + public const uint ExpectedMagic = 0x564B4D49; // IMKV + + public uint Magic; + public int Reserved; + public int EntryCount; + } + + [StructLayout(LayoutKind.Sequential, Size = 0xC)] + internal struct ImkvdbEntryHeader + { + public const uint ExpectedMagic = 0x4E454D49; // IMEN + + public uint Magic; + public int KeySize; + public int ValueSize; + } +} diff --git a/src/LibHac/Kvdb/ImkvdbReader.cs b/src/LibHac/Kvdb/ImkvdbReader.cs new file mode 100644 index 00000000..57da9c72 --- /dev/null +++ b/src/LibHac/Kvdb/ImkvdbReader.cs @@ -0,0 +1,79 @@ +using System; +using System.Runtime.CompilerServices; + +using static LibHac.Results; +using static LibHac.Kvdb.ResultsKvdb; + +namespace LibHac.Kvdb +{ + public ref struct ImkvdbReader + { + private ReadOnlySpan _data; + private int _position; + + public ImkvdbReader(ReadOnlySpan data) + { + _data = data; + _position = 0; + } + + public Result ReadHeader(out int entryCount) + { + entryCount = default; + + if (_position + Unsafe.SizeOf() > _data.Length) return ResultKvdbInvalidKeyValue; + + ref ImkvdbHeader header = ref Unsafe.As(ref Unsafe.AsRef(_data[_position])); + + if (header.Magic != ImkvdbHeader.ExpectedMagic) + { + return ResultKvdbInvalidKeyValue; + } + + entryCount = header.EntryCount; + _position += Unsafe.SizeOf(); + + return ResultSuccess; + } + + public Result GetEntrySize(out int keySize, out int valueSize) + { + keySize = default; + valueSize = default; + + if (_position + Unsafe.SizeOf() > _data.Length) return ResultKvdbInvalidKeyValue; + + ref ImkvdbEntryHeader header = ref Unsafe.As(ref Unsafe.AsRef(_data[_position])); + + if (header.Magic != ImkvdbEntryHeader.ExpectedMagic) + { + return ResultKvdbInvalidKeyValue; + } + + keySize = header.KeySize; + valueSize = header.ValueSize; + + return ResultSuccess; + } + + public Result ReadEntry(out ReadOnlySpan key, out ReadOnlySpan value) + { + key = default; + value = default; + + Result sizeResult = GetEntrySize(out int keySize, out int valueSize); + if (sizeResult.IsFailure()) return sizeResult; + + _position += Unsafe.SizeOf(); + + if (_position + keySize + valueSize > _data.Length) return ResultKvdbInvalidKeyValue; + + key = _data.Slice(_position, keySize); + value = _data.Slice(_position + keySize, valueSize); + + _position += keySize + valueSize; + + return ResultSuccess; + } + } +} diff --git a/src/LibHac/Kvdb/ImkvdbWriter.cs b/src/LibHac/Kvdb/ImkvdbWriter.cs new file mode 100644 index 00000000..fd060662 --- /dev/null +++ b/src/LibHac/Kvdb/ImkvdbWriter.cs @@ -0,0 +1,61 @@ +using System; +using System.Runtime.CompilerServices; + +namespace LibHac.Kvdb +{ + public ref struct ImkvdbWriter + { + private Span _data; + private int _position; + + public ImkvdbWriter(Span data) + { + _data = data; + _position = 0; + } + + public void WriteHeader(int entryCount) + { + if (_position + Unsafe.SizeOf() > _data.Length) throw new InvalidOperationException(); + + ref ImkvdbHeader header = ref Unsafe.As(ref _data[_position]); + + header.Magic = ImkvdbHeader.ExpectedMagic; + header.Reserved = 0; + header.EntryCount = entryCount; + + _position += Unsafe.SizeOf(); + } + + public void WriteEntry(IExportable key, IExportable value) + { + WriteEntryHeader(key.ExportSize, value.ExportSize); + Write(key); + Write(value); + } + + private void WriteEntryHeader(int keySize, int valueSize) + { + if (_position + Unsafe.SizeOf() > _data.Length) throw new InvalidOperationException(); + + ref ImkvdbEntryHeader header = ref Unsafe.As(ref _data[_position]); + + header.Magic = ImkvdbEntryHeader.ExpectedMagic; + header.KeySize = keySize; + header.ValueSize = valueSize; + + _position += Unsafe.SizeOf(); + } + + private void Write(IExportable value) + { + int valueSize = value.ExportSize; + if (_position + valueSize > _data.Length) throw new InvalidOperationException(); + + Span dest = _data.Slice(_position, valueSize); + value.ToBytes(dest); + + _position += valueSize; + } + } +} diff --git a/src/LibHac/Kvdb/KeyValueDatabase.cs b/src/LibHac/Kvdb/KeyValueDatabase.cs new file mode 100644 index 00000000..87b6b15a --- /dev/null +++ b/src/LibHac/Kvdb/KeyValueDatabase.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; + +using static LibHac.Results; +using static LibHac.Kvdb.ResultsKvdb; + +namespace LibHac.Kvdb +{ + // Todo: Save and load from file + public class KeyValueDatabase + where TKey : IComparable, IEquatable, IExportable, new() + where TValue : IExportable, new() + { + private Dictionary KvDict { get; } = new Dictionary(); + + public int Count => KvDict.Count; + + public Result Get(TKey key, out TValue value) + { + if (!KvDict.TryGetValue(key, out value)) + { + return ResultKvdbKeyNotFound; + } + + return ResultSuccess; + } + + public Result Set(TKey key, TValue value) + { + key.Freeze(); + + KvDict[key] = value; + + return ResultSuccess; + } + + public Result ReadDatabaseFromBuffer(ReadOnlySpan data) + { + var reader = new ImkvdbReader(data); + + Result headerResult = reader.ReadHeader(out int entryCount); + if (headerResult.IsFailure()) return headerResult; + + for (int i = 0; i < entryCount; i++) + { + Result entryResult = reader.ReadEntry(out ReadOnlySpan keyBytes, out ReadOnlySpan valueBytes); + if (entryResult.IsFailure()) return entryResult; + + var key = new TKey(); + var value = new TValue(); + + key.FromBytes(keyBytes); + value.FromBytes(valueBytes); + + key.Freeze(); + + KvDict.Add(key, value); + } + + return ResultSuccess; + } + + public Result WriteDatabaseToBuffer(Span output) + { + var writer = new ImkvdbWriter(output); + + writer.WriteHeader(KvDict.Count); + + foreach (KeyValuePair entry in KvDict.OrderBy(x => x.Key)) + { + writer.WriteEntry(entry.Key, entry.Value); + } + + return ResultSuccess; + } + + public int GetExportedSize() + { + int size = Unsafe.SizeOf(); + + foreach (KeyValuePair entry in KvDict) + { + size += Unsafe.SizeOf(); + size += entry.Key.ExportSize; + size += entry.Value.ExportSize; + } + + return size; + } + } +} diff --git a/src/LibHac/Kvdb/ResultsKvdb.cs b/src/LibHac/Kvdb/ResultsKvdb.cs new file mode 100644 index 00000000..333510ec --- /dev/null +++ b/src/LibHac/Kvdb/ResultsKvdb.cs @@ -0,0 +1,13 @@ +namespace LibHac.Kvdb +{ + public static class ResultsKvdb + { + public const int ModuleKvdb = 20; + + public static Result ResultKvdbTooLargeKey => new Result(ModuleKvdb, 1); + public static Result ResultKvdbKeyNotFound => new Result(ModuleKvdb, 2); + public static Result ResultKvdbAllocationFailed => new Result(ModuleKvdb, 4); + public static Result ResultKvdbInvalidKeyValue => new Result(ModuleKvdb, 5); + public static Result ResultKvdbBufferInsufficient => new Result(ModuleKvdb, 6); + } +} diff --git a/src/LibHac/Ncm/ContentMetaKey.cs b/src/LibHac/Ncm/ContentMetaKey.cs new file mode 100644 index 00000000..fda438a4 --- /dev/null +++ b/src/LibHac/Ncm/ContentMetaKey.cs @@ -0,0 +1,84 @@ +using System; +using System.Buffers.Binary; +using LibHac.Kvdb; + +namespace LibHac.Ncm +{ + public class ContentMetaKey : IComparable, IComparable, IEquatable, IExportable + { + public ulong TitleId { get; private set; } + public uint Version { get; private set; } + public byte Type { get; private set; } + public byte Flags { get; private set; } + + public int ExportSize => 0x10; + private bool _isFrozen; + + public void ToBytes(Span output) + { + if (output.Length < ExportSize) throw new InvalidOperationException("Output buffer is too small."); + + BinaryPrimitives.WriteUInt64LittleEndian(output, TitleId); + BinaryPrimitives.WriteUInt32LittleEndian(output.Slice(8), Version); + output[0xC] = Type; + output[0xD] = Flags; + } + + public void FromBytes(ReadOnlySpan input) + { + if (_isFrozen) throw new InvalidOperationException("Unable to modify frozen object."); + if (input.Length < ExportSize) throw new InvalidOperationException("Input data is too short."); + + TitleId = BinaryPrimitives.ReadUInt64LittleEndian(input); + Version = BinaryPrimitives.ReadUInt32LittleEndian(input.Slice(8)); + Type = input[0xC]; + Flags = input[0xD]; + } + + public void Freeze() => _isFrozen = true; + + public bool Equals(ContentMetaKey other) + { + return other != null && TitleId == other.TitleId && Version == other.Version && + Type == other.Type && Flags == other.Flags; + } + + public override bool Equals(object obj) + { + return obj is ContentMetaKey other && Equals(other); + } + + public override int GetHashCode() + { + unchecked + { + // ReSharper disable NonReadonlyMemberInGetHashCode + int hashCode = TitleId.GetHashCode(); + hashCode = (hashCode * 397) ^ (int)Version; + hashCode = (hashCode * 397) ^ Type.GetHashCode(); + hashCode = (hashCode * 397) ^ Flags.GetHashCode(); + return hashCode; + // ReSharper restore NonReadonlyMemberInGetHashCode + } + } + + public int CompareTo(ContentMetaKey other) + { + if (ReferenceEquals(this, other)) return 0; + if (ReferenceEquals(null, other)) return 1; + int titleIdComparison = TitleId.CompareTo(other.TitleId); + if (titleIdComparison != 0) return titleIdComparison; + int versionComparison = Version.CompareTo(other.Version); + if (versionComparison != 0) return versionComparison; + int typeComparison = Type.CompareTo(other.Type); + if (typeComparison != 0) return typeComparison; + return Flags.CompareTo(other.Flags); + } + + public int CompareTo(object obj) + { + if (obj is null) return 1; + return obj is ContentMetaKey other ? CompareTo(other) : throw new ArgumentException($"Object must be of type {nameof(ContentMetaKey)}"); + } + } +} diff --git a/src/LibHac/Result.cs b/src/LibHac/Result.cs new file mode 100644 index 00000000..d21cddde --- /dev/null +++ b/src/LibHac/Result.cs @@ -0,0 +1,28 @@ +namespace LibHac +{ + public struct Result + { + public readonly int Value; + + public Result(int value) + { + Value = value; + } + + public Result(int module, int description) + { + Value = (description << 9) | module; + } + + public int Description => (Value >> 9) & 0x1FFF; + public int Module => Value & 0x1FF; + + public bool IsSuccess() => Value == 0; + public bool IsFailure() => Value != 0; + } + + public static class Results + { + public static Result ResultSuccess => new Result(0); + } +}