From 73b472befa182332466aa64db7d9e8823df5a3ee Mon Sep 17 00:00:00 2001 From: shadowninja108 Date: Sat, 22 Sep 2018 20:19:27 -0700 Subject: [PATCH 1/5] Add partial NSO support --- LibHac/LibHac.csproj | 1 + libhac/Nso.cs | 113 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 libhac/Nso.cs diff --git a/LibHac/LibHac.csproj b/LibHac/LibHac.csproj index 59228d90..3f4cd941 100644 --- a/LibHac/LibHac.csproj +++ b/LibHac/LibHac.csproj @@ -27,6 +27,7 @@ + diff --git a/libhac/Nso.cs b/libhac/Nso.cs new file mode 100644 index 00000000..8d983c32 --- /dev/null +++ b/libhac/Nso.cs @@ -0,0 +1,113 @@ +using LibHac.Streams; +using LZ4; +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace LibHac +{ + public class Nso + { + public NsoSection[] Sections; + public RodataRelativeExtent[] RodataRelativeExtents; + public uint BssSize; + public byte[] BuildID = new byte[0x20]; + + private BinaryReader reader; + + public Nso(Stream stream) + { + reader = new BinaryReader(stream); + if (reader.ReadAscii(4) != "NSO0") + throw new InvalidDataException("NSO magic is incorrect!"); + reader.ReadUInt32(); // Version + reader.ReadUInt32(); // Reserved/Unused + BitArray flags = new BitArray(new int[] { (int) reader.ReadUInt32() }); + NsoSection textSection = new NsoSection(stream); + NsoSection rodataSection = new NsoSection(stream); + NsoSection dataSection = new NsoSection(stream); + textSection.IsCompressed = flags[0]; + textSection.CheckHash = flags[3]; + rodataSection.IsCompressed = flags[1]; + rodataSection.CheckHash = flags[4]; + dataSection.IsCompressed = flags[2]; + dataSection.CheckHash = flags[5]; + + ReadSegmentHeader(textSection); + reader.ReadUInt32(); // Module offset (TODO) + ReadSegmentHeader(rodataSection); + reader.ReadUInt32(); // Module file size + ReadSegmentHeader(dataSection); + BssSize = reader.ReadUInt32(); + reader.Read(BuildID, 0, 0x20); + textSection.CompressedSize = reader.ReadUInt32(); + rodataSection.CompressedSize = reader.ReadUInt32(); + dataSection.CompressedSize = reader.ReadUInt32(); + reader.ReadBytes(0x1C); // Padding + RodataRelativeExtents = new RodataRelativeExtent[] + { + ReadRodataRelativeExtent(), ReadRodataRelativeExtent(), ReadRodataRelativeExtent() + }; + + reader.Read(textSection.Hash, 0, 0x20); + reader.Read(rodataSection.Hash, 0, 0x20); + reader.Read(dataSection.Hash, 0, 0x20); + + Sections = new NsoSection[] {textSection, rodataSection, dataSection }; + } + + public void ReadSegmentHeader(NsoSection section) + { + section.FileOffset = reader.ReadUInt32(); + section.MemoryOffset = reader.ReadUInt32(); + section.DecompressedSize = reader.ReadUInt32(); + } + + public RodataRelativeExtent ReadRodataRelativeExtent() + { + RodataRelativeExtent extent = new RodataRelativeExtent(); + extent.RegionRodataOffset = reader.ReadUInt32(); + extent.RegionSize = reader.ReadUInt32(); + return extent; + } + + public class NsoSection + { + private Stream Stream; + + public bool IsCompressed, + CheckHash; + public uint FileOffset, + MemoryOffset, + DecompressedSize, + CompressedSize; + public byte[] Hash = new byte[0x20]; + + public NsoSection(Stream stream) + { + Stream = stream; + } + + public Stream OpenCompressedStream() + { + return new SubStream(Stream, FileOffset, CompressedSize); + } + + public Stream OpenDecompressedStream() + { + return new LZ4Stream(OpenCompressedStream(), LZ4StreamMode.Decompress); + } + } + + public class RodataRelativeExtent + { + public uint + RegionRodataOffset, + RegionSize; + } + + + } +} From e694833e0cb176a11b3d1298b71da656b307fd08 Mon Sep 17 00:00:00 2001 From: shadowninja108 Date: Sat, 22 Sep 2018 21:46:15 -0700 Subject: [PATCH 2/5] Switched to Ryujinx's implementation of LZ4 Fixed stream handling in Nso.cs --- LibHac/LibHac.csproj | 1 - libhac/Lz4.cs | 78 ++++++++++++++++++++++++++++++++++++++++++++ libhac/Nso.cs | 37 ++++++++++++++------- 3 files changed, 103 insertions(+), 13 deletions(-) create mode 100644 libhac/Lz4.cs diff --git a/LibHac/LibHac.csproj b/LibHac/LibHac.csproj index 3f4cd941..59228d90 100644 --- a/LibHac/LibHac.csproj +++ b/LibHac/LibHac.csproj @@ -27,7 +27,6 @@ - diff --git a/libhac/Lz4.cs b/libhac/Lz4.cs new file mode 100644 index 00000000..cfb49551 --- /dev/null +++ b/libhac/Lz4.cs @@ -0,0 +1,78 @@ +using System; + +namespace Ryujinx.HLE.Loaders.Compression +{ + static class Lz4 + { + public static byte[] Decompress(byte[] Cmp, int DecLength) + { + byte[] Dec = new byte[DecLength]; + + int CmpPos = 0; + int DecPos = 0; + + int GetLength(int Length) + { + byte Sum; + + if (Length == 0xf) + { + do + { + Length += (Sum = Cmp[CmpPos++]); + } + while (Sum == 0xff); + } + + return Length; + } + + do + { + byte Token = Cmp[CmpPos++]; + + int EncCount = (Token >> 0) & 0xf; + int LitCount = (Token >> 4) & 0xf; + + //Copy literal chunck + LitCount = GetLength(LitCount); + + Buffer.BlockCopy(Cmp, CmpPos, Dec, DecPos, LitCount); + + CmpPos += LitCount; + DecPos += LitCount; + + if (CmpPos >= Cmp.Length) + { + break; + } + + //Copy compressed chunck + int Back = Cmp[CmpPos++] << 0 | + Cmp[CmpPos++] << 8; + + EncCount = GetLength(EncCount) + 4; + + int EncPos = DecPos - Back; + + if (EncCount <= Back) + { + Buffer.BlockCopy(Dec, EncPos, Dec, DecPos, EncCount); + + DecPos += EncCount; + } + else + { + while (EncCount-- > 0) + { + Dec[DecPos++] = Dec[EncPos++]; + } + } + } + while (CmpPos < Cmp.Length && + DecPos < Dec.Length); + + return Dec; + } + } +} \ No newline at end of file diff --git a/libhac/Nso.cs b/libhac/Nso.cs index 8d983c32..11da2a6b 100644 --- a/libhac/Nso.cs +++ b/libhac/Nso.cs @@ -1,5 +1,5 @@ using LibHac.Streams; -using LZ4; +using Ryujinx.HLE.Loaders.Compression; using System; using System.Collections; using System.Collections.Generic; @@ -15,19 +15,20 @@ namespace LibHac public uint BssSize; public byte[] BuildID = new byte[0x20]; - private BinaryReader reader; + private SharedStreamSource StreamSource; public Nso(Stream stream) { - reader = new BinaryReader(stream); + StreamSource = new SharedStreamSource(stream); + BinaryReader reader = new BinaryReader(StreamSource.CreateStream()); if (reader.ReadAscii(4) != "NSO0") throw new InvalidDataException("NSO magic is incorrect!"); reader.ReadUInt32(); // Version reader.ReadUInt32(); // Reserved/Unused BitArray flags = new BitArray(new int[] { (int) reader.ReadUInt32() }); - NsoSection textSection = new NsoSection(stream); - NsoSection rodataSection = new NsoSection(stream); - NsoSection dataSection = new NsoSection(stream); + NsoSection textSection = new NsoSection(StreamSource); + NsoSection rodataSection = new NsoSection(StreamSource); + NsoSection dataSection = new NsoSection(StreamSource); textSection.IsCompressed = flags[0]; textSection.CheckHash = flags[3]; rodataSection.IsCompressed = flags[1]; @@ -56,26 +57,31 @@ namespace LibHac reader.Read(dataSection.Hash, 0, 0x20); Sections = new NsoSection[] {textSection, rodataSection, dataSection }; + reader.Close(); } public void ReadSegmentHeader(NsoSection section) { + BinaryReader reader = new BinaryReader(StreamSource.CreateStream()); section.FileOffset = reader.ReadUInt32(); section.MemoryOffset = reader.ReadUInt32(); - section.DecompressedSize = reader.ReadUInt32(); + section.DecompressedSize = reader.ReadUInt32(); + reader.Close(); } public RodataRelativeExtent ReadRodataRelativeExtent() { + BinaryReader reader = new BinaryReader(StreamSource.CreateStream()); RodataRelativeExtent extent = new RodataRelativeExtent(); extent.RegionRodataOffset = reader.ReadUInt32(); extent.RegionSize = reader.ReadUInt32(); + reader.Close(); return extent; } public class NsoSection { - private Stream Stream; + private readonly SharedStreamSource StreamSource; public bool IsCompressed, CheckHash; @@ -85,19 +91,26 @@ namespace LibHac CompressedSize; public byte[] Hash = new byte[0x20]; - public NsoSection(Stream stream) + public NsoSection(SharedStreamSource streamSource) { - Stream = stream; + StreamSource = streamSource; } public Stream OpenCompressedStream() { - return new SubStream(Stream, FileOffset, CompressedSize); + return StreamSource.CreateStream(FileOffset, CompressedSize); } public Stream OpenDecompressedStream() { - return new LZ4Stream(OpenCompressedStream(), LZ4StreamMode.Decompress); + return new MemoryStream(Decompress()); + } + + public byte[] Decompress() + { + byte[] compressed = new byte[CompressedSize]; + OpenCompressedStream().Read(compressed, 0, (int) CompressedSize); + return Lz4.Decompress(compressed, (int) DecompressedSize); } } From 0efc550fa61a4eebecbd957dd6d8e35c4d662c3a Mon Sep 17 00:00:00 2001 From: shadowninja108 Date: Sat, 22 Sep 2018 21:52:30 -0700 Subject: [PATCH 3/5] Better handling of the IsCompressed flag in Nso.cs --- libhac/Nso.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libhac/Nso.cs b/libhac/Nso.cs index 11da2a6b..5b412593 100644 --- a/libhac/Nso.cs +++ b/libhac/Nso.cs @@ -110,7 +110,10 @@ namespace LibHac { byte[] compressed = new byte[CompressedSize]; OpenCompressedStream().Read(compressed, 0, (int) CompressedSize); - return Lz4.Decompress(compressed, (int) DecompressedSize); + if (IsCompressed) + return Lz4.Decompress(compressed, (int)DecompressedSize); + else + return compressed; } } From 86aa80c87b32ef8b454937ac07de056f8ac06c2a Mon Sep 17 00:00:00 2001 From: shadowninja108 Date: Sun, 23 Sep 2018 00:24:14 -0700 Subject: [PATCH 4/5] Fixed Nso implementation Added more keyset printing --- LibHac/Keyset.cs | 47 ++++++++++++++++++++++++++++++++++++++++------- libhac/Nso.cs | 16 ++++++---------- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/LibHac/Keyset.cs b/LibHac/Keyset.cs index 68c256bd..0ef37cc3 100644 --- a/LibHac/Keyset.cs +++ b/LibHac/Keyset.cs @@ -269,9 +269,11 @@ namespace LibHac public static class ExternalKeys { private const int TitleKeySize = 0x10; - private static readonly Dictionary CommonKeyDict; - private static readonly Dictionary UniqueKeyDict; - private static readonly Dictionary AllKeyDict; + + public static readonly Dictionary + CommonKeyDict, + UniqueKeyDict, + AllKeyDict; static ExternalKeys() { @@ -393,12 +395,12 @@ namespace LibHac } } - public static string PrintKeys(Keyset keyset) + public static string PrintKeys(Keyset keyset, Dictionary dict) { var sb = new StringBuilder(); - int maxNameLength = CommonKeyDict.Values.Max(x => x.Name.Length); + int maxNameLength = dict.Values.Max(x => x.Name.Length); - foreach (KeyValue keySlot in CommonKeyDict.Values.OrderBy(x => x.Name)) + foreach (KeyValue keySlot in dict.Values.OrderBy(x => x.Name)) { byte[] key = keySlot.GetKey(keyset); if (key.IsEmpty()) continue; @@ -410,6 +412,37 @@ namespace LibHac return sb.ToString(); } + public static string PrintCommonKeys(Keyset keyset) + { + return PrintKeys(keyset, CommonKeyDict); + } + + public static string PrintUniqueKeys(Keyset keyset) + { + return PrintKeys(keyset, UniqueKeyDict); + } + + public static string PrintAllKeys(Keyset keyset) + { + return PrintKeys(keyset, AllKeyDict); + } + + public static string PrintTitleKeys(Keyset keyset) + { + var sb = new StringBuilder(); + int maxNameLength = keyset.TitleKeys.Values.Max(x => x.Length); + + foreach (KeyValuePair kv in keyset.TitleKeys) + { + byte[] key = kv.Key; + byte[] value = kv.Value; + var line = $"{key.ToHexString().PadRight(maxNameLength)} = {value.ToHexString()}"; + sb.AppendLine(line); + } + + return sb.ToString(); + } + private static List CreateCommonKeyList() { var keys = new List @@ -489,7 +522,7 @@ namespace LibHac return keys; } - private class KeyValue + public class KeyValue { public readonly string Name; public readonly int Size; diff --git a/libhac/Nso.cs b/libhac/Nso.cs index 5b412593..d8ea6404 100644 --- a/libhac/Nso.cs +++ b/libhac/Nso.cs @@ -36,11 +36,11 @@ namespace LibHac dataSection.IsCompressed = flags[2]; dataSection.CheckHash = flags[5]; - ReadSegmentHeader(textSection); + ReadSegmentHeader(textSection, reader); reader.ReadUInt32(); // Module offset (TODO) - ReadSegmentHeader(rodataSection); + ReadSegmentHeader(rodataSection, reader); reader.ReadUInt32(); // Module file size - ReadSegmentHeader(dataSection); + ReadSegmentHeader(dataSection, reader); BssSize = reader.ReadUInt32(); reader.Read(BuildID, 0, 0x20); textSection.CompressedSize = reader.ReadUInt32(); @@ -49,7 +49,7 @@ namespace LibHac reader.ReadBytes(0x1C); // Padding RodataRelativeExtents = new RodataRelativeExtent[] { - ReadRodataRelativeExtent(), ReadRodataRelativeExtent(), ReadRodataRelativeExtent() + ReadRodataRelativeExtent(reader), ReadRodataRelativeExtent(reader), ReadRodataRelativeExtent(reader) }; reader.Read(textSection.Hash, 0, 0x20); @@ -60,22 +60,18 @@ namespace LibHac reader.Close(); } - public void ReadSegmentHeader(NsoSection section) + public void ReadSegmentHeader(NsoSection section, BinaryReader reader) { - BinaryReader reader = new BinaryReader(StreamSource.CreateStream()); section.FileOffset = reader.ReadUInt32(); section.MemoryOffset = reader.ReadUInt32(); section.DecompressedSize = reader.ReadUInt32(); - reader.Close(); } - public RodataRelativeExtent ReadRodataRelativeExtent() + public RodataRelativeExtent ReadRodataRelativeExtent(BinaryReader reader) { - BinaryReader reader = new BinaryReader(StreamSource.CreateStream()); RodataRelativeExtent extent = new RodataRelativeExtent(); extent.RegionRodataOffset = reader.ReadUInt32(); extent.RegionSize = reader.ReadUInt32(); - reader.Close(); return extent; } From fba8a7c6e0e089a1c263b043a474cd570611979e Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Sun, 23 Sep 2018 21:18:20 -0500 Subject: [PATCH 5/5] Code style tweaks --- LibHac/Keyset.cs | 7 ++-- libhac/Nso.cs | 103 +++++++++++++++++++++-------------------------- 2 files changed, 49 insertions(+), 61 deletions(-) diff --git a/LibHac/Keyset.cs b/LibHac/Keyset.cs index 0ef37cc3..c8c15777 100644 --- a/LibHac/Keyset.cs +++ b/LibHac/Keyset.cs @@ -270,10 +270,9 @@ namespace LibHac { private const int TitleKeySize = 0x10; - public static readonly Dictionary - CommonKeyDict, - UniqueKeyDict, - AllKeyDict; + public static readonly Dictionary CommonKeyDict; + public static readonly Dictionary UniqueKeyDict; + public static readonly Dictionary AllKeyDict; static ExternalKeys() { diff --git a/libhac/Nso.cs b/libhac/Nso.cs index d8ea6404..c777c12b 100644 --- a/libhac/Nso.cs +++ b/libhac/Nso.cs @@ -1,21 +1,18 @@ -using LibHac.Streams; -using Ryujinx.HLE.Loaders.Compression; -using System; -using System.Collections; -using System.Collections.Generic; +using System.Collections; using System.IO; -using System.Text; +using LibHac.Streams; +using Ryujinx.HLE.Loaders.Compression; namespace LibHac { public class Nso { - public NsoSection[] Sections; - public RodataRelativeExtent[] RodataRelativeExtents; - public uint BssSize; - public byte[] BuildID = new byte[0x20]; + public NsoSection[] Sections { get; } + public RodataRelativeExtent[] RodataRelativeExtents { get; } + public uint BssSize { get; } + public byte[] BuildId { get; } = new byte[0x20]; - private SharedStreamSource StreamSource; + private SharedStreamSource StreamSource { get; } public Nso(Stream stream) { @@ -25,101 +22,93 @@ namespace LibHac throw new InvalidDataException("NSO magic is incorrect!"); reader.ReadUInt32(); // Version reader.ReadUInt32(); // Reserved/Unused - BitArray flags = new BitArray(new int[] { (int) reader.ReadUInt32() }); + BitArray flags = new BitArray(new[] { (int)reader.ReadUInt32() }); NsoSection textSection = new NsoSection(StreamSource); NsoSection rodataSection = new NsoSection(StreamSource); NsoSection dataSection = new NsoSection(StreamSource); textSection.IsCompressed = flags[0]; - textSection.CheckHash = flags[3]; rodataSection.IsCompressed = flags[1]; - rodataSection.CheckHash = flags[4]; dataSection.IsCompressed = flags[2]; + textSection.CheckHash = flags[3]; + rodataSection.CheckHash = flags[4]; dataSection.CheckHash = flags[5]; - ReadSegmentHeader(textSection, reader); + textSection.ReadSegmentHeader(reader); reader.ReadUInt32(); // Module offset (TODO) - ReadSegmentHeader(rodataSection, reader); + rodataSection.ReadSegmentHeader(reader); reader.ReadUInt32(); // Module file size - ReadSegmentHeader(dataSection, reader); + dataSection.ReadSegmentHeader(reader); BssSize = reader.ReadUInt32(); - reader.Read(BuildID, 0, 0x20); + reader.Read(BuildId, 0, 0x20); textSection.CompressedSize = reader.ReadUInt32(); rodataSection.CompressedSize = reader.ReadUInt32(); dataSection.CompressedSize = reader.ReadUInt32(); reader.ReadBytes(0x1C); // Padding - RodataRelativeExtents = new RodataRelativeExtent[] + RodataRelativeExtents = new[] { - ReadRodataRelativeExtent(reader), ReadRodataRelativeExtent(reader), ReadRodataRelativeExtent(reader) + new RodataRelativeExtent(reader), new RodataRelativeExtent(reader), new RodataRelativeExtent(reader) }; reader.Read(textSection.Hash, 0, 0x20); reader.Read(rodataSection.Hash, 0, 0x20); reader.Read(dataSection.Hash, 0, 0x20); - Sections = new NsoSection[] {textSection, rodataSection, dataSection }; + Sections = new[] { textSection, rodataSection, dataSection }; reader.Close(); } - public void ReadSegmentHeader(NsoSection section, BinaryReader reader) - { - section.FileOffset = reader.ReadUInt32(); - section.MemoryOffset = reader.ReadUInt32(); - section.DecompressedSize = reader.ReadUInt32(); - } - - public RodataRelativeExtent ReadRodataRelativeExtent(BinaryReader reader) - { - RodataRelativeExtent extent = new RodataRelativeExtent(); - extent.RegionRodataOffset = reader.ReadUInt32(); - extent.RegionSize = reader.ReadUInt32(); - return extent; - } - public class NsoSection { - private readonly SharedStreamSource StreamSource; + private SharedStreamSource StreamSource { get; } - public bool IsCompressed, - CheckHash; - public uint FileOffset, - MemoryOffset, - DecompressedSize, - CompressedSize; - public byte[] Hash = new byte[0x20]; + public bool IsCompressed { get; set; } + public bool CheckHash { get; set; } + public uint FileOffset { get; set; } + public uint MemoryOffset { get; set; } + public uint DecompressedSize { get; set; } + public uint CompressedSize { get; set; } + + public byte[] Hash { get; } = new byte[0x20]; public NsoSection(SharedStreamSource streamSource) { StreamSource = streamSource; } - public Stream OpenCompressedStream() + public Stream OpenSection() { return StreamSource.CreateStream(FileOffset, CompressedSize); } - public Stream OpenDecompressedStream() - { - return new MemoryStream(Decompress()); - } - - public byte[] Decompress() + public byte[] DecompressSection() { byte[] compressed = new byte[CompressedSize]; - OpenCompressedStream().Read(compressed, 0, (int) CompressedSize); + OpenSection().Read(compressed, 0, (int)CompressedSize); + if (IsCompressed) return Lz4.Decompress(compressed, (int)DecompressedSize); else return compressed; } + + internal void ReadSegmentHeader(BinaryReader reader) + { + FileOffset = reader.ReadUInt32(); + MemoryOffset = reader.ReadUInt32(); + DecompressedSize = reader.ReadUInt32(); + } } public class RodataRelativeExtent { - public uint - RegionRodataOffset, - RegionSize; + public uint RegionRodataOffset { get; } + public uint RegionSize { get; } + + public RodataRelativeExtent(BinaryReader reader) + { + RegionRodataOffset = reader.ReadUInt32(); + RegionSize = reader.ReadUInt32(); + } } - - } }