From 0300f55da52435c12d95136d2174ca632f4810d5 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Thu, 4 Oct 2018 21:40:59 -0500 Subject: [PATCH 1/4] Give RomfsDir and RomfsFile a common base --- LibHac/Romfs.cs | 98 +++----------------------------------------- LibHac/RomfsEntry.cs | 95 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 92 deletions(-) create mode 100644 LibHac/RomfsEntry.cs diff --git a/LibHac/Romfs.cs b/LibHac/Romfs.cs index 6892303f..be15a7a7 100644 --- a/LibHac/Romfs.cs +++ b/LibHac/Romfs.cs @@ -1,5 +1,4 @@ using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; using System.Text; @@ -15,7 +14,7 @@ namespace LibHac public List Files { get; } = new List(); public RomfsDir RootDir { get; } - private Dictionary FileDict { get; } + public Dictionary FileDict { get; } private SharedStreamSource StreamSource { get; } public Romfs(Stream stream) @@ -40,7 +39,7 @@ namespace LibHac { var dir = new RomfsDir(reader) { Offset = position }; Directories.Add(dir); - if (dir.ParentOffset == position) RootDir = dir; + if (dir.ParentDirOffset == position) RootDir = dir; position = (int)reader.BaseStream.Position; } } @@ -57,7 +56,8 @@ namespace LibHac } SetReferences(); - ResolveFilenames(); + RomfsEntry.ResolveFilenames(Files); + RomfsEntry.ResolveFilenames(Directories); FileDict = Files.ToDictionary(x => x.FullPath, x => x); } @@ -99,7 +99,7 @@ namespace LibHac foreach (RomfsDir dir in Directories) { - if (dir.ParentOffset >= 0 && dir.ParentOffset != dir.Offset) dir.Parent = dirDict[dir.ParentOffset]; + if (dir.ParentDirOffset >= 0 && dir.ParentDirOffset != dir.Offset) dir.ParentDir = dirDict[dir.ParentDirOffset]; if (dir.NextSiblingOffset >= 0) dir.NextSibling = dirDict[dir.NextSiblingOffset]; if (dir.FirstChildOffset >= 0) dir.FirstChild = dirDict[dir.FirstChildOffset]; if (dir.FirstFileOffset >= 0) dir.FirstFile = fileDict[dir.FirstFileOffset]; @@ -113,33 +113,6 @@ namespace LibHac if (file.NextFileHashOffset >= 0) file.NextFileHash = fileDict[file.NextFileHashOffset]; } } - - private void ResolveFilenames() - { - var list = new List(); - var sb = new StringBuilder(); - string delimiter = "/"; - foreach (RomfsFile file in Files) - { - list.Add(file.Name); - RomfsDir dir = file.ParentDir; - while (dir != null) - { - list.Add(delimiter); - list.Add(dir.Name); - dir = dir.Parent; - } - - for (int i = list.Count - 1; i >= 0; i--) - { - sb.Append(list[i]); - } - - file.FullPath = sb.ToString(); - list.Clear(); - sb.Clear(); - } - } } public class RomfsHeader @@ -170,66 +143,7 @@ namespace LibHac } } - [DebuggerDisplay("{" + nameof(Name) + "}")] - public class RomfsDir - { - public int Offset { get; set; } - public int ParentOffset { get; } - public int NextSiblingOffset { get; } - public int FirstChildOffset { get; } - public int FirstFileOffset { get; } - public int NextDirHashOffset { get; } - public int NameLength { get; } - public string Name { get; } - - public RomfsDir Parent { get; internal set; } - public RomfsDir NextSibling { get; internal set; } - public RomfsDir FirstChild { get; internal set; } - public RomfsFile FirstFile { get; internal set; } - public RomfsDir NextDirHash { get; internal set; } - - public RomfsDir(BinaryReader reader) - { - ParentOffset = reader.ReadInt32(); - NextSiblingOffset = reader.ReadInt32(); - FirstChildOffset = reader.ReadInt32(); - FirstFileOffset = reader.ReadInt32(); - NextDirHashOffset = reader.ReadInt32(); - NameLength = reader.ReadInt32(); - Name = reader.ReadUtf8(NameLength); - reader.BaseStream.Position = Util.GetNextMultiple(reader.BaseStream.Position, 4); - } - } - - [DebuggerDisplay("{" + nameof(Name) + "}")] - public class RomfsFile - { - public int Offset { get; set; } - public int ParentDirOffset { get; } - public int NextSiblingOffset { get; } - public long DataOffset { get; } - public long DataLength { get; } - public int NextFileHashOffset { get; } - public int NameLength { get; } - public string Name { get; } - - public RomfsDir ParentDir { get; internal set; } - public RomfsFile NextSibling { get; internal set; } - public RomfsFile NextFileHash { get; internal set; } - public string FullPath { get; set; } - - public RomfsFile(BinaryReader reader) - { - ParentDirOffset = reader.ReadInt32(); - NextSiblingOffset = reader.ReadInt32(); - DataOffset = reader.ReadInt64(); - DataLength = reader.ReadInt64(); - NextFileHashOffset = reader.ReadInt32(); - NameLength = reader.ReadInt32(); - Name = reader.ReadUtf8(NameLength); - reader.BaseStream.Position = Util.GetNextMultiple(reader.BaseStream.Position, 4); - } - } + public class IvfcLevel { diff --git a/LibHac/RomfsEntry.cs b/LibHac/RomfsEntry.cs new file mode 100644 index 00000000..1fa90653 --- /dev/null +++ b/LibHac/RomfsEntry.cs @@ -0,0 +1,95 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Text; + +namespace LibHac +{ + public abstract class RomfsEntry + { + public int Offset { get; set; } + public int ParentDirOffset { get; protected set; } + public int NameLength { get; protected set; } + public string Name { get; protected set; } + + public RomfsDir ParentDir { get; internal set; } + public string FullPath { get; private set; } + + internal static void ResolveFilenames(IEnumerable entries) + { + var list = new List(); + var sb = new StringBuilder(); + const string delimiter = "/"; + foreach (RomfsEntry file in entries) + { + list.Add(file.Name); + RomfsDir dir = file.ParentDir; + while (dir != null) + { + list.Add(delimiter); + list.Add(dir.Name); + dir = dir.ParentDir; + } + + for (int i = list.Count - 1; i >= 0; i--) + { + sb.Append(list[i]); + } + + file.FullPath = sb.ToString(); + list.Clear(); + sb.Clear(); + } + } + } + + [DebuggerDisplay("{" + nameof(Name) + "}")] + public class RomfsDir : RomfsEntry + { + public int NextSiblingOffset { get; } + public int FirstChildOffset { get; } + public int FirstFileOffset { get; } + public int NextDirHashOffset { get; } + + public RomfsDir NextSibling { get; internal set; } + public RomfsDir FirstChild { get; internal set; } + public RomfsFile FirstFile { get; internal set; } + public RomfsDir NextDirHash { get; internal set; } + + public RomfsDir(BinaryReader reader) + { + ParentDirOffset = reader.ReadInt32(); + NextSiblingOffset = reader.ReadInt32(); + FirstChildOffset = reader.ReadInt32(); + FirstFileOffset = reader.ReadInt32(); + NextDirHashOffset = reader.ReadInt32(); + NameLength = reader.ReadInt32(); + Name = reader.ReadUtf8(NameLength); + reader.BaseStream.Position = Util.GetNextMultiple(reader.BaseStream.Position, 4); + } + } + + [DebuggerDisplay("{" + nameof(Name) + "}")] + public class RomfsFile : RomfsEntry + { + public int NextSiblingOffset { get; } + public long DataOffset { get; } + public long DataLength { get; } + public int NextFileHashOffset { get; } + + public RomfsFile NextSibling { get; internal set; } + public RomfsFile NextFileHash { get; internal set; } + + public RomfsFile(BinaryReader reader) + { + ParentDirOffset = reader.ReadInt32(); + NextSiblingOffset = reader.ReadInt32(); + DataOffset = reader.ReadInt64(); + DataLength = reader.ReadInt64(); + NextFileHashOffset = reader.ReadInt32(); + NameLength = reader.ReadInt32(); + Name = reader.ReadUtf8(NameLength); + reader.BaseStream.Position = Util.GetNextMultiple(reader.BaseStream.Position, 4); + } + } +} From db00267ef401c5768f3427cc0a993602cb2be6f8 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Fri, 5 Oct 2018 12:23:18 -0500 Subject: [PATCH 2/4] Dispose SharedStreamSource. Temp fix for switchfs reading without title keys --- LibHac/Nca.cs | 11 +++++------ LibHac/Streams/SharedStreamSource.cs | 19 ++++++++++++++++--- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/LibHac/Nca.cs b/LibHac/Nca.cs index a42d8b9f..213c1bc2 100644 --- a/LibHac/Nca.cs +++ b/LibHac/Nca.cs @@ -16,7 +16,6 @@ namespace LibHac public byte[][] DecryptedKeys { get; } = Util.CreateJaggedArray(4, 0x10); public byte[] TitleKey { get; } public byte[] TitleKeyDec { get; } = new byte[0x10]; - private Stream Stream { get; } private SharedStreamSource StreamSource { get; } private bool KeepOpen { get; } private Nca BaseNca { get; set; } @@ -27,8 +26,7 @@ namespace LibHac { stream.Position = 0; KeepOpen = keepOpen; - Stream = stream; - StreamSource = new SharedStreamSource(stream); + StreamSource = new SharedStreamSource(stream, keepOpen); DecryptHeader(keyset, stream); CryptoType = Math.Max(Header.CryptoType, Header.CryptoType2); @@ -50,7 +48,8 @@ namespace LibHac } else { - throw new MissingKeyException("A required key is missing.", $"{Header.RightsId.ToHexString()}", KeyType.Title); + // todo enable key check when opening a section + // throw new MissingKeyException("A required key is missing.", $"{Header.RightsId.ToHexString()}", KeyType.Title); } } @@ -121,7 +120,7 @@ namespace LibHac Stream rawStream = OpenRawSection(index); NcaSection sect = Sections[index]; - if (raw) return rawStream; + if (raw || rawStream == null) return rawStream; switch (sect.Header.Type) { @@ -393,7 +392,7 @@ namespace LibHac { if (!KeepOpen) { - Stream?.Dispose(); + StreamSource?.Dispose(); } } } diff --git a/LibHac/Streams/SharedStreamSource.cs b/LibHac/Streams/SharedStreamSource.cs index 20301524..471669e6 100644 --- a/LibHac/Streams/SharedStreamSource.cs +++ b/LibHac/Streams/SharedStreamSource.cs @@ -1,15 +1,20 @@ -using System.IO; +using System; +using System.IO; namespace LibHac.Streams { - public class SharedStreamSource + public class SharedStreamSource : IDisposable { private Stream BaseStream { get; } private object Locker { get; } = new object(); + private bool KeepOpen { get; } - public SharedStreamSource(Stream baseStream) + public SharedStreamSource(Stream baseStream) : this(baseStream, true) { } + + public SharedStreamSource(Stream baseStream, bool keepOpen) { BaseStream = baseStream; + KeepOpen = keepOpen; } public SharedStream CreateStream() @@ -59,5 +64,13 @@ namespace LibHac.Streams public bool CanSeek => BaseStream.CanSeek; public bool CanWrite => BaseStream.CanWrite; public long Length => BaseStream.Length; + + public void Dispose() + { + if (KeepOpen) + { + BaseStream?.Dispose(); + } + } } } From 47a3eda0ede11ad5ee78ba0106389abdd43194ee Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Fri, 5 Oct 2018 12:29:52 -0500 Subject: [PATCH 3/4] Fix key printing --- LibHac/Keyset.cs | 7 +++---- hactoolnet/Program.cs | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/LibHac/Keyset.cs b/LibHac/Keyset.cs index 1d709495..10d7174e 100644 --- a/LibHac/Keyset.cs +++ b/LibHac/Keyset.cs @@ -396,6 +396,8 @@ namespace LibHac public static string PrintKeys(Keyset keyset, Dictionary dict) { + if(dict.Count == 0) return string.Empty; + var sb = new StringBuilder(); int maxNameLength = dict.Values.Max(x => x.Name.Length); @@ -429,13 +431,10 @@ namespace LibHac public static string PrintTitleKeys(Keyset keyset) { var sb = new StringBuilder(); - int maxNameLength = keyset.TitleKeys.Values.Max(x => x.Length); foreach (KeyValuePair kv in keyset.TitleKeys) { - byte[] key = kv.Key; - byte[] value = kv.Value; - string line = $"{key.ToHexString().PadRight(maxNameLength)} = {value.ToHexString()}"; + string line = $"{kv.Key.ToHexString()} = {kv.Value.ToHexString()}"; sb.AppendLine(line); } diff --git a/hactoolnet/Program.cs b/hactoolnet/Program.cs index 6cc18d21..3476f4e4 100644 --- a/hactoolnet/Program.cs +++ b/hactoolnet/Program.cs @@ -100,7 +100,7 @@ namespace hactoolnet ctx.Keyset.SetSdSeed(ctx.Options.SdSeed.ToBytes()); } - if (ctx.Options.OutDir != null) + if (ctx.Options.InFileType == FileType.Keygen && ctx.Options.OutDir != null) { string dir = ctx.Options.OutDir; Directory.CreateDirectory(dir); From 209d81187a52ca3d4f2cfc5f9feff9351b947ebe Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Sun, 7 Oct 2018 20:29:27 -0500 Subject: [PATCH 4/4] NCA reading improvements Add on-the-fly Partition FS hash verification Do some cleanup on NCA classes --- LibHac/Bktr.cs | 4 +- LibHac/BktrCryptoStream.cs | 18 +-- LibHac/BktrStructs.cs | 28 ++-- LibHac/IntegrityVerificationStream.cs | 18 ++- LibHac/Nca.cs | 178 ++++++++------------------ LibHac/NcaStructs.cs | 116 ++++++++++++----- LibHac/Pfs.cs | 4 +- LibHac/Savefile/Savefile.cs | 4 +- hactoolnet/ProcessNca.cs | 47 ++++--- hactoolnet/ProcessSwitchFs.cs | 2 +- 10 files changed, 207 insertions(+), 212 deletions(-) diff --git a/LibHac/Bktr.cs b/LibHac/Bktr.cs index f5f60275..edc5aec1 100644 --- a/LibHac/Bktr.cs +++ b/LibHac/Bktr.cs @@ -23,12 +23,12 @@ namespace LibHac Patch = patchRomfs ?? throw new NullReferenceException($"{nameof(patchRomfs)} cannot be null"); Base = baseRomfs ?? throw new NullReferenceException($"{nameof(baseRomfs)} cannot be null"); - IvfcLevelHeader level5 = section.Header.Bktr.IvfcHeader.LevelHeaders[5]; + IvfcLevelHeader level5 = section.Header.IvfcInfo.LevelHeaders[5]; Length = level5.LogicalOffset + level5.HashDataSize; using (var reader = new BinaryReader(patchRomfs, Encoding.Default, true)) { - patchRomfs.Position = section.Header.Bktr.RelocationHeader.Offset; + patchRomfs.Position = section.Header.BktrInfo.RelocationHeader.Offset; RelocationBlock = new RelocationBlock(reader); } diff --git a/LibHac/BktrCryptoStream.cs b/LibHac/BktrCryptoStream.cs index e39a32fc..97153636 100644 --- a/LibHac/BktrCryptoStream.cs +++ b/LibHac/BktrCryptoStream.cs @@ -8,15 +8,15 @@ namespace LibHac { public class BktrCryptoStream : Aes128CtrStream { - public SubsectionBlock SubsectionBlock { get; } - private List SubsectionEntries { get; } = new List(); + public AesSubsectionBlock AesSubsectionBlock { get; } + private List SubsectionEntries { get; } = new List(); private List SubsectionOffsets { get; } - private SubsectionEntry CurrentEntry { get; set; } + private AesSubsectionEntry CurrentEntry { get; set; } - public BktrCryptoStream(Stream baseStream, byte[] key, long offset, long length, long counterOffset, byte[] ctrHi, BktrSuperblock bktr) + public BktrCryptoStream(Stream baseStream, byte[] key, long offset, long length, long counterOffset, byte[] ctrHi, BktrPatchInfo bktr) : base(baseStream, key, offset, length, counterOffset, ctrHi) { - BktrHeader header = bktr.SubsectionHeader; + BktrHeader header = bktr.EncryptionHeader; byte[] subsectionBytes; using (var streamDec = new RandomAccessSectorStream(new Aes128CtrStream(baseStream, key, offset, length, counterOffset, ctrHi))) { @@ -27,16 +27,16 @@ namespace LibHac using (var reader = new BinaryReader(new MemoryStream(subsectionBytes))) { - SubsectionBlock = new SubsectionBlock(reader); + AesSubsectionBlock = new AesSubsectionBlock(reader); } - foreach (SubsectionBucket bucket in SubsectionBlock.Buckets) + foreach (AesSubsectionBucket bucket in AesSubsectionBlock.Buckets) { SubsectionEntries.AddRange(bucket.Entries); } // Add a subsection for the BKTR headers to make things easier - var headerSubsection = new SubsectionEntry + var headerSubsection = new AesSubsectionEntry { Offset = bktr.RelocationHeader.Offset, Counter = (uint)(ctrHi[4] << 24 | ctrHi[5] << 16 | ctrHi[6] << 8 | ctrHi[7]), @@ -92,7 +92,7 @@ namespace LibHac return totalBytesRead; } - private SubsectionEntry GetSubsectionEntry(long offset) + private AesSubsectionEntry GetSubsectionEntry(long offset) { int index = SubsectionOffsets.BinarySearch(offset); if (index < 0) index = ~index - 1; diff --git a/LibHac/BktrStructs.cs b/LibHac/BktrStructs.cs index 94901332..a64872e9 100644 --- a/LibHac/BktrStructs.cs +++ b/LibHac/BktrStructs.cs @@ -75,15 +75,15 @@ namespace LibHac } } - public class SubsectionBlock + public class AesSubsectionBlock { public uint Field0; public int BucketCount; public long Size; public long[] BaseOffsets; - public SubsectionBucket[] Buckets; + public AesSubsectionBucket[] Buckets; - public SubsectionBlock(BinaryReader reader) + public AesSubsectionBlock(BinaryReader reader) { long start = reader.BaseStream.Position; @@ -91,7 +91,7 @@ namespace LibHac BucketCount = reader.ReadInt32(); Size = reader.ReadInt64(); BaseOffsets = new long[BucketCount]; - Buckets = new SubsectionBucket[BucketCount]; + Buckets = new AesSubsectionBucket[BucketCount]; for (int i = 0; i < BucketCount; i++) { @@ -102,47 +102,47 @@ namespace LibHac for (int i = 0; i < BucketCount; i++) { - Buckets[i] = new SubsectionBucket(reader); + Buckets[i] = new AesSubsectionBucket(reader); } } } - public class SubsectionBucket + public class AesSubsectionBucket { public int BucketNum; public int EntryCount; public long VirtualOffsetEnd; - public SubsectionEntry[] Entries; - public SubsectionBucket(BinaryReader reader) + public AesSubsectionEntry[] Entries; + public AesSubsectionBucket(BinaryReader reader) { long start = reader.BaseStream.Position; BucketNum = reader.ReadInt32(); EntryCount = reader.ReadInt32(); VirtualOffsetEnd = reader.ReadInt64(); - Entries = new SubsectionEntry[EntryCount]; + Entries = new AesSubsectionEntry[EntryCount]; for (int i = 0; i < EntryCount; i++) { - Entries[i] = new SubsectionEntry(reader); + Entries[i] = new AesSubsectionEntry(reader); } reader.BaseStream.Position = start + 0x4000; } } - public class SubsectionEntry + public class AesSubsectionEntry { public long Offset; public uint Field8; public uint Counter; - public SubsectionEntry Next; + public AesSubsectionEntry Next; public long OffsetEnd; - public SubsectionEntry() { } + public AesSubsectionEntry() { } - public SubsectionEntry(BinaryReader reader) + public AesSubsectionEntry(BinaryReader reader) { Offset = reader.ReadInt64(); Field8 = reader.ReadUInt32(); diff --git a/LibHac/IntegrityVerificationStream.cs b/LibHac/IntegrityVerificationStream.cs index de6ad6fb..2c835aab 100644 --- a/LibHac/IntegrityVerificationStream.cs +++ b/LibHac/IntegrityVerificationStream.cs @@ -19,7 +19,7 @@ namespace LibHac private readonly SHA256 _hash = SHA256.Create(); public IntegrityVerificationStream(IntegrityVerificationInfo info, Stream hashStream, bool enableIntegrityChecks) - : base(info.Data, 1 << info.BlockSizePower) + : base(info.Data, info.BlockSize) { HashStream = hashStream; EnableIntegrityChecks = enableIntegrityChecks; @@ -61,6 +61,9 @@ namespace LibHac HashStream.Read(_hashBuffer, 0, DigestSize); int bytesRead = base.Read(buffer, 0, count); + int bytesToHash = SectorSize; + + if (bytesRead == 0) return 0; // If a hash is zero the data for the entire block is zero if (Type == IntegrityStreamType.Save && _hashBuffer.IsEmpty()) @@ -73,6 +76,12 @@ namespace LibHac { // Pad out unused portion of block Array.Clear(buffer, bytesRead, SectorSize - bytesRead); + + // Partition FS hashes don't pad out an incomplete block + if (Type == IntegrityStreamType.PartitionFs) + { + bytesToHash = bytesRead; + } } if (!EnableIntegrityChecks) return bytesRead; @@ -84,7 +93,7 @@ namespace LibHac _hash.TransformBlock(Salt, 0, Salt.Length, null, 0); } - _hash.TransformBlock(buffer, 0, SectorSize, null, 0); + _hash.TransformBlock(buffer, 0, bytesToHash, null, 0); _hash.TransformFinalBlock(buffer, 0, 0); byte[] hash = _hash.Hash; @@ -119,7 +128,7 @@ namespace LibHac public class IntegrityVerificationInfo { public Stream Data { get; set; } - public int BlockSizePower { get; set; } + public int BlockSize { get; set; } public byte[] Salt { get; set; } public IntegrityStreamType Type { get; set; } } @@ -127,6 +136,7 @@ namespace LibHac public enum IntegrityStreamType { Save, - RomFs + RomFs, + PartitionFs } } diff --git a/LibHac/Nca.cs b/LibHac/Nca.cs index 213c1bc2..077c1ea7 100644 --- a/LibHac/Nca.cs +++ b/LibHac/Nca.cs @@ -83,24 +83,24 @@ namespace LibHac NcaSection sect = Sections[index]; if (sect == null) throw new ArgumentOutOfRangeException(nameof(index)); - if (sect.SuperblockHashValidity == Validity.Invalid) return null; + //if (sect.SuperblockHashValidity == Validity.Invalid) return null; long offset = sect.Offset; long size = sect.Size; Stream rawStream = StreamSource.CreateStream(offset, size); - switch (sect.Header.CryptType) + switch (sect.Header.EncryptionType) { - case SectionCryptType.None: + case NcaEncryptionType.None: return rawStream; - case SectionCryptType.XTS: + case NcaEncryptionType.XTS: throw new NotImplementedException("NCA sections using XTS are not supported"); - case SectionCryptType.CTR: + case NcaEncryptionType.AesCtr: return new RandomAccessSectorStream(new Aes128CtrStream(rawStream, DecryptedKeys[2], offset, sect.Header.Ctr), false); - case SectionCryptType.BKTR: + case NcaEncryptionType.AesCtrEx: rawStream = new RandomAccessSectorStream( - new BktrCryptoStream(rawStream, DecryptedKeys[2], 0, size, offset, sect.Header.Ctr, sect.Header.Bktr), + new BktrCryptoStream(rawStream, DecryptedKeys[2], 0, size, offset, sect.Header.Ctr, sect.Header.BktrInfo), false); if (BaseNca == null) return rawStream; @@ -125,25 +125,10 @@ namespace LibHac switch (sect.Header.Type) { case SectionType.Pfs0: - PfsSuperblock pfs0Superblock = sect.Pfs0.Superblock; - - return new SubStream(rawStream, pfs0Superblock.Pfs0Offset, pfs0Superblock.Pfs0Size); + return InitIvfcForPartitionfs(sect.Header.Sha256Info, new SharedStreamSource(rawStream), enableIntegrityChecks); case SectionType.Romfs: case SectionType.Bktr: - - var romfsStreamSource = new SharedStreamSource(rawStream); - - IvfcHeader ivfc; - if (sect.Header.Type == SectionType.Romfs) - { - ivfc = sect.Header.Romfs.IvfcHeader; - } - else - { - ivfc = sect.Header.Bktr.IvfcHeader; - } - - return InitIvfcForRomfs(ivfc, romfsStreamSource, enableIntegrityChecks); + return InitIvfcForRomfs(sect.Header.IvfcInfo, new SharedStreamSource(rawStream), enableIntegrityChecks); default: throw new ArgumentOutOfRangeException(); } @@ -158,7 +143,7 @@ namespace LibHac initInfo[0] = new IntegrityVerificationInfo { Data = new MemoryStream(ivfc.MasterHash), - BlockSizePower = 0 + BlockSize = 0 }; for (int i = 1; i < ivfc.NumLevels; i++) @@ -169,7 +154,7 @@ namespace LibHac initInfo[i] = new IntegrityVerificationInfo { Data = data, - BlockSizePower = level.BlockSize, + BlockSize = 1 << level.BlockSizePower, Type = IntegrityStreamType.RomFs }; } @@ -177,6 +162,39 @@ namespace LibHac return new HierarchicalIntegrityVerificationStream(initInfo, enableIntegrityChecks); } + private static Stream InitIvfcForPartitionfs(Sha256Info sb, + SharedStreamSource pfsStreamSource, bool enableIntegrityChecks) + { + SharedStream hashStream = pfsStreamSource.CreateStream(sb.HashTableOffset, sb.HashTableSize); + SharedStream dataStream = pfsStreamSource.CreateStream(sb.DataOffset, sb.DataSize); + + var initInfo = new IntegrityVerificationInfo[3]; + + // Set the master hash + initInfo[0] = new IntegrityVerificationInfo + { + Data = new MemoryStream(sb.MasterHash), + BlockSize = 0, + Type = IntegrityStreamType.PartitionFs + }; + + initInfo[1] = new IntegrityVerificationInfo + { + Data = hashStream, + BlockSize = (int)sb.HashTableSize, + Type = IntegrityStreamType.PartitionFs + }; + + initInfo[2] = new IntegrityVerificationInfo + { + Data = dataStream, + BlockSize = sb.BlockSize, + Type = IntegrityStreamType.PartitionFs + }; + + return new HierarchicalIntegrityVerificationStream(initInfo, enableIntegrityChecks); + } + public void SetBaseNca(Nca baseNca) => BaseNca = baseNca; private void DecryptHeader(Keyset keyset, Stream stream) @@ -216,46 +234,12 @@ namespace LibHac sect.Header = header; sect.Type = header.Type; - if (sect.Type == SectionType.Pfs0) - { - sect.Pfs0 = new Pfs0Section(); - sect.Pfs0.Superblock = header.Pfs; - } - else if (sect.Type == SectionType.Romfs) - { - ProcessIvfcSection(sect); - } - return sect; } - private void ProcessIvfcSection(NcaSection sect) - { - sect.Romfs = new RomfsSection(); - sect.Romfs.Superblock = sect.Header.Romfs; - IvfcLevelHeader[] headers = sect.Romfs.Superblock.IvfcHeader.LevelHeaders; - - for (int i = 0; i < Romfs.IvfcMaxLevel; i++) - { - var level = new IvfcLevel(); - sect.Romfs.IvfcLevels[i] = level; - IvfcLevelHeader header = headers[i]; - level.DataOffset = header.LogicalOffset; - level.DataSize = header.HashDataSize; - level.HashBlockSize = 1 << header.BlockSize; - level.HashBlockCount = Util.DivideByRoundUp(level.DataSize, level.HashBlockSize); - level.HashSize = level.HashBlockCount * 0x20; - - if (i != 0) - { - level.HashOffset = sect.Romfs.IvfcLevels[i - 1].DataOffset; - } - } - } - private void CheckBktrKey(NcaSection sect) { - long offset = sect.Header.Bktr.SubsectionHeader.Offset; + long offset = sect.Header.BktrInfo.EncryptionHeader.Offset; using (var streamDec = new RandomAccessSectorStream(new Aes128CtrStream(GetStream(), DecryptedKeys[2], sect.Offset, sect.Size, sect.Offset, sect.Header.Ctr))) { var reader = new BinaryReader(streamDec); @@ -283,16 +267,16 @@ namespace LibHac case SectionType.Invalid: break; case SectionType.Pfs0: - PfsSuperblock pfs0 = sect.Header.Pfs; + Sha256Info pfs0 = sect.Header.Sha256Info; expected = pfs0.MasterHash; offset = pfs0.HashTableOffset; size = pfs0.HashTableSize; break; case SectionType.Romfs: - IvfcHeader ivfc = sect.Header.Romfs.IvfcHeader; + IvfcHeader ivfc = sect.Header.IvfcInfo; expected = ivfc.MasterHash; offset = ivfc.LevelHeaders[0].LogicalOffset; - size = 1 << ivfc.LevelHeaders[0].BlockSize; + size = 1 << ivfc.LevelHeaders[0].BlockSizePower; break; case SectionType.Bktr: CheckBktrKey(sect); @@ -308,7 +292,7 @@ namespace LibHac stream.Read(hashTable, 0, hashTable.Length); sect.SuperblockHashValidity = Crypto.CheckMemoryHashTable(hashTable, expected, 0, hashTable.Length); - if (sect.Type == SectionType.Romfs) sect.Romfs.IvfcLevels[0].HashValidity = sect.SuperblockHashValidity; + // todo if (sect.Type == SectionType.Romfs) sect.Romfs.IvfcLevels[0].HashValidity = sect.SuperblockHashValidity; } public void VerifySection(int index, IProgressReport logger = null) @@ -323,71 +307,16 @@ namespace LibHac case SectionType.Invalid: break; case SectionType.Pfs0: - VerifyPfs0(stream, sect.Pfs0, logger); + // todo VerifyPfs0(stream, sect.Pfs0, logger); break; case SectionType.Romfs: - VerifyIvfc(stream, sect.Romfs.IvfcLevels, logger); + // todo VerifyIvfc(stream, sect.Romfs.IvfcLevels, logger); break; case SectionType.Bktr: break; } } - - private void VerifyPfs0(Stream section, Pfs0Section pfs0, IProgressReport logger = null) - { - PfsSuperblock sb = pfs0.Superblock; - var table = new byte[sb.HashTableSize]; - section.Position = sb.HashTableOffset; - section.Read(table, 0, table.Length); - - pfs0.Validity = VerifyHashTable(section, table, sb.Pfs0Offset, sb.Pfs0Size, sb.BlockSize, false, logger); - } - - private void VerifyIvfc(Stream section, IvfcLevel[] levels, IProgressReport logger = null) - { - for (int i = 1; i < levels.Length; i++) - { - logger?.LogMessage($" Verifying IVFC Level {i}..."); - IvfcLevel level = levels[i]; - var table = new byte[level.HashSize]; - section.Position = level.HashOffset; - section.Read(table, 0, table.Length); - level.HashValidity = VerifyHashTable(section, table, level.DataOffset, level.DataSize, level.HashBlockSize, true, logger); - } - } - - private Validity VerifyHashTable(Stream section, byte[] hashTable, long dataOffset, long dataLen, long blockSize, bool isFinalBlockFull, IProgressReport logger = null) - { - const int hashSize = 0x20; - var currentBlock = new byte[blockSize]; - var expectedHash = new byte[hashSize]; - long blockCount = Util.DivideByRoundUp(dataLen, blockSize); - int curBlockSize = (int)blockSize; - section.Position = dataOffset; - logger?.SetTotal(blockCount); - - for (long i = 0; i < blockCount; i++) - { - var remaining = (dataLen - i * blockSize); - if (remaining < blockSize) - { - Array.Clear(currentBlock, 0, currentBlock.Length); - if (!isFinalBlockFull) curBlockSize = (int)remaining; - } - Array.Copy(hashTable, i * hashSize, expectedHash, 0, hashSize); - section.Read(currentBlock, 0, curBlockSize); - - if (Crypto.CheckMemoryHashTable(currentBlock, expectedHash, 0, curBlockSize) == Validity.Invalid) - { - return Validity.Invalid; - } - - logger?.ReportAdd(1); - } - - return Validity.Valid; - } - + public void Dispose() { if (!KeepOpen) @@ -399,7 +328,6 @@ namespace LibHac public class NcaSection { - public Stream Stream { get; set; } public NcaFsHeader Header { get; set; } public SectionType Type { get; set; } public int SectionNum { get; set; } @@ -407,8 +335,6 @@ namespace LibHac public long Size { get; set; } public Validity SuperblockHashValidity { get; set; } - public Pfs0Section Pfs0 { get; set; } - public RomfsSection Romfs { get; set; } public bool IsExefs { get; internal set; } } diff --git a/LibHac/NcaStructs.cs b/LibHac/NcaStructs.cs index 16cf5763..5ad9ef55 100644 --- a/LibHac/NcaStructs.cs +++ b/LibHac/NcaStructs.cs @@ -84,48 +84,66 @@ namespace LibHac public class NcaFsHeader { - public byte Field0; - public byte Field1; - public SectionPartitionType PartitionType; - public SectionFsType FsType; - public SectionCryptType CryptType; + public short Version; + public NcaFormatType FormatType; + public NcaHashType HashType; + public NcaEncryptionType EncryptionType; public SectionType Type; - public PfsSuperblock Pfs; - public RomfsSuperblock Romfs; - public BktrSuperblock Bktr; + public IvfcHeader IvfcInfo; + public Sha256Info Sha256Info; + public BktrPatchInfo BktrInfo; + public byte[] Ctr; public NcaFsHeader(BinaryReader reader) { long start = reader.BaseStream.Position; - Field0 = reader.ReadByte(); - Field1 = reader.ReadByte(); - PartitionType = (SectionPartitionType)reader.ReadByte(); - FsType = (SectionFsType)reader.ReadByte(); - CryptType = (SectionCryptType)reader.ReadByte(); + Version = reader.ReadInt16(); + FormatType = (NcaFormatType)reader.ReadByte(); + HashType = (NcaHashType)reader.ReadByte(); + EncryptionType = (NcaEncryptionType)reader.ReadByte(); reader.BaseStream.Position += 3; - if (PartitionType == SectionPartitionType.Pfs0 && FsType == SectionFsType.Pfs0) + switch (HashType) + { + case NcaHashType.Sha256: + Sha256Info = new Sha256Info(reader); + break; + case NcaHashType.Ivfc: + IvfcInfo = new IvfcHeader(reader); + break; + } + + if (EncryptionType == NcaEncryptionType.AesCtrEx) + { + BktrInfo = new BktrPatchInfo(); + + reader.BaseStream.Position = start + 0x100; + + BktrInfo.RelocationHeader = new BktrHeader(reader); + BktrInfo.EncryptionHeader = new BktrHeader(reader); + } + + if (FormatType == NcaFormatType.Pfs0) { Type = SectionType.Pfs0; - Pfs = new PfsSuperblock(reader); } - else if (PartitionType == SectionPartitionType.Romfs && FsType == SectionFsType.Romfs) + else if (FormatType == NcaFormatType.Romfs) { - if (CryptType == SectionCryptType.BKTR) + if (EncryptionType == NcaEncryptionType.AesCtrEx) { Type = SectionType.Bktr; - Bktr = new BktrSuperblock(reader); } else { Type = SectionType.Romfs; - Romfs = new RomfsSuperblock(reader); } } + reader.BaseStream.Position = start + 0x140; Ctr = reader.ReadBytes(8).Reverse().ToArray(); + reader.BaseStream.Position = start + 512; } } @@ -156,10 +174,16 @@ namespace LibHac } } + public class BktrPatchInfo + { + public BktrHeader RelocationHeader; + public BktrHeader EncryptionHeader; + } + public class IvfcHeader { public string Magic; - public uint Id; + public int Version; public uint MasterHashSize; public uint NumLevels; public IvfcLevelHeader[] LevelHeaders = new IvfcLevelHeader[6]; @@ -169,7 +193,8 @@ namespace LibHac public IvfcHeader(BinaryReader reader) { Magic = reader.ReadAscii(4); - Id = reader.ReadUInt32(); + Version = reader.ReadInt16(); + reader.BaseStream.Position += 2; MasterHashSize = reader.ReadUInt32(); NumLevels = reader.ReadUInt32(); @@ -187,24 +212,46 @@ namespace LibHac { public long LogicalOffset; public long HashDataSize; - public int BlockSize; + public int BlockSizePower; public uint Reserved; public IvfcLevelHeader(BinaryReader reader) { LogicalOffset = reader.ReadInt64(); HashDataSize = reader.ReadInt64(); - BlockSize = reader.ReadInt32(); + BlockSizePower = reader.ReadInt32(); Reserved = reader.ReadUInt32(); } } + public class Sha256Info + { + public byte[] MasterHash; + public int BlockSize; // In bytes + public uint Always2; + public long HashTableOffset; + public long HashTableSize; + public long DataOffset; + public long DataSize; + + public Sha256Info(BinaryReader reader) + { + MasterHash = reader.ReadBytes(0x20); + BlockSize = reader.ReadInt32(); + Always2 = reader.ReadUInt32(); + HashTableOffset = reader.ReadInt64(); + HashTableSize = reader.ReadInt64(); + DataOffset = reader.ReadInt64(); + DataSize = reader.ReadInt64(); + } + } + public class BktrHeader { public long Offset; public long Size; public uint Magic; - public uint Field14; + public uint Version; public uint NumEntries; public uint Field1C; @@ -213,7 +260,7 @@ namespace LibHac Offset = reader.ReadInt64(); Size = reader.ReadInt64(); Magic = reader.ReadUInt32(); - Field14 = reader.ReadUInt32(); + Version = reader.ReadUInt32(); NumEntries = reader.ReadUInt32(); Field1C = reader.ReadUInt32(); } @@ -281,21 +328,24 @@ namespace LibHac Gamecard } - public enum SectionCryptType + public enum NcaEncryptionType { - None = 1, + Auto, + None, XTS, - CTR, - BKTR + AesCtr, + AesCtrEx } - public enum SectionFsType + public enum NcaHashType { - Pfs0 = 2, - Romfs + Auto, + None, + Sha256, + Ivfc } - public enum SectionPartitionType + public enum NcaFormatType { Romfs, Pfs0 diff --git a/LibHac/Pfs.cs b/LibHac/Pfs.cs index 490b3906..a3d8ab1b 100644 --- a/LibHac/Pfs.cs +++ b/LibHac/Pfs.cs @@ -71,7 +71,7 @@ namespace LibHac public class PfsSuperblock { public byte[] MasterHash; /* SHA-256 hash of the hash table. */ - public uint BlockSize; /* In bytes. */ + public int BlockSize; /* In bytes. */ public uint Always2; public long HashTableOffset; /* Normally zero. */ public long HashTableSize; @@ -81,7 +81,7 @@ namespace LibHac public PfsSuperblock(BinaryReader reader) { MasterHash = reader.ReadBytes(0x20); - BlockSize = reader.ReadUInt32(); + BlockSize = reader.ReadInt32(); Always2 = reader.ReadUInt32(); HashTableOffset = reader.ReadInt64(); HashTableSize = reader.ReadInt64(); diff --git a/LibHac/Savefile/Savefile.cs b/LibHac/Savefile/Savefile.cs index d87dc2f8..015e47ac 100644 --- a/LibHac/Savefile/Savefile.cs +++ b/LibHac/Savefile/Savefile.cs @@ -130,7 +130,7 @@ namespace LibHac.Savefile initInfo[0] = new IntegrityVerificationInfo { Data = new MemoryStream(Header.MasterHashA), - BlockSizePower = 0, + BlockSize = 0, Type = IntegrityStreamType.Save }; @@ -145,7 +145,7 @@ namespace LibHac.Savefile initInfo[i] = new IntegrityVerificationInfo { Data = data, - BlockSizePower = level.BlockSize, + BlockSize = 1 << level.BlockSizePower, Salt = new HMACSHA256(Encoding.ASCII.GetBytes(SaltSources[i - 1])).ComputeHash(ivfc.SaltSource), Type = IntegrityStreamType.Save }; diff --git a/hactoolnet/ProcessNca.cs b/hactoolnet/ProcessNca.cs index d5d6e840..464852f8 100644 --- a/hactoolnet/ProcessNca.cs +++ b/hactoolnet/ProcessNca.cs @@ -178,34 +178,43 @@ namespace hactoolnet void PrintPfs0(NcaSection sect) { - PfsSuperblock sBlock = sect.Pfs0.Superblock; - PrintItem(sb, colLen, $" Superblock Hash{sect.SuperblockHashValidity.GetValidityString()}:", sBlock.MasterHash); - sb.AppendLine($" Hash Table{sect.Pfs0.Validity.GetValidityString()}:"); + Sha256Info hashInfo = sect.Header.Sha256Info; - PrintItem(sb, colLen, " Offset:", $"0x{sBlock.HashTableOffset:x12}"); - PrintItem(sb, colLen, " Size:", $"0x{sBlock.HashTableSize:x12}"); - PrintItem(sb, colLen, " Block Size:", $"0x{sBlock.BlockSize:x}"); - PrintItem(sb, colLen, " PFS0 Offset:", $"0x{sBlock.Pfs0Offset:x12}"); - PrintItem(sb, colLen, " PFS0 Size:", $"0x{sBlock.Pfs0Size:x12}"); + PrintItem(sb, colLen, $" Superblock Hash{sect.SuperblockHashValidity.GetValidityString()}:", hashInfo.MasterHash); + // todo sb.AppendLine($" Hash Table{sect.Pfs0.Validity.GetValidityString()}:"); + sb.AppendLine($" Hash Table:"); + + PrintItem(sb, colLen, " Offset:", $"0x{hashInfo.HashTableOffset:x12}"); + PrintItem(sb, colLen, " Size:", $"0x{hashInfo.HashTableSize:x12}"); + PrintItem(sb, colLen, " Block Size:", $"0x{hashInfo.BlockSize:x}"); + PrintItem(sb, colLen, " PFS0 Offset:", $"0x{hashInfo.DataOffset:x12}"); + PrintItem(sb, colLen, " PFS0 Size:", $"0x{hashInfo.DataSize:x12}"); } void PrintRomfs(NcaSection sect) { - RomfsSuperblock sBlock = sect.Romfs.Superblock; - IvfcLevel[] levels = sect.Romfs.IvfcLevels; + IvfcHeader ivfcInfo = sect.Header.IvfcInfo; - PrintItem(sb, colLen, $" Superblock Hash{sect.SuperblockHashValidity.GetValidityString()}:", sBlock.IvfcHeader.MasterHash); - PrintItem(sb, colLen, " Magic:", sBlock.IvfcHeader.Magic); - PrintItem(sb, colLen, " ID:", $"{sBlock.IvfcHeader.Id:x8}"); + PrintItem(sb, colLen, $" Superblock Hash{sect.SuperblockHashValidity.GetValidityString()}:", ivfcInfo.MasterHash); + PrintItem(sb, colLen, " Magic:", ivfcInfo.Magic); + PrintItem(sb, colLen, " Version:", $"{ivfcInfo.Version:x8}"); for (int i = 0; i < Romfs.IvfcMaxLevel; i++) { - IvfcLevel level = levels[i]; - sb.AppendLine($" Level {i}{level.HashValidity.GetValidityString()}:"); - PrintItem(sb, colLen, " Data Offset:", $"0x{level.DataOffset:x12}"); - PrintItem(sb, colLen, " Data Size:", $"0x{level.DataSize:x12}"); - PrintItem(sb, colLen, " Hash Offset:", $"0x{level.HashOffset:x12}"); - PrintItem(sb, colLen, " Hash BlockSize:", $"0x{level.HashBlockSize:x8}"); + IvfcLevelHeader level = ivfcInfo.LevelHeaders[i]; + long hashOffset = 0; + + if (i != 0) + { + hashOffset = ivfcInfo.LevelHeaders[i - 1].LogicalOffset; + } + + // todo sb.AppendLine($" Level {i}{level.HashValidity.GetValidityString()}:"); + sb.AppendLine($" Level {i}:"); + PrintItem(sb, colLen, " Data Offset:", $"0x{level.LogicalOffset:x12}"); + PrintItem(sb, colLen, " Data Size:", $"0x{level.HashDataSize:x12}"); + PrintItem(sb, colLen, " Hash Offset:", $"0x{hashOffset:x12}"); + PrintItem(sb, colLen, " Hash BlockSize:", $"0x{1 << level.BlockSizePower:x8}"); } } } diff --git a/hactoolnet/ProcessSwitchFs.cs b/hactoolnet/ProcessSwitchFs.cs index 816e9b15..b23bc75b 100644 --- a/hactoolnet/ProcessSwitchFs.cs +++ b/hactoolnet/ProcessSwitchFs.cs @@ -170,7 +170,7 @@ namespace hactoolnet foreach (NcaSection sect in nca.Sections.Where(x => x != null)) { - Console.WriteLine($" {sect.SectionNum} {sect.Type} {sect.Header.CryptType} {sect.SuperblockHashValidity}"); + Console.WriteLine($" {sect.SectionNum} {sect.Type} {sect.Header.EncryptionType} {sect.SuperblockHashValidity}"); } }