From a64cbeca5b05010c90c89aba56c5ea6cbeca3dc9 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 8 Oct 2018 21:04:39 -0500 Subject: [PATCH] Continue NCA improvements - Read only the NCA header when first opening an NCA. This allows for reading of partial NCAs and slightly improves performance when opening an NCA. - Add a separate NCA method to validate master hashes now that it's not automatically done when opening the NCA. - Fix possible hang when reading BKTR sections. - When keys required to decrypt an NCA are missing, throw an exception with information about the missing keys. - Add more sanity checks when reading an NCA. Hactoolnet: Don't hard-crash when hitting an unhandled exception --- LibHac/BktrCryptoStream.cs | 2 + LibHac/Keyset.cs | 6 +- LibHac/Nca.cs | 224 +++++++++++++++++++++++++--------- LibHac/NcaStructs.cs | 10 +- LibHac/Util.cs | 8 +- hactoolnet/ProcessNca.cs | 5 +- hactoolnet/ProcessSwitchFs.cs | 2 +- hactoolnet/Program.cs | 108 +++++++++------- 8 files changed, 257 insertions(+), 108 deletions(-) diff --git a/LibHac/BktrCryptoStream.cs b/LibHac/BktrCryptoStream.cs index 97153636..7e415c37 100644 --- a/LibHac/BktrCryptoStream.cs +++ b/LibHac/BktrCryptoStream.cs @@ -78,6 +78,8 @@ namespace LibHac int bytesToRead = (int)Math.Min(CurrentEntry.OffsetEnd - Position, count); int bytesRead = base.Read(buffer, outPos, bytesToRead); + if (bytesRead == 0) break; + outPos += bytesRead; totalBytesRead += bytesRead; count -= bytesRead; diff --git a/LibHac/Keyset.cs b/LibHac/Keyset.cs index 10d7174e..9f25203f 100644 --- a/LibHac/Keyset.cs +++ b/LibHac/Keyset.cs @@ -246,7 +246,7 @@ namespace LibHac Crypto.DecryptEcb(headerKek, HeaderKeySource, HeaderKey, 0x20); } - private void DeriveSdCardKeys() + public void DeriveSdCardKeys() { var sdKek = new byte[0x10]; Crypto.GenerateKek(MasterKeys[0], SdCardKekSource, sdKek, AesKekGenerationSource, AesKeyGenerationSource); @@ -264,6 +264,8 @@ namespace LibHac Crypto.DecryptEcb(sdKek, SdCardKeySourcesSpecific[k], SdCardKeys[k], 0x20); } } + + internal static readonly string[] KakNames = {"application", "ocean", "system"}; } public static class ExternalKeys @@ -396,7 +398,7 @@ namespace LibHac public static string PrintKeys(Keyset keyset, Dictionary dict) { - if(dict.Count == 0) return string.Empty; + if (dict.Count == 0) return string.Empty; var sb = new StringBuilder(); int maxNameLength = dict.Values.Max(x => x.Name.Length); diff --git a/LibHac/Nca.cs b/LibHac/Nca.cs index 077c1ea7..ff2c00de 100644 --- a/LibHac/Nca.cs +++ b/LibHac/Nca.cs @@ -1,6 +1,5 @@ using System; using System.IO; -using System.Linq; using LibHac.Streams; using LibHac.XTSSharp; @@ -20,6 +19,9 @@ namespace LibHac private bool KeepOpen { get; } private Nca BaseNca { get; set; } + private bool IsMissingTitleKey { get; set; } + private string MissingKeyName { get; set; } + public NcaSection[] Sections { get; } = new NcaSection[4]; public Nca(Keyset keyset, Stream stream, bool keepOpen) @@ -38,19 +40,20 @@ namespace LibHac { DecryptKeyArea(keyset); } + else if (keyset.TitleKeys.TryGetValue(Header.RightsId, out byte[] titleKey)) + { + if (keyset.Titlekeks[CryptoType].IsEmpty()) + { + MissingKeyName = $"titlekek_{CryptoType:x2}"; + } + + TitleKey = titleKey; + Crypto.DecryptEcb(keyset.Titlekeks[CryptoType], titleKey, TitleKeyDec, 0x10); + DecryptedKeys[2] = TitleKeyDec; + } else { - if (keyset.TitleKeys.TryGetValue(Header.RightsId, out byte[] titleKey)) - { - TitleKey = titleKey; - Crypto.DecryptEcb(keyset.Titlekeks[CryptoType], titleKey, TitleKeyDec, 0x10); - DecryptedKeys[2] = TitleKeyDec; - } - else - { - // todo enable key check when opening a section - // throw new MissingKeyException("A required key is missing.", $"{Header.RightsId.ToHexString()}", KeyType.Title); - } + IsMissingTitleKey = true; } for (int i = 0; i < 4; i++) @@ -58,36 +61,69 @@ namespace LibHac NcaSection section = ParseSection(i); if (section == null) continue; Sections[i] = section; - ValidateSuperblockHash(i); + ValidateMasterHash(i); } - foreach (NcaSection pfsSection in Sections.Where(x => x != null && x.Type == SectionType.Pfs0)) - { - Stream sectionStream = OpenSection(pfsSection.SectionNum, false, false); - if (sectionStream == null) continue; + //foreach (NcaSection pfsSection in Sections.Where(x => x != null && x.Type == SectionType.Pfs0)) + //{ + // Stream sectionStream = OpenSection(pfsSection.SectionNum, false, false); + // if (sectionStream == null) continue; - var pfs = new Pfs(sectionStream); - if (!pfs.FileExists("main.npdm")) continue; + // var pfs = new Pfs(sectionStream); + // if (!pfs.FileExists("main.npdm")) continue; - pfsSection.IsExefs = true; - } + // pfsSection.IsExefs = true; + //} } + /// + /// Opens a of the underlying NCA file. + /// + /// A that provides access to the entire raw NCA file. public Stream GetStream() { return StreamSource.CreateStream(); } + public bool CanOpenSection(int index) + { + if (index < 0 || index > 3) throw new ArgumentOutOfRangeException(nameof(index)); + + NcaSection sect = Sections[index]; + if (sect == null) return false; + + return sect.Header.EncryptionType == NcaEncryptionType.None || !IsMissingTitleKey && string.IsNullOrWhiteSpace(MissingKeyName); + } + private Stream OpenRawSection(int index) { - NcaSection sect = Sections[index]; - if (sect == null) throw new ArgumentOutOfRangeException(nameof(index)); + if (index < 0 || index > 3) throw new ArgumentOutOfRangeException(nameof(index)); - //if (sect.SuperblockHashValidity == Validity.Invalid) return null; + NcaSection sect = Sections[index]; + if (sect == null) return null; + + if (sect.Header.EncryptionType != NcaEncryptionType.None) + { + if (IsMissingTitleKey) + { + throw new MissingKeyException("Unable to decrypt NCA section.", Header.RightsId.ToHexString(), KeyType.Title); + } + + if (!string.IsNullOrWhiteSpace(MissingKeyName)) + { + throw new MissingKeyException("Unable to decrypt NCA section.", MissingKeyName, KeyType.Common); + } + } long offset = sect.Offset; long size = sect.Size; + if (!Util.IsSubRange(offset, size, StreamSource.Length)) + { + throw new InvalidDataException( + $"Section offset (0x{offset:x}) and length (0x{size:x}) fall outside the total NCA length (0x{StreamSource.Length:x})."); + } + Stream rawStream = StreamSource.CreateStream(offset, size); switch (sect.Header.EncryptionType) @@ -104,10 +140,9 @@ namespace LibHac false); if (BaseNca == null) return rawStream; - NcaSection baseSect = BaseNca.Sections.FirstOrDefault(x => x.Type == SectionType.Romfs); - if (baseSect == null) throw new InvalidDataException("Base NCA has no RomFS section"); + Stream baseStream = BaseNca.OpenSection(ProgramPartitionType.Data, true, false); + if (baseStream == null) throw new InvalidDataException("Base NCA has no RomFS section"); - Stream baseStream = BaseNca.OpenSection(baseSect.SectionNum, true, false); return new Bktr(rawStream, baseStream, sect); default: @@ -115,25 +150,50 @@ namespace LibHac } } + /// + /// Opens one of the sections in the current . + /// + /// The index of the NCA section to open. Valid indexes are 0-3. + /// to open the raw section with hash metadata. + /// to enable data integrity checks when reading the section. + /// Only applies if is . + /// A that provides access to the specified section. if the section does not exist. + /// The specified is outside the valid range. public Stream OpenSection(int index, bool raw, bool enableIntegrityChecks) { Stream rawStream = OpenRawSection(index); NcaSection sect = Sections[index]; + NcaFsHeader header = sect.Header; if (raw || rawStream == null) return rawStream; - switch (sect.Header.Type) + // If it's a patch section without a base, return the raw section because it has no hash data + if (header.EncryptionType == NcaEncryptionType.AesCtrEx && BaseNca == null) return rawStream; + + switch (header.HashType) { - case SectionType.Pfs0: - return InitIvfcForPartitionfs(sect.Header.Sha256Info, new SharedStreamSource(rawStream), enableIntegrityChecks); - case SectionType.Romfs: - case SectionType.Bktr: - return InitIvfcForRomfs(sect.Header.IvfcInfo, new SharedStreamSource(rawStream), enableIntegrityChecks); + case NcaHashType.Sha256: + return InitIvfcForPartitionfs(header.Sha256Info, new SharedStreamSource(rawStream), enableIntegrityChecks); + case NcaHashType.Ivfc: + return InitIvfcForRomfs(header.IvfcInfo, new SharedStreamSource(rawStream), enableIntegrityChecks); + default: throw new ArgumentOutOfRangeException(); } } + /// + /// Opens one of the sections in the current . For use with type NCAs. + /// + /// The type of section to open. + /// to open the raw section with hash metadata. + /// to enable data integrity checks when reading the section. + /// Only applies if is . + /// A that provides access to the specified section. if the section does not exist. + /// The specified is outside the valid range. + public Stream OpenSection(ProgramPartitionType type, bool raw, bool enableIntegrityChecks) => + OpenSection((int)type, raw, enableIntegrityChecks); + private static HierarchicalIntegrityVerificationStream InitIvfcForRomfs(IvfcHeader ivfc, SharedStreamSource romfsStreamSource, bool enableIntegrityChecks) { @@ -195,10 +255,31 @@ namespace LibHac return new HierarchicalIntegrityVerificationStream(initInfo, enableIntegrityChecks); } + /// + /// Sets a base to use when reading patches. + /// + /// The base public void SetBaseNca(Nca baseNca) => BaseNca = baseNca; + /// + /// Validates the master hash and store the result in for each . + /// + public void ValidateMasterHashes() + { + for (int i = 0; i < 4; i++) + { + if (Sections[i] == null) continue; + ValidateMasterHash(i); + } + } + private void DecryptHeader(Keyset keyset, Stream stream) { + if (keyset.HeaderKey.IsEmpty()) + { + throw new MissingKeyException("Unable to decrypt NCA header.", "header_key", KeyType.Common); + } + var headerBytes = new byte[0xC00]; Xts xts = XtsAes128.Create(keyset.HeaderKey); using (var headerDec = new RandomAccessSectorStream(new XtsSectorStream(stream, xts, 0x200))) @@ -213,6 +294,12 @@ namespace LibHac private void DecryptKeyArea(Keyset keyset) { + if (keyset.KeyAreaKeys[CryptoType][Header.KaekInd].IsEmpty()) + { + MissingKeyName = $"key_area_key_{Keyset.KakNames[Header.KaekInd]}_{CryptoType:x2}"; + return; + } + for (int i = 0; i < 4; i++) { Crypto.DecryptEcb(keyset.KeyAreaKeys[CryptoType][Header.KaekInd], Header.EncryptedKeys[i], @@ -239,6 +326,10 @@ namespace LibHac private void CheckBktrKey(NcaSection sect) { + // The encryption subsection table in the bktr partition contains the length of the entire partition. + // The encryption table is always located immediately following the partition data + // Decrypt this value and compare it to the encryption table offset found in the NCA header + long offset = sect.Header.BktrInfo.EncryptionHeader.Offset; using (var streamDec = new RandomAccessSectorStream(new Aes128CtrStream(GetStream(), DecryptedKeys[2], sect.Offset, sect.Size, sect.Offset, sect.Header.Ctr))) { @@ -248,17 +339,23 @@ namespace LibHac if (size != offset) { - sect.SuperblockHashValidity = Validity.Invalid; + sect.MasterHashValidity = Validity.Invalid; } } } - private void ValidateSuperblockHash(int index) + private void ValidateMasterHash(int index) { if (Sections[index] == null) throw new ArgumentOutOfRangeException(nameof(index)); NcaSection sect = Sections[index]; - byte[] expected = null; + if (!CanOpenSection(index)) + { + sect.MasterHashValidity = Validity.MissingKey; + return; + } + + byte[] expected = sect.GetMasterHash(); long offset = 0; long size = 0; @@ -267,16 +364,12 @@ namespace LibHac case SectionType.Invalid: break; case SectionType.Pfs0: - Sha256Info pfs0 = sect.Header.Sha256Info; - expected = pfs0.MasterHash; - offset = pfs0.HashTableOffset; - size = pfs0.HashTableSize; + offset = sect.Header.Sha256Info.HashTableOffset; + size = sect.Header.Sha256Info.HashTableSize; break; case SectionType.Romfs: - IvfcHeader ivfc = sect.Header.IvfcInfo; - expected = ivfc.MasterHash; - offset = ivfc.LevelHeaders[0].LogicalOffset; - size = 1 << ivfc.LevelHeaders[0].BlockSizePower; + offset = sect.Header.IvfcInfo.LevelHeaders[0].LogicalOffset; + size = 1 << sect.Header.IvfcInfo.LevelHeaders[0].BlockSizePower; break; case SectionType.Bktr: CheckBktrKey(sect); @@ -284,39 +377,33 @@ namespace LibHac } Stream stream = OpenSection(index, true, false); - if (stream == null) return; - if (expected == null) return; var hashTable = new byte[size]; stream.Position = offset; stream.Read(hashTable, 0, hashTable.Length); - sect.SuperblockHashValidity = Crypto.CheckMemoryHashTable(hashTable, expected, 0, hashTable.Length); - // todo if (sect.Type == SectionType.Romfs) sect.Romfs.IvfcLevels[0].HashValidity = sect.SuperblockHashValidity; + sect.MasterHashValidity = Crypto.CheckMemoryHashTable(hashTable, expected, 0, hashTable.Length); } public void VerifySection(int index, IProgressReport logger = null) { if (Sections[index] == null) throw new ArgumentOutOfRangeException(nameof(index)); + NcaSection sect = Sections[index]; - Stream stream = OpenSection(index, true, false); + Stream stream = OpenSection(index, false, true); logger?.LogMessage($"Verifying section {index}..."); - switch (sect.Type) + switch (sect.Header.HashType) { - case SectionType.Invalid: + case NcaHashType.Sha256: break; - case SectionType.Pfs0: - // todo VerifyPfs0(stream, sect.Pfs0, logger); - break; - case SectionType.Romfs: - // todo VerifyIvfc(stream, sect.Romfs.IvfcLevels, logger); - break; - case SectionType.Bktr: + case NcaHashType.Ivfc: break; + default: + throw new ArgumentOutOfRangeException(); } } - + public void Dispose() { if (!KeepOpen) @@ -333,9 +420,26 @@ namespace LibHac public int SectionNum { get; set; } public long Offset { get; set; } public long Size { get; set; } - public Validity SuperblockHashValidity { get; set; } + public Validity MasterHashValidity { get; set; } public bool IsExefs { get; internal set; } + + public byte[] GetMasterHash() + { + var hash = new byte[Crypto.Sha256DigestSize]; + + switch (Header.HashType) + { + case NcaHashType.Sha256: + Array.Copy(Header.Sha256Info.MasterHash, hash, Crypto.Sha256DigestSize); + break; + case NcaHashType.Ivfc: + Array.Copy(Header.IvfcInfo.MasterHash, hash, Crypto.Sha256DigestSize); + break; + } + + return hash; + } } public static class NcaExtensions diff --git a/LibHac/NcaStructs.cs b/LibHac/NcaStructs.cs index 5ad9ef55..7d2a7df3 100644 --- a/LibHac/NcaStructs.cs +++ b/LibHac/NcaStructs.cs @@ -312,6 +312,13 @@ namespace LibHac public IvfcLevel[] IvfcLevels { get; set; } = new IvfcLevel[Romfs.IvfcMaxLevel]; } + public enum ProgramPartitionType + { + Code, + Data, + Logo + }; + public enum ContentType { Program, @@ -363,6 +370,7 @@ namespace LibHac { Unchecked, Invalid, - Valid + Valid, + MissingKey } } diff --git a/LibHac/Util.cs b/LibHac/Util.cs index 8eebbae7..2e7c5e2a 100644 --- a/LibHac/Util.cs +++ b/LibHac/Util.cs @@ -399,10 +399,16 @@ namespace LibHac case 2: return "3.0.1-3.0.2"; case 3: return "4.0.0-4.1.0"; case 4: return "5.0.0-5.1.0"; - case 5: return "6.0.0"; + case 5: return "6.0.0-6.0.1"; default: return "Unknown"; } } + + public static bool IsSubRange(long startIndex, long subLength, long length) + { + bool isOutOfRange = startIndex < 0 || startIndex > length || subLength < 0 || startIndex > length - subLength; + return !isOutOfRange; + } } public class ByteArray128BitComparer : EqualityComparer diff --git a/hactoolnet/ProcessNca.cs b/hactoolnet/ProcessNca.cs index 464852f8..3d5948e1 100644 --- a/hactoolnet/ProcessNca.cs +++ b/hactoolnet/ProcessNca.cs @@ -13,6 +13,7 @@ namespace hactoolnet using (var file = new FileStream(ctx.Options.InFile, FileMode.Open, FileAccess.Read)) { var nca = new Nca(ctx.Keyset, file, false); + nca.ValidateMasterHashes(); if (ctx.Options.BaseNca != null) { @@ -180,7 +181,7 @@ namespace hactoolnet { Sha256Info hashInfo = sect.Header.Sha256Info; - PrintItem(sb, colLen, $" Superblock Hash{sect.SuperblockHashValidity.GetValidityString()}:", hashInfo.MasterHash); + PrintItem(sb, colLen, $" Superblock Hash{sect.MasterHashValidity.GetValidityString()}:", hashInfo.MasterHash); // todo sb.AppendLine($" Hash Table{sect.Pfs0.Validity.GetValidityString()}:"); sb.AppendLine($" Hash Table:"); @@ -195,7 +196,7 @@ namespace hactoolnet { IvfcHeader ivfcInfo = sect.Header.IvfcInfo; - PrintItem(sb, colLen, $" Superblock Hash{sect.SuperblockHashValidity.GetValidityString()}:", ivfcInfo.MasterHash); + PrintItem(sb, colLen, $" Superblock Hash{sect.MasterHashValidity.GetValidityString()}:", ivfcInfo.MasterHash); PrintItem(sb, colLen, " Magic:", ivfcInfo.Magic); PrintItem(sb, colLen, " Version:", $"{ivfcInfo.Version:x8}"); diff --git a/hactoolnet/ProcessSwitchFs.cs b/hactoolnet/ProcessSwitchFs.cs index b23bc75b..74813824 100644 --- a/hactoolnet/ProcessSwitchFs.cs +++ b/hactoolnet/ProcessSwitchFs.cs @@ -170,7 +170,7 @@ namespace hactoolnet foreach (NcaSection sect in nca.Sections.Where(x => x != null)) { - Console.WriteLine($" {sect.SectionNum} {sect.Type} {sect.Header.EncryptionType} {sect.SuperblockHashValidity}"); + Console.WriteLine($" {sect.SectionNum} {sect.Type} {sect.Header.EncryptionType} {sect.MasterHashValidity}"); } } diff --git a/hactoolnet/Program.cs b/hactoolnet/Program.cs index 3476f4e4..1c12bafa 100644 --- a/hactoolnet/Program.cs +++ b/hactoolnet/Program.cs @@ -8,6 +8,27 @@ namespace hactoolnet public static class Program { public static void Main(string[] args) + { + try + { + Run(args); + } + catch (MissingKeyException ex) + { + string name = ex.Type == KeyType.Title ? $"Title key for rights ID {ex.Name}" : ex.Name; + Console.WriteLine($"\nERROR: {ex.Message}\nA required key is missing.\nKey name: {name}\n"); + } + catch (Exception ex) + { + Console.WriteLine($"\nERROR: {ex.Message}\n"); + + Console.WriteLine("Additional information:"); + Console.WriteLine(ex.GetType().FullName); + Console.WriteLine(ex.StackTrace); + } + } + + private static void Run(string[] args) { Console.OutputEncoding = Encoding.UTF8; var ctx = new Context(); @@ -25,47 +46,52 @@ namespace hactoolnet return; } - switch (ctx.Options.InFileType) - { - case FileType.Nca: - ProcessNca.Process(ctx); - break; - case FileType.Pfs0: - case FileType.Nsp: - ProcessNsp.Process(ctx); - break; - case FileType.Romfs: - ProcessRomfs.Process(ctx); - break; - case FileType.Nax0: - break; - case FileType.SwitchFs: - ProcessSwitchFs.Process(ctx); - break; - case FileType.Save: - ProcessSave.Process(ctx); - break; - case FileType.Xci: - ProcessXci.Process(ctx); - break; - case FileType.Keygen: - ProcessKeygen(ctx); - break; - case FileType.Pk11: - ProcessPackage.ProcessPk11(ctx); - break; - case FileType.Pk21: - ProcessPackage.ProcessPk21(ctx); - break; - case FileType.Kip1: - ProcessKip.ProcessKip1(ctx); - break; - case FileType.Ini1: - ProcessKip.ProcessIni1(ctx); - break; - default: - throw new ArgumentOutOfRangeException(); - } + RunTask(ctx); + } + } + + private static void RunTask(Context ctx) + { + switch (ctx.Options.InFileType) + { + case FileType.Nca: + ProcessNca.Process(ctx); + break; + case FileType.Pfs0: + case FileType.Nsp: + ProcessNsp.Process(ctx); + break; + case FileType.Romfs: + ProcessRomfs.Process(ctx); + break; + case FileType.Nax0: + break; + case FileType.SwitchFs: + ProcessSwitchFs.Process(ctx); + break; + case FileType.Save: + ProcessSave.Process(ctx); + break; + case FileType.Xci: + ProcessXci.Process(ctx); + break; + case FileType.Keygen: + ProcessKeygen(ctx); + break; + case FileType.Pk11: + ProcessPackage.ProcessPk11(ctx); + break; + case FileType.Pk21: + ProcessPackage.ProcessPk21(ctx); + break; + case FileType.Kip1: + ProcessKip.ProcessKip1(ctx); + break; + case FileType.Ini1: + ProcessKip.ProcessIni1(ctx); + break; + default: + throw new ArgumentOutOfRangeException(); } }