Continue NCA improvements

- Read only the NCA header when first opening an NCA. This allows for reading of partial NCAs and slightly improves performance when opening an NCA.
- Add a separate NCA method to validate master hashes now that it's not automatically done when opening the NCA.
- Fix possible hang when reading BKTR sections.
- When keys required to decrypt an NCA are missing, throw an exception with information about the missing keys.
- Add more sanity checks when reading an NCA.

Hactoolnet: Don't hard-crash when hitting an unhandled exception

LibHac/BktrCryptoStream.cs

@@ -78,6 +78,8 @@ namespace LibHac
                 int bytesToRead = (int)Math.Min(CurrentEntry.OffsetEnd - Position, count);
                 int bytesRead = base.Read(buffer, outPos, bytesToRead);
 
+                if (bytesRead == 0) break;
+
                 outPos += bytesRead;
                 totalBytesRead += bytesRead;
                 count -= bytesRead;

LibHac/Keyset.cs

@@ -246,7 +246,7 @@ namespace LibHac
             Crypto.DecryptEcb(headerKek, HeaderKeySource, HeaderKey, 0x20);
         }
 
-        private void DeriveSdCardKeys()
+        public void DeriveSdCardKeys()
         {
             var sdKek = new byte[0x10];
             Crypto.GenerateKek(MasterKeys[0], SdCardKekSource, sdKek, AesKekGenerationSource, AesKeyGenerationSource);
@@ -264,6 +264,8 @@ namespace LibHac
                 Crypto.DecryptEcb(sdKek, SdCardKeySourcesSpecific[k], SdCardKeys[k], 0x20);
             }
         }
+
+        internal static readonly string[] KakNames = {"application", "ocean", "system"};
     }
 
     public static class ExternalKeys

LibHac/Nca.cs

@@ -1,6 +1,5 @@
 using System;
 using System.IO;
-using System.Linq;
 using LibHac.Streams;
 using LibHac.XTSSharp;
 
@@ -20,6 +19,9 @@ namespace LibHac
         private bool KeepOpen { get; }
         private Nca BaseNca { get; set; }
 
+        private bool IsMissingTitleKey { get; set; }
+        private string MissingKeyName { get; set; }
+
         public NcaSection[] Sections { get; } = new NcaSection[4];
 
         public Nca(Keyset keyset, Stream stream, bool keepOpen)
@@ -38,19 +40,20 @@ namespace LibHac
             {
                 DecryptKeyArea(keyset);
             }
-            else
+            else if (keyset.TitleKeys.TryGetValue(Header.RightsId, out byte[] titleKey))
             {
-                if (keyset.TitleKeys.TryGetValue(Header.RightsId, out byte[] titleKey))
+                if (keyset.Titlekeks[CryptoType].IsEmpty())
                 {
+                    MissingKeyName = $"titlekek_{CryptoType:x2}";
+                }
+
                 TitleKey = titleKey;
                 Crypto.DecryptEcb(keyset.Titlekeks[CryptoType], titleKey, TitleKeyDec, 0x10);
                 DecryptedKeys[2] = TitleKeyDec;
             }
             else
             {
-                    // todo enable key check when opening a section
+                IsMissingTitleKey = true;
-                    // throw new MissingKeyException("A required key is missing.", $"{Header.RightsId.ToHexString()}", KeyType.Title);
-                }
             }
 
             for (int i = 0; i < 4; i++)
@@ -58,36 +61,69 @@ namespace LibHac
                 NcaSection section = ParseSection(i);
                 if (section == null) continue;
                 Sections[i] = section;
-                ValidateSuperblockHash(i);
+                ValidateMasterHash(i);
             }
 
-            foreach (NcaSection pfsSection in Sections.Where(x => x != null && x.Type == SectionType.Pfs0))
+            //foreach (NcaSection pfsSection in Sections.Where(x => x != null && x.Type == SectionType.Pfs0))
-            {
+            //{
-                Stream sectionStream = OpenSection(pfsSection.SectionNum, false, false);
+            //    Stream sectionStream = OpenSection(pfsSection.SectionNum, false, false);
-                if (sectionStream == null) continue;
+            //    if (sectionStream == null) continue;
 
-                var pfs = new Pfs(sectionStream);
+            //    var pfs = new Pfs(sectionStream);
-                if (!pfs.FileExists("main.npdm")) continue;
+            //    if (!pfs.FileExists("main.npdm")) continue;
 
-                pfsSection.IsExefs = true;
+            //    pfsSection.IsExefs = true;
-            }
+            //}
         }
 
+        /// <summary>
+        /// Opens a <see cref="Stream"/> of the underlying NCA file.
+        /// </summary>
+        /// <returns>A <see cref="Stream"/> that provides access to the entire raw NCA file.</returns>
         public Stream GetStream()
         {
             return StreamSource.CreateStream();
         }
 
+        public bool CanOpenSection(int index)
+        {
+            if (index < 0 || index > 3) throw new ArgumentOutOfRangeException(nameof(index));
+
+            NcaSection sect = Sections[index];
+            if (sect == null) return false;
+
+            return sect.Header.EncryptionType == NcaEncryptionType.None || !IsMissingTitleKey && string.IsNullOrWhiteSpace(MissingKeyName);
+        }
+
         private Stream OpenRawSection(int index)
         {
-            NcaSection sect = Sections[index];
+            if (index < 0 || index > 3) throw new ArgumentOutOfRangeException(nameof(index));
-            if (sect == null) throw new ArgumentOutOfRangeException(nameof(index));
 
-            //if (sect.SuperblockHashValidity == Validity.Invalid) return null;
+            NcaSection sect = Sections[index];
+            if (sect == null) return null;
+
+            if (sect.Header.EncryptionType != NcaEncryptionType.None)
+            {
+                if (IsMissingTitleKey)
+                {
+                    throw new MissingKeyException("Unable to decrypt NCA section.", Header.RightsId.ToHexString(), KeyType.Title);
+                }
+
+                if (!string.IsNullOrWhiteSpace(MissingKeyName))
+                {
+                    throw new MissingKeyException("Unable to decrypt NCA section.", MissingKeyName, KeyType.Common);
+                }
+            }
 
             long offset = sect.Offset;
             long size = sect.Size;
 
+            if (!Util.IsSubRange(offset, size, StreamSource.Length))
+            {
+                throw new InvalidDataException(
+                    $"Section offset (0x{offset:x}) and length (0x{size:x}) fall outside the total NCA length (0x{StreamSource.Length:x}).");
+            }
+
             Stream rawStream = StreamSource.CreateStream(offset, size);
 
             switch (sect.Header.EncryptionType)
@@ -104,10 +140,9 @@ namespace LibHac
                         false);
                     if (BaseNca == null) return rawStream;
 
-                    NcaSection baseSect = BaseNca.Sections.FirstOrDefault(x => x.Type == SectionType.Romfs);
+                    Stream baseStream = BaseNca.OpenSection(ProgramPartitionType.Data, true, false);
-                    if (baseSect == null) throw new InvalidDataException("Base NCA has no RomFS section");
+                    if (baseStream == null) throw new InvalidDataException("Base NCA has no RomFS section");
 
-                    Stream baseStream = BaseNca.OpenSection(baseSect.SectionNum, true, false);
                     return new Bktr(rawStream, baseStream, sect);
 
                 default:
@@ -115,25 +150,50 @@ namespace LibHac
             }
         }
 
+        /// <summary>
+        /// Opens one of the sections in the current <see cref="Nca"/>.
+        /// </summary>
+        /// <param name="index">The index of the NCA section to open. Valid indexes are 0-3.</param>
+        /// <param name="raw"><see langword="true"/> to open the raw section with hash metadata.</param>
+        /// <param name="enableIntegrityChecks"><see langword="true"/> to enable data integrity checks when reading the section.
+        /// Only applies if <paramref name="raw"/> is <see langword="false"/>.</param>
+        /// <returns>A <see cref="Stream"/> that provides access to the specified section. <see langword="null"/> if the section does not exist.</returns>
+        /// <exception cref="ArgumentOutOfRangeException">The specified <paramref name="index"/> is outside the valid range.</exception>
         public Stream OpenSection(int index, bool raw, bool enableIntegrityChecks)
         {
             Stream rawStream = OpenRawSection(index);
             NcaSection sect = Sections[index];
+            NcaFsHeader header = sect.Header;
 
             if (raw || rawStream == null) return rawStream;
 
-            switch (sect.Header.Type)
+            // If it's a patch section without a base, return the raw section because it has no hash data
+            if (header.EncryptionType == NcaEncryptionType.AesCtrEx && BaseNca == null) return rawStream;
+
+            switch (header.HashType)
             {
-                case SectionType.Pfs0:
+                case NcaHashType.Sha256:
-                    return InitIvfcForPartitionfs(sect.Header.Sha256Info, new SharedStreamSource(rawStream), enableIntegrityChecks);
+                    return InitIvfcForPartitionfs(header.Sha256Info, new SharedStreamSource(rawStream), enableIntegrityChecks);
-                case SectionType.Romfs:
+                case NcaHashType.Ivfc:
-                case SectionType.Bktr:
+                    return InitIvfcForRomfs(header.IvfcInfo, new SharedStreamSource(rawStream), enableIntegrityChecks);
-                    return InitIvfcForRomfs(sect.Header.IvfcInfo, new SharedStreamSource(rawStream), enableIntegrityChecks);
+
                 default:
                     throw new ArgumentOutOfRangeException();
             }
         }
 
+        /// <summary>
+        /// Opens one of the sections in the current <see cref="Nca"/>. For use with <see cref="ContentType.Program"/> type NCAs.
+        /// </summary>
+        /// <param name="type">The type of section to open.</param>
+        /// <param name="raw"><see langword="true"/> to open the raw section with hash metadata.</param>
+        /// <param name="enableIntegrityChecks"><see langword="true"/> to enable data integrity checks when reading the section.
+        /// Only applies if <paramref name="raw"/> is <see langword="false"/>.</param>
+        /// <returns>A <see cref="Stream"/> that provides access to the specified section. <see langword="null"/> if the section does not exist.</returns>
+        /// <exception cref="ArgumentOutOfRangeException">The specified <paramref name="type"/> is outside the valid range.</exception>
+        public Stream OpenSection(ProgramPartitionType type, bool raw, bool enableIntegrityChecks) =>
+            OpenSection((int)type, raw, enableIntegrityChecks);
+
         private static HierarchicalIntegrityVerificationStream InitIvfcForRomfs(IvfcHeader ivfc,
             SharedStreamSource romfsStreamSource, bool enableIntegrityChecks)
         {
@@ -195,10 +255,31 @@ namespace LibHac
             return new HierarchicalIntegrityVerificationStream(initInfo, enableIntegrityChecks);
         }
 
+        /// <summary>
+        /// Sets a base <see cref="Nca"/> to use when reading patches.
+        /// </summary>
+        /// <param name="baseNca">The base <see cref="Nca"/></param>
         public void SetBaseNca(Nca baseNca) => BaseNca = baseNca;
 
+        /// <summary>
+        /// Validates the master hash and store the result in <see cref="NcaSection.MasterHashValidity"/> for each <see cref="NcaSection"/>.
+        /// </summary>
+        public void ValidateMasterHashes()
+        {
+            for (int i = 0; i < 4; i++)
+            {
+                if (Sections[i] == null) continue;
+                ValidateMasterHash(i);
+            }
+        }
+
         private void DecryptHeader(Keyset keyset, Stream stream)
         {
+            if (keyset.HeaderKey.IsEmpty())
+            {
+                throw new MissingKeyException("Unable to decrypt NCA header.", "header_key", KeyType.Common);
+            }
+
             var headerBytes = new byte[0xC00];
             Xts xts = XtsAes128.Create(keyset.HeaderKey);
             using (var headerDec = new RandomAccessSectorStream(new XtsSectorStream(stream, xts, 0x200)))
@@ -213,6 +294,12 @@ namespace LibHac
 
         private void DecryptKeyArea(Keyset keyset)
         {
+            if (keyset.KeyAreaKeys[CryptoType][Header.KaekInd].IsEmpty())
+            {
+                MissingKeyName = $"key_area_key_{Keyset.KakNames[Header.KaekInd]}_{CryptoType:x2}";
+                return;
+            }
+
             for (int i = 0; i < 4; i++)
             {
                 Crypto.DecryptEcb(keyset.KeyAreaKeys[CryptoType][Header.KaekInd], Header.EncryptedKeys[i],
@@ -239,6 +326,10 @@ namespace LibHac
 
         private void CheckBktrKey(NcaSection sect)
         {
+            // The encryption subsection table in the bktr partition contains the length of the entire partition.
+            // The encryption table is always located immediately following the partition data
+            // Decrypt this value and compare it to the encryption table offset found in the NCA header
+
             long offset = sect.Header.BktrInfo.EncryptionHeader.Offset;
             using (var streamDec = new RandomAccessSectorStream(new Aes128CtrStream(GetStream(), DecryptedKeys[2], sect.Offset, sect.Size, sect.Offset, sect.Header.Ctr)))
             {
@@ -248,17 +339,23 @@ namespace LibHac
 
                 if (size != offset)
                 {
-                    sect.SuperblockHashValidity = Validity.Invalid;
+                    sect.MasterHashValidity = Validity.Invalid;
                 }
             }
         }
 
-        private void ValidateSuperblockHash(int index)
+        private void ValidateMasterHash(int index)
         {
             if (Sections[index] == null) throw new ArgumentOutOfRangeException(nameof(index));
             NcaSection sect = Sections[index];
 
-            byte[] expected = null;
+            if (!CanOpenSection(index))
+            {
+                sect.MasterHashValidity = Validity.MissingKey;
+                return;
+            }
+
+            byte[] expected = sect.GetMasterHash();
             long offset = 0;
             long size = 0;
 
@@ -267,16 +364,12 @@ namespace LibHac
                 case SectionType.Invalid:
                     break;
                 case SectionType.Pfs0:
-                    Sha256Info pfs0 = sect.Header.Sha256Info;
+                    offset = sect.Header.Sha256Info.HashTableOffset;
-                    expected = pfs0.MasterHash;
+                    size = sect.Header.Sha256Info.HashTableSize;
-                    offset = pfs0.HashTableOffset;
-                    size = pfs0.HashTableSize;
                     break;
                 case SectionType.Romfs:
-                    IvfcHeader ivfc = sect.Header.IvfcInfo;
+                    offset = sect.Header.IvfcInfo.LevelHeaders[0].LogicalOffset;
-                    expected = ivfc.MasterHash;
+                    size = 1 << sect.Header.IvfcInfo.LevelHeaders[0].BlockSizePower;
-                    offset = ivfc.LevelHeaders[0].LogicalOffset;
-                    size = 1 << ivfc.LevelHeaders[0].BlockSizePower;
                     break;
                 case SectionType.Bktr:
                     CheckBktrKey(sect);
@@ -284,36 +377,30 @@ namespace LibHac
             }
 
             Stream stream = OpenSection(index, true, false);
-            if (stream == null) return;
-            if (expected == null) return;
 
             var hashTable = new byte[size];
             stream.Position = offset;
             stream.Read(hashTable, 0, hashTable.Length);
 
-            sect.SuperblockHashValidity = Crypto.CheckMemoryHashTable(hashTable, expected, 0, hashTable.Length);
+            sect.MasterHashValidity = Crypto.CheckMemoryHashTable(hashTable, expected, 0, hashTable.Length);
-            // todo if (sect.Type == SectionType.Romfs) sect.Romfs.IvfcLevels[0].HashValidity = sect.SuperblockHashValidity;
         }
 
         public void VerifySection(int index, IProgressReport logger = null)
         {
             if (Sections[index] == null) throw new ArgumentOutOfRangeException(nameof(index));
+
             NcaSection sect = Sections[index];
-            Stream stream = OpenSection(index, true, false);
+            Stream stream = OpenSection(index, false, true);
             logger?.LogMessage($"Verifying section {index}...");
 
-            switch (sect.Type)
+            switch (sect.Header.HashType)
             {
-                case SectionType.Invalid:
+                case NcaHashType.Sha256:
                     break;
-                case SectionType.Pfs0:
+                case NcaHashType.Ivfc:
-                    // todo VerifyPfs0(stream, sect.Pfs0, logger);
-                    break;
-                case SectionType.Romfs:
-                    // todo VerifyIvfc(stream, sect.Romfs.IvfcLevels, logger);
-                    break;
-                case SectionType.Bktr:
                     break;
+                default:
+                    throw new ArgumentOutOfRangeException();
             }
         }
 
@@ -333,9 +420,26 @@ namespace LibHac
         public int SectionNum { get; set; }
         public long Offset { get; set; }
         public long Size { get; set; }
-        public Validity SuperblockHashValidity { get; set; }
+        public Validity MasterHashValidity { get; set; }
 
         public bool IsExefs { get; internal set; }
+
+        public byte[] GetMasterHash()
+        {
+            var hash = new byte[Crypto.Sha256DigestSize];
+
+            switch (Header.HashType)
+            {
+                case NcaHashType.Sha256:
+                    Array.Copy(Header.Sha256Info.MasterHash, hash, Crypto.Sha256DigestSize);
+                    break;
+                case NcaHashType.Ivfc:
+                    Array.Copy(Header.IvfcInfo.MasterHash, hash, Crypto.Sha256DigestSize);
+                    break;
+            }
+
+            return hash;
+        }
     }
 
     public static class NcaExtensions

LibHac/NcaStructs.cs

@@ -312,6 +312,13 @@ namespace LibHac
         public IvfcLevel[] IvfcLevels { get; set; } = new IvfcLevel[Romfs.IvfcMaxLevel];
     }
 
+    public enum ProgramPartitionType
+    {
+        Code,
+        Data,
+        Logo
+    };
+
     public enum ContentType
     {
         Program,
@@ -363,6 +370,7 @@ namespace LibHac
     {
         Unchecked,
         Invalid,
-        Valid
+        Valid,
+        MissingKey
     }
 }

LibHac/Util.cs

@@ -399,10 +399,16 @@ namespace LibHac
                 case 2: return "3.0.1-3.0.2";
                 case 3: return "4.0.0-4.1.0";
                 case 4: return "5.0.0-5.1.0";
-                case 5: return "6.0.0";
+                case 5: return "6.0.0-6.0.1";
                 default: return "Unknown";
             }
         }
+
+        public static bool IsSubRange(long startIndex, long subLength, long length)
+        {
+            bool isOutOfRange = startIndex < 0 || startIndex > length || subLength < 0 || startIndex > length - subLength;
+            return !isOutOfRange;
+        }
     }
 
     public class ByteArray128BitComparer : EqualityComparer<byte[]>

hactoolnet/ProcessNca.cs

@@ -13,6 +13,7 @@ namespace hactoolnet
             using (var file = new FileStream(ctx.Options.InFile, FileMode.Open, FileAccess.Read))
             {
                 var nca = new Nca(ctx.Keyset, file, false);
+                nca.ValidateMasterHashes();
 
                 if (ctx.Options.BaseNca != null)
                 {
@@ -180,7 +181,7 @@ namespace hactoolnet
             {
                 Sha256Info hashInfo = sect.Header.Sha256Info;
 
-                PrintItem(sb, colLen, $"        Superblock Hash{sect.SuperblockHashValidity.GetValidityString()}:", hashInfo.MasterHash);
+                PrintItem(sb, colLen, $"        Superblock Hash{sect.MasterHashValidity.GetValidityString()}:", hashInfo.MasterHash);
                 // todo sb.AppendLine($"        Hash Table{sect.Pfs0.Validity.GetValidityString()}:");
                 sb.AppendLine($"        Hash Table:");
 
@@ -195,7 +196,7 @@ namespace hactoolnet
             {
                 IvfcHeader ivfcInfo = sect.Header.IvfcInfo;
 
-                PrintItem(sb, colLen, $"        Superblock Hash{sect.SuperblockHashValidity.GetValidityString()}:", ivfcInfo.MasterHash);
+                PrintItem(sb, colLen, $"        Superblock Hash{sect.MasterHashValidity.GetValidityString()}:", ivfcInfo.MasterHash);
                 PrintItem(sb, colLen, "        Magic:", ivfcInfo.Magic);
                 PrintItem(sb, colLen, "        Version:", $"{ivfcInfo.Version:x8}");
 

hactoolnet/ProcessSwitchFs.cs

@@ -170,7 +170,7 @@ namespace hactoolnet
 
                     foreach (NcaSection sect in nca.Sections.Where(x => x != null))
                     {
-                        Console.WriteLine($"        {sect.SectionNum} {sect.Type} {sect.Header.EncryptionType} {sect.SuperblockHashValidity}");
+                        Console.WriteLine($"        {sect.SectionNum} {sect.Type} {sect.Header.EncryptionType} {sect.MasterHashValidity}");
                     }
                 }
 

hactoolnet/Program.cs

@@ -8,6 +8,27 @@ namespace hactoolnet
     public static class Program
     {
         public static void Main(string[] args)
+        {
+            try
+            {
+                Run(args);
+            }
+            catch (MissingKeyException ex)
+            {
+                string name = ex.Type == KeyType.Title ? $"Title key for rights ID {ex.Name}" : ex.Name;
+                Console.WriteLine($"\nERROR: {ex.Message}\nA required key is missing.\nKey name: {name}\n");
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"\nERROR: {ex.Message}\n");
+
+                Console.WriteLine("Additional information:");
+                Console.WriteLine(ex.GetType().FullName);
+                Console.WriteLine(ex.StackTrace);
+            }
+        }
+
+        private static void Run(string[] args)
         {
             Console.OutputEncoding = Encoding.UTF8;
             var ctx = new Context();
@@ -25,6 +46,12 @@ namespace hactoolnet
                     return;
                 }
 
+                RunTask(ctx);
+            }
+        }
+
+        private static void RunTask(Context ctx)
+        {
             switch (ctx.Options.InFileType)
             {
                 case FileType.Nca:
@@ -67,7 +94,6 @@ namespace hactoolnet
                     throw new ArgumentOutOfRangeException();
             }
         }
-        }
 
         private static void OpenKeyset(Context ctx)
         {