Improve XCI open performance

src/LibHac/IO/PartitionFileSystem.cs

@@ -8,6 +8,7 @@ namespace LibHac.IO
 {
     public class PartitionFileSystem : IFileSystem
     {
+        // todo Re-add way of checking a file hash
         public PartitionFileSystemHeader Header { get; }
         public int HeaderSize { get; }
         public PartitionFileEntry[] Files { get; }
@@ -28,26 +29,6 @@ namespace LibHac.IO
             BaseStorage = storage;
         }
 
-        public void CreateDirectory(string path)
-        {
-            throw new NotSupportedException();
-        }
-
-        public void CreateFile(string path, long size, CreateFileOptions options)
-        {
-            throw new NotSupportedException();
-        }
-
-        public void DeleteDirectory(string path)
-        {
-            throw new NotSupportedException();
-        }
-
-        public void DeleteFile(string path)
-        {
-            throw new NotSupportedException();
-        }
-
         public IDirectory OpenDirectory(string path, OpenDirectoryMode mode)
         {
             return new PartitionDirectory(this, path, mode);
@@ -70,16 +51,6 @@ namespace LibHac.IO
             return new PartitionFile(BaseStorage, HeaderSize + entry.Offset, entry.Size, mode);
         }
 
-        public void RenameDirectory(string srcPath, string dstPath)
-        {
-            throw new NotSupportedException();
-        }
-
-        public void RenameFile(string srcPath, string dstPath)
-        {
-            throw new NotSupportedException();
-        }
-
         public bool DirectoryExists(string path)
         {
             path = PathTools.Normalize(path);
@@ -104,10 +75,13 @@ namespace LibHac.IO
             throw new FileNotFoundException(path);
         }
 
-        public void Commit()
+        public void CreateDirectory(string path) => throw new NotSupportedException();
-        {
+        public void CreateFile(string path, long size, CreateFileOptions options) => throw new NotSupportedException();
-            throw new NotSupportedException();
+        public void DeleteDirectory(string path) => throw new NotSupportedException();
-        }
+        public void DeleteFile(string path) => throw new NotSupportedException();
+        public void RenameDirectory(string srcPath, string dstPath) => throw new NotSupportedException();
+        public void RenameFile(string srcPath, string dstPath) => throw new NotSupportedException();
+        public void Commit() => throw new NotSupportedException();
     }
 
     public enum PartitionFileSystemType
@@ -160,17 +134,6 @@ namespace LibHac.IO
                 reader.BaseStream.Position = stringTableOffset + Files[i].StringTableOffset;
                 Files[i].Name = reader.ReadAsciiZ();
             }
-
-
-            if (Type == PartitionFileSystemType.Hashed)
-            {
-                for (int i = 0; i < NumFiles; i++)
-                {
-                    reader.BaseStream.Position = HeaderSize + Files[i].Offset;
-                    Files[i].HashValidity = Crypto.CheckMemoryHashTable(reader.ReadBytes(Files[i].HashedRegionSize), Files[i].Hash, 0, Files[i].HashedRegionSize);
-                }
-            }
-
         }
 
         private static int GetFileEntrySize(PartitionFileSystemType type)
@@ -193,7 +156,7 @@ namespace LibHac.IO
         public long Offset;
         public long Size;
         public uint StringTableOffset;
-        public long Reserved;
+        public long HashedRegionOffset;
         public int HashedRegionSize;
         public byte[] Hash;
         public string Name;
@@ -207,12 +170,12 @@ namespace LibHac.IO
             if (type == PartitionFileSystemType.Hashed)
             {
                 HashedRegionSize = reader.ReadInt32();
-                Reserved = reader.ReadInt64();
+                HashedRegionOffset = reader.ReadInt64();
                 Hash = reader.ReadBytes(Crypto.Sha256DigestSize);
             }
             else
             {
-                Reserved = reader.ReadUInt32();
+                reader.BaseStream.Position += 4;
             }
         }
     }

src/LibHac/Xci.cs

@@ -1,65 +1,65 @@
-using System.Collections.Generic;
+using LibHac.IO;
-using System.Linq;
-using LibHac.IO;
 
 namespace LibHac
 {
     public class Xci
     {
-        private const string RootPartitionName = "rootpt";
-        private const string UpdatePartitionName = "update";
-        private const string NormalPartitionName = "normal";
-        private const string SecurePartitionName = "secure";
-        private const string LogoPartitionName = "logo";
-
         public XciHeader Header { get; }
 
-        public XciPartition RootPartition { get; }
+        private IStorage BaseStorage { get; }
-        public XciPartition UpdatePartition { get; }
+        private object InitLocker { get; } = new object();
-        public XciPartition NormalPartition { get; }
+        private XciPartition RootPartition { get; set; }
-        public XciPartition SecurePartition { get; }
-        public XciPartition LogoPartition { get; }
-
-        public List<XciPartition> Partitions { get; } = new List<XciPartition>();
 
         public Xci(Keyset keyset, IStorage storage)
         {
+            BaseStorage = storage;
             Header = new XciHeader(keyset, storage.AsStream());
-            IStorage hfs0Storage = storage.Slice(Header.PartitionFsHeaderAddress);
+        }
 
-            RootPartition = new XciPartition(hfs0Storage)
+        public bool HasPartition(XciPartitionType type)
+        {
+            if (type == XciPartitionType.Root) return true;
+
+            return GetRootPartition().FileExists(type.GetFileName());
+        }
+
+        public XciPartition OpenPartition(XciPartitionType type)
+        {
+            XciPartition root = GetRootPartition();
+            if (type == XciPartitionType.Root) return root;
+
+            IStorage partitionStorage = root.OpenFile(type.GetFileName(), OpenMode.Read).AsStorage();
+            return new XciPartition(partitionStorage);
+        }
+
+        private XciPartition GetRootPartition()
+        {
+            if (RootPartition != null) return RootPartition;
+
+            InitializeRootPartition();
+
+            return RootPartition;
+        }
+
+        private void InitializeRootPartition()
+        {
+            lock (InitLocker)
             {
-                Name = RootPartitionName,
+                if (RootPartition != null) return;
-                Offset = Header.PartitionFsHeaderAddress,
-                HashValidity = Header.PartitionFsHeaderValidity
-            };
 
-            Partitions.Add(RootPartition);
+                IStorage rootStorage = BaseStorage.Slice(Header.RootPartitionOffset);
 
-            foreach (PartitionFileEntry file in RootPartition.Files)
+                RootPartition = new XciPartition(rootStorage)
-            {
-                IFile partitionFile = RootPartition.OpenFile(file, OpenMode.Read);
-
-                var partition = new XciPartition(partitionFile.AsStorage())
                 {
-                    Name = file.Name,
+                    Offset = Header.RootPartitionOffset,
-                    Offset = Header.PartitionFsHeaderAddress + RootPartition.HeaderSize + file.Offset,
+                    HashValidity = Header.PartitionFsHeaderValidity
-                    HashValidity = file.HashValidity
                 };
-
-                Partitions.Add(partition);
             }
-
-            UpdatePartition = Partitions.FirstOrDefault(x => x.Name == UpdatePartitionName);
-            NormalPartition = Partitions.FirstOrDefault(x => x.Name == NormalPartitionName);
-            SecurePartition = Partitions.FirstOrDefault(x => x.Name == SecurePartitionName);
-            LogoPartition = Partitions.FirstOrDefault(x => x.Name == LogoPartitionName);
         }
     }
 
     public class XciPartition : PartitionFileSystem
     {
-        public string Name { get; internal set; }
         public long Offset { get; internal set; }
         public Validity HashValidity { get; set; } = Validity.Unchecked;
 

src/LibHac/XciHeader.cs

@@ -42,9 +42,9 @@ namespace LibHac
         public ulong PackageId { get; set; }
         public long ValidDataEndPage { get; set; }
         public byte[] AesCbcIv { get; set; }
-        public long PartitionFsHeaderAddress { get; set; }
+        public long RootPartitionOffset { get; set; }
-        public long PartitionFsHeaderSize { get; set; }
+        public long RootPartitionHeaderSize { get; set; }
-        public byte[] PartitionFsHeaderHash { get; set; }
+        public byte[] RootPartitionHeaderHash { get; set; }
         public byte[] InitialDataHash { get; set; }
         public int SelSec { get; set; }
         public int SelT1Key { get; set; }
@@ -70,7 +70,6 @@ namespace LibHac
         {
             using (var reader = new BinaryReader(stream, Encoding.Default, true))
             {
-
                 Signature = reader.ReadBytes(SignatureSize);
                 Magic = reader.ReadAscii(4);
                 if (Magic != HeaderMagic)
@@ -96,9 +95,9 @@ namespace LibHac
                 ValidDataEndPage = reader.ReadInt64();
                 AesCbcIv = reader.ReadBytes(Crypto.Aes128Size);
                 Array.Reverse(AesCbcIv);
-                PartitionFsHeaderAddress = reader.ReadInt64();
+                RootPartitionOffset = reader.ReadInt64();
-                PartitionFsHeaderSize = reader.ReadInt64();
+                RootPartitionHeaderSize = reader.ReadInt64();
-                PartitionFsHeaderHash = reader.ReadBytes(Crypto.Sha256DigestSize);
+                RootPartitionHeaderHash = reader.ReadBytes(Crypto.Sha256DigestSize);
                 InitialDataHash = reader.ReadBytes(Crypto.Sha256DigestSize);
                 SelSec = reader.ReadInt32();
                 SelT1Key = reader.ReadInt32();
@@ -107,7 +106,6 @@ namespace LibHac
 
                 if (!keyset.XciHeaderKey.IsEmpty())
                 {
-
                     byte[] encHeader = reader.ReadBytes(EncryptedHeaderSize);
                     var decHeader = new byte[EncryptedHeaderSize];
                     Crypto.DecryptCbc(keyset.XciHeaderKey, AesCbcIv, encHeader, decHeader, EncryptedHeaderSize);
@@ -128,12 +126,9 @@ namespace LibHac
                     }
                 }
 
-                reader.BaseStream.Position = PartitionFsHeaderAddress;
+                reader.BaseStream.Position = RootPartitionOffset;
-                PartitionFsHeaderValidity = Crypto.CheckMemoryHashTable(reader.ReadBytes((int)PartitionFsHeaderSize), PartitionFsHeaderHash, 0, (int)PartitionFsHeaderSize);
+                PartitionFsHeaderValidity = Crypto.CheckMemoryHashTable(reader.ReadBytes((int)RootPartitionHeaderSize), RootPartitionHeaderHash, 0, (int)RootPartitionHeaderSize);
-
             }
-
-
         }
     }
 
@@ -160,4 +155,30 @@ namespace LibHac
         ClockRate25 = 10551312,
         ClockRate50
     }
+
+    public enum XciPartitionType
+    {
+        Update,
+        Normal,
+        Secure,
+        Logo,
+        Root
+    }
+
+    public static class XciExtensions
+    {
+        public static string GetFileName(this XciPartitionType type)
+        {
+            switch (type)
+            {
+                case XciPartitionType.Update: return "update";
+                case XciPartitionType.Normal: return "normal";
+                case XciPartitionType.Secure: return "secure";
+                case XciPartitionType.Logo: return "logo";
+                case XciPartitionType.Root: return "root";
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(type), type, null);
+            }
+        }
+    }
 }

src/hactoolnet/ProcessXci.cs

@@ -1,4 +1,5 @@
-using System.IO;
+using System;
+using System.IO;
 using System.Linq;
 using System.Text;
 using LibHac;
@@ -19,32 +20,32 @@ namespace hactoolnet
 
                 if (ctx.Options.RootDir != null)
                 {
-                    xci.RootPartition?.Extract(ctx.Options.RootDir, ctx.Logger);
+                    xci.OpenPartition(XciPartitionType.Root).Extract(ctx.Options.RootDir, ctx.Logger);
                 }
 
-                if (ctx.Options.UpdateDir != null)
+                if (ctx.Options.UpdateDir != null && xci.HasPartition(XciPartitionType.Update))
                 {
-                    xci.UpdatePartition?.Extract(ctx.Options.UpdateDir, ctx.Logger);
+                    xci.OpenPartition(XciPartitionType.Update).Extract(ctx.Options.UpdateDir, ctx.Logger);
                 }
 
-                if (ctx.Options.NormalDir != null)
+                if (ctx.Options.NormalDir != null && xci.HasPartition(XciPartitionType.Normal))
                 {
-                    xci.NormalPartition?.Extract(ctx.Options.NormalDir, ctx.Logger);
+                    xci.OpenPartition(XciPartitionType.Normal).Extract(ctx.Options.NormalDir, ctx.Logger);
                 }
 
-                if (ctx.Options.SecureDir != null)
+                if (ctx.Options.SecureDir != null && xci.HasPartition(XciPartitionType.Secure))
                 {
-                    xci.SecurePartition?.Extract(ctx.Options.SecureDir, ctx.Logger);
+                    xci.OpenPartition(XciPartitionType.Secure).Extract(ctx.Options.SecureDir, ctx.Logger);
                 }
 
-                if (ctx.Options.LogoDir != null)
+                if (ctx.Options.LogoDir != null && xci.HasPartition(XciPartitionType.Logo))
                 {
-                    xci.LogoPartition?.Extract(ctx.Options.LogoDir, ctx.Logger);
+                    xci.OpenPartition(XciPartitionType.Logo).Extract(ctx.Options.LogoDir, ctx.Logger);
                 }
 
-                if (ctx.Options.OutDir != null && xci.RootPartition != null)
+                if (ctx.Options.OutDir != null)
                 {
-                    XciPartition root = xci.RootPartition;
+                    XciPartition root = xci.OpenPartition(XciPartitionType.Root);
                     if (root == null)
                     {
                         ctx.Logger.LogMessage("Could not find root partition");
@@ -123,7 +124,9 @@ namespace hactoolnet
 
         private static Nca GetXciMainNca(Xci xci, Context ctx)
         {
-            if (xci.SecurePartition == null)
+            XciPartition partition = xci.OpenPartition(XciPartitionType.Secure);
+
+            if (partition == null)
             {
                 ctx.Logger.LogMessage("Could not find secure partition");
                 return null;
@@ -131,9 +134,9 @@ namespace hactoolnet
 
             Nca mainNca = null;
 
-            foreach (PartitionFileEntry fileEntry in xci.SecurePartition.Files.Where(x => x.Name.EndsWith(".nca")))
+            foreach (PartitionFileEntry fileEntry in partition.Files.Where(x => x.Name.EndsWith(".nca")))
             {
-                IStorage ncaStorage = xci.SecurePartition.OpenFile(fileEntry, OpenMode.Read).AsStorage();
+                IStorage ncaStorage = partition.OpenFile(fileEntry, OpenMode.Read).AsStorage();
                 var nca = new Nca(ctx.Keyset, ncaStorage, true);
 
                 if (nca.Header.ContentType == ContentType.Program)
@@ -156,28 +159,35 @@ namespace hactoolnet
 
             PrintItem(sb, colLen, "Magic:", xci.Header.Magic);
             PrintItem(sb, colLen, $"Header Signature{xci.Header.SignatureValidity.GetValidityString()}:", xci.Header.Signature);
-            PrintItem(sb, colLen, $"Header Hash{xci.Header.PartitionFsHeaderValidity.GetValidityString()}:", xci.Header.PartitionFsHeaderHash);
+            PrintItem(sb, colLen, $"Header Hash{xci.Header.PartitionFsHeaderValidity.GetValidityString()}:", xci.Header.RootPartitionHeaderHash);
             PrintItem(sb, colLen, "Cartridge Type:", GetCartridgeType(xci.Header.RomSize));
             PrintItem(sb, colLen, "Cartridge Size:", $"0x{Util.MediaToReal(xci.Header.ValidDataEndPage + 1):x12}");
             PrintItem(sb, colLen, "Header IV:", xci.Header.AesCbcIv);
 
-            foreach (XciPartition partition in xci.Partitions.OrderBy(x => x.Offset))
+            PrintPartition(sb, colLen, xci.OpenPartition(XciPartitionType.Root), XciPartitionType.Root);
+
+            foreach (XciPartitionType type in Enum.GetValues(typeof(XciPartitionType)))
             {
-                PrintPartition(sb, colLen, partition);
+                if (type == XciPartitionType.Root || !xci.HasPartition(type)) continue;
+
+                XciPartition partition = xci.OpenPartition(type);
+                PrintPartition(sb, colLen, partition, type);
             }
 
             return sb.ToString();
         }
 
-        private static void PrintPartition(StringBuilder sb, int colLen, XciPartition partition)
+        private static void PrintPartition(StringBuilder sb, int colLen, XciPartition partition, XciPartitionType type)
         {
             const int fileNameLen = 57;
 
-            sb.AppendLine($"{GetDisplayName(partition.Name)} Partition:{partition.HashValidity.GetValidityString()}");
+            sb.AppendLine($"{type.ToString()} Partition:{partition.HashValidity.GetValidityString()}");
             PrintItem(sb, colLen, "    Magic:", partition.Header.Magic);
             PrintItem(sb, colLen, "    Offset:", $"{partition.Offset:x12}");
             PrintItem(sb, colLen, "    Number of files:", partition.Files.Length);
 
+            string name = type.GetFileName();
+
             if (partition.Files.Length > 0 && partition.Files.Length < 100)
             {
                 for (int i = 0; i < partition.Files.Length; i++)
@@ -186,26 +196,13 @@ namespace hactoolnet
 
                     string label = i == 0 ? "    Files:" : "";
                     string offsets = $"{file.Offset:x12}-{file.Offset + file.Size:x12}{file.HashValidity.GetValidityString()}";
-                    string data = $"{partition.Name}:/{file.Name}".PadRight(fileNameLen) + offsets;
+                    string data = $"{name}:/{file.Name}".PadRight(fileNameLen) + offsets;
 
                     PrintItem(sb, colLen, label, data);
                 }
             }
         }
 
-        private static string GetDisplayName(string name)
-        {
-            switch (name)
-            {
-                case "rootpt": return "Root";
-                case "update": return "Update";
-                case "normal": return "Normal";
-                case "secure": return "Secure";
-                case "logo": return "Logo";
-                default: return name;
-            }
-        }
-
         private static string GetCartridgeType(RomSize size)
         {
             switch (size)