From 7e5c8c4e8ed9d412802ed5bde164cb7158b2e694 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Tue, 29 Jan 2019 15:58:48 -0600 Subject: [PATCH] Improve XCI open performance --- src/LibHac/IO/PartitionFileSystem.cs | 59 ++++----------------- src/LibHac/Xci.cs | 78 ++++++++++++++-------------- src/LibHac/XciHeader.cs | 47 ++++++++++++----- src/hactoolnet/ProcessXci.cs | 65 +++++++++++------------ 4 files changed, 115 insertions(+), 134 deletions(-) diff --git a/src/LibHac/IO/PartitionFileSystem.cs b/src/LibHac/IO/PartitionFileSystem.cs index 75b62faa..b8792c56 100644 --- a/src/LibHac/IO/PartitionFileSystem.cs +++ b/src/LibHac/IO/PartitionFileSystem.cs @@ -8,6 +8,7 @@ namespace LibHac.IO { public class PartitionFileSystem : IFileSystem { + // todo Re-add way of checking a file hash public PartitionFileSystemHeader Header { get; } public int HeaderSize { get; } public PartitionFileEntry[] Files { get; } @@ -28,26 +29,6 @@ namespace LibHac.IO BaseStorage = storage; } - public void CreateDirectory(string path) - { - throw new NotSupportedException(); - } - - public void CreateFile(string path, long size, CreateFileOptions options) - { - throw new NotSupportedException(); - } - - public void DeleteDirectory(string path) - { - throw new NotSupportedException(); - } - - public void DeleteFile(string path) - { - throw new NotSupportedException(); - } - public IDirectory OpenDirectory(string path, OpenDirectoryMode mode) { return new PartitionDirectory(this, path, mode); @@ -70,16 +51,6 @@ namespace LibHac.IO return new PartitionFile(BaseStorage, HeaderSize + entry.Offset, entry.Size, mode); } - public void RenameDirectory(string srcPath, string dstPath) - { - throw new NotSupportedException(); - } - - public void RenameFile(string srcPath, string dstPath) - { - throw new NotSupportedException(); - } - public bool DirectoryExists(string path) { path = PathTools.Normalize(path); @@ -104,10 +75,13 @@ namespace LibHac.IO throw new FileNotFoundException(path); } - public void Commit() - { - throw new NotSupportedException(); - } + public void CreateDirectory(string path) => throw new NotSupportedException(); + public void CreateFile(string path, long size, CreateFileOptions options) => throw new NotSupportedException(); + public void DeleteDirectory(string path) => throw new NotSupportedException(); + public void DeleteFile(string path) => throw new NotSupportedException(); + public void RenameDirectory(string srcPath, string dstPath) => throw new NotSupportedException(); + public void RenameFile(string srcPath, string dstPath) => throw new NotSupportedException(); + public void Commit() => throw new NotSupportedException(); } public enum PartitionFileSystemType @@ -160,17 +134,6 @@ namespace LibHac.IO reader.BaseStream.Position = stringTableOffset + Files[i].StringTableOffset; Files[i].Name = reader.ReadAsciiZ(); } - - - if (Type == PartitionFileSystemType.Hashed) - { - for (int i = 0; i < NumFiles; i++) - { - reader.BaseStream.Position = HeaderSize + Files[i].Offset; - Files[i].HashValidity = Crypto.CheckMemoryHashTable(reader.ReadBytes(Files[i].HashedRegionSize), Files[i].Hash, 0, Files[i].HashedRegionSize); - } - } - } private static int GetFileEntrySize(PartitionFileSystemType type) @@ -193,7 +156,7 @@ namespace LibHac.IO public long Offset; public long Size; public uint StringTableOffset; - public long Reserved; + public long HashedRegionOffset; public int HashedRegionSize; public byte[] Hash; public string Name; @@ -207,12 +170,12 @@ namespace LibHac.IO if (type == PartitionFileSystemType.Hashed) { HashedRegionSize = reader.ReadInt32(); - Reserved = reader.ReadInt64(); + HashedRegionOffset = reader.ReadInt64(); Hash = reader.ReadBytes(Crypto.Sha256DigestSize); } else { - Reserved = reader.ReadUInt32(); + reader.BaseStream.Position += 4; } } } diff --git a/src/LibHac/Xci.cs b/src/LibHac/Xci.cs index 5b713e2b..72f55f13 100644 --- a/src/LibHac/Xci.cs +++ b/src/LibHac/Xci.cs @@ -1,65 +1,65 @@ -using System.Collections.Generic; -using System.Linq; -using LibHac.IO; +using LibHac.IO; namespace LibHac { public class Xci { - private const string RootPartitionName = "rootpt"; - private const string UpdatePartitionName = "update"; - private const string NormalPartitionName = "normal"; - private const string SecurePartitionName = "secure"; - private const string LogoPartitionName = "logo"; - public XciHeader Header { get; } - public XciPartition RootPartition { get; } - public XciPartition UpdatePartition { get; } - public XciPartition NormalPartition { get; } - public XciPartition SecurePartition { get; } - public XciPartition LogoPartition { get; } - - public List Partitions { get; } = new List(); + private IStorage BaseStorage { get; } + private object InitLocker { get; } = new object(); + private XciPartition RootPartition { get; set; } public Xci(Keyset keyset, IStorage storage) { + BaseStorage = storage; Header = new XciHeader(keyset, storage.AsStream()); - IStorage hfs0Storage = storage.Slice(Header.PartitionFsHeaderAddress); + } - RootPartition = new XciPartition(hfs0Storage) + public bool HasPartition(XciPartitionType type) + { + if (type == XciPartitionType.Root) return true; + + return GetRootPartition().FileExists(type.GetFileName()); + } + + public XciPartition OpenPartition(XciPartitionType type) + { + XciPartition root = GetRootPartition(); + if (type == XciPartitionType.Root) return root; + + IStorage partitionStorage = root.OpenFile(type.GetFileName(), OpenMode.Read).AsStorage(); + return new XciPartition(partitionStorage); + } + + private XciPartition GetRootPartition() + { + if (RootPartition != null) return RootPartition; + + InitializeRootPartition(); + + return RootPartition; + } + + private void InitializeRootPartition() + { + lock (InitLocker) { - Name = RootPartitionName, - Offset = Header.PartitionFsHeaderAddress, - HashValidity = Header.PartitionFsHeaderValidity - }; + if (RootPartition != null) return; - Partitions.Add(RootPartition); + IStorage rootStorage = BaseStorage.Slice(Header.RootPartitionOffset); - foreach (PartitionFileEntry file in RootPartition.Files) - { - IFile partitionFile = RootPartition.OpenFile(file, OpenMode.Read); - - var partition = new XciPartition(partitionFile.AsStorage()) + RootPartition = new XciPartition(rootStorage) { - Name = file.Name, - Offset = Header.PartitionFsHeaderAddress + RootPartition.HeaderSize + file.Offset, - HashValidity = file.HashValidity + Offset = Header.RootPartitionOffset, + HashValidity = Header.PartitionFsHeaderValidity }; - - Partitions.Add(partition); } - - UpdatePartition = Partitions.FirstOrDefault(x => x.Name == UpdatePartitionName); - NormalPartition = Partitions.FirstOrDefault(x => x.Name == NormalPartitionName); - SecurePartition = Partitions.FirstOrDefault(x => x.Name == SecurePartitionName); - LogoPartition = Partitions.FirstOrDefault(x => x.Name == LogoPartitionName); } } public class XciPartition : PartitionFileSystem { - public string Name { get; internal set; } public long Offset { get; internal set; } public Validity HashValidity { get; set; } = Validity.Unchecked; diff --git a/src/LibHac/XciHeader.cs b/src/LibHac/XciHeader.cs index a46f24cf..f945c8a3 100644 --- a/src/LibHac/XciHeader.cs +++ b/src/LibHac/XciHeader.cs @@ -42,9 +42,9 @@ namespace LibHac public ulong PackageId { get; set; } public long ValidDataEndPage { get; set; } public byte[] AesCbcIv { get; set; } - public long PartitionFsHeaderAddress { get; set; } - public long PartitionFsHeaderSize { get; set; } - public byte[] PartitionFsHeaderHash { get; set; } + public long RootPartitionOffset { get; set; } + public long RootPartitionHeaderSize { get; set; } + public byte[] RootPartitionHeaderHash { get; set; } public byte[] InitialDataHash { get; set; } public int SelSec { get; set; } public int SelT1Key { get; set; } @@ -70,7 +70,6 @@ namespace LibHac { using (var reader = new BinaryReader(stream, Encoding.Default, true)) { - Signature = reader.ReadBytes(SignatureSize); Magic = reader.ReadAscii(4); if (Magic != HeaderMagic) @@ -96,9 +95,9 @@ namespace LibHac ValidDataEndPage = reader.ReadInt64(); AesCbcIv = reader.ReadBytes(Crypto.Aes128Size); Array.Reverse(AesCbcIv); - PartitionFsHeaderAddress = reader.ReadInt64(); - PartitionFsHeaderSize = reader.ReadInt64(); - PartitionFsHeaderHash = reader.ReadBytes(Crypto.Sha256DigestSize); + RootPartitionOffset = reader.ReadInt64(); + RootPartitionHeaderSize = reader.ReadInt64(); + RootPartitionHeaderHash = reader.ReadBytes(Crypto.Sha256DigestSize); InitialDataHash = reader.ReadBytes(Crypto.Sha256DigestSize); SelSec = reader.ReadInt32(); SelT1Key = reader.ReadInt32(); @@ -107,7 +106,6 @@ namespace LibHac if (!keyset.XciHeaderKey.IsEmpty()) { - byte[] encHeader = reader.ReadBytes(EncryptedHeaderSize); var decHeader = new byte[EncryptedHeaderSize]; Crypto.DecryptCbc(keyset.XciHeaderKey, AesCbcIv, encHeader, decHeader, EncryptedHeaderSize); @@ -128,12 +126,9 @@ namespace LibHac } } - reader.BaseStream.Position = PartitionFsHeaderAddress; - PartitionFsHeaderValidity = Crypto.CheckMemoryHashTable(reader.ReadBytes((int)PartitionFsHeaderSize), PartitionFsHeaderHash, 0, (int)PartitionFsHeaderSize); - + reader.BaseStream.Position = RootPartitionOffset; + PartitionFsHeaderValidity = Crypto.CheckMemoryHashTable(reader.ReadBytes((int)RootPartitionHeaderSize), RootPartitionHeaderHash, 0, (int)RootPartitionHeaderSize); } - - } } @@ -160,4 +155,30 @@ namespace LibHac ClockRate25 = 10551312, ClockRate50 } + + public enum XciPartitionType + { + Update, + Normal, + Secure, + Logo, + Root + } + + public static class XciExtensions + { + public static string GetFileName(this XciPartitionType type) + { + switch (type) + { + case XciPartitionType.Update: return "update"; + case XciPartitionType.Normal: return "normal"; + case XciPartitionType.Secure: return "secure"; + case XciPartitionType.Logo: return "logo"; + case XciPartitionType.Root: return "root"; + default: + throw new ArgumentOutOfRangeException(nameof(type), type, null); + } + } + } } diff --git a/src/hactoolnet/ProcessXci.cs b/src/hactoolnet/ProcessXci.cs index 02b06b74..62867d31 100644 --- a/src/hactoolnet/ProcessXci.cs +++ b/src/hactoolnet/ProcessXci.cs @@ -1,4 +1,5 @@ -using System.IO; +using System; +using System.IO; using System.Linq; using System.Text; using LibHac; @@ -19,32 +20,32 @@ namespace hactoolnet if (ctx.Options.RootDir != null) { - xci.RootPartition?.Extract(ctx.Options.RootDir, ctx.Logger); + xci.OpenPartition(XciPartitionType.Root).Extract(ctx.Options.RootDir, ctx.Logger); } - if (ctx.Options.UpdateDir != null) + if (ctx.Options.UpdateDir != null && xci.HasPartition(XciPartitionType.Update)) { - xci.UpdatePartition?.Extract(ctx.Options.UpdateDir, ctx.Logger); + xci.OpenPartition(XciPartitionType.Update).Extract(ctx.Options.UpdateDir, ctx.Logger); } - if (ctx.Options.NormalDir != null) + if (ctx.Options.NormalDir != null && xci.HasPartition(XciPartitionType.Normal)) { - xci.NormalPartition?.Extract(ctx.Options.NormalDir, ctx.Logger); + xci.OpenPartition(XciPartitionType.Normal).Extract(ctx.Options.NormalDir, ctx.Logger); } - if (ctx.Options.SecureDir != null) + if (ctx.Options.SecureDir != null && xci.HasPartition(XciPartitionType.Secure)) { - xci.SecurePartition?.Extract(ctx.Options.SecureDir, ctx.Logger); + xci.OpenPartition(XciPartitionType.Secure).Extract(ctx.Options.SecureDir, ctx.Logger); } - if (ctx.Options.LogoDir != null) + if (ctx.Options.LogoDir != null && xci.HasPartition(XciPartitionType.Logo)) { - xci.LogoPartition?.Extract(ctx.Options.LogoDir, ctx.Logger); + xci.OpenPartition(XciPartitionType.Logo).Extract(ctx.Options.LogoDir, ctx.Logger); } - if (ctx.Options.OutDir != null && xci.RootPartition != null) + if (ctx.Options.OutDir != null) { - XciPartition root = xci.RootPartition; + XciPartition root = xci.OpenPartition(XciPartitionType.Root); if (root == null) { ctx.Logger.LogMessage("Could not find root partition"); @@ -123,7 +124,9 @@ namespace hactoolnet private static Nca GetXciMainNca(Xci xci, Context ctx) { - if (xci.SecurePartition == null) + XciPartition partition = xci.OpenPartition(XciPartitionType.Secure); + + if (partition == null) { ctx.Logger.LogMessage("Could not find secure partition"); return null; @@ -131,9 +134,9 @@ namespace hactoolnet Nca mainNca = null; - foreach (PartitionFileEntry fileEntry in xci.SecurePartition.Files.Where(x => x.Name.EndsWith(".nca"))) + foreach (PartitionFileEntry fileEntry in partition.Files.Where(x => x.Name.EndsWith(".nca"))) { - IStorage ncaStorage = xci.SecurePartition.OpenFile(fileEntry, OpenMode.Read).AsStorage(); + IStorage ncaStorage = partition.OpenFile(fileEntry, OpenMode.Read).AsStorage(); var nca = new Nca(ctx.Keyset, ncaStorage, true); if (nca.Header.ContentType == ContentType.Program) @@ -156,28 +159,35 @@ namespace hactoolnet PrintItem(sb, colLen, "Magic:", xci.Header.Magic); PrintItem(sb, colLen, $"Header Signature{xci.Header.SignatureValidity.GetValidityString()}:", xci.Header.Signature); - PrintItem(sb, colLen, $"Header Hash{xci.Header.PartitionFsHeaderValidity.GetValidityString()}:", xci.Header.PartitionFsHeaderHash); + PrintItem(sb, colLen, $"Header Hash{xci.Header.PartitionFsHeaderValidity.GetValidityString()}:", xci.Header.RootPartitionHeaderHash); PrintItem(sb, colLen, "Cartridge Type:", GetCartridgeType(xci.Header.RomSize)); PrintItem(sb, colLen, "Cartridge Size:", $"0x{Util.MediaToReal(xci.Header.ValidDataEndPage + 1):x12}"); PrintItem(sb, colLen, "Header IV:", xci.Header.AesCbcIv); - foreach (XciPartition partition in xci.Partitions.OrderBy(x => x.Offset)) + PrintPartition(sb, colLen, xci.OpenPartition(XciPartitionType.Root), XciPartitionType.Root); + + foreach (XciPartitionType type in Enum.GetValues(typeof(XciPartitionType))) { - PrintPartition(sb, colLen, partition); + if (type == XciPartitionType.Root || !xci.HasPartition(type)) continue; + + XciPartition partition = xci.OpenPartition(type); + PrintPartition(sb, colLen, partition, type); } return sb.ToString(); } - private static void PrintPartition(StringBuilder sb, int colLen, XciPartition partition) + private static void PrintPartition(StringBuilder sb, int colLen, XciPartition partition, XciPartitionType type) { const int fileNameLen = 57; - sb.AppendLine($"{GetDisplayName(partition.Name)} Partition:{partition.HashValidity.GetValidityString()}"); + sb.AppendLine($"{type.ToString()} Partition:{partition.HashValidity.GetValidityString()}"); PrintItem(sb, colLen, " Magic:", partition.Header.Magic); PrintItem(sb, colLen, " Offset:", $"{partition.Offset:x12}"); PrintItem(sb, colLen, " Number of files:", partition.Files.Length); + string name = type.GetFileName(); + if (partition.Files.Length > 0 && partition.Files.Length < 100) { for (int i = 0; i < partition.Files.Length; i++) @@ -186,26 +196,13 @@ namespace hactoolnet string label = i == 0 ? " Files:" : ""; string offsets = $"{file.Offset:x12}-{file.Offset + file.Size:x12}{file.HashValidity.GetValidityString()}"; - string data = $"{partition.Name}:/{file.Name}".PadRight(fileNameLen) + offsets; + string data = $"{name}:/{file.Name}".PadRight(fileNameLen) + offsets; PrintItem(sb, colLen, label, data); } } } - private static string GetDisplayName(string name) - { - switch (name) - { - case "rootpt": return "Root"; - case "update": return "Update"; - case "normal": return "Normal"; - case "secure": return "Secure"; - case "logo": return "Logo"; - default: return name; - } - } - private static string GetCartridgeType(RomSize size) { switch (size)