From 0ed67d87df2de27ae2586aaaebaf257686d7b9b9 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Sat, 19 Jan 2019 21:05:25 -0600 Subject: [PATCH] Add NDV0 support --- src/LibHac/BitTools.cs | 11 +++ src/LibHac/IO/DeltaFragment.cs | 160 +++++++++++++++++++++++++++++++++ src/LibHac/IO/FileReader.cs | 30 +++++++ src/hactoolnet/CliParser.cs | 9 +- src/hactoolnet/Options.cs | 3 + src/hactoolnet/ProcessDelta.cs | 81 +++++++++++++++++ src/hactoolnet/Program.cs | 3 + 7 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 src/LibHac/BitTools.cs create mode 100644 src/LibHac/IO/DeltaFragment.cs create mode 100644 src/hactoolnet/ProcessDelta.cs diff --git a/src/LibHac/BitTools.cs b/src/LibHac/BitTools.cs new file mode 100644 index 00000000..ba50248f --- /dev/null +++ b/src/LibHac/BitTools.cs @@ -0,0 +1,11 @@ +namespace LibHac +{ + public static class BitTools + { + public static int SignExtend32(int value, int bits) + { + int shift = 8 * sizeof(int) - bits; + return (value << shift) >> shift; + } + } +} diff --git a/src/LibHac/IO/DeltaFragment.cs b/src/LibHac/IO/DeltaFragment.cs new file mode 100644 index 00000000..4d279383 --- /dev/null +++ b/src/LibHac/IO/DeltaFragment.cs @@ -0,0 +1,160 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace LibHac.IO +{ + public class DeltaFragment + { + private const string Ndv0Magic = "NDV0"; + private IStorage Original { get; set; } + private IStorage Delta { get; } + public DeltaFragmentHeader Header { get; } + private List Segments { get; } = new List(); + + public DeltaFragment(IStorage delta, IStorage originalData) : this(delta) + { + SetBaseStorage(originalData); + } + + public DeltaFragment(IStorage delta) + { + Delta = delta; + + if (Delta.Length < 0x40) throw new InvalidDataException("Delta file is too small."); + + Header = new DeltaFragmentHeader(new StorageFile(delta, OpenMode.Read)); + + if (Header.Magic != Ndv0Magic) throw new InvalidDataException("NDV0 magic value is missing."); + + long fragmentSize = Header.FragmentHeaderSize + Header.FragmentBodySize; + if (Delta.Length < fragmentSize) + { + throw new InvalidDataException($"Delta file is smaller than the header indicates. (0x{fragmentSize} bytes)"); + } + + ParseDeltaStructure(); + } + + public void SetBaseStorage(IStorage baseStorage) + { + Original = baseStorage; + + if (Original.Length != Header.OriginalSize) + { + throw new InvalidDataException($"Original file size does not match the size in the delta header. (0x{Header.OriginalSize} bytes)"); + } + } + + public IStorage GetPatchedStorage() + { + if (Original == null) throw new InvalidOperationException("Cannot apply a delta patch without a base file."); + + var storages = new List(); + + foreach (DeltaFragmentSegment segment in Segments) + { + IStorage source = segment.IsInOriginal ? Original : Delta; + + // todo Do this without tons of SubStorages + Storage sub = source.Slice(segment.SourceOffset, segment.Size); + + storages.Add(sub); + } + + return new ConcatenationStorage(storages, true); + } + + private void ParseDeltaStructure() + { + var reader = new FileReader(new StorageFile(Delta, OpenMode.Read)); + + reader.Position = Header.FragmentHeaderSize; + + long offset = 0; + + while (offset < Header.NewSize) + { + ReadSegmentHeader(reader, out int size, out int seek); + + if (seek > 0) + { + var segment = new DeltaFragmentSegment() + { + SourceOffset = offset, + Size = seek, + IsInOriginal = true + }; + + Segments.Add(segment); + offset += seek; + } + + if (size > 0) + { + var segment = new DeltaFragmentSegment() + { + SourceOffset = reader.Position, + Size = size, + IsInOriginal = false + }; + + Segments.Add(segment); + offset += size; + } + + reader.Position += size; + } + } + + private static void ReadSegmentHeader(FileReader reader, out int size, out int seek) + { + byte type = reader.ReadUInt8(); + + int seekBytes = (type & 3) + 1; + int sizeBytes = ((type >> 3) & 3) + 1; + + size = ReadInt(reader, sizeBytes); + seek = ReadInt(reader, seekBytes); + } + + private static int ReadInt(FileReader reader, int bytes) + { + switch (bytes) + { + case 1: return reader.ReadUInt8(); + case 2: return reader.ReadUInt16(); + case 3: return reader.ReadUInt24(); + case 4: return reader.ReadInt32(); + default: return 0; + } + } + } + + internal class DeltaFragmentSegment + { + public long SourceOffset { get; set; } + public int Size { get; set; } + public bool IsInOriginal { get; set; } + } + + public class DeltaFragmentHeader + { + public string Magic { get; } + public long OriginalSize { get; } + public long NewSize { get; } + public long FragmentHeaderSize { get; } + public long FragmentBodySize { get; } + + public DeltaFragmentHeader(IFile header) + { + var reader = new FileReader(header); + + Magic = reader.ReadAscii(4); + OriginalSize = reader.ReadInt64(8); + NewSize = reader.ReadInt64(); + FragmentHeaderSize = reader.ReadInt64(); + FragmentBodySize = reader.ReadInt64(); + } + } +} diff --git a/src/LibHac/IO/FileReader.cs b/src/LibHac/IO/FileReader.cs index 4639c61d..8977b960 100644 --- a/src/LibHac/IO/FileReader.cs +++ b/src/LibHac/IO/FileReader.cs @@ -1,6 +1,7 @@ using System; using System.Diagnostics; using System.Runtime.InteropServices; +using System.Text; namespace LibHac.IO { @@ -62,6 +63,20 @@ namespace LibHac.IO return MemoryMarshal.Read(_buffer); } + public int ReadUInt24(long offset, bool updatePosition) + { + FillBuffer(offset, 3, updatePosition); + + return MemoryMarshal.Read(_buffer) & 0xFFFFFF; + } + + public int ReadInt24(long offset, bool updatePosition) + { + FillBuffer(offset, 3, updatePosition); + + return BitTools.SignExtend32(MemoryMarshal.Read(_buffer), 24); + } + public uint ReadUInt32(long offset, bool updatePosition) { FillBuffer(offset, sizeof(uint), updatePosition); @@ -120,10 +135,21 @@ namespace LibHac.IO if (updatePosition) Position = offset + destination.Length; } + public string ReadAscii(long offset, int length, bool updatePosition) + { + var bytes = new byte[length]; + _file.Read(bytes, offset); + + if (updatePosition) Position = offset + length; + return Encoding.ASCII.GetString(bytes); + } + public byte ReadUInt8(long offset) => ReadUInt8(offset, true); public sbyte ReadInt8(long offset) => ReadInt8(offset, true); public ushort ReadUInt16(long offset) => ReadUInt16(offset, true); public short ReadInt16(long offset) => ReadInt16(offset, true); + public int ReadUInt24(long offset) => ReadUInt24(offset, true); + public int ReadInt24(long offset) => ReadInt24(offset, true); public uint ReadUInt32(long offset) => ReadUInt32(offset, true); public int ReadInt32(long offset) => ReadInt32(offset, true); public ulong ReadUInt64(long offset) => ReadUInt64(offset, true); @@ -132,11 +158,14 @@ namespace LibHac.IO public double ReadDouble(long offset) => ReadDouble(offset, true); public byte[] ReadBytes(long offset, int length) => ReadBytes(offset, length, true); public void ReadBytes(Span destination, long offset) => ReadBytes(destination, offset, true); + public string ReadAscii(long offset, int length) => ReadAscii(offset, length, true); public byte ReadUInt8() => ReadUInt8(Position, true); public sbyte ReadInt8() => ReadInt8(Position, true); public ushort ReadUInt16() => ReadUInt16(Position, true); public short ReadInt16() => ReadInt16(Position, true); + public int ReadUInt24() => ReadUInt24(Position, true); + public int ReadInt24() => ReadInt24(Position, true); public uint ReadUInt32() => ReadUInt32(Position, true); public int ReadInt32() => ReadInt32(Position, true); public ulong ReadUInt64() => ReadUInt64(Position, true); @@ -145,5 +174,6 @@ namespace LibHac.IO public double ReadDouble() => ReadDouble(Position, true); public byte[] ReadBytes(int length) => ReadBytes(Position, length, true); public void ReadBytes(Span destination) => ReadBytes(destination, Position, true); + public string ReadAscii(int length) => ReadAscii(Position, length, true); } } diff --git a/src/hactoolnet/CliParser.cs b/src/hactoolnet/CliParser.cs index 1ebb1ffb..cbe21ec1 100644 --- a/src/hactoolnet/CliParser.cs +++ b/src/hactoolnet/CliParser.cs @@ -32,11 +32,13 @@ namespace hactoolnet new CliOption("debugoutdir", 1, (o, a) => o.DebugOutDir = a[0]), new CliOption("savedir", 1, (o, a) => o.SaveOutDir = a[0]), new CliOption("outdir", 1, (o, a) => o.OutDir = a[0]), + new CliOption("outfile", 1, (o, a) => o.OutFile = a[0]), new CliOption("plaintext", 1, (o, a) => o.PlaintextOut = a[0]), new CliOption("nspout", 1, (o, a) => o.NspOut = a[0]), new CliOption("sdseed", 1, (o, a) => o.SdSeed = a[0]), new CliOption("sdpath", 1, (o, a) => o.SdPath = a[0]), new CliOption("basenca", 1, (o, a) => o.BaseNca = a[0]), + new CliOption("basefile", 1, (o, a) => o.BaseFile = a[0]), new CliOption("rootdir", 1, (o, a) => o.RootDir = a[0]), new CliOption("updatedir", 1, (o, a) => o.UpdateDir = a[0]), new CliOption("normaldir", 1, (o, a) => o.NormalDir = a[0]), @@ -159,9 +161,10 @@ namespace hactoolnet sb.AppendLine(" -y, --verify Verify all hashes in the input file."); sb.AppendLine(" -h, --enablehash Enable hash checks when reading the input file."); sb.AppendLine(" -k, --keyset Load keys from an external file."); - sb.AppendLine(" -t, --intype=type Specify input file type [nca, xci, romfs, pk11, pk21, ini1, kip1, switchfs, save, keygen]"); + sb.AppendLine(" -t, --intype=type Specify input file type [nca, xci, romfs, pk11, pk21, ini1, kip1, switchfs, save, ndv0 keygen]"); sb.AppendLine(" --titlekeys Load title keys from an external file."); sb.AppendLine("NCA options:"); + sb.AppendLine(" --plaintext Specify file path for saving a decrypted copy of the NCA."); sb.AppendLine(" --section0 Specify Section 0 file path."); sb.AppendLine(" --section1 Specify Section 1 file path."); sb.AppendLine(" --section2 Specify Section 2 file path."); @@ -216,6 +219,10 @@ namespace hactoolnet sb.AppendLine(" --sign Sign the save file. (Requires device_key in key file)"); sb.AppendLine(" --listfiles List files in save file."); sb.AppendLine(" --replacefile Replaces a file in the save data"); + sb.AppendLine("NDV0 (Delta) options:"); + sb.AppendLine(" Input delta patch can be a delta NCA file or a delta fragment file."); + sb.AppendLine(" --basefile Specify base file path."); + sb.AppendLine(" --outfile Specify patched file path."); sb.AppendLine("Keygen options:"); sb.AppendLine(" --outdir Specify directory path to save key files to."); diff --git a/src/hactoolnet/Options.cs b/src/hactoolnet/Options.cs index b7e3d697..f5f00e41 100644 --- a/src/hactoolnet/Options.cs +++ b/src/hactoolnet/Options.cs @@ -23,11 +23,13 @@ namespace hactoolnet public string DebugOutDir; public string SaveOutDir; public string OutDir; + public string OutFile; public string PlaintextOut; public string SdSeed; public string NspOut; public string SdPath; public string BaseNca; + public string BaseFile; public string RootDir; public string UpdateDir; public string NormalDir; @@ -70,6 +72,7 @@ namespace hactoolnet Pk21, Kip1, Ini1, + Ndv0, Bench } diff --git a/src/hactoolnet/ProcessDelta.cs b/src/hactoolnet/ProcessDelta.cs new file mode 100644 index 00000000..56ad4e51 --- /dev/null +++ b/src/hactoolnet/ProcessDelta.cs @@ -0,0 +1,81 @@ +using System; +using System.IO; +using System.Runtime.InteropServices; +using System.Text; +using LibHac; +using LibHac.IO; +using static hactoolnet.Print; + +namespace hactoolnet +{ + internal static class ProcessDelta + { + private const uint Ndv0Magic = 0x3056444E; + private const string FragmentFileName = "fragment"; + + public static void Process(Context ctx) + { + using (var deltaFile = new StreamStorage(new FileStream(ctx.Options.InFile, FileMode.Open, FileAccess.Read), false)) + { + + IStorage deltaStorage = deltaFile; + Span magic = stackalloc byte[4]; + deltaFile.Read(magic, 0); + + if (MemoryMarshal.Read(magic) != Ndv0Magic) + { + try + { + var nca = new Nca(ctx.Keyset, deltaStorage, true); + IFileSystem fs = nca.OpenSectionFileSystem(0, IntegrityCheckLevel.ErrorOnInvalid); + + if (!fs.FileExists(FragmentFileName)) + { + throw new FileNotFoundException("Specified NCA does not contain a delta fragment"); + } + + deltaStorage = new FileStorage(fs.OpenFile(FragmentFileName, OpenMode.Read)); + } + catch (InvalidDataException) { } // Ignore non-NCA3 files + } + + var delta = new DeltaFragment(deltaStorage); + + if (ctx.Options.BaseFile != null) + { + using (var baseFile = new StreamStorage(new FileStream(ctx.Options.BaseFile, FileMode.Open, FileAccess.Read), false)) + { + delta.SetBaseStorage(baseFile); + + if (ctx.Options.OutFile != null) + { + using (var outFile = new FileStream(ctx.Options.OutFile, FileMode.OpenOrCreate, FileAccess.ReadWrite)) + { + IStorage patchedStorage = delta.GetPatchedStorage(); + patchedStorage.CopyToStream(outFile, patchedStorage.Length, ctx.Logger); + } + } + } + } + + ctx.Logger.LogMessage(delta.Print()); + } + } + + private static string Print(this DeltaFragment delta) + { + int colLen = 36; + var sb = new StringBuilder(); + sb.AppendLine(); + + sb.AppendLine("Delta Fragment:"); + PrintItem(sb, colLen, "Magic:", delta.Header.Magic); + PrintItem(sb, colLen, "Base file size:", $"0x{delta.Header.OriginalSize:x12}"); + PrintItem(sb, colLen, "New file size:", $"0x{delta.Header.NewSize:x12}"); + PrintItem(sb, colLen, "Fragment header size:", $"0x{delta.Header.FragmentHeaderSize:x12}"); + PrintItem(sb, colLen, "Fragment body size:", $"0x{delta.Header.FragmentBodySize:x12}"); + + return sb.ToString(); + } + } +} diff --git a/src/hactoolnet/Program.cs b/src/hactoolnet/Program.cs index 51b62f7d..05623145 100644 --- a/src/hactoolnet/Program.cs +++ b/src/hactoolnet/Program.cs @@ -90,6 +90,9 @@ namespace hactoolnet case FileType.Ini1: ProcessKip.ProcessIni1(ctx); break; + case FileType.Ndv0: + ProcessDelta.Process(ctx); + break; case FileType.Bench: ProcessBench.Process(ctx); break;