From 1a7e452a89350edc76a9136abb5116cde381a576 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Sat, 1 Sep 2018 14:23:45 -0500 Subject: [PATCH] Add Duplex FS handler for save files --- LibHac/Savefile/DuplexBitmap.cs | 41 +++++++++++++ LibHac/Savefile/DuplexFs.cs | 94 ++++++++++++++++++++++++++++++ LibHac/Savefile/Header.cs | 48 +++++++++++++-- LibHac/Savefile/LayeredDuplexFs.cs | 75 ++++++++++++++++++++++++ LibHac/Savefile/Savefile.cs | 28 ++++++++- hactoolnet/CliParser.cs | 2 +- hactoolnet/Program.cs | 37 ++++++------ 7 files changed, 300 insertions(+), 25 deletions(-) create mode 100644 LibHac/Savefile/DuplexBitmap.cs create mode 100644 LibHac/Savefile/DuplexFs.cs create mode 100644 LibHac/Savefile/LayeredDuplexFs.cs diff --git a/LibHac/Savefile/DuplexBitmap.cs b/LibHac/Savefile/DuplexBitmap.cs new file mode 100644 index 00000000..03fc2f54 --- /dev/null +++ b/LibHac/Savefile/DuplexBitmap.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections; +using System.IO; + +namespace LibHac.Savefile +{ + public class DuplexBitmap + { + private Stream Data { get; } + public BitArray Bitmap { get; } + + public DuplexBitmap(Stream bitmapStream, int lengthBits) + { + Data = bitmapStream; + Bitmap = new BitArray(lengthBits); + ReadBitmap(lengthBits); + } + + private void ReadBitmap(int lengthBits) + { + uint mask = unchecked((uint)(1 << 31)); + var reader = new BinaryReader(Data); + int bitsRemaining = lengthBits; + int bitmapPos = 0; + + while (bitsRemaining > 0) + { + int bitsToRead = Math.Min(bitsRemaining, 32); + uint val = reader.ReadUInt32(); + + for (int i = 0; i < bitsToRead; i++) + { + Bitmap[bitmapPos] = (val & mask) != 0; + bitmapPos++; + bitsRemaining--; + val <<= 1; + } + } + } + } +} diff --git a/LibHac/Savefile/DuplexFs.cs b/LibHac/Savefile/DuplexFs.cs new file mode 100644 index 00000000..70ae4fb3 --- /dev/null +++ b/LibHac/Savefile/DuplexFs.cs @@ -0,0 +1,94 @@ +using System; +using System.IO; + +namespace LibHac.Savefile +{ + public class DuplexFs : Stream + { + private int BlockSize{ get; } + private Stream BitmapStream { get; } + private Stream DataA { get; } + private Stream DataB { get; } + private DuplexBitmap Bitmap { get; } + + public DuplexFs(Stream bitmap, Stream dataA, Stream dataB, int blockSize) + { + if (dataA.Length != dataB.Length) + { + throw new InvalidDataException("Both data streams must be the same length"); + } + + BlockSize = blockSize; + BitmapStream = bitmap; + DataA = dataA; + DataB = dataB; + Bitmap = new DuplexBitmap(BitmapStream, (int)(bitmap.Length * 8)); + Length = dataA.Length; + } + + public override void Flush() + { + throw new NotImplementedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + long remaining = Math.Min(count, Length - Position); + if (remaining <= 0) return 0; + int outOffset = offset; + int totalBytesRead = 0; + + while (remaining > 0) + { + int blockNum = (int)(Position / BlockSize); + int blockPos = (int)(Position % BlockSize); + int bytesToRead = (int)Math.Min(remaining, BlockSize - blockPos); + + var data = Bitmap.Bitmap[blockNum] ? DataB : DataA; + data.Position = blockNum * BlockSize + blockPos; + + data.Read(buffer, outOffset, bytesToRead); + outOffset += bytesToRead; + totalBytesRead += bytesToRead; + remaining -= bytesToRead; + Position += bytesToRead; + } + + return totalBytesRead; + } + + public override long Seek(long offset, SeekOrigin origin) + { + switch (origin) + { + case SeekOrigin.Begin: + Position = offset; + break; + case SeekOrigin.Current: + Position += offset; + break; + case SeekOrigin.End: + Position = Length - offset; + break; + } + + return Position; + } + + public override void SetLength(long value) + { + throw new NotImplementedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotImplementedException(); + } + + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => false; + public override long Length { get; } + public override long Position { get; set; } + } +} diff --git a/LibHac/Savefile/Header.cs b/LibHac/Savefile/Header.cs index 03fe0f99..77fcd75e 100644 --- a/LibHac/Savefile/Header.cs +++ b/LibHac/Savefile/Header.cs @@ -8,6 +8,7 @@ namespace LibHac.Savefile public byte[] Cmac { get; set; } public FsLayout Layout { get; set; } public JournalHeader Journal { get; set; } + public DuplexHeader Duplex { get; set; } public SaveHeader Save { get; set; } public RemapHeader FileRemap { get; set; } @@ -34,6 +35,9 @@ namespace LibHac.Savefile reader.BaseStream.Position = 0x100; Layout = new FsLayout(reader); + reader.BaseStream.Position = 0x300; + Duplex = new DuplexHeader(reader); + reader.BaseStream.Position = 0x408; Journal = new JournalHeader(reader); @@ -46,14 +50,14 @@ namespace LibHac.Savefile MetaRemap = new RemapHeader(reader); reader.BaseStream.Position = Layout.MasterHashOffset0; - MasterHashA = reader.ReadBytes((int) Layout.MasterHashSize); + MasterHashA = reader.ReadBytes((int)Layout.MasterHashSize); reader.BaseStream.Position = Layout.MasterHashOffset1; - MasterHashB = reader.ReadBytes((int) Layout.MasterHashSize); + MasterHashB = reader.ReadBytes((int)Layout.MasterHashSize); reader.BaseStream.Position = Layout.L1BitmapOffset0; - DuplexMasterA = reader.ReadBytes((int) Layout.L1BitmapSize); + DuplexMasterA = reader.ReadBytes((int)Layout.L1BitmapSize); reader.BaseStream.Position = Layout.L1BitmapOffset1; - DuplexMasterB = reader.ReadBytes((int) Layout.L1BitmapSize); + DuplexMasterB = reader.ReadBytes((int)Layout.L1BitmapSize); reader.BaseStream.Position = Layout.FileMapEntryOffset; FileMapEntries = new MapEntry[FileRemap.MapEntryCount]; @@ -193,6 +197,42 @@ namespace LibHac.Savefile } } + public class DuplexHeader + { + public string Magic { get; } + public uint MagicNum { get; } + public DuplexInfo[] Layers { get; } = new DuplexInfo[3]; + + public DuplexHeader(BinaryReader reader) + { + Magic = reader.ReadAscii(4); + MagicNum = reader.ReadUInt32(); + + for (int i = 0; i < Layers.Length; i++) + { + Layers[i] = new DuplexInfo(reader); + } + } + } + + public class DuplexInfo + { + public long Offset { get; } + public long Length { get; set; } + public int BlockSizePower { get; set; } + public int BlockSize { get; set; } + + public DuplexInfo() { } + + public DuplexInfo(BinaryReader reader) + { + Offset = reader.ReadInt64(); + Length = reader.ReadInt64(); + BlockSizePower = reader.ReadInt32(); + BlockSize = 1 << BlockSizePower; + } + } + public class JournalHeader { public string Magic { get; } diff --git a/LibHac/Savefile/LayeredDuplexFs.cs b/LibHac/Savefile/LayeredDuplexFs.cs new file mode 100644 index 00000000..124082f1 --- /dev/null +++ b/LibHac/Savefile/LayeredDuplexFs.cs @@ -0,0 +1,75 @@ +using System.IO; + +namespace LibHac.Savefile +{ + public class LayeredDuplexFs : Stream + { + private DuplexFs[] Layers { get; } + private DuplexFs DataLayer { get; } + + public LayeredDuplexFs(DuplexFsLayerInfo[] layers, bool masterBit) + { + Layers = new DuplexFs[layers.Length - 1]; + + for (int i = 0; i < Layers.Length; i++) + { + Stream bitmap; + + if (i == 0) + { + bitmap = masterBit ? layers[0].DataB : layers[0].DataA; + } + else + { + bitmap = Layers[i - 1]; + } + + Layers[i] = new DuplexFs(bitmap, layers[i + 1].DataA, layers[i + 1].DataB, layers[i + 1].Info.BlockSize); + } + + DataLayer = Layers[Layers.Length - 1]; + } + + public override void Flush() + { + throw new System.NotImplementedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + return DataLayer.Read(buffer, offset, count); + } + + public override long Seek(long offset, SeekOrigin origin) + { + return DataLayer.Seek(offset, origin); + } + + public override void SetLength(long value) + { + throw new System.NotImplementedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new System.NotImplementedException(); + } + + public override bool CanRead => DataLayer.CanRead; + public override bool CanSeek => DataLayer.CanSeek; + public override bool CanWrite => DataLayer.CanWrite; + public override long Length => DataLayer.Length; + public override long Position + { + get => DataLayer.Position; + set => DataLayer.Position = value; + } + } + + public class DuplexFsLayerInfo + { + public Stream DataA { get; set; } + public Stream DataB { get; set; } + public DuplexInfo Info { get; set; } + } +} diff --git a/LibHac/Savefile/Savefile.cs b/LibHac/Savefile/Savefile.cs index 8d7e062e..82072ff9 100644 --- a/LibHac/Savefile/Savefile.cs +++ b/LibHac/Savefile/Savefile.cs @@ -20,6 +20,7 @@ namespace LibHac.Savefile public Stream DuplexL1B { get; } public Stream DuplexDataA { get; } public Stream DuplexDataB { get; } + public LayeredDuplexFs DuplexData { get; } public Stream JournalData { get; } public Stream JournalTable { get; } @@ -46,14 +47,37 @@ namespace LibHac.Savefile FileRemapSource = new SharedStreamSource(FileRemap); + var duplexLayers = new DuplexFsLayerInfo[3]; + + duplexLayers[0] = new DuplexFsLayerInfo + { + DataA = new MemoryStream(Header.DuplexMasterA), + DataB = new MemoryStream(Header.DuplexMasterB), + Info = Header.Duplex.Layers[0] + }; + + duplexLayers[1] = new DuplexFsLayerInfo + { + DataA = FileRemapSource.CreateStream(layout.DuplexL1OffsetA, layout.DuplexL1Size), + DataB = FileRemapSource.CreateStream(layout.DuplexL1OffsetB, layout.DuplexL1Size), + Info = Header.Duplex.Layers[1] + }; + + duplexLayers[2] = new DuplexFsLayerInfo + { + DataA = FileRemapSource.CreateStream(layout.DuplexDataOffsetA, layout.DuplexDataSize), + DataB = FileRemapSource.CreateStream(layout.DuplexDataOffsetB, layout.DuplexDataSize), + Info = Header.Duplex.Layers[2] + }; + DuplexL1A = FileRemapSource.CreateStream(layout.DuplexL1OffsetA, layout.DuplexL1Size); DuplexL1B = FileRemapSource.CreateStream(layout.DuplexL1OffsetB, layout.DuplexL1Size); DuplexDataA = FileRemapSource.CreateStream(layout.DuplexDataOffsetA, layout.DuplexDataSize); DuplexDataB = FileRemapSource.CreateStream(layout.DuplexDataOffsetB, layout.DuplexDataSize); JournalData = FileRemapSource.CreateStream(layout.JournalDataOffset, layout.JournalDataSizeB + layout.SizeReservedArea); - var duplexData = layout.DuplexIndex == 0 ? DuplexDataA : DuplexDataB; - MetaRemap = new RemapStream(duplexData, Header.MetaMapEntries, Header.MetaRemap.MapSegmentCount); + DuplexData = new LayeredDuplexFs(duplexLayers, Header.Layout.DuplexIndex == 1); + MetaRemap = new RemapStream(DuplexData, Header.MetaMapEntries, Header.MetaRemap.MapSegmentCount); MetaRemapSource = new SharedStreamSource(MetaRemap); JournalTable = MetaRemapSource.CreateStream(layout.JournalTableOffset, layout.JournalTableSize); diff --git a/hactoolnet/CliParser.cs b/hactoolnet/CliParser.cs index 7659882e..e9bce688 100644 --- a/hactoolnet/CliParser.cs +++ b/hactoolnet/CliParser.cs @@ -95,7 +95,7 @@ namespace hactoolnet i += option.ArgsNeeded; } - if (!inputSpecified && options.InFileType != FileType.Keygen) + if (!inputSpecified && options.InFileType != FileType.Keygen && !options.RunCustom) { PrintWithUsage("Input file must be specified"); return null; diff --git a/hactoolnet/Program.cs b/hactoolnet/Program.cs index 1778bfce..567c376c 100644 --- a/hactoolnet/Program.cs +++ b/hactoolnet/Program.cs @@ -474,26 +474,27 @@ namespace hactoolnet var dir = ctx.Options.DebugOutDir; Directory.CreateDirectory(dir); - File.WriteAllBytes(Path.Combine(dir, "L1_0_MasterHashA"), save.Header.MasterHashA); - File.WriteAllBytes(Path.Combine(dir, "L1_1_MasterHashB"), save.Header.MasterHashB); - File.WriteAllBytes(Path.Combine(dir, "L1_2_DuplexMasterA"), save.Header.DuplexMasterA); - File.WriteAllBytes(Path.Combine(dir, "L1_3_DuplexMasterB"), save.Header.DuplexMasterB); - save.DuplexL1A.WriteAllBytes(Path.Combine(dir, "L0_0_DuplexL1A"), ctx.Logger); - save.DuplexL1B.WriteAllBytes(Path.Combine(dir, "L0_1_DuplexL1B"), ctx.Logger); - save.DuplexDataA.WriteAllBytes(Path.Combine(dir, "L0_2_DuplexDataA"), ctx.Logger); - save.DuplexDataB.WriteAllBytes(Path.Combine(dir, "L0_3_DuplexDataB"), ctx.Logger); - save.JournalData.WriteAllBytes(Path.Combine(dir, "L0_4_JournalData"), ctx.Logger); + File.WriteAllBytes(Path.Combine(dir, "L0_0_MasterHashA"), save.Header.MasterHashA); + File.WriteAllBytes(Path.Combine(dir, "L0_1_MasterHashB"), save.Header.MasterHashB); + File.WriteAllBytes(Path.Combine(dir, "L0_2_DuplexMasterA"), save.Header.DuplexMasterA); + File.WriteAllBytes(Path.Combine(dir, "L0_3_DuplexMasterB"), save.Header.DuplexMasterB); + save.DuplexL1A.WriteAllBytes(Path.Combine(dir, "L0_4_DuplexL1A"), ctx.Logger); + save.DuplexL1B.WriteAllBytes(Path.Combine(dir, "L0_5_DuplexL1B"), ctx.Logger); + save.DuplexDataA.WriteAllBytes(Path.Combine(dir, "L0_6_DuplexDataA"), ctx.Logger); + save.DuplexDataB.WriteAllBytes(Path.Combine(dir, "L0_7_DuplexDataB"), ctx.Logger); + save.JournalData.WriteAllBytes(Path.Combine(dir, "L0_9_JournalData"), ctx.Logger); - save.JournalTable.WriteAllBytes(Path.Combine(dir, "L1_0_JournalTable"), ctx.Logger); - save.JournalBitmapUpdatedPhysical.WriteAllBytes(Path.Combine(dir, "L1_1_JournalBitmapUpdatedPhysical"), ctx.Logger); - save.JournalBitmapUpdatedVirtual.WriteAllBytes(Path.Combine(dir, "L1_2_JournalBitmapUpdatedVirtual"), ctx.Logger); - save.JournalBitmapUnassigned.WriteAllBytes(Path.Combine(dir, "L1_3_JournalBitmapUnassigned"), ctx.Logger); - save.JournalLayer1Hash.WriteAllBytes(Path.Combine(dir, "L1_4_Layer1Hash"), ctx.Logger); - save.JournalLayer2Hash.WriteAllBytes(Path.Combine(dir, "L1_5_Layer2Hash"), ctx.Logger); - save.JournalLayer3Hash.WriteAllBytes(Path.Combine(dir, "L1_6_Layer3Hash"), ctx.Logger); - save.JournalFat.WriteAllBytes(Path.Combine(dir, "L1_7_FAT"), ctx.Logger); + save.DuplexData.WriteAllBytes(Path.Combine(dir, "L1_0_DuplexData"), ctx.Logger); + save.JournalTable.WriteAllBytes(Path.Combine(dir, "L2_0_JournalTable"), ctx.Logger); + save.JournalBitmapUpdatedPhysical.WriteAllBytes(Path.Combine(dir, "L2_1_JournalBitmapUpdatedPhysical"), ctx.Logger); + save.JournalBitmapUpdatedVirtual.WriteAllBytes(Path.Combine(dir, "L2_2_JournalBitmapUpdatedVirtual"), ctx.Logger); + save.JournalBitmapUnassigned.WriteAllBytes(Path.Combine(dir, "L2_3_JournalBitmapUnassigned"), ctx.Logger); + save.JournalLayer1Hash.WriteAllBytes(Path.Combine(dir, "L2_4_Layer1Hash"), ctx.Logger); + save.JournalLayer2Hash.WriteAllBytes(Path.Combine(dir, "L2_5_Layer2Hash"), ctx.Logger); + save.JournalLayer3Hash.WriteAllBytes(Path.Combine(dir, "L2_6_Layer3Hash"), ctx.Logger); + save.JournalFat.WriteAllBytes(Path.Combine(dir, "L2_7_FAT"), ctx.Logger); - save.JournalStreamSource.CreateStream().WriteAllBytes(Path.Combine(dir, "L2_0_SaveData"), ctx.Logger); + save.JournalStreamSource.CreateStream().WriteAllBytes(Path.Combine(dir, "L3_0_SaveData"), ctx.Logger); } } }