diff --git a/build/_build.csproj.DotSettings b/build/_build.csproj.DotSettings deleted file mode 100644 index 9aac7d8e..00000000 --- a/build/_build.csproj.DotSettings +++ /dev/null @@ -1,24 +0,0 @@ - - False - Implicit - Implicit - ExpressionBody - 0 - NEXT_LINE - True - False - 120 - IF_OWNER_IS_SINGLE_LINE - WRAP_IF_LONG - False - <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> - <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> - True - True - True - True - True - True - True - True - True diff --git a/src/LibHac/IO/HierarchicalIntegrityVerificationStorage.cs b/src/LibHac/IO/HierarchicalIntegrityVerificationStorage.cs index d143eae8..fc3e5895 100644 --- a/src/LibHac/IO/HierarchicalIntegrityVerificationStorage.cs +++ b/src/LibHac/IO/HierarchicalIntegrityVerificationStorage.cs @@ -149,6 +149,14 @@ namespace LibHac.IO return result; } + public void FsTrim() + { + foreach (IntegrityVerificationStorage level in IntegrityStorages) + { + level.FsTrim(); + } + } + private static readonly string[] SaltSources = { "HierarchicalIntegrityVerificationStorage::Master", diff --git a/src/LibHac/IO/IntegrityVerificationStorage.cs b/src/LibHac/IO/IntegrityVerificationStorage.cs index cdf15f13..d5e41cee 100644 --- a/src/LibHac/IO/IntegrityVerificationStorage.cs +++ b/src/LibHac/IO/IntegrityVerificationStorage.cs @@ -2,6 +2,7 @@ using System.Buffers; using System.IO; using System.Security.Cryptography; +using LibHac.IO.Save; namespace LibHac.IO { @@ -171,7 +172,7 @@ namespace LibHac.IO if (Type == IntegrityStorageType.Save) { // This bit is set on all save hashes - hash[0x1F] |= 0x80; + hash[0x1F] |= 0b10000000; } return hash; @@ -183,6 +184,24 @@ namespace LibHac.IO HashStorage.Flush(); base.Flush(); } + + public void FsTrim() + { + if (Type != IntegrityStorageType.Save) return; + + Span digest = stackalloc byte[DigestSize]; + + for (int i = 0; i < SectorCount; i++) + { + long hashPos = i * DigestSize; + HashStorage.Read(digest, hashPos); + + if (!Util.IsEmpty(digest)) continue; + + int dataOffset = i * SectorSize; + BaseStorage.Fill(SaveDataFileSystem.TrimFillValue, dataOffset, SectorSize); + } + } } /// diff --git a/src/LibHac/IO/Save/AllocationTable.cs b/src/LibHac/IO/Save/AllocationTable.cs index 045ae2f4..b12797cf 100644 --- a/src/LibHac/IO/Save/AllocationTable.cs +++ b/src/LibHac/IO/Save/AllocationTable.cs @@ -312,7 +312,7 @@ namespace LibHac.IO.Save int fillOffset = BlockToEntryIndex(index + 2) * EntrySize; int fillLength = (length - 3) * EntrySize; - BaseStorage.Slice(fillOffset, fillLength).Fill(0x00); + BaseStorage.Slice(fillOffset, fillLength).Fill(SaveDataFileSystem.TrimFillValue); } nodesIterated++; @@ -329,7 +329,7 @@ namespace LibHac.IO.Save public void FsTrim() { int tableSize = BlockToEntryIndex(Header.AllocationTableBlockCount) * EntrySize; - BaseStorage.Slice(tableSize).Fill(0x00); + BaseStorage.Slice(tableSize).Fill(SaveDataFileSystem.TrimFillValue); } private void ReadEntries(int entryIndex, Span entries) diff --git a/src/LibHac/IO/Save/DuplexStorage.cs b/src/LibHac/IO/Save/DuplexStorage.cs index 2b241ded..2f0e8968 100644 --- a/src/LibHac/IO/Save/DuplexStorage.cs +++ b/src/LibHac/IO/Save/DuplexStorage.cs @@ -77,5 +77,17 @@ namespace LibHac.IO.Save } public override long GetSize() => _length; + + public void FsTrim() + { + int blockCount = (int)(DataA.GetSize() / BlockSize); + + for (int i = 0; i < blockCount; i++) + { + IStorage dataToClear = Bitmap.Bitmap[i] ? DataA : DataB; + + dataToClear.Slice(i * BlockSize, BlockSize).Fill(SaveDataFileSystem.TrimFillValue); + } + } } } diff --git a/src/LibHac/IO/Save/HierarchicalDuplexStorage.cs b/src/LibHac/IO/Save/HierarchicalDuplexStorage.cs index 1a32bd0b..d1230fcd 100644 --- a/src/LibHac/IO/Save/HierarchicalDuplexStorage.cs +++ b/src/LibHac/IO/Save/HierarchicalDuplexStorage.cs @@ -48,6 +48,14 @@ namespace LibHac.IO.Save } public override long GetSize() => _length; + + public void FsTrim() + { + foreach (DuplexStorage layer in Layers) + { + layer.FsTrim(); + } + } } public class DuplexFsLayerInfo diff --git a/src/LibHac/IO/Save/HierarchicalSaveFileTable.cs b/src/LibHac/IO/Save/HierarchicalSaveFileTable.cs index 4c45b465..13064da8 100644 --- a/src/LibHac/IO/Save/HierarchicalSaveFileTable.cs +++ b/src/LibHac/IO/Save/HierarchicalSaveFileTable.cs @@ -235,6 +235,12 @@ namespace LibHac.IO.Save return true; } + public void TrimFreeEntries() + { + DirectoryTable.TrimFreeEntries(); + FileTable.TrimFreeEntries(); + } + [StructLayout(LayoutKind.Sequential, Pack = 1)] private struct TableEntry where T : struct { diff --git a/src/LibHac/IO/Save/JournalMap.cs b/src/LibHac/IO/Save/JournalMap.cs index 8acf3500..fc89bcdc 100644 --- a/src/LibHac/IO/Save/JournalMap.cs +++ b/src/LibHac/IO/Save/JournalMap.cs @@ -4,6 +4,7 @@ namespace LibHac.IO.Save { public class JournalMap { + private int MapEntryLength = 8; public JournalMapHeader Header { get; } private JournalMapEntry[] Entries { get; } @@ -55,6 +56,21 @@ namespace LibHac.IO.Save public IStorage GetModifiedPhysicalBlocksStorage() => ModifiedPhysicalBlocks.AsReadOnly(); public IStorage GetModifiedVirtualBlocksStorage() => ModifiedVirtualBlocks.AsReadOnly(); public IStorage GetFreeBlocksStorage() => FreeBlocks.AsReadOnly(); + + public void FsTrim() + { + int virtualBlockCount = Header.MainDataBlockCount; + int physicalBlockCount = virtualBlockCount + Header.JournalBlockCount; + + int blockMapLength = virtualBlockCount * MapEntryLength; + int physicalBitmapLength = Util.AlignUp(physicalBlockCount, 32) / 8; + int virtualBitmapLength = Util.AlignUp(virtualBlockCount, 32) / 8; + + MapStorage.Slice(blockMapLength).Fill(SaveDataFileSystem.TrimFillValue); + FreeBlocks.Slice(physicalBitmapLength).Fill(SaveDataFileSystem.TrimFillValue); + ModifiedPhysicalBlocks.Slice(physicalBitmapLength).Fill(SaveDataFileSystem.TrimFillValue); + ModifiedVirtualBlocks.Slice(virtualBitmapLength).Fill(SaveDataFileSystem.TrimFillValue); + } } public class JournalMapHeader diff --git a/src/LibHac/IO/Save/JournalStorage.cs b/src/LibHac/IO/Save/JournalStorage.cs index 5c871d89..a29fdc26 100644 --- a/src/LibHac/IO/Save/JournalStorage.cs +++ b/src/LibHac/IO/Save/JournalStorage.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.IO; namespace LibHac.IO.Save @@ -85,6 +86,22 @@ namespace LibHac.IO.Save public IStorage GetBaseStorage() => BaseStorage.AsReadOnly(); public IStorage GetHeaderStorage() => HeaderStorage.AsReadOnly(); + + public void FsTrim() + { + // todo replace with a bitmap reader class when added + BitArray bitmap = new DuplexBitmap(Map.GetFreeBlocksStorage(), + Map.Header.JournalBlockCount + Map.Header.MainDataBlockCount).Bitmap; + + for (int i = 0; i < bitmap.Length; i++) + { + if (!bitmap[i]) continue; + + BaseStorage.Fill(SaveDataFileSystem.TrimFillValue, i * BlockSize, BlockSize); + } + + Map.FsTrim(); + } } public class JournalHeader diff --git a/src/LibHac/IO/Save/RemapStorage.cs b/src/LibHac/IO/Save/RemapStorage.cs index 3661f1e7..12374c3d 100644 --- a/src/LibHac/IO/Save/RemapStorage.cs +++ b/src/LibHac/IO/Save/RemapStorage.cs @@ -1,11 +1,14 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; namespace LibHac.IO.Save { public class RemapStorage : StorageBase { + private const int MapEntryLength = 0x20; + private IStorage BaseStorage { get; } private IStorage HeaderStorage { get; } private IStorage MapEntryStorage { get; } @@ -45,6 +48,8 @@ namespace LibHac.IO.Save protected override void ReadImpl(Span destination, long offset) { + if (destination.Length == 0) return; + MapEntry entry = GetMapEntry(offset); long inPos = offset; @@ -71,6 +76,8 @@ namespace LibHac.IO.Save protected override void WriteImpl(ReadOnlySpan source, long offset) { + if (source.Length == 0) return; + MapEntry entry = GetMapEntry(offset); long inPos = offset; @@ -178,6 +185,15 @@ namespace LibHac.IO.Save { return ~GetOffsetMask(); } + + public void FsTrim() + { + int mapEntriesLength = Header.MapEntryCount * MapEntryLength; + long dataEnd = MapEntries.Max(x => x.PhysicalOffsetEnd); + + MapEntryStorage.Slice(mapEntriesLength).Fill(SaveDataFileSystem.TrimFillValue); + BaseStorage.Slice(dataEnd).Fill(SaveDataFileSystem.TrimFillValue); + } } public class RemapHeader diff --git a/src/LibHac/IO/Save/SaveDataFileSystem.cs b/src/LibHac/IO/Save/SaveDataFileSystem.cs index a10762d3..81543067 100644 --- a/src/LibHac/IO/Save/SaveDataFileSystem.cs +++ b/src/LibHac/IO/Save/SaveDataFileSystem.cs @@ -5,7 +5,11 @@ namespace LibHac.IO.Save { public class SaveDataFileSystem : IFileSystem { + internal const byte TrimFillValue = 0; + public Header Header { get; } + private bool IsFirstHeaderInUse { get; } + public IStorage BaseStorage { get; } public bool LeaveOpen { get; } @@ -17,7 +21,7 @@ namespace LibHac.IO.Save public HierarchicalDuplexStorage DuplexStorage { get; } public JournalStorage JournalStorage { get; } - public HierarchicalIntegrityVerificationStorage JournalIvfcStorage { get; } + public HierarchicalIntegrityVerificationStorage CoreDataIvfcStorage { get; } public HierarchicalIntegrityVerificationStorage FatIvfcStorage { get; } private Keyset Keyset { get; } @@ -28,7 +32,24 @@ namespace LibHac.IO.Save LeaveOpen = leaveOpen; Keyset = keyset; - Header = new Header(BaseStorage, keyset); + var headerA = new Header(BaseStorage, keyset); + var headerB = new Header(BaseStorage.Slice(0x4000), keyset); + + if (headerA.HeaderHashValidity == Validity.Valid) + { + IsFirstHeaderInUse = true; + } + else if (headerB.HeaderHashValidity == Validity.Valid) + { + IsFirstHeaderInUse = false; + } + else + { + throw new InvalidDataException("Savedata header is not valid."); + } + + Header = IsFirstHeaderInUse ? headerA : headerB; + FsLayout layout = Header.Layout; IStorage dataRemapBase = BaseStorage.Slice(layout.FileMapDataOffset, layout.FileMapDataSize); @@ -54,7 +75,7 @@ namespace LibHac.IO.Save JournalStorage = new JournalStorage(journalData, Header.JournalHeader, journalMapInfo, leaveOpen); - JournalIvfcStorage = InitJournalIvfcStorage(integrityCheckLevel); + CoreDataIvfcStorage = InitJournalIvfcStorage(integrityCheckLevel); IStorage fatStorage = MetaRemapStorage.Slice(layout.FatOffset, layout.FatSize); @@ -64,7 +85,7 @@ namespace LibHac.IO.Save fatStorage = FatIvfcStorage; } - SaveDataFileSystemCore = new SaveDataFileSystemCore(JournalIvfcStorage, fatStorage, Header.SaveHeader); + SaveDataFileSystemCore = new SaveDataFileSystemCore(CoreDataIvfcStorage, fatStorage, Header.SaveHeader); } private static HierarchicalDuplexStorage InitDuplexStorage(IStorage baseStorage, Header header) @@ -175,8 +196,8 @@ namespace LibHac.IO.Save public bool Commit(Keyset keyset) { - JournalIvfcStorage.Flush(); - FatIvfcStorage.Flush(); + CoreDataIvfcStorage.Flush(); + FatIvfcStorage?.Flush(); Stream headerStream = BaseStorage.AsStream(); @@ -208,15 +229,24 @@ namespace LibHac.IO.Save public void FsTrim() { + MetaRemapStorage.FsTrim(); + DataRemapStorage.FsTrim(); + DuplexStorage.FsTrim(); + JournalStorage.FsTrim(); + CoreDataIvfcStorage.FsTrim(); + FatIvfcStorage?.FsTrim(); SaveDataFileSystemCore.FsTrim(); + + int unusedHeaderOffset = IsFirstHeaderInUse ? 0x4000 : 0; + BaseStorage.Slice(unusedHeaderOffset, 0x4000).Fill(TrimFillValue); } public Validity Verify(IProgressReport logger = null) { - Validity journalValidity = JournalIvfcStorage.Validate(true, logger); - JournalIvfcStorage.SetLevelValidities(Header.Ivfc); + Validity journalValidity = CoreDataIvfcStorage.Validate(true, logger); + CoreDataIvfcStorage.SetLevelValidities(Header.Ivfc); - if (FatIvfcStorage == null)return journalValidity; + if (FatIvfcStorage == null) return journalValidity; Validity fatValidity = FatIvfcStorage.Validate(true, logger); FatIvfcStorage.SetLevelValidities(Header.Ivfc); diff --git a/src/LibHac/IO/Save/SaveDataFileSystemCore.cs b/src/LibHac/IO/Save/SaveDataFileSystemCore.cs index d902e08f..6666315d 100644 --- a/src/LibHac/IO/Save/SaveDataFileSystemCore.cs +++ b/src/LibHac/IO/Save/SaveDataFileSystemCore.cs @@ -143,7 +143,7 @@ namespace LibHac.IO.Save { AllocationTable.FsTrimList(fileInfo.StartBlock); - OpenFatStorage(fileInfo.StartBlock).Slice(fileInfo.Length).Fill(0); + OpenFatStorage(fileInfo.StartBlock).Slice(fileInfo.Length).Fill(SaveDataFileSystem.TrimFillValue); } } @@ -152,7 +152,9 @@ namespace LibHac.IO.Save AllocationTable.FsTrimList(freeIndex); - OpenFatStorage(freeIndex).Fill(0); + OpenFatStorage(freeIndex).Fill(SaveDataFileSystem.TrimFillValue); + + FileTable.TrimFreeEntries(); } private AllocationTableStorage OpenFatStorage(int blockIndex) diff --git a/src/LibHac/IO/Save/SaveFsList.cs b/src/LibHac/IO/Save/SaveFsList.cs index 726c5ac2..6b203842 100644 --- a/src/LibHac/IO/Save/SaveFsList.cs +++ b/src/LibHac/IO/Save/SaveFsList.cs @@ -226,6 +226,30 @@ namespace LibHac.IO.Save Free(index); } + public void TrimFreeEntries() + { + Span entryBytes = stackalloc byte[_sizeOfEntry]; + Span name = entryBytes.Slice(4, MaxNameLength); + ref SaveFsEntry entry = ref GetEntryFromBytes(entryBytes); + + ReadEntry(FreeListHeadIndex, out entry); + + int index = entry.Next; + + while (entry.Next > 0) + { + ReadEntry(index, out entry); + + entry.Parent = 0; + entry.Value = default; + name.Fill(SaveDataFileSystem.TrimFillValue); + + WriteEntry(index, ref entry); + + index = entry.Next; + } + } + private int GetListCapacity() { Span buf = stackalloc byte[sizeof(int)]; diff --git a/src/LibHac/IO/StorageBase.cs b/src/LibHac/IO/StorageBase.cs index c423cc89..04195b3b 100644 --- a/src/LibHac/IO/StorageBase.cs +++ b/src/LibHac/IO/StorageBase.cs @@ -56,8 +56,8 @@ namespace LibHac.IO protected void ValidateParameters(ReadOnlySpan span, long offset) { if (_isDisposed) throw new ObjectDisposedException(null); - if (span == null) throw new ArgumentNullException(nameof(span)); if (offset < 0) throw new ArgumentOutOfRangeException(nameof(offset), "Argument must be non-negative."); + long length = GetSize(); if (length != -1 && !CanAutoExpand) diff --git a/src/LibHac/IO/StorageExtensions.cs b/src/LibHac/IO/StorageExtensions.cs index 9be88414..3adae66e 100644 --- a/src/LibHac/IO/StorageExtensions.cs +++ b/src/LibHac/IO/StorageExtensions.cs @@ -98,31 +98,35 @@ namespace LibHac.IO } public static void Fill(this IStorage input, byte value, IProgressReport progress = null) + { + input.Fill(value, 0, input.GetSize(), progress); + } + + public static void Fill(this IStorage input, byte value, long offset, long count, IProgressReport progress = null) { const int threshold = 0x400; - long length = input.GetSize(); - if (length > threshold) + if (count > threshold) { - input.FillLarge(value, progress); + input.FillLarge(value, offset, count, progress); return; } - Span buf = stackalloc byte[(int)length]; + Span buf = stackalloc byte[(int)count]; buf.Fill(value); - input.Write(buf, 0); + input.Write(buf, offset); } - private static void FillLarge(this IStorage input, byte value, IProgressReport progress = null) + private static void FillLarge(this IStorage input, byte value, long offset, long count, IProgressReport progress = null) { const int bufferSize = 0x4000; - long remaining = input.GetSize(); + long remaining = count; if (remaining < 0) throw new ArgumentException("Storage must have an explicit length"); progress?.SetTotal(remaining); - long pos = 0; + long pos = offset; byte[] buffer = ArrayPool.Shared.Rent(bufferSize); try diff --git a/src/LibHac/Util.cs b/src/LibHac/Util.cs index f66486da..71de4130 100644 --- a/src/LibHac/Util.cs +++ b/src/LibHac/Util.cs @@ -134,8 +134,6 @@ namespace LibHac public static bool IsEmpty(this ReadOnlySpan span) { - if (span == null) throw new ArgumentNullException(nameof(span)); - for (int i = 0; i < span.Length; i++) { if (span[i] != 0)