Add fstrim capability to savedata (#54)

* Properly commit version 4 savedata. Remove null span checks

* Add FsTrim to IntegrityVerificationStorage

* Remove dotsettings file

* Add trimming to all savedata layers

build/_build.csproj.DotSettings

@@ -1,24 +0,0 @@
-<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
-	<s:Boolean x:Key="/Default/CodeInspection/ImplicitNullability/EnableFields/@EntryValue">False</s:Boolean>
-	<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpCodeStyle/DEFAULT_INTERNAL_MODIFIER/@EntryValue">Implicit</s:String>
-	<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpCodeStyle/DEFAULT_PRIVATE_MODIFIER/@EntryValue">Implicit</s:String>
-	<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpCodeStyle/METHOD_OR_OPERATOR_BODY/@EntryValue">ExpressionBody</s:String>
-	<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpCodeStyle/ThisQualifier/INSTANCE_MEMBERS_QUALIFY_MEMBERS/@EntryValue">0</s:String>
-	<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/ANONYMOUS_METHOD_DECLARATION_BRACES/@EntryValue">NEXT_LINE</s:String>
-	<s:Boolean x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/KEEP_USER_LINEBREAKS/@EntryValue">True</s:Boolean>
-	<s:Boolean x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/WRAP_AFTER_INVOCATION_LPAR/@EntryValue">False</s:Boolean>
-	<s:Int64 x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/MAX_ATTRIBUTE_LENGTH_FOR_SAME_LINE/@EntryValue">120</s:Int64>
-	<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/PLACE_FIELD_ATTRIBUTE_ON_SAME_LINE_EX/@EntryValue">IF_OWNER_IS_SINGLE_LINE</s:String>
-	<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/WRAP_ARGUMENTS_STYLE/@EntryValue">WRAP_IF_LONG</s:String>
-	<s:Boolean x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/PLACE_SIMPLE_ANONYMOUSMETHOD_ON_SINGLE_LINE/@EntryValue">False</s:Boolean>
-	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/PredefinedNamingRules/=PrivateInstanceFields/@EntryIndexedValue">&lt;Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /&gt;</s:String>
-	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/PredefinedNamingRules/=PrivateStaticFields/@EntryIndexedValue">&lt;Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /&gt;</s:String>
-	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ECSharpAttributeForSingleLineMethodUpgrade/@EntryIndexedValue">True</s:Boolean>
-	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ECSharpKeepExistingMigration/@EntryIndexedValue">True</s:Boolean>
-	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ECSharpPlaceEmbeddedOnSameLineMigration/@EntryIndexedValue">True</s:Boolean>
-	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ECSharpRenamePlacementToArrangementMigration/@EntryIndexedValue">True</s:Boolean>
-	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ECSharpUseContinuousIndentInsideBracesMigration/@EntryIndexedValue">True</s:Boolean>
-	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002EAddAccessorOwnerDeclarationBracesMigration/@EntryIndexedValue">True</s:Boolean>
-	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002ECSharpPlaceAttributeOnSameLineMigration/@EntryIndexedValue">True</s:Boolean>
-	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002EMigrateBlankLinesAroundFieldToBlankLinesAroundProperty/@EntryIndexedValue">True</s:Boolean>
-	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002EMigrateThisQualifierSettings/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

src/LibHac/IO/HierarchicalIntegrityVerificationStorage.cs

@@ -149,6 +149,14 @@ namespace LibHac.IO
             return result;
         }
 
+        public void FsTrim()
+        {
+            foreach (IntegrityVerificationStorage level in IntegrityStorages)
+            {
+                level.FsTrim();
+            }
+        }
+
         private static readonly string[] SaltSources =
         {
             "HierarchicalIntegrityVerificationStorage::Master",

src/LibHac/IO/IntegrityVerificationStorage.cs

@@ -2,6 +2,7 @@
 using System.Buffers;
 using System.IO;
 using System.Security.Cryptography;
+using LibHac.IO.Save;
 
 namespace LibHac.IO
 {
@@ -171,7 +172,7 @@ namespace LibHac.IO
                 if (Type == IntegrityStorageType.Save)
                 {
                     // This bit is set on all save hashes
-                    hash[0x1F] |= 0x80;
+                    hash[0x1F] |= 0b10000000;
                 }
 
                 return hash;
@@ -183,6 +184,24 @@ namespace LibHac.IO
             HashStorage.Flush();
             base.Flush();
         }
+
+        public void FsTrim()
+        {
+            if (Type != IntegrityStorageType.Save) return;
+
+            Span<byte> digest = stackalloc byte[DigestSize];
+
+            for (int i = 0; i < SectorCount; i++)
+            {
+                long hashPos = i * DigestSize;
+                HashStorage.Read(digest, hashPos);
+
+                if (!Util.IsEmpty(digest)) continue;
+
+                int dataOffset = i * SectorSize;
+                BaseStorage.Fill(SaveDataFileSystem.TrimFillValue, dataOffset, SectorSize);
+            }
+        }
     }
 
     /// <summary>

src/LibHac/IO/Save/AllocationTable.cs

@@ -312,7 +312,7 @@ namespace LibHac.IO.Save
                     int fillOffset = BlockToEntryIndex(index + 2) * EntrySize;
                     int fillLength = (length - 3) * EntrySize;
 
-                    BaseStorage.Slice(fillOffset, fillLength).Fill(0x00);
+                    BaseStorage.Slice(fillOffset, fillLength).Fill(SaveDataFileSystem.TrimFillValue);
                 }
 
                 nodesIterated++;
@@ -329,7 +329,7 @@ namespace LibHac.IO.Save
         public void FsTrim()
         {
             int tableSize = BlockToEntryIndex(Header.AllocationTableBlockCount) * EntrySize;
-            BaseStorage.Slice(tableSize).Fill(0x00);
+            BaseStorage.Slice(tableSize).Fill(SaveDataFileSystem.TrimFillValue);
         }
 
         private void ReadEntries(int entryIndex, Span<AllocationTableEntry> entries)

src/LibHac/IO/Save/DuplexStorage.cs

@@ -77,5 +77,17 @@ namespace LibHac.IO.Save
         }
 
         public override long GetSize() => _length;
+
+        public void FsTrim()
+        {
+            int blockCount = (int)(DataA.GetSize() / BlockSize);
+
+            for (int i = 0; i < blockCount; i++)
+            {
+                IStorage dataToClear = Bitmap.Bitmap[i] ? DataA : DataB;
+
+                dataToClear.Slice(i * BlockSize, BlockSize).Fill(SaveDataFileSystem.TrimFillValue);
+            }
+        }
     }
 }

src/LibHac/IO/Save/HierarchicalDuplexStorage.cs

@@ -48,6 +48,14 @@ namespace LibHac.IO.Save
         }
 
         public override long GetSize() => _length;
+
+        public void FsTrim()
+        {
+            foreach (DuplexStorage layer in Layers)
+            {
+                layer.FsTrim();
+            }
+        }
     }
 
     public class DuplexFsLayerInfo

src/LibHac/IO/Save/HierarchicalSaveFileTable.cs

@@ -235,6 +235,12 @@ namespace LibHac.IO.Save
             return true;
         }
 
+        public void TrimFreeEntries()
+        {
+            DirectoryTable.TrimFreeEntries();
+            FileTable.TrimFreeEntries();
+        }
+
         [StructLayout(LayoutKind.Sequential, Pack = 1)]
         private struct TableEntry<T> where T : struct
         {

src/LibHac/IO/Save/JournalMap.cs

@@ -4,6 +4,7 @@ namespace LibHac.IO.Save
 {
     public class JournalMap
     {
+        private int MapEntryLength = 8;
         public JournalMapHeader Header { get; }
         private JournalMapEntry[] Entries { get; }
 
@@ -55,6 +56,21 @@ namespace LibHac.IO.Save
         public IStorage GetModifiedPhysicalBlocksStorage() => ModifiedPhysicalBlocks.AsReadOnly();
         public IStorage GetModifiedVirtualBlocksStorage() => ModifiedVirtualBlocks.AsReadOnly();
         public IStorage GetFreeBlocksStorage() => FreeBlocks.AsReadOnly();
+
+        public void FsTrim()
+        {
+            int virtualBlockCount = Header.MainDataBlockCount;
+            int physicalBlockCount = virtualBlockCount + Header.JournalBlockCount;
+
+            int blockMapLength = virtualBlockCount * MapEntryLength;
+            int physicalBitmapLength = Util.AlignUp(physicalBlockCount, 32) / 8;
+            int virtualBitmapLength = Util.AlignUp(virtualBlockCount, 32) / 8;
+
+            MapStorage.Slice(blockMapLength).Fill(SaveDataFileSystem.TrimFillValue);
+            FreeBlocks.Slice(physicalBitmapLength).Fill(SaveDataFileSystem.TrimFillValue);
+            ModifiedPhysicalBlocks.Slice(physicalBitmapLength).Fill(SaveDataFileSystem.TrimFillValue);
+            ModifiedVirtualBlocks.Slice(virtualBitmapLength).Fill(SaveDataFileSystem.TrimFillValue);
+        }
     }
 
     public class JournalMapHeader

src/LibHac/IO/Save/JournalStorage.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections;
 using System.IO;
 
 namespace LibHac.IO.Save
@@ -85,6 +86,22 @@ namespace LibHac.IO.Save
 
         public IStorage GetBaseStorage() => BaseStorage.AsReadOnly();
         public IStorage GetHeaderStorage() => HeaderStorage.AsReadOnly();
+
+        public void FsTrim()
+        {
+            // todo replace with a bitmap reader class when added
+            BitArray bitmap = new DuplexBitmap(Map.GetFreeBlocksStorage(),
+                Map.Header.JournalBlockCount + Map.Header.MainDataBlockCount).Bitmap;
+
+            for (int i = 0; i < bitmap.Length; i++)
+            {
+                if (!bitmap[i]) continue;
+
+                BaseStorage.Fill(SaveDataFileSystem.TrimFillValue, i * BlockSize, BlockSize);
+            }
+
+            Map.FsTrim();
+        }
     }
 
     public class JournalHeader

src/LibHac/IO/Save/RemapStorage.cs

@@ -1,11 +1,14 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 
 namespace LibHac.IO.Save
 {
     public class RemapStorage : StorageBase
     {
+        private const int MapEntryLength = 0x20;
+
         private IStorage BaseStorage { get; }
         private IStorage HeaderStorage { get; }
         private IStorage MapEntryStorage { get; }
@@ -45,6 +48,8 @@ namespace LibHac.IO.Save
 
         protected override void ReadImpl(Span<byte> destination, long offset)
         {
+            if (destination.Length == 0) return;
+
             MapEntry entry = GetMapEntry(offset);
 
             long inPos = offset;
@@ -71,6 +76,8 @@ namespace LibHac.IO.Save
 
         protected override void WriteImpl(ReadOnlySpan<byte> source, long offset)
         {
+            if (source.Length == 0) return;
+
             MapEntry entry = GetMapEntry(offset);
 
             long inPos = offset;
@@ -178,6 +185,15 @@ namespace LibHac.IO.Save
         {
             return ~GetOffsetMask();
         }
+
+        public void FsTrim()
+        {
+            int mapEntriesLength = Header.MapEntryCount * MapEntryLength;
+            long dataEnd = MapEntries.Max(x => x.PhysicalOffsetEnd);
+
+            MapEntryStorage.Slice(mapEntriesLength).Fill(SaveDataFileSystem.TrimFillValue);
+            BaseStorage.Slice(dataEnd).Fill(SaveDataFileSystem.TrimFillValue);
+        }
     }
 
     public class RemapHeader

src/LibHac/IO/Save/SaveDataFileSystem.cs

@@ -5,7 +5,11 @@ namespace LibHac.IO.Save
 {
     public class SaveDataFileSystem : IFileSystem
     {
+        internal const byte TrimFillValue = 0;
+
         public Header Header { get; }
+        private bool IsFirstHeaderInUse { get; }
+
         public IStorage BaseStorage { get; }
         public bool LeaveOpen { get; }
 
@@ -17,7 +21,7 @@ namespace LibHac.IO.Save
         public HierarchicalDuplexStorage DuplexStorage { get; }
         public JournalStorage JournalStorage { get; }
 
-        public HierarchicalIntegrityVerificationStorage JournalIvfcStorage { get; }
+        public HierarchicalIntegrityVerificationStorage CoreDataIvfcStorage { get; }
         public HierarchicalIntegrityVerificationStorage FatIvfcStorage { get; }
 
         private Keyset Keyset { get; }
@@ -28,7 +32,24 @@ namespace LibHac.IO.Save
             LeaveOpen = leaveOpen;
             Keyset = keyset;
 
-            Header = new Header(BaseStorage, keyset);
+            var headerA = new Header(BaseStorage, keyset);
+            var headerB = new Header(BaseStorage.Slice(0x4000), keyset);
+
+            if (headerA.HeaderHashValidity == Validity.Valid)
+            {
+                IsFirstHeaderInUse = true;
+            }
+            else if (headerB.HeaderHashValidity == Validity.Valid)
+            {
+                IsFirstHeaderInUse = false;
+            }
+            else
+            {
+                throw new InvalidDataException("Savedata header is not valid.");
+            }
+
+            Header = IsFirstHeaderInUse ? headerA : headerB;
+
             FsLayout layout = Header.Layout;
 
             IStorage dataRemapBase = BaseStorage.Slice(layout.FileMapDataOffset, layout.FileMapDataSize);
@@ -54,7 +75,7 @@ namespace LibHac.IO.Save
 
             JournalStorage = new JournalStorage(journalData, Header.JournalHeader, journalMapInfo, leaveOpen);
 
-            JournalIvfcStorage = InitJournalIvfcStorage(integrityCheckLevel);
+            CoreDataIvfcStorage = InitJournalIvfcStorage(integrityCheckLevel);
 
             IStorage fatStorage = MetaRemapStorage.Slice(layout.FatOffset, layout.FatSize);
 
@@ -64,7 +85,7 @@ namespace LibHac.IO.Save
                 fatStorage = FatIvfcStorage;
             }
 
-            SaveDataFileSystemCore = new SaveDataFileSystemCore(JournalIvfcStorage, fatStorage, Header.SaveHeader);
+            SaveDataFileSystemCore = new SaveDataFileSystemCore(CoreDataIvfcStorage, fatStorage, Header.SaveHeader);
         }
 
         private static HierarchicalDuplexStorage InitDuplexStorage(IStorage baseStorage, Header header)
@@ -175,8 +196,8 @@ namespace LibHac.IO.Save
 
         public bool Commit(Keyset keyset)
         {
-            JournalIvfcStorage.Flush();
+            CoreDataIvfcStorage.Flush();
-            FatIvfcStorage.Flush();
+            FatIvfcStorage?.Flush();
 
             Stream headerStream = BaseStorage.AsStream();
 
@@ -208,15 +229,24 @@ namespace LibHac.IO.Save
 
         public void FsTrim()
         {
+            MetaRemapStorage.FsTrim();
+            DataRemapStorage.FsTrim();
+            DuplexStorage.FsTrim();
+            JournalStorage.FsTrim();
+            CoreDataIvfcStorage.FsTrim();
+            FatIvfcStorage?.FsTrim();
             SaveDataFileSystemCore.FsTrim();
+
+            int unusedHeaderOffset = IsFirstHeaderInUse ? 0x4000 : 0;
+            BaseStorage.Slice(unusedHeaderOffset, 0x4000).Fill(TrimFillValue);
         }
 
         public Validity Verify(IProgressReport logger = null)
         {
-            Validity journalValidity = JournalIvfcStorage.Validate(true, logger);
+            Validity journalValidity = CoreDataIvfcStorage.Validate(true, logger);
-            JournalIvfcStorage.SetLevelValidities(Header.Ivfc);
+            CoreDataIvfcStorage.SetLevelValidities(Header.Ivfc);
 
-            if (FatIvfcStorage == null)return journalValidity;
+            if (FatIvfcStorage == null) return journalValidity;
 
             Validity fatValidity = FatIvfcStorage.Validate(true, logger);
             FatIvfcStorage.SetLevelValidities(Header.Ivfc);

src/LibHac/IO/Save/SaveDataFileSystemCore.cs

@@ -143,7 +143,7 @@ namespace LibHac.IO.Save
                 {
                     AllocationTable.FsTrimList(fileInfo.StartBlock);
 
-                    OpenFatStorage(fileInfo.StartBlock).Slice(fileInfo.Length).Fill(0);
+                    OpenFatStorage(fileInfo.StartBlock).Slice(fileInfo.Length).Fill(SaveDataFileSystem.TrimFillValue);
                 }
             }
 
@@ -152,7 +152,9 @@ namespace LibHac.IO.Save
 
             AllocationTable.FsTrimList(freeIndex);
 
-            OpenFatStorage(freeIndex).Fill(0);
+            OpenFatStorage(freeIndex).Fill(SaveDataFileSystem.TrimFillValue);
+
+            FileTable.TrimFreeEntries();
         }
 
         private AllocationTableStorage OpenFatStorage(int blockIndex)

src/LibHac/IO/Save/SaveFsList.cs

@@ -226,6 +226,30 @@ namespace LibHac.IO.Save
             Free(index);
         }
 
+        public void TrimFreeEntries()
+        {
+            Span<byte> entryBytes = stackalloc byte[_sizeOfEntry];
+            Span<byte> name = entryBytes.Slice(4, MaxNameLength);
+            ref SaveFsEntry entry = ref GetEntryFromBytes(entryBytes);
+
+            ReadEntry(FreeListHeadIndex, out entry);
+
+            int index = entry.Next;
+
+            while (entry.Next > 0)
+            {
+                ReadEntry(index, out entry);
+
+                entry.Parent = 0;
+                entry.Value = default;
+                name.Fill(SaveDataFileSystem.TrimFillValue);
+
+                WriteEntry(index, ref entry);
+
+                index = entry.Next;
+            }
+        }
+
         private int GetListCapacity()
         {
             Span<byte> buf = stackalloc byte[sizeof(int)];

src/LibHac/IO/StorageBase.cs

@@ -56,8 +56,8 @@ namespace LibHac.IO
         protected void ValidateParameters(ReadOnlySpan<byte> span, long offset)
         {
             if (_isDisposed) throw new ObjectDisposedException(null);
-            if (span == null) throw new ArgumentNullException(nameof(span));
             if (offset < 0) throw new ArgumentOutOfRangeException(nameof(offset), "Argument must be non-negative.");
+
             long length = GetSize();
 
             if (length != -1 && !CanAutoExpand)

src/LibHac/IO/StorageExtensions.cs

@@ -98,31 +98,35 @@ namespace LibHac.IO
         }
 
         public static void Fill(this IStorage input, byte value, IProgressReport progress = null)
+        {
+            input.Fill(value, 0, input.GetSize(), progress);
+        }
+
+        public static void Fill(this IStorage input, byte value, long offset, long count, IProgressReport progress = null)
         {
             const int threshold = 0x400;
 
-            long length = input.GetSize();
+            if (count > threshold)
-            if (length > threshold)
             {
-                input.FillLarge(value, progress);
+                input.FillLarge(value, offset, count, progress);
                 return;
             }
 
-            Span<byte> buf = stackalloc byte[(int)length];
+            Span<byte> buf = stackalloc byte[(int)count];
             buf.Fill(value);
 
-            input.Write(buf, 0);
+            input.Write(buf, offset);
         }
 
-        private static void FillLarge(this IStorage input, byte value, IProgressReport progress = null)
+        private static void FillLarge(this IStorage input, byte value, long offset, long count, IProgressReport progress = null)
         {
             const int bufferSize = 0x4000;
 
-            long remaining = input.GetSize();
+            long remaining = count;
             if (remaining < 0) throw new ArgumentException("Storage must have an explicit length");
             progress?.SetTotal(remaining);
 
-            long pos = 0;
+            long pos = offset;
 
             byte[] buffer = ArrayPool<byte>.Shared.Rent(bufferSize);
             try

src/LibHac/Util.cs

@@ -134,8 +134,6 @@ namespace LibHac
 
         public static bool IsEmpty(this ReadOnlySpan<byte> span)
         {
-            if (span == null) throw new ArgumentNullException(nameof(span));
-
             for (int i = 0; i < span.Length; i++)
             {
                 if (span[i] != 0)