mirrors/LibHac
1
0
Fork 0
mirror of https://github.com/Thealexbarney/LibHac.git synced 2024-11-14 10:49:41 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add fstrim capability to savedata (#54)

Browse source 
* Properly commit version 4 savedata. Remove null span checks

* Add FsTrim to IntegrityVerificationStorage

* Remove dotsettings file

* Add trimming to all savedata layers
This commit is contained in:
Alex Barney 2019-04-26 09:48:57 -05:00 committed by GitHub
parent fbdf6b3be9
commit cc9ca448f0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 185 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
build/_build.csproj.DotSettings
View file

@ -1,24 +0,0 @@
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
<s:Boolean x:Key="/Default/CodeInspection/ImplicitNullability/EnableFields/@EntryValue">False</s:Boolean>
<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpCodeStyle/DEFAULT_INTERNAL_MODIFIER/@EntryValue">Implicit</s:String>
<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpCodeStyle/DEFAULT_PRIVATE_MODIFIER/@EntryValue">Implicit</s:String>
<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpCodeStyle/METHOD_OR_OPERATOR_BODY/@EntryValue">ExpressionBody</s:String>
<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpCodeStyle/ThisQualifier/INSTANCE_MEMBERS_QUALIFY_MEMBERS/@EntryValue">0</s:String>
<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/ANONYMOUS_METHOD_DECLARATION_BRACES/@EntryValue">NEXT_LINE</s:String>
<s:Boolean x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/KEEP_USER_LINEBREAKS/@EntryValue">True</s:Boolean>
<s:Boolean x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/WRAP_AFTER_INVOCATION_LPAR/@EntryValue">False</s:Boolean>
<s:Int64 x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/MAX_ATTRIBUTE_LENGTH_FOR_SAME_LINE/@EntryValue">120</s:Int64>
<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/PLACE_FIELD_ATTRIBUTE_ON_SAME_LINE_EX/@EntryValue">IF_OWNER_IS_SINGLE_LINE</s:String>
<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/WRAP_ARGUMENTS_STYLE/@EntryValue">WRAP_IF_LONG</s:String>
<s:Boolean x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/PLACE_SIMPLE_ANONYMOUSMETHOD_ON_SINGLE_LINE/@EntryValue">False</s:Boolean>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/PredefinedNamingRules/=PrivateInstanceFields/@EntryIndexedValue">&lt;Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /&gt;</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/PredefinedNamingRules/=PrivateStaticFields/@EntryIndexedValue">&lt;Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /&gt;</s:String>
<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ECSharpAttributeForSingleLineMethodUpgrade/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ECSharpKeepExistingMigration/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ECSharpPlaceEmbeddedOnSameLineMigration/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ECSharpRenamePlacementToArrangementMigration/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ECSharpUseContinuousIndentInsideBracesMigration/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002EAddAccessorOwnerDeclarationBracesMigration/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002ECSharpPlaceAttributeOnSameLineMigration/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002EMigrateBlankLinesAroundFieldToBlankLinesAroundProperty/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002EMigrateThisQualifierSettings/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

8
src/LibHac/IO/HierarchicalIntegrityVerificationStorage.cs
View file

@ -149,6 +149,14 @@ namespace LibHac.IO
return result;
}
public void FsTrim()
{
foreach (IntegrityVerificationStorage level in IntegrityStorages)
{
level.FsTrim();
}
}
private static readonly string[] SaltSources =
{
"HierarchicalIntegrityVerificationStorage::Master",

21
src/LibHac/IO/IntegrityVerificationStorage.cs
View file

@ -2,6 +2,7 @@
using System.Buffers;
using System.IO;
using System.Security.Cryptography;
using LibHac.IO.Save;
namespace LibHac.IO
{
@ -171,7 +172,7 @@ namespace LibHac.IO
if (Type == IntegrityStorageType.Save)
{
// This bit is set on all save hashes
hash[0x1F] |= 0x80;
hash[0x1F] |= 0b10000000;
}
return hash;
@ -183,6 +184,24 @@ namespace LibHac.IO
HashStorage.Flush();
base.Flush();
}
public void FsTrim()
{
if (Type != IntegrityStorageType.Save) return;
Span<byte> digest = stackalloc byte[DigestSize];
for (int i = 0; i < SectorCount; i++)
{
long hashPos = i * DigestSize;
HashStorage.Read(digest, hashPos);
if (!Util.IsEmpty(digest)) continue;
int dataOffset = i * SectorSize;
BaseStorage.Fill(SaveDataFileSystem.TrimFillValue, dataOffset, SectorSize);
}
}
}
/// <summary>

4
src/LibHac/IO/Save/AllocationTable.cs
View file

@ -312,7 +312,7 @@ namespace LibHac.IO.Save
int fillOffset = BlockToEntryIndex(index + 2) * EntrySize;
int fillLength = (length - 3) * EntrySize;
BaseStorage.Slice(fillOffset, fillLength).Fill(0x00);
BaseStorage.Slice(fillOffset, fillLength).Fill(SaveDataFileSystem.TrimFillValue);
}
nodesIterated++;
@ -329,7 +329,7 @@ namespace LibHac.IO.Save
public void FsTrim()
{
int tableSize = BlockToEntryIndex(Header.AllocationTableBlockCount) * EntrySize;
BaseStorage.Slice(tableSize).Fill(0x00);
BaseStorage.Slice(tableSize).Fill(SaveDataFileSystem.TrimFillValue);
}
private void ReadEntries(int entryIndex, Span<AllocationTableEntry> entries)

12
src/LibHac/IO/Save/DuplexStorage.cs
View file

@ -77,5 +77,17 @@ namespace LibHac.IO.Save
}
public override long GetSize() => _length;
public void FsTrim()
{
int blockCount = (int)(DataA.GetSize() / BlockSize);
for (int i = 0; i < blockCount; i++)
{
IStorage dataToClear = Bitmap.Bitmap[i] ? DataA : DataB;
dataToClear.Slice(i * BlockSize, BlockSize).Fill(SaveDataFileSystem.TrimFillValue);
}
}
}
}

8
src/LibHac/IO/Save/HierarchicalDuplexStorage.cs
View file

@ -48,6 +48,14 @@ namespace LibHac.IO.Save
}
public override long GetSize() => _length;
public void FsTrim()
{
foreach (DuplexStorage layer in Layers)
{
layer.FsTrim();
}
}
}
public class DuplexFsLayerInfo

6
src/LibHac/IO/Save/HierarchicalSaveFileTable.cs
View file

@ -235,6 +235,12 @@ namespace LibHac.IO.Save
return true;
}
public void TrimFreeEntries()
{
DirectoryTable.TrimFreeEntries();
FileTable.TrimFreeEntries();
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
private struct TableEntry<T> where T : struct
{

16
src/LibHac/IO/Save/JournalMap.cs
View file

@ -4,6 +4,7 @@ namespace LibHac.IO.Save
{
public class JournalMap
{
private int MapEntryLength = 8;
public JournalMapHeader Header { get; }
private JournalMapEntry[] Entries { get; }
@ -55,6 +56,21 @@ namespace LibHac.IO.Save
public IStorage GetModifiedPhysicalBlocksStorage() => ModifiedPhysicalBlocks.AsReadOnly();
public IStorage GetModifiedVirtualBlocksStorage() => ModifiedVirtualBlocks.AsReadOnly();
public IStorage GetFreeBlocksStorage() => FreeBlocks.AsReadOnly();
public void FsTrim()
{
int virtualBlockCount = Header.MainDataBlockCount;
int physicalBlockCount = virtualBlockCount + Header.JournalBlockCount;
int blockMapLength = virtualBlockCount * MapEntryLength;
int physicalBitmapLength = Util.AlignUp(physicalBlockCount, 32) / 8;
int virtualBitmapLength = Util.AlignUp(virtualBlockCount, 32) / 8;
MapStorage.Slice(blockMapLength).Fill(SaveDataFileSystem.TrimFillValue);
FreeBlocks.Slice(physicalBitmapLength).Fill(SaveDataFileSystem.TrimFillValue);
ModifiedPhysicalBlocks.Slice(physicalBitmapLength).Fill(SaveDataFileSystem.TrimFillValue);
ModifiedVirtualBlocks.Slice(virtualBitmapLength).Fill(SaveDataFileSystem.TrimFillValue);
}
}
public class JournalMapHeader

17
src/LibHac/IO/Save/JournalStorage.cs
View file

@ -1,4 +1,5 @@
using System;
using System.Collections;
using System.IO;
namespace LibHac.IO.Save
@ -85,6 +86,22 @@ namespace LibHac.IO.Save
public IStorage GetBaseStorage() => BaseStorage.AsReadOnly();
public IStorage GetHeaderStorage() => HeaderStorage.AsReadOnly();
public void FsTrim()
{
// todo replace with a bitmap reader class when added
BitArray bitmap = new DuplexBitmap(Map.GetFreeBlocksStorage(),
Map.Header.JournalBlockCount + Map.Header.MainDataBlockCount).Bitmap;
for (int i = 0; i < bitmap.Length; i++)
{
if (!bitmap[i]) continue;
BaseStorage.Fill(SaveDataFileSystem.TrimFillValue, i * BlockSize, BlockSize);
}
Map.FsTrim();
}
}
public class JournalHeader

16
src/LibHac/IO/Save/RemapStorage.cs
View file

@ -1,11 +1,14 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace LibHac.IO.Save
{
public class RemapStorage : StorageBase
{
private const int MapEntryLength = 0x20;
private IStorage BaseStorage { get; }
private IStorage HeaderStorage { get; }
private IStorage MapEntryStorage { get; }
@ -45,6 +48,8 @@ namespace LibHac.IO.Save
protected override void ReadImpl(Span<byte> destination, long offset)
{
if (destination.Length == 0) return;
MapEntry entry = GetMapEntry(offset);
long inPos = offset;
@ -71,6 +76,8 @@ namespace LibHac.IO.Save
protected override void WriteImpl(ReadOnlySpan<byte> source, long offset)
{
if (source.Length == 0) return;
MapEntry entry = GetMapEntry(offset);
long inPos = offset;
@ -178,6 +185,15 @@ namespace LibHac.IO.Save
{
return ~GetOffsetMask();
}
public void FsTrim()
{
int mapEntriesLength = Header.MapEntryCount * MapEntryLength;
long dataEnd = MapEntries.Max(x => x.PhysicalOffsetEnd);
MapEntryStorage.Slice(mapEntriesLength).Fill(SaveDataFileSystem.TrimFillValue);
BaseStorage.Slice(dataEnd).Fill(SaveDataFileSystem.TrimFillValue);
}
}
public class RemapHeader

46
src/LibHac/IO/Save/SaveDataFileSystem.cs
View file

@ -5,7 +5,11 @@ namespace LibHac.IO.Save
{
public class SaveDataFileSystem : IFileSystem
{
internal const byte TrimFillValue = 0;
public Header Header { get; }
private bool IsFirstHeaderInUse { get; }
public IStorage BaseStorage { get; }
public bool LeaveOpen { get; }
@ -17,7 +21,7 @@ namespace LibHac.IO.Save
public HierarchicalDuplexStorage DuplexStorage { get; }
public JournalStorage JournalStorage { get; }
public HierarchicalIntegrityVerificationStorage JournalIvfcStorage { get; }
public HierarchicalIntegrityVerificationStorage CoreDataIvfcStorage { get; }
public HierarchicalIntegrityVerificationStorage FatIvfcStorage { get; }
private Keyset Keyset { get; }
@ -28,7 +32,24 @@ namespace LibHac.IO.Save
LeaveOpen = leaveOpen;
Keyset = keyset;
Header = new Header(BaseStorage, keyset);
var headerA = new Header(BaseStorage, keyset);
var headerB = new Header(BaseStorage.Slice(0x4000), keyset);
if (headerA.HeaderHashValidity == Validity.Valid)
{
IsFirstHeaderInUse = true;
}
else if (headerB.HeaderHashValidity == Validity.Valid)
{
IsFirstHeaderInUse = false;
}
else
{
throw new InvalidDataException("Savedata header is not valid.");
}
Header = IsFirstHeaderInUse ? headerA : headerB;
FsLayout layout = Header.Layout;
IStorage dataRemapBase = BaseStorage.Slice(layout.FileMapDataOffset, layout.FileMapDataSize);
@ -54,7 +75,7 @@ namespace LibHac.IO.Save
JournalStorage = new JournalStorage(journalData, Header.JournalHeader, journalMapInfo, leaveOpen);
JournalIvfcStorage = InitJournalIvfcStorage(integrityCheckLevel);
CoreDataIvfcStorage = InitJournalIvfcStorage(integrityCheckLevel);
IStorage fatStorage = MetaRemapStorage.Slice(layout.FatOffset, layout.FatSize);
@ -64,7 +85,7 @@ namespace LibHac.IO.Save
fatStorage = FatIvfcStorage;
}
SaveDataFileSystemCore = new SaveDataFileSystemCore(JournalIvfcStorage, fatStorage, Header.SaveHeader);
SaveDataFileSystemCore = new SaveDataFileSystemCore(CoreDataIvfcStorage, fatStorage, Header.SaveHeader);
}
private static HierarchicalDuplexStorage InitDuplexStorage(IStorage baseStorage, Header header)
@ -175,8 +196,8 @@ namespace LibHac.IO.Save
public bool Commit(Keyset keyset)
{
JournalIvfcStorage.Flush();
FatIvfcStorage.Flush();
CoreDataIvfcStorage.Flush();
FatIvfcStorage?.Flush();
Stream headerStream = BaseStorage.AsStream();
@ -208,13 +229,22 @@ namespace LibHac.IO.Save
public void FsTrim()
{
MetaRemapStorage.FsTrim();
DataRemapStorage.FsTrim();
DuplexStorage.FsTrim();
JournalStorage.FsTrim();
CoreDataIvfcStorage.FsTrim();
FatIvfcStorage?.FsTrim();
SaveDataFileSystemCore.FsTrim();
int unusedHeaderOffset = IsFirstHeaderInUse ? 0x4000 : 0;
BaseStorage.Slice(unusedHeaderOffset, 0x4000).Fill(TrimFillValue);
}
public Validity Verify(IProgressReport logger = null)
{
Validity journalValidity = JournalIvfcStorage.Validate(true, logger);
JournalIvfcStorage.SetLevelValidities(Header.Ivfc);
Validity journalValidity = CoreDataIvfcStorage.Validate(true, logger);
CoreDataIvfcStorage.SetLevelValidities(Header.Ivfc);
if (FatIvfcStorage == null) return journalValidity;

6
src/LibHac/IO/Save/SaveDataFileSystemCore.cs
View file

@ -143,7 +143,7 @@ namespace LibHac.IO.Save
{
AllocationTable.FsTrimList(fileInfo.StartBlock);
OpenFatStorage(fileInfo.StartBlock).Slice(fileInfo.Length).Fill(0);
OpenFatStorage(fileInfo.StartBlock).Slice(fileInfo.Length).Fill(SaveDataFileSystem.TrimFillValue);
}
}
@ -152,7 +152,9 @@ namespace LibHac.IO.Save
AllocationTable.FsTrimList(freeIndex);
OpenFatStorage(freeIndex).Fill(0);
OpenFatStorage(freeIndex).Fill(SaveDataFileSystem.TrimFillValue);
FileTable.TrimFreeEntries();
}
private AllocationTableStorage OpenFatStorage(int blockIndex)

24
src/LibHac/IO/Save/SaveFsList.cs
View file

@ -226,6 +226,30 @@ namespace LibHac.IO.Save
Free(index);
}
public void TrimFreeEntries()
{
Span<byte> entryBytes = stackalloc byte[_sizeOfEntry];
Span<byte> name = entryBytes.Slice(4, MaxNameLength);
ref SaveFsEntry entry = ref GetEntryFromBytes(entryBytes);
ReadEntry(FreeListHeadIndex, out entry);
int index = entry.Next;
while (entry.Next > 0)
{
ReadEntry(index, out entry);
entry.Parent = 0;
entry.Value = default;
name.Fill(SaveDataFileSystem.TrimFillValue);
WriteEntry(index, ref entry);
index = entry.Next;
}
}
private int GetListCapacity()
{
Span<byte> buf = stackalloc byte[sizeof(int)];

2
src/LibHac/IO/StorageBase.cs
View file

@ -56,8 +56,8 @@ namespace LibHac.IO
protected void ValidateParameters(ReadOnlySpan<byte> span, long offset)
{
if (_isDisposed) throw new ObjectDisposedException(null);
if (span == null) throw new ArgumentNullException(nameof(span));
if (offset < 0) throw new ArgumentOutOfRangeException(nameof(offset), "Argument must be non-negative.");
long length = GetSize();
if (length != -1 && !CanAutoExpand)

20
src/LibHac/IO/StorageExtensions.cs
View file

@ -98,31 +98,35 @@ namespace LibHac.IO
}
public static void Fill(this IStorage input, byte value, IProgressReport progress = null)
{
input.Fill(value, 0, input.GetSize(), progress);
}
public static void Fill(this IStorage input, byte value, long offset, long count, IProgressReport progress = null)
{
const int threshold = 0x400;
long length = input.GetSize();
if (length > threshold)
if (count > threshold)
{
input.FillLarge(value, progress);
input.FillLarge(value, offset, count, progress);
return;
}
Span<byte> buf = stackalloc byte[(int)length];
Span<byte> buf = stackalloc byte[(int)count];
buf.Fill(value);
input.Write(buf, 0);
input.Write(buf, offset);
}
private static void FillLarge(this IStorage input, byte value, IProgressReport progress = null)
private static void FillLarge(this IStorage input, byte value, long offset, long count, IProgressReport progress = null)
{
const int bufferSize = 0x4000;
long remaining = input.GetSize();
long remaining = count;
if (remaining < 0) throw new ArgumentException("Storage must have an explicit length");
progress?.SetTotal(remaining);
long pos = 0;
long pos = offset;
byte[] buffer = ArrayPool<byte>.Shared.Rent(bufferSize);
try

2
src/LibHac/Util.cs
View file

@ -134,8 +134,6 @@ namespace LibHac
public static bool IsEmpty(this ReadOnlySpan<byte> span)
{
if (span == null) throw new ArgumentNullException(nameof(span));
for (int i = 0; i < span.Length; i++)
{
if (span[i] != 0)