From 9a234575ac6a25426206b885a4563c10495c6a9b Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 31 Dec 2018 22:30:30 -0700 Subject: [PATCH] Add PartitionFileSystem --- src/LibHac/IO/FileSystemExtensions.cs | 38 +++-- src/LibHac/IO/PartitionDirectory.cs | 49 ++++++ src/LibHac/IO/PartitionFile.cs | 48 ++++++ src/LibHac/IO/PartitionFileSystem.cs | 208 ++++++++++++++++++++++++++ src/LibHac/IO/RomFsFileSystem.cs | 7 + src/LibHac/Nca.cs | 19 ++- src/hactoolnet/ProcessNca.cs | 5 +- src/hactoolnet/ProcessNsp.cs | 6 +- 8 files changed, 362 insertions(+), 18 deletions(-) create mode 100644 src/LibHac/IO/PartitionDirectory.cs create mode 100644 src/LibHac/IO/PartitionFile.cs create mode 100644 src/LibHac/IO/PartitionFileSystem.cs diff --git a/src/LibHac/IO/FileSystemExtensions.cs b/src/LibHac/IO/FileSystemExtensions.cs index dc0683e1..99ee1452 100644 --- a/src/LibHac/IO/FileSystemExtensions.cs +++ b/src/LibHac/IO/FileSystemExtensions.cs @@ -6,16 +6,15 @@ namespace LibHac.IO { public static class FileSystemExtensions { - // todo add progress logging - public static void CopyDirectory(this IDirectory source, IDirectory dest) + public static void CopyDirectory(this IDirectory source, IDirectory dest, IProgressReport logger = null) { IFileSystem sourceFs = source.ParentFileSystem; IFileSystem destFs = dest.ParentFileSystem; foreach (DirectoryEntry entry in source.Read()) { - string subSrcPath = source.FullPath + '/' + entry.Name; - string subDstPath = dest.FullPath + '/' + entry.Name; + string subSrcPath = PathTools.Normalize(source.FullPath + '/' + entry.Name); + string subDstPath = PathTools.Normalize(dest.FullPath + '/' + entry.Name); if (entry.Type == DirectoryEntryType.Directory) { @@ -23,7 +22,7 @@ namespace LibHac.IO IDirectory subSrcDir = sourceFs.OpenDirectory(subSrcPath, OpenDirectoryMode.All); IDirectory subDstDir = destFs.OpenDirectory(subDstPath, OpenDirectoryMode.All); - subSrcDir.CopyDirectory(subDstDir); + subSrcDir.CopyDirectory(subDstDir, logger); } if (entry.Type == DirectoryEntryType.File) @@ -33,12 +32,28 @@ namespace LibHac.IO using (IFile srcFile = sourceFs.OpenFile(subSrcPath, OpenMode.Read)) using (IFile dstFile = destFs.OpenFile(subDstPath, OpenMode.Write)) { - srcFile.CopyTo(dstFile); + logger?.LogMessage(subSrcPath); + srcFile.CopyTo(dstFile, logger); } } } } + public static void CopyFileSystem(this IFileSystem source, IFileSystem dest, IProgressReport logger = null) + { + IDirectory sourceRoot = source.OpenDirectory("/", OpenDirectoryMode.All); + IDirectory destRoot = dest.OpenDirectory("/", OpenDirectoryMode.All); + + sourceRoot.CopyDirectory(destRoot, logger); + } + + public static void Extract(this IFileSystem source, string destinationPath, IProgressReport logger = null) + { + var destFs = new LocalFileSystem(destinationPath); + + source.CopyFileSystem(destFs, logger); + } + public static IEnumerable EnumerateEntries(this IDirectory directory) { IFileSystem fs = directory.ParentFileSystem; @@ -57,10 +72,10 @@ namespace LibHac.IO } } - // todo add progress logging - public static void CopyTo(this IFile file, IFile dest) + public static void CopyTo(this IFile file, IFile dest, IProgressReport logger = null) { const int bufferSize = 0x8000; + logger?.SetTotal(file.GetSize()); byte[] buffer = ArrayPool.Shared.Rent(bufferSize); try @@ -72,9 +87,14 @@ namespace LibHac.IO { dest.Write(buffer.AsSpan(0, bytesRead), inOffset); inOffset += bytesRead; + logger?.ReportAdd(bytesRead); } } - finally { ArrayPool.Shared.Return(buffer); } + finally + { + ArrayPool.Shared.Return(buffer); + logger?.SetTotal(0); + } } } } diff --git a/src/LibHac/IO/PartitionDirectory.cs b/src/LibHac/IO/PartitionDirectory.cs new file mode 100644 index 00000000..54214b7d --- /dev/null +++ b/src/LibHac/IO/PartitionDirectory.cs @@ -0,0 +1,49 @@ +using System.Collections.Generic; +using System.IO; + +namespace LibHac.IO +{ + public class PartitionDirectory : IDirectory + { + IFileSystem IDirectory.ParentFileSystem => ParentFileSystem; + public PartitionFileSystem ParentFileSystem { get; } + public string FullPath { get; } + + private OpenDirectoryMode Mode { get; } + + public PartitionDirectory(PartitionFileSystem fs, string path, OpenDirectoryMode mode) + { + path = PathTools.Normalize(path); + + if (path != "/") throw new DirectoryNotFoundException(); + + ParentFileSystem = fs; + FullPath = path; + Mode = mode; + } + + + public IEnumerable Read() + { + if (Mode.HasFlag(OpenDirectoryMode.Files)) + { + foreach (PartitionFileEntry entry in ParentFileSystem.Files) + { + yield return new DirectoryEntry(entry.Name, '/' + entry.Name, DirectoryEntryType.File, entry.Size); + } + } + } + + public int GetEntryCount() + { + int count = 0; + + if (Mode.HasFlag(OpenDirectoryMode.Files)) + { + count += ParentFileSystem.Files.Length; + } + + return count; + } + } +} \ No newline at end of file diff --git a/src/LibHac/IO/PartitionFile.cs b/src/LibHac/IO/PartitionFile.cs new file mode 100644 index 00000000..fa1a5083 --- /dev/null +++ b/src/LibHac/IO/PartitionFile.cs @@ -0,0 +1,48 @@ +using System; + +namespace LibHac.IO +{ + public class PartitionFile : FileBase + { + private IStorage BaseStorage { get; } + private long Offset { get; } + private long Size { get; } + + public PartitionFile(IStorage baseStorage, long offset, long size, OpenMode mode) + { + Mode = mode; + BaseStorage = baseStorage; + Offset = offset; + Size = size; + } + + public override int Read(Span destination, long offset) + { + int toRead = ValidateReadParamsAndGetSize(destination, offset); + + long storageOffset = Offset + offset; + BaseStorage.Read(destination.Slice(0, toRead), storageOffset); + + return toRead; + } + + public override void Write(ReadOnlySpan source, long offset) + { + throw new NotImplementedException(); + } + + public override void Flush() + { + } + + public override long GetSize() + { + return Size; + } + + public override void SetSize(long size) + { + throw new NotSupportedException(); + } + } +} diff --git a/src/LibHac/IO/PartitionFileSystem.cs b/src/LibHac/IO/PartitionFileSystem.cs new file mode 100644 index 00000000..aea87a20 --- /dev/null +++ b/src/LibHac/IO/PartitionFileSystem.cs @@ -0,0 +1,208 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace LibHac.IO +{ + public class PartitionFileSystem : IFileSystem + { + public PartitionFileSystemHeader Header { get; } + public int HeaderSize { get; } + public PartitionFileEntry[] Files { get; } + + private Dictionary FileDict { get; } + private IStorage BaseStorage { get; } + + public PartitionFileSystem(IStorage storage) + { + using (var reader = new BinaryReader(storage.AsStream(), Encoding.Default, true)) + { + Header = new PartitionFileSystemHeader(reader); + } + + HeaderSize = Header.HeaderSize; + Files = Header.Files; + FileDict = Header.Files.ToDictionary(x => x.Name, x => x); + BaseStorage = storage; + } + + public void CreateDirectory(string path) + { + throw new NotSupportedException(); + } + + public void CreateFile(string path, long size) + { + throw new NotSupportedException(); + } + + public void DeleteDirectory(string path) + { + throw new NotSupportedException(); + } + + public void DeleteFile(string path) + { + throw new NotSupportedException(); + } + + public IDirectory OpenDirectory(string path, OpenDirectoryMode mode) + { + return new PartitionDirectory(this, path, mode); + } + + public IFile OpenFile(string path, OpenMode mode) + { + path = PathTools.Normalize(path).TrimStart('/'); + + if (!FileDict.TryGetValue(path, out PartitionFileEntry entry)) + { + throw new FileNotFoundException(); + } + + return OpenFile(entry, mode); + } + + public IFile OpenFile(PartitionFileEntry entry, OpenMode mode) + { + return new PartitionFile(BaseStorage, HeaderSize + entry.Offset, entry.Size, mode); + } + + public void RenameDirectory(string srcPath, string dstPath) + { + throw new NotSupportedException(); + } + + public void RenameFile(string srcPath, string dstPath) + { + throw new NotSupportedException(); + } + + public bool DirectoryExists(string path) + { + path = PathTools.Normalize(path); + return path == "/"; + } + + public bool FileExists(string path) + { + path = PathTools.Normalize(path).TrimStart('/'); + + return FileDict.ContainsKey(path); + } + + public void Commit() + { + throw new NotSupportedException(); + } + } + + public enum PartitionFileSystemType + { + Pfs0, + Hfs0 + } + + public class PartitionFileSystemHeader + { + public string Magic; + public int NumFiles; + public int StringTableSize; + public long Reserved; + public PartitionFileSystemType Type; + public int HeaderSize; + public PartitionFileEntry[] Files; + + public PartitionFileSystemHeader(BinaryReader reader) + { + Magic = reader.ReadAscii(4); + NumFiles = reader.ReadInt32(); + StringTableSize = reader.ReadInt32(); + Reserved = reader.ReadInt32(); + + switch (Magic) + { + case "PFS0": + Type = PartitionFileSystemType.Pfs0; + break; + case "HFS0": + Type = PartitionFileSystemType.Hfs0; + break; + default: + throw new InvalidDataException($"Invalid Partition FS type \"{Magic}\""); + } + + int entrySize = GetFileEntrySize(Type); + int stringTableOffset = 16 + entrySize * NumFiles; + HeaderSize = stringTableOffset + StringTableSize; + + Files = new PartitionFileEntry[NumFiles]; + for (int i = 0; i < NumFiles; i++) + { + Files[i] = new PartitionFileEntry(reader, Type) { Index = i }; + } + + for (int i = 0; i < NumFiles; i++) + { + reader.BaseStream.Position = stringTableOffset + Files[i].StringTableOffset; + Files[i].Name = reader.ReadAsciiZ(); + } + + + if (Type == PartitionFileSystemType.Hfs0) + { + for (int i = 0; i < NumFiles; i++) + { + reader.BaseStream.Position = HeaderSize + Files[i].Offset; + Files[i].HashValidity = Crypto.CheckMemoryHashTable(reader.ReadBytes(Files[i].HashedRegionSize), Files[i].Hash, 0, Files[i].HashedRegionSize); + } + } + + } + + private static int GetFileEntrySize(PartitionFileSystemType type) + { + switch (type) + { + case PartitionFileSystemType.Pfs0: + return 24; + case PartitionFileSystemType.Hfs0: + return 0x40; + default: + throw new ArgumentOutOfRangeException(nameof(type), type, null); + } + } + } + + public class PartitionFileEntry + { + public int Index; + public long Offset; + public long Size; + public uint StringTableOffset; + public long Reserved; + public int HashedRegionSize; + public byte[] Hash; + public string Name; + public Validity HashValidity = Validity.Unchecked; + + public PartitionFileEntry(BinaryReader reader, PartitionFileSystemType type) + { + Offset = reader.ReadInt64(); + Size = reader.ReadInt64(); + StringTableOffset = reader.ReadUInt32(); + if (type == PartitionFileSystemType.Hfs0) + { + HashedRegionSize = reader.ReadInt32(); + Reserved = reader.ReadInt64(); + Hash = reader.ReadBytes(Crypto.Sha256DigestSize); + } + else + { + Reserved = reader.ReadUInt32(); + } + } + } +} diff --git a/src/LibHac/IO/RomFsFileSystem.cs b/src/LibHac/IO/RomFsFileSystem.cs index e0356c1e..5163ce2e 100644 --- a/src/LibHac/IO/RomFsFileSystem.cs +++ b/src/LibHac/IO/RomFsFileSystem.cs @@ -24,6 +24,7 @@ namespace LibHac.IO byte[] dirMetaTable; byte[] fileMetaTable; + using (var reader = new BinaryReader(BaseStorage.AsStream(), Encoding.Default, true)) { Header = new RomfsHeader(reader); @@ -117,6 +118,8 @@ namespace LibHac.IO public IFile OpenFile(string path, OpenMode mode) { + path = PathTools.Normalize(path); + if (!FileDict.TryGetValue(path, out RomfsFile file)) { throw new FileNotFoundException(); @@ -147,11 +150,15 @@ namespace LibHac.IO public bool DirectoryExists(string path) { + path = PathTools.Normalize(path); + return DirectoryDict.ContainsKey(path); } public bool FileExists(string path) { + path = PathTools.Normalize(path); + return FileDict.ContainsKey(path); } } diff --git a/src/LibHac/Nca.cs b/src/LibHac/Nca.cs index 556c73bf..569e1f55 100644 --- a/src/LibHac/Nca.cs +++ b/src/LibHac/Nca.cs @@ -255,13 +255,24 @@ namespace LibHac return new HierarchicalIntegrityVerificationStorage(initInfo, integrityCheckLevel, leaveOpen); } - public IFileSystem OpenSectionFileSystem(int index) + public IFileSystem OpenSectionFileSystem(int index, IntegrityCheckLevel integrityCheckLevel) { - IStorage storage = OpenSection(index, false, IntegrityCheckLevel.ErrorOnInvalid, true); + if (Sections[index] == null) throw new ArgumentOutOfRangeException(nameof(index)); + NcaFsHeader header = Sections[index].Header; - var fs = new RomFsFileSystem(storage); + IStorage storage = OpenSection(index, false, integrityCheckLevel, true); - return fs; + switch (header.Type) + { + case SectionType.Pfs0: + return new PartitionFileSystem(storage); + case SectionType.Romfs: + return new RomFsFileSystem(storage); + case SectionType.Bktr: + return new RomFsFileSystem(storage); + default: + throw new ArgumentOutOfRangeException(); + } } /// diff --git a/src/hactoolnet/ProcessNca.cs b/src/hactoolnet/ProcessNca.cs index 63c8ca5b..2b13b6c3 100644 --- a/src/hactoolnet/ProcessNca.cs +++ b/src/hactoolnet/ProcessNca.cs @@ -75,7 +75,7 @@ namespace hactoolnet if (ctx.Options.RomfsOutDir != null) { - var romfs = new Romfs(nca.OpenSection(section.SectionNum, false, ctx.Options.IntegrityLevel, true)); + IFileSystem romfs = nca.OpenSectionFileSystem(section.SectionNum, ctx.Options.IntegrityLevel); romfs.Extract(ctx.Options.RomfsOutDir, ctx.Logger); } } @@ -103,7 +103,8 @@ namespace hactoolnet if (ctx.Options.ExefsOutDir != null) { - nca.ExtractSection(section.SectionNum, ctx.Options.ExefsOutDir, ctx.Options.IntegrityLevel, ctx.Logger); + IFileSystem pfs = nca.OpenSectionFileSystem(section.SectionNum, ctx.Options.IntegrityLevel); + pfs.Extract(ctx.Options.ExefsOutDir, ctx.Logger); } } diff --git a/src/hactoolnet/ProcessNsp.cs b/src/hactoolnet/ProcessNsp.cs index d441d4c0..1c0a8a0b 100644 --- a/src/hactoolnet/ProcessNsp.cs +++ b/src/hactoolnet/ProcessNsp.cs @@ -13,7 +13,7 @@ namespace hactoolnet { using (var file = new FileStream(ctx.Options.InFile, FileMode.Open, FileAccess.Read)) { - var pfs = new Pfs(file.AsStorage()); + var pfs = new PartitionFileSystem(file.AsStorage()); ctx.Logger.LogMessage(pfs.Print()); if (ctx.Options.OutDir != null) @@ -23,7 +23,7 @@ namespace hactoolnet } } - private static string Print(this Pfs pfs) + private static string Print(this PartitionFileSystem pfs) { const int colLen = 36; const int fileNameLen = 39; @@ -38,7 +38,7 @@ namespace hactoolnet for (int i = 0; i < pfs.Files.Length; i++) { - PfsFileEntry file = pfs.Files[i]; + PartitionFileEntry file = pfs.Files[i]; string label = i == 0 ? "Files:" : ""; string offsets = $"{file.Offset:x12}-{file.Offset + file.Size:x12}{file.HashValidity.GetValidityString()}";