From a14b513b59a2fecc49c56c530bb1ce639e9eb810 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Fri, 21 Jun 2019 00:50:24 -0500 Subject: [PATCH] hactoolnet: Add an access log option and enable it for NCAs Progress bars for NCA extraction now show the overall progress instead of per-file progress --- src/hactoolnet/AccessLog.cs | 54 ++++++++++++++++ src/hactoolnet/CliParser.cs | 2 + src/hactoolnet/ConsoleAccessLog.cs | 15 ----- src/hactoolnet/FsUtils.cs | 99 ++++++++++++++++++++++++++++++ src/hactoolnet/Options.cs | 2 + src/hactoolnet/ProcessNca.cs | 35 +++++++++-- src/hactoolnet/Program.cs | 36 ++++++++--- 7 files changed, 213 insertions(+), 30 deletions(-) create mode 100644 src/hactoolnet/AccessLog.cs delete mode 100644 src/hactoolnet/ConsoleAccessLog.cs create mode 100644 src/hactoolnet/FsUtils.cs diff --git a/src/hactoolnet/AccessLog.cs b/src/hactoolnet/AccessLog.cs new file mode 100644 index 00000000..89226520 --- /dev/null +++ b/src/hactoolnet/AccessLog.cs @@ -0,0 +1,54 @@ +using System; +using System.IO; +using System.Runtime.CompilerServices; +using LibHac; +using LibHac.Fs.Accessors; + +namespace hactoolnet +{ + public class ConsoleAccessLog : IAccessLog + { + public void Log(TimeSpan startTime, TimeSpan endTime, int handleId, string message, [CallerMemberName] string caller = "") + { + Console.WriteLine(CommonAccessLog.BuildLogLine(startTime, endTime, handleId, message, caller)); + } + } + + public class ProgressReportAccessLog : IAccessLog + { + private IProgressReport Logger { get; } + public ProgressReportAccessLog(IProgressReport logger) + { + Logger = logger; + } + + public void Log(TimeSpan startTime, TimeSpan endTime, int handleId, string message, [CallerMemberName] string caller = "") + { + Logger.LogMessage(CommonAccessLog.BuildLogLine(startTime, endTime, handleId, message, caller)); + } + } + + public class TextWriterAccessLog : IAccessLog + { + private TextWriter Logger { get; } + + public TextWriterAccessLog(TextWriter logger) + { + Logger = logger; + } + + public void Log(TimeSpan startTime, TimeSpan endTime, int handleId, string message, [CallerMemberName] string caller = "") + { + Logger.WriteLine(CommonAccessLog.BuildLogLine(startTime, endTime, handleId, message, caller)); + } + } + + public static class CommonAccessLog + { + public static string BuildLogLine(TimeSpan startTime, TimeSpan endTime, int handleId, string message, + string caller) + { + return $"FS_ACCESS: {{ start: {(long)startTime.TotalMilliseconds,9}, end: {(long)endTime.TotalMilliseconds,9}, handle: 0x{handleId:x8}, function: \"{caller}\"{message} }}"; + } + } +} diff --git a/src/hactoolnet/CliParser.cs b/src/hactoolnet/CliParser.cs index 03394f98..3f8ab7f1 100644 --- a/src/hactoolnet/CliParser.cs +++ b/src/hactoolnet/CliParser.cs @@ -18,6 +18,7 @@ namespace hactoolnet new CliOption("keyset", 'k', 1, (o, a) => o.Keyfile = a[0]), new CliOption("titlekeys", 1, (o, a) => o.TitleKeyFile = a[0]), new CliOption("consolekeys", 1, (o, a) => o.ConsoleKeyFile = a[0]), + new CliOption("accesslog", 1, (o, a) => o.AccessLog = a[0]), new CliOption("section0", 1, (o, a) => o.SectionOut[0] = a[0]), new CliOption("section1", 1, (o, a) => o.SectionOut[1] = a[0]), new CliOption("section2", 1, (o, a) => o.SectionOut[2] = a[0]), @@ -170,6 +171,7 @@ namespace hactoolnet sb.AppendLine(" -k, --keyset Load keys from an external file."); sb.AppendLine(" -t, --intype=type Specify input file type [nca, xci, romfs, pfs0, pk11, pk21, ini1, kip1, switchfs, save, ndv0, keygen, romfsbuild, pfsbuild]"); sb.AppendLine(" --titlekeys Load title keys from an external file."); + sb.AppendLine(" --accesslog Specify the access log file path."); sb.AppendLine("NCA options:"); sb.AppendLine(" --plaintext Specify file path for saving a decrypted copy of the NCA."); sb.AppendLine(" --header Specify Header file path."); diff --git a/src/hactoolnet/ConsoleAccessLog.cs b/src/hactoolnet/ConsoleAccessLog.cs deleted file mode 100644 index 70813b1f..00000000 --- a/src/hactoolnet/ConsoleAccessLog.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; -using System.Runtime.CompilerServices; -using LibHac.Fs.Accessors; - -namespace hactoolnet -{ - public class ConsoleAccessLog : IAccessLogger - { - public void Log(TimeSpan startTime, TimeSpan endTime, int handleId, string message, [CallerMemberName] string caller = "") - { - Console.WriteLine( - $"FS_ACCESS: {{ start: {startTime.Milliseconds,9}, end: {endTime.Milliseconds,9}, handle: 0x{handleId:x8}, function: \"{caller}\"{message} }}"); - } - } -} diff --git a/src/hactoolnet/FsUtils.cs b/src/hactoolnet/FsUtils.cs new file mode 100644 index 00000000..94eda776 --- /dev/null +++ b/src/hactoolnet/FsUtils.cs @@ -0,0 +1,99 @@ +using System; +using System.Buffers; +using LibHac; +using LibHac.Fs; +using LibHac.Fs.Accessors; + +namespace hactoolnet +{ + public static class FsUtils + { + public static void CopyDirectoryWithProgress(FileSystemManager fs, string sourcePath, string destPath, + CreateFileOptions options = CreateFileOptions.None, IProgressReport logger = null) + { + try + { + logger?.SetTotal(GetTotalSize(fs, sourcePath)); + + CopyDirectoryWithProgressInternal(fs, sourcePath, destPath, options, logger); + } + finally + { + logger?.SetTotal(0); + } + } + + private static void CopyDirectoryWithProgressInternal(FileSystemManager fs, string sourcePath, string destPath, + CreateFileOptions options, IProgressReport logger) + { + using (DirectoryHandle sourceHandle = fs.OpenDirectory(sourcePath, OpenDirectoryMode.All)) + { + foreach (DirectoryEntry entry in fs.ReadDirectory(sourceHandle)) + { + string subSrcPath = PathTools.Normalize(PathTools.Combine(sourcePath, entry.Name)); + string subDstPath = PathTools.Normalize(PathTools.Combine(destPath, entry.Name)); + + if (entry.Type == DirectoryEntryType.Directory) + { + fs.CreateDirectory(subDstPath); + + CopyDirectoryWithProgressInternal(fs, subSrcPath, subDstPath, options, logger); + } + + if (entry.Type == DirectoryEntryType.File) + { + logger?.LogMessage(subSrcPath); + fs.CreateFile(subDstPath, entry.Size, options); + + CopyFileWithProgress(fs, subSrcPath, subDstPath, logger); + } + } + } + } + + public static long GetTotalSize(FileSystemManager fs, string path, string searchPattern = "*") + { + long size = 0; + + foreach (DirectoryEntry entry in fs.EnumerateEntries(path, searchPattern)) + { + size += entry.Size; + } + + return size; + } + + public static void CopyFileWithProgress(FileSystemManager fs, string sourcePath, string destPath, IProgressReport logger = null) + { + using (FileHandle sourceHandle = fs.OpenFile(sourcePath, OpenMode.Read)) + using (FileHandle destHandle = fs.OpenFile(destPath, OpenMode.Write | OpenMode.Append)) + { + const int maxBufferSize = 1024 * 1024; + + long fileSize = fs.GetFileSize(sourceHandle); + int bufferSize = (int)Math.Min(maxBufferSize, fileSize); + + byte[] buffer = ArrayPool.Shared.Rent(bufferSize); + try + { + for (long offset = 0; offset < fileSize; offset += bufferSize) + { + int toRead = (int)Math.Min(fileSize - offset, bufferSize); + Span buf = buffer.AsSpan(0, toRead); + + fs.ReadFile(sourceHandle, buf, offset); + fs.WriteFile(destHandle, buf, offset); + + logger?.ReportAdd(toRead); + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + + fs.FlushFile(destHandle); + } + } + } +} diff --git a/src/hactoolnet/Options.cs b/src/hactoolnet/Options.cs index e1765ed6..1a327eaf 100644 --- a/src/hactoolnet/Options.cs +++ b/src/hactoolnet/Options.cs @@ -15,6 +15,7 @@ namespace hactoolnet public string Keyfile; public string TitleKeyFile; public string ConsoleKeyFile; + public string AccessLog; public string[] SectionOut = new string[4]; public string[] SectionOutDir = new string[4]; public string HeaderOut; @@ -89,5 +90,6 @@ namespace hactoolnet public Options Options; public Keyset Keyset; public ProgressBar Logger; + public Horizon Horizon; } } diff --git a/src/hactoolnet/ProcessNca.cs b/src/hactoolnet/ProcessNca.cs index fda5aabb..590619fc 100644 --- a/src/hactoolnet/ProcessNca.cs +++ b/src/hactoolnet/ProcessNca.cs @@ -42,8 +42,17 @@ namespace hactoolnet if (ctx.Options.SectionOutDir[i] != null) { - IFileSystem fs = OpenFileSystem(i); - fs.Extract(ctx.Options.SectionOutDir[i], ctx.Logger); + FileSystemManager fs = ctx.Horizon.Fs; + + string mountName = $"section{i}"; + + fs.Register(mountName, OpenFileSystem(i)); + fs.Register("output", new LocalFileSystem(ctx.Options.SectionOutDir[i])); + + FsUtils.CopyDirectoryWithProgress(fs, mountName + ":/", "output:/", logger: ctx.Logger); + + fs.Unmount(mountName); + fs.Unmount("output"); } if (ctx.Options.Validate && nca.SectionExists(i)) @@ -84,8 +93,15 @@ namespace hactoolnet if (ctx.Options.RomfsOutDir != null) { - IFileSystem fs = OpenFileSystemByType(NcaSectionType.Data); - fs.Extract(ctx.Options.RomfsOutDir, ctx.Logger); + FileSystemManager fs = ctx.Horizon.Fs; + + fs.Register("rom", OpenFileSystemByType(NcaSectionType.Data)); + fs.Register("output", new LocalFileSystem(ctx.Options.RomfsOutDir)); + + FsUtils.CopyDirectoryWithProgress(fs, "rom:/", "output:/", logger: ctx.Logger); + + fs.Unmount("rom"); + fs.Unmount("output"); } if (ctx.Options.ReadBench) @@ -131,8 +147,15 @@ namespace hactoolnet if (ctx.Options.ExefsOutDir != null) { - IFileSystem fs = OpenFileSystemByType(NcaSectionType.Code); - fs.Extract(ctx.Options.ExefsOutDir, ctx.Logger); + FileSystemManager fs = ctx.Horizon.Fs; + + fs.Register("code", OpenFileSystemByType(NcaSectionType.Code)); + fs.Register("output", new LocalFileSystem(ctx.Options.ExefsOutDir)); + + FsUtils.CopyDirectoryWithProgress(fs, "code:/", "output:/", logger: ctx.Logger); + + fs.Unmount("code"); + fs.Unmount("output"); } } diff --git a/src/hactoolnet/Program.cs b/src/hactoolnet/Program.cs index fa683f20..531fb176 100644 --- a/src/hactoolnet/Program.cs +++ b/src/hactoolnet/Program.cs @@ -37,18 +37,36 @@ namespace hactoolnet ctx.Options = CliParser.Parse(args); if (ctx.Options == null) return false; - using (var logger = new ProgressBar()) + StreamWriter logWriter = null; + + try { - ctx.Logger = logger; - OpenKeyset(ctx); - - if (ctx.Options.RunCustom) + using (var logger = new ProgressBar()) { - CustomTask(ctx); - return true; - } + ctx.Logger = logger; + ctx.Horizon = new Horizon(new TimeSpanTimer()); - RunTask(ctx); + if (ctx.Options.AccessLog != null) + { + logWriter = new StreamWriter(ctx.Options.AccessLog); + var accessLog = new TextWriterAccessLog(logWriter); + ctx.Horizon.Fs.SetAccessLog(true, accessLog); + } + + OpenKeyset(ctx); + + if (ctx.Options.RunCustom) + { + CustomTask(ctx); + return true; + } + + RunTask(ctx); + } + } + finally + { + logWriter?.Dispose(); } return true;