hactoolnet: Add an access log option and enable it for NCAs

Progress bars for NCA extraction now show the overall progress instead of per-file progress

src/hactoolnet/AccessLog.cs

@@ -0,0 +1,54 @@
+using System;
+using System.IO;
+using System.Runtime.CompilerServices;
+using LibHac;
+using LibHac.Fs.Accessors;
+
+namespace hactoolnet
+{
+    public class ConsoleAccessLog : IAccessLog
+    {
+        public void Log(TimeSpan startTime, TimeSpan endTime, int handleId, string message, [CallerMemberName] string caller = "")
+        {
+            Console.WriteLine(CommonAccessLog.BuildLogLine(startTime, endTime, handleId, message, caller));
+        }
+    }
+
+    public class ProgressReportAccessLog : IAccessLog
+    {
+        private IProgressReport Logger { get; }
+        public ProgressReportAccessLog(IProgressReport logger)
+        {
+            Logger = logger;
+        }
+
+        public void Log(TimeSpan startTime, TimeSpan endTime, int handleId, string message, [CallerMemberName] string caller = "")
+        {
+            Logger.LogMessage(CommonAccessLog.BuildLogLine(startTime, endTime, handleId, message, caller));
+        }
+    }
+
+    public class TextWriterAccessLog : IAccessLog
+    {
+        private TextWriter Logger { get; }
+
+        public TextWriterAccessLog(TextWriter logger)
+        {
+            Logger = logger;
+        }
+
+        public void Log(TimeSpan startTime, TimeSpan endTime, int handleId, string message, [CallerMemberName] string caller = "")
+        {
+            Logger.WriteLine(CommonAccessLog.BuildLogLine(startTime, endTime, handleId, message, caller));
+        }
+    }
+
+    public static class CommonAccessLog
+    {
+        public static string BuildLogLine(TimeSpan startTime, TimeSpan endTime, int handleId, string message,
+            string caller)
+        {
+            return $"FS_ACCESS: {{ start: {(long)startTime.TotalMilliseconds,9}, end: {(long)endTime.TotalMilliseconds,9}, handle: 0x{handleId:x8}, function: \"{caller}\"{message} }}";
+        }
+    }
+}

src/hactoolnet/CliParser.cs

@@ -18,6 +18,7 @@ namespace hactoolnet
             new CliOption("keyset", 'k', 1, (o, a) => o.Keyfile = a[0]),
             new CliOption("titlekeys", 1, (o, a) => o.TitleKeyFile = a[0]),
             new CliOption("consolekeys", 1, (o, a) => o.ConsoleKeyFile = a[0]),
+            new CliOption("accesslog", 1, (o, a) => o.AccessLog = a[0]),
             new CliOption("section0", 1, (o, a) => o.SectionOut[0] = a[0]),
             new CliOption("section1", 1, (o, a) => o.SectionOut[1] = a[0]),
             new CliOption("section2", 1, (o, a) => o.SectionOut[2] = a[0]),
@@ -170,6 +171,7 @@ namespace hactoolnet
             sb.AppendLine("  -k, --keyset         Load keys from an external file.");
             sb.AppendLine("  -t, --intype=type    Specify input file type [nca, xci, romfs, pfs0, pk11, pk21, ini1, kip1, switchfs, save, ndv0, keygen, romfsbuild, pfsbuild]");
             sb.AppendLine("  --titlekeys <file>   Load title keys from an external file.");
+            sb.AppendLine("  --accesslog <file>   Specify the access log file path.");
             sb.AppendLine("NCA options:");
             sb.AppendLine("  --plaintext <file>   Specify file path for saving a decrypted copy of the NCA.");
             sb.AppendLine("  --header <file>      Specify Header file path.");

src/hactoolnet/ConsoleAccessLog.cs

@@ -1,15 +0,0 @@
-using System;
-using System.Runtime.CompilerServices;
-using LibHac.Fs.Accessors;
-
-namespace hactoolnet
-{
-    public class ConsoleAccessLog : IAccessLogger
-    {
-        public void Log(TimeSpan startTime, TimeSpan endTime, int handleId, string message, [CallerMemberName] string caller = "")
-        {
-            Console.WriteLine(
-                $"FS_ACCESS: {{ start: {startTime.Milliseconds,9}, end: {endTime.Milliseconds,9}, handle: 0x{handleId:x8}, function: \"{caller}\"{message} }}");
-        }
-    }
-}

src/hactoolnet/FsUtils.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Buffers;
+using LibHac;
+using LibHac.Fs;
+using LibHac.Fs.Accessors;
+
+namespace hactoolnet
+{
+    public static class FsUtils
+    {
+        public static void CopyDirectoryWithProgress(FileSystemManager fs, string sourcePath, string destPath,
+            CreateFileOptions options = CreateFileOptions.None, IProgressReport logger = null)
+        {
+            try
+            {
+                logger?.SetTotal(GetTotalSize(fs, sourcePath));
+
+                CopyDirectoryWithProgressInternal(fs, sourcePath, destPath, options, logger);
+            }
+            finally
+            {
+                logger?.SetTotal(0);
+            }
+        }
+
+        private static void CopyDirectoryWithProgressInternal(FileSystemManager fs, string sourcePath, string destPath,
+            CreateFileOptions options, IProgressReport logger)
+        {
+            using (DirectoryHandle sourceHandle = fs.OpenDirectory(sourcePath, OpenDirectoryMode.All))
+            {
+                foreach (DirectoryEntry entry in fs.ReadDirectory(sourceHandle))
+                {
+                    string subSrcPath = PathTools.Normalize(PathTools.Combine(sourcePath, entry.Name));
+                    string subDstPath = PathTools.Normalize(PathTools.Combine(destPath, entry.Name));
+
+                    if (entry.Type == DirectoryEntryType.Directory)
+                    {
+                        fs.CreateDirectory(subDstPath);
+
+                        CopyDirectoryWithProgressInternal(fs, subSrcPath, subDstPath, options, logger);
+                    }
+
+                    if (entry.Type == DirectoryEntryType.File)
+                    {
+                        logger?.LogMessage(subSrcPath);
+                        fs.CreateFile(subDstPath, entry.Size, options);
+
+                        CopyFileWithProgress(fs, subSrcPath, subDstPath, logger);
+                    }
+                }
+            }
+        }
+
+        public static long GetTotalSize(FileSystemManager fs, string path, string searchPattern = "*")
+        {
+            long size = 0;
+
+            foreach (DirectoryEntry entry in fs.EnumerateEntries(path, searchPattern))
+            {
+                size += entry.Size;
+            }
+
+            return size;
+        }
+
+        public static void CopyFileWithProgress(FileSystemManager fs, string sourcePath, string destPath, IProgressReport logger = null)
+        {
+            using (FileHandle sourceHandle = fs.OpenFile(sourcePath, OpenMode.Read))
+            using (FileHandle destHandle = fs.OpenFile(destPath, OpenMode.Write | OpenMode.Append))
+            {
+                const int maxBufferSize = 1024 * 1024;
+
+                long fileSize = fs.GetFileSize(sourceHandle);
+                int bufferSize = (int)Math.Min(maxBufferSize, fileSize);
+
+                byte[] buffer = ArrayPool<byte>.Shared.Rent(bufferSize);
+                try
+                {
+                    for (long offset = 0; offset < fileSize; offset += bufferSize)
+                    {
+                        int toRead = (int)Math.Min(fileSize - offset, bufferSize);
+                        Span<byte> buf = buffer.AsSpan(0, toRead);
+
+                        fs.ReadFile(sourceHandle, buf, offset);
+                        fs.WriteFile(destHandle, buf, offset);
+
+                        logger?.ReportAdd(toRead);
+                    }
+                }
+                finally
+                {
+                    ArrayPool<byte>.Shared.Return(buffer);
+                }
+
+                fs.FlushFile(destHandle);
+            }
+        }
+    }
+}

src/hactoolnet/Options.cs

@@ -15,6 +15,7 @@ namespace hactoolnet
         public string Keyfile;
         public string TitleKeyFile;
         public string ConsoleKeyFile;
+        public string AccessLog;
         public string[] SectionOut = new string[4];
         public string[] SectionOutDir = new string[4];
         public string HeaderOut;
@@ -89,5 +90,6 @@ namespace hactoolnet
         public Options Options;
         public Keyset Keyset;
         public ProgressBar Logger;
+        public Horizon Horizon;
     }
 }

src/hactoolnet/ProcessNca.cs

@@ -42,8 +42,17 @@ namespace hactoolnet
 
                     if (ctx.Options.SectionOutDir[i] != null)
                     {
-                        IFileSystem fs = OpenFileSystem(i);
+                        FileSystemManager fs = ctx.Horizon.Fs;
-                        fs.Extract(ctx.Options.SectionOutDir[i], ctx.Logger);
+
+                        string mountName = $"section{i}";
+
+                        fs.Register(mountName, OpenFileSystem(i));
+                        fs.Register("output", new LocalFileSystem(ctx.Options.SectionOutDir[i]));
+
+                        FsUtils.CopyDirectoryWithProgress(fs, mountName + ":/", "output:/", logger: ctx.Logger);
+
+                        fs.Unmount(mountName);
+                        fs.Unmount("output");
                     }
 
                     if (ctx.Options.Validate && nca.SectionExists(i))
@@ -84,8 +93,15 @@ namespace hactoolnet
 
                     if (ctx.Options.RomfsOutDir != null)
                     {
-                        IFileSystem fs = OpenFileSystemByType(NcaSectionType.Data);
+                        FileSystemManager fs = ctx.Horizon.Fs;
-                        fs.Extract(ctx.Options.RomfsOutDir, ctx.Logger);
+
+                        fs.Register("rom", OpenFileSystemByType(NcaSectionType.Data));
+                        fs.Register("output", new LocalFileSystem(ctx.Options.RomfsOutDir));
+
+                        FsUtils.CopyDirectoryWithProgress(fs, "rom:/", "output:/", logger: ctx.Logger);
+
+                        fs.Unmount("rom");
+                        fs.Unmount("output");
                     }
 
                     if (ctx.Options.ReadBench)
@@ -131,8 +147,15 @@ namespace hactoolnet
 
                     if (ctx.Options.ExefsOutDir != null)
                     {
-                        IFileSystem fs = OpenFileSystemByType(NcaSectionType.Code);
+                        FileSystemManager fs = ctx.Horizon.Fs;
-                        fs.Extract(ctx.Options.ExefsOutDir, ctx.Logger);
+
+                        fs.Register("code", OpenFileSystemByType(NcaSectionType.Code));
+                        fs.Register("output", new LocalFileSystem(ctx.Options.ExefsOutDir));
+
+                        FsUtils.CopyDirectoryWithProgress(fs, "code:/", "output:/", logger: ctx.Logger);
+
+                        fs.Unmount("code");
+                        fs.Unmount("output");
                     }
                 }
 

src/hactoolnet/Program.cs

@@ -37,9 +37,22 @@ namespace hactoolnet
             ctx.Options = CliParser.Parse(args);
             if (ctx.Options == null) return false;
 
+            StreamWriter logWriter = null;
+
+            try
+            {
                 using (var logger = new ProgressBar())
                 {
                     ctx.Logger = logger;
+                    ctx.Horizon = new Horizon(new TimeSpanTimer());
+
+                    if (ctx.Options.AccessLog != null)
+                    {
+                        logWriter = new StreamWriter(ctx.Options.AccessLog);
+                        var accessLog = new TextWriterAccessLog(logWriter);
+                        ctx.Horizon.Fs.SetAccessLog(true, accessLog);
+                    }
+
                     OpenKeyset(ctx);
 
                     if (ctx.Options.RunCustom)
@@ -50,6 +63,11 @@ namespace hactoolnet
 
                     RunTask(ctx);
                 }
+            }
+            finally
+            {
+                logWriter?.Dispose();
+            }
 
             return true;
         }