From f94c6e83e9ec11bdb0074f3bec7bf0d42c96c28e Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 31 Dec 2018 19:15:35 -0700 Subject: [PATCH] Add LocalFileSystem --- src/LibHac/IO/CachedStorage.cs | 1 + src/LibHac/IO/DirectoryEntry.cs | 4 +- src/LibHac/IO/FileBase.cs | 68 ++++++++++++++- src/LibHac/IO/FileSystemExtensions.cs | 88 ++++++++++--------- src/LibHac/IO/IDirectory.cs | 7 +- src/LibHac/IO/IFile.cs | 4 +- src/LibHac/IO/IFileSystem.cs | 4 +- src/LibHac/IO/LocalDirectory.cs | 66 +++++++++++++++ src/LibHac/IO/LocalFile.cs | 49 +++++++++++ src/LibHac/IO/LocalFileSystem.cs | 116 ++++++++++++++++++++++++++ src/LibHac/IO/RomFsDirectory.cs | 23 ++--- src/LibHac/IO/RomFsFile.cs | 8 +- src/LibHac/IO/RomFsFileSystem.cs | 7 +- 13 files changed, 377 insertions(+), 68 deletions(-) create mode 100644 src/LibHac/IO/LocalDirectory.cs create mode 100644 src/LibHac/IO/LocalFile.cs create mode 100644 src/LibHac/IO/LocalFileSystem.cs diff --git a/src/LibHac/IO/CachedStorage.cs b/src/LibHac/IO/CachedStorage.cs index 37a38908..1ca66a73 100644 --- a/src/LibHac/IO/CachedStorage.cs +++ b/src/LibHac/IO/CachedStorage.cs @@ -22,6 +22,7 @@ namespace LibHac.IO for (int i = 0; i < cacheSize; i++) { + // todo why is this rented? var block = new CacheBlock { Buffer = ArrayPool.Shared.Rent(blockSize) }; Blocks.AddLast(block); } diff --git a/src/LibHac/IO/DirectoryEntry.cs b/src/LibHac/IO/DirectoryEntry.cs index 591e9118..a04c48ed 100644 --- a/src/LibHac/IO/DirectoryEntry.cs +++ b/src/LibHac/IO/DirectoryEntry.cs @@ -3,12 +3,14 @@ public class DirectoryEntry { public string Name { get; } + public string FullPath { get; } public DirectoryEntryType Type { get; } public long Size { get; } - public DirectoryEntry(string name, DirectoryEntryType type, long size) + public DirectoryEntry(string name, string fullPath, DirectoryEntryType type, long size) { Name = name; + FullPath = PathTools.Normalize(fullPath); Type = type; Size = size; } diff --git a/src/LibHac/IO/FileBase.cs b/src/LibHac/IO/FileBase.cs index a2872aa3..964a3fd5 100644 --- a/src/LibHac/IO/FileBase.cs +++ b/src/LibHac/IO/FileBase.cs @@ -4,20 +4,80 @@ namespace LibHac.IO { public abstract class FileBase : IFile { + private bool _isDisposed; + public abstract int Read(Span destination, long offset); public abstract void Write(ReadOnlySpan source, long offset); public abstract void Flush(); public abstract long GetSize(); - public abstract long SetSize(); + public abstract void SetSize(long size); - protected int GetAvailableSizeAndValidate(ReadOnlySpan span, long offset) + protected OpenMode Mode { get; set; } + + protected int ValidateReadParamsAndGetSize(ReadOnlySpan span, long offset) { - long fileLength = GetSize(); + if (_isDisposed) throw new ObjectDisposedException(null); + + if ((Mode & OpenMode.Read) == 0) throw new NotSupportedException("File does not allow reading."); + if (span == null) throw new ArgumentNullException(nameof(span)); + if (offset < 0) throw new ArgumentOutOfRangeException(nameof(offset), "Argument must be non-negative."); + + long fileSize = GetSize(); + int size = span.Length; + + if (offset > fileSize) throw new ArgumentOutOfRangeException(nameof(offset), "Offset must be less than the file size."); + + return (int)Math.Min(fileSize - offset, size); + } + + protected void ValidateWriteParams(ReadOnlySpan span, long offset) + { + if (_isDisposed) throw new ObjectDisposedException(null); + + if ((Mode & OpenMode.Write) == 0) throw new NotSupportedException("File does not allow writing."); if (span == null) throw new ArgumentNullException(nameof(span)); if (offset < 0) throw new ArgumentOutOfRangeException(nameof(offset), "Argument must be non-negative."); - return (int)Math.Min(fileLength - offset, span.Length); + long fileSize = GetSize(); + int size = span.Length; + + if (offset + size > fileSize) + { + if ((Mode & OpenMode.Append) == 0) + { + throw new NotSupportedException("File does not allow appending."); + } + + SetSize(offset + size); + } + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (_isDisposed) return; + + if (disposing) + { + Flush(); + } + + _isDisposed = true; } } + + [Flags] + public enum OpenMode + { + Read = 1, + Write = 2, + Append = 4, + ReadWrite = Read | Write + } } diff --git a/src/LibHac/IO/FileSystemExtensions.cs b/src/LibHac/IO/FileSystemExtensions.cs index 143dfba9..dc0683e1 100644 --- a/src/LibHac/IO/FileSystemExtensions.cs +++ b/src/LibHac/IO/FileSystemExtensions.cs @@ -1,68 +1,80 @@ using System; +using System.Buffers; using System.Collections.Generic; -using System.IO; namespace LibHac.IO { public static class FileSystemExtensions { - public static void Extract(this IFileSystem fs, string outDir) + // todo add progress logging + public static void CopyDirectory(this IDirectory source, IDirectory dest) { - IDirectory root = fs.OpenDirectory("/", OpenDirectoryMode.All); + IFileSystem sourceFs = source.ParentFileSystem; + IFileSystem destFs = dest.ParentFileSystem; - foreach (string filename in root.EnumerateFiles()) + foreach (DirectoryEntry entry in source.Read()) { - //Console.WriteLine(filename); - IFile file = fs.OpenFile(filename); - string outPath = Path.Combine(outDir, filename.TrimStart('/')); + string subSrcPath = source.FullPath + '/' + entry.Name; + string subDstPath = dest.FullPath + '/' + entry.Name; - string directoryName = Path.GetDirectoryName(outPath); - if(!string.IsNullOrWhiteSpace(directoryName)) Directory.CreateDirectory(directoryName); - - using (var outFile = new FileStream(outPath, FileMode.Create, FileAccess.ReadWrite)) - { - file.CopyTo(outFile); - } - } - } - - public static IEnumerable EnumerateFiles(this IDirectory directory) - { - DirectoryEntry[] entries = directory.Read(); - - foreach (DirectoryEntry entry in entries) - { if (entry.Type == DirectoryEntryType.Directory) { - foreach (string a in EnumerateFiles(directory.ParentFileSystem.OpenDirectory(entry.Name, OpenDirectoryMode.All))) - { - yield return a; - } + destFs.CreateDirectory(subDstPath); + IDirectory subSrcDir = sourceFs.OpenDirectory(subSrcPath, OpenDirectoryMode.All); + IDirectory subDstDir = destFs.OpenDirectory(subDstPath, OpenDirectoryMode.All); + + subSrcDir.CopyDirectory(subDstDir); } if (entry.Type == DirectoryEntryType.File) { - yield return entry.Name; + destFs.CreateFile(subDstPath, entry.Size); + + using (IFile srcFile = sourceFs.OpenFile(subSrcPath, OpenMode.Read)) + using (IFile dstFile = destFs.OpenFile(subDstPath, OpenMode.Write)) + { + srcFile.CopyTo(dstFile); + } } } } - public static void CopyTo(this IFile file, Stream output) + public static IEnumerable EnumerateEntries(this IDirectory directory) + { + IFileSystem fs = directory.ParentFileSystem; + + foreach (DirectoryEntry entry in directory.Read()) + { + yield return entry; + if (entry.Type != DirectoryEntryType.Directory) continue; + + IDirectory subDir = fs.OpenDirectory(directory.FullPath + '/' + entry.Name, OpenDirectoryMode.All); + + foreach (DirectoryEntry subEntry in subDir.EnumerateEntries()) + { + yield return subEntry; + } + } + } + + // todo add progress logging + public static void CopyTo(this IFile file, IFile dest) { const int bufferSize = 0x8000; - long remaining = file.GetSize(); - long inOffset = 0; - var buffer = new byte[bufferSize]; - while (remaining > 0) + byte[] buffer = ArrayPool.Shared.Rent(bufferSize); + try { - int toWrite = (int)Math.Min(buffer.Length, remaining); - file.Read(buffer.AsSpan(0, toWrite), inOffset); + long inOffset = 0; - output.Write(buffer, 0, toWrite); - remaining -= toWrite; - inOffset += toWrite; + int bytesRead; + while ((bytesRead = file.Read(buffer, inOffset)) != 0) + { + dest.Write(buffer.AsSpan(0, bytesRead), inOffset); + inOffset += bytesRead; + } } + finally { ArrayPool.Shared.Return(buffer); } } } } diff --git a/src/LibHac/IO/IDirectory.cs b/src/LibHac/IO/IDirectory.cs index a26793d6..a75a6a84 100644 --- a/src/LibHac/IO/IDirectory.cs +++ b/src/LibHac/IO/IDirectory.cs @@ -1,10 +1,13 @@ -namespace LibHac.IO +using System.Collections.Generic; + +namespace LibHac.IO { public interface IDirectory { IFileSystem ParentFileSystem { get; } + string FullPath { get; } - DirectoryEntry[] Read(); + IEnumerable Read(); int GetEntryCount(); } } \ No newline at end of file diff --git a/src/LibHac/IO/IFile.cs b/src/LibHac/IO/IFile.cs index 00f26f66..bc137701 100644 --- a/src/LibHac/IO/IFile.cs +++ b/src/LibHac/IO/IFile.cs @@ -2,12 +2,12 @@ namespace LibHac.IO { - public interface IFile + public interface IFile : IDisposable { int Read(Span destination, long offset); void Write(ReadOnlySpan source, long offset); void Flush(); long GetSize(); - long SetSize(); + void SetSize(long size); } } \ No newline at end of file diff --git a/src/LibHac/IO/IFileSystem.cs b/src/LibHac/IO/IFileSystem.cs index 5b40db81..b8ddfd3f 100644 --- a/src/LibHac/IO/IFileSystem.cs +++ b/src/LibHac/IO/IFileSystem.cs @@ -4,17 +4,17 @@ namespace LibHac.IO { public interface IFileSystem { - void Commit(); void CreateDirectory(string path); void CreateFile(string path, long size); void DeleteDirectory(string path); void DeleteFile(string path); IDirectory OpenDirectory(string path, OpenDirectoryMode mode); - IFile OpenFile(string path); + IFile OpenFile(string path, OpenMode mode); void RenameDirectory(string srcPath, string dstPath); void RenameFile(string srcPath, string dstPath); bool DirectoryExists(string path); bool FileExists(string path); + void Commit(); } [Flags] diff --git a/src/LibHac/IO/LocalDirectory.cs b/src/LibHac/IO/LocalDirectory.cs new file mode 100644 index 00000000..c2686b71 --- /dev/null +++ b/src/LibHac/IO/LocalDirectory.cs @@ -0,0 +1,66 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace LibHac.IO +{ + public class LocalDirectory : IDirectory + { + public IFileSystem ParentFileSystem { get; } + public string FullPath { get; } + + private string LocalPath { get; } + private OpenDirectoryMode Mode { get; } + private DirectoryInfo DirInfo { get; } + + public LocalDirectory(LocalFileSystem fs, string path, OpenDirectoryMode mode) + { + ParentFileSystem = fs; + FullPath = path; + LocalPath = fs.ResolveLocalPath(path); + Mode = mode; + + DirInfo = new DirectoryInfo(LocalPath); + } + + public IEnumerable Read() + { + var entries = new List(); + + if (Mode.HasFlag(OpenDirectoryMode.Directories)) + { + foreach (DirectoryInfo dir in DirInfo.EnumerateDirectories()) + { + entries.Add(new DirectoryEntry(dir.Name, FullPath + '/' + dir.Name, DirectoryEntryType.Directory, 0)); + } + } + + if (Mode.HasFlag(OpenDirectoryMode.Files)) + { + foreach (FileInfo file in DirInfo.EnumerateFiles()) + { + entries.Add(new DirectoryEntry(file.Name, FullPath + '/' + file.Name, DirectoryEntryType.File, file.Length)); + } + } + + return entries.ToArray(); + } + + public int GetEntryCount() + { + int count = 0; + + if (Mode.HasFlag(OpenDirectoryMode.Directories)) + { + count += Directory.EnumerateDirectories(LocalPath).Count(); + } + + if (Mode.HasFlag(OpenDirectoryMode.Files)) + { + count += Directory.EnumerateFiles(LocalPath).Count(); + } + + return count; + } + } +} diff --git a/src/LibHac/IO/LocalFile.cs b/src/LibHac/IO/LocalFile.cs new file mode 100644 index 00000000..5974bb55 --- /dev/null +++ b/src/LibHac/IO/LocalFile.cs @@ -0,0 +1,49 @@ +using System; +using System.IO; + +namespace LibHac.IO +{ + public class LocalFile : FileBase + { + private string Path { get; } + private StreamStorage Storage { get; } + + public LocalFile(string path, OpenMode mode) + { + Path = path; + Mode = mode; + Storage = new StreamStorage(new FileStream(Path, FileMode.Open), false); + } + + public override int Read(Span destination, long offset) + { + int toRead = ValidateReadParamsAndGetSize(destination, offset); + + Storage.Read(destination.Slice(0, toRead), offset); + + return toRead; + } + + public override void Write(ReadOnlySpan source, long offset) + { + ValidateWriteParams(source, offset); + + Storage.Write(source, offset); + } + + public override void Flush() + { + Storage.Flush(); + } + + public override long GetSize() + { + return Storage.Length; + } + + public override void SetSize(long size) + { + throw new NotImplementedException(); + } + } +} diff --git a/src/LibHac/IO/LocalFileSystem.cs b/src/LibHac/IO/LocalFileSystem.cs new file mode 100644 index 00000000..7b26b4ab --- /dev/null +++ b/src/LibHac/IO/LocalFileSystem.cs @@ -0,0 +1,116 @@ +using System; +using System.IO; + +namespace LibHac.IO +{ + public class LocalFileSystem : IFileSystem + { + private string BasePath { get; } + + public LocalFileSystem(string basePath) + { + BasePath = Path.GetFullPath(basePath); + } + + internal string ResolveLocalPath(string path) + { + return Path.Combine(BasePath, path.TrimStart('/')); + } + + public void CreateDirectory(string path) + { + path = PathTools.Normalize(path); + Directory.CreateDirectory(ResolveLocalPath(path)); + } + + public void CreateFile(string path, long size) + { + path = PathTools.Normalize(path); + string localPath = ResolveLocalPath(path); + string localDir = Path.GetDirectoryName(localPath); + + if (localDir != null) Directory.CreateDirectory(localDir); + + using (FileStream stream = File.Create(localPath)) + { + stream.SetLength(size); + } + } + + public void DeleteDirectory(string path) + { + path = PathTools.Normalize(path); + + string resolveLocalPath = ResolveLocalPath(path); + Directory.Delete(resolveLocalPath); + } + + public void DeleteFile(string path) + { + path = PathTools.Normalize(path); + + string resolveLocalPath = ResolveLocalPath(path); + File.Delete(resolveLocalPath); + } + + public IDirectory OpenDirectory(string path, OpenDirectoryMode mode) + { + path = PathTools.Normalize(path); + + return new LocalDirectory(this, path, mode); + } + + public IFile OpenFile(string path, OpenMode mode) + { + path = PathTools.Normalize(path); + + string localPath = ResolveLocalPath(path); + return new LocalFile(localPath, mode); + } + + public void RenameDirectory(string srcPath, string dstPath) + { + srcPath = PathTools.Normalize(srcPath); + dstPath = PathTools.Normalize(dstPath); + + string srcLocalPath = ResolveLocalPath(srcPath); + string dstLocalPath = ResolveLocalPath(dstPath); + + string directoryName = Path.GetDirectoryName(dstLocalPath); + if (directoryName != null) Directory.CreateDirectory(directoryName); + Directory.Move(srcLocalPath, dstLocalPath); + } + + public void RenameFile(string srcPath, string dstPath) + { + srcPath = PathTools.Normalize(srcPath); + dstPath = PathTools.Normalize(dstPath); + + string srcLocalPath = ResolveLocalPath(srcPath); + string dstLocalPath = ResolveLocalPath(dstPath); + string dstLocalDir = Path.GetDirectoryName(dstLocalPath); + + if (dstLocalDir != null) Directory.CreateDirectory(dstLocalDir); + File.Move(srcLocalPath, dstLocalPath); + } + + public bool DirectoryExists(string path) + { + path = PathTools.Normalize(path); + + return Directory.Exists(ResolveLocalPath(path)); + } + + public bool FileExists(string path) + { + path = PathTools.Normalize(path); + + return File.Exists(ResolveLocalPath(path)); + } + + public void Commit() + { + throw new NotImplementedException(); + } + } +} diff --git a/src/LibHac/IO/RomFsDirectory.cs b/src/LibHac/IO/RomFsDirectory.cs index 6c25c864..3355197c 100644 --- a/src/LibHac/IO/RomFsDirectory.cs +++ b/src/LibHac/IO/RomFsDirectory.cs @@ -1,16 +1,20 @@ -using System.IO; +using System.Collections.Generic; +using System.IO; namespace LibHac.IO { public class RomFsDirectory : IDirectory { public IFileSystem ParentFileSystem { get; } + public string FullPath { get; } private RomfsDir Directory { get; } private OpenDirectoryMode Mode { get; } public RomFsDirectory(RomFsFileSystem fs, string path, OpenDirectoryMode mode) { + path = PathTools.Normalize(path); + if (!fs.DirectoryDict.TryGetValue(path, out RomfsDir dir)) { throw new DirectoryNotFoundException(path); @@ -18,25 +22,20 @@ namespace LibHac.IO ParentFileSystem = fs; Directory = dir; + FullPath = path; Mode = mode; } - public DirectoryEntry[] Read() + public IEnumerable Read() { - int count = GetEntryCount(); - - var entries = new DirectoryEntry[count]; - int index = 0; - if (Mode.HasFlag(OpenDirectoryMode.Directories)) { RomfsDir dirEntry = Directory.FirstChild; while (dirEntry != null) { - entries[index] = new DirectoryEntry(dirEntry.FullPath, DirectoryEntryType.Directory, 0); + yield return new DirectoryEntry(dirEntry.Name, FullPath + '/' + dirEntry.Name, DirectoryEntryType.Directory, 0); dirEntry = dirEntry.NextSibling; - index++; } } @@ -46,14 +45,10 @@ namespace LibHac.IO while (fileEntry != null) { - entries[index] = - new DirectoryEntry(fileEntry.FullPath, DirectoryEntryType.File, fileEntry.DataLength); + yield return new DirectoryEntry(fileEntry.Name, FullPath + '/' + fileEntry.Name, DirectoryEntryType.File, fileEntry.DataLength); fileEntry = fileEntry.NextSibling; - index++; } } - - return entries; } public int GetEntryCount() diff --git a/src/LibHac/IO/RomFsFile.cs b/src/LibHac/IO/RomFsFile.cs index 9b41d4f3..2f96586c 100644 --- a/src/LibHac/IO/RomFsFile.cs +++ b/src/LibHac/IO/RomFsFile.cs @@ -10,6 +10,7 @@ namespace LibHac.IO public RomFsFile(IStorage baseStorage, long offset, long size) { + Mode = OpenMode.Read; BaseStorage = baseStorage; Offset = offset; Size = size; @@ -17,9 +18,9 @@ namespace LibHac.IO public override int Read(Span destination, long offset) { - long storageOffset = Offset + offset; - int toRead = GetAvailableSizeAndValidate(destination, offset); + int toRead = ValidateReadParamsAndGetSize(destination, offset); + long storageOffset = Offset + offset; BaseStorage.Read(destination.Slice(0, toRead), storageOffset); return toRead; @@ -32,7 +33,6 @@ namespace LibHac.IO public override void Flush() { - throw new NotImplementedException(); } public override long GetSize() @@ -40,7 +40,7 @@ namespace LibHac.IO return Size; } - public override long SetSize() + public override void SetSize(long size) { throw new NotSupportedException(); } diff --git a/src/LibHac/IO/RomFsFileSystem.cs b/src/LibHac/IO/RomFsFileSystem.cs index 078b5f6c..e0356c1e 100644 --- a/src/LibHac/IO/RomFsFileSystem.cs +++ b/src/LibHac/IO/RomFsFileSystem.cs @@ -115,13 +115,18 @@ namespace LibHac.IO return new RomFsDirectory(this, path, mode); } - public IFile OpenFile(string path) + public IFile OpenFile(string path, OpenMode mode) { if (!FileDict.TryGetValue(path, out RomfsFile file)) { throw new FileNotFoundException(); } + if (mode != OpenMode.Read) + { + throw new ArgumentOutOfRangeException(nameof(mode), "RomFs files must be opened read-only."); + } + return OpenFile(file); }