diff --git a/Net/CliParser.cs b/Net/CliParser.cs index eaeb9440..b6f7c2bc 100644 --- a/Net/CliParser.cs +++ b/Net/CliParser.cs @@ -15,7 +15,9 @@ namespace Net new CliOption("title", 1, (o, a) => o.TitleId = ParseTitleId(a[0])), new CliOption("version", 1, (o, a) => o.Version = ParseVersion(a[0])), new CliOption("did", 1, (o, a) => o.DeviceId = ParseTitleId(a[0])), - new CliOption("cert", 1, (o, a) => o.CertFile = a[0]) + new CliOption("cert", 1, (o, a) => o.CertFile = a[0]), + new CliOption("commoncert", 1, (o, a) => o.CommonCertFile = a[0]), + new CliOption("metadata", 0, (o, a) => o.GetMetadata = true) }; public static Options Parse(string[] args) diff --git a/Net/Database.cs b/Net/Database.cs new file mode 100644 index 00000000..55cb07e2 --- /dev/null +++ b/Net/Database.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using libhac; +using Newtonsoft.Json; + +namespace Net +{ + public class Database + { + public Dictionary Titles { get; set; } = new Dictionary(); + public DateTime VersionListTime { get; set; } + + public string Serialize() + { + return JsonConvert.SerializeObject(this); + } + + public static Database Deserialize(string filename) + { + var text = File.ReadAllText(filename); + return JsonConvert.DeserializeObject(text); + } + + public bool IsVersionListCurrent() + { + return VersionListTime.AddDays(1) > DateTime.UtcNow; + } + + public void ImportVersionList(VersionList list) + { + foreach (var title in list.titles) + { + var mainId = long.Parse(title.id, NumberStyles.HexNumber, CultureInfo.InvariantCulture); + long updateId = 0; + bool isUpdate = (mainId & 0x800) != 0; + if (isUpdate) + { + updateId = mainId; + mainId &= ~0x800; + } + + if (!Titles.TryGetValue(mainId, out TitleMetadata titleDb)) + { + titleDb = new TitleMetadata(); + Titles.Add(mainId, titleDb); + } + + titleDb.Id = mainId; + titleDb.UpdateId = updateId; + titleDb.MaxVersion = title.version; + + int maxVersionShort = title.version >> 16; + for(int i = 0; i <= maxVersionShort; i++) + { + var version = i << 16; + + if (!titleDb.Versions.TryGetValue(version, out TitleVersion versionDb)) + { + versionDb = new TitleVersion {Version = version}; + titleDb.Versions.Add(version, versionDb); + } + } + } + } + } + + public class TitleMetadata + { + public long Id { get; set; } + public long UpdateId { get; set; } + public int MaxVersion { get; set; } + public Dictionary Versions { get; set; } = new Dictionary(); + + + } + + public class TitleVersion + { + public bool Exists { get; set; } = true; + public int Version { get; set; } + public Cnmt ContentMetadata { get; set; } + public Nacp Control { get; set; } + } +} diff --git a/Net/Json.cs b/Net/Json.cs new file mode 100644 index 00000000..53287db3 --- /dev/null +++ b/Net/Json.cs @@ -0,0 +1,32 @@ +// ReSharper disable InconsistentNaming +// ReSharper disable CollectionNeverUpdated.Global +using System.Collections.Generic; +using System.IO; +using Newtonsoft.Json; + +namespace Net +{ + public static class Json + { + public static VersionList ReadVersionList(string filename) + { + var text = File.ReadAllText(filename); + var versionList = JsonConvert.DeserializeObject(text); + return versionList; + } + } + + public class VersionList + { + public List titles { get; set; } + public int format_version { get; set; } + public long last_modified { get; set; } + } + + public class VersionListTitle + { + public string id { get; set; } + public int version { get; set; } + public int required_version { get; set; } + } +} diff --git a/Net/Net.csproj b/Net/Net.csproj index 48fc5f7e..2e9dd834 100644 --- a/Net/Net.csproj +++ b/Net/Net.csproj @@ -6,6 +6,10 @@ 7.3 + + + + diff --git a/Net/NetContext.cs b/Net/NetContext.cs index 53807432..91e2d572 100644 --- a/Net/NetContext.cs +++ b/Net/NetContext.cs @@ -10,11 +10,15 @@ namespace Net internal class NetContext { private X509Certificate2 Certificate { get; set; } + private X509Certificate2 CertificateCommon { get; set; } private string Eid { get; } = "lp1"; private ulong Did { get; } private string Firmware { get; } = "5.1.0-3.0"; private string CachePath { get; } = "titles"; private Context ToolCtx { get; } + public Database Db { get; } + + private const string VersionUrl = "https://tagaya.hac.lp1.eshop.nintendo.net/tagaya/hac_versionlist"; public NetContext(Context ctx) { @@ -24,6 +28,25 @@ namespace Net { SetCertificate(ctx.Options.CertFile); } + + if (ctx.Options.CommonCertFile != null) + { + CertificateCommon = new X509Certificate2(ctx.Options.CommonCertFile, "shop"); + } + + var databaseFile = Path.Combine(CachePath, "database.json"); + if (!File.Exists(databaseFile)) + { + File.WriteAllText(databaseFile, new Database().Serialize()); + } + Db = Database.Deserialize(databaseFile); + } + + public void Save() + { + var databaseFile = Path.Combine(CachePath, "database.json"); + + File.WriteAllText(databaseFile, Db.Serialize()); } public void SetCertificate(string filename) @@ -35,6 +58,8 @@ namespace Net { using (var stream = GetCnmtFile(titleId, version)) { + if (stream == null) return null; + var nca = new Nca(ToolCtx.Keyset, stream, true); Stream sect = nca.OpenSection(0, false); var pfs0 = new Pfs0(sect); @@ -68,7 +93,7 @@ namespace Net if (cnmtFiles.Length > 1) { - throw new FileNotFoundException($"More than cnmt file exists for {titleId:x16}v{version}"); + throw new FileNotFoundException($"More than 1 cnmt file exists for {titleId:x16}v{version}"); } return null; @@ -77,7 +102,7 @@ namespace Net public Nacp GetControl(ulong titleId, int version) { var cnmt = GetCnmt(titleId, version); - var controlEntry = cnmt.ContentEntries.FirstOrDefault(x => x.Type == CnmtContentType.Control); + var controlEntry = cnmt?.ContentEntries.FirstOrDefault(x => x.Type == CnmtContentType.Control); if (controlEntry == null) return null; var controlNca = GetNcaFile(titleId, version, controlEntry.NcaId.ToHexString()); @@ -109,6 +134,12 @@ namespace Net var titleDir = GetTitleDir(titleId, version); var ncaId = GetMetadataNcaId(titleId, version); + if (ncaId == null) + { + Console.WriteLine($"Could not get {titleId:x16}v{version} metadata"); + return; + } + var filename = $"{ncaId.ToLower()}.cnmt.nca"; var filePath = Path.Combine(titleDir, filename); DownloadFile(GetMetaUrl(ncaId), filePath); @@ -117,6 +148,7 @@ namespace Net public void DownloadFile(string url, string filePath) { var response = Request("GET", url); + if (response == null) return; using (var responseStream = response.GetResponseStream()) using (var outStream = new FileStream(filePath, FileMode.Create, FileAccess.ReadWrite)) { @@ -151,10 +183,34 @@ namespace Net using (WebResponse response = Request("HEAD", url)) { - return response.Headers.Get("X-Nintendo-Content-ID"); + return response?.Headers.Get("X-Nintendo-Content-ID"); } } + public VersionList GetVersionList() + { + var filename = Path.Combine(CachePath, "hac_versionlist"); + VersionList list = null; + if (Db.IsVersionListCurrent() && File.Exists(filename)) + { + return Json.ReadVersionList(filename); + } + + DownloadVersionList(); + if (File.Exists(filename)) + { + list = Json.ReadVersionList(filename); + } + + return list; + } + + public void DownloadVersionList() + { + DownloadFile(VersionUrl, Path.Combine(CachePath, "hac_versionlist")); + Db.VersionListTime = DateTime.UtcNow; + } + private string GetAtumUrl() { return $"https://atum.hac.{Eid}.d4c.nintendo.net"; @@ -164,11 +220,22 @@ namespace Net { HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url); request.ClientCertificates.Add(Certificate); - request.UserAgent = string.Format("NintendoSDK Firmware/{0} (platform:NX; did:{1}; eid:{2})", Firmware, Did, Eid); + request.UserAgent = $"NintendoSDK Firmware/{Firmware} (platform:NX; did:{Did}; eid:{Eid})"; request.Method = method; - ServicePointManager.ServerCertificateValidationCallback = ((sender, certificate, chain, sslPolicyErrors) => true); - if (((HttpWebResponse)request.GetResponse()).StatusCode != HttpStatusCode.OK) { Console.WriteLine("http error"); return null; } - return request.GetResponse(); + ServicePointManager.ServerCertificateValidationCallback = (sender, certificate, chain, sslPolicyErrors) => true; + + try + { + if (((HttpWebResponse)request.GetResponse()).StatusCode == HttpStatusCode.OK) + return request.GetResponse(); + } + catch (WebException ex) + { + Console.WriteLine(ex.Message); + } + + Console.WriteLine("http error"); + return null; } } } diff --git a/Net/Options.cs b/Net/Options.cs index 1ba2ce3d..b18b7c80 100644 --- a/Net/Options.cs +++ b/Net/Options.cs @@ -11,6 +11,8 @@ namespace Net public int Version; public ulong DeviceId; public string CertFile; + public string CommonCertFile; + public bool GetMetadata; } internal class Context diff --git a/Net/Program.cs b/Net/Program.cs index f9013f9b..4ba46e6e 100644 --- a/Net/Program.cs +++ b/Net/Program.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq; using System.Text; using libhac; @@ -26,32 +27,73 @@ namespace Net private static void ProcessNet(Context ctx) { - if (ctx.Options.DeviceId == 0) { CliParser.PrintWithUsage("A non-zero Device ID must be set."); return; } + if (ctx.Options.GetMetadata) + { + GetMetadata(new NetContext(ctx), ctx.Logger); + return; + } + + if (ctx.Options.TitleId == 0) + { + CliParser.PrintWithUsage("A non-zero Title ID must be set."); + return; + } + var tid = ctx.Options.TitleId; var ver = ctx.Options.Version; var net = new NetContext(ctx); - //GetControls(net); - var cnmt = net.GetCnmt(tid, ver); + if (cnmt == null) return; + ctx.Logger.LogMessage($"Title is of type {cnmt.Type} and has {cnmt.ContentEntries.Length} content entries"); + var control = net.GetControl(tid, ver); + if (control != null) + { + ctx.Logger.LogMessage($"Title has name {control.Languages[0].Title}"); + } foreach (var entry in cnmt.ContentEntries) { - Console.WriteLine($"{entry.NcaId.ToHexString()} {entry.Type}"); + ctx.Logger.LogMessage($"{entry.NcaId.ToHexString()} {entry.Type}"); net.GetNcaFile(tid, ver, entry.NcaId.ToHexString()); } - - var control = net.GetControl(tid, ver); - ; } - private static void GetControls(NetContext net) + private static void GetMetadata(NetContext net, IProgressReport logger = null) { + var versionList = net.GetVersionList(); + net.Db.ImportVersionList(versionList); + net.Save(); + + foreach (var title in net.Db.Titles.Values) + { + foreach (var version in title.Versions.Values.Where(x => x.Exists)) + { + var titleId = version.Version == 0 ? title.Id : title.UpdateId; + try + { + var control = net.GetControl((ulong)titleId, version.Version); + version.Control = control; + if (control == null) version.Exists = false; + logger?.LogMessage($"{titleId}v{version.Version}"); + //Thread.Sleep(300); + } + catch (Exception ex) + { + Console.WriteLine($"Failed getting {titleId}v{version.Version}\n{ex.Message}"); + } + } + // net.Save(); + } + + net.Save(); + return; + var titles = GetTitleIds("titles.txt"); foreach (var title in titles) @@ -108,4 +150,3 @@ namespace Net } } } - diff --git a/Net/Util.cs b/Net/Util.cs new file mode 100644 index 00000000..79386939 --- /dev/null +++ b/Net/Util.cs @@ -0,0 +1,9 @@ +using System; + +namespace Net +{ + public static class Util + { + public static long ToUnixTime(this DateTime inputTime) => (long)(inputTime - new DateTime(1970, 1, 1)).TotalSeconds; + } +}