Net database

Net/CliParser.cs

@@ -15,7 +15,9 @@ namespace Net
             new CliOption("title", 1, (o, a) => o.TitleId = ParseTitleId(a[0])),
             new CliOption("version", 1, (o, a) => o.Version = ParseVersion(a[0])),
             new CliOption("did", 1, (o, a) => o.DeviceId = ParseTitleId(a[0])),
-            new CliOption("cert", 1, (o, a) => o.CertFile = a[0])
+            new CliOption("cert", 1, (o, a) => o.CertFile = a[0]),
+            new CliOption("commoncert", 1, (o, a) => o.CommonCertFile = a[0]),
+            new CliOption("metadata", 0, (o, a) => o.GetMetadata = true)
         };
 
         public static Options Parse(string[] args)

Net/Database.cs

@@ -0,0 +1,86 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using libhac;
+using Newtonsoft.Json;
+
+namespace Net
+{
+    public class Database
+    {
+        public Dictionary<long, TitleMetadata> Titles { get; set; } = new Dictionary<long, TitleMetadata>();
+        public DateTime VersionListTime { get; set; }
+
+        public string Serialize()
+        {
+            return JsonConvert.SerializeObject(this);
+        }
+
+        public static Database Deserialize(string filename)
+        {
+            var text = File.ReadAllText(filename);
+            return JsonConvert.DeserializeObject<Database>(text);
+        }
+
+        public bool IsVersionListCurrent()
+        {
+            return VersionListTime.AddDays(1) > DateTime.UtcNow;
+        }
+
+        public void ImportVersionList(VersionList list)
+        {
+            foreach (var title in list.titles)
+            {
+                var mainId = long.Parse(title.id, NumberStyles.HexNumber, CultureInfo.InvariantCulture);
+                long updateId = 0;
+                bool isUpdate = (mainId & 0x800) != 0;
+                if (isUpdate)
+                {
+                    updateId = mainId;
+                    mainId &= ~0x800;
+                }
+
+                if (!Titles.TryGetValue(mainId, out TitleMetadata titleDb))
+                {
+                    titleDb = new TitleMetadata();
+                    Titles.Add(mainId, titleDb);
+                }
+
+                titleDb.Id = mainId;
+                titleDb.UpdateId = updateId;
+                titleDb.MaxVersion = title.version;
+
+                int maxVersionShort = title.version >> 16;
+                for(int i = 0; i <= maxVersionShort; i++)
+                {
+                    var version = i << 16;
+
+                    if (!titleDb.Versions.TryGetValue(version, out TitleVersion versionDb))
+                    {
+                        versionDb = new TitleVersion {Version = version};
+                        titleDb.Versions.Add(version, versionDb);
+                    }
+                }
+            }
+        }
+    }
+
+    public class TitleMetadata
+    {
+        public long Id { get; set; }
+        public long UpdateId { get; set; }
+        public int MaxVersion { get; set; }
+        public Dictionary<int, TitleVersion> Versions { get; set; } = new Dictionary<int, TitleVersion>();
+
+
+    }
+
+    public class TitleVersion
+    {
+        public bool Exists { get; set; } = true;
+        public int Version { get; set; }
+        public Cnmt ContentMetadata { get; set; }
+        public Nacp Control { get; set; }
+    }
+}

Net/Json.cs

@@ -0,0 +1,32 @@
+// ReSharper disable InconsistentNaming
+// ReSharper disable CollectionNeverUpdated.Global
+using System.Collections.Generic;
+using System.IO;
+using Newtonsoft.Json;
+
+namespace Net
+{
+    public static class Json
+    {
+        public static VersionList ReadVersionList(string filename)
+        {
+            var text = File.ReadAllText(filename);
+            var versionList = JsonConvert.DeserializeObject<VersionList>(text);
+            return versionList;
+        }
+    }
+
+    public class VersionList
+    {
+        public List<VersionListTitle> titles { get; set; }
+        public int format_version { get; set; }
+        public long last_modified { get; set; }
+    }
+
+    public class VersionListTitle
+    {
+        public string id { get; set; }
+        public int version { get; set; }
+        public int required_version { get; set; }
+    }
+}

Net/Net.csproj

@@ -6,6 +6,10 @@
     <LangVersion>7.3</LangVersion>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\libhac\libhac.csproj" />
   </ItemGroup>

Net/NetContext.cs

@@ -10,11 +10,15 @@ namespace Net
     internal class NetContext
     {
         private X509Certificate2 Certificate { get; set; }
+        private X509Certificate2 CertificateCommon { get; set; }
         private string Eid { get; } = "lp1";
         private ulong Did { get; }
         private string Firmware { get; } = "5.1.0-3.0";
         private string CachePath { get; } = "titles";
         private Context ToolCtx { get; }
+        public Database Db { get; }
+
+        private const string VersionUrl = "https://tagaya.hac.lp1.eshop.nintendo.net/tagaya/hac_versionlist";
 
         public NetContext(Context ctx)
         {
@@ -24,6 +28,25 @@ namespace Net
             {
                 SetCertificate(ctx.Options.CertFile);
             }
+
+            if (ctx.Options.CommonCertFile != null)
+            {
+                CertificateCommon = new X509Certificate2(ctx.Options.CommonCertFile, "shop");
+            }
+
+            var databaseFile = Path.Combine(CachePath, "database.json");
+            if (!File.Exists(databaseFile))
+            {
+                File.WriteAllText(databaseFile, new Database().Serialize());
+            }
+            Db = Database.Deserialize(databaseFile);
+        }
+
+        public void Save()
+        {
+            var databaseFile = Path.Combine(CachePath, "database.json");
+
+            File.WriteAllText(databaseFile, Db.Serialize());
         }
 
         public void SetCertificate(string filename)
@@ -35,6 +58,8 @@ namespace Net
         {
             using (var stream = GetCnmtFile(titleId, version))
             {
+                if (stream == null) return null;
+
                 var nca = new Nca(ToolCtx.Keyset, stream, true);
                 Stream sect = nca.OpenSection(0, false);
                 var pfs0 = new Pfs0(sect);
@@ -68,7 +93,7 @@ namespace Net
 
             if (cnmtFiles.Length > 1)
             {
-                throw new FileNotFoundException($"More than cnmt file exists for {titleId:x16}v{version}");
+                throw new FileNotFoundException($"More than 1 cnmt file exists for {titleId:x16}v{version}");
             }
 
             return null;
@@ -77,7 +102,7 @@ namespace Net
         public Nacp GetControl(ulong titleId, int version)
         {
             var cnmt = GetCnmt(titleId, version);
-            var controlEntry = cnmt.ContentEntries.FirstOrDefault(x => x.Type == CnmtContentType.Control);
+            var controlEntry = cnmt?.ContentEntries.FirstOrDefault(x => x.Type == CnmtContentType.Control);
             if (controlEntry == null) return null;
 
             var controlNca = GetNcaFile(titleId, version, controlEntry.NcaId.ToHexString());
@@ -109,6 +134,12 @@ namespace Net
             var titleDir = GetTitleDir(titleId, version);
 
             var ncaId = GetMetadataNcaId(titleId, version);
+            if (ncaId == null)
+            {
+                Console.WriteLine($"Could not get {titleId:x16}v{version} metadata");
+                return;
+            }
+
             var filename = $"{ncaId.ToLower()}.cnmt.nca";
             var filePath = Path.Combine(titleDir, filename);
             DownloadFile(GetMetaUrl(ncaId), filePath);
@@ -117,6 +148,7 @@ namespace Net
         public void DownloadFile(string url, string filePath)
         {
             var response = Request("GET", url);
+            if (response == null) return;
             using (var responseStream = response.GetResponseStream())
             using (var outStream = new FileStream(filePath, FileMode.Create, FileAccess.ReadWrite))
             {
@@ -151,10 +183,34 @@ namespace Net
 
             using (WebResponse response = Request("HEAD", url))
             {
-                return response.Headers.Get("X-Nintendo-Content-ID");
+                return response?.Headers.Get("X-Nintendo-Content-ID");
             }
         }
 
+        public VersionList GetVersionList()
+        {
+            var filename = Path.Combine(CachePath, "hac_versionlist");
+            VersionList list = null;
+            if (Db.IsVersionListCurrent() && File.Exists(filename))
+            {
+                return Json.ReadVersionList(filename);
+            }
+
+            DownloadVersionList();
+            if (File.Exists(filename))
+            {
+                list = Json.ReadVersionList(filename);
+            }
+
+            return list;
+        }
+
+        public void DownloadVersionList()
+        {
+            DownloadFile(VersionUrl, Path.Combine(CachePath, "hac_versionlist"));
+            Db.VersionListTime = DateTime.UtcNow;
+        }
+
         private string GetAtumUrl()
         {
             return $"https://atum.hac.{Eid}.d4c.nintendo.net";
@@ -164,11 +220,22 @@ namespace Net
         {
             HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
             request.ClientCertificates.Add(Certificate);
-            request.UserAgent = string.Format("NintendoSDK Firmware/{0} (platform:NX; did:{1}; eid:{2})", Firmware, Did, Eid);
+            request.UserAgent = $"NintendoSDK Firmware/{Firmware} (platform:NX; did:{Did}; eid:{Eid})";
             request.Method = method;
-            ServicePointManager.ServerCertificateValidationCallback = ((sender, certificate, chain, sslPolicyErrors) => true);
+            ServicePointManager.ServerCertificateValidationCallback = (sender, certificate, chain, sslPolicyErrors) => true;
-            if (((HttpWebResponse)request.GetResponse()).StatusCode != HttpStatusCode.OK) { Console.WriteLine("http error"); return null; }
+
-            return request.GetResponse();
+            try
+            {
+                if (((HttpWebResponse)request.GetResponse()).StatusCode == HttpStatusCode.OK)
+                    return request.GetResponse();
+            }
+            catch (WebException ex)
+            {
+                Console.WriteLine(ex.Message);
+            }
+
+            Console.WriteLine("http error");
+            return null;
         }
     }
 }

Net/Options.cs

@@ -11,6 +11,8 @@ namespace Net
         public int Version;
         public ulong DeviceId;
         public string CertFile;
+        public string CommonCertFile;
+        public bool GetMetadata;
     }
 
     internal class Context

Net/Program.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
+using System.Linq;
 using System.Text;
 using libhac;
 
@@ -26,32 +27,73 @@ namespace Net
 
         private static void ProcessNet(Context ctx)
         {
-            
             if (ctx.Options.DeviceId == 0)
             {
                 CliParser.PrintWithUsage("A non-zero Device ID must be set.");
                 return;
             }
 
+            if (ctx.Options.GetMetadata)
+            {
+                GetMetadata(new NetContext(ctx), ctx.Logger);
+                return;
+            }
+
+            if (ctx.Options.TitleId == 0)
+            {
+                CliParser.PrintWithUsage("A non-zero Title ID must be set.");
+                return;
+            }
+
             var tid = ctx.Options.TitleId;
             var ver = ctx.Options.Version;
 
             var net = new NetContext(ctx);
-            //GetControls(net);
-
             var cnmt = net.GetCnmt(tid, ver);
+            if (cnmt == null) return;
+            ctx.Logger.LogMessage($"Title is of type {cnmt.Type} and has {cnmt.ContentEntries.Length} content entries");
+            var control = net.GetControl(tid, ver);
+            if (control != null)
+            {
+                ctx.Logger.LogMessage($"Title has name {control.Languages[0].Title}");
+            }
             foreach (var entry in cnmt.ContentEntries)
             {
-                Console.WriteLine($"{entry.NcaId.ToHexString()} {entry.Type}");
+                ctx.Logger.LogMessage($"{entry.NcaId.ToHexString()} {entry.Type}");
                 net.GetNcaFile(tid, ver, entry.NcaId.ToHexString());
             }
-
-            var control = net.GetControl(tid, ver);
-            ;
         }
 
-        private static void GetControls(NetContext net)
+        private static void GetMetadata(NetContext net, IProgressReport logger = null)
         {
+            var versionList = net.GetVersionList();
+            net.Db.ImportVersionList(versionList);
+            net.Save();
+
+            foreach (var title in net.Db.Titles.Values)
+            {
+                foreach (var version in title.Versions.Values.Where(x => x.Exists))
+                {
+                    var titleId = version.Version == 0 ? title.Id : title.UpdateId;
+                    try
+                    {
+                        var control = net.GetControl((ulong)titleId, version.Version);
+                        version.Control = control;
+                        if (control == null) version.Exists = false;
+                        logger?.LogMessage($"{titleId}v{version.Version}");
+                        //Thread.Sleep(300);
+                    }
+                    catch (Exception ex)
+                    {
+                        Console.WriteLine($"Failed getting {titleId}v{version.Version}\n{ex.Message}");
+                    }
+                }
+               // net.Save();
+            }
+
+            net.Save();
+            return;
+
             var titles = GetTitleIds("titles.txt");
 
             foreach (var title in titles)
@@ -108,4 +150,3 @@ namespace Net
         }
     }
 }
-

Net/Util.cs

@@ -0,0 +1,9 @@
+using System;
+
+namespace Net
+{
+    public static class Util
+    {
+        public static long ToUnixTime(this DateTime inputTime) => (long)(inputTime - new DateTime(1970, 1, 1)).TotalSeconds;
+    }
+}