fix: read game version 1.1.0 master data and story scenes

scripts/gen_typetrees.py

@@ -53,20 +53,32 @@
     ("Adv", "AdvSystem.Rendering", "AdvFieldRendererFeature"),
     ("App.Runtime", "App.Config", "AppConfigDefaultData"),
     ("DOTween.dll", "DG.Tweening.Core", "DOTweenSettings"),
+    ("Fwk", "Fwk.Cam", "CameraManager"),
     ("Fwk", "Fwk.Rendering", "ScreenCaptureRendererFeature"),
     ("Fwk", "Fwk.Sound", "SoundVolumeSettings"),
+    ("Fwk", "Fwk.UI", "UICameraController"),
+    ("Fwk", "Fwk.UI", "UIInputController"),
+    ("Fwk", "Fwk.UI", "UIManager"),
+    ("Fwk", "Fwk.UI", "UIScreenSizeWatcher"),
     ("Fwk", "Fwk.UI.Rendering", "UIRendererFeature"),
     ("SiriusAsset", "App.GachaAnimation.Blur.DualKawase", "DualKawaseBlurFeature"),
     ("SiriusAsset", "App.GachaAnimation.ChromaticAberration", "ChromaticAberrationFeature"),
     ("SiriusAsset", "App.GachaAnimation.Paraffin", "ParaffinFeature"),
     ("SiriusAsset", "App.GachaAnimation.PostProcessing", "CameraBlendFeature"),
     ("SiriusAsset", "App.GachaAnimation.PostProcessing", "RadialBlurFeature"),
     ("SiriusAsset", "App.UI.Text.DropShadow", "TextDropShadowFeature"),
+    ("Unity.InputSystem", "UnityEngine.InputSystem.UI", "InputSystemUIInputModule"),
     ("Unity.RenderPipelines.Universal.Runtime", "UnityEngine.Rendering.Universal", "PostProcessData"),
+    ("Unity.RenderPipelines.Universal.Runtime", "UnityEngine.Rendering.Universal", "UniversalAdditionalCameraData"),
     ("Unity.RenderPipelines.Universal.Runtime", "UnityEngine.Rendering.Universal", "UniversalRenderPipelineAsset"),
     ("Unity.RenderPipelines.Universal.Runtime", "UnityEngine.Rendering.Universal", "UniversalRendererData"),
     ("Unity.TextMeshPro", "TMPro", "TMP_Settings"),
     ("Unity.TextMeshPro", "TMPro", "TMP_StyleSheet"),
+    ("UnityEngine.UI", "UnityEngine.EventSystems", "EventSystem"),
+    ("UnityEngine.UI", "UnityEngine.EventSystems", "Physics2DRaycaster"),
+    ("UnityEngine.UI", "UnityEngine.UI", "CanvasScaler"),
+    ("UnityEngine.UI", "UnityEngine.UI", "GraphicRaycaster"),
+    ("UnityEngine.UI", "UnityEngine.UI", "Image"),
 )
 DEFAULT_OUT = Path(__file__).resolve().parents[1] / "src" / "nnnotes" / "typetrees"
 

src/nnnotes/advui.py

@@ -900,14 +900,14 @@ def mono(owner, pptr, cls: str, what: str):
     managers = [c for c in comps if c is not None and c.type.name == "MonoBehaviour" and ex.script_class(c) == "UIManager"]
     if len(managers) != 1:
         raise RuntimeError(f"{UI_MANAGER_KEY}: {len(managers)} UIManager on the prefab root")
-    ctrl = mono(managers[0], managers[0].read_typetree()["_cameraController"], "UICameraController",
+    ctrl = mono(managers[0], ex.typetree(managers[0])["_cameraController"], "UICameraController",
                 "UIManager._cameraController")
-    ctrl_tt = ctrl.read_typetree()
+    ctrl_tt = ex.typetree(ctrl)
     cam = ex.deref(ctrl, ctrl_tt["_uiCamera"]) if ctrl_tt["_uiCamera"] else None
     if cam is None or cam.type.name != "Camera":
         raise RuntimeError(f"{UI_MANAGER_KEY}: UICameraController._uiCamera is not a Camera")
     data = mono(ctrl, ctrl_tt["_uiCameraData"], "UniversalAdditionalCameraData", "UICameraController._uiCameraData")
-    cam_tt, data_tt = cam.read_typetree(), data.read_typetree()
+    cam_tt, data_tt = ex.typetree(cam), ex.typetree(data)
     go_pid = cam_tt["m_GameObject"]["m_PathID"]
     if data_tt["m_GameObject"]["m_PathID"] != go_pid:
         raise RuntimeError(f"{UI_MANAGER_KEY}: _uiCamera and _uiCameraData are on different GameObjects")

src/nnnotes/deckdata.py

@@ -131,6 +131,10 @@
 ))
 
 
+# Tables a master data version may leave out (no rows then): the master data of newer game versions does not list them.
+OPTIONAL_TABLES = frozenset({"MasterArenaMusic"})
+
+
 class DeckDataError(ValueError):
     """An input the deck data cannot be made from (the message names it)."""
 
@@ -214,9 +218,12 @@ def read(names):
 
 def read_master(src: MasterSource, key, tables=None) -> tuple[dict[str, list[dict]], dict[str, str]]:
     """The rows (`_allData`) of every table of `tables` (default: TABLES) and the SHA-256 of each file as served.
+    A table of OPTIONAL_TABLES the manifest does not list has no rows and no SHA-256.
     `key`: a master.MasterKey (unused for decoded master data: None)."""
     from . import master
     names = {t: f"{t}.bin" for t in (tables if tables is not None else (t for t, _ in TABLES))}
+    absent = {t for t, n in names.items() if t in OPTIONAL_TABLES and n not in src.hashes}
+    names = {t: n for t, n in names.items() if t not in absent}
     unlisted = [t for t, n in names.items() if n not in src.hashes]
     if unlisted:
         raise DeckDataError(f"master data {src.where}: {MANIFEST} lists no {', '.join(unlisted)}")
@@ -246,6 +253,7 @@ def read_master(src: MasterSource, key, tables=None) -> tuple[dict[str, list[dic
         if not isinstance(rows, list) or not all(isinstance(r, dict) for r in rows):
             raise DeckDataError(f"master data {src.where}: {f} has no `_allData` rows")
         tables[t], shas[t] = rows, sha
+    tables.update((t, []) for t in absent)
     return tables, shas
 
 

src/nnnotes/export.py

@@ -26,6 +26,8 @@
 import math
 import re
 import struct
+import types
+import warnings
 import zlib
 from pathlib import Path
 
@@ -359,7 +361,8 @@ def __init__(self, cat: Catalog, out_dir: Path, player=None, inline_meshes: bool
         self.clip_format = clip_format
         self.follow = tuple(follow)
         self.defer_writes = defer_writes
-        self.script_class = class_of or script_class
+        base_class = class_of or script_class
+        self.script_class = lambda o: self.player.script(o)[2] if self._in_boot(o) else base_class(o)
         self.font_stubs = font_stubs
         self._font_bundles: frozenset | None = None
         self._unwritten: dict[str, object] = {}    # defer_writes: file -> Texture2D
@@ -385,13 +388,56 @@ def __init__(self, cat: Catalog, out_dir: Path, player=None, inline_meshes: bool
         self._gos_at_cache: dict[tuple, list] = {}
         self._path_sources: list[tuple[str, dict]] = []
         self._crcs: dict[str, int] = {}
+        self.unresolved: dict[str, int] = {}        # bundle file name -> references exported as null
+        self._boot_env = None                      # the boot scene of the player data (builtin_root)
+        self._boot_graph: SceneGraph | None = None
+        self._boot_files: set[int] = set()
 
     # --- loading ---------------------------------------------------------
+    def builtin_root(self, key: str):
+        """The root GameObject of the player's boot scene (level0 of the APK's boot data) that stands for the
+        catalog key `key`, None when the catalog lists the key or there is none. Game versions that build the app
+        managers into the boot scene have no catalog key for them; the GameObject has the last name of the key."""
+        if self.player is None or self.cat.has(key):
+            return None
+        name = key.rsplit("/", 1)[-1]
+        env = self._player_scene()
+        graph = self._boot_graph
+        hits = [o for o in env.objects if o.type.name == "GameObject" and graph.go[o.path_id]["m_Name"] == name
+                and not graph.tf[graph.tf_of_go[o.path_id]]["m_Father"]["m_PathID"]]
+        return hits[0] if len(hits) == 1 else None
+
+    def _in_boot(self, o) -> bool:
+        """Is `o` an object of the boot scene taken from the player data (its scripts are stripped of typetrees)?"""
+        return self._boot_env is not None and o.type.name == "MonoBehaviour" and id(o.assets_file) in self._boot_files
+
+    def typetree(self, o) -> dict:
+        """The fields of an object: its own typetree, or for a MonoBehaviour of the boot scene (header included) the
+        player data's embedded typetree of its class."""
+        if not self._in_boot(o):
+            return o.read_typetree()
+        raw = o.get_raw_data()
+        game_object, enabled, script = struct.unpack_from("<iq", raw, 0), raw[12], struct.unpack_from("<iq", raw, 16)
+        pptr = lambda v: {"m_FileID": v[0], "m_PathID": v[1]}
+        return {"m_GameObject": pptr(game_object), "m_Enabled": enabled, "m_Script": pptr(script),
+                **self.player.mono(o)}
+
+    def _player_scene(self):
+        """The objects of the boot scene of the player data (one serialized file, so path ids are unique)."""
+        if self._boot_env is None:
+            objects = [o for o in self.player.env.objects if o.assets_file.name == "level0"]
+            self._boot_env = types.SimpleNamespace(objects=objects)
+            self._boot_files = {id(o.assets_file) for o in objects}
+            self._boot_graph = SceneGraph(self._boot_env)
+        return self._boot_env
+
     def load(self, key: str):
         """(env, graph) of a key's bundle closure, loaded once."""
         if key in self._loaded:
             return self._loaded[key]
-        if self.font_stubs is None:
+        if self.builtin_root(key) is not None:
+            env = self._player_scene()
+        elif self.font_stubs is None:
             env = load_closure(self.cat.fetch_key(key))
         else:
             env = load_closure(self.cat.fetch_key(key, self.font_bundles()))
@@ -442,9 +488,16 @@ def deref(self, owner, pptr: dict):
             return deref(owner, pptr)
         except FileNotFoundError:
             rec = self.font_stubs.objects.get(pptr["m_PathID"]) if self.font_stubs is not None else None
-            if rec is None:
+            if rec is not None:
+                return FontStub(rec, pptr["m_PathID"])
+            name = external_path(owner, pptr)
+            if not name.lower().rsplit("/", 1)[-1].startswith("cab-"):
                 raise
-            return FontStub(rec, pptr["m_PathID"])
+            # a bundle file no bundle of the catalog provides: the reference is exported as null
+            if name not in self.unresolved:
+                warnings.warn(f"{name}: no bundle of the catalog holds this file; references to it are null", stacklevel=2)
+            self.unresolved[name] = self.unresolved.get(name, 0) + 1
+            return None
 
     def graph(self, o) -> SceneGraph:
         return self._graph_of_file[id(o.assets_file)]
@@ -1358,7 +1411,7 @@ def scriptable(self, o) -> dict:
         return {**ref, **self.value(o, {x: y for x, y in tt.items() if x not in HEADER})}
 
     def component(self, o) -> dict:
-        tt = o.read_typetree()
+        tt = self.typetree(o)
         self._register_names(o, tt)
         body = {k: v for k, v in tt.items() if k not in HEADER}
         out = {"type": o.type.name}
@@ -1418,11 +1471,17 @@ def _container_asset(self, env, internal_id: str):
     def key_object(self, key: str):
         """The object an addressable key names (its bundle's container asset)."""
         env, _ = self.load(key)
+        root = self.builtin_root(key)
+        if root is not None:
+            return root
         return self._container_asset(env, self.cat._entry(key)["internal_id"])
 
     def prefab(self, key: str, loaded=None) -> dict:
         """A prefab or scene as a node list; `loaded` = (env, graph) from load(key)."""
         env, graph = loaded or self.load(key)
+        root = self.builtin_root(key)
+        if root is not None:
+            return {"key": key, "nodes": self.hierarchy(env, graph, graph.tf_of_go[root.path_id])}
         iid = self.cat._entry(key)["internal_id"]
         if iid.endswith(".unity"):
             scene_files = set()