From d647d3de5337ba3ba65b855ee90e843dafa9e8d2 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 14 Oct 2026 21:53:17 +0800 Subject: [PATCH 01/91] refactor!: keep downloads, results and runs in one depot `[paths] depot` (`--depot`, `NNNOTES_PATHS_DEPOT`) names one directory for everything a configuration reads and computes: - `downloads/`: the game files as downloaded, under their own names (catalogs, decrypted bundles, raw CDN files, files read from the APK, one directory per catalog source); - `cas/`, `ac/`, `inputs/`, `catalogs/`, `runs/` and `costs/`: the results of the stages by content, their task results, input identities, catalog versions and run manifests. The `depot` command group replaces the `store` group and adds `depot gc`, which removes the runs, results and contents that no kept run reaches (`--keep N` latest runs per selection, `--dry-run`). Task inputs locate their bytes with the locator kinds `depot`, `download` (a path in `downloads/`), `file` and `catalog`. The catalog API speaks of downloads: `Catalog(catalog_bytes, download_dir)`, `Catalog.downloaded`, `Catalog.downloaded_raw`, `Catalog.file_name`, `jp.Source.download_dir`, and `jp.open_catalog` takes the downloads directory. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 +- README.zh-CN.md | 8 +- docs/architecture.md | 10 +- docs/assets.md | 90 +++--- docs/cli.md | 38 ++- docs/commands.md | 31 +- docs/configuration.md | 35 ++- docs/contracts.md | 43 +-- docs/determinism.md | 6 +- docs/export.md | 2 +- docs/jp.md | 14 +- docs/music-data.md | 4 +- docs/quickstart.md | 6 +- docs/schemas.md | 6 +- docs/ui.md | 2 +- docs/views.md | 8 +- src/nnnotes/cli/__init__.py | 6 +- src/nnnotes/cli/assets.py | 151 +++++----- src/nnnotes/cli/browse.py | 21 +- src/nnnotes/cli/bundle.py | 4 +- src/nnnotes/cli/catalog.py | 26 +- src/nnnotes/cli/common.py | 8 +- src/nnnotes/cli/{store.py => depot.py} | 41 ++- src/nnnotes/cli/export.py | 8 +- src/nnnotes/cli/musicdata.py | 2 +- src/nnnotes/content/ui.py | 4 +- src/nnnotes/formats/cri.py | 2 +- src/nnnotes/musicdata/deckdata.py | 12 +- src/nnnotes/pipeline/__init__.py | 2 +- src/nnnotes/pipeline/catalogdb.py | 28 +- src/nnnotes/pipeline/census.py | 6 +- src/nnnotes/pipeline/contract.py | 2 +- src/nnnotes/pipeline/cristages.py | 84 +++--- src/nnnotes/pipeline/{store.py => depot.py} | 171 ++++++++--- src/nnnotes/pipeline/layout.py | 36 +-- src/nnnotes/pipeline/link.py | 28 +- src/nnnotes/pipeline/modelstages.py | 28 +- src/nnnotes/pipeline/objexport.py | 40 +-- src/nnnotes/pipeline/orchestrate.py | 50 ++-- src/nnnotes/pipeline/plan.py | 20 +- src/nnnotes/pipeline/stages.py | 48 +-- src/nnnotes/pipeline/views.py | 26 +- src/nnnotes/pipeline/voices.py | 42 +-- src/nnnotes/schemas/catalogs.schema.json | 2 +- src/nnnotes/schemas/common.schema.json | 6 +- src/nnnotes/schemas/music-data.schema.json | 2 +- src/nnnotes/schemas/run.schema.json | 2 +- src/nnnotes/site/sitefiles.py | 2 +- src/nnnotes/source/catalog.py | 78 ++--- src/nnnotes/source/jp.py | 15 +- src/nnnotes/source/nnnotes.example.toml | 12 +- src/nnnotes/source/session.py | 31 +- tests/cli/test_assets.py | 198 ++++++------ tests/cli/test_cli.py | 49 +-- tests/cli/test_voices.py | 12 +- tests/content/test_ui.py | 2 +- tests/musicdata/test_deckdata.py | 8 +- tests/musicdata/test_musicdata.py | 4 +- tests/pipeline/test_catalogdb.py | 32 +- tests/pipeline/test_census.py | 20 +- tests/pipeline/test_contract.py | 14 +- tests/pipeline/test_cristages.py | 168 +++++------ .../pipeline/{test_store.py => test_depot.py} | 58 ++-- tests/pipeline/test_depot_gc.py | 74 +++++ tests/pipeline/test_goldens.py | 42 +-- tests/pipeline/test_layout.py | 86 +++--- tests/pipeline/test_link.py | 50 ++-- tests/pipeline/test_modelstages.py | 114 +++---- tests/pipeline/test_objexport.py | 282 +++++++++--------- tests/pipeline/test_orchestrate.py | 36 +-- tests/pipeline/test_plan.py | 76 ++--- tests/pipeline/test_stages.py | 90 +++--- tests/pipeline/test_views.py | 70 ++--- tests/pipeline/test_voices.py | 80 ++--- tests/pipeline/test_voices_story.py | 46 +-- tests/site/test_live_options_web.py | 2 +- tests/site/test_pipeline.py | 8 +- tests/site/test_readsets.py | 2 +- tests/source/test_catalog.py | 24 +- tests/source/test_catalog_lazy.py | 46 +-- tests/source/test_catalog_versions.py | 40 +-- tests/source/test_configfile.py | 8 +- tests/source/test_jp.py | 64 ++-- 83 files changed, 1688 insertions(+), 1474 deletions(-) rename src/nnnotes/cli/{store.py => depot.py} (54%) rename src/nnnotes/pipeline/{store.py => depot.py} (75%) rename tests/pipeline/{test_store.py => test_depot.py} (83%) create mode 100644 tests/pipeline/test_depot_gc.py diff --git a/README.md b/README.md index 84ce615..d1836c4 100644 --- a/README.md +++ b/README.md @@ -47,13 +47,13 @@ questions: `nnnotes config init --user --no-input --set SECTION.KEY=VALUE ...`. | `config init\|set\|unset\|check\|path` | write, edit and check the config file | | `servers` | the game API's server list: regions with their CDN and API roots | | `catalog keys\|browse` | the addressable keys of a region's catalog; a local web page to browse catalogs and bundles | -| `catalog versions\|import\|fetch\|diff` | catalog versions kept in the store, and what changed between two | -| `bundle pull` | fetch and decrypt the bundle closure of keys into the cache | +| `catalog versions\|import\|fetch\|diff` | catalog versions kept in the depot, and what changed between two | +| `bundle pull` | fetch and decrypt the bundle closure of keys into the depot | | `master version\|download\|decode` | the master data version a region serves, its download and its decoding to JSON | -| `export assets` | every object of the selected bundles in common formats, through the content-addressed store | +| `export assets` | every object of the selected bundles in common formats, through the content-addressed depot | | `export episode\|story\|live\|live2d\|spot\|room\|shader\|audio\|ui` | one subject: an ADV episode, a story, a live, a Live2D model, a spot, a room, shaders, a cue sheet, the UI library | | `apk settings\|hca-key` | the player's graphics settings and the CRI HCA keycode of `base.apk` | -| `store plan\|run\|verify` | what `export assets` would run and why, single tasks, the store's check | +| `depot plan\|run\|verify\|gc` | what `export assets` would run and why, single tasks, the depot's check and collection | | `voices list\|search\|summary\|get` | the character and story voices of the master data | | `site build` | the static site of ournotes-player: charts, Live2D models and stories, in several regions and languages | | `music-data` | every song and chart in one JSON file, with the deck model's chart statistics | diff --git a/README.zh-CN.md b/README.zh-CN.md index 243f5f2..0d49d32 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -45,13 +45,13 @@ nnnotes site build -o out/site --all --player path/to/ournotes-player | `config init\|set\|unset\|check\|path` | 写入、修改和检查配置文件 | | `servers` | 游戏 API 的服务器列表：各区服及其 CDN 与 API 根地址 | | `catalog keys\|browse` | 区服 catalog 的资源键；在本地网页中浏览 catalog 与资源包 | -| `catalog versions\|import\|fetch\|diff` | store 中保存的 catalog 版本，以及两个版本之间的变化 | -| `bundle pull` | 下载并解密资源键的依赖闭包到缓存 | +| `catalog versions\|import\|fetch\|diff` | depot 中保存的 catalog 版本，以及两个版本之间的变化 | +| `bundle pull` | 下载并解密资源键的依赖闭包到 depot | | `master version\|download\|decode` | 区服当前的 masterdata 版本、下载，以及解码为 JSON | -| `export assets` | 经内容寻址的 store，把所选资源包中的每个对象导出为通用格式 | +| `export assets` | 经内容寻址的 depot，把所选资源包中的每个对象导出为通用格式 | | `export episode\|story\|live\|live2d\|spot\|room\|shader\|audio\|ui` | 单个对象：ADV 剧情集、完整剧情、谱面、Live2D 模型、据点、房间、着色器、cue sheet、UI 库 | | `apk settings\|hca-key` | `base.apk` 中的画面设置与 CRI HCA 密钥 | -| `store plan\|run\|verify` | `export assets` 将运行什么及原因、单独运行任务、检查 store | +| `depot plan\|run\|verify\|gc` | `export assets` 将运行什么及原因、单独运行任务、检查和回收 depot | | `voices list\|search\|summary\|get` | masterdata 中的角色语音与剧情语音 | | `site build` | ournotes-player 静态站点：谱面、Live2D 模型与剧情，支持多个区服与语言 | | `music-data` | 全部歌曲与谱面的单个 JSON 文件，附组卡模型的谱面统计 | diff --git a/docs/architecture.md b/docs/architecture.md index e259fe3..8b5646b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,7 +10,7 @@ subpackage of `nnnotes` and imports only the layers below it. |---|---|---| | Command line | `nnnotes.cli` | one module per command group; parses options, reads the configuration, prints summaries and `--json` reports | | Products | `nnnotes.site`, `nnnotes.musicdata` | the ournotes-player site (charts, models, stories, indexes); the music data file and its deck statistics | -| Pipeline | `nnnotes.pipeline` | stages, task descriptions and results, the content-addressed store, plans, layouts, the catalog versions, the views of the master data | +| Pipeline | `nnnotes.pipeline` | stages, task descriptions and results, the content-addressed depot, plans, layouts, the catalog versions, the views of the master data | | Content | `nnnotes.content` | the game's subjects as exports: ADV episodes and stories, lives and scores, spots, rooms, the UI library | | Unity objects | `nnnotes.unity` | bundle reading, the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | | File formats | `nnnotes.formats` | formats read from bytes: CRI ACB/AWB and HCA, USM movies, CRI Lips data, shaders, motions | @@ -27,12 +27,12 @@ and print, and the work they run is in the library modules below them. Every setting comes from the configuration ([sources and precedence](configuration.md#sources-and-precedence)). `nnnotes.source.session.Session` turns one configuration into the data sources a command needs: the catalog of a region, its decoded master data and the master data version it serves, the APK, the bundle and master data keys and -the store root. It reads a setting when it is first needed and raises a `ConfigError` naming the setting when it is missing or wrong; +the depot root. It reads a setting when it is first needed and raises a `ConfigError` naming the setting when it is missing or wrong; the command line turns that error into exit status 2. The catalog (`nnnotes.source.catalog.Catalog`) resolves an addressable key to its bundle and the bundle's dependency closure, takes a bundle from the APK when the APK has it and from the CDN otherwise, and keeps fetched and -decrypted bundles in the cache directory. +decrypted bundles in the depot's `downloads/` directory. ## Two ways to export @@ -42,8 +42,8 @@ decrypted bundles in the cache directory. - **Every asset.** `export assets` splits the work into stages ([stages](stages.md)). A task is one stage applied to one subject (a bundle, a content id, a view); its key covers exactly what its output depends on: the stage and converter versions, the input contents and the options ([contracts](contracts.md)). Results and their files are - stored once by content hash in the store, and the output directory is laid out from the store. A second run finds - every unchanged result in the store and only lays out what changed; `store plan` says what would run and why + stored once by content hash in the depot, and the output directory is laid out from the depot. A second run finds + every unchanged result in the depot and only lays out what changed; `depot plan` says what would run and why ([asset export](assets.md)). Both ways use the same readers below the pipeline layer, so a texture, shader, mesh or audio stream is converted by diff --git a/docs/assets.md b/docs/assets.md index 4e089bf..f52f1e6 100644 --- a/docs/assets.md +++ b/docs/assets.md @@ -5,29 +5,29 @@ meshes as binary glTF, text assets and fonts as their stored bytes, prefabs, mat class as JSON. It also writes the semantic views of the master data (cards, characters, stamps, …) with the files of the objects they name. The work is split into stages ([stages.md](stages.md)); each stage runs one task per subject (a bundle, a view, …), and a task's result is stored under a key that covers exactly what its output depends on -([contracts.md](contracts.md)). A second run over the same data finds every result in the store and only +([contracts.md](contracts.md)). A second run over the same data finds every result in the depot and only rewrites what changed. ``` nnnotes export assets -o out/assets nnnotes export assets -o out/cards --select group:membercard --views cards -nnnotes store plan --select key:Image/Jacket/ -nnnotes store plan --check && echo "up to date" +nnnotes depot plan --select key:Image/Jacket/ +nnnotes depot plan --check && echo "up to date" ``` ## Commands ``` nnnotes export assets -o OUT [--layout original[,cas]] [--select group:G|key:PREFIX|bundle:GLOB ...] [--views all|none|a,b] - [--store DIR] [--workers N] [--memory GiB] [--fetch-workers N] [--stored-catalog L|SHA] + [--depot DIR] [--workers N] [--memory GiB] [--fetch-workers N] [--stored-catalog L|SHA] [--png-level N] [--flac-level N] [--only-class C,...] [--link auto|clone|hard|copy] [--strict] [--dry-run] [--explain] -nnnotes store plan [the selection and parameters of export] [-o OUT] [--json] [--since RUN] [--check] [--census] +nnnotes depot plan [the selection and parameters of export] [-o OUT] [--json] [--since RUN] [--check] [--census] [--emit-tasks DIR] [--why TASK] -nnnotes store run TASK.json|DIR [...] [--store DIR] [--fetch] [--force] +nnnotes depot run TASK.json|DIR [...] [--depot DIR] [--fetch] [--force] nnnotes catalog versions [--json] | import FILE [--label L] [--apk-catalog FILE] [--resource-version V] | fetch [--label L] | diff OLD NEW [--json] -nnnotes store verify [--quick] [--workers N] +nnnotes depot verify [--quick] [--workers N] ``` ### export assets @@ -38,24 +38,24 @@ nnnotes store verify [--quick] [--workers N] | `--select SEL` | what to export (repeatable, the union; default everything): `group:G` the bundles of an Addressables group (the stable bundle name up to `_assets_`, else up to its first `_`), `key:PREFIX` the bundles and raw files the catalog keys under `PREFIX` load, `bundle:GLOB` the bundles and raw files whose stable name matches | | `--views` | the semantic views to build ([views.md](views.md)): `all`, `none` or names, comma-separated; default `all` when master data is set, else `none` | | `--layout` | `original`, `cas` or both (below) | -| `--store DIR` | the store (`[paths] store`; default `/store`) | +| `--depot DIR` | the depot (`[paths] depot`) | | `--workers N` | worker processes (default: the CPUs the process may run on, its affinity mask as `taskset` or a container's cpuset sets it; `0`: every task in this process) | | `--memory GiB` | memory budget (default: 80 % of the physical memory where it can be read, else none) | -| `--fetch-workers N` | parallel downloads of bundles not in the cache (default 8) | -| `--stored-catalog` | a catalog version of the store (`catalog versions`) instead of the current catalog | +| `--fetch-workers N` | parallel downloads of bundles not downloaded yet (default 8) | +| `--stored-catalog` | a catalog version of the depot (`catalog versions`) instead of the current catalog | | `--png-level N` | zlib level 0-9 of the PNG files (default 6); part of the task keys | | `--flac-level N` | FLAC compression level 0-12 of the CRI audio and the movies' audio (default 8; every level decodes to the same samples; 12 writes files 0.2-1.3 % smaller in about five times the encoding time); part of the task keys | | `--only-class C,...` | export only objects of these Unity classes; part of the task keys | -| `--link` | how the layout files are made from the store (below): `auto` (default; `[export] link`), `clone`, `hard` or `copy` | +| `--link` | how the layout files are made from the depot (below): `auto` (default; `[export] link`), `clone`, `hard` or `copy` | | `--strict` | exit 1 when a view has gaps: a required role whose address names no catalog key or no fitting sub-object (empty values and roles that do not apply are not gaps); in `voices`, a voice whose cue sheet has no catalog key or whose decoded sheet lacks its cue, and a story episode without a catalog key | | `--dry-run` | print the plan and stop | -| `--explain` | print every setting of the run first: store, context, selection, pipeline, stages not installed, parameters, workers, memory budget, recycling, layouts | +| `--explain` | print every setting of the run first: depot, context, selection, pipeline, stages not installed, parameters, workers, memory budget, recycling, layouts | A run resolves the context (region, language, catalog version) and indexes the catalog, identifies the selected -bundles (the sha256 of each decrypted bundle, remembered by path, size and modification time; bundles not in the -cache are fetched), takes the census of each bundle, links the scripts and addresses, exports the bundles, runs the +bundles (the sha256 of each decrypted bundle, remembered by path, size and modification time; bundles not +downloaded yet are fetched into the depot's `downloads/`), takes the census of each bundle, links the scripts and addresses, exports the bundles, runs the stages that need those results (atlas sprites, views), writes the layouts and the reports. The selected raw files -(CRI audio and movies) not in the cache are fetched while the bundle stages run. The census also covers +(CRI audio and movies) not downloaded yet are fetched while the bundle stages run. The census also covers the bundles the selected ones depend on, so script classes and atlas textures in other bundles resolve. Tasks run in spawned worker processes that stay alive between tasks (no more of them start than a stage has tasks to @@ -64,16 +64,16 @@ the running tasks and its own fits the budget, and a task estimated above the bu replaced after 500 tasks or when its resident memory after a task is above 3 GiB, or above 40 % of its part of the memory budget (the budget divided by the workers) when that is less, but not below 256 MiB: a worker keeps much of its largest task's memory, which the budget does not count (`--explain` prints the threshold). The estimates come -from each stage's cost model, scaled per stage by the costs measured in earlier runs of the store -(`/costs/calibration.json`, updated after every export; a task's measured peak counts from its worker's -memory at its start). None of this changes an output: a run with one worker and a run with many write the same store +from each stage's cost model, scaled per stage by the costs measured in earlier runs of the depot +(`/costs/calibration.json`, updated after every export; a task's measured peak counts from its worker's +memory at its start). None of this changes an output: a run with one worker and a run with many write the same depot and the same files. A failed object is reported with its task's result and stays failed until something in its key changes. A task that fails as a whole (an exception, an input that cannot be read, a worker that dies) leaves no result and runs again next time. -### store plan +### depot plan What `export assets` with the same flags would do, without running or fetching anything: every task with its status (`hit`: its result is stored, `run`, `unknown`: it reads the result of a task that would run first, or of a bundle @@ -90,21 +90,21 @@ gives per stage its counts and its first 20 tasks that are not hits; `--json` ha | `--why TASK` | one task: its status, reasons, and its key parts beside those of the previous run | Reasons: `new-input`, `input-changed`, `stage-version`, `atom-changed`, `params-changed`, `context-changed`, -`previous-failed`, `result-missing`, `forced`; hits are `unchanged` or `cached` (the result is in the store from +`previous-failed`, `result-missing`, `forced`; hits are `unchanged` or `cached` (the result is in the depot from another run); unknown tasks are `pending` on a task, or on `fetch:` for a bundle whose content is not known yet. See [contracts.md](contracts.md#plan-nnnotesplan1). -### store run +### depot run -Runs task descriptions (`nnnotes.task/1`, as `plan --emit-tasks` writes them) into a store: each is checked (its key +Runs task descriptions (`nnnotes.task/1`, as `plan --emit-tasks` writes them) into a depot: each is checked (its key is recomputed, its stage must be installed at its version), its inputs are read by content id through their -locators (the store, the cache, a file, or with `--fetch` the catalog), and its result is stored exactly as `export assets` -would store it. Several tasks run one after another in one process; stages that do not read bundles never load +locators (the depot, its downloads, a file, or with `--fetch` the catalog), and its result is stored exactly as +`export assets` would store it. Several tasks run one after another in one process; stages that do not read bundles never load UnityPy. `--force` runs tasks whose result is stored again (the new result must be identical). ### catalog versions, import, fetch, diff -The catalog versions of a store: `import` a catalog file (with the APK's catalog from `[paths] apk` or +The catalog versions of a depot: `import` a catalog file (with the APK's catalog from `[paths] apk` or `--apk-catalog`), `fetch` the current catalog of the configured region and language, `list` them, and `diff` two of them (labels, version ids or sha256 prefixes): keys added, removed and changed, bundles and raw files by stable name, the keys that reach a changed file, and, when a run over each version has stored its address table, the objects by @@ -112,14 +112,22 @@ stable address (an object keeps its stable address while its object id changes w is labelled by `--label`, else by its resource version, else by the first 12 hex digits of its sha256. `fetch` records the resource version it downloads (none for `main`); `import` takes `--resource-version`, else the JP catalog's source version, else the version in a file name `catalog__.bin` as `fetch` and the -cache name them (none for `main`). `export assets` imports the current catalog when it is not imported yet. +downloads name them (none for `main`). `export assets` imports the current catalog when it is not imported yet. -### store verify +### depot verify Checks that every stored content has the sha256 its name says (`--quick`: sizes only) and that every result names its own key, has the key of its parts and names stored contents of the right size. Exit 1 when a problem is found. `--workers` hashes in parallel (default: the CPUs the process may use; `0`: one at a time). +### depot gc + +Removes what no kept run reaches. The kept runs are, for each selection, the `--keep N` latest (default: every run; +the latest run of a selection is always kept). A kept run reaches the results of its tasks, the contents those +results name (their artifacts, and the inputs their task keys name when the depot holds them) and its layout +manifests with the files they list. Every other run, result (with its measured cost) and content is removed. +`--dry-run` counts what would be removed. A layout made of hard links or clones keeps its files. + ## Layouts A layout places the artifacts of a run as files; it never changes their bytes. With one layout the files are @@ -150,16 +158,16 @@ manifest; the run manifest and the printed summary name the method and why it wa | `--link` | the files | extra space | an edit of a file | |---|---|---|---| -| `clone` | clones (reflinks) of the store objects: Linux FICLONE (Btrfs, XFS, bcachefs, …), macOS APFS `clonefile` | none until a file is written (a quota that counts each file's blocks, such as an XFS project quota, counts them in full) | changes that file only | -| `hard` | hard links to the store objects (same file system) | none | fails, except for the superuser: the store objects are read-only (on Windows a delete fails too) | +| `clone` | clones (reflinks) of the depot objects: Linux FICLONE (Btrfs, XFS, bcachefs, …), macOS APFS `clonefile` | none until a file is written (a quota that counts each file's blocks, such as an XFS project quota, counts them in full) | changes that file only | +| `hard` | hard links to the depot objects (same file system) | none | fails, except for the superuser: the depot objects are read-only (on Windows a delete fails too) | | `copy` | copies | the whole layout again | changes that file only | | `auto` (default) | `clone` where the file system makes clones, else `hard` on the same file system, else `copy` | | | `auto` tries each method once, on a temporary file, before the run starts. An explicit `clone` or `hard` that does not work there is a usage error (exit 2) before anything runs; a file that fails with the chosen method is reported (summary, layout report, exit 1), left out of the manifest and placed by the next run: no file falls back to -another method. The store writes its objects read-only; a layout clears the attribute of a file it owns before it -replaces or removes it where the system needs that (Windows), and the store object stays read-only. +another method. The depot writes its objects read-only; a layout clears the attribute of a file it owns before it +replaces or removes it where the system needs that (Windows), and the depot object stays read-only. `cas` is `assets/.` per distinct content, plus the manifest. Each layout follows from the other: `cas` from the `original` manifest alone, `original` from `cas` and the results. The views are the exception: `original` @@ -171,7 +179,7 @@ has them with the paths of their objects, `cas` the view documents they are made | File | Content | |---|---| -| `run.json` | the run manifest (`nnnotes.run/1`): context, selection, parameters, stage versions, every task with its key and status, the layout manifests, a summary; also stored as `/runs/.json` | +| `run.json` | the run manifest (`nnnotes.run/1`): context, selection, parameters, stage versions, every task with its key and status, the layout manifests, a summary; also stored as `/runs/.json` | | `coverage.json` | per class the census total and the count of every item status (`exported`, `contained`, `generic`, `unsupported`, `failed`, and `missing`: objects no task covered; an object two tasks cover, such as a sprite whose image `sprite.crop` makes, counts once); per reason code its count and five examples | | `failures.json` | failed tasks and failed objects, with their codes and messages | | `sources.json` | files of the selection that could not be read: `source.absent` (a file the catalog names that is not there, or an APK file without `[paths] apk`), `source.error` (the download failed) | @@ -185,28 +193,28 @@ and the reports directory. | Command | 0 | 1 | 2 | 3 | 4 | |---|---|---|---|---|---| | `export assets` | complete (unsupported objects allowed) | a task, an object or a download failed (reports written); with `--strict` also view gaps | usage or settings | aborted: a setting or tool a task needs is missing, or interrupted; results written so far stay valid | | -| `store plan` | | with `--check`: something would run or change | usage or settings | | | -| `store run` | every task ran or hit | a task failed or its result is partial | usage or settings | | a task is incompatible (edited, unknown stage or version); nothing of the batch runs | -| `store verify` | no problem | problems found | usage or settings | | | +| `depot plan` | | with `--check`: something would run or change | usage or settings | | | +| `depot run` | every task ran or hit | a task failed or its result is partial | usage or settings | | a task is incompatible (edited, unknown stage or version); nothing of the batch runs | +| `depot verify` | no problem | problems found | usage or settings | | | ## Running tasks elsewhere -Task descriptions carry everything a task needs; any machine with the same nnnotes version and access to the store +Task descriptions carry everything a task needs; any machine with the same nnnotes version and access to the depot can run them. A task can only be described once the content of its inputs is known, so the bundles are identified first (`plan --census` fetches and hashes them and takes their census). Then plan the tasks that can run now, run them in batches on as many workers as you like, and plan again until nothing is left; `export assets` finally writes the layouts and reports (every task is a hit by then). ```sh -store=/data/nnnotes-store -nnnotes store plan --store "$store" --census > /dev/null +depot=/data/nnnotes-depot +nnnotes depot plan --depot "$depot" --census > /dev/null for round in 1 2 3 4 5 6 7 8; do rm -rf tasks - nnnotes store plan --store "$store" --emit-tasks tasks > /dev/null + nnnotes depot plan --depot "$depot" --emit-tasks tasks > /dev/null ls tasks/*.json > /dev/null 2>&1 || break # nothing left to run - ls tasks/*.json | xargs -n 50 -P 16 nnnotes store run --store "$store" --fetch + ls tasks/*.json | xargs -n 50 -P 16 nnnotes depot run --depot "$depot" --fetch done -nnnotes export assets --store "$store" -o out/assets +nnnotes export assets --depot "$depot" -o out/assets ``` Each round runs the tasks whose inputs are known: the catalog index and the censuses, then the script and address diff --git a/docs/cli.md b/docs/cli.md index 1b38fe7..7077399 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -11,10 +11,9 @@ to standard error. Exit status 2 is a usage or settings error. | `--config` `FILE` | TOML config file (else NNNOTES_CONFIG, else ./nnnotes.toml) | | `--region` `REGION` | region: a [servers.] table ([catalog] region) | | `--language` `LANG` | catalog and client language: ja, en, zh-Hant, zh-Hans or ko ([catalog] language) | -| `--catalog` `FILE` | catalog .bin file ([paths] catalog; else downloaded into the cache) | +| `--catalog` `FILE` | catalog .bin file ([paths] catalog; else downloaded into the depot) | | `--catalog-version` `VERSION` | the resource version of an international catalog ([catalog] version; else the version the game API serves when an API root is configured, otherwise main) | -| `--cache` `DIR` | cache directory ([paths] cache) | -| `--store` `DIR` | store directory ([paths] store; default /store) | +| `--depot` `DIR` | depot directory ([paths] depot) | | `--master` `DIR` | decoded master data directory ([paths] master) | | `--apk` `FILE` | base.apk ([paths] apk) | | `--ffmpeg` `EXE` | ffmpeg executable ([paths] ffmpeg; else on PATH) | @@ -97,7 +96,7 @@ browse the configured regions' catalogs and bundles in a local web page ## nnnotes catalog versions -the catalog versions imported into the store (--json: nnnotes.catalogs/1) +the catalog versions imported into the depot (--json: nnnotes.catalogs/1) ## nnnotes catalog import @@ -112,7 +111,7 @@ import a catalog file (with the APK's catalog when [paths] apk is set) ## nnnotes catalog fetch -fetch the region's current catalog and import it into the store +fetch the region's current catalog and import it into the depot | Argument | Description | | --- | --- | @@ -120,7 +119,7 @@ fetch the region's current catalog and import it into the store ## nnnotes catalog diff -keys, bundles, raw files (and objects) that differ between two catalog versions of the store (--json: nnnotes.catalog-diff/1) +keys, bundles, raw files (and objects) that differ between two catalog versions of the depot (--json: nnnotes.catalog-diff/1) | Argument | Description | | --- | --- | @@ -129,7 +128,7 @@ keys, bundles, raw files (and objects) that differ between two catalog versions ## nnnotes bundle pull -fetch the bundle closure of keys into the cache (prints the cached bundle files) +fetch the bundle closure of keys into the depot (prints the downloaded bundle files) | Argument | Description | | --- | --- | @@ -164,14 +163,14 @@ Exactly one of `--version`, `--latest`. ## nnnotes export assets -every object of the selected bundles -> common formats (the store and its layouts) +every object of the selected bundles -> common formats (the depot and its layouts) | Argument | Description | | --- | --- | | `-o`, `--out` `OUT` | output directory (required) | | `--select` `SEL` | group:, key: or bundle: (repeatable; default: everything) | | `--views` `all\|none\|a,b` | semantic views to build (default: all when master data is set, else none) | -| `--stored-catalog` `LABEL\|SHA` | a catalog version of the store (`catalog versions`; default: the current catalog) | +| `--stored-catalog` `LABEL\|SHA` | a catalog version of the depot (`catalog versions`; default: the current catalog) | | `--png-level` `N` | PNG compression level 0-9 (changes the output) | | `--flac-level` `N` | FLAC compression level 0-12 of the CRI audio (default 8; every level decodes to the same samples; changes the files) | | `--only-class` `C,...` | export only objects of these classes | @@ -180,7 +179,7 @@ every object of the selected bundles -> common formats (the store and its layout | `--fetch-workers` `N` | parallel downloads (default 8) | | `--layout` `LAYOUT` | original, cas or original,cas (default original) | | `--explain` | print every setting of the run first | -| `--link` `LINK` | how the layout files are made from the store: auto (default; [export] link): a clone where the file system makes them, else a hard link (read-only), else a copy; or that method only (one of `auto`, `clone`, `hard`, `copy`) | +| `--link` `LINK` | how the layout files are made from the depot: auto (default; [export] link): a clone where the file system makes them, else a hard link (read-only), else a copy; or that method only (one of `auto`, `clone`, `hard`, `copy`) | | `--strict` | exit 1 when a view has gaps | | `--dry-run` | print the plan, run nothing | @@ -305,7 +304,7 @@ find the HCA keycode of the CRI audio (never printed) | --- | --- | | `--write` `DIR` | write it as DIR/.hcakey (the file vgmstream reads) | -## nnnotes store plan +## nnnotes depot plan what `export assets` would run and why, without running it (--json: nnnotes.plan/1) @@ -314,7 +313,7 @@ what `export assets` would run and why, without running it (--json: nnnotes.plan | `-o`, `--out` `OUT` | output directory of the export (counts the layout files to write) | | `--select` `SEL` | group:, key: or bundle: (repeatable; default: everything) | | `--views` `all\|none\|a,b` | semantic views to build (default: all when master data is set, else none) | -| `--stored-catalog` `LABEL\|SHA` | a catalog version of the store (`catalog versions`; default: the current catalog) | +| `--stored-catalog` `LABEL\|SHA` | a catalog version of the depot (`catalog versions`; default: the current catalog) | | `--png-level` `N` | PNG compression level 0-9 (changes the output) | | `--flac-level` `N` | FLAC compression level 0-12 of the CRI audio (default 8; every level decodes to the same samples; changes the files) | | `--only-class` `C,...` | export only objects of these classes | @@ -329,9 +328,9 @@ what `export assets` would run and why, without running it (--json: nnnotes.plan | `--emit-tasks` `DIR` | write the runnable tasks as task descriptions | | `--why` `TASK` | the reasons and key parts of one task | -## nnnotes store run +## nnnotes depot run -run task descriptions (nnnotes.task/1) into the store +run task descriptions (nnnotes.task/1) into the depot | Argument | Description | | --- | --- | @@ -339,7 +338,7 @@ run task descriptions (nnnotes.task/1) into the store | `--fetch` | fetch inputs through their catalog locators | | `--force` | run tasks whose result is stored (it must not change) | -## nnnotes store verify +## nnnotes depot verify check every stored content and result @@ -348,6 +347,15 @@ check every stored content and result | `--quick` | check sizes only, do not hash the contents | | `--workers` `N` | parallel hashing (default the CPUs this process may use; 0: one at a time) | +## nnnotes depot gc + +remove the runs, results and contents no kept run reaches + +| Argument | Description | +| --- | --- | +| `--keep` `N` | keep the N latest runs of each selection (default: every run; the latest run of a selection is kept) | +| `--dry-run` | count what would be removed, remove nothing | + ## nnnotes voices list the voices matching the filters (the text column in the --language language, default [catalog] language) diff --git a/docs/commands.md b/docs/commands.md index a5188e5..a6b8a37 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -5,11 +5,11 @@ nnnotes [options] ``` Global options (before or after the command name): `--config`, `--region`, `--language`, `--catalog`, -`--catalog-version`, `--cache`, `--store`, `--master`, `--apk`, `--ffmpeg`, `--vgmstream`, `--node` and `--json`, +`--catalog-version`, `--depot`, `--master`, `--apk`, `--ffmpeg`, `--vgmstream`, `--node` and `--json`, plus `--version` and `--help`. They set the settings described in [configuration.md](configuration.md), which also lists the settings each command needs. `nnnotes --help` prints the options of a command; -[cli.md](cli.md) lists every command with its options. The asset export commands (`export assets`, `store plan`, -`store run`, `store verify` and the catalog versions of the store) are described in [assets.md](assets.md). +[cli.md](cli.md) lists every command with its options. The asset export commands (`export assets`, `depot plan`, +`depot run`, `depot verify` and the catalog versions of the depot) are described in [assets.md](assets.md). Commands that write files take the output path with `-o/--out` (required). A command prints a short summary on standard output; with `--json` it prints its report as JSON (UTF-8). Every JSON file is written by one @@ -38,26 +38,27 @@ Write the config file from the template (asking in a terminal, else from `--set` check every setting's origin and format, and list where the config file is looked up. No `config` command prints a setting's value. Details in [configuration.md](configuration.md#writing-the-config-file). -## Cache +## Downloads -`bundle pull` and every extractor read bundles through the cache (`[paths] cache`): +`bundle pull` and every extractor read the game files from the depot's `downloads/` directory (`[paths] depot`), +which keeps the files under their own names: ``` -/catalog_main_.bin the legacy main catalog (explicit main or no API/version pin) -/bundles/ bundles of the dependency closures, decrypted (UnityFS) -/raw/ raw CDN files stored as they are (e.g. CRI cue sheet data) -/catalogs// catalogs downloaded by `catalog browse` -/international/// - versioned catalog, bundles and raw files; isolated from old main caches +/downloads/catalog_main_.bin the main catalog (explicit main or no API/version pin) +/downloads/bundles/ bundles of the dependency closures, decrypted (UnityFS) +/downloads/raw/ raw CDN files stored as they are (e.g. CRI cue sheet data) +/downloads/catalogs// catalogs downloaded by `catalog browse` +/downloads/international/// + versioned catalog, bundles and raw files of one CDN and version ``` -Files already in the cache are not downloaded again. When `[paths] apk` is set, the APK's own catalog is merged with +Files already downloaded are not downloaded again. When `[paths] apk` is set, the APK's own catalog is merged with the region's and bundles that ship inside the APK are read from it. For international regions with a configured API, catalog-based commands discover `resource_version` first and download `catalog__.bin`. `catalog fetch` labels the bytes with the version used to select that file, rather than querying a newer label after downloading main. Use `--catalog-version VERSION` or -the [version settings](configuration.md) to pin builds and use cached data offline. An explicit `--catalog FILE` +the [version settings](configuration.md) to pin builds and use downloaded data offline. An explicit `--catalog FILE` bypasses discovery. JP keeps its existing Version-driven path. ## catalog keys @@ -85,8 +86,8 @@ it, decrypted; `bundles/` lists every bundle directly. nnnotes bundle pull KEY [KEY ...] ``` -Fetches the bundle closure (the key's location and all its dependencies) of each key into the cache and prints the -cached path of every bundle. +Fetches the bundle closure (the key's location and all its dependencies) of each key into the depot's downloads and +prints the path of every bundle. ## servers diff --git a/docs/configuration.md b/docs/configuration.md index 2cfcb5a..f0764aa 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -59,7 +59,7 @@ No `config` command prints a setting's value. The config files these commands wr The global flags go before or after the command name: ```bash -nnnotes --config ~/nnnotes.toml --cache /data/nnnotes-cache bundle pull +nnnotes --config ~/nnnotes.toml --depot /data/nnnotes-depot bundle pull NNNOTES_PATHS_MASTER=/data/master nnnotes export episode 10462 -o episode.json ``` @@ -101,8 +101,7 @@ exist when they are given. | `[bootstrap] api` | `NNNOTES_BOOTSTRAP_API` | — | API root that serves the server list (`nnnotes servers`), same format | | `[client] version` | `NNNOTES_CLIENT_VERSION` | — | client version sent to the game's API, e.g. `1.0.1`; unset: the `versionName` of `[paths] apk` | | `[paths] catalog` | `NNNOTES_PATHS_CATALOG` | `--catalog` | a catalog `.bin` file to read instead of discovering/downloading a catalog; explicit files stay usable offline | -| `[paths] cache` | `NNNOTES_PATHS_CACHE` | `--cache` | cache directory (created when missing) | -| `[paths] store` | `NNNOTES_PATHS_STORE` | `--store` | store directory of the asset export ([assets.md](assets.md)); unset: `<[paths] cache>/store` | +| `[paths] depot` | `NNNOTES_PATHS_DEPOT` | `--depot` | depot directory (created when missing): the downloaded game files, the results of the stages, catalog versions and runs ([commands.md](commands.md#downloads), [contracts.md](contracts.md#depot)) | | `[paths] master` | `NNNOTES_PATHS_MASTER` | `--master` | decoded master data directory, one `.json` per table; the flag overrides `[servers.] master` | | `[paths] apk` | `NNNOTES_PATHS_APK` | `--apk` | the game's `base.apk` (the story export reads the CRI Lips library from it, or from `split_config.arm64_v8a.apk` next to it) | | `[paths] ffmpeg` | `NNNOTES_PATHS_FFMPEG` | `--ffmpeg` | `ffmpeg` executable; unset: `ffmpeg` on `PATH` | @@ -111,7 +110,7 @@ exist when they are given. | `[paths] player` | `NNNOTES_PATHS_PLAYER` | `--player` (of `site build`) | ournotes-player checkout (built) or installed package | | `[paths] fonts.` | `NNNOTES_PATHS_FONTS_` (e.g. `NNNOTES_PATHS_FONTS_ZH_HANT`) | `--font =PATH` (of `site build`) | font file (OpenType or TrueType) the story text of a language is drawn with (`--fonts open`); also read for the other story languages whose game fonts fall back to that language's font | | `[paths] fonts.emoji` | `NNNOTES_PATHS_FONTS_EMOJI` | `--font emoji=PATH` (of `site build`) | colour emoji font (PNG bitmap glyphs: CBDT or sbix) the stories' emoji sprites are drawn from (`--fonts open`); Noto Color Emoji (SIL Open Font License 1.1) is the tested font. Optional: without it the sprites keep their layout with empty glyphs, listed in `ui/fonts.json` `coverage.sprites.missing` | -| `[export] link` | `NNNOTES_EXPORT_LINK` | `--link` (of `export assets`) | how `export assets` makes the layout files from the store: `auto` (unset), `clone`, `hard` or `copy` ([assets.md](assets.md)) | +| `[export] link` | `NNNOTES_EXPORT_LINK` | `--link` (of `export assets`) | how `export assets` makes the layout files from the depot: `auto` (unset), `clone`, `hard` or `copy` ([assets.md](assets.md)) | Hex values are case-insensitive; a `0x` prefix and surrounding whitespace are ignored. All values are TOML strings except `languages` (array of strings). @@ -136,15 +135,15 @@ nnnotes --region tw --catalog-version 1.0.0.201 export live2d MODEL_ID -o out/mo ``` Alternatively set `NNNOTES_SERVERS_TW_CATALOG_VERSION` in the build environment; workers inherit this pin. -`--catalog-version` selects a remote resource release; `--stored-catalog LABEL|SHA` of `export assets` and `store -plan` selects a catalog version of the store. To inspect the historical main catalog, explicitly pin `main`. -Versioned caches are isolated by CDN root and resource version; a cached pin needs the CDN setting to identify -its directory but performs no API/catalog download. A missing versioned file never falls back to main. +`--catalog-version` selects a remote resource release; `--stored-catalog LABEL|SHA` of `export assets` and `depot +plan` selects a catalog version of the depot. To inspect the historical main catalog, explicitly pin `main`. +Versioned downloads are isolated by CDN root and resource version; a pin whose catalog is downloaded needs the CDN +setting to identify its directory but performs no API/catalog download. A missing versioned file never falls back to main. ## What each command needs -"Catalog" below means: `[paths] cache`; either `[paths] catalog` or `[catalog] language` (the catalog is then -downloaded into the cache on first use); and, for what must be downloaded, `[catalog] region` and that region's +"Catalog" below means: `[paths] depot`; either `[paths] catalog` or `[catalog] language` (the catalog is then +downloaded into the depot on first use); and, for what must be downloaded, `[catalog] region` and that region's `cdn`, `[bundle] key` and `nonce_seed`. `[paths] apk` is optional for the catalog: when it is set, the APK's own catalog and bundles are merged in, and bundles that ship inside the APK can only be read with it. @@ -157,8 +156,8 @@ story text); it is not a setting. | Command | Settings | |---|---| -| `catalog keys` | `[paths] cache`; `[paths] catalog`, or `[catalog] language` (plus region and `cdn` while the catalog is not in the cache); no bundle key | -| `catalog browse` | `[paths] cache`, `[bundle] key` + `nonce_seed`, and per region `cdn` + `languages` (`name` optional) | +| `catalog keys` | `[paths] depot`; `[paths] catalog`, or `[catalog] language` (plus region and `cdn` while the catalog is not downloaded); no bundle key | +| `catalog browse` | `[paths] depot`, `[bundle] key` + `nonce_seed`, and per region `cdn` + `languages` (`name` optional) | | `bundle pull` | catalog | | `master decode` | `[master] key` + `iv` | | `master download` | `[catalog] region` and that region's `cdn`; `--latest` also that region's `api` and the client version | @@ -178,13 +177,13 @@ story text); it is not a setting. | `site build --live2d` / `--all-live2d` | catalog (bundles from the CDN of the site's first region), `[paths] apk`, `[paths] player`; not `node`; master data only for the model names (optional: without it `models.json` has no names) | | `site build --player-only` / `--reingest-json` | `[paths] player` | | `music-data` | catalog, `[master] key` + `iv` (not with `--decoded-master`); `--master-files` also `[catalog] region`; `--apk-master` also `[paths] apk`; `--decoded-master` also `[catalog] region` and master data with its `MasterManifest.json`; `[paths] apk` (optional otherwise) for the client version | -| `export assets`, `store plan` | the store (`[paths] store` or `[paths] cache`); catalog (bundles are fetched into the cache); `[paths] apk` for the bundles inside the APK (without it they are reported as `source.absent`); master data for `--views`; with `--stored-catalog` a catalog version of the store instead of the current catalog | -| `store run` | the store; `[paths] cache` for inputs located in the cache; `--fetch` also what fetching needs (region, `cdn`, bundle key, `[paths] apk`) | -| `catalog versions` / `import` / `diff`, `store verify` | the store; `import` reads the APK's catalog from `[paths] apk` when it is set | -| `catalog fetch` | the store, `[catalog] region` and `language`, that region's `cdn`; its `api` (optional) for the resource version label | +| `export assets`, `depot plan` | catalog (bundles are fetched into the depot); `[paths] apk` for the bundles inside the APK (without it they are reported as `source.absent`); master data for `--views`; with `--stored-catalog` a catalog version of the depot instead of the current catalog | +| `depot run` | the depot; `--fetch` also what fetching needs (region, `cdn`, bundle key, `[paths] apk`) | +| `catalog versions` / `import` / `diff`, `depot verify` | the depot; `import` reads the APK's catalog from `[paths] apk` when it is set | +| `catalog fetch` | the depot, `[catalog] region` and `language`, that region's `cdn`; its `api` (optional) for the resource version label | The region, its `cdn`, `[bundle] key` and `nonce_seed` are read only when a file must be downloaded: when the -catalog and every bundle a command needs are in the cache, none of them is needed. A download that needs a missing +catalog and every bundle a command needs are downloaded, none of them is needed. A download that needs a missing one stops the command with one line naming the file and the setting (see Errors). ## Errors @@ -194,7 +193,7 @@ the setting, its environment variable and its flag where there is one; it never ``` nnnotes: setting catalog.region is not set: give it as `region` in the [catalog] table of the config file, the environment variable NNNOTES_CATALOG_REGION or --region -nnnotes: .bundle is not in the cache: setting bundle.key is not set: give it as `key` in the [bundle] table of the config file, the environment variable NNNOTES_BUNDLE_KEY +nnnotes: .bundle is not downloaded: setting bundle.key is not set: give it as `key` in the [bundle] table of the config file, the environment variable NNNOTES_BUNDLE_KEY nnnotes: setting bundle.key: must be 16 bytes (32 hex digits) nnnotes: setting paths.apk: file not found nnnotes: ffmpeg not found on PATH: give its path as `ffmpeg` in the [paths] table of the config file, the environment variable NNNOTES_PATHS_FFMPEG or --ffmpeg diff --git a/docs/contracts.md b/docs/contracts.md index c9e0f2d..930f5ab 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -55,7 +55,7 @@ One stage applied to one subject, with everything needed to run it "params": {"png": {"encoder": "pillow", "level": 6}, "blobMin": 4096}, "atoms": {"png.encode": "pillow-12.3.0/1", "texture.decode": "unitypy-1.25.3/1"}, "inputs": [{"role": "bundle", "sha256": "…", "size": 123456, "name": "membercard_…_3fa…c2.bundle", - "locators": [{"kind": "store"}, {"kind": "cache", "path": "bundles/membercard_…_3fa…c2.bundle"}]}], + "locators": [{"kind": "depot"}, {"kind": "download", "path": "bundles/membercard_…_3fa…c2.bundle"}]}], "context": {"scripts": {"CAB-…:12345": "Assembly-CSharp|Fwk.Sound|SplitAcbData"}}, "key": "…", "cost": {"cpuSeconds": 1.2, "peakBytes": 450000000}} ``` @@ -83,10 +83,10 @@ Locator kinds, tried in order to find an input's bytes (each checked against the | Kind | Where | |---|---| -| `store` | the store's `cas/` | -| `cache` | `path` relative to the cache directory | +| `depot` | the depot's `cas/` | +| `download` | `path` relative to the depot's `downloads/` | | `file` | an absolute `path` | -| `catalog` | fetched through a catalog version of the store (`catalog`: its version id, `location`: the location id in its index) | +| `catalog` | fetched through a catalog version of the depot (`catalog`: its version id, `location`: the location id in its index) | ## Artifact record: nnnotes.artifact/1 @@ -142,7 +142,7 @@ stage, an input that cannot be read, a worker process that dies) leaves no resul What one run covered ([run.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/run.schema.json)): the context (region, language, the catalog version, and the master data its views read: the version named by `MasterManifest.json` and the sha256 of each table), the selection, the normalized parameters per stage, the stage versions, every task with its -key (`null` when it could not be described) and status (`hit`: the result was in the store, `ran`, `failed`), the +key (`null` when it could not be described) and status (`hit`: the result was in the depot, `ran`, `failed`), the sha256 of each layout manifest written, and a summary (task and item counts). The run id is the key hash of the context, selection, params, stages, the tasks' ids and keys, and the layouts, so a run with the same inputs has the same id whether its tasks ran or hit. The run log (`runs/.log.jsonl`) holds what is not deterministic: @@ -181,11 +181,11 @@ task id in the previous run manifest: | `params-changed` | run | a parameter changed (`path`, `old`, `new`) | | `context-changed` | run | a consumed context entry changed (`entry`) | | `previous-failed` | run | the task failed in the previous run | -| `result-missing` | run | same key as before, but its result is not in the store | +| `result-missing` | run | same key as before, but its result is not in the depot | | `key-changed` | run | the key changed for none of the reasons above | | `forced` | run | asked to run again | | `unchanged` | hit | same key as in the previous run | -| `cached` | hit | the result is in the store from another run | +| `cached` | hit | the result is in the depot from another run | | `pending` | unknown | the task reads the result of `task`, which would run first (`fetch:`: an input file whose content is not known yet) | ## Layout manifest: nnnotes.layout/1 @@ -221,8 +221,8 @@ Documents written by particular stages (artifacts of no object: their ids are `< | `nnnotes.artifacts/1` | `link.artifacts` | [artifacts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/artifacts.schema.json) | | `nnnotes.view/1` | `view.` | [view.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/view.schema.json) | -The catalog versions of a store, `nnnotes.catalogs/1` ([catalogs.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalogs.schema.json), -`/catalogs/index.json`), and the difference between two of them, `nnnotes.catalog-diff/1` +The catalog versions of a depot, `nnnotes.catalogs/1` ([catalogs.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalogs.schema.json), +`/catalogs/index.json`), and the difference between two of them, `nnnotes.catalog-diff/1` ([catalog-diff.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalog-diff.schema.json), `nnnotes catalog diff --json`), are written by the commands. @@ -230,23 +230,24 @@ The music data file of `nnnotes music-data`, `nnnotes.music-data/2` ([music-data.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/music-data.schema.json)), names its format in a `format` field; it is described in [music-data.md](music-data.md). -## Store +## Depot -The store is a directory, and the cache of all stage work: +The depot is a directory: the downloaded game files, and the results of all stage work by content: ``` -/cas/sha256/ab/ bytes by content id, written once (no extension: records carry it) -/ac/ab/.json results by task key (written after the bytes they name) -/inputs/.jsonl input identities: name -> sha256, size, with the path, size and mtime of the +/downloads/ the game files as downloaded, under their own names ([commands.md](commands.md#downloads)) +/cas/sha256/ab/ bytes by content id, written once (no extension: records carry it) +/ac/ab/.json results by task key (written after the bytes they name) +/inputs/.jsonl input identities: name -> sha256, size, with the path, size and mtime of the file that was hashed (bundles.jsonl, raw.jsonl, files.jsonl) -/catalogs/ versioned catalogs -/runs/.json run manifests -/runs/.log.jsonl the log of the run's last execution -/runs/selections/ the id of the latest run of a selection (key hash of the selection list) -/costs/.json the measured cost of a task's last run +/catalogs/ versioned catalogs +/runs/.json run manifests +/runs/.log.jsonl the log of the run's last execution +/runs/selections/ the id of the latest run of a selection (key hash of the selection list) +/costs/.json the measured cost of a task's last run ``` Every file is written to a temporary name and renamed into place, so concurrent writers never see or leave a partial file and an interrupted task leaves no result. `cas/` and `ac/` depend only on the tasks run: two runs of the -same tasks, with any number of workers, leave identical trees. `inputs/`, `runs/*.log.jsonl` and `costs/` are -local and not deterministic. +same tasks, with any number of workers, leave identical trees. `downloads/`, `inputs/`, `runs/*.log.jsonl` and +`costs/` are local and not deterministic. diff --git a/docs/determinism.md b/docs/determinism.md index 550828c..5b7113d 100644 --- a/docs/determinism.md +++ b/docs/determinism.md @@ -19,7 +19,7 @@ fixes the bytes, what can change them, and how the tests hold it. textures, shaders and audio are converted from the stored data with fixed encoder settings. - **No clocks.** No output holds the time of a run or a path of the machine it ran on. A result of the asset export holds no input name, locator, timestamp or traceback; the measured cost of a task (CPU and wall seconds, memory) is - kept in the store's cost report, apart from the results. + kept in the depot's cost report, apart from the results. ## What can change the bytes @@ -32,7 +32,7 @@ Another version of an encoder can write the same content as other bytes: while lossy formats depend on the encoder version. - zlib and brotli encode the compressed assets of a site. -So the guarantee is per installation: pin the versions to reproduce a result exactly. The store keeps this visible: +So the guarantee is per installation: pin the versions to reproduce a result exactly. The depot keeps this visible: a task's key includes the implementation ids of the atoms it uses, with the versions of the libraries they call, so another installation computes other keys instead of mixing results. @@ -44,7 +44,7 @@ another installation computes other keys instead of mixing results. library versions of `.github/goldens-constraints.txt` and sets `GOLDENS_STRICT=1`; elsewhere a record made with other library versions is skipped. - **Two runs, one result.** The tests run exports twice over synthetic inputs and compare the bytes, and compare - one worker with several: a store built with four workers equals one built with one, and parallel audio decoding + one worker with several: a depot built with four workers equals one built with one, and parallel audio decoding equals sequential decoding. - **Writer tests.** The JSON writer's tests cover the non-finite literals, the NaN error and the line endings. diff --git a/docs/export.md b/docs/export.md index 8251fdd..6cce4fc 100644 --- a/docs/export.md +++ b/docs/export.md @@ -1,7 +1,7 @@ # Export stories and lives The `export` commands each write one subject as a directory or file. They read the catalog, fetch and decrypt the -bundles they need into the cache, and need decoded master data where the subject is defined by it +bundles they need into the depot, and need decoded master data where the subject is defined by it ([what each command needs](configuration.md#what-each-command-needs)). To export every object of many bundles at once, use [`export assets`](assets.md) instead. diff --git a/docs/jp.md b/docs/jp.md index df5067e..63d0862 100644 --- a/docs/jp.md +++ b/docs/jp.md @@ -25,7 +25,7 @@ apk = "/path/to/jp/base.apk" master = "/path/to/jp/master" [paths] -cache = "/path/to/cache" +depot = "/path/to/depot" ``` Keep `[bundle] key/nonce_seed` and `[master] key/iv` in your configuration. Master keys are unnecessary when @@ -79,17 +79,17 @@ the JP hash in `provenance.catalog.resourceHash`. JP catalogs are `catalog_main.bin` in a version/hash directory, without a language suffix. The HTTP payload can be gzip; nnnotes keeps its original bytes and SHA-256, and bounds decompression before parsing. Remote bundle and CRI locations use `{Fwk.Resource.RemoteAssetDir}`. These are indexed, resolved and downloaded through the selected -snapshot, including in `catalog browse`, the other `catalog` commands, `export assets`, `store plan` and -`store run`. +snapshot, including in `catalog browse`, the other `catalog` commands, `export assets`, `depot plan` and +`depot run`. -The cache lives under `/jp////`. It cannot reuse the -international catalog cache. Embedded files are further isolated under `apk//`, so an APK -update cannot reuse old local bundles when the CDN snapshot stays unchanged. Each cached `catalog_main.bin` has a `catalog_main.bin.source.json` with its SHA-256 +The downloads live under `/downloads/jp////`, apart from the +international downloads. Embedded files are further isolated under `apk//`, so an APK +update cannot reuse old local bundles when the CDN snapshot stays unchanged. Each downloaded `catalog_main.bin` has a `catalog_main.bin.source.json` with its SHA-256 and public source metadata. For an offline `--catalog` or `catalog import`, copy the pair together. Credentials are reacquired only when a missing file must be downloaded. Catalog database records include this source metadata, and JP catalog identity includes version/hash even when -the catalog bytes remain the same. Old international records retain their previous IDs. Cached historical files +the catalog bytes remain the same. Old international records retain their previous IDs. Downloaded historical files can be read offline; downloading a missing historical file fails if Version now selects a different snapshot. Refresh the catalog to use the new snapshot. Historical local bundles require the APK set matching the imported APK catalog. A JP master download likewise rejects a different current master snapshot. diff --git a/docs/music-data.md b/docs/music-data.md index dae9ac1..1361ce9 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -54,7 +54,7 @@ Each master data file is checked against the SHA-256 the manifest lists and deco With `--decoded-master` the tables are read as decoded, and the master data version and each file's SHA-256 are the manifest's: the decoded tables cannot be checked against the files as served, so the file records what the manifest lists (the same values as `--master-files` on those files). -Charts are read from the catalog of `[catalog] language` (bundles fetched into the cache as for every command); the +Charts are read from the catalog of `[catalog] language` (bundles fetched into the depot as for every command); the BGM length from the cue sheet's ACB (its `CueTable` and `WaveformTable`, no audio is decoded). `FILE` ending in `.gz` is written gzip-compressed. The command prints a summary (`songs`, `charts`, `deck`: the deck model's commit, `unplayable`, `bytes`, `sha256` of the file, ...). @@ -102,7 +102,7 @@ A **text** is an object with one string per language of `languages` (`{"ja": ... |---|---| | `region` | the region whose master data this is (a configured region name), or `embedded` for the APK's master data | | `client.versionName`, `client.versionCode` | the APK's version name and code (null without `[paths] apk`) | -| `catalog.resourceVersion` | the resource version used to select the downloaded catalog, or recorded for an explicit file in the catalog store (`nnnotes catalog fetch` / `import`); null when unknown | +| `catalog.resourceVersion` | the resource version used to select the downloaded catalog, or recorded for an explicit file in the catalog depot (`nnnotes catalog fetch` / `import`); null when unknown | | `catalog.sha256` | SHA-256 of the remote catalog file the charts were read with | | `master.source` | `api` (`--master-files`, `--decoded-master`: the region's files) or `embedded` (`--apk-master`) | | `master.version` | the `version` of the master data manifest | diff --git a/docs/quickstart.md b/docs/quickstart.md index 93d1c43..8c8cc44 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -21,7 +21,7 @@ nnnotes config init --user ``` In a terminal this asks for each setting in turn, among them the bundle and master data keys, the names of your -regions with each region's CDN base and API root, the catalog language, and the paths of the cache, the APK and the +regions with each region's CDN base and API root, the catalog language, and the paths of the depot, the APK and the tools. Enter leaves a setting empty; the keys are read without echo. The file is written to the per-user location, which every command reads from any directory (`nnnotes config path` shows it). @@ -41,7 +41,7 @@ set it. nnnotes catalog keys --prefix Adv/ --limit 5 ``` -The first command downloads the region's catalog into the cache; the keys it lists are the addressable keys of the +The first command downloads the region's catalog into the depot; the keys it lists are the addressable keys of the game's assets. `nnnotes catalog browse` serves a local web page to browse the catalogs and bundles of every configured region. @@ -74,5 +74,5 @@ where the next story reuses them. Add `--json` to any command to get its report - [Export stories and lives](export.md): the other single-subject exports and their options. - [Build an ournotes-player site](site.md): charts, models and stories as one static site. -- [Export every asset](assets.md): every object of the selected bundles through the content-addressed store. +- [Export every asset](assets.md): every object of the selected bundles through the content-addressed depot. - [Command reference](cli.md): every command and option. diff --git a/docs/schemas.md b/docs/schemas.md index 051f6fe..04a7d89 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -19,7 +19,7 @@ schemas.load("plan") # the schema as a dict | `artifacts` | `nnnotes.artifacts/1` | the artifact table: the artifacts of every object, with content id, size and extension | | `catalog-diff` | `nnnotes.catalog-diff/1` | the difference between two catalog versions (`catalog diff`) | | `catalog-index` | `nnnotes.catalog-index/1` | the index of a catalog pair: remote and APK catalogs, every location in catalog order | -| `catalogs` | `nnnotes.catalogs/1` | the catalog versions of a store (`catalog versions`) | +| `catalogs` | `nnnotes.catalogs/1` | the catalog versions of a depot (`catalog versions`) | | `census` | `nnnotes.census/1` | the census of one bundle: serialized files, type trees and objects, read without decoding | | `common` | | definitions shared by the other schemas | | `cost` | `nnnotes.cost/1` | the measured cost of a task's last run | @@ -27,11 +27,11 @@ schemas.load("plan") # the schema as a dict | `failures` | `nnnotes.failures/1` | the failed tasks and items of a run | | `layout` | `nnnotes.layout/1` | the files a layout owns in its output directory | | `music-data` | `nnnotes.music-data/2` | every song and chart of one master data version ([music data](music-data.md)) | -| `plan` | `nnnotes.plan/1` | what a run would do and why (`store plan`) | +| `plan` | `nnnotes.plan/1` | what a run would do and why (`depot plan`) | | `result` | `nnnotes.result/1` | the result of a task, a function of its key alone | | `run` | `nnnotes.run/1` | a run manifest: context, selection, tasks with keys and statuses, layouts | | `scripts` | `nnnotes.scripts/1` | every MonoScript of the censuses, by object id | -| `task` | `nnnotes.task/1` | one stage applied to one subject, with everything needed to run it (`store run`) | +| `task` | `nnnotes.task/1` | one stage applied to one subject, with everything needed to run it (`depot run`) | | `view` | `nnnotes.view/1` | a semantic view of a master table, each row with its names and the objects it names | The documents of the asset export are described in [contracts](contracts.md), the views in [views](views.md). The diff --git a/docs/ui.md b/docs/ui.md index 57c1686..5251ce8 100644 --- a/docs/ui.md +++ b/docs/ui.md @@ -18,7 +18,7 @@ index entry and makes the command exit with status 1. Existing unrelated entries The command uses the same `ApkSet` reader as the existing extractors: one APK, base.apk with adjacent splits, a directory of splits, or an APKS/XAPK archive. Explicit `--apk` wins over the selected region's APK configuration. -The configured cache and bundle decryption settings are used when needed. Readable unencrypted local bundles do +The configured depot and bundle decryption settings are used when needed. Readable unencrypted local bundles do not require a key. Default Unity resources are read from the selected APK set through the existing player reader. ```sh diff --git a/docs/views.md b/docs/views.md index b3cb362..af73b43 100644 --- a/docs/views.md +++ b/docs/views.md @@ -109,7 +109,7 @@ resolver's name). ## Stages `nnnotes.pipeline.views:stages` gives one stage per view, `view.` (docs/stages.md), run by `nnnotes export assets ---views` and planned by `nnnotes store plan`: +--views` and planned by `nnnotes depot plan`: - Subjects: the catalog subjects of `link.addresses` (the export command has one, `main`), when master data is set and the view is selected (the planning facts `master` and `views`). @@ -132,7 +132,7 @@ resolver's name). ## Views ### `backgrounds` -Profile backgrounds: the rows store full addresses. Table `MasterBackground`. Vars: `path` = `_assetPath`, `thumbnail` +Profile backgrounds: the rows depot full addresses. Table `MasterBackground`. Vars: `path` = `_assetPath`, `thumbnail` = `_thumbnailAssetPath`. Names: `name` = `_nameTextId`, `description` = `_descriptionTextId`. Prefixes: `Image/Background/`, `thumbnail/Background/`. @@ -331,7 +331,7 @@ Song jackets. Table `MasterLiveMusic`. Vars: `jacket` = `_jacketAssetName`. Name ### `login_bonuses` -Login bonus sheets: the rows store full addresses. Table `MasterLoginBonus`. Vars: `sheet` = `_sheetImageAsset`, +Login bonus sheets: the rows depot full addresses. Table `MasterLoginBonus`. Vars: `sheet` = `_sheetImageAsset`, `background` = `_backgroundImageAsset`. Names: `name` = `_nameTextID`. Prefixes: `LoginBonus/`. | Role | Address | Expect | Condition | Required | @@ -399,7 +399,7 @@ Skill icons (a skill names its icon by id). Table `MasterSkillIcon`. Vars: `icon ### `spots` -Home spots: the rows store full addresses. Table `MasterHomeSpot`. Vars: `thumbnail` = `_thumbnailAssetPath`, +Home spots: the rows depot full addresses. Table `MasterHomeSpot`. Vars: `thumbnail` = `_thumbnailAssetPath`, `background` = `_backgroundAssetPath`, `situation` = `_situationAssetPath`. Names: `name` = `_nameTextId`. Prefixes: `Image/Spot/`. diff --git a/src/nnnotes/cli/__init__.py b/src/nnnotes/cli/__init__.py index 0020406..0aa5979 100644 --- a/src/nnnotes/cli/__init__.py +++ b/src/nnnotes/cli/__init__.py @@ -13,7 +13,7 @@ nnnotes master version|download|decode nnnotes export assets|episode|story|live|live2d|spot|room|shader|audio|ui ... -o OUT nnnotes apk settings -o FILE | hca-key [--write DIR] - nnnotes store plan|run|verify + nnnotes depot plan|run|verify|gc nnnotes voices list|search|summary|get nnnotes site build -o SITE nnnotes music-data -o FILE @@ -36,10 +36,10 @@ from .. import __version__ from ..source.config import ConfigError from ..source.gameapi import GameApiError -from . import apk, bundle, catalog, config, export, master, musicdata, servers, site, store, voices +from . import apk, bundle, catalog, config, depot, export, master, musicdata, servers, site, voices from .common import GLOBALS, Formatter, fill_globals, load_config -GROUPS = (config, servers, catalog, bundle, master, export, apk, store, voices, site, musicdata) +GROUPS = (config, servers, catalog, bundle, master, export, apk, depot, voices, site, musicdata) def build_parser() -> argparse.ArgumentParser: diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index 3536ff9..55d7a7b 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -1,22 +1,22 @@ -"""The asset commands: the default orchestration of the stages (`export assets`), its plan and single tasks (`store -plan`, `store run`). +"""The asset commands: the default orchestration of the stages (`export assets`), its plan and single tasks (`depot +plan`, `depot run`). nnnotes export assets -o OUT [--layout original[,cas]] [--select group:G|key:PREFIX|bundle:GLOB ...] [--views all|none|a,b] [--workers N] [--memory GiB] [--fetch-workers N] [--stored-catalog L|SHA] [--png-level N] [--flac-level N] [--only-class C,...] [--link auto|clone|hard|copy] [--strict] [--dry-run] [--explain] - nnnotes store plan [the selection and parameters of export assets] [-o OUT] [--since RUN] [--check] [--census] + nnnotes depot plan [the selection and parameters of export assets] [-o OUT] [--since RUN] [--check] [--census] [--emit-tasks DIR] [--why TASK] - nnnotes store run TASK.json|DIR [...] [--fetch] [--force] + nnnotes depot run TASK.json|DIR [...] [--fetch] [--force] The pipeline is the registry STAGE_SOURCES in order: each entry names the module and attribute of a stage (a Stage class) or of a family of stages (a function returning them, `view.`). An entry whose module or attribute is not installed is left out of the pipeline and named by --explain. Stage modules import heavy libraries (UnityPy) only -when a task runs, so `store plan`, and `store run` of stages that do not read bundles, never load them. +when a task runs, so `depot plan`, and `depot run` of stages that do not read bundles, never load them. Facts the stages read while their tasks are described (stages.Env): - catalogs {"main": [Input remote, Input apk]}: the catalog version's files in /catalogs/ + catalogs {"main": [Input remote, Input apk]}: the catalog version's files in /catalogs/ index the catalog index document (nnnotes.catalog-index/1) bundles {stable bundle name: Input "bundle"} of the selected bundles and their dependencies (a bundle whose bytes are not known yet: a Pending "fetch:") @@ -34,11 +34,11 @@ dependency is its own bundle), so script and atlas references resolve. Task cost estimates (scheduling only: never part of a key) are each stage's estimate scaled per stage by the -costs measured in earlier runs (/costs/calibration.json, updated after every export). +costs measured in earlier runs (/costs/calibration.json, updated after every export). Exit codes: export assets 0 complete (unsupported objects allowed), 1 item or task failures (reports written), 2 -usage or settings, 3 aborted (results written so far stay valid); store run 0, 1 (a task failed or a result is -partial), 2, 4 an incompatible task (nothing of the batch runs); store plan --check 1 when anything would run or +usage or settings, 3 aborted (results written so far stay valid); depot run 0, 1 (a task failed or a result is +partial), 2, 4 an incompatible task (nothing of the batch runs); depot plan --check 1 when anything would run or change. """ @@ -147,7 +147,7 @@ class Installed: def installed(names=None) -> Installed: - """The stages of STAGE_SOURCES (`names`: only the sources of these stage names, for `store run`).""" + """The stages of STAGE_SOURCES (`names`: only the sources of these stage names, for `depot run`).""" stages, of, missing = {}, {}, [] for src in STAGE_SOURCES: if names is not None and not any(src.matches(n) for n in names): @@ -257,19 +257,20 @@ def _bundle_key(cfg): class CatalogFetcher: """The fetcher of "catalog" locators ({catalog: version id, location: location id}): the file is fetched through - that catalog version of the store (the settings' CDN, bundle key and APK) into the cache. Crosses to worker - processes as its settings.""" + that catalog version of the depot (the settings' CDN, bundle key and APK) into the depot's downloads/. Crosses to + worker processes as its settings.""" - def __init__(self, store_root_, cache, cfg): - self.root, self.cache, self.cfg = Path(store_root_), Path(cache) if cache else None, cfg + def __init__(self, root, cfg): + self.root, self.cfg = Path(root), cfg + self.downloads = self.root / "downloads" self._catalogs: dict = {} self._lock = threading.Lock() def __getstate__(self): - return {"root": self.root, "cache": self.cache, "cfg": self.cfg} + return {"root": self.root, "cfg": self.cfg} def __setstate__(self, state): - self.__init__(state["root"], state["cache"], state["cfg"]) + self.__init__(state["root"], state["cfg"]) def catalog(self, vid: str): """(Catalog, {location id: location}) of a stored catalog version.""" @@ -279,34 +280,32 @@ def catalog(self, vid: str): with self._lock: hit = self._catalogs.get(vid) if hit is None: - if self.cache is None: - raise self.cfg.missing("paths", "cache") db = catalogdb.CatalogDB(self.root) v = next((x for x in db.versions() if x["id"] == vid), None) if v is None: - raise FileNotFoundError(f"catalog version {vid[:12]} is not in the store") + raise FileNotFoundError(f"catalog version {vid[:12]} is not in the depot") remote, apk = db.catalog_bytes(v) cfg, region = self.cfg, v.get("region") region = region or cfg.get("catalog", "region") if region: cfg = cfg.for_region(region) - source, client, cache = None, None, self.cache + source, client, downloads = None, None, self.downloads if v.get("source") is not None: from ..source.jp import Client, Source source = Source.from_dict(v["source"]) client = Client(cfg, cfg.region()) - cache = source.cache_dir(cache) + downloads = source.download_dir(downloads) cat = Catalog( remote, - cache, + downloads, cdn=lambda: cfg.cdn(cfg.region()), bundle_key=lambda: _bundle_key(cfg), apk_catalog=apk, source=source, client=client, ) - # Replay imported offsets; the current APK is needed only for uncached local files. + # Replay imported offsets; the current APK is needed only for local files not downloaded yet. cat.apk = cfg.path("paths", "apk") hit = self._catalogs[vid] = (cat, catalogdb.by_id(catalogdb.index(remote, apk))) return hit @@ -327,12 +326,12 @@ def fetch_location(self, vid: str, lid: str) -> Path: return cat.fetch_raw({"internal_id": iid}) def _apk_file(self, internal_id: str, cat) -> Path: - """A raw APK file through its snapshot's cache; only a cache miss needs the matching APK.""" + """A raw APK file through its snapshot's downloads; only a file not read yet needs the matching APK.""" from ..core.cache import write_atomic from ..source.catalog import APK_AA_DIR rel = apk_rel(internal_id) - dst = cat.local_cache_dir() / "raw" / rel + dst = cat.local_download_dir() / "raw" / rel if not (dst.is_file() and dst.stat().st_size > 0): if cat.apk is None: raise self.cfg.missing("paths", "apk") @@ -354,7 +353,7 @@ def apk_rel(internal_id: str) -> str: return internal_id[len(LOCAL_PREFIX) :].lstrip("/") if internal_id.startswith(LOCAL_PREFIX) else internal_id -def _worker_init(store, cfg) -> None: +def _worker_init(depot, cfg) -> None: """Every worker process: the settings of the command.""" use(cfg) @@ -366,7 +365,7 @@ def _log(message: str) -> None: # ---------------------------------------------------------------- cost calibration -CALIBRATION = "calibration.json" # /costs/calibration.json +CALIBRATION = "calibration.json" # /costs/calibration.json MIN_SAMPLES = 10 @@ -388,7 +387,7 @@ def estimate(self, subject, env, inputs): def load_calibration(root: Path) -> dict: - """{stage: {"cpu": factor, "peak": factor, "samples": n}} of a store ({} when none was measured).""" + """{stage: {"cpu": factor, "peak": factor, "samples": n}} of a depot ({} when none was measured).""" try: doc = contract.loads((Path(root) / "costs" / CALIBRATION).read_bytes()) except (OSError, ValueError): @@ -442,26 +441,25 @@ def calibrate(log: list[dict], factors: dict, worker_bytes: int) -> dict: def write_calibration(root: Path, factors: dict) -> None: - from ..pipeline.store import write_file + from ..pipeline.depot import write_file write_file(Path(root) / "costs" / CALIBRATION, contract.encode({"stages": factors})) # ---------------------------------------------------------------- the workspace of a command class Workspace: - """What export and plan share: the store, the catalog version, its index, the selection, the pipeline and its + """What export and plan share: the depot, the catalog version, its index, the selection, the pipeline and its parameters, the facts.""" def __init__(self, args, cfg, log=_log): from ..pipeline.catalogdb import CatalogDB - from ..pipeline.store import Store + from ..pipeline.depot import Depot self.args, self.cfg, self.log = args, cfg, log - self.root = Session(cfg).store_root() - self.cache = cfg.path("paths", "cache") + self.root = Session(cfg).depot_root() self.apk = cfg.path("paths", "apk") - self.fetcher = CatalogFetcher(self.root, self.cache, cfg) - self.store = Store(self.root, cache=self.cache, fetch=self.fetcher) + self.fetcher = CatalogFetcher(self.root, cfg) + self.depot = Depot(self.root, fetch=self.fetcher) self.db = CatalogDB(self.root) try: self.selection = parse_selection(getattr(args, "select", None)) @@ -521,24 +519,24 @@ def catalogs_fact(self) -> dict: def index_task(self) -> Task: stage = self.installed.stages["catalog.index"] - return describe(stage, CATALOG_SUBJECT, None, Env(self.store, {"catalogs": self.catalogs_fact()})) + return describe(stage, CATALOG_SUBJECT, None, Env(self.depot, {"catalogs": self.catalogs_fact()})) def index(self) -> dict: - """The catalog index: from the store when catalog.index has run, else computed here.""" + """The catalog index: from the depot when catalog.index has run, else computed here.""" if self._index is None: from ..pipeline import catalogdb t = self.index_task() - doc = self.store.result(t.key) + doc = self.depot.result(t.key) if doc is not None: (rec,) = [a for a in doc["artifacts"] if a["id"] == contract.artifact_id(t.id, "index")] - self._index = self.store.document(rec["content"]["sha256"]) + self._index = self.depot.document(rec["content"]["sha256"]) else: self._index = catalogdb.index(self.remote, self.apk_catalog) return self._index def run_index(self) -> None: - execute(self.index_task(), self.store, self.installed.stages) + execute(self.index_task(), self.depot, self.installed.stages) self._index = None def context(self) -> dict: @@ -695,7 +693,7 @@ def facts(self, pipeline: list[str], fetch: bool, background: bool = False) -> d facts["raw"] = Later(self._later) else: facts["raw"] = job[0](*job[1:]) - facts["boot"] = boot_input(self.store, self.apk) + facts["boot"] = boot_input(self.depot, self.apk) self.selected = sel self._facts, self._names = facts, {} return facts @@ -713,14 +711,15 @@ def problems(self) -> list[dict]: """Files of the selection that could not be read: {kind, stable, name, code, message}, sorted.""" return sorted(self._problems, key=lambda p: (p["kind"], p["stable"])) - def _cache_rel(self, kind: str, entry: dict, loc: dict) -> str: + def _download_rel(self, kind: str, entry: dict, loc: dict) -> str: + """Where the file of a catalog entry is in the depot's downloads/ (the path of its "download" locator).""" from ..source.addressables import remote_path prefix = "" if self.version.get("source"): from ..source.jp import Source - prefix = Source.from_dict(self.version["source"]).cache_dir(Path(".")).as_posix() + "/" + prefix = Source.from_dict(self.version["source"]).download_dir(Path(".")).as_posix() + "/" if not entry.get("remote", True) and self.version.get("apk"): prefix += "apk/" + self.version["apk"]["sha256"] + "/" if kind == "bundle": @@ -739,15 +738,15 @@ def _inputs(self, kind: str, entries: list[dict], fetch: bool) -> Inputs: out, todo = {}, [] for e in entries: loc = locs[e["location"]] - rel = self._cache_rel(kind, e, loc) - locators = ([{"kind": "cache", "path": rel}] if rel else []) + [ + rel = self._download_rel(kind, e, loc) + locators = ([{"kind": "download", "path": rel}] if rel else []) + [ {"kind": "catalog", "catalog": self.version["id"], "location": e["location"]} ] - path = self.cache / rel if self.cache is not None and rel else None + path = self.depot.downloads / rel if rel else None if path is not None and path.is_file() and path.stat().st_size > 0: todo.append((e, locators, path)) continue - known = self.store.named(memo_kind, memo_prefix + e["name"]) if e["name"] != e["stable"] else None + known = self.depot.named(memo_kind, memo_prefix + e["name"]) if e["name"] != e["stable"] else None if known is not None: # a hashed file name: its recorded identity holds out[e["stable"]] = Input(kind, known[0], known[1], e["name"], tuple(locators)) elif not e["remote"] and self.apk is None: @@ -766,7 +765,7 @@ def one(job): try: if path is None: path = self.fetcher.fetch_location(self.version["id"], e["location"]) - sha, size = self.store.identify(path, memo_kind, memo_prefix + e["name"]) + sha, size = self.depot.identify(path, memo_kind, memo_prefix + e["name"]) except ConfigError: raise except KeyError as x: # not in the APK @@ -809,8 +808,7 @@ def explain(self, pipeline: list[str], params: dict, orch=None, layouts=(), plac ctx = self.context() c = ctx["catalog"] lines = [ - f"store: {self.root}", - f"cache: {self.cache if self.cache is not None else 'not set'}", + f"depot: {self.root}", f"context: region {ctx['region'] or '-'}, language {ctx['language'] or '-'}, catalog " f"{c['label'] or '-'} ({c['id'][:12]}), APK catalog {(c['apkCatalogSha'] or '-')[:12]}", f"selection: {', '.join(self.selection)}", @@ -837,13 +835,13 @@ def explain(self, pipeline: list[str], params: dict, orch=None, layouts=(), plac # ---------------------------------------------------------------- plan def previous(self, since: str | None) -> dict | None: if not since: - return self.store.latest_run(self.selection) - doc = self.store.run(since) if contract.is_sha256(since) else None + return self.depot.latest_run(self.selection) + doc = self.depot.run(since) if contract.is_sha256(since) else None if doc is None: runs = sorted(p.name[:-5] for p in (self.root / "runs").glob("*.json") if p.name.startswith(since)) if len(runs) != 1: usage(self.args, f"--since {since}: {'no run' if not runs else f'{len(runs)} runs'} with this id") - doc = self.store.run(runs[0]) + doc = self.depot.run(runs[0]) return doc def plan(self, pipeline: list[str], params: dict, *, since=None, out=None, layouts=()): @@ -853,7 +851,7 @@ def plan(self, pipeline: list[str], params: dict, *, since=None, out=None, layou p = plan_mod.plan( self.scheduled, pipeline, - self.store, + self.depot, params=params, facts=facts, previous=self.previous(since), @@ -867,7 +865,7 @@ def plan(self, pipeline: list[str], params: dict, *, since=None, out=None, layou hits = [(n["id"], n["key"]) for n in p.doc["nodes"] if n["status"] == "hit" and self._output(n["stage"])] placed = self.layout_input(dict(hits)) for tid, key in hits: - placed.add(tid, self.store.committed(key, tid)) + placed.add(tid, self.depot.committed(key, tid)) for name, d in layout_dirs(Path(out), layouts).items(): entries, _ = self.layout_entries(name, placed, store_derived=False) p.doc["summary"]["layouts"][name] = layout.diff(d, entries) @@ -885,7 +883,7 @@ def layout_input(self, tasks: dict) -> LayoutInput: stages, asked = self.installed.stages, _Asked() makers = sorted(t for t in tasks if hasattr(stages[contract.parse_task_id(t)[0]], "derived")) for tid in makers: - stages[contract.parse_task_id(tid)[0]].derived(tid, self.store.result(tasks[tid]), self.store, asked) + stages[contract.parse_task_id(tid)[0]].derived(tid, self.depot.result(tasks[tid]), self.depot, asked) return LayoutInput(asked.objects_asked, makers) def layout_entries(self, name: str, placed: LayoutInput, *, store_derived: bool = True) -> tuple[list[dict], dict]: @@ -938,7 +936,7 @@ def names(self, stage_name: str) -> dict: stage = self.installed.stages[stage_name] got = {} if hasattr(stage, "names"): - env = Env(self.store, self._facts) + env = Env(self.depot, self._facts) for tid, key in sorted(self._done.items()): env.done(tid, key) got = stage.names(env) @@ -958,8 +956,8 @@ def _derived(self, sources, entries: list[dict], store_derived: bool, objects: d out = [] for s in makers: stage = stages[contract.parse_task_id(s.task)[0]] - for rel, data in stage.derived(s.task, s.result, self.store, paths): - sha = self.store.put(data) if store_derived else contract.sha256(data) + for rel, data in stage.derived(s.task, s.result, self.depot, paths): + sha = self.depot.put(data) if store_derived else contract.sha256(data) out.append( { "path": layout.escape_path(rel.split("/")), @@ -1129,7 +1127,7 @@ def view_gaps(results) -> int: def _write_report(d: Path, name: str, doc) -> Path: - from ..pipeline.store import write_file + from ..pipeline.depot import write_file p = d / name write_file(p, contract.encode(doc)) @@ -1149,7 +1147,7 @@ def cmd_export(args, cfg): pipeline = ws.pipeline() params = ws.params(pipeline) orch = Orchestrator( - ws.store, + ws.depot, ws.scheduled, workers=ws.workers(), memory=ws.memory(), @@ -1229,7 +1227,7 @@ def visit(tid, doc): written, shas, unplaced = {}, {}, 0 for name, d in layout_dirs(out, names).items(): entries, report = ws.layout_entries(name, placed) - w = layout.materialize(d, name, {}, entries, layout.from_store(ws.store), link=placement[0]) + w = layout.materialize(d, name, {}, entries, layout.from_depot(ws.depot), link=placement[0]) shas[name] = w["sha256"] written[name] = { "dir": str(d), @@ -1316,7 +1314,7 @@ def cmd_plan(args, cfg): ws.run_index() facts = ws.facts(["catalog.index", "unity.census"], fetch=True) Orchestrator( - ws.store, + ws.depot, ws.scheduled, workers=ws.workers(), memory=ws.memory(), @@ -1348,7 +1346,7 @@ def why(ws: Workspace, p, tid: str, since=None) -> dict: new = None if node["key"] is not None: t = next((t for t in p.tasks if t.key == node["key"]), None) - doc = None if t is not None else ws.store.result(node["key"]) + doc = None if t is not None else ws.depot.result(node["key"]) new = t.key_parts() if t is not None else (doc["keyParts"] if doc else None) prev = ws.previous(since) old_task = next((t for t in prev.get("tasks", []) if t["id"] == tid), None) if prev is not None else None @@ -1356,7 +1354,7 @@ def why(ws: Workspace, p, tid: str, since=None) -> dict: if prev is not None and old_task is not None: previous = {"run": prev["id"], "key": old_task.get("key"), "status": old_task["status"]} if old_task.get("key"): - doc = ws.store.result(old_task["key"]) + doc = ws.depot.result(old_task["key"]) old = doc["keyParts"] if doc else None return { "task": tid, @@ -1379,7 +1377,7 @@ def why_lines(r: dict) -> list[str]: ] -# ---------------------------------------------------------------- store run +# ---------------------------------------------------------------- depot run def run_lines(r: dict) -> list[str]: lines = [] for t in r["tasks"]: @@ -1398,8 +1396,8 @@ def _task_files(items) -> list[Path]: def cmd_run_stage(args, cfg): + from ..pipeline.depot import Depot from ..pipeline.orchestrate import run_one - from ..pipeline.store import Store problems, tasks = [], [] for f in _task_files(args.tasks): @@ -1423,13 +1421,12 @@ def cmd_run_stage(args, cfg): for m in problems: print(f"nnnotes: incompatible task: {m}", file=sys.stderr) sys.exit(EXIT_INCOMPATIBLE) - root = Session(cfg).store_root() - cache = cfg.path("paths", "cache") - store = Store(root, cache=cache, fetch=CatalogFetcher(root, cache, cfg) if args.fetch else None) + root = Session(cfg).depot_root() + depot = Depot(root, fetch=CatalogFetcher(root, cfg) if args.fetch else None) use(cfg) out, bad = [], 0 for t in tasks: - r = run_one(t, store, inst.stages, args.force) + r = run_one(t, depot, inst.stages, args.force) if r["ok"]: out.append({"id": t.id, "key": t.key, "status": r["status"], "result": r["resultStatus"]}) bad += r["resultStatus"] != "ok" @@ -1449,9 +1446,9 @@ def cmd_run_stage(args, cfg): # ---------------------------------------------------------------- catalog versions -def addresses_of(store, vid: str) -> dict | None: +def addresses_of(depot, vid: str) -> dict | None: """The address table (link.addresses) of the latest run over a catalog version, None when no run has one.""" - runs = sorted((store.root / "runs").glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True) + runs = sorted((depot.root / "runs").glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True) for p in runs: try: doc = contract.loads(p.read_bytes()) @@ -1467,10 +1464,10 @@ def addresses_of(store, vid: str) -> dict | None: ), None, ) - res = store.result(t["key"]) if t else None + res = depot.result(t["key"]) if t else None if res: (a,) = res["artifacts"] - return contract.loads(store.read(a["content"]["sha256"])) + return contract.loads(depot.read(a["content"]["sha256"])) return None @@ -1486,7 +1483,7 @@ def _gib(s: str) -> float: def run_args(c) -> None: - """The selection and parameters shared by `export assets` and `store plan`.""" + """The selection and parameters shared by `export assets` and `depot plan`.""" c.add_argument( "--select", action="append", @@ -1501,7 +1498,7 @@ def run_args(c) -> None: c.add_argument( "--stored-catalog", metavar="LABEL|SHA", - help="a catalog version of the store (`catalog versions`; default: the current catalog)", + help="a catalog version of the depot (`catalog versions`; default: the current catalog)", ) c.add_argument("--png-level", type=int, metavar="N", help="PNG compression level 0-9 (changes the output)") c.add_argument( diff --git a/src/nnnotes/cli/browse.py b/src/nnnotes/cli/browse.py index 9e384f6..93f2c79 100644 --- a/src/nnnotes/cli/browse.py +++ b/src/nnnotes/cli/browse.py @@ -49,13 +49,14 @@ class Region: class Server(ThreadingHTTPServer): - """The browser's HTTP server: its regions, the bundle key, the cache, and the catalogs read so far.""" + """The browser's HTTP server: its regions, the bundle key, the depot's downloads/, and the catalogs read so + far.""" - def __init__(self, address, regions: list[Region], bundle_key: BundleKey, cache: Path): + def __init__(self, address, regions: list[Region], bundle_key: BundleKey, downloads: Path): super().__init__(address, Handler) self.regions = {r.name: r for r in regions} self.bundle_key = bundle_key - self.cache = Path(cache) + self.downloads = Path(downloads) self.catalogs: dict[tuple[str, str], tuple[dict, dict]] = {} self.jp_catalogs: dict[tuple[str, str], Catalog] = {} @@ -87,12 +88,14 @@ def catalog(self, region: Region, language: str): key = (region.name, language) if key not in self.server.catalogs: if region.config is not None and region.config.provider(region.name) == "jp": - cat = open_catalog(region.config, region.name, bundle_key=self.server.bundle_key) + cat = open_catalog(region.config, region.name, self.server.downloads, bundle_key=self.server.bundle_key) self.server.catalogs[key] = browse(cat.entries) self.server.jp_catalogs[key] = cat return self.server.catalogs[key] version = Session(region.config).catalog_version(region.name) if region.config is not None else "main" - cat = Catalog.load(language, self.server.cache / "catalogs" / region.name, cdn=region.cdn, version=version) + cat = Catalog.load( + language, self.server.downloads / "catalogs" / region.name, cdn=region.cdn, version=version + ) self.server.catalogs[key] = browse(cat.entries) return self.server.catalogs[key] @@ -174,10 +177,10 @@ def do_GET(self): self.listing(region.label + "/" + language + "/" + prefix, rows) -def serve(regions: list[Region], bundle_key: BundleKey, cache: Path, port: int, host: str = "127.0.0.1") -> None: - """Browse the catalogs of `regions` and download decrypted bundles on `host:port` (catalogs cached under - /catalogs//).""" - server = Server((host, port), regions, bundle_key, cache) +def serve(regions: list[Region], bundle_key: BundleKey, downloads: Path, port: int, host: str = "127.0.0.1") -> None: + """Browse the catalogs of `regions` and download decrypted bundles on `host:port` (catalogs downloaded into + `downloads`/catalogs//).""" + server = Server((host, port), regions, bundle_key, downloads) print(f"nnnotes catalog browse: serving on {host}:{port}", flush=True) try: server.serve_forever() diff --git a/src/nnnotes/cli/bundle.py b/src/nnnotes/cli/bundle.py index 72a94bb..14b98a6 100644 --- a/src/nnnotes/cli/bundle.py +++ b/src/nnnotes/cli/bundle.py @@ -1,4 +1,4 @@ -"""`nnnotes bundle`: fetch bundles into the cache.""" +"""`nnnotes bundle`: fetch bundles into the depot's downloads.""" from __future__ import annotations @@ -16,6 +16,6 @@ def cmd_pull(args, cfg): def register(sub) -> None: bs = group(sub, "bundle", "the bundles of the catalog") c = command( - bs, "pull", cmd_pull, "fetch the bundle closure of keys into the cache (prints the cached bundle files)" + bs, "pull", cmd_pull, "fetch the bundle closure of keys into the depot (prints the downloaded bundle files)" ) c.add_argument("keys", nargs="+", metavar="KEY", help="addressable keys (`catalog keys` lists them)") diff --git a/src/nnnotes/cli/catalog.py b/src/nnnotes/cli/catalog.py index bc055ca..f1f6f14 100644 --- a/src/nnnotes/cli/catalog.py +++ b/src/nnnotes/cli/catalog.py @@ -1,4 +1,4 @@ -"""`nnnotes catalog`: the region's addressable keys, a local catalog browser, and the catalog versions of the store +"""`nnnotes catalog`: the region's addressable keys, a local catalog browser, and the catalog versions of the depot (import, fetch, list, diff).""" from __future__ import annotations @@ -40,14 +40,14 @@ def cmd_browse(args, cfg): if not langs: raise cfg.missing(f"servers.{r}", "languages") regions.append(Region(r, cfg.get(f"servers.{r}", "name") or r, cfg.cdn(r), langs, cfg)) - serve(regions, Session(cfg).bundle_key(), cfg.require_path("paths", "cache"), args.port, args.host) + serve(regions, Session(cfg).bundle_key(), Session(cfg).downloads(), args.port, args.host) -# ---------------------------------------------------------------- catalog versions of the store +# ---------------------------------------------------------------- catalog versions of the depot def _db(cfg): from ..pipeline.catalogdb import CatalogDB - return CatalogDB(Session(cfg).store_root()) + return CatalogDB(Session(cfg).depot_root()) def version_line(v: dict) -> str: @@ -145,7 +145,7 @@ def cmd_fetch(args, cfg): if cfg.provider(region) == "jp": from ..source.jp import open_catalog - cat = open_catalog(cfg, region, apk=cfg.path("paths", "apk")) + cat = open_catalog(cfg, region, Session(cfg).downloads(), apk=cfg.path("paths", "apk")) src, source = cat.sources(), cat.source assert source is not None # a JP catalog names its version source v = _db(cfg).add( @@ -163,7 +163,7 @@ def cmd_fetch(args, cfg): cdn = cfg.cdn(region) language = cfg.require("catalog", "language") version = Session(cfg).catalog_version(region) - filename = Catalog.cache_file(language, Path("."), version=version).name + filename = Catalog.file_name(language, version=version) try: data, hash_text = catalogdb.fetch(cdn, language, version=version) except OSError as e: # URLError, HTTPError, timeouts (the message has no URL) @@ -190,7 +190,7 @@ def cmd_fetch(args, cfg): def cmd_diff(args, cfg): from ..pipeline import catalogdb - from ..pipeline.store import Store + from ..pipeline.depot import Depot from .assets import addresses_of db = _db(cfg) @@ -200,9 +200,9 @@ def cmd_diff(args, cfg): vs.append(db.get(ref)) except (KeyError, ValueError) as e: usage(args, f"{ref}: {e.args[0] if e.args else e}") - store = Store(Session(cfg).store_root()) + depot = Depot(Session(cfg).depot_root()) a, b = vs - d = catalogdb.diff(db.index(a), db.index(b), addresses_of(store, a["id"]), addresses_of(store, b["id"])) + d = catalogdb.diff(db.index(a), db.index(b), addresses_of(depot, a["id"]), addresses_of(depot, b["id"])) if args.json: report(args, d) else: @@ -210,14 +210,14 @@ def cmd_diff(args, cfg): def register(sub) -> None: - cs = group(sub, "catalog", "the region's catalog: its keys, a local browser, and the catalog versions of the store") + cs = group(sub, "catalog", "the region's catalog: its keys, a local browser, and the catalog versions of the depot") c = command(cs, "keys", cmd_keys, "the addressable keys of the catalog (the total on standard error)") c.add_argument("--prefix", default="", help="only the keys that start with this prefix") c.add_argument("--limit", type=int, default=200, help="at most N keys (default 200; 0: all)") c = command(cs, "browse", cmd_browse, "browse the configured regions' catalogs and bundles in a local web page") c.add_argument("--port", type=int, default=8000, help="port (default 8000)") c.add_argument("--host", default="127.0.0.1", help="address to listen on (default 127.0.0.1)") - command(cs, "versions", cmd_versions, "the catalog versions imported into the store (--json: nnnotes.catalogs/1)") + command(cs, "versions", cmd_versions, "the catalog versions imported into the depot (--json: nnnotes.catalogs/1)") c = command(cs, "import", cmd_import, "import a catalog file (with the APK's catalog when [paths] apk is set)") c.add_argument("file", help="catalog__.bin") c.add_argument("--label", help="label of the version (default: its resource version, else the sha256 prefix)") @@ -227,13 +227,13 @@ def register(sub) -> None: help="the game's resource version of this catalog (default: the JP source's version, else the " "version in the file name catalog__.bin)", ) - c = command(cs, "fetch", cmd_fetch, "fetch the region's current catalog and import it into the store") + c = command(cs, "fetch", cmd_fetch, "fetch the region's current catalog and import it into the depot") c.add_argument("--label", help="label of the version (default: the resource version when an API root is set)") c = command( cs, "diff", cmd_diff, - "keys, bundles, raw files (and objects) that differ between two catalog versions of the store " + "keys, bundles, raw files (and objects) that differ between two catalog versions of the depot " "(--json: nnnotes.catalog-diff/1)", ) c.add_argument("old", help="label, version id or sha256 prefix") diff --git a/src/nnnotes/cli/common.py b/src/nnnotes/cli/common.py index 40a786b..257dd59 100644 --- a/src/nnnotes/cli/common.py +++ b/src/nnnotes/cli/common.py @@ -23,13 +23,12 @@ ("catalog", "language"): ("language", "--language"), ("catalog", "version"): ("catalog_version", "--catalog-version"), ("paths", "catalog"): ("catalog", "--catalog"), - ("paths", "cache"): ("cache", "--cache"), ("paths", "master"): ("master", "--master"), ("paths", "apk"): ("apk", "--apk"), ("paths", "ffmpeg"): ("ffmpeg", "--ffmpeg"), ("paths", "vgmstream"): ("vgmstream", "--vgmstream"), ("paths", "node"): ("node", "--node"), - ("paths", "store"): ("store", "--store"), + ("paths", "depot"): ("depot", "--depot"), ("paths", "player"): ("player", "--player"), } # the global options: dest -> (flag, metavar, help); --player is an option of `site build` only @@ -41,15 +40,14 @@ "LANG", "catalog and client language: ja, en, zh-Hant, zh-Hans or ko ([catalog] language)", ), - "catalog": ("--catalog", "FILE", "catalog .bin file ([paths] catalog; else downloaded into the cache)"), + "catalog": ("--catalog", "FILE", "catalog .bin file ([paths] catalog; else downloaded into the depot)"), "catalog_version": ( "--catalog-version", "VERSION", "the resource version of an international catalog ([catalog] version; else the version the game API " "serves when an API root is configured, otherwise main)", ), - "cache": ("--cache", "DIR", "cache directory ([paths] cache)"), - "store": ("--store", "DIR", "store directory ([paths] store; default /store)"), + "depot": ("--depot", "DIR", "depot directory ([paths] depot)"), "master": ("--master", "DIR", "decoded master data directory ([paths] master)"), "apk": ("--apk", "FILE", "base.apk ([paths] apk)"), "ffmpeg": ("--ffmpeg", "EXE", "ffmpeg executable ([paths] ffmpeg; else on PATH)"), diff --git a/src/nnnotes/cli/store.py b/src/nnnotes/cli/depot.py similarity index 54% rename from src/nnnotes/cli/store.py rename to src/nnnotes/cli/depot.py index 4736a4f..77f5d69 100644 --- a/src/nnnotes/cli/store.py +++ b/src/nnnotes/cli/depot.py @@ -1,5 +1,5 @@ -"""`nnnotes store`: what `export assets` would run and why (plan), task descriptions run into the store (run), and the -check of every stored content and result (verify).""" +"""`nnnotes depot`: what `export assets` would run and why (plan), task descriptions run into the depot (run), the +check of every stored content and result (verify), and the removal of what no kept run reaches (gc).""" from __future__ import annotations @@ -8,21 +8,39 @@ from ..source.config import usable_cpus from ..source.session import Session from . import assets -from .common import command, group, report, workers_arg +from .common import command, group, report, usage, workers_arg def cmd_verify(args, cfg): - from ..pipeline.store import Store + from ..pipeline.depot import Depot workers = usable_cpus() if args.workers is None else args.workers - r = Store(Session(cfg).store_root()).verify(quick=args.quick, workers=workers) + r = Depot(Session(cfg).depot_root()).verify(quick=args.quick, workers=workers) report(args, r) if r["problems"]: sys.exit(assets.EXIT_FAILED) +def cmd_gc(args, cfg): + from ..pipeline.depot import Depot + + if args.keep is not None and args.keep < 1: + usage(args, "--keep must be at least 1") + r = Depot(Session(cfg).depot_root()).gc(keep=args.keep, dry_run=args.dry_run) + report( + args, + r, + lambda r: [ + f"{'would remove' if args.dry_run else 'removed'}: {r['runs']['removed']} runs, " + f"{r['results']['removed']} results, {r['contents']['removed']} contents " + f"({r['bytesRemoved'] / 1e6:.1f} MB)", + f"kept: {r['runs']['kept']} runs, {r['results']['kept']} results, {r['contents']['kept']} contents", + ], + ) + + def register(sub) -> None: - ss = group(sub, "store", "the store of `export assets`: its plan, single tasks, its check") + ss = group(sub, "depot", "the depot: its plan, single tasks, its check, its collection") c = command( ss, @@ -38,7 +56,7 @@ def register(sub) -> None: c.add_argument("--emit-tasks", metavar="DIR", help="write the runnable tasks as task descriptions") c.add_argument("--why", metavar="TASK", help="the reasons and key parts of one task") - c = command(ss, "run", assets.cmd_run_stage, "run task descriptions (nnnotes.task/1) into the store") + c = command(ss, "run", assets.cmd_run_stage, "run task descriptions (nnnotes.task/1) into the depot") c.add_argument("tasks", nargs="+", metavar="TASK", help="task description files, or directories of them") c.add_argument("--fetch", action="store_true", help="fetch inputs through their catalog locators") c.add_argument("--force", action="store_true", help="run tasks whose result is stored (it must not change)") @@ -46,3 +64,12 @@ def register(sub) -> None: c = command(ss, "verify", cmd_verify, "check every stored content and result") c.add_argument("--quick", action="store_true", help="check sizes only, do not hash the contents") workers_arg(c, "parallel hashing", "the CPUs this process may use", zero="one at a time") + + c = command(ss, "gc", cmd_gc, "remove the runs, results and contents no kept run reaches") + c.add_argument( + "--keep", + type=int, + metavar="N", + help="keep the N latest runs of each selection (default: every run; the latest run of a selection is kept)", + ) + c.add_argument("--dry-run", action="store_true", help="count what would be removed, remove nothing") diff --git a/src/nnnotes/cli/export.py b/src/nnnotes/cli/export.py index a9bbfb6..a59b440 100644 --- a/src/nnnotes/cli/export.py +++ b/src/nnnotes/cli/export.py @@ -1,4 +1,4 @@ -"""`nnnotes export`: the selected bundles of the catalog into the store and its layouts (`assets`), and single +"""`nnnotes export`: the selected bundles of the catalog into the depot and its layouts (`assets`), and single subjects into a directory or file: an ADV episode, a story, a live, a Live2D model, a spot, a room, shaders, a cue sheet, the APK's UI library.""" @@ -235,20 +235,20 @@ def cmd_ui(args, cfg): def register(sub) -> None: - es = group(sub, "export", "game data -> common formats: the store's asset export and single subjects") + es = group(sub, "export", "game data -> common formats: the depot's asset export and single subjects") c = command( es, "assets", assets.cmd_export, - "every object of the selected bundles -> common formats (the store and its layouts)", + "every object of the selected bundles -> common formats (the depot and its layouts)", ) out_arg(c, "output directory") assets.run_args(c) c.add_argument( "--link", choices=("auto", "clone", "hard", "copy"), - help="how the layout files are made from the store: auto (default; [export] link): a clone where " + help="how the layout files are made from the depot: auto (default; [export] link): a clone where " "the file system makes them, else a hard link (read-only), else a copy; or that method only", ) c.add_argument("--strict", action="store_true", help="exit 1 when a view has gaps") diff --git a/src/nnnotes/cli/musicdata.py b/src/nnnotes/cli/musicdata.py index dab1236..c39d7e7 100644 --- a/src/nnnotes/cli/musicdata.py +++ b/src/nnnotes/cli/musicdata.py @@ -41,7 +41,7 @@ def cmd_music_data(args, cfg): None if args.no_bgm else musicdata.catalog_bgm(cat), region=region, client=deckdata.apk_client(apk) if apk is not None else {}, - catalog=deckdata.catalog_info(cat, session.store_root()), + catalog=deckdata.catalog_info(cat, session.depot_root()), deck=deck, full=args.full, jacket=musicdata.catalog_jacket(cat) if args.jackets else None, diff --git a/src/nnnotes/content/ui.py b/src/nnnotes/content/ui.py index 1e929b8..559348e 100644 --- a/src/nnnotes/content/ui.py +++ b/src/nnnotes/content/ui.py @@ -397,9 +397,7 @@ def export_apk_library( "region": cfg.get("catalog", "region"), "client": {"versionName": manifest_version_name(manifest), "versionCode": manifest_version_code(manifest)}, } - cat = Catalog( - catalog_data, cfg.require_path("paths", "cache"), bundle_key=lambda: Session(cfg).bundle_key(), apk=apk - ) + cat = Catalog(catalog_data, Session(cfg).downloads(), bundle_key=lambda: Session(cfg).bundle_key(), apk=apk) keys = keys or cat.keys(prefix) if limit is not None: keys = keys[:limit] diff --git a/src/nnnotes/formats/cri.py b/src/nnnotes/formats/cri.py index ccce5da..f105342 100644 --- a/src/nnnotes/formats/cri.py +++ b/src/nnnotes/formats/cri.py @@ -22,7 +22,7 @@ `decode` writes one file per vgmstream stream (a name repeated within the sheet, e.g. the layered waveforms of one cue, gets `_`), `cues.json` (first stream per name) and `streams.json` (every stream in vgmstream order). `decode_acb_bytes` does the same for ACB bytes -given directly (no catalog, no cache: the cri.audio stage keeps its results in the store); `split_acb` and +given directly (no catalog, no cache: the cri.audio stage keeps its results in the depot); `split_acb` and `embedded_acb` give the ACB bytes of the two in-bundle layouts from their fields. HCA streams are decrypted with the keycode from the game's own boot data (see diff --git a/src/nnnotes/musicdata/deckdata.py b/src/nnnotes/musicdata/deckdata.py index c714947..a6f903e 100644 --- a/src/nnnotes/musicdata/deckdata.py +++ b/src/nnnotes/musicdata/deckdata.py @@ -452,28 +452,28 @@ def apk_client(apk) -> dict: return {"versionName": manifest_version_name(data), "versionCode": manifest_version_code(data)} -def catalog_info(cat, store_root=None) -> dict: +def catalog_info(cat, depot_root=None) -> dict: """{resourceVersion, sha256} of a catalog's remote catalog file (resource_version).""" sha = hashlib.sha256(cat.sources()["remote"]).hexdigest() source = getattr(cat, "source", None) return { "resourceVersion": source.version if source - else (getattr(cat, "resource_version", None) or resource_version(store_root, sha)), + else (getattr(cat, "resource_version", None) or resource_version(depot_root, sha)), "sha256": sha, **({"resourceHash": source.hash} if source else {}), } -def resource_version(store_root, remote_sha: str) -> str | None: - """The resource version recorded for the remote catalog `remote_sha` by the catalog versions of a store +def resource_version(depot_root, remote_sha: str) -> str | None: + """The resource version recorded for the remote catalog `remote_sha` by the catalog versions of a depot (the latest import that has one), None when none is recorded. Read only.""" - if store_root is None: + if depot_root is None: return None from ..pipeline.catalogdb import CatalogDB try: - versions = CatalogDB(store_root).versions() + versions = CatalogDB(depot_root).versions() except (OSError, ValueError): return None for v in reversed(versions): diff --git a/src/nnnotes/pipeline/__init__.py b/src/nnnotes/pipeline/__init__.py index 261d5d6..4c7284d 100644 --- a/src/nnnotes/pipeline/__init__.py +++ b/src/nnnotes/pipeline/__init__.py @@ -1 +1 @@ -"""The content-addressed export pipeline: stages, tasks, the store, layouts and the atoms.""" +"""The content-addressed export pipeline: stages, tasks, the depot, layouts and the atoms.""" diff --git a/src/nnnotes/pipeline/catalogdb.py b/src/nnnotes/pipeline/catalogdb.py index b1545c6..fd3655d 100644 --- a/src/nnnotes/pipeline/catalogdb.py +++ b/src/nnnotes/pipeline/catalogdb.py @@ -1,4 +1,4 @@ -"""Catalog versions: the catalog index, a store of catalog versions, and the difference between two versions. +"""Catalog versions: the catalog index, a depot of catalog versions, and the difference between two versions. The catalog index (nnnotes.catalog-index/1) is everything a catalog pair says, decoded (addressables.parse_locations): the remote catalog and, when given, the APK's local catalog. It holds every location (primary key, internal id, @@ -9,10 +9,10 @@ references to APK files may come from another build). The index is a function of the catalog bytes alone. A catalog version is identified by the sha256 of its remote catalog and of its APK catalog (none without an APK). -The versions live in the store: +The versions live in the depot: - /catalogs/.bin each catalog file (remote and APK catalogs alike), by content - /catalogs/index.json the versions (nnnotes.catalogs/1): id, labels, region, language, the two + /catalogs/.bin each catalog file (remote and APK catalogs alike), by content + /catalogs/index.json the versions (nnnotes.catalogs/1): id, labels, region, language, the two catalogs (sha256, size, buildResultHash), the CDN's `.hash` text when it was served (an opaque change signal), resource version, APK version name @@ -35,8 +35,8 @@ from ..source.catalog import APK_CATALOG, Catalog, file_name, location_kind from . import contract from .contract import Cost, Input +from .depot import write_file from .stages import Output, Stage -from .store import write_file CATALOGS = "nnnotes.catalogs/1" DIFF = "nnnotes.catalog-diff/1" @@ -365,7 +365,7 @@ def format_diff(d: dict) -> str: return "\n".join(lines) + "\n" -# ---------------------------------------------------------------- the store of versions +# ---------------------------------------------------------------- the depot of versions def version_id(remote_sha: str, apk_sha: str | None, source: dict | None = None) -> str: """The id of a catalog version: the key hash of its two catalogs' content ids.""" identity = {"remote": remote_sha, "apk": apk_sha} @@ -399,7 +399,7 @@ def apk_version(apk: Path | None) -> str | None: def fetch(cdn: str, language: str, timeout: float = 120, *, version: str = "main") -> tuple[bytes, str | None]: """The remote catalog of `language` from a CDN base, and the text of its `.hash` file (None when not served).""" - base = cdn.rstrip("/") + "/asset/Android/" + Catalog.cache_file(language, Path("."), version=version).name + base = cdn.rstrip("/") + "/asset/Android/" + Catalog.file_name(language, version=version) with urllib.request.urlopen(base, timeout=timeout) as r: data = r.read() try: @@ -411,10 +411,10 @@ def fetch(cdn: str, language: str, timeout: float = 120, *, version: str = "main class CatalogDB: - """The catalog versions of a store (/catalogs/).""" + """The catalog versions of a depot (/catalogs/).""" - def __init__(self, store_root): - self.root = Path(store_root) / "catalogs" + def __init__(self, depot_root): + self.root = Path(depot_root) / "catalogs" def path(self, sha: str) -> Path: if not contract.is_sha256(sha): @@ -540,7 +540,7 @@ def index(self, version: dict) -> dict: return index(*self.catalog_bytes(version)) def inputs(self, version: dict) -> list[Input]: - """The inputs of the catalog.index task of a version (roles "remote" and "apk"), read from this store.""" + """The inputs of the catalog.index task of a version (roles "remote" and "apk"), read from this depot.""" out = [] for role in CATALOG_NAMES: rec = version[role] @@ -579,12 +579,12 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs) return Cost(0.2 + size / 10e6, (60 << 20) + 40 * size) - def run(self, task, store) -> Output: + def run(self, task, depot) -> Output: roles = {i.role for i in task.inputs} doc = index( - store.input_bytes(task.input("remote")), store.input_bytes(task.input("apk")) if "apk" in roles else None + depot.input_bytes(task.input("remote")), depot.input_bytes(task.input("apk")) if "apk" in roles else None ) - content = store.add(contract.encode(doc), "json") + content = depot.add(contract.encode(doc), "json") rec = contract.artifact( contract.artifact_id(task.id, "index"), content, diff --git a/src/nnnotes/pipeline/census.py b/src/nnnotes/pipeline/census.py index 4eb71e2..2875e36 100644 --- a/src/nnnotes/pipeline/census.py +++ b/src/nnnotes/pipeline/census.py @@ -330,9 +330,9 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs) return Cost(0.002 + size / 40e6, (32 << 20) + 12 * size) - def run(self, task, store) -> Output: - doc = census_file(store.open_input(task.input("bundle"))) - content = store.add(contract.encode(doc), "json") + def run(self, task, depot) -> Output: + doc = census_file(depot.open_input(task.input("bundle"))) + content = depot.add(contract.encode(doc), "json") rec = contract.artifact( contract.artifact_id(task.id, "census"), content, diff --git a/src/nnnotes/pipeline/contract.py b/src/nnnotes/pipeline/contract.py index f2dd92b..8c958c8 100644 --- a/src/nnnotes/pipeline/contract.py +++ b/src/nnnotes/pipeline/contract.py @@ -56,7 +56,7 @@ RESULT_STATUSES = ("ok", "partial") RUN_TASK_STATUSES = ("hit", "ran", "failed") # locator kinds of a task input: where the bytes can be read (never part of the key) -LOCATOR_KINDS = ("store", "cache", "file", "catalog") +LOCATOR_KINDS = ("depot", "download", "file", "catalog") # an object's primary artifact is the one whose role comes first here; its other artifacts are secondary PRIMARY_ROLES = ("image", "mesh", "audio", "video", "font", "data", "prefab", "json") MEDIA_TYPES = { diff --git a/src/nnnotes/pipeline/cristages.py b/src/nnnotes/pipeline/cristages.py index 87f314d..8d4d794 100644 --- a/src/nnnotes/pipeline/cristages.py +++ b/src/nnnotes/pipeline/cristages.py @@ -70,30 +70,30 @@ # ---------------------------------------------------------------- inputs shared by the stages -def boot_input(store, apk) -> Input | None: +def boot_input(depot, apk) -> Input | None: """The input "boot" of the CRI stages: the boot data of an APK (`assets/bin/Data/data.unity3d`, where the HCA - keycode is read), stored in `store` once; the APK is hashed once (store.identify memo) and its boot data is + keycode is read), stored in `depot` once; the APK is hashed once (depot.identify memo) and its boot data is remembered by the APK's content id. A path that is not an .apk / .zip is the boot data itself. None without an APK.""" if apk is None: return None apk = Path(apk) if not apk.is_dir() and apk.suffix.lower() not in (".apk", ".zip", ".apks", ".xapk"): - sha, size = store.identify(apk) + sha, size = depot.identify(apk) return Input("boot", sha, size, apk.name, ({"kind": "file", "path": str(apk.resolve())},)) if apk.is_dir(): with ApkSet(apk) as z: data = z.read(BOOT_IN_APK) - sha = store.put(data) - return Input("boot", sha, len(data), "data.unity3d", ({"kind": "store"},)) - apk_sha, _ = store.identify(apk) + sha = depot.put(data) + return Input("boot", sha, len(data), "data.unity3d", ({"kind": "depot"},)) + apk_sha, _ = depot.identify(apk) name = f"boot:{apk_sha}" - known = store.named("files", name) - if known is None or not store.has(*known): + known = depot.named("files", name) + if known is None or not depot.has(*known): with ApkSet(apk) as z: data = z.read(BOOT_IN_APK) - known = store.identify(store.path(store.put(data)), "files", name) - return Input("boot", known[0], known[1], "data.unity3d", ({"kind": "store"},)) + known = depot.identify(depot.path(depot.put(data)), "files", name) + return Input("boot", known[0], known[1], "data.unity3d", ({"kind": "depot"},)) def _boot(env) -> Input: @@ -103,14 +103,14 @@ def _boot(env) -> Input: return inp -def hca_key(store, inp: Input) -> int: +def hca_key(depot, inp: Input) -> int: """The HCA keycode in the boot data `inp` (crikey.find_key), read once per content and process.""" with _lock: if inp.sha256 in _keys: return _keys[inp.sha256] from ..formats import crikey - key = crikey.find_key(store.open_input(inp)) + key = crikey.find_key(depot.open_input(inp)) with _lock: _keys[inp.sha256] = key return key @@ -153,19 +153,19 @@ def _tool(which: str) -> str: return tool(which, "vgmstream-cli" if which == "vgmstream" else "ffmpeg") -def _magic_of(store, inp: Input, name: str) -> bytes: - """The first 4 bytes of a raw file, from a local copy (the store, the cache, a file); one that is not fetched +def _magic_of(depot, inp: Input, name: str) -> bytes: + """The first 4 bytes of a raw file, from a local copy (the depot, the cache, a file); one that is not fetched yet is Pending("fetch:").""" with _lock: m = _magic.get(inp.sha256) if m is None: - from .store import InputMissing + from .depot import InputMissing local = Input( inp.role, inp.sha256, inp.size, inp.name, tuple(loc for loc in inp.locators if loc.get("kind") != "catalog") ) try: - path = store.open_input(local) + path = depot.open_input(local) except InputMissing: raise Pending(f"fetch:{inp.name or name}") from None with open(path, "rb") as f: @@ -187,7 +187,7 @@ def _raw(env) -> dict[str, tuple[Input, bytes]]: raise Pending(f"fetch:{stable}") if isinstance(inp, Exception): raise inp - out[stable] = (inp, _magic_of(env.store, inp, stable)) + out[stable] = (inp, _magic_of(env.depot, inp, stable)) except Pending as p: waiting.append(p) if waiting: @@ -279,11 +279,11 @@ def _check_atoms(self, task, want: dict) -> None: ) def _artifact( - self, task, store, role: str, data: bytes, ext: str, kind: str, fmt: str, facts: dict, media: str | None = None + self, task, depot, role: str, data: bytes, ext: str, kind: str, fmt: str, facts: dict, media: str | None = None ) -> dict: return contract.artifact( contract.artifact_id(task.id, role), - store.add(data, ext, media), + depot.add(data, ext, media), contract.provenance(task), {"kind": kind, "format": fmt, "facts": facts}, ) @@ -353,11 +353,11 @@ def entry(sha: str) -> dict: a = raw[other][0] e["awb"] = Input("awb", a.sha256, a.size, a.name, a.locators) for tid in tids: - for a in _export_artifacts(env.store, env.key(tid), ACB_KIND): + for a in _export_artifacts(env.depot, env.key(tid), ACB_KIND): c = a["content"] e = entry(c["sha256"]) if e["acb"] is None: - e["acb"] = Input("acb", c["sha256"], c["size"], None, ({"kind": "store"},)) + e["acb"] = Input("acb", c["sha256"], c["size"], None, ({"kind": "depot"},)) e["depends"].add(tid) sheet = (a["semantics"].get("facts") or {}).get("cueSheet") if sheet: @@ -378,7 +378,7 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs if i.role == "acb") return Cost(0.3 + size * 2.0e-6, (64 << 20) + 8 * size) - def run(self, task, store) -> Output: + def run(self, task, depot) -> Output: from ..formats import cri vgm, ff = _tool("vgmstream"), _tool("ffmpeg") @@ -390,8 +390,8 @@ def run(self, task, store) -> Output: "unsupported.cri.awb_external", "the cue sheet's waveforms are streamed from an external AWB" ), ) - acb = store.input_bytes(task.input("acb")) - key = hca_key(store, task.input("boot")) + acb = depot.input_bytes(task.input("acb")) + key = hca_key(depot, task.input("boot")) level = int(task.params["flac"]["level"]) with tempfile.TemporaryDirectory(prefix="nnnotes-cri-") as tmp: out = Path(tmp) @@ -410,7 +410,7 @@ def run(self, task, store) -> Output: arts = [ self._artifact( task, - store, + depot, s["file"], (out / s["file"]).read_bytes(), "flac", @@ -423,7 +423,7 @@ def run(self, task, store) -> Output: arts.append( self._artifact( task, - store, + depot, "cues.json", (out / "cues.json").read_bytes(), "json", @@ -435,7 +435,7 @@ def run(self, task, store) -> Output: arts.append( self._artifact( task, - store, + depot, "streams.json", (out / "streams.json").read_bytes(), "json", @@ -447,11 +447,11 @@ def run(self, task, store) -> Output: return self._item(task, arts) -def _export_artifacts(store, key: str, kind: str) -> list[dict]: +def _export_artifacts(depot, key: str, kind: str) -> list[dict]: """The artifact records of semantics kind `kind` (cri.acb, cri.usm) of a unity.export result (a result without the kind is not parsed).""" try: - data = store.result_path(key).read_bytes() + data = depot.result_path(key).read_bytes() except OSError: return [] if f'"{kind}"'.encode() not in data: @@ -521,9 +521,9 @@ def entry(inp: Input) -> dict: e["names"] |= cat.keys.get(stable) or {stable} for tid in tids: bundle = contract.parse_task_id(tid, EXPORT)[1] - for a in _export_artifacts(env.store, env.key(tid), USM_KIND): + for a in _export_artifacts(env.depot, env.key(tid), USM_KIND): c = a["content"] - e = entry(Input("usm", c["sha256"], c["size"], None, ({"kind": "store"},))) + e = entry(Input("usm", c["sha256"], c["size"], None, ({"kind": "depot"},))) e["depends"].add(tid) movie = (a["semantics"].get("facts") or {}).get("movie") keys = {k for k in cat.bundle_keys.get(bundle, ()) if movie and k.rsplit("/", 1)[-1] == movie} @@ -542,13 +542,13 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs if i.role == "usm") return Cost(0.3 + size * 1.1e-7, (96 << 20) + 12 * size) - def run(self, task, store) -> Output: + def run(self, task, depot) -> Output: from ..formats import usm ff, vgm = _tool("ffmpeg"), _tool("vgmstream") self._check_atoms(task, movie_atoms(ff, vgm)) - movie = store.input_bytes(task.input("usm")) - key = hca_key(store, task.input("boot")) + movie = depot.input_bytes(task.input("usm")) + key = hca_key(depot, task.input("boot")) try: streams = usm.demux(movie, key) except usm.Unsupported as e: @@ -556,7 +556,7 @@ def run(self, task, store) -> Output: arts, why, tracks = [], None, [] for s in streams: if s.kind == "subtitle": - made, problem = self._subtitle(task, store, s) + made, problem = self._subtitle(task, depot, s) arts += made if problem is None: tracks.append(s) @@ -573,7 +573,7 @@ def run(self, task, store) -> Output: kind = "audio.stream" else: facts, kind = usm.video_facts(s), "video.stream" if s.kind == "video" else "video.alpha" - arts.append(self._artifact(task, store, f"{s.name}.{ext}", s.data, ext, kind, fmt, facts, media)) + arts.append(self._artifact(task, depot, f"{s.name}.{ext}", s.data, ext, kind, fmt, facts, media)) tracks.append(s) out_fmt = task.params["format"] with tempfile.TemporaryDirectory(prefix="nnnotes-usm-") as tmp: @@ -591,11 +591,11 @@ def run(self, task, store) -> Output: facts = {"tracks": [_track(s, out_fmt, reencoded) for s in tracks]} fmt, media = MOVIE_FORMATS[out_fmt] arts.append( - self._artifact(task, store, dst.name, dst.read_bytes(), out_fmt, "video.movie", fmt, facts, media) + self._artifact(task, depot, dst.name, dst.read_bytes(), out_fmt, "video.movie", fmt, facts, media) ) return self._item(task, arts, why) - def _subtitle(self, task, store, s) -> tuple[list[dict], str | None]: + def _subtitle(self, task, depot, s) -> tuple[list[dict], str | None]: """The artifacts of a subtitle channel and why it is not a track (None when it is): the records as JSON, and as SubRip and WebVTT when every text is UTF-8; the bytes as stored (`.sbt`) when they are not records.""" from ..core.jsonio import dumps @@ -606,12 +606,12 @@ def _subtitle(self, task, store, s) -> tuple[list[dict], str | None]: except ValueError as e: return [ self._artifact( - task, store, f"{s.name}.sbt", s.data, "sbt", "subtitle.stream", "sbt", {"channel": s.channel} + task, depot, f"{s.name}.sbt", s.data, "sbt", "subtitle.stream", "sbt", {"channel": s.channel} ) ], f"{s.name}: {e}" facts = {"channel": s.channel, "records": len(records)} doc = dumps(usm.subtitle_json(s.channel, records), indent=1, ensure_ascii=False).encode() - arts = [self._artifact(task, store, f"{s.name}.json", doc, "json", "subtitle.records", "json", facts)] + arts = [self._artifact(task, depot, f"{s.name}.json", doc, "json", "subtitle.records", "json", facts)] try: srt, vtt = usm.subtitle_srt(records), usm.subtitle_webvtt(records) except UnicodeDecodeError: @@ -619,7 +619,7 @@ def _subtitle(self, task, store, s) -> tuple[list[dict], str | None]: arts.append( self._artifact( task, - store, + depot, f"{s.name}.srt", srt.encode(), "srt", @@ -631,7 +631,7 @@ def _subtitle(self, task, store, s) -> tuple[list[dict], str | None]: ) arts.append( self._artifact( - task, store, f"{s.name}.vtt", vtt.encode(), "vtt", "subtitle.track", "webvtt", facts, "text/vtt" + task, depot, f"{s.name}.vtt", vtt.encode(), "vtt", "subtitle.track", "webvtt", facts, "text/vtt" ) ) return arts, None diff --git a/src/nnnotes/pipeline/store.py b/src/nnnotes/pipeline/depot.py similarity index 75% rename from src/nnnotes/pipeline/store.py rename to src/nnnotes/pipeline/depot.py index 8e8acc5..027fae0 100644 --- a/src/nnnotes/pipeline/store.py +++ b/src/nnnotes/pipeline/depot.py @@ -1,21 +1,24 @@ -"""The content-addressed store of the asset stages: a documented format, and the cache of their work. +"""The depot: the files a configuration downloads, the results of the stages by content, and the record of runs. - /cas/sha256/ab/ bytes by content id, write-once (no extension: the records carry it) - /ac/ab/.json task results (nnnotes.result/1) by task key; written after every byte they + /downloads/ the game files as downloaded: catalogs, decrypted bundles (bundles/), + raw files (raw/), files read from the APK, each source in its own + directory (international///, jp////) + /cas/sha256/ab/ bytes by content id, write-once (no extension: the records carry it) + /ac/ab/.json task results (nnnotes.result/1) by task key; written after every byte they name is in cas/, so a result present is a complete result (the commit) - /inputs/.jsonl input identities: name -> sha256, size, with the (path, size, mtime) of the + /inputs/.jsonl input identities: name -> sha256, size, with the (path, size, mtime) of the file that was hashed (a memo: a file seen unchanged is not hashed again); bundles.jsonl, raw.jsonl, files.jsonl (files verified by path only) - /catalogs/ versioned catalogs (catalogdb) - /runs/.json run manifests (nnnotes.run/1), deterministic - /runs/.log.jsonl the log of the run's last execution (timings, workers, errors) - /runs/selections/ the id of the latest run of a selection (digest of its selection) - /costs/.json the measured cost of a task's last run (nnnotes.cost/1) + /catalogs/ versioned catalogs (catalogdb) + /runs/.json run manifests (nnnotes.run/1), deterministic + /runs/.log.jsonl the log of the run's last execution (timings, workers, errors) + /runs/selections/ the id of the latest run of a selection (digest of its selection) + /costs/.json the measured cost of a task's last run (nnnotes.cost/1) Every file is written through a temporary file and a rename, so concurrent writers (threads, processes, machines on a shared file system) never see or leave a partial file, and a crash before a result's entry leaves no false hit. The files of cas/ are read-only (a layout may hard-link them: an edit of such a file fails instead of changing the -store). +depot). Only costs/, runs/ and inputs/ hold non-deterministic data; cas/ and ac/ are a function of the tasks run. """ @@ -39,15 +42,15 @@ class InputMissing(FileNotFoundError): """No locator of an input gives a file with its content id.""" -class StoreConflict(RuntimeError): - """A result already in the store differs from the one being committed for the same key: the stage's output is +class DepotConflict(RuntimeError): + """A result already in the depot differs from the one being committed for the same key: the stage's output is not a function of its key (a nondeterministic stage, or a change without a version bump).""" def _replace(tmp: Path, dst: Path, same) -> bool: """Rename `tmp` onto `dst`; False when another writer put `dst` first (`same(dst)` true) and `tmp` was dropped. On some systems a rename onto (or a read of) a file another process is replacing or reading fails for a - moment: retried, as sitefiles.Store does.""" + moment: retried, as sitefiles.Depot does.""" try: for i in range(20): try: @@ -61,7 +64,7 @@ def _replace(tmp: Path, dst: Path, same) -> bool: except PermissionError: pass time.sleep(0.1 * (i + 1)) - raise RuntimeError(f"could not store {dst}") + raise RuntimeError(f"could not depot {dst}") except BaseException: _drop(tmp) raise @@ -90,7 +93,7 @@ def _read(p: Path) -> bytes | None: def _write(dst: Path, data: bytes, same=None, read_only: bool = False) -> bool: """Write `data` to `dst` atomically (creating the directory); with `same`, keep an existing `dst` for which - same(dst) holds; `read_only`: the file is read-only (store objects). True when this call wrote the file.""" + same(dst) holds; `read_only`: the file is read-only (depot objects). True when this call wrote the file.""" dst.parent.mkdir(parents=True, exist_ok=True) tmp = temp_path(dst) try: @@ -157,29 +160,33 @@ def _lines(path: Path): continue -class Store: - """A store directory. `cache`: the directory "cache" locators are relative to; `fetch(input, locator) -> Path`: - how a "catalog" locator is fetched (none: such locators are skipped).""" +class Depot: + """A depot directory. `fetch(input, locator) -> Path`: how a "catalog" locator is fetched (none: such locators + are skipped).""" - def __init__(self, root, *, cache=None, fetch=None): + def __init__(self, root, *, fetch=None): self.root = Path(root) - self.cache = Path(cache) if cache else None self.fetch = fetch self.written = self.reused = 0 - self._have: set[str] = set() # content ids this store object wrote or found + self._have: set[str] = set() # content ids this depot object wrote or found self._memo: dict | None = None # (path, size, mtime_ns) -> (sha256, size) self._names: dict[str, dict] = {} # kind -> {name: (sha256, size)} self._docs: dict[str, dict] = {} # the documents document() decoded last self._lock = threading.Lock() - def __getstate__(self): # a store crosses to worker processes as its settings - return {"root": self.root, "cache": self.cache, "fetch": self.fetch} + def __getstate__(self): # a depot crosses to worker processes as its settings + return {"root": self.root, "fetch": self.fetch} def __setstate__(self, state): - self.__init__(state["root"], cache=state["cache"], fetch=state["fetch"]) + self.__init__(state["root"], fetch=state["fetch"]) def __repr__(self) -> str: - return f"Store({str(self.root)!r})" + return f"Depot({str(self.root)!r})" + + @property + def downloads(self) -> Path: + """The directory of downloaded game files ("download" locators are relative to it).""" + return self.root / "downloads" @property def catalogs(self) -> Path: @@ -315,7 +322,7 @@ def committed(self, key: str, task: str) -> dict: """The committed result of a task key that has one (a FileNotFoundError naming `task` when it is missing).""" doc = self.result(key) if doc is None: - raise FileNotFoundError(f"{task}: its result {key} is not in the store") + raise FileNotFoundError(f"{task}: its result {key} is not in the depot") return doc def has_result(self, key: str) -> bool: @@ -323,7 +330,7 @@ def has_result(self, key: str) -> bool: def commit(self, doc: dict) -> bool: """Write a result entry, after checking that the bytes of every artifact it names are stored. Write-once: an - entry already there must be identical (else StoreConflict). True when this call wrote it.""" + entry already there must be identical (else DepotConflict). True when this call wrote it.""" if doc.get("schema") != contract.RESULT: raise ValueError("not a result document") for a in doc["artifacts"]: @@ -338,7 +345,7 @@ def same(p: Path) -> bool: if old is None: return False if old != data: - raise StoreConflict(f"result {doc['key']} differs from the one in the store ({dst})") + raise DepotConflict(f"result {doc['key']} differs from the one in the depot ({dst})") return True if same(dst): @@ -401,27 +408,24 @@ def named(self, kind: str, name: str) -> tuple[str, int] | None: return self._names.get(kind, {}).get(name) def open_input(self, inp: Input) -> Path: - """A file holding the bytes of `inp`: its locators in order (none: the store), each checked against the - content id (store: by name and size; files: hashed, memoized). Raises InputMissing.""" + """A file holding the bytes of `inp`: its locators in order (none: the depot), each checked against the + content id (depot: by name and size; files: hashed, memoized). Raises InputMissing.""" tried = [] - for loc in inp.locators or ({"kind": "store"},): + for loc in inp.locators or ({"kind": "depot"},): kind = loc["kind"] try: - if kind == "store": + if kind == "depot": if self.has(inp.sha256, inp.size): return self.path(inp.sha256) - tried.append("store: absent") + tried.append("depot: absent") continue if kind == "catalog": if self.fetch is None: tried.append("catalog: no fetcher") continue p = Path(self.fetch(inp, loc)) - elif kind == "cache": - if self.cache is None: - tried.append("cache: no cache directory") - continue - p = self.cache / loc["path"] + elif kind == "download": + p = self.downloads / loc["path"] else: p = Path(loc["path"]) if not p.is_file() or p.stat().st_size != inp.size: @@ -440,7 +444,7 @@ def input_bytes(self, inp: Input) -> bytes: # ---------------------------------------------------------------- verification def verify(self, *, quick: bool = False, workers: int = 8, limit: int = 100) -> dict: - """Check the store: every cas/ file has the content its name says (`quick`: only that it is a file of a + """Check the depot: every cas/ file has the content its name says (`quick`: only that it is a file of a well-formed name; its size is checked against the results naming it), every ac/ entry is a result whose name is its key, whose key is the key of its parts and whose artifacts' bytes are stored with their size. -> {"contents", "results", "problems": [{path, problem}] (the first `limit`, sorted), "problemCount"}.""" @@ -540,3 +544,92 @@ def cost(self, key: str) -> dict | None: return contract.loads((self.root / "costs" / f"{key}.json").read_bytes()) except OSError: return None + + # ---------------------------------------------------------------- collection + def gc(self, *, keep: int | None = None, dry_run: bool = False) -> dict: + """Remove what no kept run reaches. The kept runs: of each selection the `keep` latest by the time their + manifest was written (None: every run), and always the run the selection names as its latest. A kept run + reaches the results of its tasks, the contents those results name (their artifacts, and the inputs their key + parts name when the depot holds them) and the layout manifests it names with the contents they list. + Removed: the other runs (manifest and log), results and their measured costs, and contents. `dry_run`: count + only. -> {"runs", "results", "contents": {"kept", "removed"}, "bytesRemoved"}.""" + runs_dir = self.root / "runs" + manifests = sorted(p for p in runs_dir.glob("*.json") if p.is_file()) + latest = set() + for p in (runs_dir / "selections").glob("*"): + try: + latest.add(p.read_text(encoding="ascii").strip()) + except OSError: + continue + by_selection: dict[str, list[tuple[int, Path, dict]]] = {} + for p in manifests: + try: + doc = contract.loads(p.read_bytes()) + except (OSError, ValueError): + continue + sel = contract.digest(list(doc.get("selection") or ())) + by_selection.setdefault(sel, []).append((p.stat().st_mtime_ns, p, doc)) + kept, dropped = [], [] + for runs in by_selection.values(): + runs.sort(key=lambda r: r[0], reverse=True) + for i, (_, p, doc) in enumerate(runs): + (kept if keep is None or i < keep or doc.get("id") in latest else dropped).append((p, doc)) + keys: set[str] = set() + shas: set[str] = set() + for _, doc in kept: + for t in doc.get("tasks") or (): + if t.get("key"): + keys.add(t["key"]) + for sha in (doc.get("layouts") or {}).values(): + shas.add(sha) + if self.has(sha): + _collect_shas(contract.loads(self.read(sha)), shas) + for key in keys: + res = self.result(key) + if res is None: + continue + shas.update(a["content"]["sha256"] for a in res.get("artifacts", ())) + shas.update(sha for _, sha in (res.get("keyParts") or {}).get("inputs", ())) + removed_bytes = 0 + out: dict = {"runs": {"kept": len(kept), "removed": len(dropped)}} + for p, _ in dropped: + if not dry_run: + p.unlink(missing_ok=True) + p.with_name(p.name[: -len(".json")] + ".log.jsonl").unlink(missing_ok=True) + n_kept = n_removed = 0 + for p in sorted((self.root / "ac").glob("*/*.json")): + key = p.name[: -len(".json")] + if key in keys: + n_kept += 1 + continue + n_removed += 1 + if not dry_run: + p.unlink(missing_ok=True) + (self.root / "costs" / f"{key}.json").unlink(missing_ok=True) + out["results"] = {"kept": n_kept, "removed": n_removed} + n_kept = n_removed = 0 + for p in sorted((self.root / "cas" / "sha256").glob("*/*")): + if p.name in shas: + n_kept += 1 + continue + n_removed += 1 + removed_bytes += p.stat().st_size + if not dry_run: + _drop(p) + self._have.discard(p.name) + out["contents"] = {"kept": n_kept, "removed": n_removed} + out["bytesRemoved"] = removed_bytes + return out + + +def _collect_shas(doc, out: set[str]) -> None: + """Add every "sha256" value of a document (a layout manifest names its files' contents so).""" + if isinstance(doc, dict): + sha = doc.get("sha256") + if isinstance(sha, str) and contract.is_sha256(sha): + out.add(sha) + for v in doc.values(): + _collect_shas(v, out) + elif isinstance(doc, list): + for v in doc: + _collect_shas(v, out) diff --git a/src/nnnotes/pipeline/layout.py b/src/nnnotes/pipeline/layout.py index 6bc7495..1ef6591 100644 --- a/src/nnnotes/pipeline/layout.py +++ b/src/nnnotes/pipeline/layout.py @@ -39,9 +39,9 @@ place is reported and left out of the manifest (the next run tries it again); no file falls back to another method. Placement (PLACEMENTS; probe() picks one per run for `auto` and checks an explicit one): a clone (reflink: Linux -FICLONE, macOS clonefile) shares the store object's blocks until either file is written, so it is independent and -takes no space; a hard link is the store object itself, read-only like every object of the store (an edit fails -instead of changing the store); a copy is independent and writable, and takes the space again. Every method gives +FICLONE, macOS clonefile) shares the depot object's blocks until either file is written, so it is independent and +takes no space; a hard link is the depot object itself, read-only like every object of the depot (an edit fails +instead of changing the depot); a copy is independent and writable, and takes the space again. Every method gives the same files and the same manifest. """ @@ -57,7 +57,7 @@ from ..core.cache import temp_path from . import contract -from .store import write_doc, write_file +from .depot import write_doc, write_file MANIFEST = "manifest.json" JOURNAL = ".nnnotes-layout-journal.json" @@ -366,9 +366,9 @@ def paths(entries_: list[dict]) -> dict[str, str]: # ---------------------------------------------------------------- writing -def from_store(store): - """A byte source reading the store's cas/.""" - return store.path +def from_depot(depot): + """A byte source reading the depot's cas/.""" + return depot.path def from_layout(directory): @@ -429,7 +429,7 @@ def diff(out_dir, entries_: list[dict]) -> dict: def materialize(out_dir, name: str, params: dict, entries_: list[dict], source, *, link: str = "copy") -> dict: """Write layout `name` into `out_dir` (see the module documentation). `source(sha256) -> Path` gives the bytes - (from_store, from_layout); `link`: the placement (PLACEMENTS). -> {"manifest": the layout document (without + (from_depot, from_layout); `link`: the placement (PLACEMENTS). -> {"manifest": the layout document (without the files not placed), "sha256": of its file, "write", "remove", "keep", "cloned", "linked", "copied", "failed": [{path, message}] of the files not placed}.""" if link not in PLACEMENTS: @@ -449,7 +449,7 @@ def materialize(out_dir, name: str, params: dict, entries_: list[dict], source, stats = {"cloned": 0, "linked": 0, "copied": 0} failed = [] - def restore(p): # a replaced or removed hard link's store object: read-only again + def restore(p): # a replaced or removed hard link's depot object: read-only again if p in owned: try: _read_only(Path(source(owned[p][0]))) @@ -498,7 +498,7 @@ def _place(src: Path, dst: Path, link: str, stats: dict, restore) -> None: if link == "clone": clone(src, tmp) elif link == "hard": - _read_only(src) # an object stored before the store made them read-only + _read_only(src) # an object stored before the depot made them read-only os.link(src, tmp) else: shutil.copyfile(src, tmp) @@ -513,7 +513,7 @@ def _place(src: Path, dst: Path, link: str, stats: dict, restore) -> None: def _read_only(p: Path) -> None: - """Make a store object read-only (its hard links in layouts are the object itself).""" + """Make a depot object read-only (its hard links in layouts are the object itself).""" try: mode = os.stat(p).st_mode if mode & 0o222: @@ -524,7 +524,7 @@ def _read_only(p: Path) -> None: def _writable(p: Path) -> bool: """Clear the read-only attribute of a layout file about to be replaced or removed, where the system refuses - to do either to a read-only file (Windows; the attribute belongs to the file, so a hard link's store object + to do either to a read-only file (Windows; the attribute belongs to the file, so a hard link's depot object loses it too: True, the caller sets it again). Elsewhere nothing: False.""" if os.name != "nt": return False @@ -570,7 +570,7 @@ def clone(src: Path, dst: Path) -> None: def _why(e: OSError, what: str) -> str: if e.errno in _ELSEWHERE: - return "store on another file system" + return "depot on another file system" if "platform" in str(e): return f"{what} unsupported on this platform" code = e.errno if e.errno is None else errno.errorcode.get(e.errno, e.errno) @@ -578,7 +578,7 @@ def _why(e: OSError, what: str) -> str: def probe(out_dir, store_dir, want: str = "auto") -> tuple[str, str]: - """The placement of a layout in `out_dir` whose files come from the store objects in `store_dir`, for `want` + """The placement of a layout in `out_dir` whose files come from the depot objects in `store_dir`, for `want` (LINKS), tried once on a temporary file in each: (placement, reason). auto: a clone where the file system makes them, else a hard link on the same file system, else a copy. An explicit clone or hard link that does not work raises LayoutError.""" @@ -586,10 +586,10 @@ def probe(out_dir, store_dir, want: str = "auto") -> tuple[str, str]: raise ValueError(f"unknown link mode {want!r} (one of {', '.join(LINKS)})") if want == "copy": return "copy", "copy: as asked" - out, store = Path(out_dir), Path(store_dir) + out, depot = Path(out_dir), Path(store_dir) out.mkdir(parents=True, exist_ok=True) - store.mkdir(parents=True, exist_ok=True) - src, made = temp_path(store / "probe"), [] + depot.mkdir(parents=True, exist_ok=True) + src, made = temp_path(depot / "probe"), [] try: with open(src, "xb") as f: f.write(b"nnnotes placement probe\n") @@ -598,7 +598,7 @@ def probe(out_dir, store_dir, want: str = "auto") -> tuple[str, str]: made.append(temp_path(out / "probe")) try: clone(src, made[-1]) - return "clone", "clone: the file system shares the store objects' blocks" + return "clone", "clone: the file system shares the depot objects' blocks" except OSError as e: why = _why(e, "clone") if want == "clone": diff --git a/src/nnnotes/pipeline/link.py b/src/nnnotes/pipeline/link.py index f0222c4..9a6ea97 100644 --- a/src/nnnotes/pipeline/link.py +++ b/src/nnnotes/pipeline/link.py @@ -246,8 +246,8 @@ def _census_input(env, tid: str) -> Input: return env.input_of(tid, "census", as_role="census:" + contract.parse_task_id(tid, census.CensusStage.name)[1]) -def _read(store, task, role: str) -> dict: - return contract.loads(store.input_bytes(task.input(role))) +def _read(depot, task, role: str) -> dict: + return contract.loads(depot.input_bytes(task.input(role))) class ScriptsStage(Stage): @@ -275,9 +275,9 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs) return Cost(0.05 + size / 50e6, (40 << 20) + 8 * size) - def run(self, task, store) -> Output: - doc = scripts(_read(store, task, i.role) for i in task.inputs) - content = store.add(contract.encode(doc), "json") + def run(self, task, depot) -> Output: + doc = scripts(_read(depot, task, i.role) for i in task.inputs) + content = depot.add(contract.encode(doc), "json") rec = contract.artifact( contract.artifact_id(task.id, "scripts"), content, @@ -303,7 +303,7 @@ def subjects(self, env) -> list[str]: def inputs(self, subject: str, env) -> list[Input]: tid = contract.task_id(catalogdb.IndexStage.name, subject) idx = env.input_of(tid, "index") - stables = {b["stable"] for b in env.store.document(idx.sha256)["bundles"]} + stables = {b["stable"] for b in env.depot.document(idx.sha256)["bundles"]} wanted = {contract.task_id(census.CensusStage.name, s) for s in stables} pending = [t for t in env.pending(census.CensusStage.name) if t in wanted] if pending: @@ -314,13 +314,13 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs) return Cost(0.5 + size / 50e6, (200 << 20) + size // 2) - def run(self, task, store) -> Output: - idx = _read(store, task, "index") + def run(self, task, depot) -> Output: + idx = _read(depot, task, "index") docs = ( - (i.role.split(":", 1)[1], _read(store, task, i.role)) for i in task.inputs if i.role.startswith("census:") + (i.role.split(":", 1)[1], _read(depot, task, i.role)) for i in task.inputs if i.role.startswith("census:") ) doc = addresses(idx, docs) - content = store.add(contract.encode(doc), "json") + content = depot.add(contract.encode(doc), "json") facts = { "bundles": len(doc["bundles"]), "keys": len(doc["keys"]), @@ -426,16 +426,16 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: n = len(self._tasks(env)) return Cost(0.05 + 0.002 * n, (60 << 20) + 200_000 * n) - def run(self, task, store) -> Output: + def run(self, task, depot) -> Output: def results(): for tid, key in sorted(task.context.get("results", {}).items()): - doc = store.result(key) + doc = depot.result(key) if doc is None: - raise FileNotFoundError(f"result of {tid} ({key}) is not in the store") + raise FileNotFoundError(f"result of {tid} ({key}) is not in the depot") yield tid, doc objects, contained, arts = artifact_table(results()) - content = store.add_chunks(encode_artifacts(objects, contained, arts), "json") + content = depot.add_chunks(encode_artifacts(objects, contained, arts), "json") facts = {"objects": len(objects), "contained": len(contained), "artifacts": len(arts)} rec = contract.artifact( contract.artifact_id(task.id, "artifacts"), diff --git a/src/nnnotes/pipeline/modelstages.py b/src/nnnotes/pipeline/modelstages.py index b7059dc..1fadad3 100644 --- a/src/nnnotes/pipeline/modelstages.py +++ b/src/nnnotes/pipeline/modelstages.py @@ -163,18 +163,18 @@ def closure_bundles(self, key: str) -> list[str]: class TaskCatalog: """What the model extractors ask of a Catalog (live2d.extract_model through export.Exporter: fetch_key, entry, apk), answered from a task: its key, the internal id of the key's location and the bundles of the key's closure - in load order, each bundle read through the store (Store.open_input: its locators, checked against its content + in load order, each bundle read through the depot (Depot.open_input: its locators, checked against its content id). Another key is not in it (KeyError). `apk` is not None: the extractors need the APK's bundles, and those of the closure are inputs of the task like the others.""" - def __init__(self, key: str, internal_id: str, bundles: list[Input], store): - self.key, self.internal_id, self.bundles, self.store = key, internal_id, list(bundles), store + def __init__(self, key: str, internal_id: str, bundles: list[Input], depot): + self.key, self.internal_id, self.bundles, self.depot = key, internal_id, list(bundles), depot self.apk = APK_FROM_INPUTS @classmethod - def from_task(cls, task, store) -> TaskCatalog: + def from_task(cls, task, depot) -> TaskCatalog: """The catalog of a live2d.model task: its subject, context internalId and inputs "bundle:".""" - return cls(task.subject, task.context["internalId"], _ordered(task), store) + return cls(task.subject, task.context["internalId"], _ordered(task), depot) def _check(self, key: str) -> None: if key != self.key: @@ -203,7 +203,7 @@ def resolve(self, key: str) -> list[Bundle]: def fetch_key(self, key: str) -> list[Path]: """The files of the closure's bundles, in load order.""" self._check(key) - return [self.store.open_input(inp) for inp in self.bundles] + return [self.depot.open_input(inp) for inp in self.bundles] # ---------------------------------------------------------------- live2d.model @@ -267,11 +267,11 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs) return Cost(0.9 + size * 2.2e-7, (420 << 20) + 10 * size) - def run(self, task, store) -> Output: + def run(self, task, depot) -> Output: _check_atoms(task, self.ATOMS) from ..unity import live2d - cat = TaskCatalog.from_task(task, store) + cat = TaskCatalog.from_task(task, depot) arts = [] with tempfile.TemporaryDirectory(prefix="nnnotes-live2d-") as d: # TaskCatalog answers the catalog calls of the model extractor @@ -283,7 +283,7 @@ def run(self, task, store) -> Output: arts.append( contract.artifact( contract.artifact_id(task.id, rel), - store.add(p.read_bytes(), ext), + depot.add(p.read_bytes(), ext), contract.provenance(task), {"kind": kind, "format": ext, "facts": facts}, ) @@ -301,14 +301,14 @@ def skeleton_index(env, selected) -> dict: pending = env.pending(CENSUS) if pending: raise Pending(pending[0]) - table = contract.loads(env.store.read(env.input_of(SCRIPTS_TASK, "scripts").sha256))["scripts"] + table = contract.loads(env.depot.read(env.input_of(SCRIPTS_TASK, "scripts").sha256))["scripts"] scripts = {oid for oid, entry in table.items() if entry.rsplit("|", 1)[-1] == SKELETON_CLASS} files: dict[str, str] = {} externals: dict[str, list[str]] = {} subjects: dict[str, tuple] = {} for tid in env.tasks(CENSUS): stable = contract.parse_task_id(tid, CENSUS)[1] - doc = contract.loads(env.store.read(env.artifact(tid, "census")["content"]["sha256"])) + doc = contract.loads(env.depot.read(env.artifact(tid, "census")["content"]["sha256"])) for f in doc["files"]: files.setdefault(f["name"].casefold(), stable) externals.setdefault(f["name"].casefold(), [x["file"] for x in f["externals"]]) @@ -411,13 +411,13 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs) return Cost(0.27 + size * 1.9e-7, (440 << 20) + 20 * size) - def run(self, task, store) -> Output: + def run(self, task, depot) -> Output: _check_atoms(task, self.ATOMS) from ..content import spot from ..unity import reader as unity oid = task.context["object"] - paths = [store.open_input(i) for i in task.inputs if i.role.startswith("bundle:")] + paths = [depot.open_input(i) for i in task.inputs if i.role.startswith("bundle:")] env = unity.load_closure(paths) file, pid = contract.parse_object_id(oid) written: dict[str, bytes] = {} @@ -446,7 +446,7 @@ def run(self, task, store) -> Output: arts.append( contract.artifact( contract.artifact_id(task.id, name), - store.add(data, ext), + depot.add(data, ext), contract.provenance(task), {"kind": kind, "format": ext, "facts": facts}, ) diff --git a/src/nnnotes/pipeline/objexport.py b/src/nnnotes/pipeline/objexport.py index 2e885bd..a1fbb0d 100644 --- a/src/nnnotes/pipeline/objexport.py +++ b/src/nnnotes/pipeline/objexport.py @@ -309,10 +309,10 @@ class RefExporter(_ExporterBase): code runs here in reference mode: `ref` gives $ref values, `deref` resolves inside the bundle only (a MonoScript of another bundle from the script table), and MonoBehaviour classes come from the task's script table.""" - def __init__(self, view, task, store, fns: dict): + def __init__(self, view, task, depot, fns: dict): # no catalog: the objects come from the task's bundle view super().__init__(None, Path("."), class_of=self._script_name) # pyright: ignore[reportArgumentType] - self.view, self.task, self.store, self.fns = view, task, store, fns + self.view, self.task, self.depot, self.fns = view, task, depot, fns self.blob_min = int(task.params["blobMin"]) self.png_level = int(task.params["png"]["level"]) self.scripts = dict(task.context.get("scripts", {})) @@ -631,11 +631,11 @@ def _emit( refs=(), owner: str | None = None, ) -> tuple[str, dict]: - """Store `data` and stage the artifact "#" (owner: the object `key`, else a task-level name).""" + """Depot `data` and stage the artifact "#" (owner: the object `key`, else a task-level name).""" aid = contract.artifact_id(owner or contract.object_id(*key), role) if aid in self.emitted or any(a["id"] == aid for a in self.staged): raise ValueError(f"artifact {aid} made twice") - content = self.store.add(data, ext) + content = self.depot.add(data, ext) prov = contract.provenance(self.task, atoms=self._atoms(self.conv), obj=None if owner else self._facts(key)) sem = {"kind": kind, "format": fmt, "facts": facts} if issues: @@ -1428,15 +1428,15 @@ def _check_atoms(task, reader: Impl | None = None) -> None: class _Docs: - """A few JSON documents of the store by content id (the planning reads each census several times).""" + """A few JSON documents of the depot by content id (the planning reads each census several times).""" def __init__(self, size: int = 64): self.size, self.docs = size, {} - def get(self, store, sha: str) -> dict: + def get(self, depot, sha: str) -> dict: doc = self.docs.get(sha) if doc is None: - doc = contract.loads(store.read(sha)) + doc = contract.loads(depot.read(sha)) if len(self.docs) >= self.size: self.docs.pop(next(iter(self.docs))) self.docs[sha] = doc @@ -1513,7 +1513,7 @@ def inputs(self, subject: str, env) -> list[Input]: def census(self, subject: str, env) -> dict: inp = env.input_of(contract.task_id(CENSUS, subject), "census") - return self._docs.get(env.store, inp.sha256) + return self._docs.get(env.depot, inp.sha256) def uses(self, subject: str, env) -> dict: """The atoms and converters of the classes the bundle's census lists; with the task's `classes` (env.params @@ -1533,7 +1533,7 @@ def context(self, subject: str, env) -> dict: refs = census_mod.script_refs(self.census(subject, env)) if not refs: return {} - table = self._docs.get(env.store, env.input_of(SCRIPTS_TASK, "scripts").sha256)["scripts"] + table = self._docs.get(env.depot, env.input_of(SCRIPTS_TASK, "scripts").sha256)["scripts"] return {"scripts": link.script_context(table, refs)} def depends(self, subject: str, env) -> list[str]: @@ -1551,12 +1551,12 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: n = res = 0 return Cost(0.12 + size * 3.3e-7 + n * 8.8e-5 + res * 6.6e-8, (80 << 20) + 80 * size + 17500 * n) - def run(self, task, store) -> Output: + def run(self, task, depot) -> Output: reader = self.reader if self.reader is not None else resolve("reader") _check_atoms(task, reader) - view = reader.fn(store.input_bytes(task.input("bundle"))) + view = reader.fn(depot.input_bytes(task.input("bundle"))) fns = {n: resolve(n).fn for n in task.atoms if n in ATOM_NAMES and n != "reader"} - arts, items = exporter_type()(view, task, store, fns).run_bundle() + arts, items = exporter_type()(view, task, depot, fns).run_bundle() return Output(arts, items) @@ -1570,7 +1570,7 @@ def _census_classes(env, subject: str) -> dict: def _store_input(role: str, record: dict) -> Input: c = record["content"] - return Input(role, c["sha256"], c["size"], None, ({"kind": "store"},)) + return Input(role, c["sha256"], c["size"], None, ({"kind": "depot"},)) def _pptr_oid(v) -> str | None: @@ -1674,11 +1674,11 @@ def inputs(self, subject: str, env) -> list[Input]: idx, atlas, reqs = self._subject(subject, env) rec = idx["atlases"][atlas][1] ins = [_store_input("atlas", rec)] - entries = _entries(self._docs.get(env.store, rec["content"]["sha256"])) + entries = _entries(self._docs.get(env.depot, rec["content"]["sha256"])) needed = set() for owner, _tid, meta in reqs: ins.append(_store_input("sprite:" + owner, meta)) - doc = self._docs.get(env.store, meta["content"]["sha256"]) + doc = self._docs.get(env.depot, meta["content"]["sha256"]) entry = _entry(entries, doc["sprite"].get("m_RenderDataKey")) if entry is not None: needed.update(t for t in (_pptr_oid(entry.get("texture")), _pptr_oid(entry.get("alphaTexture"))) if t) @@ -1697,7 +1697,7 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: n = sum(1 for i in inputs if i.role.startswith("sprite:")) return Cost(0.05 + n * 0.01 + tex * 5e-8, (72 << 20) + 40 * tex) - def run(self, task, store) -> Output: + def run(self, task, depot) -> Output: from PIL import Image from .atoms import sprite as sprite_atom @@ -1705,7 +1705,7 @@ def run(self, task, store) -> Output: _check_atoms(task) crop, encode = resolve("sprite.crop").fn, resolve("png.encode").fn level = int(task.params["png"]["level"]) - entries = _entries(contract.loads(store.input_bytes(task.input("atlas")))) + entries = _entries(contract.loads(depot.input_bytes(task.input("atlas")))) images = {} def image(oid: str): @@ -1714,7 +1714,7 @@ def image(oid: str): inp = task.input("texture:" + oid) except KeyError: raise Unsupported("no_data.sprite", f"no image of the atlas texture {oid}") from None - im = Image.open(io.BytesIO(store.input_bytes(inp))) + im = Image.open(io.BytesIO(depot.input_bytes(inp))) im.load() images[oid] = im return images[oid] @@ -1724,14 +1724,14 @@ def image(oid: str): if not inp.role.startswith("sprite:"): continue oid = inp.role[len("sprite:") :] - meta = contract.loads(store.input_bytes(inp)) + meta = contract.loads(depot.input_bytes(inp)) try: img = _crop_meta(meta, entries, image, crop, sprite_atom) aid = contract.artifact_id(oid, "image") arts.append( contract.artifact( aid, - store.add(encode(img, level), "png"), + depot.add(encode(img, level), "png"), contract.provenance(task, obj=meta.get("object")), { "kind": "sprite.image", diff --git a/src/nnnotes/pipeline/orchestrate.py b/src/nnnotes/pipeline/orchestrate.py index b3b7f62..6d54736 100644 --- a/src/nnnotes/pipeline/orchestrate.py +++ b/src/nnnotes/pipeline/orchestrate.py @@ -2,7 +2,7 @@ Stages run in the order given; a stage's tasks are described once the stages before it are done (Env: the facts of the caller plus the keys of the tasks already done), so every input is known when a task is described. Per stage: -a task whose result is in the store is a hit; the others run in parallel, then the next stage starts. +a task whose result is in the depot is a hit; the others run in parallel, then the next stage starts. The pool: up to `workers` spawned processes (no more than a stage has tasks to run), long-lived (each imports what its stages need once), one task at a time each. Tasks start in order of decreasing estimated CPU (longest @@ -39,7 +39,7 @@ from .stages import Env, Pending, Stage, describe, execute if TYPE_CHECKING: - from .store import Store + from .depot import Depot WORKER_BYTES = 70 << 20 # resident memory of an idle worker process (interpreter and imports), per worker # failure codes of tasks (failed items carry the codes of their stages) @@ -101,13 +101,13 @@ def failure(task: str, stage: str, code: str, message: str, inp: str | None = No def _error_code(e: BaseException) -> str: from .contract import IncompatibleTask - from .store import InputMissing, StoreConflict + from .depot import DepotConflict, InputMissing if isinstance(e, IncompatibleTask): return "task.incompatible" if isinstance(e, InputMissing): return "task.input" - if isinstance(e, StoreConflict): + if isinstance(e, DepotConflict): return "task.conflict" if isinstance(e, MemoryError): return "task.memory" @@ -116,14 +116,14 @@ def _error_code(e: BaseException) -> str: return "task.error" -def run_one(task: Task, store, stages: dict, force: bool) -> dict: +def run_one(task: Task, depot, stages: dict, force: bool) -> dict: """Execute one task and measure it: the reply a worker sends (status, result status, cost, or error). The cost's peak memory is the process's, with the resident memory at the task's start (what the process held from its imports and earlier tasks) beside it.""" start = rss() reset_peak() try: - ex = execute(task, store, stages, force=force) + ex = execute(task, depot, stages, force=force) except ConfigError as e: return {"ok": False, "code": "config", "message": str(e), "trace": "", "rss": rss()} except Exception as e: @@ -137,14 +137,14 @@ def run_one(task: Task, store, stages: dict, force: bool) -> dict: reply = {"ok": True, "status": ex.status, "resultStatus": ex.result_status, "rss": rss()} if ex.cost is not None: cost = dict(ex.cost, peakRssBytes=peak_rss(), startRssBytes=start) - store.write_cost(task.key, cost) + depot.write_cost(task.key, cost) reply["cost"] = cost return reply -def _worker_main(conn, store, stages: dict, initializer, initargs) -> None: +def _worker_main(conn, depot, stages: dict, initializer, initargs) -> None: if initializer is not None: - initializer(store, *initargs) + initializer(depot, *initargs) while True: try: msg = conn.recv() @@ -158,16 +158,16 @@ def _worker_main(conn, store, stages: dict, initializer, initargs) -> None: except Exception as e: conn.send({"ok": False, "code": "task.incompatible", "message": str(e), "trace": "", "rss": rss()}) continue - conn.send(run_one(task, store, stages, force)) + conn.send(run_one(task, depot, stages, force)) class _Worker: - def __init__(self, ctx, n: int, store, stages, initializer, initargs): + def __init__(self, ctx, n: int, depot, stages, initializer, initargs): self.n = n self.conn, child = ctx.Pipe() self.proc = ctx.Process( target=_worker_main, - args=(child, store, stages, initializer, initargs), + args=(child, depot, stages, initializer, initargs), name=f"nnnotes-worker-{n}", daemon=True, ) @@ -251,7 +251,7 @@ class Run: The reports (summary, failures, coverage) come from one reading of every result (scan), kept while the tasks stay the same.""" - store: Store + depot: Depot context: dict selection: list params: dict @@ -267,7 +267,7 @@ def results(self): for tid in sorted(self.tasks): t = self.tasks[tid] if t["status"] != "failed": - yield tid, self.store.committed(t["key"], tid) + yield tid, self.depot.committed(t["key"], tid) def scan(self, visit=None, census=None) -> None: """Read every result once, by task id, for the reports; `visit(task id, result)` sees each as well (what @@ -328,7 +328,7 @@ def write(self, layouts: dict | None = None, placement: dict | None = None) -> d """Write the run manifest (layouts: {name: sha256 of its layout manifest}; placement: how their files were made) and this execution's log.""" doc = self.manifest(layouts, placement) - self.store.write_run(doc, self.log) + self.depot.write_run(doc, self.log) return doc def coverage(self, census: dict | None = None) -> dict: @@ -447,12 +447,12 @@ def coverage(results, census: dict | None = None) -> dict: class Orchestrator: - """Runs pipelines of stages into a store (see the module documentation). `initializer(store, *initargs)` runs - first in every worker (settings, caches, the store's fetcher); it must be importable (spawned processes).""" + """Runs pipelines of stages into a depot (see the module documentation). `initializer(depot, *initargs)` runs + first in every worker (settings, caches, the depot's fetcher); it must be importable (spawned processes).""" def __init__( self, - store, + depot, stages: dict[str, Stage], *, workers: int = 0, @@ -464,7 +464,7 @@ def __init__( initargs: tuple = (), log=_log, ): - self.store, self.stages = store, stages + self.depot, self.stages = depot, stages self.workers, self.memory, self.worker_bytes = max(0, int(workers)), memory, int(worker_bytes) self.recycle_tasks, self.recycle_rss = recycle_tasks, recycle_rss self.initializer, self.initargs = initializer, tuple(initargs) @@ -504,7 +504,7 @@ def run( ) -> Run: """Run the stages named in `pipeline`, in order. `params`: {stage: parameters}; `facts`: the planning facts; `context` / `selection`: recorded in the run manifest; `force`: task ids or stage names ("*": all) - run again even when their result is in the store (the new result must be identical).""" + run again even when their result is in the depot (the new result must be identical).""" params = params or {} unknown = [n for n in pipeline if n not in self.stages] if unknown: @@ -514,8 +514,8 @@ def run( if late: raise ValueError(f"stage {n} runs before {', '.join(late)}, which it reads") norm = {n: self.stages[n].normalize(params.get(n)) for n in pipeline} - run = Run(self.store, dict(context or {}), list(selection), norm, {n: self.stages[n].version for n in pipeline}) - env = Env(self.store, facts) + run = Run(self.depot, dict(context or {}), list(selection), norm, {n: self.stages[n].version for n in pipeline}) + env = Env(self.depot, facts) force = set(force) for line in self.explain(): run.log.append({"event": "knob", "value": line}) @@ -564,7 +564,7 @@ def _stage(self, stage: Stage, params: dict, env: Env, force: set, run: Run) -> forced = "*" in force or stage.name in force todo = [] for t in sorted(tasks, key=lambda t: t.id): - if not (forced or t.id in force) and self.store.has_result(t.key): + if not (forced or t.id in force) and self.depot.has_result(t.key): run.tasks[t.id] = {"id": t.id, "key": t.key, "status": "hit"} env.done(t.id, t.key) else: @@ -605,7 +605,7 @@ def _execute(self, tasks: list[Task], force, run: Run) -> dict[str, dict]: if t is None: break self._record(run, t, sched, None) - replies[t.id] = run_one(t, self.store, self.stages, force(t)) + replies[t.id] = run_one(t, self.depot, self.stages, force(t)) self._done(run, t, replies[t.id], None) sched.finished(t.id) if replies[t.id].get("code") == "config": @@ -617,7 +617,7 @@ def _execute(self, tasks: list[Task], force, run: Run) -> dict[str, dict]: def _spawn(self) -> _Worker: self._started += 1 - return _Worker(mp.get_context("spawn"), self._started, self.store, self.stages, self.initializer, self.initargs) + return _Worker(mp.get_context("spawn"), self._started, self.depot, self.stages, self.initializer, self.initargs) def _pool_run(self, sched: Scheduler, force, replies: dict, run: Run) -> None: while len(self._pool) < min(self.workers, len(sched.queue)): # no more processes than tasks diff --git a/src/nnnotes/pipeline/plan.py b/src/nnnotes/pipeline/plan.py index 781edba..1cba6ee 100644 --- a/src/nnnotes/pipeline/plan.py +++ b/src/nnnotes/pipeline/plan.py @@ -1,6 +1,6 @@ """Plan: what a run of a pipeline would do, and why, without running anything. -Every subject of every stage is a node. A node is a `hit` (its result is in the store), `run` (it would run), or +Every subject of every stage is a node. A node is a `hit` (its result is in the depot), `run` (it would run), or `unknown` (it reads the result of a task that would run first, or of one that has not run: its inputs are not known yet). Why a node runs is the difference between its key parts and those of the same task id in a previous run manifest (the latest of the same selection, or one named with --since): @@ -12,9 +12,9 @@ params-changed{path,old,new} an output-affecting parameter (path: /name/sub...) context-changed{entry} a consumed context entry previous-failed the task failed in the previous run - result-missing same key as before, but the result is not in the store + result-missing same key as before, but the result is not in the depot forced asked to run again -Hit reasons: `unchanged` (same key as before), `cached` (the key's result is in the store from another run, e.g. +Hit reasons: `unchanged` (same key as before), `cached` (the key's result is in the depot from another run, e.g. content that changed back). Unknown: `pending{task}`. The plan document (nnnotes.plan/1) lists the nodes in pipeline order with key, status, reasons and cost estimate, @@ -27,9 +27,9 @@ from pathlib import Path from . import contract +from .depot import write_file from .orchestrate import WORKER_BYTES from .stages import Env, Pending, Stage, describe -from .store import write_file def _leaves(old, new, path: str = ""): @@ -138,7 +138,7 @@ def _short(v) -> str: def plan( stages: dict[str, Stage], pipeline: list[str], - store, + depot, *, params: dict | None = None, facts: dict | None = None, @@ -154,7 +154,7 @@ def plan( `memory`, `worker_bytes`: as the orchestrator's, for the peak-memory estimate; `layouts`: {name: {write, remove, keep}} (layout.diff).""" params = params or {} - env = Env(store, facts) + env = Env(depot, facts) force = set(force) prev = {t["id"]: t for t in (previous or {}).get("tasks", [])} nodes, edges, runnable = [], set(), [] @@ -198,13 +198,13 @@ def plan( ) continue old = prev.get(tid) - if not (forced or tid in force) and store.has_result(t.key): + if not (forced or tid in force) and depot.has_result(t.key): env.done(tid, t.key) reasons = [{"code": "unchanged" if old is not None and old.get("key") == t.key else "cached"}] status = "hit" else: env.waiting(tid) - reasons = _run_reasons(t, old, store, forced or tid in force) + reasons = _run_reasons(t, old, depot, forced or tid in force) status = "run" runnable.append(t) nodes.append( @@ -220,7 +220,7 @@ def plan( return Plan(_doc(nodes, edges, runnable, workers, memory, worker_bytes, layouts or {}), runnable) -def _run_reasons(t, old: dict | None, store, forced: bool) -> list[dict]: +def _run_reasons(t, old: dict | None, depot, forced: bool) -> list[dict]: if forced: return [{"code": "forced"}] if old is None: @@ -230,7 +230,7 @@ def _run_reasons(t, old: dict | None, store, forced: bool) -> list[dict]: out.append({"code": "previous-failed"}) if old.get("key") == t.key: return out or [{"code": "result-missing"}] - prior = store.result(old["key"]) if old.get("key") else None + prior = depot.result(old["key"]) if old.get("key") else None if prior is not None: reasons = why(prior["keyParts"], t.key_parts()) or [{"code": "key-changed"}] for r in reasons: diff --git a/src/nnnotes/pipeline/stages.py b/src/nnnotes/pipeline/stages.py index 95ab75f..ced2e4b 100644 --- a/src/nnnotes/pipeline/stages.py +++ b/src/nnnotes/pipeline/stages.py @@ -10,13 +10,13 @@ context(subject, env) the context entries its task consumes (only those) depends(subject, env) the task ids whose results it reads (the plan's edges) estimate(subject, env, inputs) the expected cost (CPU seconds, peak bytes) - run(task, store) the work: reads the inputs through the store, writes the bytes of its artifacts - into the store, returns their records and one item per covered object + run(task, depot) the work: reads the inputs through the depot, writes the bytes of its artifacts + into the depot, returns their records and one item per covered object A stage that needs the output of a task that has not run raises Pending from inputs/uses/context (the environment does it for it: Env.key / Env.result). While a task is described, `env.params` holds its normalized parameters, for answers that depend on them (the atoms of the converters a parameter selects). `describe` turns the answers into a -Task; `execute` runs a Task once and commits its result (the store's action cache) after the artifacts' bytes are +Task; `execute` runs a Task once and commits its result (the depot's action cache) after the artifacts' bytes are stored. The output of `run` must be a function of the task's key parts: a change of what a stage writes is a change of its version (golden tests check). Deterministic per-object failures are items (status `failed`, cached with the result); an exception out of `run` @@ -58,7 +58,7 @@ def __init__(self, task: str, failed: bool = False): @dataclass class Output: - """What Stage.run returns: artifact records (their bytes already in the store) and one item per object.""" + """What Stage.run returns: artifact records (their bytes already in the depot) and one item per object.""" artifacts: list = field(default_factory=list) items: list = field(default_factory=list) @@ -99,7 +99,7 @@ def depends(self, subject: str, env: Env) -> list[str]: def estimate(self, subject: str, env: Env, inputs: list[Input]) -> Cost: return Cost(0.0, sum(i.size for i in inputs)) - def run(self, task: Task, store) -> Output: + def run(self, task: Task, depot) -> Output: raise NotImplementedError @@ -118,11 +118,11 @@ class Env: the tasks of the current plan or run with their keys. `key` / `result` raise Pending for a task that has not run (or failed); a stage lets that propagate.""" - def __init__(self, store, facts: dict | None = None): - self.store = store + def __init__(self, depot, facts: dict | None = None): + self.depot = depot self.facts = dict(facts or {}) self.params: dict = {} # the normalized parameters of the task being described - self._keys: dict[str, str] = {} # task id -> key, for tasks whose result is in the store + self._keys: dict[str, str] = {} # task id -> key, for tasks whose result is in the depot self._open: dict[str, bool] = {} # task id -> failed?, for tasks without a result def fact(self, name: str): @@ -148,7 +148,7 @@ def key(self, tid: str) -> str: raise Pending(tid, self._open.get(tid, False)) def result(self, tid: str) -> dict: - doc = self.store.result(self.key(tid)) + doc = self.depot.result(self.key(tid)) if doc is None: raise Pending(tid) return doc @@ -175,9 +175,9 @@ def artifact(self, tid: str, role: str) -> dict: raise KeyError(f"result of {tid} has no artifact {aid}") def input_of(self, tid: str, role: str, as_role: str | None = None) -> Input: - """A task input made of the artifact "#" (read from the store).""" + """A task input made of the artifact "#" (read from the depot).""" c = self.artifact(tid, role)["content"] - return Input(as_role or role, c["sha256"], c["size"], None, ({"kind": "store"},)) + return Input(as_role or role, c["sha256"], c["size"], None, ({"kind": "depot"},)) def describe(stage: Stage, subject: str, params: dict | None, env: Env) -> Task: @@ -202,7 +202,7 @@ def describe(stage: Stage, subject: str, params: dict | None, env: Env) -> Task: @dataclass class Execution: - """The outcome of execute: `status` hit (the result was in the store) or ran; the measured cost of a run.""" + """The outcome of execute: `status` hit (the result was in the depot) or ran; the measured cost of a run.""" task: str key: str @@ -211,10 +211,10 @@ class Execution: cost: dict | None = None -def execute(task: Task, store, stages: dict[str, Stage], *, force: bool = False) -> Execution: +def execute(task: Task, depot, stages: dict[str, Stage], *, force: bool = False) -> Execution: """Run `task` once: verify that this installation has its stage at its version and can read every input with its content id, run the stage, store its artifacts' bytes, then commit the result (write-once; a result already - in the store is a hit unless `force`, which runs again and requires identical output). Exceptions fail the task + in the depot is a hit unless `force`, which runs again and requires identical output). Exceptions fail the task and leave no result. The measured cost (never part of a key) counts the CPU time of this process and of the child processes the stage ran and waited for (childCpuSeconds; null where the platform does not report it).""" stage = stages.get(task.stage) @@ -226,16 +226,16 @@ def execute(task: Task, store, stages: dict[str, Stage], *, force: bool = False) f"the task was described for version {task.version}" ) if not force: - hit = store.result(task.key) + hit = depot.result(task.key) if hit is not None: return Execution(task.id, task.key, "hit", hit["status"]) for i in task.inputs: - store.open_input(i) + depot.open_input(i) children = children_cpu() cpu, wall = time.process_time(), time.perf_counter() - out = stage.run(task, store) + out = stage.run(task, depot) doc = contract.result(task, out.artifacts, out.items) - store.commit(doc) + depot.commit(doc) own = time.process_time() - cpu now = children_cpu() child = None if children is None or now is None else max(0.0, now - children) @@ -273,17 +273,17 @@ def library_versions(dists) -> dict[str, str]: return out -def golden(stage: Stage, fixtures: dict, store, libraries=()) -> dict: +def golden(stage: Stage, fixtures: dict, depot, libraries=()) -> dict: """The golden record of a stage: its version, the atoms its fixtures use, the versions of `libraries`, and per - fixture the key and the sha256 of the canonical result. `fixtures`: {name: callable(store) -> Task} (each - builds its inputs into the store).""" + fixture the key and the sha256 of the canonical result. `fixtures`: {name: callable(depot) -> Task} (each + builds its inputs into the depot).""" atoms, out = {}, {} for name in sorted(fixtures): - task = fixtures[name](store) + task = fixtures[name](depot) if task.stage != stage.name: raise ValueError(f"fixture {name} is a task of {task.stage}, not {stage.name}") - execute(task, store, {stage.name: stage}, force=True) - doc = store.result(task.key) + execute(task, depot, {stage.name: stage}, force=True) + doc = depot.result(task.key) atoms.update(task.atoms) out[name] = {"key": task.key, "result": contract.sha256(contract.encode(doc))} return { diff --git a/src/nnnotes/pipeline/views.py b/src/nnnotes/pipeline/views.py index 926956d..18c7012 100644 --- a/src/nnnotes/pipeline/views.py +++ b/src/nnnotes/pipeline/views.py @@ -862,14 +862,14 @@ def _address_index(env, tid: str) -> AddressIndex: sha = env.artifact(tid, "addresses")["content"]["sha256"] ix = _index_cache.get(sha) if ix is None: - ix = _index_cache[sha] = AddressIndex.from_addresses(contract.loads(env.store.read(sha))) + ix = _index_cache[sha] = AddressIndex.from_addresses(contract.loads(env.depot.read(sha))) while len(_index_cache) > 2: _index_cache.popitem(last=False) return ix -def _stored(store, role: str, data: bytes) -> Input: - return Input(role, store.put(data), len(data), None, ({"kind": "store"},)) +def _stored(depot, role: str, data: bytes) -> Input: + return Input(role, depot.put(data), len(data), None, ({"kind": "depot"},)) def _rank(aid: str) -> tuple: @@ -953,7 +953,7 @@ def master_inputs(self, env) -> list[Input]: p = mdir / f"{t}.json" if not p.is_file(): raise ViewError(f"view {self.view}: master table {t} is not in the master data") - sha, size = env.store.identify(p) + sha, size = env.depot.identify(p) out.append(Input(MASTER_ROLE + t, sha, size, p.name, ({"kind": "file", "path": str(p)},))) return out @@ -966,27 +966,27 @@ def inputs(self, subject: str, env) -> list[Input]: rec = Recorder(index) build(self.rules, self.view, tables, rec) return masters + [ - _stored(env.store, "addresses", contract.encode(rec.subset())), - _stored(env.store, "rules", self.rules_bytes), + _stored(env.depot, "addresses", contract.encode(rec.subset())), + _stored(env.depot, "rules", self.rules_bytes), ] def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs) return Cost(0.05 + size / 20e6, (48 << 20) + 16 * size) - def run(self, task, store) -> Output: - rules = contract.loads(store.input_bytes(task.input("rules"))) + def run(self, task, depot) -> Output: + rules = contract.loads(depot.input_bytes(task.input("rules"))) check_rules(rules) if list(rules["views"]) != [self.view] or rules["views"][self.view]["version"] != task.version: raise ViewError(f"task {task.id}: its rules are not those of view {self.view} version {task.version}") tables = { - i.role[len(MASTER_ROLE) :]: _cached_rows(i.sha256, functools.partial(store.input_bytes, i)) + i.role[len(MASTER_ROLE) :]: _cached_rows(i.sha256, functools.partial(depot.input_bytes, i)) for i in task.inputs if i.role.startswith(MASTER_ROLE) } - index = AddressIndex.from_subset(contract.loads(store.input_bytes(task.input("addresses")))) + index = AddressIndex.from_subset(contract.loads(depot.input_bytes(task.input("addresses")))) doc = build(rules, self.view, tables, index) - content = store.add(contract.encode(doc), "json") + content = depot.add(contract.encode(doc), "json") entries: dict[str, int] = {} for c in doc["coverage"]["roles"].values(): for s, n in c["counts"].items(): @@ -1005,13 +1005,13 @@ def run(self, task, store) -> Output: ) return Output([rec], []) - def derived(self, task_id: str, result: dict, store, paths) -> list[tuple[str, bytes]]: + def derived(self, task_id: str, result: dict, depot, paths) -> list[tuple[str, bytes]]: """The view document with the layout paths of its objects (with_paths), at view_path.""" aid = contract.artifact_id(task_id, VIEW_ROLE) rec = next((a for a in result["artifacts"] if a["id"] == aid), None) if rec is None: raise ValueError(f"result of {task_id} has no artifact {aid}") - doc = contract.loads(store.read(rec["content"]["sha256"])) + doc = contract.loads(depot.read(rec["content"]["sha256"])) subject = contract.parse_task_id(task_id, self.name)[1] return [(view_path(self.view, subject), contract.encode(with_paths(doc, paths)))] diff --git a/src/nnnotes/pipeline/voices.py b/src/nnnotes/pipeline/voices.py index d0f80e4..52eefa9 100644 --- a/src/nnnotes/pipeline/voices.py +++ b/src/nnnotes/pipeline/voices.py @@ -828,7 +828,7 @@ def _address_parts(env, tid: str) -> tuple: sha = env.artifact(tid, "addresses")["content"]["sha256"] hit = _address_cache.get(sha) if hit is None: - doc = contract.loads(env.store.read(sha)) + doc = contract.loads(env.depot.read(sha)) hit = (views.AddressIndex.from_addresses(doc), dict(doc.get("files") or {})) _address_cache.clear() _address_cache[sha] = hit @@ -925,7 +925,7 @@ def _plan(self, subject: str, env) -> tuple: tuple((t, env.key(t)) for t in tids), tuple((t, env.key(t)) for t in exports), tuple(i.sha256 for i in masters), - id(env.store), + id(env.depot), id(env.facts.get("index")), ) if self._memo is not None and self._memo[0] == sig: @@ -978,21 +978,21 @@ def _episodes(self, env, index, tables, files: dict, exports: set) -> tuple[dict c = a["content"] out.setdefault(asset, {})[part] = {"object": oid, "artifact": aid, "sha256": c["sha256"]} contents.setdefault( - c["sha256"], Input(f"adv:{c['sha256']}", c["sha256"], c["size"], None, ({"kind": "store"},)) + c["sha256"], Input(f"adv:{c['sha256']}", c["sha256"], c["size"], None, ({"kind": "depot"},)) ) read.add(tid) return dict(sorted(out.items())), [contents[k] for k in sorted(contents)], sorted(read) def inputs(self, subject: str, env) -> list[Input]: masters, subset, sheets, acbs, episodes, contents, _ = self._plan(subject, env) - store = env.store + depot = env.depot return ( masters + [ - views._stored(store, "addresses", contract.encode(subset)), - views._stored(store, "rules", self.rules_bytes), - views._stored(store, "sheets", contract.encode(sheets)), - views._stored(store, "episodes", contract.encode(episodes)), + views._stored(depot, "addresses", contract.encode(subset)), + views._stored(depot, "rules", self.rules_bytes), + views._stored(depot, "sheets", contract.encode(sheets)), + views._stored(depot, "episodes", contract.encode(episodes)), ] + acbs + contents @@ -1005,7 +1005,7 @@ def _masters(self, env) -> list[Input]: p = mdir / f"{t}.json" if not p.is_file(): raise views.ViewError(f"view {VIEW}: master table {t} is not in the master data") - sha, size = env.store.identify(p) + sha, size = env.depot.identify(p) out.append(Input(views.MASTER_ROLE + t, sha, size, p.name, ({"kind": "file", "path": str(p)},))) return out @@ -1022,30 +1022,30 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: + 12 * max(story, default=0), ) - def run(self, task, store) -> Output: - rules = contract.loads(store.input_bytes(task.input("rules"))) + def run(self, task, depot) -> Output: + rules = contract.loads(depot.input_bytes(task.input("rules"))) check_rules(rules) if rules[VIEW]["version"] != task.version: raise views.ViewError(f"task {task.id}: its rules are not those of view {VIEW} version {task.version}") masters = [i for i in task.inputs if i.role.startswith(views.MASTER_ROLE)] tables = { - i.role[len(views.MASTER_ROLE) :]: views._cached_rows(i.sha256, functools.partial(store.input_bytes, i)) + i.role[len(views.MASTER_ROLE) :]: views._cached_rows(i.sha256, functools.partial(depot.input_bytes, i)) for i in masters } - index = views.AddressIndex.from_subset(contract.loads(store.input_bytes(task.input("addresses")))) - sheets = contract.loads(store.input_bytes(task.input("sheets"))) + index = views.AddressIndex.from_subset(contract.loads(depot.input_bytes(task.input("addresses")))) + sheets = contract.loads(depot.input_bytes(task.input("sheets"))) results = task.context.get("audio", {}) roles = {i.role for i in task.inputs} cues: dict[str, dict] = {} audio = {} for sheet, s in sorted(sheets.items()): key = results.get(s["task"]) - res = store.result(key) if key is not None else None + res = depot.result(key) if key is not None else None if res is None: raise views.ViewError(f"task {task.id}: no result of {s['task']}") role = f"acb:{s['acb']}" if role in roles and s["acb"] not in cues: - cues[s["acb"]] = acb.cue_streams(store.input_bytes(task.input(role))) + cues[s["acb"]] = acb.cue_streams(depot.input_bytes(task.input(role))) audio[sheet] = sheet_audio(res, cues.get(s["acb"])) snapshot = { "tables": {i.role[len(views.MASTER_ROLE) :]: i.sha256 for i in masters}, @@ -1057,13 +1057,13 @@ def run(self, task, store) -> Output: if "episodes" in roles: snapshot["episodes"] = task.input("episodes").sha256 column = rules[VIEW]["sound"]["sheet"] - for asset, parts in sorted(contract.loads(store.input_bytes(task.input("episodes"))).items()): - got = {p: store.input_bytes(task.input(f"adv:{x['sha256']}")) for p, x in parts.items()} + for asset, parts in sorted(contract.loads(depot.input_bytes(task.input("episodes"))).items()): + got = {p: depot.input_bytes(task.input(f"adv:{x['sha256']}")) for p, x in parts.items()} story[asset] = storyvoices.episode( got.get("script"), got.get("text"), got.get("sound"), got.get("sheets"), sheet_column=column ) doc = build(rules, tables, index, audio, snapshot, story) - content = store.add(contract.encode(doc), "json") + content = depot.add(contract.encode(doc), "json") facts = { "rows": len(doc["rows"]), "entries": counts(doc), @@ -1078,13 +1078,13 @@ def run(self, task, store) -> Output: ) return Output([rec], []) - def derived(self, task_id: str, result: dict, store, paths) -> list[tuple[str, bytes]]: + def derived(self, task_id: str, result: dict, depot, paths) -> list[tuple[str, bytes]]: """The voices document with the layout path of every stream, at views/voices.json.""" aid = contract.artifact_id(task_id, views.VIEW_ROLE) rec = next((a for a in result["artifacts"] if a["id"] == aid), None) if rec is None: raise ValueError(f"result of {task_id} has no artifact {aid}") - doc = contract.loads(store.read(rec["content"]["sha256"])) + doc = contract.loads(depot.read(rec["content"]["sha256"])) subject = contract.parse_task_id(task_id, self.name)[1] return [(views.view_path(VIEW, subject), contract.encode(with_paths(doc, paths)))] diff --git a/src/nnnotes/schemas/catalogs.schema.json b/src/nnnotes/schemas/catalogs.schema.json index 7a05fb7..f37287b 100644 --- a/src/nnnotes/schemas/catalogs.schema.json +++ b/src/nnnotes/schemas/catalogs.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "urn:nnnotes:schema:catalogs", "title": "nnnotes.catalogs/1", - "description": "The catalog versions of a store (/catalogs/index.json), in import order: id (key hash of the two catalogs' content ids), labels, region, language, the remote and APK catalogs (content id, size, build result hash), the CDN's .hash text when it was served, resource version and APK version name.", + "description": "The catalog versions of a depot (/catalogs/index.json), in import order: id (key hash of the two catalogs' content ids), labels, region, language, the remote and APK catalogs (content id, size, build result hash), the CDN's .hash text when it was served, resource version and APK version name.", "type": "object", "required": [ "schema", diff --git a/src/nnnotes/schemas/common.schema.json b/src/nnnotes/schemas/common.schema.json index d11e92a..d36e735 100644 --- a/src/nnnotes/schemas/common.schema.json +++ b/src/nnnotes/schemas/common.schema.json @@ -103,8 +103,8 @@ "properties": { "kind": { "enum": [ - "store", - "cache", + "depot", + "download", "file", "catalog" ] @@ -125,7 +125,7 @@ "properties": { "kind": { "enum": [ - "cache", + "download", "file" ] } diff --git a/src/nnnotes/schemas/music-data.schema.json b/src/nnnotes/schemas/music-data.schema.json index 148c345..b5c1934 100644 --- a/src/nnnotes/schemas/music-data.schema.json +++ b/src/nnnotes/schemas/music-data.schema.json @@ -69,7 +69,7 @@ "string", "null" ], - "description": "the resource version recorded for the catalog in the catalog store, null when none is recorded" + "description": "the resource version recorded for the catalog in the catalog depot, null when none is recorded" }, "resourceHash": { "type": "string", diff --git a/src/nnnotes/schemas/run.schema.json b/src/nnnotes/schemas/run.schema.json index c032502..3e814b7 100644 --- a/src/nnnotes/schemas/run.schema.json +++ b/src/nnnotes/schemas/run.schema.json @@ -171,7 +171,7 @@ } }, "placement": { - "description": "How the layouts' files were made from the store (not part of the id, and not of the layout manifests, which every method writes alike): `clone` (sharing the store objects' blocks), `hard` (hard links to the read-only store objects) or `copy`, with the reason `auto` chose it or `as asked`.", + "description": "How the layouts' files were made from the depot (not part of the id, and not of the layout manifests, which every method writes alike): `clone` (sharing the depot objects' blocks), `hard` (hard links to the read-only depot objects) or `copy`, with the reason `auto` chose it or `as asked`.", "type": "object", "required": [ "method", diff --git a/src/nnnotes/site/sitefiles.py b/src/nnnotes/site/sitefiles.py index 158faae..aded184 100644 --- a/src/nnnotes/site/sitefiles.py +++ b/src/nnnotes/site/sitefiles.py @@ -286,7 +286,7 @@ def _lock_fetches(cat, lock) -> None: f = getattr(cat, name) def locked(x, _f=f, _name=name): - hit = cat.cached(x) if _name == "fetch" else cat.cached_raw(x) + hit = cat.downloaded(x) if _name == "fetch" else cat.downloaded_raw(x) if hit is not None: return hit with lock: diff --git a/src/nnnotes/source/catalog.py b/src/nnnotes/source/catalog.py index 94654b9..cd5979f 100644 --- a/src/nnnotes/source/catalog.py +++ b/src/nnnotes/source/catalog.py @@ -1,8 +1,8 @@ -"""Addressables catalog access: dependency closures, a bundle cache, remote and APK-local bundles. +"""Addressables catalog access: dependency closures, downloaded files, remote and APK-local bundles. Remote bundles are fetched from the region's CDN and decrypted (addressables.decrypt); local bundles (shipped inside the APK) are read from a user-supplied base.apk. Nothing here bundles or redistributes game data: it only reads -what the user points it at into a local cache the user controls. +what the user points it at into a directory of downloads the user controls (the depot's downloads/). """ from __future__ import annotations @@ -44,7 +44,7 @@ T = TypeVar("T") -# a setting given as its value, or as a function that reads it when a file not in the cache needs it +# a setting given as its value, or as a function that reads it when a file not downloaded yet needs it Lazy: TypeAlias = "T | Callable[[], T] | None" @@ -94,8 +94,9 @@ class Catalog: shared prefabs) and the remote catalog (downloadable content). Entry offsets are per-catalog, so APK entries are moved into their own offset namespace. - `cdn`: the region's CDN base (needed only to download what the cache lacks); `bundle_key`: the bundle - decryption key (needed only for bundles not yet in the cache). Either may be a function that returns it, called + `download_dir`: the directory of this catalog's downloaded files. `cdn`: the region's CDN base (needed only to + download what is not downloaded yet); `bundle_key`: the bundle decryption key (needed only for bundles not + downloaded yet). Either may be a function that returns it, called the first time it is needed (a ConfigError it raises is raised naming the file that needed the setting). `apk_catalog`: the stored catalog to replay without opening an APK; JP local reads verify it before extraction. """ @@ -103,7 +104,7 @@ class Catalog: def __init__( self, catalog_bytes: bytes, - cache_dir: Path, + download_dir: Path, *, cdn: Lazy[str] = None, bundle_key: Lazy[BundleKey] = None, @@ -114,8 +115,8 @@ def __init__( resource_version: str | None = None, ): self._cdn, self._bundle_key = cdn, bundle_key - self.cache_dir = Path(cache_dir) - self.cache_dir.mkdir(parents=True, exist_ok=True) + self.download_dir = Path(download_dir) + self.download_dir.mkdir(parents=True, exist_ok=True) self.apk = Path(apk) if apk else None self.source, self.client = source, client self.resource_version = resource_version @@ -138,11 +139,11 @@ def check_apk(self, archive: ApkSet) -> None: if not matches: raise ConfigError("JP historical catalog needs the APK set it was imported with") - def local_cache_dir(self) -> Path: + def local_download_dir(self) -> Path: """JP embedded files also depend on the APK catalog, independently of the CDN snapshot.""" if self.source is not None and "apk" in self._sources: - return self.cache_dir / "apk" / hashlib.sha256(self._sources["apk"]).hexdigest() - return self.cache_dir + return self.download_dir / "apk" / hashlib.sha256(self._sources["apk"]).hexdigest() + return self.download_dir def _parse(self) -> tuple[list[Location], dict[int, Location], dict[str, list[Location]]]: """(entries, by offset, by primary key), parsed the first time a lookup needs them (a command that only @@ -177,35 +178,38 @@ def _by_key(self) -> dict[str, list[Location]]: def load( cls, language: str, - cache_dir: Path, + download_dir: Path, *, cdn: Lazy[str] = None, bundle_key: Lazy[BundleKey] = None, apk: Path | None = None, version: str = "main", ) -> Catalog: - """The remote catalog of `language` from the cache, downloaded from the CDN on first use (`cdn` / - `bundle_key`: as for Catalog). A versioned catalog is isolated by CDN root and version; main retains - its legacy offline cache path. Missing versioned files never fall back to main.""" - cache_dir = Path(cache_dir) - filename = cls.cache_file(language, Path("."), version=version).name + """The remote catalog of `language` from `download_dir`, downloaded from the CDN on first use (`cdn` / + `bundle_key`: as for Catalog). A versioned catalog is isolated by CDN root and version + (international///); main is at the top of `download_dir`. Missing versioned + files never fall back to main.""" + download_dir = Path(download_dir) + filename = cls.file_name(language, version=version) if version != "main": cdn = _setting(cdn, filename) if not cdn: - raise ConfigError("a versioned catalog needs its CDN root to identify the cache") - cache_dir = cache_dir / "international" / hashlib.sha256(cdn.rstrip("/").encode()).hexdigest() / version - cat = cache_dir / filename + raise ConfigError("a versioned catalog needs its CDN root to identify its downloads") + download_dir = ( + download_dir / "international" / hashlib.sha256(cdn.rstrip("/").encode()).hexdigest() / version + ) + cat = download_dir / filename if not cat.exists(): root = _setting(cdn, cat.name) if not root: - raise FileNotFoundError(f"{cat} not cached and no CDN base given") + raise FileNotFoundError(f"{cat} not downloaded and no CDN base given") cat.parent.mkdir(parents=True, exist_ok=True) data = download(root.rstrip("/") + f"/asset/Android/{cat.name}") parse(data) # Never install a corrupt/error response as a reusable catalog. _write_atomic(cat, data) return cls( cat.read_bytes(), - cache_dir, + download_dir, cdn=cdn, bundle_key=bundle_key, apk=apk, @@ -229,11 +233,11 @@ def cdn(self) -> str | None: return None if callable(v) or not v else v.rstrip("/") @staticmethod - def cache_file(language: str, cache_dir: Path, *, version: str = "main") -> Path: - """Where the remote catalog of `language` is cached.""" + def file_name(language: str, *, version: str = "main") -> str: + """The file name of the remote catalog of `language` (on the CDN and in a download directory).""" check_catalog_version(version) check_catalog_version(language) - return Path(cache_dir) / f"catalog_{version}_{language}.bin" + return f"catalog_{version}_{language}.bin" # --- lookup ------------------------------------------------------------ def keys(self, prefix: str = "") -> list[str]: @@ -336,20 +340,20 @@ def resolve(self, key: str) -> list[Bundle]: return list(out.values()) # --- fetch ------------------------------------------------------------- - def cached(self, b: Bundle) -> Path | None: - """The file fetch(b) returns when the bundle is in the cache already, else None.""" - dst = (self.cache_dir if b.remote else self.local_cache_dir()) / "bundles" / b.name + def downloaded(self, b: Bundle) -> Path | None: + """The file fetch(b) returns when the bundle is downloaded already, else None.""" + dst = (self.download_dir if b.remote else self.local_download_dir()) / "bundles" / b.name return dst if dst.is_file() and dst.stat().st_size > 0 else None - def cached_raw(self, e: Location) -> Path | None: - """The file fetch_raw(e) returns when it is in the cache already, else None.""" + def downloaded_raw(self, e: Location) -> Path | None: + """The file fetch_raw(e) returns when it is downloaded already, else None.""" rel = remote_path(e["internal_id"]) - dst = self.cache_dir / "raw" / rel.lstrip("/") if rel is not None else None + dst = self.download_dir / "raw" / rel.lstrip("/") if rel is not None else None return dst if dst is not None and dst.is_file() and dst.stat().st_size > 0 else None def fetch(self, b: Bundle) -> Path: """Local path to the decrypted bundle (CDN download or APK read).""" - dst = (self.cache_dir if b.remote else self.local_cache_dir()) / "bundles" / b.name + dst = (self.download_dir if b.remote else self.local_download_dir()) / "bundles" / b.name dst.parent.mkdir(parents=True, exist_ok=True) if dst.exists() and dst.stat().st_size > 0: return dst @@ -374,7 +378,7 @@ def _url(self, internal_id: str) -> str: name = internal_id.rsplit("/", 1)[-1] cdn = self._cdn_for(name) if not cdn: - raise RuntimeError(f"{name} not cached and no CDN base given") + raise RuntimeError(f"{name} not downloaded and no CDN base given") path = remote_path(internal_id) if path is None: raise ValueError(f"{internal_id}: not a remote location") @@ -411,7 +415,7 @@ def fetch_raw(self, e: Location) -> Path: rel = remote_path(iid) if rel is None: raise ValueError(f"not a CDN asset: {iid}") - dst = self.cache_dir / "raw" / rel.lstrip("/") + dst = self.download_dir / "raw" / rel.lstrip("/") dst.parent.mkdir(parents=True, exist_ok=True) if not (dst.exists() and dst.stat().st_size > 0): _write_atomic(dst, self._download(self._url(iid))) @@ -430,7 +434,7 @@ def apk_bundle(self, name_contains: str) -> Path: if len(names) != 1: raise KeyError(f"{name_contains!r}: {len(names)} APK bundles match") name = names[0].rsplit("/", 1)[1] - dst = self.local_cache_dir() / "bundles" / name + dst = self.local_download_dir() / "bundles" / name if not (dst.exists() and dst.stat().st_size > 0): dst.parent.mkdir(parents=True, exist_ok=True) self.check_apk(z) @@ -491,10 +495,10 @@ def _urlopen(url: str, timeout: float) -> bytes: def _setting(v: Lazy[T], needed_by: str) -> T | None: - """`v`, or what it returns when it is a function; its ConfigError names `needed_by` (what is not cached).""" + """`v`, or what it returns when it is a function; its ConfigError names `needed_by` (what is not downloaded).""" if not callable(v): return cast("T | None", v) try: return cast("T", v()) except ConfigError as e: - raise ConfigError(f"{needed_by} is not in the cache: {e}") from None + raise ConfigError(f"{needed_by} is not downloaded: {e}") from None diff --git a/src/nnnotes/source/jp.py b/src/nnnotes/source/jp.py index 7763ec2..9cf141d 100644 --- a/src/nnnotes/source/jp.py +++ b/src/nnnotes/source/jp.py @@ -153,9 +153,10 @@ def from_dict(cls, data: object) -> Source: raise ValueError("invalid JP source metadata") return cls(**fields) - def cache_dir(self, cache: str | os.PathLike[str]) -> Path: + def download_dir(self, downloads: str | os.PathLike[str]) -> Path: + """The directory of this snapshot's files in `downloads`: jp///.""" site = hashlib.sha256(self.cdn.encode()).hexdigest()[:16] - return Path(cache) / "jp" / site / self.version / self.hash + return Path(downloads) / "jp" / site / self.version / self.hash @dataclass(frozen=True) @@ -303,13 +304,15 @@ def write_source(path: str | os.PathLike[str], data: bytes, source: Source) -> N def open_catalog( cfg: Config, region: str, + downloads: Path, *, catalog_file: str | os.PathLike[str] | None = None, bundle_key: BundleKey | Callable[[], BundleKey] | None = None, apk: Path | None = None, ) -> Catalog: """The catalog of the JP region `region`: `catalog_file` with its .source.json sidecar, else the catalog of the - asset version the API serves now (from the cache, downloaded on first use).""" + asset version the API serves now (from `downloads`, the depot's directory of downloaded files, downloaded on first + use).""" client = Client(cfg, region) if catalog_file is not None: data = Path(catalog_file).read_bytes() @@ -318,7 +321,7 @@ def open_catalog( source = client.observe().source if source is None: raise gameapi.GameApiError("JP Version: no Android asset version applies to this client") - path = source.cache_dir(cfg.require_path("paths", "cache")) / "catalog_main.bin" + path = source.download_dir(downloads) / "catalog_main.bin" if path.exists(): data = path.read_bytes() if not sidecar(path).exists(): @@ -328,7 +331,7 @@ def open_catalog( write_atomic(path, data) write_source(path, data, source) elif read_source(path, data) != source: - raise ConfigError("JP cached catalog source differs from the selected version") + raise ConfigError("JP downloaded catalog source differs from the selected version") else: data = client.get(source.catalog_url, source=source, limit=CATALOG_LIMIT) parse_header(data) @@ -337,7 +340,7 @@ def open_catalog( write_source(path, data, source) return Catalog( data, - source.cache_dir(cfg.require_path("paths", "cache")), + source.download_dir(downloads), bundle_key=bundle_key, apk=apk, source=source, diff --git a/src/nnnotes/source/nnnotes.example.toml b/src/nnnotes/source/nnnotes.example.toml index 8f97483..0221b2b 100644 --- a/src/nnnotes/source/nnnotes.example.toml +++ b/src/nnnotes/source/nnnotes.example.toml @@ -61,11 +61,9 @@ version = "" [paths] # catalog .bin file to read instead of downloading; JP source metadata is stored in .source.json catalog = "" -# cache directory: downloaded catalogs, decrypted bundles, raw CDN files (flag --cache) -cache = "" -# store of the asset export: contents, task results, catalog versions, run manifests; empty: /store -# (flag --store) -store = "" +# depot: downloaded game files, the results of the stages by content, catalog versions, run manifests +# (flag --depot) +depot = "" # decoded master data directory, one
.json per table (`nnnotes master decode`) (flag --master) master = "" # the game's base.apk: APK-local bundles, player settings, CRI HCA keycode (flag --apk) @@ -80,6 +78,6 @@ node = "" player = "" [export] -# how `nnnotes export assets` makes the layout files from the store: auto (a clone where the file system makes them, else -# a hard link to the read-only store object, else a copy), clone, hard or copy; empty: auto (flag --link) +# how `nnnotes export assets` makes the layout files from the depot: auto (a clone where the file system makes them, else +# a hard link to the read-only depot object, else a copy), clone, hard or copy; empty: auto (flag --link) link = "" diff --git a/src/nnnotes/source/session.py b/src/nnnotes/source/session.py index 7ad2c88..aba1841 100644 --- a/src/nnnotes/source/session.py +++ b/src/nnnotes/source/session.py @@ -12,7 +12,7 @@ from . import gameapi from .addressables import BundleKey from .catalog import Catalog -from .config import Config, ConfigError, check_catalog_version, describe +from .config import Config, ConfigError, check_catalog_version from .master import MasterKey __all__ = ["Session"] @@ -49,18 +49,13 @@ def required_path(self, section: str, key: str, kind: str = "file") -> Path: assert p is not None # require_path: the setting is set return p - def store_root(self) -> Path: - """The store directory: `[paths] store`, else the store/ directory of `[paths] cache`.""" - p = self.cfg.path("paths", "store") - if p is not None: - return p - cache = self.cfg.path("paths", "cache") - if cache is not None: - return cache / "store" - raise ConfigError( - "setting paths.store is not set (nor paths.cache, whose store/ directory is the default): " - f"give it as {describe('paths', 'store', '--store')}" - ) + def depot_root(self) -> Path: + """The depot directory: `[paths] depot`, required.""" + return self.cfg.require_path("paths", "depot") + + def downloads(self) -> Path: + """The directory of the depot that holds the downloaded game files (its downloads/ directory).""" + return self.depot_root() / "downloads" def apk(self, region: str | None = None) -> Path: """`[paths] apk` of `region`, required.""" @@ -106,20 +101,20 @@ def catalog(self, bundles: bool = True, region: str | None = None) -> Catalog: """The catalog of `[catalog] language` (merged with the APK's when `[paths] apk` is set), fetching from the CDN of `region`. International versioned catalogs are isolated by CDN root and resource version. `bundles`: bundles will be fetched; else only the catalog is read. An explicit `[paths] catalog` file bypasses - discovery; a version pin bypasses the API but needs the CDN root to identify the cache. The bundle key is + discovery; a version pin bypasses the API but needs the CDN root to identify the downloads. The bundle key is only read for a missing encrypted bundle.""" cfg = self.cfg.for_region(region) if region else self.cfg session = Session(cfg) - cache = cfg.require_path("paths", "cache") + downloads = session.downloads() catbin = session.path("paths", "catalog") apk = session.path("paths", "apk") key = (lambda: session.bundle_key()) if bundles else None if cfg.provider() == "jp": from .jp import open_catalog - return open_catalog(cfg, cfg.region(), catalog_file=catbin, bundle_key=key, apk=apk) + return open_catalog(cfg, cfg.region(), downloads, catalog_file=catbin, bundle_key=key, apk=apk) cdn = (lambda: cfg.cdn(region or cfg.region())) if bundles or catbin is None else None if catbin is not None: - return Catalog(catbin.read_bytes(), cache, cdn=cdn, bundle_key=key, apk=apk) + return Catalog(catbin.read_bytes(), downloads, cdn=cdn, bundle_key=key, apk=apk) language = cfg.require("catalog", "language") - return Catalog.load(language, cache, cdn=cdn, bundle_key=key, apk=apk, version=session.catalog_version()) + return Catalog.load(language, downloads, cdn=cdn, bundle_key=key, apk=apk, version=session.catalog_version()) diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index 0c7089b..8778e07 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -1,4 +1,4 @@ -"""The asset commands on synthetic catalogs: export assets, store plan, store run, catalog versions, store verify. +"""The asset commands on synthetic catalogs: export assets, depot plan, depot run, catalog versions, depot verify. Bundles here are UnityFS-signed files whose body is the census document a real census would find; the census and export stages of the pipeline are replaced by small stages that read them (the registry is patched), the catalog, @@ -19,8 +19,8 @@ from nnnotes.pipeline import contract from nnnotes.pipeline.census import CensusStage from nnnotes.pipeline.contract import Cost +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Output, Stage -from nnnotes.pipeline.store import Store from nnnotes.source.config import ConfigError from tests.helpers import synth @@ -134,27 +134,28 @@ def catalog_bytes(extra=()) -> bytes: return synth.CatalogWriter().build(entries) -def setup_data(tmp_path, docs=None, cached=None, extra=()) -> dict: - """A catalog file and a cache with the (decrypted) bundles; -> the global flags.""" +def setup_data(tmp_path, docs=None, downloaded=None, extra=(), depots=("s",)) -> dict: + """A catalog file and, in the downloads of each depot, the (decrypted) bundles; -> the global flags.""" docs = censuses() if docs is None else docs - cache = tmp_path / "cache" - (cache / "bundles").mkdir(parents=True, exist_ok=True) - for n, doc in docs.items(): - if cached is None or n in cached: - (cache / "bundles" / NAMES[n]).write_bytes(bundle_bytes(doc)) + for depot in depots: + downloads = tmp_path / depot / "downloads" + (downloads / "bundles").mkdir(parents=True, exist_ok=True) + for n, doc in docs.items(): + if downloaded is None or n in downloaded: + (downloads / "bundles" / NAMES[n]).write_bytes(bundle_bytes(doc)) cat = tmp_path / "catalog.bin" cat.write_bytes(catalog_bytes(extra)) - return {"cache": cache, "catalog": cat} + return {"downloads": tmp_path / depots[0] / "downloads", "catalog": cat} # ---------------------------------------------------------------- stand-in stages class FakeCensus(CensusStage): """unity.census over the synthetic bundles (their body is the census).""" - def run(self, task, store): - data = store.input_bytes(task.input("bundle")) + def run(self, task, depot): + data = depot.input_bytes(task.input("bundle")) doc = contract.loads(data[8:]) - content = store.add(contract.encode(doc), "json") + content = depot.add(contract.encode(doc), "json") rec = contract.artifact( contract.artifact_id(task.id, "census"), content, @@ -193,9 +194,9 @@ def inputs(self, subject, env): def estimate(self, subject, env, inputs): return Cost(0.01, 1 << 20) - def run(self, task, store): - doc = contract.loads(store.input_bytes(task.input("census"))) - scripts = contract.loads(store.input_bytes(task.input("scripts")))["scripts"] + def run(self, task, depot): + doc = contract.loads(depot.input_bytes(task.input("census"))) + scripts = contract.loads(depot.input_bytes(task.input("scripts")))["scripts"] if any(o["class"] == "Crash" for f in doc["files"] for o in f["objects"]): raise ConfigError("setting paths.tool is not set") container = {(e["file"], e["pathId"]): e["path"] for e in doc["assetBundles"][0]["container"]} @@ -229,7 +230,7 @@ def run(self, task, store): arts.append( contract.artifact( aid, - store.add(contract.encode(body), "json"), + depot.add(contract.encode(body), "json"), contract.provenance(task, obj=facts), {"kind": "fake.json"}, ) @@ -275,7 +276,7 @@ def nn(capsys, *argv): def flags(d: dict) -> list: - return ["--cache", d["cache"], "--catalog", d["catalog"]] + return ["--catalog", d["catalog"]] def tree(root: Path, skip=()) -> dict: @@ -287,7 +288,7 @@ def tree(root: Path, skip=()) -> dict: return out -def export(capsys, tmp_path, d, store, out, *more): +def export(capsys, tmp_path, d, depot, out, *more): return nn( capsys, *flags(d), @@ -295,8 +296,8 @@ def export(capsys, tmp_path, d, store, out, *more): "assets", "-o", tmp_path / out, - "--store", - tmp_path / store, + "--depot", + tmp_path / depot, "--workers", "0", "--json", @@ -331,8 +332,8 @@ def test_select_adds_the_dependencies_to_the_census(): # ---------------------------------------------------------------- export def test_export_writes_layouts_reports_and_hits_the_second_time(tmp_path, capsys, fake): - d = setup_data(tmp_path) - code, out, err = export(capsys, tmp_path, d, "store", "out") + d = setup_data(tmp_path, depots=("depot",)) + code, out, err = export(capsys, tmp_path, d, "depot", "out") assert code == 1, err # the broken object is a failed item summary = json.loads(out) assert summary["items"] == {"exported": 8, "contained": 4, "generic": 0, "unsupported": 0, "failed": 1} @@ -358,23 +359,23 @@ def test_export_writes_layouts_reports_and_hits_the_second_time(tmp_path, capsys "failed": 0, } run = contract.loads((reports / "run.json").read_bytes()) - assert Store(tmp_path / "store").run(run["id"]) == run + assert Depot(tmp_path / "depot").run(run["id"]) == run assert run["context"]["catalog"]["sha"] == contract.sha256(d["catalog"].read_bytes()) first = files - code, out, _ = export(capsys, tmp_path, d, "store", "out") + code, out, _ = export(capsys, tmp_path, d, "depot", "out") again = json.loads(out) assert code == 1 and again["tasks"]["ran"] == 0 and again["run"] == summary["run"] assert again["layouts"]["original"]["write"] == 0 same = lambda t: {k: v for k, v in t.items() if k != f"{assets.REPORTS}/run.json"} # noqa: E731 assert same(tree(tmp_path / "out")) == same(first) # the run's statuses are hits now code, out, _ = nn( - capsys, *flags(d), "store", "plan", "-o", tmp_path / "out", "--store", tmp_path / "store", "--check" + capsys, *flags(d), "depot", "plan", "-o", tmp_path / "out", "--depot", tmp_path / "depot", "--check" ) assert code == 0, out def test_export_is_the_same_with_worker_processes(tmp_path, capsys, fake): - d = setup_data(tmp_path) + d = setup_data(tmp_path, depots=("s0", "s2")) export(capsys, tmp_path, d, "s0", "o0") code, _, err = nn( capsys, @@ -383,7 +384,7 @@ def test_export_is_the_same_with_worker_processes(tmp_path, capsys, fake): "assets", "-o", tmp_path / "o2", - "--store", + "--depot", tmp_path / "s2", "--workers", "2", @@ -391,37 +392,38 @@ def test_export_is_the_same_with_worker_processes(tmp_path, capsys, fake): "original,cas", ) assert code == 1, err - assert tree(tmp_path / "s0", ["inputs", "runs", "costs"]) == tree(tmp_path / "s2", ["inputs", "runs", "costs"]) + skip = ["inputs", "runs", "costs", "downloads"] + assert tree(tmp_path / "s0", skip) == tree(tmp_path / "s2", skip) o0, o2 = tree(tmp_path / "o0"), tree(tmp_path / "o2" / "original") assert {k: v for k, v in o0.items() if not k.startswith(assets.REPORTS)} == o2 cas = tree(tmp_path / "o2" / "cas") assert all(p.startswith("assets/") or p == "manifest.json" for p in cas) -def test_plan_emit_tasks_and_run_stage_give_the_exported_store(tmp_path, capsys, fake): - d = setup_data(tmp_path) +def test_plan_emit_tasks_and_run_stage_give_the_exported_depot(tmp_path, capsys, fake): + d = setup_data(tmp_path, depots=("orch", "q")) export(capsys, tmp_path, d, "orch", "o") rounds = 0 while True: tasks = tmp_path / f"round{rounds}" code, out, err = nn( - capsys, *flags(d), "store", "plan", "--store", tmp_path / "q", "--emit-tasks", tasks, "--json" + capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "q", "--emit-tasks", tasks, "--json" ) assert code == 0, err doc = json.loads(out) if not any(n["status"] == "run" for n in doc["nodes"]): break - code, out, err = nn(capsys, *flags(d), "store", "run", tasks, "--store", tmp_path / "q") + code, out, err = nn(capsys, *flags(d), "depot", "run", tasks, "--depot", tmp_path / "q") assert code in (0, 1), err rounds += 1 assert rounds == 4 # index + census; scripts + addresses; export; artifacts - skip = ["inputs", "runs", "costs", "catalogs"] + skip = ["inputs", "runs", "costs", "catalogs", "downloads"] assert tree(tmp_path / "orch", skip) == tree(tmp_path / "q", skip) def test_plan_explains_and_waits_for_bundles_not_fetched(tmp_path, capsys, fake): - d = setup_data(tmp_path, cached={"card_1", "shared", "bg_x"}) - code, out, _ = nn(capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--json", "--check") + d = setup_data(tmp_path, downloaded={"card_1", "shared", "bg_x"}) + code, out, _ = nn(capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "s", "--json", "--check") assert code == 1 nodes = {n["id"]: n for n in json.loads(out)["nodes"]} card2 = contract.stable_bundle_name(NAMES["card_2"]) @@ -429,21 +431,21 @@ def test_plan_explains_and_waits_for_bundles_not_fetched(tmp_path, capsys, fake) assert nodes[f"unity.census:{card2}"]["reasons"] == [{"code": "pending", "task": f"fetch:{NAMES['card_2']}"}] assert nodes["link.scripts:all"]["status"] == "unknown" assert nodes["catalog.index:main"]["reasons"] == [{"code": "new-input"}] - code, out, _ = nn(capsys, *flags(d), "store", "plan", "--store", tmp_path / "s") + code, out, _ = nn(capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "s") assert "unknown" in out and "tasks:" in out def test_parameters_change_keys_and_are_explained(tmp_path, capsys, fake): d = setup_data(tmp_path) export(capsys, tmp_path, d, "s", "o") - base = [*flags(d), "store", "plan", "--store", tmp_path / "s", "--json"] + base = [*flags(d), "depot", "plan", "--depot", tmp_path / "s", "--json"] code, out, _ = nn(capsys, *base, "--png-level", "3") runs = [n for n in json.loads(out)["nodes"] if n["status"] == "run"] assert {n["stage"] for n in runs} == {"unity.export"} assert runs[0]["reasons"] == [{"code": "params-changed", "path": "/png/level", "old": 6, "new": 3}] tid = runs[0]["id"] code, out, _ = nn( - capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--png-level", "3", "--why", tid, "--json" + capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "s", "--png-level", "3", "--why", tid, "--json" ) w = json.loads(out) assert w["keyParts"]["old"]["params"]["png"]["level"] == 6 and w["keyParts"]["new"]["params"]["png"]["level"] == 3 @@ -465,7 +467,7 @@ def test_usage_errors(tmp_path, capsys, fake, monkeypatch): code, _, err = export(capsys, tmp_path, d, "s", "o", "--flac-level", "5") assert code == 2 and "no stage of the pipeline encodes FLAC" in err code, _, err = nn(capsys, "export", "assets", "-o", tmp_path / "o") - assert code == 2 and "paths.store" in err # no store and no cache + assert code == 2 and "paths.depot" in err # no depot def test_the_flac_level_is_a_parameter_of_the_stages_that_encode_flac(tmp_path, capsys, fake, monkeypatch): @@ -473,7 +475,7 @@ def test_the_flac_level_is_a_parameter_of_the_stages_that_encode_flac(tmp_path, d = setup_data(tmp_path) assert export(capsys, tmp_path, d, "s", "o", "--flac-level", "13")[0] == 2 export(capsys, tmp_path, d, "s", "o") - base = [*flags(d), "store", "plan", "--store", tmp_path / "s", "--flac-level", "5"] + base = [*flags(d), "depot", "plan", "--depot", tmp_path / "s", "--flac-level", "5"] code, out, _ = nn(capsys, *base, "--json") tid = next(n["id"] for n in json.loads(out)["nodes"] if n["stage"] == "unity.export") code, out, _ = nn(capsys, *base, "--why", tid, "--json") @@ -550,7 +552,7 @@ def test_export_dry_run_and_explain(tmp_path, capsys, fake): # ---------------------------------------------------------------- run-stage def test_run_stage_refuses_incompatible_tasks(tmp_path, capsys, fake): d = setup_data(tmp_path) - nn(capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--emit-tasks", tmp_path / "t") + nn(capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "s", "--emit-tasks", tmp_path / "t") (f,) = [ p for p in sorted((tmp_path / "t").glob("*.json")) @@ -559,7 +561,7 @@ def test_run_stage_refuses_incompatible_tasks(tmp_path, capsys, fake): doc = contract.loads(f.read_bytes()) edited = tmp_path / "edited.json" edited.write_bytes(contract.encode(dict(doc, params={"x": 1}))) - code, _, err = nn(capsys, "store", "run", f, edited, "--store", tmp_path / "s") + code, _, err = nn(capsys, "depot", "run", f, edited, "--depot", tmp_path / "s") assert code == 4 and "incompatible task" in err assert not (tmp_path / "s" / "ac").exists() # nothing of the batch ran other = tmp_path / "other.json" @@ -567,8 +569,8 @@ def test_run_stage_refuses_incompatible_tasks(tmp_path, capsys, fake): other.write_bytes( contract.encode(contract.Task("catalog.index", 9, t.subject, t.params, t.atoms, t.inputs, t.context).to_json()) ) - assert nn(capsys, "store", "run", other, "--store", tmp_path / "s")[0] == 4 - code, out, _ = nn(capsys, "store", "run", f, "--store", tmp_path / "s", "--json") + assert nn(capsys, "depot", "run", other, "--depot", tmp_path / "s")[0] == 4 + code, out, _ = nn(capsys, "depot", "run", f, "--depot", tmp_path / "s", "--json") assert code == 0 and json.loads(out)["summary"] == {"tasks": 1, "hit": 0, "ran": 1, "failed": 0} @@ -585,19 +587,19 @@ def main(argv): cli.main(argv) except SystemExit as e: assert not e.code, e.code -main(args + ["store", "plan", "--store", {str(tmp_path / "s")!r}, "--emit-tasks", {str(tmp_path / "t")!r}]) +main(args + ["depot", "plan", "--depot", {str(tmp_path / "s")!r}, "--emit-tasks", {str(tmp_path / "t")!r}]) index = [str(p) for p in Path({str(tmp_path / "t")!r}).glob("*.json") if contract.loads(p.read_bytes())["stage"]["name"] == "catalog.index"] assert len(index) == 1, index -main(["store", "run", *index, "--store", {str(tmp_path / "s")!r}]) -main(args + ["store", "plan", "--store", {str(tmp_path / "s")!r}, "--json"]) +main(["depot", "run", *index, "--depot", {str(tmp_path / "s")!r}]) +main(args + ["depot", "plan", "--depot", {str(tmp_path / "s")!r}, "--json"]) assert "UnityPy" not in sys.modules, "UnityPy imported" """ r = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, cwd=tmp_path) assert r.returncode == 0, r.stderr[-3000:] -# ---------------------------------------------------------------- catalogs and store +# ---------------------------------------------------------------- catalogs and depot def test_catalog_versions_import_list_and_diff(tmp_path, capsys): a = tmp_path / "a.bin" a.write_bytes(catalog_bytes()) @@ -608,7 +610,7 @@ def test_catalog_versions_import_list_and_diff(tmp_path, capsys): ] b = tmp_path / "b.bin" b.write_bytes(synth.CatalogWriter().build(entries)) - s = ["--store", tmp_path / "s"] + s = ["--depot", tmp_path / "s"] assert nn(capsys, "catalog", "import", a, "--label", "v1", *s)[0] == 0 code, out, _ = nn(capsys, "catalog", "import", b, *s) assert code == 0 and contract.sha256(b.read_bytes())[:12] in out @@ -625,7 +627,7 @@ def test_catalog_versions_import_list_and_diff(tmp_path, capsys): def test_catalog_import_takes_the_resource_version_from_the_file_name(tmp_path, capsys): - s = ["--store", tmp_path / "s"] + s = ["--depot", tmp_path / "s"] named = tmp_path / "catalog_1.0.0.201_en.bin" named.write_bytes(catalog_bytes()) main = tmp_path / "catalog_main_en.bin" @@ -650,31 +652,31 @@ def test_catalog_import_resource_version_option(): catalog_cli.import_resource_version("../x", file, None) -def test_store_verify(tmp_path, capsys, fake): +def test_depot_verify(tmp_path, capsys, fake): d = setup_data(tmp_path) export(capsys, tmp_path, d, "s", "o") - code, out, _ = nn(capsys, "store", "verify", "--store", tmp_path / "s", "--json") + code, out, _ = nn(capsys, "depot", "verify", "--depot", tmp_path / "s", "--json") r = json.loads(out) assert code == 0 and r["problemCount"] == 0 and r["contents"] > 0 and r["results"] > 0 victim = next(p for p in sorted((tmp_path / "s" / "cas").rglob("*")) if p.is_file()) - assert os.stat(victim).st_mode & 0o222 == 0 # store objects are read-only + assert os.stat(victim).st_mode & 0o222 == 0 # depot objects are read-only os.chmod(victim, 0o666) victim.write_bytes(victim.read_bytes() + b"x") - code, out, _ = nn(capsys, "store", "verify", "--store", tmp_path / "s", "--json") + code, out, _ = nn(capsys, "depot", "verify", "--depot", tmp_path / "s", "--json") problems = {p["problem"] for p in json.loads(out)["problems"]} assert code == 1 and "content differs from its id" in problems assert any(p.endswith("bytes of another size") for p in problems) - code, out, _ = nn(capsys, "store", "verify", "--quick", "--store", tmp_path / "s", "--json") + code, out, _ = nn(capsys, "depot", "verify", "--quick", "--depot", tmp_path / "s", "--json") assert code == 1 and all("another size" in p["problem"] for p in json.loads(out)["problems"]) # ---------------------------------------------------------------- the command line def test_the_commands_are_part_of_nnnotes(tmp_path, capsys, fake): try: - cli.build_parser().parse_args(["store", "verify"]) + cli.build_parser().parse_args(["depot", "verify"]) except SystemExit: pytest.skip("the command line does not register the asset commands") - assert ("paths", "store") in common.FLAG_SETTINGS + assert ("paths", "depot") in common.FLAG_SETTINGS d = setup_data(tmp_path) with pytest.raises(SystemExit) as e: cli.main( @@ -684,7 +686,7 @@ def test_the_commands_are_part_of_nnnotes(tmp_path, capsys, fake): "assets", "-o", str(tmp_path / "o"), - "--store", + "--depot", str(tmp_path / "s"), "--workers", "0", @@ -694,7 +696,7 @@ def test_the_commands_are_part_of_nnnotes(tmp_path, capsys, fake): capsys.readouterr() with pytest.raises(SystemExit) as e: cli.main( - [*map(str, flags(d)), "store", "plan", "-o", str(tmp_path / "o"), "--store", str(tmp_path / "s"), "--check"] + [*map(str, flags(d)), "depot", "plan", "-o", str(tmp_path / "o"), "--depot", str(tmp_path / "s"), "--check"] ) assert e.value.code == 0 @@ -721,21 +723,21 @@ def inputs(self, subject, env): def estimate(self, subject, env, inputs): return Cost(0.01, 1 << 20) - def run(self, task, store): + def run(self, task, depot): oids = ["CAB-none:9"] for inp in task.inputs: - for f in contract.loads(store.input_bytes(inp))["files"]: + for f in contract.loads(depot.input_bytes(inp))["files"]: oids += [contract.object_id(f["name"], o["pathId"]) for o in f["objects"]] rec = contract.artifact( contract.artifact_id(task.id, "objects"), - store.add(contract.encode(sorted(oids)), "json"), + depot.add(contract.encode(sorted(oids)), "json"), contract.provenance(task), {"kind": "fake.objects"}, ) return Output([rec], []) - def derived(self, task_id, result, store, paths): - oids = contract.loads(store.read(result["artifacts"][0]["content"]["sha256"])) + def derived(self, task_id, result, depot, paths): + oids = contract.loads(depot.read(result["artifacts"][0]["content"]["sha256"])) doc = {o: {a: paths.artifact(a) for a in paths.objects(o)} for o in oids} return [("derived/objects.json", contract.encode(doc))] @@ -755,7 +757,7 @@ def test_derived_documents_read_the_paths_of_the_objects_they_name(tmp_path, cap assert doc[card] == {f"{card}#json": "Assets/Game/Card/1/full.png.json"} assert doc[f"{CABS['card_1']}:1"] == doc[card] # contained in it assert (tmp_path / "o" / "Assets" / "Game" / "Card" / "1" / "full.png.json").is_file() - code, out, _ = nn(capsys, *flags(d), "store", "plan", "-o", tmp_path / "o", "--store", tmp_path / "s", "--check") + code, out, _ = nn(capsys, *flags(d), "depot", "plan", "-o", tmp_path / "o", "--depot", tmp_path / "s", "--check") assert code == 0, out @@ -782,7 +784,7 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps "assets", "-o", tmp_path / "o", - "--store", + "--depot", tmp_path / "s", "--workers", "0", @@ -803,16 +805,16 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps assert not any(p.startswith("_tasks/view.") for p in placed) # placed through the derived documents doc = contract.loads((tmp_path / "o" / "original" / "views" / "cards.json").read_bytes()) assert doc["layout"] == "original" and doc["view"] == "cards" - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") run = contract.loads((tmp_path / "o" / assets.REPORTS / "run.json").read_bytes()) v = validators() checked = set() for t in run["tasks"]: - res = store.result(t["key"]) + res = depot.result(t["key"]) for a in res["artifacts"]: if a["content"]["ext"] != "json": continue - body = contract.loads(store.read(a["content"]["sha256"])) + body = contract.loads(depot.read(a["content"]["sha256"])) name = body.get("schema") if isinstance(body, dict) else None if name in v: errors = [e.message for e in v[name].iter_errors(body)] @@ -831,9 +833,9 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps *flags(d), "--master", master, - "store", + "depot", "plan", - "--store", + "--depot", tmp_path / "s", "--json", "--views", @@ -856,7 +858,7 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps errors = [e.message for e in v[name].iter_errors(body)] assert errors == [], (name, errors[:3]) vid = run["context"]["catalog"]["id"] - code, out, _ = nn(capsys, "catalog", "diff", vid[:12], vid[:12], "--json", "--store", tmp_path / "s") + code, out, _ = nn(capsys, "catalog", "diff", vid[:12], vid[:12], "--json", "--depot", tmp_path / "s") diff = json.loads(out) assert diff["objects"]["summary"]["unchanged"] > 0 # the address tables of the runs assert [e.message for e in v["nnnotes.catalog-diff/1"].iter_errors(diff)] == [] @@ -927,12 +929,12 @@ def test_export_records_the_calibration_and_plan_uses_it(tmp_path, capsys, fake) assert contract.loads(cal.read_bytes()) == {"stages": {}} # four bundles: too few assets.write_calibration(tmp_path / "s", {"unity.export": {"cpu": 2.0, "peak": 3.0, "samples": 50}}) code, out, err = nn( - capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--json", "--png-level", "1", "--explain" + capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "s", "--json", "--png-level", "1", "--explain" ) assert "cost calibration unity.export: CPU x2.0, peak memory x3.0 (50 measured tasks)" in err runs = [n for n in json.loads(out)["nodes"] if n["stage"] == "unity.export"] assert runs and all(n["cost"] == {"cpuSeconds": 0.02, "peakBytes": 3 << 20} for n in runs) - code, out, _ = nn(capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--check") + code, out, _ = nn(capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "s", "--check") assert code == 0 # keys do not change @@ -958,10 +960,10 @@ def subjects(self, env): def names(self, env): return {"s1": ["Cri/Sound/y", "Cri/Sound/x"]} - def run(self, task, store): + def run(self, task, depot): arts = [ contract.artifact( - contract.artifact_id(task.id, role), store.add(data, ext), contract.provenance(task), {"kind": "toy"} + contract.artifact_id(task.id, role), depot.add(data, ext), contract.provenance(task), {"kind": "toy"} ) for role, data, ext in (("a.flac", b"fLaC" + task.subject.encode(), "flac"), ("cues.json", b"[]", "json")) ] @@ -982,7 +984,7 @@ def uses(self, subject, env): cfg = active() return {"tool": str(cfg.path("paths", "ffmpeg")) if cfg is not None else "unset"} - def run(self, task, store): + def run(self, task, depot): return Output([], []) @@ -1023,7 +1025,7 @@ def test_plan_and_export_describe_with_the_same_settings(tmp_path, capsys, monke d = setup_data(tmp_path, censuses(broken=False)) tool = tmp_path / "ffmpeg-bin" code, _, err = nn( - capsys, *flags(d), "--ffmpeg", tool, "store", "plan", "--store", tmp_path / "s", "--emit-tasks", tmp_path / "t" + capsys, *flags(d), "--ffmpeg", tool, "depot", "plan", "--depot", tmp_path / "s", "--emit-tasks", tmp_path / "t" ) assert code == 0, err (planned,) = [ @@ -1041,7 +1043,7 @@ def test_plan_and_export_describe_with_the_same_settings(tmp_path, capsys, monke "assets", "-o", tmp_path / "o", - "--store", + "--depot", tmp_path / "s", "--workers", "0", @@ -1061,14 +1063,14 @@ def test_plan_knows_the_files_the_apk_does_not_have(tmp_path, capsys, fake, monk ) d = setup_data(tmp_path, censuses(broken=False), extra=[("apkonly.bundle", synth.local("apkonly.bundle"), [])]) monkeypatch.setattr("nnnotes.pipeline.catalogdb.apk_version", lambda apk: None) - code, out, err = nn(capsys, *flags(d), "--apk", apk, "store", "plan", "--store", tmp_path / "s", "--json") + code, out, err = nn(capsys, *flags(d), "--apk", apk, "depot", "plan", "--depot", tmp_path / "s", "--json") nodes = {n["id"]: n for n in json.loads(out)["nodes"]} assert "unity.census:apkonly" not in nodes # not pending on a fetch that cannot happen code, out, err = nn( - capsys, *flags(d), "--apk", apk, "store", "plan", "--store", tmp_path / "s", "--emit-tasks", tmp_path / "t" + capsys, *flags(d), "--apk", apk, "depot", "plan", "--depot", tmp_path / "s", "--emit-tasks", tmp_path / "t" ) - assert nn(capsys, *flags(d), "store", "run", tmp_path / "t", "--store", tmp_path / "s")[0] == 0 - code, out, _ = nn(capsys, *flags(d), "--apk", apk, "store", "plan", "--store", tmp_path / "s", "--json") + assert nn(capsys, *flags(d), "depot", "run", tmp_path / "t", "--depot", tmp_path / "s")[0] == 0 + code, out, _ = nn(capsys, *flags(d), "--apk", apk, "depot", "plan", "--depot", tmp_path / "s", "--json") nodes = {n["id"]: n for n in json.loads(out)["nodes"]} assert nodes["link.scripts:all"]["status"] == "run" @@ -1076,7 +1078,7 @@ def test_plan_knows_the_files_the_apk_does_not_have(tmp_path, capsys, fake, monk def test_plan_census_takes_the_censuses_first(tmp_path, capsys, fake): d = setup_data(tmp_path, censuses(broken=False)) code, out, err = nn( - capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--census", "--workers", "0", "--json" + capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "s", "--census", "--workers", "0", "--json" ) assert code == 0, err nodes = json.loads(out)["nodes"] @@ -1105,7 +1107,7 @@ def test_names_see_the_tasks_that_ran(tmp_path, capsys, monkeypatch): assert code == 0, err placed = tree(tmp_path / "o") assert "Named/4/a.flac" in placed and "Named/0/a.flac" not in placed # four bundles were exported - code, out, _ = nn(capsys, *flags(d), "store", "plan", "-o", tmp_path / "o", "--store", tmp_path / "s", "--check") + code, out, _ = nn(capsys, *flags(d), "depot", "plan", "-o", tmp_path / "o", "--depot", tmp_path / "s", "--check") assert code == 0, out # plan places them the same way @@ -1151,8 +1153,8 @@ def subjects(self, env): def inputs(self, subject, env): return [env.fact("raw")[subject]] - def run(self, task, store): - content = store.add(store.input_bytes(task.input("raw")), "acb") + def run(self, task, depot): + content = depot.add(depot.input_bytes(task.input("raw")), "acb") return Output( [ contract.artifact( @@ -1192,7 +1194,7 @@ def raw_export(capsys, tmp_path, d, apk): "assets", "-o", tmp_path / "o", - "--store", + "--depot", tmp_path / "s", "--workers", "0", @@ -1202,7 +1204,7 @@ def raw_export(capsys, tmp_path, d, apk): def test_raw_files_of_the_apk_are_read_from_it(tmp_path, capsys, monkeypatch): apk, name = raw_apk(tmp_path, monkeypatch) d = setup_data(tmp_path, censuses(broken=False)) - code, out, err = nn(capsys, *flags(d), "--apk", apk, "store", "plan", "--store", tmp_path / "s", "--json") + code, out, err = nn(capsys, *flags(d), "--apk", apk, "depot", "plan", "--depot", tmp_path / "s", "--json") nodes = {n["id"]: n for n in json.loads(out)["nodes"]} assert nodes["cri.audio:cri_assets_embcri/sound/initialse"]["reasons"] == [ {"code": "pending", "task": f"fetch:{name}"} @@ -1216,7 +1218,7 @@ def test_raw_files_of_the_apk_are_read_from_it(tmp_path, capsys, monkeypatch): "assets", "-o", tmp_path / "o", - "--store", + "--depot", tmp_path / "s", "--workers", "0", @@ -1224,13 +1226,13 @@ def test_raw_files_of_the_apk_are_read_from_it(tmp_path, capsys, monkeypatch): ) assert code == 0, err assert json.loads(out)["sources"] == {} - assert (d["cache"] / "raw" / "Android" / name).read_bytes() == b"@UTF sheet" # where a CDN file would be + assert (d["downloads"] / "raw" / "Android" / name).read_bytes() == b"@UTF sheet" # where a CDN file would be run = contract.loads((tmp_path / "o" / assets.REPORTS / "run.json").read_bytes()) assert {t["id"]: t["status"] for t in run["tasks"]}["cri.audio:cri_assets_embcri/sound/initialse"] == "ran" code, out, _ = nn( - capsys, *flags(d), "--apk", apk, "store", "plan", "-o", tmp_path / "o", "--store", tmp_path / "s", "--check" + capsys, *flags(d), "--apk", apk, "depot", "plan", "-o", tmp_path / "o", "--depot", tmp_path / "s", "--check" ) - assert code == 0, out # read from the cache now + assert code == 0, out # read from the downloads now def test_the_raw_files_are_fetched_while_the_bundle_stages_run(tmp_path, capsys, monkeypatch): @@ -1246,11 +1248,11 @@ def inputs(self, kind, entries, fetch): order.append(("raw fetch", threading.current_thread() is threading.main_thread())) return real_inputs(self, kind, entries, fetch) - def run(self, task, store): + def run(self, task, depot): if not exported.is_set(): order.append("unity.export") exported.set() - return real_run(self, task, store) + return real_run(self, task, depot) monkeypatch.setattr(assets.Workspace, "_inputs", inputs) monkeypatch.setattr(FakeExport, "run", run) @@ -1261,7 +1263,7 @@ def run(self, task, store): assert "fetched while the bundle stages run" in err and "0 raw files not readable" in err run_doc = contract.loads((tmp_path / "o" / assets.REPORTS / "run.json").read_bytes()) assert {t["id"]: t["status"] for t in run_doc["tasks"]}["cri.audio:cri_assets_embcri/sound/initialse"] == "ran" - assert (d["cache"] / "raw" / "Android" / name).read_bytes() == b"@UTF sheet" + assert (d["downloads"] / "raw" / "Android" / name).read_bytes() == b"@UTF sheet" def test_a_setting_the_raw_fetch_lacks_aborts_the_run(tmp_path, capsys, monkeypatch): @@ -1309,7 +1311,7 @@ def interrupted(*a, **kw): code, _, err = raw_export(capsys, tmp_path, d, apk) assert code == 3 and "interrupted" in err assert ended.wait(10) - assert not (d["cache"] / "raw" / "Android" / name).exists() # not fetched after the interrupt + assert not (d["downloads"] / "raw" / "Android" / name).exists() # not fetched after the interrupt def test_worker_counts_follow_the_cpus_this_process_may_use(monkeypatch): diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 36cb2cf..0f85f43 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -64,7 +64,7 @@ def test_version_and_help(capsys): "master", "export", "apk", - "store", + "depot", "voices", "site", "music-data", @@ -133,8 +133,8 @@ def test_catalog_listing(catalog_file, tmp_path, capsys): [ "--catalog", str(catalog_file), - "--cache", - str(tmp_path / "cache"), + "--depot", + str(tmp_path / "depot"), "catalog", "keys", "--prefix", @@ -148,11 +148,11 @@ def test_catalog_listing(catalog_file, tmp_path, capsys): def test_missing_settings_exit_2_and_name_the_setting(catalog_file, tmp_path, cdn, capsys, monkeypatch): - base = ["--catalog", str(catalog_file), "--cache", str(tmp_path / "cache")] + base = ["--catalog", str(catalog_file), "--depot", str(tmp_path / "depot")] code, out, err = run(base + ["bundle", "pull", "Char/A"], capsys) assert code == 2 and out == "" assert err.strip() == ( - "nnnotes: a_01.bundle is not in the cache: setting catalog.region is not set: give it " + "nnnotes: a_01.bundle is not downloaded: setting catalog.region is not set: give it " "as `region` in the [catalog] table of the config file, the environment variable " "NNNOTES_CATALOG_REGION or --region (no config file was found: `nnnotes config init` " "writes one to fill in)" @@ -164,7 +164,7 @@ def test_missing_settings_exit_2_and_name_the_setting(catalog_file, tmp_path, cd assert code == 2 and "bundle.key" in err and "NNNOTES_BUNDLE_KEY" in err assert cdn not in err and len(err.strip().splitlines()) == 1 code, _, err = run(["bundle", "pull", "Char/A"], capsys) - assert code == 2 and "paths.cache" in err and "NNNOTES_PATHS_CACHE" in err and "--cache" in err + assert code == 2 and "paths.depot" in err and "NNNOTES_PATHS_DEPOT" in err and "--depot" in err def test_malformed_setting_names_setting_not_value(catalog_file, tmp_path, cdn, capsys, monkeypatch): @@ -176,8 +176,8 @@ def test_malformed_setting_names_setting_not_value(catalog_file, tmp_path, cdn, [ "--catalog", str(catalog_file), - "--cache", - str(tmp_path / "cache"), + "--depot", + str(tmp_path / "depot"), "--region", "zz", "bundle", @@ -187,14 +187,14 @@ def test_malformed_setting_names_setting_not_value(catalog_file, tmp_path, cdn, capsys, ) assert code == 2 and err.strip() == ( - "nnnotes: a_01.bundle is not in the cache: setting bundle.key: must be 16 bytes (32 hex digits)" + "nnnotes: a_01.bundle is not downloaded: setting bundle.key: must be 16 bytes (32 hex digits)" ) assert bad not in err def test_missing_path_on_disk(tmp_path, capsys): code, _, err = run( - ["--cache", str(tmp_path / "c"), "--catalog", str(tmp_path / "absent.bin"), "catalog", "keys"], capsys + ["--depot", str(tmp_path / "c"), "--catalog", str(tmp_path / "absent.bin"), "catalog", "keys"], capsys ) assert code == 2 and "setting paths.catalog: file" in err @@ -206,14 +206,14 @@ def test_pull_from_config_file(catalog_file, tmp_path, cdn, capsys): f'[bundle]\nkey = "{KEY_HEX}"\nnonce_seed = "{SEED_HEX}"\n' f'[catalog]\nregion = "zz"\nlanguage = "xx"\n' f'[servers.zz]\ncdn = "{cdn}"\n' - f'[paths]\ncatalog = "../{catalog_file.name}"\ncache = "cache"\n', + f'[paths]\ncatalog = "../{catalog_file.name}"\ndepot = "depot"\n', encoding="utf-8", ) code, out, err = run(["--config", str(conf), "bundle", "pull", "Char/A", "Char/B"], capsys) assert code == 0, err - cached = tmp_path / "conf" / "cache" / "bundles" / "a_01.bundle" - assert cached.read_bytes() == synth.fake_bundle("a_01.bundle") - assert [Path(line).resolve() for line in out.splitlines()] == [cached.resolve()] * 2 + pulled = tmp_path / "conf" / "depot" / "downloads" / "bundles" / "a_01.bundle" + assert pulled.read_bytes() == synth.fake_bundle("a_01.bundle") + assert [Path(line).resolve() for line in out.splitlines()] == [pulled.resolve()] * 2 assert KEY_HEX not in out + err @@ -263,13 +263,14 @@ def test_web_needs_the_player(tmp_path, capsys): assert code == 2 and "--player" in err and "NNNOTES_PATHS_PLAYER" in err -def test_catalog_from_the_cache_needs_no_region(catalog_file, tmp_path, capsys): - cache = tmp_path / "cache" - code, _, err = run(["--cache", str(cache), "--language", "xx", "catalog", "keys"], capsys) +def test_catalog_from_the_downloads_needs_no_region(catalog_file, tmp_path, capsys): + depot = tmp_path / "depot" + downloads = depot / "downloads" + code, _, err = run(["--depot", str(depot), "--language", "xx", "catalog", "keys"], capsys) assert code == 2 and "catalog.region" in err and "NNNOTES_CATALOG_REGION" in err - cache.mkdir() - (cache / "catalog_main_xx.bin").write_bytes(catalog_file.read_bytes()) - code, out, err = run(["--cache", str(cache), "--language", "xx", "catalog", "keys", "--prefix", "Char/"], capsys) + downloads.mkdir(parents=True) + (downloads / "catalog_main_xx.bin").write_bytes(catalog_file.read_bytes()) + code, out, err = run(["--depot", str(depot), "--language", "xx", "catalog", "keys", "--prefix", "Char/"], capsys) assert code == 0, err assert out.splitlines() == ["Char/A", "Char/B"] @@ -284,8 +285,8 @@ def test_apk_is_a_required_setting(argv, catalog_file, tmp_path, cdn, capsys, mo [ "--catalog", str(catalog_file), - "--cache", - str(tmp_path / "cache"), + "--depot", + str(tmp_path / "depot"), "--region", "zz", "--master", @@ -342,8 +343,8 @@ def data(catalog_file, tmp_path, cdn, monkeypatch): return [ "--catalog", str(catalog_file), - "--cache", - str(tmp_path / "cache"), + "--depot", + str(tmp_path / "depot"), "--region", "zz", "--language", diff --git a/tests/cli/test_voices.py b/tests/cli/test_voices.py index 28d9396..722f070 100644 --- a/tests/cli/test_voices.py +++ b/tests/cli/test_voices.py @@ -7,7 +7,7 @@ from nnnotes import cli from nnnotes.content import storyvoices from nnnotes.formats import cri -from nnnotes.pipeline.store import Store +from nnnotes.pipeline.depot import Depot from nnnotes.source.session import Session from tests.pipeline import test_voices as tv from tests.pipeline import test_voices_story as ts @@ -47,7 +47,7 @@ def decode(cat, sheet, out_dir, key=None, fmt="flac", **kw): def test_list_search_summary(tmp_path, capsysbinary): - out = tv.export_dir(tmp_path, Store(tmp_path / "s")) + out = tv.export_dir(tmp_path, Depot(tmp_path / "s")) command(["voices", "list", "--from", str(out), "--character", "Alpha", "--category", "Talk.CharacterRankUp"]) line = capsysbinary.readouterr().out.decode("utf-8") assert ( @@ -76,8 +76,8 @@ def test_without_an_export_the_content_is_built(tmp_path, capsysbinary, session) def test_get_copies_the_exported_files(tmp_path, capsysbinary): - store = Store(tmp_path / "s") - out = tv.export_dir(tmp_path, store) + depot = Depot(tmp_path / "s") + out = tv.export_dir(tmp_path, depot) dst = tmp_path / "got" command(["voices", "get", "MasterHomeSpot:10001", "-o", str(dst), "--from", str(out)]) assert sorted(p.name for p in dst.iterdir()) == ["IntroRandom.flac", "IntroRandom_2.flac", "voice.json"] @@ -114,7 +114,7 @@ def test_get_decodes_at_the_flac_level_given(tmp_path, capsysbinary, monkeypatch monkeypatch.setattr(cri, "acb_data", lambda cat, sheet: ({"acb": tv.sheet_acb(sheet)}, cri.RAW_ACB)) monkeypatch.setattr(cri, "decode", decode) monkeypatch.setattr(Session, "catalog", lambda self, bundles=True, region=None: tv.Catalog()) - out = tv.export_dir(tmp_path, Store(tmp_path / "s")) + out = tv.export_dir(tmp_path, Depot(tmp_path / "s")) command(["voices", "get", "101", "-o", str(tmp_path / "g"), "--from", str(out), "--flac-level", "5"]) assert decode.calls == [("VoiceSystem_01", "flac", 5)] for argv, message in ( @@ -127,7 +127,7 @@ def test_get_decodes_at_the_flac_level_given(tmp_path, capsysbinary, monkeypatch def test_story_rows_are_filtered(tmp_path, capsysbinary): - out = ts.export_dir(tmp_path, Store(tmp_path / "s")) + out = ts.export_dir(tmp_path, Depot(tmp_path / "s")) command( ["voices", "list", "--from", str(out), "--source", "story", "--episode", "900001", "--status", "ok"], tv.Cfg("en"), diff --git a/tests/content/test_ui.py b/tests/content/test_ui.py index d1380bc..f17ebab 100644 --- a/tests/content/test_ui.py +++ b/tests/content/test_ui.py @@ -176,7 +176,7 @@ def test_cli_selects_the_jp_split_apk_and_keeps_public_provenance(tmp_path, expo { "catalog": {"region": "jp"}, "servers": {"jp": {"apk": str(apk)}}, - "paths": {"apk": "wrong-global.apk", "cache": str(tmp_path / "cache")}, + "paths": {"apk": "wrong-global.apk", "depot": str(tmp_path / "depot")}, }, environ={}, ) diff --git a/tests/musicdata/test_deckdata.py b/tests/musicdata/test_deckdata.py index 5a9fa6a..36729b6 100644 --- a/tests/musicdata/test_deckdata.py +++ b/tests/musicdata/test_deckdata.py @@ -608,10 +608,10 @@ def test_decoded_master_checks(tmp_path): def test_resource_version_from_the_catalog_store(tmp_path): remote = synth.CatalogWriter().build([("a_01.bundle", synth.remote("a_01.bundle"), [])]) sha = hashlib.sha256(remote).hexdigest() - assert deckdata.resource_version(tmp_path / "store", sha) is None - CatalogDB(tmp_path / "store").add(remote, None, region="xx", language="en", resource_version="1.2.3") - assert deckdata.resource_version(tmp_path / "store", sha) == "1.2.3" - assert deckdata.resource_version(tmp_path / "store", "0" * 64) is None + assert deckdata.resource_version(tmp_path / "depot", sha) is None + CatalogDB(tmp_path / "depot").add(remote, None, region="xx", language="en", resource_version="1.2.3") + assert deckdata.resource_version(tmp_path / "depot", sha) == "1.2.3" + assert deckdata.resource_version(tmp_path / "depot", "0" * 64) is None assert deckdata.resource_version(None, sha) is None diff --git a/tests/musicdata/test_musicdata.py b/tests/musicdata/test_musicdata.py index f7d4d3f..8cd8c4d 100644 --- a/tests/musicdata/test_musicdata.py +++ b/tests/musicdata/test_musicdata.py @@ -1260,7 +1260,7 @@ def test_command(tmp_path, capsys, monkeypatch): monkeypatch.setenv("NNNOTES_MASTER_IV", synth.MASTER_IV.hex()) monkeypatch.setattr(Session, "catalog", lambda self, **kw: FakeCatalog(CHARTS)) monkeypatch.setattr(score, "fetch_chart", lambda cat, name: cat.charts[name]) - base = ["--region", "xx", "--cache", str(tmp_path / "cache"), "music-data", "--master-files", str(d), "--no-bgm"] + base = ["--region", "xx", "--depot", str(tmp_path / "depot"), "music-data", "--master-files", str(d), "--no-bgm"] code, stdout, err = run(base + ["--no-deck", "-o", str(out)], capsys) assert code == 0, err r = json.loads(stdout) @@ -1297,7 +1297,7 @@ def test_command_decoded_master(tmp_path, capsys, monkeypatch): fake = FakeDeck() real = musicdata.Deck monkeypatch.setattr(musicdata, "Deck", lambda **kw: real(module=fake, **kw)) - common = ["--region", "xx", "--cache", str(tmp_path / "cache")] + common = ["--region", "xx", "--depot", str(tmp_path / "depot")] options = ["--no-bgm", "--full", "-o"] code, stdout, err = run( common + ["--master", str(dec), "music-data", "--decoded-master"] + options + [str(out)], capsys diff --git a/tests/pipeline/test_catalogdb.py b/tests/pipeline/test_catalogdb.py index 1b40ba2..8a2c34b 100644 --- a/tests/pipeline/test_catalogdb.py +++ b/tests/pipeline/test_catalogdb.py @@ -15,8 +15,8 @@ stable_raw_name, version_id, ) +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env, describe, execute, registry -from nnnotes.pipeline.store import Store from nnnotes.source.addressables import parse, parse_header, parse_keys, parse_locations from nnnotes.source.catalog import APK_OFFSET_BASE, Catalog, file_name, location_kind from tests.helpers import synth @@ -158,18 +158,18 @@ def test_location_kind_and_file_name(): def test_catalog_bundles_raw_files_locations(tmp_path): remote = build(remote_entries()) - cat = Catalog(remote, tmp_path / "cache") + cat = Catalog(remote, tmp_path / "downloads") assert [b.name for b in cat.bundles()] == [f"chara_a_{H1}.bundle", f"chara_b_{H2}.bundle", "shared.bundle"] assert [b.remote for b in cat.bundles()] == [True, True, False] assert [file_name(e["internal_id"]) for e in cat.raw_files()] == [RAW] - assert cat.cached_raw(cat.raw_files()[0]) is None + assert cat.downloaded_raw(cat.raw_files()[0]) is None assert cat.sources() == {"remote": remote} locs = cat.locations() assert [e["offset"] for e in locs] == [e["offset"] for e in cat.entries] assert {e["catalog"] for e in locs} == {"remote"} assert [e["kind"] for e in locs] == ["asset", "asset", "bundle", "bundle", "asset", "bundle", "raw", "asset"] - with_apk = Catalog(remote, tmp_path / "cache", apk=make_apk(tmp_path / "base.apk")) + with_apk = Catalog(remote, tmp_path / "downloads", apk=make_apk(tmp_path / "base.apk")) shared = [b for b in with_apk.bundles() if b.name == "shared.bundle"] assert len(shared) == 1 and shared[0].offset >= APK_OFFSET_BASE # the APK catalog's location wins assert [file_name(e["internal_id"]) for e in with_apk.raw_files()] == ["cri_emb/voice_" + H4, RAW] @@ -292,9 +292,9 @@ def test_diff_with_addresses(): assert "~ object chara_a/Assets/Game/Char/A.png (object, byteSize)" in format_diff(d) -# ---------------------------------------------------------------- versions in the store +# ---------------------------------------------------------------- versions in the depot def test_catalog_db(tmp_path): - db = CatalogDB(tmp_path / "store") + db = CatalogDB(tmp_path / "depot") assert db.versions() == [] remote, apk = build(remote_entries()), build(APK) v = db.add(remote, apk, region="r1", language="xx", hash_text="00ff" * 8) @@ -320,7 +320,7 @@ def test_catalog_db(tmp_path): assert db.index(v) == index(remote, apk) ins = db.inputs(v) assert [i.role for i in ins] == ["remote", "apk"] and ins[0].locators[0]["path"] == str(db.path(rsha)) - assert (tmp_path / "store" / "catalogs" / "index.json").read_bytes() == contract.encode( + assert (tmp_path / "depot" / "catalogs" / "index.json").read_bytes() == contract.encode( {"schema": "nnnotes.catalogs/1", "versions": db.versions()} ) assert default_label("ab" * 32) == "ab" * 6 and default_label("ab" * 32, "9.9") == "9.9" @@ -338,22 +338,22 @@ def test_fetch_with_and_without_hash(tmp_path): # ---------------------------------------------------------------- stage def test_index_stage(tmp_path): - db = CatalogDB(tmp_path / "store") + db = CatalogDB(tmp_path / "depot") remote, apk = build(remote_entries()), build(APK) v = db.add(remote, apk) - store = Store(tmp_path / "store") + depot = Depot(tmp_path / "depot") stage = IndexStage() - env = Env(store, {"catalogs": {"main": db.inputs(v)}}) + env = Env(depot, {"catalogs": {"main": db.inputs(v)}}) assert stage.subjects(env) == ["main"] task = describe(stage, "main", None, env) assert task.id == "catalog.index:main" - ex = execute(task, store, registry(stage)) + ex = execute(task, depot, registry(stage)) assert ex.status == "ran" and ex.result_status == "ok" - (rec,) = store.result(task.key)["artifacts"] + (rec,) = depot.result(task.key)["artifacts"] assert rec["id"] == "catalog.index:main#index" and rec["semantics"]["facts"]["bundles"] == 3 - assert contract.loads(store.read(rec["content"]["sha256"])) == index(remote, apk) - assert execute(task, store, registry(stage)).status == "hit" - only_remote = Env(store, {"catalogs": {"main": db.inputs(v)[:1]}}) + assert contract.loads(depot.read(rec["content"]["sha256"])) == index(remote, apk) + assert execute(task, depot, registry(stage)).status == "hit" + only_remote = Env(depot, {"catalogs": {"main": db.inputs(v)[:1]}}) assert describe(stage, "main", None, only_remote).key != task.key with pytest.raises(ValueError, match="expected remote"): - describe(stage, "main", None, Env(store, {"catalogs": {"main": db.inputs(v)[1:]}})) + describe(stage, "main", None, Env(depot, {"catalogs": {"main": db.inputs(v)[1:]}})) diff --git a/tests/pipeline/test_census.py b/tests/pipeline/test_census.py index 40647a5..933a34b 100644 --- a/tests/pipeline/test_census.py +++ b/tests/pipeline/test_census.py @@ -5,8 +5,8 @@ from nnnotes.pipeline import census, contract from nnnotes.pipeline.contract import Input +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry -from nnnotes.pipeline.store import Store from nnnotes.unity import reader as unity SCRIPTS = "CAB-5c1e0000000000000000000000000000" @@ -300,26 +300,26 @@ def load(path): return scene_bundle()[0] monkeypatch.setattr(unity, "load", load) - store = Store(tmp_path / "store") - sha, size = store.identify(bundle, "bundles", bundle.name) + depot = Depot(tmp_path / "depot") + sha, size = depot.identify(bundle, "bundles", bundle.name) inp = Input("bundle", sha, size, bundle.name, ({"kind": "file", "path": str(bundle)},)) stage = census.CensusStage() - env = Env(store, {"bundles": {"x": inp}}) + env = Env(depot, {"bundles": {"x": inp}}) assert stage.subjects(env) == ["x"] task = describe(stage, "x", None, env) assert task.id == "unity.census:x" and task.atoms == census.CensusStage.ATOMS assert task.atoms["reader"].startswith("unitypy-") assert task.cost.peak_bytes > size - assert execute(task, store, registry(stage)).status == "ran" - doc = store.result(task.key) + assert execute(task, depot, registry(stage)).status == "ran" + doc = depot.result(task.key) (rec,) = doc["artifacts"] assert rec["id"] == "unity.census:x#census" and rec["semantics"]["facts"]["objects"] == 7 assert rec["provenance"]["inputs"] == [{"role": "bundle", "sha256": sha}] - assert contract.loads(store.read(rec["content"]["sha256"])) == census.census_env(scene_bundle()[0]) + assert contract.loads(depot.read(rec["content"]["sha256"])) == census.census_env(scene_bundle()[0]) assert loaded == [str(bundle)] - assert execute(task, store, registry(stage)).status == "hit" and len(loaded) == 1 + assert execute(task, depot, registry(stage)).status == "hit" and len(loaded) == 1 with pytest.raises(ValueError, match="expected 'bundle'"): - describe(stage, "x", None, Env(store, {"bundles": {"x": Input("other", sha, size)}})) + describe(stage, "x", None, Env(depot, {"bundles": {"x": Input("other", sha, size)}})) for waiting in (None, Pending("fetch:x_1.bundle")): with pytest.raises(Pending, match="fetch:x"): - describe(stage, "x", None, Env(store, {"bundles": {"x": waiting}})) + describe(stage, "x", None, Env(depot, {"bundles": {"x": waiting}})) diff --git a/tests/pipeline/test_contract.py b/tests/pipeline/test_contract.py index aaaa396..a0f1026 100644 --- a/tests/pipeline/test_contract.py +++ b/tests/pipeline/test_contract.py @@ -11,8 +11,8 @@ from nnnotes.pipeline import contract, layout from nnnotes.pipeline import plan as plan_mod from nnnotes.pipeline.contract import Cost, IncompatibleTask, Input, Task +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.orchestrate import FAILURE_CODES, Orchestrator -from nnnotes.pipeline.store import Store from tests.pipeline.test_stages import PIPELINE, toy_files, toy_stages ROOT = Path(__file__).resolve().parents[2] @@ -94,7 +94,7 @@ def a_task(**kw) -> Task: subject="b", params={"png": {"level": 6}}, atoms={"png.encode": "pillow/1"}, - inputs=(Input("bundle", SHA, 10, "b_x.bundle", ({"kind": "cache", "path": "bundles/b_x.bundle"},)),), + inputs=(Input("bundle", SHA, 10, "b_x.bundle", ({"kind": "download", "path": "bundles/b_x.bundle"},)),), context={"scripts": {"CAB-1:1": "A|B|C"}}, cost=Cost(1.5, 100), ) @@ -248,13 +248,13 @@ def test_every_document_has_a_valid_schema(): def test_the_documents_of_a_run_validate(tmp_path): v = validators() - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") facts = {"files": toy_files(tmp_path / "in", {"a": "x\n\nFAIL\n?q", "b": "CRASH", "c": "y"})} - p = plan_mod.plan(toy_stages(), PIPELINE, store, facts=facts, layouts={"cas": {"write": 1, "remove": 0, "keep": 0}}) - run = Orchestrator(store, toy_stages(), log=lambda m: None).run(PIPELINE, facts=facts, selection=["all"]) + p = plan_mod.plan(toy_stages(), PIPELINE, depot, facts=facts, layouts={"cas": {"write": 1, "remove": 0, "keep": 0}}) + run = Orchestrator(depot, toy_stages(), log=lambda m: None).run(PIPELINE, facts=facts, selection=["all"]) sources = [layout.Source(tid, doc) for tid, doc in run.results()] entries, _ = layout.entries("original", sources) - w = layout.materialize(tmp_path / "out", "original", {}, entries, layout.from_store(store)) + w = layout.materialize(tmp_path / "out", "original", {}, entries, layout.from_depot(depot)) manifest = run.write({"original": w["sha256"]}) docs = [ (contract.PLAN, p.doc), @@ -268,7 +268,7 @@ def test_the_documents_of_a_run_validate(tmp_path): docs.append((contract.RESULT, doc)) docs += [(contract.ARTIFACT, a) for a in doc["artifacts"]] keys = [t["key"] for t in manifest["tasks"] if t["status"] == "ran"] - docs += [(contract.COST, store.cost(k)) for k in keys] + docs += [(contract.COST, depot.cost(k)) for k in keys] for name, doc in docs: errors = [e.message for e in v[name].iter_errors(contract.loads(contract.dumps(doc)))] assert errors == [], (name, errors) diff --git a/tests/pipeline/test_cristages.py b/tests/pipeline/test_cristages.py index 89d9002..6881be6 100644 --- a/tests/pipeline/test_cristages.py +++ b/tests/pipeline/test_cristages.py @@ -18,8 +18,8 @@ from nnnotes.pipeline import contract, cristages from nnnotes.pipeline.contract import IncompatibleTask, Input, Task from nnnotes.pipeline.cristages import AUDIO, AudioStage, MovieStage +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry -from nnnotes.pipeline.store import Store from tests.formats.test_usm import ADX, HCA, KEY, MPEG1, VP9, audio_header, build, chunk, rand, sbt, video_header from tests.unity.test_speed_exact import FakeTools @@ -79,11 +79,11 @@ def tools(): yield fake -def file_input(store, tmp: Path, stable: str, data: bytes, role="raw") -> Input: +def file_input(depot, tmp: Path, stable: str, data: bytes, role="raw") -> Input: p = tmp / "files" / stable.replace("/", "_") p.parent.mkdir(parents=True, exist_ok=True) p.write_bytes(data) - sha, size = store.identify(p, "raw", stable) + sha, size = depot.identify(p, "raw", stable) return Input(role, sha, size, f"{stable}_{'0' * 32}", ({"kind": "file", "path": str(p)},)) @@ -106,26 +106,26 @@ def catalog_index(keys: dict, bundles=()) -> dict: } -def planning(store, tmp, files: dict, keys=None, boot=True, bundles=()) -> Env: - raw = {s: file_input(store, tmp, s, d) for s, d in files.items()} +def planning(depot, tmp, files: dict, keys=None, boot=True, bundles=()) -> Env: + raw = {s: file_input(depot, tmp, s, d) for s, d in files.items()} b = tmp / "data.unity3d" b.write_bytes(b"UnityFS boot data") facts = { "raw": raw, "index": catalog_index(keys or {}, bundles), - "boot": cristages.boot_input(store, b) if boot else None, + "boot": cristages.boot_input(depot, b) if boot else None, } - return Env(store, facts) + return Env(depot, facts) -def export_result(store, env, subject: str, acbs: dict, usms=None) -> str: +def export_result(depot, env, subject: str, acbs: dict, usms=None) -> str: """A committed unity.export result holding `cri.acb` artifacts {cue sheet: ACB bytes} and `cri.usm` artifacts {movie asset name: USM bytes}.""" - t = Task(EXPORT, 1, subject, {}, {}, (Input("bundle", store.put(subject.encode()), len(subject)),)) + t = Task(EXPORT, 1, subject, {}, {}, (Input("bundle", depot.put(subject.encode()), len(subject)),)) recs = [ contract.artifact( f"CAB-{subject}:{i + 1}#acb", - store.add(data, "acb"), + depot.add(data, "acb"), contract.provenance(t), {"kind": "cri.acb", "format": "acb", "facts": {"cueSheet": sheet, "layout": "embedded"}}, ) @@ -134,13 +134,13 @@ def export_result(store, env, subject: str, acbs: dict, usms=None) -> str: recs += [ contract.artifact( f"CAB-{subject}:{100 + i}#usm", - store.add(data, "usm"), + depot.add(data, "usm"), contract.provenance(t), {"kind": "cri.usm", "format": "usm", "facts": {"movie": name}}, ) for i, (name, data) in enumerate(sorted((usms or {}).items())) ] - store.commit(contract.result(t, recs, [])) + depot.commit(contract.result(t, recs, [])) env.done(t.id, t.key) return t.id @@ -150,52 +150,52 @@ def export_result(store, env, subject: str, acbs: dict, usms=None) -> str: def run(stage, env, subject, params=None): t = describe(stage, subject, params, env) - execute(t, env.store, registry(stage), force=True) - return t, env.store.result(t.key) + execute(t, env.depot, registry(stage), force=True) + return t, env.depot.result(t.key) def art(doc, aid): return next(a for a in doc["artifacts"] if a["id"] == aid) -def load(store, doc, aid): - return store.read(art(doc, aid)["content"]["sha256"]) +def load(depot, doc, aid): + return depot.read(art(doc, aid)["content"]["sha256"]) # ---------------------------------------------------------------- the key's source def test_boot_input_from_an_apk_is_stored_once(tmp_path): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") apk = tmp_path / "base.apk" with zipfile.ZipFile(apk, "w") as z: z.writestr(cristages.BOOT_IN_APK, b"UnityFS boot") z.writestr("classes.dex", b"dex") - a = cristages.boot_input(store, apk) + a = cristages.boot_input(depot, apk) assert (a.role, a.sha256, a.size, a.name, a.locators) == ( "boot", contract.sha256(b"UnityFS boot"), 12, "data.unity3d", - ({"kind": "store"},), + ({"kind": "depot"},), ) - assert store.read(a.sha256) == b"UnityFS boot" + assert depot.read(a.sha256) == b"UnityFS boot" with mock.patch.object(zipfile, "ZipFile", side_effect=AssertionError("read again")): - assert cristages.boot_input(Store(tmp_path / "s"), apk) == a # remembered by the APK's content + assert cristages.boot_input(Depot(tmp_path / "s"), apk) == a # remembered by the APK's content d = tmp_path / "data.unity3d" d.write_bytes(b"UnityFS other") - b = cristages.boot_input(store, d) + b = cristages.boot_input(depot, d) assert b.sha256 == contract.sha256(b"UnityFS other") and b.locators[0]["kind"] == "file" - assert cristages.boot_input(store, None) is None + assert cristages.boot_input(depot, None) is None def test_hca_key_is_read_once_per_content(tmp_path): - store = Store(tmp_path / "s") - inp = Input("boot", store.put(b"boot"), 4, None, ({"kind": "store"},)) + depot = Depot(tmp_path / "s") + inp = Input("boot", depot.put(b"boot"), 4, None, ({"kind": "depot"},)) calls = [] with ( mock.patch.object(cristages, "_keys", {}), mock.patch.object(crikey, "find_key", lambda p: calls.append(Path(p).read_bytes()) or 77), ): - assert cristages.hca_key(store, inp) == cristages.hca_key(store, inp) == 77 + assert cristages.hca_key(depot, inp) == cristages.hca_key(depot, inp) == 77 assert calls == [b"boot"] @@ -217,15 +217,15 @@ def test_tool_ids_name_the_version_and_the_executable(tmp_path): # ---------------------------------------------------------------- cri.audio def test_audio_subjects_by_content(tmp_path, tools): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") other = b"@UTF" + b"another sheet" env = planning( - store, + depot, tmp_path, {"cri_assets_cri/sound/bgm_x": ACB, "cri_assets_cri/video/m": usm_file()[0]}, {"Cri/Sound/bgm_x": ["cri_assets_cri/sound/bgm_x"], "Cri/Video/m": ["cri_assets_cri/video/m"]}, ) - tid = export_result(store, env, "cri_bundle", {"bgm_x": ACB, "se_live": other}) + tid = export_result(depot, env, "cri_bundle", {"bgm_x": ACB, "se_live": other}) stage = AudioStage() sha, other_sha = contract.sha256(ACB), contract.sha256(other) assert stage.subjects(env) == sorted([sha, other_sha]) # the raw ACB and its bundle copy: one @@ -236,20 +236,20 @@ def test_audio_subjects_by_content(tmp_path, tools): assert t.input("acb").locators[0]["kind"] == "file" and t.atoms == FAKE_ATOMS assert t.params == {"flac": {"level": 8}} b = describe(stage, other_sha, None, env) - assert b.input("acb").locators == ({"kind": "store"},) - waiting = Env(store, env.facts) + assert b.input("acb").locators == ({"kind": "depot"},) + waiting = Env(depot, env.facts) waiting.waiting(tid) with pytest.raises(Pending, match=tid): stage.subjects(waiting) with pytest.raises(ValueError, match="set \\[paths\\] apk"): - describe(stage, sha, None, planning(store, tmp_path, {"cri_assets_cri/sound/bgm_x": ACB}, boot=False)) + describe(stage, sha, None, planning(depot, tmp_path, {"cri_assets_cri/sound/bgm_x": ACB}, boot=False)) def test_audio_waits_for_raw_files_not_fetched(tmp_path, tools): from nnnotes.cli.assets import Inputs - store = Store(tmp_path / "s") - env = planning(store, tmp_path, {"cri_assets_cri/sound/a": ACB}) + depot = Depot(tmp_path / "s") + env = planning(depot, tmp_path, {"cri_assets_cri/sound/a": ACB}) env.facts["raw"] = Inputs({**env.facts["raw"], "cri_assets_cri/sound/b": Pending("fetch:b_1")}) with pytest.raises(Pending, match="fetch:b_1"): AudioStage().subjects(env) @@ -260,8 +260,8 @@ def test_audio_waits_for_raw_files_not_fetched(tmp_path, tools): def test_audio_artifacts_are_the_files_of_cri_decode(tmp_path, tools, monkeypatch): - store = Store(tmp_path / "s") - env = planning(store, tmp_path, {"cri_assets_cri/sound/bgm_x": ACB}) + depot = Depot(tmp_path / "s") + env = planning(depot, tmp_path, {"cri_assets_cri/sound/bgm_x": ACB}) sha = contract.sha256(ACB) t, doc = run(AudioStage(), env, sha) monkeypatch.setattr(cri, "acb_data", lambda cat, sheet: ({"acb": ACB}, cri.RAW_ACB)) @@ -276,7 +276,7 @@ def test_audio_artifacts_are_the_files_of_cri_decode(tmp_path, tools, monkeypatc assert files == ["a.flac", "a_3.flac", "b.flac", "c.flac", "c_5.flac", "cues.json", "streams.json"] assert sorted(a["id"] for a in doc["artifacts"]) == [f"{t.id}#{f}" for f in files] for f in files: - assert load(store, doc, f"{t.id}#{f}") == (ref / f).read_bytes(), f + assert load(depot, doc, f"{t.id}#{f}") == (ref / f).read_bytes(), f s = art(doc, f"{t.id}#a_3.flac") assert s["content"]["mediaType"] == "audio/flac" and "object" not in s["provenance"] assert s["semantics"] == { @@ -300,7 +300,7 @@ def test_audio_artifacts_are_the_files_of_cri_decode(tmp_path, tools, monkeypatc def test_audio_two_runs_and_levels(tmp_path, tools): - a, b = Store(tmp_path / "a"), Store(tmp_path / "b") + a, b = Depot(tmp_path / "a"), Depot(tmp_path / "b") ea, eb = planning(a, tmp_path / "x", {"s/a": ACB}), planning(b, tmp_path / "y", {"s/a": ACB}) ta, da = run(AudioStage(), ea, contract.sha256(ACB)) tb, db = run(AudioStage(), eb, contract.sha256(ACB)) @@ -313,9 +313,9 @@ def test_audio_two_runs_and_levels(tmp_path, tools): def test_an_acb_with_an_external_awb_is_unsupported(tmp_path, tools): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") env = planning( - store, + depot, tmp_path, {"cri_assets_cri/sound/v": ACB, "cri_assets_cri/sound/v_awb": AWB}, {"Cri/Sound/v": ["cri_assets_cri/sound/v", "cri_assets_cri/sound/v_awb"]}, @@ -340,22 +340,22 @@ def test_an_acb_with_an_external_awb_is_unsupported(tmp_path, tools): def test_a_task_of_other_tools_is_incompatible(tmp_path, tools): - store = Store(tmp_path / "s") - env = planning(store, tmp_path, {"s/a": ACB}) + depot = Depot(tmp_path / "s") + env = planning(depot, tmp_path, {"s/a": ACB}) t = describe(AudioStage(), contract.sha256(ACB), None, env) with mock.patch.object(cristages, "tool_id", lambda which, exe: f"{which}-other/1"): with pytest.raises(IncompatibleTask, match="flac.encode, hca.decode differ"): - execute(t, store, registry(AudioStage())) + execute(t, depot, registry(AudioStage())) # ---------------------------------------------------------------- cri.movie def test_movie_streams_as_stored_and_the_mkv(tmp_path, tools): from nnnotes.formats import usm - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") data, frames = usm_file() env = planning( - store, + depot, tmp_path, {"membercard/1/movie/anime_part1": data, "cri_assets_cri/sound/x": ACB}, {"MemberCard/1/movie/anime_part1": ["membercard/1/movie/anime_part1"]}, @@ -367,8 +367,8 @@ def test_movie_streams_as_stored_and_the_mkv(tmp_path, tools): t, doc = run(stage, env, sha) assert t.params == {"format": "mkv", "flac": {"level": 8}} and set(t.atoms) == MOVIE_ATOMS video, audio = usm.demux(data, KEY) - assert load(store, doc, f"{t.id}#video.ivf") == video.data == b"".join(frames) - assert load(store, doc, f"{t.id}#audio.adx") == audio.data + assert load(depot, doc, f"{t.id}#video.ivf") == video.data == b"".join(frames) + assert load(depot, doc, f"{t.id}#audio.adx") == audio.data assert art(doc, f"{t.id}#video.ivf")["semantics"] == { "kind": "video.stream", "format": "ivf", @@ -382,20 +382,20 @@ def test_movie_streams_as_stored_and_the_mkv(tmp_path, tools): {"kind": "audio", "channel": 0, "codec": "flac", "source": "adx"}, ] } - assert b"-c:a|flac|-compression_level|8" in load(store, doc, mkv["id"]) + assert b"-c:a|flac|-compression_level|8" in load(depot, doc, mkv["id"]) assert doc["items"][0]["status"] == "exported" and doc["items"][0]["class"] == "USM" with fake_tools(key=KEY): tw, dw = run(stage, env, sha, {"format": "webm"}) webm = art(dw, f"{tw.id}#movie.webm") assert tw.params == {"format": "webm"} and webm["content"]["mediaType"] == "video/webm" - assert b"libopus" in load(store, dw, webm["id"]) and b"-c:v|copy" in load(store, dw, webm["id"]) + assert b"libopus" in load(depot, dw, webm["id"]) and b"-c:v|copy" in load(depot, dw, webm["id"]) assert f"{tw.id}#movie.mkv" not in {a["id"] for a in dw["artifacts"]} def test_movie_without_audio(tmp_path, tools): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") data, _ = usm_file(audio=False) - env = planning(store, tmp_path, {"v/m": data}) + env = planning(depot, tmp_path, {"v/m": data}) with fake_tools(key=KEY): t, doc = run(MovieStage(), env, contract.sha256(data)) assert sorted(a["id"] for a in doc["artifacts"]) == [f"{t.id}#movie.mkv", f"{t.id}#video.ivf"] @@ -420,9 +420,9 @@ def test_movie_every_stream_kind(tmp_path, tools): channels: every stream as stored, the subtitles as JSON / SubRip / WebVTT, every track in the Matroska file.""" from nnnotes.formats import usm - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") data = build(EVERY) - env = planning(store, tmp_path, {"v/m": data}) + env = planning(depot, tmp_path, {"v/m": data}) with fake_tools(key=KEY) as fake: t, doc = run(MovieStage(), env, contract.sha256(data)) roles = sorted(contract.parse_artifact_id(a["id"])[1] for a in doc["artifacts"]) @@ -444,21 +444,21 @@ def test_movie_every_stream_kind(tmp_path, tools): ) assert doc["items"][0]["status"] == "exported" streams = {s.name: s for s in usm.demux(data, KEY)} - assert load(store, doc, f"{t.id}#alpha.m1v") == streams["alpha"].data == b"".join(MPEG1) - assert load(store, doc, f"{t.id}#audio_2.hca") == b"".join(HCA) + assert load(depot, doc, f"{t.id}#alpha.m1v") == streams["alpha"].data == b"".join(MPEG1) + assert load(depot, doc, f"{t.id}#audio_2.hca") == b"".join(HCA) alpha = art(doc, f"{t.id}#alpha.m1v") assert alpha["content"]["mediaType"] == "video/mpeg" and alpha["semantics"]["kind"] == "video.alpha" assert alpha["semantics"]["facts"]["alphaType"] == 1 and alpha["semantics"]["facts"]["displayHeight"] == 180 assert art(doc, f"{t.id}#audio_2.hca")["semantics"]["facts"]["cipher"] == 56 - assert json.loads(load(store, doc, f"{t.id}#subtitle.json")) == { + assert json.loads(load(depot, doc, f"{t.id}#subtitle.json")) == { "channel": 0, "records": [ {"language": 0, "timeUnit": 1000, "start": 0, "duration": 1500, "text": "こんにちは", "terminator": 1}, {"language": 0, "timeUnit": 1000, "start": 2000, "duration": 1000, "text": "b", "terminator": 1}, ], } - assert load(store, doc, f"{t.id}#subtitle_1.srt") == b"1\n00:00:00,000 --> 00:00:01,500\nen\n\n" - assert load(store, doc, f"{t.id}#subtitle.vtt").startswith( + assert load(depot, doc, f"{t.id}#subtitle_1.srt") == b"1\n00:00:00,000 --> 00:00:01,500\nen\n\n" + assert load(depot, doc, f"{t.id}#subtitle.vtt").startswith( "WEBVTT\n\n00:00:00.000 --> 00:00:01.500\nこんにちは\n".encode() ) assert ["-i", "-o"] in fake.calls # vgmstream decoded the HCA stream @@ -472,7 +472,7 @@ def test_movie_every_stream_kind(tmp_path, tools): ("subtitle", 0, "srt"), ("subtitle", 1, "srt"), ] - args = load(store, doc, mkv["id"]).split(b"\x00")[0] + args = load(depot, doc, mkv["id"]).split(b"\x00")[0] assert ( b"|+genpts|-f|mpegvideo|-i|video.m1v|" in args and b"|-f|wav|-i|audio_2.wav|" in args @@ -485,9 +485,9 @@ def test_movie_every_stream_kind(tmp_path, tools): def test_movie_webm_forms(tmp_path, tools): """format webm: MPEG-1 video with its alpha re-encoded as VP9 with alpha; the first audio stream only; no subtitles.""" - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") data = build(EVERY) - env = planning(store, tmp_path, {"v/m": data}) + env = planning(depot, tmp_path, {"v/m": data}) with fake_tools(key=KEY): t, doc = run(MovieStage(), env, contract.sha256(data), {"format": "webm"}) webm = art(doc, f"{t.id}#movie.webm") @@ -496,7 +496,7 @@ def test_movie_webm_forms(tmp_path, tools): {"kind": "alpha", "channel": 0, "codec": "vp9", "source": "mpeg1"}, {"kind": "audio", "channel": 0, "codec": "opus", "source": "adx"}, ] - args = load(store, doc, webm["id"]) + args = load(depot, doc, webm["id"]) assert b"alphamerge,format=yuva420p" in args and b"|-c:v|libvpx-vp9|-b:v|0|-crf|20|" in args assert b"audio_1" not in args and b"srt" not in args @@ -504,7 +504,7 @@ def test_movie_webm_forms(tmp_path, tools): def test_movie_subtitles_that_cannot_be_read(tmp_path, tools): """Subtitle bytes that are not records: stored as they are (`.sbt`); a text that is not UTF-8: the JSON only. Either way the movie is `unsupported` with its other artifacts, and the Matroska file has no track for them.""" - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") data = build( [ (b"@SFV", 0, None, VP9, "video"), @@ -512,7 +512,7 @@ def test_movie_subtitles_that_cannot_be_read(tmp_path, tools): (b"@SBT", 1, None, [sbt(0, 1000, 0, 10, b"\x82\xa0")], None), ] ) - env = planning(store, tmp_path, {"v/m": data}) + env = planning(depot, tmp_path, {"v/m": data}) with fake_tools(key=KEY): t, doc = run(MovieStage(), env, contract.sha256(data)) (item,) = doc["items"] @@ -524,7 +524,7 @@ def test_movie_subtitles_that_cannot_be_read(tmp_path, tools): "video.ivf", ] assert sorted(item["artifacts"]) == sorted(a["id"] for a in doc["artifacts"]) - assert json.loads(load(store, doc, f"{t.id}#subtitle_1.json"))["records"][0]["textHex"] == "82a0" + assert json.loads(load(depot, doc, f"{t.id}#subtitle_1.json"))["records"][0]["textHex"] == "82a0" assert art(doc, f"{t.id}#movie.mkv")["semantics"]["facts"]["tracks"] == [ {"kind": "video", "channel": 0, "codec": "vp9"} ] @@ -539,9 +539,9 @@ def test_movie_subtitles_that_cannot_be_read(tmp_path, tools): ], ) def test_unsupported_movies(tmp_path, tools, streams, extra, code): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") data, _ = usm_file(streams=streams, extra=extra) - env = planning(store, tmp_path, {"v/m": data}) + env = planning(depot, tmp_path, {"v/m": data}) with fake_tools(key=KEY): t, doc = run(MovieStage(), env, contract.sha256(data)) (item,) = doc["items"] @@ -551,11 +551,11 @@ def test_unsupported_movies(tmp_path, tools, streams, extra, code): def test_movie_held_in_a_bundle(tmp_path, tools): """A USM artifact of unity.export is a subject like a raw USM (one task for both copies), named by the keys of its bundle that end in the movie asset's name; it waits for unity.export.""" - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") held, _ = usm_file(audio=False) raw, _ = usm_file() env = planning( - store, + depot, tmp_path, {"cri_assets_cri/video/m": raw}, { @@ -565,7 +565,7 @@ def test_movie_held_in_a_bundle(tmp_path, tools): }, bundles=("embcri_video_common",), ) - tid = export_result(store, env, "embcri_video_common", {}, {"warmup": held, "loose": raw}) + tid = export_result(depot, env, "embcri_video_common", {}, {"warmup": held, "loose": raw}) stage = MovieStage() sha, raw_sha = contract.sha256(held), contract.sha256(raw) assert stage.subjects(env) == sorted([sha, raw_sha]) @@ -573,10 +573,10 @@ def test_movie_held_in_a_bundle(tmp_path, tools): assert stage.depends(sha, env) == [tid] and stage.depends(raw_sha, env) == [tid] with fake_tools(key=KEY): t = describe(stage, sha, None, env) - assert t.input("usm").locators == ({"kind": "store"},) and sorted(i.role for i in t.inputs) == ["boot", "usm"] + assert t.input("usm").locators == ({"kind": "depot"},) and sorted(i.role for i in t.inputs) == ["boot", "usm"] t, doc = run(stage, env, sha) - assert doc["items"][0]["status"] == "exported" and load(store, doc, f"{t.id}#video.ivf") == b"".join(VP9) - waiting = Env(store, env.facts) + assert doc["items"][0]["status"] == "exported" and load(depot, doc, f"{t.id}#video.ivf") == b"".join(VP9) + waiting = Env(depot, env.facts) waiting.waiting(tid) with pytest.raises(Pending, match=tid): stage.subjects(waiting) @@ -623,9 +623,9 @@ class GoldenAudio(AudioStage): def ATOMS(self) -> dict: return dict(FAKE_ATOMS) - def run(self, task, store): + def run(self, task, depot): with fake_tools(): - return super().run(task, store) + return super().run(task, depot) class GoldenMovie(MovieStage): @@ -639,15 +639,15 @@ def ATOMS(self) -> dict: "hca.decode": "vgmstream-fake/1", } - def run(self, task, store): + def run(self, task, depot): with fake_tools(key=KEY): - return super().run(task, store) + return super().run(task, depot) def _fixture(stage, role, data: bytes, params=None): - def build(store): - sha = store.put(data) - boot = store.put(b"UnityFS boot data") + def build(depot): + sha = depot.put(data) + boot = depot.put(b"UnityFS boot data") return Task( stage.name, stage.version, @@ -655,8 +655,8 @@ def build(store): stage.normalize(params), stage.ATOMS, ( - Input(role, sha, len(data), None, ({"kind": "store"},)), - Input("boot", boot, 17, None, ({"kind": "store"},)), + Input(role, sha, len(data), None, ({"kind": "depot"},)), + Input("boot", boot, 17, None, ({"kind": "depot"},)), ), ) @@ -705,6 +705,6 @@ def test_golden_providers_run_twice_alike(tmp_path): for provider in (audio_golden, movie_golden): p = provider() - a = golden(p["stage"], p["fixtures"], Store(tmp_path / f"{p['stage'].name}a"), p["libraries"]) - b = golden(p["stage"], p["fixtures"], Store(tmp_path / f"{p['stage'].name}b"), p["libraries"]) + a = golden(p["stage"], p["fixtures"], Depot(tmp_path / f"{p['stage'].name}a"), p["libraries"]) + b = golden(p["stage"], p["fixtures"], Depot(tmp_path / f"{p['stage'].name}b"), p["libraries"]) assert a == b and golden_problems(a, b, p["stage"]) == [] diff --git a/tests/pipeline/test_store.py b/tests/pipeline/test_depot.py similarity index 83% rename from tests/pipeline/test_store.py rename to tests/pipeline/test_depot.py index 56c596b..c81bb48 100644 --- a/tests/pipeline/test_store.py +++ b/tests/pipeline/test_depot.py @@ -1,4 +1,4 @@ -"""The content-addressed store: write-once bytes, results as the commit point, input identities, runs.""" +"""The content-addressed depot: write-once bytes, results as the commit point, input identities, runs.""" import hashlib import multiprocessing as mp @@ -8,24 +8,24 @@ import pytest from nnnotes.pipeline import contract -from nnnotes.pipeline import store as store_mod +from nnnotes.pipeline import depot as depot_mod from nnnotes.pipeline.contract import Input, Task -from nnnotes.pipeline.store import InputMissing, Store, StoreConflict +from nnnotes.pipeline.depot import Depot, DepotConflict, InputMissing -def a_result(s: Store, task: Task, payload: bytes) -> dict: +def a_result(s: Depot, task: Task, payload: bytes) -> dict: aid = contract.artifact_id(task.id, "data.bin") art = contract.artifact(aid, s.add(payload, "bin"), contract.provenance(task), {"kind": "test"}) return contract.result(task, [art], []) def test_put_is_write_once_and_content_addressed(tmp_path): - s = Store(tmp_path) + s = Depot(tmp_path) sha = s.put(b"hello") assert sha == hashlib.sha256(b"hello").hexdigest() assert s.path(sha) == tmp_path / "cas" / "sha256" / sha[:2] / sha and s.read(sha) == b"hello" assert s.put(b"hello") == sha and (s.written, s.reused) == (1, 1) - assert Store(tmp_path).put(b"hello") == sha # another writer finds it + assert Depot(tmp_path).put(b"hello") == sha # another writer finds it assert s.add(b"{}", "JSON") == { "sha256": contract.sha256(b"{}"), "size": 2, @@ -40,7 +40,7 @@ def test_put_is_write_once_and_content_addressed(tmp_path): def test_commit_needs_the_bytes_and_is_write_once(tmp_path): - s = Store(tmp_path) + s = Depot(tmp_path) task = Task("t.x", 1, "a") doc = a_result(s, task, b"payload") assert s.result(task.key) is None @@ -49,7 +49,7 @@ def test_commit_needs_the_bytes_and_is_write_once(tmp_path): assert s.result_path(task.key).read_bytes() == contract.encode(doc) assert s.commit(doc) is False # identical: kept other = dict(doc, items=[contract.item("F:1", "unsupported", why=contract.reason("x"))]) - with pytest.raises(StoreConflict): + with pytest.raises(DepotConflict): s.commit(other) t2 = Task("t.x", 1, "b") missing = contract.result( @@ -70,7 +70,7 @@ def test_commit_needs_the_bytes_and_is_write_once(tmp_path): def _crash_after_put(root: str, key_file: str) -> None: - s = Store(root) + s = Depot(root) task = Task("t.x", 1, "crash") Path(key_file).write_text(task.key) a_result(s, task, b"written before the crash") # bytes stored, result never committed @@ -82,7 +82,7 @@ def test_a_crash_before_the_result_leaves_no_hit(tmp_path): p.start() p.join(60) assert p.exitcode == 9 - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") key = (tmp_path / "k").read_text() assert s.result(key) is None and not s.has_result(key) assert s.has(contract.sha256(b"written before the crash")) @@ -90,7 +90,7 @@ def test_a_crash_before_the_result_leaves_no_hit(tmp_path): def _writer(root: str, n: int, barrier) -> None: - s = Store(root) + s = Depot(root) barrier.wait() for i in range(40): task = Task("t.x", 1, f"s{i % 10}") @@ -108,7 +108,7 @@ def test_concurrent_writers_across_processes(tmp_path): for p in procs: p.join(120) assert [p.exitcode for p in procs] == [0, 0, 0, 0] - s = Store(tmp_path) + s = Depot(tmp_path) for i in range(10): task = Task("t.x", 1, f"s{i}") doc = s.result(task.key) @@ -123,13 +123,13 @@ def test_concurrent_writers_across_processes(tmp_path): def test_identify_memoizes_by_path_size_and_mtime(tmp_path, monkeypatch): f = tmp_path / "in.bin" f.write_bytes(b"abc") - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") assert s.identify(f, "bundles", "in.bin") == (contract.sha256(b"abc"), 3) assert s.named("bundles", "in.bin") == (contract.sha256(b"abc"), 3) calls = [] - real = store_mod.hashlib.file_digest - monkeypatch.setattr(store_mod.hashlib, "file_digest", lambda *a: calls.append(1) or real(*a)) - fresh = Store(tmp_path / "s") # the memo is on disk + real = depot_mod.hashlib.file_digest + monkeypatch.setattr(depot_mod.hashlib, "file_digest", lambda *a: calls.append(1) or real(*a)) + fresh = Depot(tmp_path / "s") # the memo is on disk assert fresh.identify(f) == (contract.sha256(b"abc"), 3) and calls == [] assert fresh.named("bundles", "in.bin")[1] == 3 and fresh.named("bundles", "other") is None f.write_bytes(b"abcd") @@ -137,15 +137,15 @@ def test_identify_memoizes_by_path_size_and_mtime(tmp_path, monkeypatch): assert fresh.identify(f) == (contract.sha256(b"abcd"), 4) and calls == [1] with open(tmp_path / "s" / "inputs" / "files.jsonl", "a", encoding="utf-8") as g: g.write('{"torn": ') # a writer that died mid-line - assert Store(tmp_path / "s").identify(f) == (contract.sha256(b"abcd"), 4) + assert Depot(tmp_path / "s").identify(f) == (contract.sha256(b"abcd"), 4) def test_open_input_tries_locators_in_order(tmp_path): - s = Store(tmp_path / "s", cache=tmp_path / "cache") + s = Depot(tmp_path / "s") data = b"bundle bytes" sha = contract.sha256(data) - (tmp_path / "cache" / "bundles").mkdir(parents=True) - (tmp_path / "cache" / "bundles" / "b.bundle").write_bytes(data) + (tmp_path / "s" / "downloads" / "bundles").mkdir(parents=True) + (tmp_path / "s" / "downloads" / "bundles" / "b.bundle").write_bytes(data) (tmp_path / "wrong.bin").write_bytes(b"other bytes!") inp = Input( "bundle", @@ -153,12 +153,12 @@ def test_open_input_tries_locators_in_order(tmp_path): len(data), "b.bundle", ( - {"kind": "store"}, + {"kind": "depot"}, {"kind": "file", "path": str(tmp_path / "wrong.bin")}, - {"kind": "cache", "path": "bundles/b.bundle"}, + {"kind": "download", "path": "bundles/b.bundle"}, ), ) - assert s.open_input(inp) == tmp_path / "cache" / "bundles" / "b.bundle" + assert s.open_input(inp) == tmp_path / "s" / "downloads" / "bundles" / "b.bundle" s.put(data) assert s.open_input(inp) == s.path(sha) lonely = Input("bundle", contract.sha256(b"x"), 1, "x", ({"kind": "catalog", "catalog": "c", "internalId": "i"},)) @@ -167,12 +167,12 @@ def test_open_input_tries_locators_in_order(tmp_path): (tmp_path / "fetched").write_bytes(b"x") s.fetch = lambda inp, loc: tmp_path / "fetched" assert s.open_input(lonely) == tmp_path / "fetched" - with pytest.raises(InputMissing, match="store: absent"): + with pytest.raises(InputMissing, match="depot: absent"): s.open_input(Input("x", contract.sha256(b"y"), 1)) def test_runs_costs_and_the_latest_run_of_a_selection(tmp_path): - s = Store(tmp_path) + s = Depot(tmp_path) doc = contract.run_doc( context={"region": "r"}, selection=["group:a"], @@ -190,17 +190,17 @@ def test_runs_costs_and_the_latest_run_of_a_selection(tmp_path): assert s.catalogs == tmp_path / "catalogs" -def test_a_store_crosses_processes_as_its_settings(tmp_path): +def test_a_depot_crosses_processes_as_its_settings(tmp_path): import pickle - s = Store(tmp_path / "s", cache=tmp_path / "c") + s = Depot(tmp_path / "s") s.put(b"x") t = pickle.loads(pickle.dumps(s)) - assert (t.root, t.cache, t.written) == (s.root, s.cache, 0) + assert (t.root, t.downloads, t.written) == (s.root, s.downloads, 0) def test_write_doc_writes_the_canonical_bytes_as_it_encodes(tmp_path): - from nnnotes.pipeline.store import write_doc + from nnnotes.pipeline.depot import write_doc docs = [ {"b": [1, 2, {"é": "x\n"}], "a": {}, "c": [], "d": None, "e": 1.5, "f": "立ち絵" * 1000}, diff --git a/tests/pipeline/test_depot_gc.py b/tests/pipeline/test_depot_gc.py new file mode 100644 index 0000000..326cc6d --- /dev/null +++ b/tests/pipeline/test_depot_gc.py @@ -0,0 +1,74 @@ +"""The collection of the depot: what kept runs reach stays, the rest goes.""" + +import os + +from nnnotes.pipeline import contract +from nnnotes.pipeline.contract import Input, Task +from nnnotes.pipeline.depot import Depot + + +def commit(d: Depot, task: Task, payload: bytes) -> dict: + aid = contract.artifact_id(task.id, "data.bin") + art = contract.artifact(aid, d.add(payload, "bin"), contract.provenance(task), {"kind": "test"}) + doc = contract.result(task, [art], []) + d.commit(doc) + return doc + + +def write_run(d: Depot, selection, tasks, layouts=None, mtime=None) -> dict: + doc = contract.run_doc( + context={"region": "r"}, + selection=selection, + params={}, + stages={"t.x": 1}, + tasks=[{"id": t.id, "key": t.key, "status": "ran"} for t in tasks], + layouts=layouts or {}, + summary={}, + ) + p = d.write_run(doc) + if mtime is not None: + os.utime(p, ns=(mtime, mtime)) + return doc + + +def test_gc_keeps_what_the_kept_runs_reach(tmp_path): + d = Depot(tmp_path) + stored = d.put(b"a stored input") + new = Task("t.x", 1, "a", inputs=(Input("doc", stored, 14, None, ({"kind": "depot"},)),)) + old = Task("t.x", 1, "b") + new_art = commit(d, new, b"new")["artifacts"][0]["content"]["sha256"] + old_art = commit(d, old, b"old")["artifacts"][0]["content"]["sha256"] + orphan = d.put(b"no result names it") + first = write_run(d, ["group:a"], [old], mtime=1_000_000_000) + latest = write_run(d, ["group:a"], [new], mtime=2_000_000_000) + + counted = d.gc(keep=1, dry_run=True) + assert counted["runs"] == {"kept": 1, "removed": 1} and counted["contents"] == {"kept": 2, "removed": 2} + assert d.run(first["id"]) is not None and d.has(orphan) # a dry run removes nothing + + r = d.gc(keep=1) + assert r["runs"] == {"kept": 1, "removed": 1} and r["results"] == {"kept": 1, "removed": 1} + assert r["contents"] == {"kept": 2, "removed": 2} + assert d.run(first["id"]) is None and d.run(latest["id"]) is not None + assert d.result(new.key) is not None and d.result(old.key) is None + assert d.has(new_art) and d.has(stored) and not d.has(old_art) and not d.has(orphan) + + +def test_gc_keeps_every_run_by_default_and_the_latest_of_a_selection(tmp_path): + d = Depot(tmp_path) + a, b = Task("t.x", 1, "a"), Task("t.x", 1, "b") + commit(d, a, b"a") + commit(d, b, b"b") + write_run(d, ["group:a"], [a], mtime=2_000_000_000) + latest = write_run(d, ["group:a"], [b], mtime=1_000_000_000) # written last: the selection's latest run + assert d.gc()["results"] == {"kept": 2, "removed": 0} + r = d.gc(keep=1) + assert r["runs"] == {"kept": 2, "removed": 0} and d.latest_run(["group:a"]) == latest + + +def test_gc_keeps_the_files_of_a_layout_a_run_names(tmp_path): + d = Depot(tmp_path) + placed = d.put(b"a derived file") + manifest = d.add(contract.encode({"files": [{"path": "x.json", "sha256": placed, "size": 14}]}), "json") + write_run(d, ["group:a"], [], layouts={"original": manifest["sha256"]}) + assert d.gc()["contents"] == {"kept": 2, "removed": 0} and d.has(placed) diff --git a/tests/pipeline/test_goldens.py b/tests/pipeline/test_goldens.py index c2e0d1d..bc0dc7c 100644 --- a/tests/pipeline/test_goldens.py +++ b/tests/pipeline/test_goldens.py @@ -6,8 +6,8 @@ {"stage": "unity.export", "version": 1, "atoms": {...}, "libraries": {"Pillow": "12.3.0", ...}, "provider": ":", "fixtures": {"": {"key": "...", "result": "..."}}} -The provider (a function of a test module) returns {"stage": Stage, "fixtures": {name: callable(store) -> Task}, -"libraries": [distribution names]}; each fixture builds its synthetic inputs into the store. Every atom the stage +The provider (a function of a test module) returns {"stage": Stage, "fixtures": {name: callable(depot) -> Task}, +"libraries": [distribution names]}; each fixture builds its synthetic inputs into the depot. Every atom the stage declares must be used by a fixture. A record made with other library versions is skipped (another encoder version may write other bytes for the same content) unless GOLDENS_STRICT=1, as in the pinned CI job, where it fails. GOLDENS_UPDATE=1 rewrites the records from the current code (after a version bump). @@ -20,8 +20,8 @@ import pytest from nnnotes.pipeline import contract +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Output, golden, golden_problems, library_versions -from nnnotes.pipeline.store import Store from tests.pipeline.test_stages import Src GOLDEN_DIR = Path(__file__).resolve().parents[1] / "golden" @@ -32,9 +32,9 @@ def provided(spec: str) -> dict: return getattr(importlib.import_module(module), name)() -def record(spec: str, store: Store) -> dict: +def record(spec: str, depot: Depot) -> dict: p = provided(spec) - doc = golden(p["stage"], p["fixtures"], store, p.get("libraries", ())) + doc = golden(p["stage"], p["fixtures"], depot, p.get("libraries", ())) return dict(doc, provider=spec) @@ -43,31 +43,31 @@ def test_golden_record(path, tmp_path): expected = contract.loads(path.read_bytes()) p = provided(expected["provider"]) if os.environ.get("GOLDENS_UPDATE") == "1": - path.write_bytes(contract.encode(record(expected["provider"], Store(tmp_path / "store")))) + path.write_bytes(contract.encode(record(expected["provider"], Depot(tmp_path / "depot")))) return installed = library_versions(expected["libraries"]) if installed != expected["libraries"]: if os.environ.get("GOLDENS_STRICT") != "1": pytest.skip(f"recorded with {expected['libraries']}, installed {installed}") pytest.fail(f"the pinned job has {installed}, the record needs {expected['libraries']}") - actual = record(expected["provider"], Store(tmp_path / "store")) + actual = record(expected["provider"], Depot(tmp_path / "depot")) assert golden_problems(expected, actual, p["stage"]) == [] # ---------------------------------------------------------------- the machinery itself (toy stage) def text_fixture(text: str): - def build(store): + def build(depot): from nnnotes.pipeline.contract import Input, Task data = text.encode() - sha = store.put(data) + sha = depot.put(data) return Task( "toy.src", 1, text[:4], {"suffix": ""}, dict(Src.ATOMS), - (Input("file", sha, len(data), None, ({"kind": "store"},)),), + (Input("file", sha, len(data), None, ({"kind": "depot"},)),), ) return build @@ -82,16 +82,16 @@ def toy_provider(): def test_a_regenerated_record_matches(tmp_path): - a = record("tests.pipeline.test_goldens:toy_provider", Store(tmp_path / "a")) - b = record("tests.pipeline.test_goldens:toy_provider", Store(tmp_path / "b")) + a = record("tests.pipeline.test_goldens:toy_provider", Depot(tmp_path / "a")) + b = record("tests.pipeline.test_goldens:toy_provider", Depot(tmp_path / "b")) assert a == b and golden_problems(a, b, Src()) == [] assert a["libraries"] == library_versions(["pytest"]) and set(a["fixtures"]) == {"plain", "items"} assert a["atoms"] == Src.ATOMS and a["version"] == 1 class Changed(Src): - def run(self, task, store): - out = super().run(task, store) + def run(self, task, depot): + out = super().run(task, depot) return Output( out.artifacts, [dict(i, reason=contract.reason("new", "x")) if i["status"] == "unsupported" else i for i in out.items], @@ -99,21 +99,21 @@ def run(self, task, store): def test_an_output_change_without_a_version_bump_fails(tmp_path): - expected = record("tests.pipeline.test_goldens:toy_provider", Store(tmp_path / "a")) + expected = record("tests.pipeline.test_goldens:toy_provider", Depot(tmp_path / "a")) fixtures = toy_provider()["fixtures"] - actual = golden(Changed(), fixtures, Store(tmp_path / "b"), ["pytest"]) + actual = golden(Changed(), fixtures, Depot(tmp_path / "b"), ["pytest"]) assert golden_problems(expected, actual) == ["toy.src: fixture items: output changed without a version bump"] bumped = Changed() bumped.version = 2 - actual = golden(bumped, {n: _versioned(f, 2) for n, f in fixtures.items()}, Store(tmp_path / "c"), ["pytest"]) + actual = golden(bumped, {n: _versioned(f, 2) for n, f in fixtures.items()}, Depot(tmp_path / "c"), ["pytest"]) problems = golden_problems(expected, actual) assert problems[0] == "toy.src: version 2, golden record has 1: regenerate the golden record" assert "toy.src: fixture plain: task key changed" in problems def _versioned(build, version): - def b(store): - t = build(store) + def b(depot): + t = build(depot) return contract.Task(t.stage, version, t.subject, t.params, t.atoms, t.inputs, t.context) return b @@ -123,14 +123,14 @@ def test_every_declared_atom_needs_a_fixture(tmp_path): class More(Src): ATOMS = {**Src.ATOMS, "mesh.glb": "toy-glb/1"} - doc = record("tests.pipeline.test_goldens:toy_provider", Store(tmp_path / "a")) + doc = record("tests.pipeline.test_goldens:toy_provider", Depot(tmp_path / "a")) assert golden_problems(doc, doc, More()) == ["toy.src: no fixture uses mesh.glb"] missing = dict(doc, fixtures={"plain": doc["fixtures"]["plain"]}) assert golden_problems(missing, doc) == ["toy.src: fixture items has no golden entry"] def test_records_are_canonical_documents(tmp_path): - doc = record("tests.pipeline.test_goldens:toy_provider", Store(tmp_path / "a")) + doc = record("tests.pipeline.test_goldens:toy_provider", Depot(tmp_path / "a")) path = tmp_path / "toy.src.json" path.write_bytes(contract.encode(doc)) assert contract.loads(path.read_bytes()) == doc diff --git a/tests/pipeline/test_layout.py b/tests/pipeline/test_layout.py index 924d992..b4a8655 100644 --- a/tests/pipeline/test_layout.py +++ b/tests/pipeline/test_layout.py @@ -8,11 +8,11 @@ from nnnotes.pipeline import contract, layout from nnnotes.pipeline.contract import Task +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.layout import LayoutError, Source, escape_segment, unescape_path -from nnnotes.pipeline.store import Store -def bundle(store, subject, objects, loose=(), root=True): +def bundle(depot, subject, objects, loose=(), root=True): """A unity.export-like result: objects [(file, pathId, class, name, container, {role: (bytes, ext)})], loose artifacts of no object [(role, bytes, ext)].""" task = Task("unity.export", 1, subject) @@ -26,14 +26,14 @@ def bundle(store, subject, objects, loose=(), root=True): for role, (data, ext) in roles.items(): aid = contract.artifact_id(oid, role) arts.append( - contract.artifact(aid, store.add(data, ext), contract.provenance(task, obj=obj), {"kind": "test"}) + contract.artifact(aid, depot.add(data, ext), contract.provenance(task, obj=obj), {"kind": "test"}) ) ids.append(aid) items.append(contract.item(oid, "exported", artifacts=ids, cls=cls)) for role, data, ext in loose: arts.append( contract.artifact( - contract.artifact_id(task.id, role), store.add(data, ext), contract.provenance(task), {"kind": "test"} + contract.artifact_id(task.id, role), depot.add(data, ext), contract.provenance(task), {"kind": "test"} ) ) return Source(task.id, contract.result(task, arts, items), f"_bundles/{subject}" if root else None) @@ -60,7 +60,7 @@ def test_escaping_is_reversible_and_safe(): def test_objects_of_one_container(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") src = bundle( s, "card", @@ -104,7 +104,7 @@ def test_objects_of_one_container(tmp_path): def test_the_primary_is_the_class_the_extension_names(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") src = bundle( s, "b", @@ -118,7 +118,7 @@ def test_the_primary_is_the_class_the_extension_names(tmp_path): def test_objects_without_container_and_artifacts_of_no_object(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") src = bundle( s, "b1", @@ -143,7 +143,7 @@ def test_objects_without_container_and_artifacts_of_no_object(tmp_path): def test_a_bundle_of_several_serialized_files_names_them(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") src = bundle( s, "scene", @@ -161,7 +161,7 @@ def test_a_bundle_of_several_serialized_files_names_them(tmp_path): def test_a_container_in_two_bundles_is_told_apart_by_bundle(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") a = bundle( s, "group_a", @@ -181,7 +181,7 @@ def test_a_container_in_two_bundles_is_told_apart_by_bundle(tmp_path): def test_the_catalog_case_and_case_collisions(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") src = bundle( s, "b", @@ -219,7 +219,7 @@ def test_equal_content_at_one_path_shares_the_file(): def two_bundles(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") a = bundle( s, "a", @@ -245,8 +245,8 @@ def test_cas_and_original_derive_from_each_other(tmp_path): f"assets/{a['content']['sha256']}.{a['content']['ext']}" for src in sources for a in src.result["artifacts"] } orig, _ = layout.entries("original", sources) - w_cas = layout.materialize(tmp_path / "cas", "cas", {}, cas, layout.from_store(s)) - w_orig = layout.materialize(tmp_path / "orig", "original", {}, orig, layout.from_store(s)) + w_cas = layout.materialize(tmp_path / "cas", "cas", {}, cas, layout.from_depot(s)) + w_orig = layout.materialize(tmp_path / "orig", "original", {}, orig, layout.from_depot(s)) assert w_cas["manifest"]["schema"] == contract.LAYOUT and w_orig["write"] == len(orig) # original from cas + the records; cas from the original manifest alone layout.materialize(tmp_path / "orig2", "original", {}, orig, layout.from_layout(tmp_path / "cas")) @@ -265,15 +265,15 @@ def test_materialize_is_incremental_and_touches_only_its_files(tmp_path): (out / "assets").mkdir(parents=True) (out / "notes.txt").write_text("mine") # a file of the user orig, _ = layout.entries("original", sources) - first = layout.materialize(out, "original", {}, orig, layout.from_store(s)) + first = layout.materialize(out, "original", {}, orig, layout.from_depot(s)) assert layout.diff(out, orig) == {"write": 0, "remove": 0, "keep": len(orig)} - again = layout.materialize(out, "original", {}, orig, layout.from_store(s)) + again = layout.materialize(out, "original", {}, orig, layout.from_depot(s)) assert (again["write"], again["remove"], again["keep"]) == (0, 0, len(orig)) assert again["sha256"] == first["sha256"] (out / "assets" / "t.png").unlink() # damaged output is written again changed, _ = layout.entries("original", sources[:1]) # bundle b dropped assert layout.diff(out, changed) == {"write": 1, "remove": 1, "keep": len(changed) - 1} - w = layout.materialize(out, "original", {}, changed, layout.from_store(s)) + w = layout.materialize(out, "original", {}, changed, layout.from_depot(s)) assert (w["write"], w["remove"]) == (1, 1) assert not (out / "assets" / "text").exists() and (out / "notes.txt").read_text() == "mine" assert (out / "assets" / "t.png").exists() and not (out / layout.JOURNAL).exists() @@ -286,7 +286,7 @@ def test_an_interrupted_run_is_reconciled(tmp_path): (out / "half").mkdir(parents=True) (out / "half" / "written.png").write_bytes(b"partial") # written by a run that was interrupted (out / layout.JOURNAL).write_bytes(contract.encode({"write": ["half/written.png"], "remove": []})) - w = layout.materialize(out, "original", {}, orig, layout.from_store(s)) + w = layout.materialize(out, "original", {}, orig, layout.from_depot(s)) assert w["remove"] == 1 and not (out / "half").exists() and not (out / layout.JOURNAL).exists() @@ -297,27 +297,27 @@ def test_a_file_that_is_not_the_layouts_is_never_overwritten(tmp_path): (out / "assets").mkdir(parents=True) (out / "assets" / "t.png").write_bytes(b"someone else's") with pytest.raises(LayoutError, match="not this layout's: assets/t.png"): - layout.materialize(out, "original", {}, orig, layout.from_store(s)) + layout.materialize(out, "original", {}, orig, layout.from_depot(s)) assert (out / "assets" / "t.png").read_bytes() == b"someone else's" (out / "assets" / "t.png").write_bytes(png("t")[0]) # the right content: adopted - w = layout.materialize(out, "original", {}, orig, layout.from_store(s)) + w = layout.materialize(out, "original", {}, orig, layout.from_depot(s)) assert w["write"] == len(orig) def test_hard_links(tmp_path): s, sources = two_bundles(tmp_path) cas, _ = layout.entries("cas", sources) - w = layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_store(s), link="hard") + w = layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_depot(s), link="hard") assert w["linked"] == w["write"] and w["copied"] == w["cloned"] == 0 and w["failed"] == [] e = cas[0] assert os.path.samefile(tmp_path / "out" / e["path"], s.path(e["sha256"])) - assert os.stat(tmp_path / "out" / e["path"]).st_mode & 0o222 == 0 # the store object itself: read-only + assert os.stat(tmp_path / "out" / e["path"]).st_mode & 0o222 == 0 # the depot object itself: read-only if not (hasattr(os, "geteuid") and os.geteuid() == 0): # (the superuser writes anyway) with pytest.raises(PermissionError): (tmp_path / "out" / e["path"]).write_bytes(b"edited") for bad in ("soft", "auto"): with pytest.raises(ValueError): - layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_store(s), link=bad) + layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_depot(s), link=bad) def _clone_by_copy(src, dst): @@ -335,20 +335,20 @@ def fail(*a, **k): def test_the_probe_prefers_clone_then_hard_link_then_copy(tmp_path, monkeypatch): import errno - out, store = tmp_path / "out", tmp_path / "s" / "cas" / "sha256" + out, depot = tmp_path / "out", tmp_path / "s" / "cas" / "sha256" monkeypatch.setattr(layout, "clone", _clone_by_copy) - assert layout.probe(out, store)[0] == "clone" and layout.probe(out, store, "clone")[0] == "clone" + assert layout.probe(out, depot)[0] == "clone" and layout.probe(out, depot, "clone")[0] == "clone" monkeypatch.setattr(layout, "clone", _no(errno.EOPNOTSUPP)) - how, why = layout.probe(out, store) + how, why = layout.probe(out, depot) assert how == "hard" and why.startswith("hard links: clone unsupported on this file system (") with pytest.raises(LayoutError, match="--link clone: clone unsupported"): - layout.probe(out, store, "clone") + layout.probe(out, depot, "clone") monkeypatch.setattr(layout.os, "link", _no(errno.EXDEV)) - assert layout.probe(out, store) == ("copy", "copy: store on another file system") - with pytest.raises(LayoutError, match="--link hard: store on another file system"): - layout.probe(out, store, "hard") - assert layout.probe(out, store, "copy") == ("copy", "copy: as asked") - assert list(out.iterdir()) == [] and list(store.iterdir()) == [] # nothing left behind + assert layout.probe(out, depot) == ("copy", "copy: depot on another file system") + with pytest.raises(LayoutError, match="--link hard: depot on another file system"): + layout.probe(out, depot, "hard") + assert layout.probe(out, depot, "copy") == ("copy", "copy: as asked") + assert list(out.iterdir()) == [] and list(depot.iterdir()) == [] # nothing left behind def test_every_placement_gives_the_same_files_and_manifest(tmp_path, monkeypatch): @@ -357,13 +357,13 @@ def test_every_placement_gives_the_same_files_and_manifest(tmp_path, monkeypatch monkeypatch.setattr(layout, "clone", _clone_by_copy) got = {} for how in layout.PLACEMENTS: - w = layout.materialize(tmp_path / how, "original", {}, orig, layout.from_store(s), link=how) + w = layout.materialize(tmp_path / how, "original", {}, orig, layout.from_depot(s), link=how) counts = {k: w[k] for k in ("cloned", "linked", "copied")} assert counts[{"clone": "cloned", "hard": "linked", "copy": "copied"}[how]] == w["write"] == len(orig) got[how] = (files(tmp_path / how), w["sha256"]) assert got["clone"] == got["hard"] == got["copy"] (tmp_path / "copy" / orig[0]["path"]).write_bytes(b"a copy is writable") - assert all(os.stat(s.path(e["sha256"])).st_mode & 0o222 == 0 for e in orig) # store objects: read-only + assert all(os.stat(s.path(e["sha256"])).st_mode & 0o222 == 0 for e in orig) # depot objects: read-only def test_a_file_the_placement_cannot_make_is_reported_and_placed_next_time(tmp_path, monkeypatch): @@ -374,29 +374,29 @@ def test_a_file_the_placement_cannot_make_is_reported_and_placed_next_time(tmp_p real = os.link victim = s.path(cas[0]["sha256"]) monkeypatch.setattr(layout.os, "link", lambda a, b: _no(errno.EMLINK)() if Path(a) == victim else real(a, b)) - w = layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_store(s), link="hard") + w = layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_depot(s), link="hard") assert [f["path"] for f in w["failed"]] == [cas[0]["path"]] and w["write"] == len(cas) - 1 assert cas[0]["path"] not in {e["path"] for e in layout.read_manifest(tmp_path / "out")["entries"]} monkeypatch.setattr(layout.os, "link", real) - w = layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_store(s), link="hard") + w = layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_depot(s), link="hard") assert w["failed"] == [] and w["write"] == 1 and len(layout.read_manifest(tmp_path / "out")["entries"]) == len(cas) def test_a_layout_replaces_and_removes_its_read_only_files(tmp_path): s, sources = two_bundles(tmp_path) cas, _ = layout.entries("cas", sources) - layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_store(s), link="hard") - layout.materialize(tmp_path / "out", "cas", {}, cas[1:], layout.from_store(s), link="copy") # remove one + layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_depot(s), link="hard") + layout.materialize(tmp_path / "out", "cas", {}, cas[1:], layout.from_depot(s), link="copy") # remove one assert not (tmp_path / "out" / cas[0]["path"]).exists() - assert os.stat(s.path(cas[0]["sha256"])).st_mode & 0o222 == 0 # its store object stays read-only + assert os.stat(s.path(cas[0]["sha256"])).st_mode & 0o222 == 0 # its depot object stays read-only other = [dict(cas[1], path=cas[2]["path"])] + [e for e in cas[1:] if e["path"] != cas[2]["path"]] - layout.materialize(tmp_path / "out", "cas", {}, other, layout.from_store(s), link="copy") # replace one + layout.materialize(tmp_path / "out", "cas", {}, other, layout.from_depot(s), link="copy") # replace one assert (tmp_path / "out" / cas[2]["path"]).read_bytes() == s.read(cas[1]["sha256"]) assert os.stat(s.path(cas[2]["sha256"])).st_mode & 0o222 == 0 def test_an_objects_artifacts_from_several_results_are_placed_together(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") export = bundle( s, "atlas_b", @@ -421,16 +421,16 @@ def test_an_objects_artifacts_from_several_results_are_placed_together(tmp_path) def test_derivation_keeps_lower_case_extensions(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") src = bundle(s, "b", [("CAB-1", 1, "Texture2D", "x", "Assets/X.PNG", {"image": png("x")})]) orig, _ = layout.entries("original", [src]) assert [e["path"] for e in orig] == ["Assets/X.PNG"] - layout.materialize(tmp_path / "o", "original", {}, orig, layout.from_store(s)) + layout.materialize(tmp_path / "o", "original", {}, orig, layout.from_depot(s)) assert layout.cas_from_manifest(layout.read_manifest(tmp_path / "o")) == layout.entries("cas", [src])[0] def test_objects_without_container_go_under_the_root_of_their_file(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") export = bundle(s, "sprites_b", [("CAB-s", 3, "Sprite", "loose", None, {"meta": (b"{}", "json")})]) crop = bundle(s, "atlas_a:7", [("CAB-s", 3, "Sprite", "loose", None, {"image": png("c")})]) crop.root = "_tasks/sprite.crop/atlas_a:7" diff --git a/tests/pipeline/test_link.py b/tests/pipeline/test_link.py index f6d5f3f..9f24dc9 100644 --- a/tests/pipeline/test_link.py +++ b/tests/pipeline/test_link.py @@ -3,8 +3,8 @@ from nnnotes.pipeline import catalogdb, census, contract, link from nnnotes.pipeline.catalogdb import CatalogDB, IndexStage, index from nnnotes.pipeline.contract import Input +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry -from nnnotes.pipeline.store import Store from tests.helpers import synth HA, HB, HC = "a" * 32, "b" * 32, "c" * 32 @@ -241,32 +241,32 @@ def test_diff_addresses(): # ---------------------------------------------------------------- stages -def census_task(store, subject, doc): +def census_task(depot, subject, doc): """A census task with a stored result made of `doc` (no bundle read).""" data = contract.encode(doc) inp = Input("bundle", contract.sha256(subject.encode()), 1, subject + ".bundle") - task = describe(census.CensusStage(), subject, None, Env(store, {"bundles": {subject: inp}})) - content = store.add(data, "json") + task = describe(census.CensusStage(), subject, None, Env(depot, {"bundles": {subject: inp}})) + content = depot.add(data, "json") rec = contract.artifact( contract.artifact_id(task.id, "census"), content, contract.provenance(task), {"kind": "unity.census", "format": "json", "facts": census.facts(doc)}, ) - store.commit(contract.result(task, [rec], [])) + depot.commit(contract.result(task, [rec], [])) return task def test_link_stages(tmp_path): - store = Store(tmp_path / "store") - db = CatalogDB(tmp_path / "store") + depot = Depot(tmp_path / "depot") + db = CatalogDB(tmp_path / "depot") v = db.add(synth.CatalogWriter().build(catalog_entries())) - env = Env(store, {"catalogs": {"main": db.inputs(v)}}) + env = Env(depot, {"catalogs": {"main": db.inputs(v)}}) itask = describe(IndexStage(), "main", None, env) - execute(itask, store, registry(IndexStage())) + execute(itask, depot, registry(IndexStage())) env.done(itask.id, itask.key) docs = censuses() - tasks = {s: census_task(store, s, d) for s, d in docs.items()} + tasks = {s: census_task(depot, s, d) for s, d in docs.items()} env.waiting(tasks["card_a"].id) # not run yet for s, t in tasks.items(): if s != "card_a": @@ -282,27 +282,27 @@ def test_link_stages(tmp_path): env.done(tasks["card_a"].id, tasks["card_a"].key) st = describe(stages["link.scripts"], "all", None, env) assert st.id == "link.scripts:all" and [i.role for i in st.inputs] == ["census:scripts"] - execute(st, store, stages) - (rec,) = store.result(st.key)["artifacts"] + execute(st, depot, stages) + (rec,) = depot.result(st.key)["artifacts"] assert rec["id"] == "link.scripts:all#scripts" and rec["semantics"]["facts"] == {"scripts": 2} - assert contract.loads(store.read(rec["content"]["sha256"])) == link.scripts([docs["scripts"]]) + assert contract.loads(depot.read(rec["content"]["sha256"])) == link.scripts([docs["scripts"]]) at = describe(stages["link.addresses"], "main", None, env) assert [i.role for i in at.inputs] == ["census:card_a", "census:scene_x", "census:shared", "index"] - execute(at, store, stages) - (rec,) = store.result(at.key)["artifacts"] + execute(at, depot, stages) + (rec,) = depot.result(at.key)["artifacts"] assert rec["id"] == "link.addresses:main#addresses" and rec["semantics"]["facts"]["objects"] == 5 expected = link.addresses(index(*db.catalog_bytes(v)), {s: docs[s] for s in ("card_a", "scene_x", "shared")}) - assert contract.loads(store.read(rec["content"]["sha256"])) == expected - assert execute(at, store, stages).status == "hit" + assert contract.loads(depot.read(rec["content"]["sha256"])) == expected + assert execute(at, depot, stages).status == "hit" def test_artifacts_table_and_stage(tmp_path): - store = Store(tmp_path / "store") + depot = Depot(tmp_path / "depot") arts, items = [], [] for pid, status in ((1, "exported"), (2, "contained"), (3, "failed")): oid = contract.object_id(CAB_A, pid) if status == "exported": - c = store.add(b"{}\n", "json") + c = depot.add(b"{}\n", "json") aid = contract.artifact_id(oid, "json") arts.append({"id": aid, "content": c}) items.append(contract.item(oid, "exported", artifacts=[aid])) @@ -315,9 +315,9 @@ def test_artifacts_table_and_stage(tmp_path): assert table["contained"] == {f"{CAB_A}:2": f"{CAB_A}:1#json"} assert table["artifacts"][f"{CAB_A}:1#json"]["ext"] == "json" stage = link.ArtifactsStage() - env = Env(store, {}) + env = Env(depot, {}) export = contract.Task("unity.export", 1, "card_a") - store.commit( + depot.commit( contract.result( export, [contract.artifact(a["id"], a["content"], contract.provenance(export), {"kind": "x"}) for a in arts], @@ -331,10 +331,10 @@ def test_artifacts_table_and_stage(tmp_path): env.waiting("sprite.crop:atlas", failed=True) # failed tasks are skipped t = describe(stage, "all", None, env) assert t.context == {"results": {export.id: export.key}} and t.inputs == () - execute(t, store, registry(stage)) - (rec,) = store.result(t.key)["artifacts"] + execute(t, depot, registry(stage)) + (rec,) = depot.result(t.key)["artifacts"] assert rec["id"] == "link.artifacts:all#artifacts" - assert contract.loads(store.read(rec["content"]["sha256"]))["objects"] == table["objects"] + assert contract.loads(depot.read(rec["content"]["sha256"]))["objects"] == table["objects"] def test_the_artifact_table_is_written_entry_by_entry_with_the_same_bytes(): @@ -372,7 +372,7 @@ def res(n, oid, aid, *, within=None): def test_store_add_chunks_is_add_of_the_joined_bytes(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") c = s.add_chunks(iter([b"ab", b"", "é".encode()]), ".JSON") assert c == contract.content("abé".encode(), "json") and s.read(c["sha256"]) == "abé".encode() assert s.add_chunks([b"a", "bé".encode()], "json") == c and (s.written, s.reused) == (1, 1) diff --git a/tests/pipeline/test_modelstages.py b/tests/pipeline/test_modelstages.py index 0df3759..49afcd8 100644 --- a/tests/pipeline/test_modelstages.py +++ b/tests/pipeline/test_modelstages.py @@ -12,9 +12,9 @@ from nnnotes.pipeline.catalogdb import index from nnnotes.pipeline.census import CensusStage from nnnotes.pipeline.contract import IncompatibleTask, Input, Task +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.modelstages import Closures, Live2DStage, SpineStage, TaskCatalog from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry -from nnnotes.pipeline.store import Store from nnnotes.source.catalog import Catalog from tests.helpers import synth @@ -54,7 +54,7 @@ def make_apk(path, entries=APK): def test_closures_are_those_of_catalog_resolve(tmp_path): remote = build(remote_entries()) - cat = Catalog(remote, tmp_path / "cache", apk=make_apk(tmp_path / "base.apk")) + cat = Catalog(remote, tmp_path / "downloads", apk=make_apk(tmp_path / "base.apk")) doc = index(remote, build(APK)) c = Closures(doc) names = {b["stable"]: b["name"] for b in doc["bundles"]} @@ -84,23 +84,23 @@ def bundles_fact(doc, data, locators=True): out = {} for b in doc["bundles"]: body = data[b["stable"]] - locs = ({"kind": "cache", "path": f"bundles/{b['name']}"},) if locators else () + locs = ({"kind": "download", "path": f"bundles/{b['name']}"},) if locators else () out[b["stable"]] = Input("bundle", contract.sha256(body), len(body), b["name"], locs) return out -def live2d_env(store, entries=None, data=None, selected=None, locators=True): +def live2d_env(depot, entries=None, data=None, selected=None, locators=True): doc = index(build(entries or remote_entries()), build(APK)) data = data or contents() bundles = bundles_fact(doc, data, locators) - return Env(store, {"index": doc, "bundles": bundles, "selected": sorted(selected or bundles)}) + return Env(depot, {"index": doc, "bundles": bundles, "selected": sorted(selected or bundles)}) def test_live2d_subjects_inputs_and_context(tmp_path): stage = Live2DStage() - env = live2d_env(Store(tmp_path / "s")) + env = live2d_env(Depot(tmp_path / "s")) assert stage.subjects(env) == [M1, M2] # model keys with their bundle - assert stage.subjects(live2d_env(Store(tmp_path / "s"), selected=["model_m2", "shared"])) == [M2] + assert stage.subjects(live2d_env(Depot(tmp_path / "s"), selected=["model_m2", "shared"])) == [M2] t = describe(stage, M1, None, env) assert t.id == "live2d.model:" + M1 and t.params == {} and t.atoms == Live2DStage.ATOMS assert [(i.role, i.name) for i in t.inputs] == [ @@ -111,32 +111,32 @@ def test_live2d_subjects_inputs_and_context(tmp_path): ] assert t.inputs[2].sha256 == contract.sha256(contents()["model_m1"]) assert t.context == {"internalId": PREFAB} - assert t.inputs[0].locators == ({"kind": "cache", "path": "bundles/monoscripts.bundle"},) + assert t.inputs[0].locators == ({"kind": "download", "path": "bundles/monoscripts.bundle"},) assert set(Live2DStage.ATOMS) == {"live2d.extract_model"} assert all(lib.lower() in Live2DStage.ATOMS["live2d.extract_model"] for lib in modelstages.LIBRARIES) def test_live2d_keys_follow_the_closure_contents_and_order_only(tmp_path): - stage, store = Live2DStage(), Store(tmp_path / "s") - key = describe(stage, M1, None, live2d_env(store)).key + stage, depot = Live2DStage(), Depot(tmp_path / "s") + key = describe(stage, M1, None, live2d_env(depot)).key # other file names, no locators, another catalog with more keys: the same task renamed = [(e[0].replace(H[1], H[8]), e[1].replace(H[1], H[8]), *e[2:]) for e in remote_entries()] renamed.append(("Other/key", "Assets/Game/other.asset", [3])) - assert describe(stage, M1, None, live2d_env(store, renamed, locators=False)).key == key + assert describe(stage, M1, None, live2d_env(depot, renamed, locators=False)).key == key # a bundle of the closure changed, the closure's order changed, the key's location changed: another task - assert describe(stage, M1, None, live2d_env(store, data=contents(b"!"))).key != key - assert describe(stage, M1, None, live2d_env(store, remote_entries(order=(3, 2, 5)))).key != key - assert describe(stage, M1, None, live2d_env(store, remote_entries(m1_prefab=PREFAB + "x"))).key != key + assert describe(stage, M1, None, live2d_env(depot, data=contents(b"!"))).key != key + assert describe(stage, M1, None, live2d_env(depot, remote_entries(order=(3, 2, 5)))).key != key + assert describe(stage, M1, None, live2d_env(depot, remote_entries(m1_prefab=PREFAB + "x"))).key != key # a bundle outside the closure changed: the same task other = dict(contents(), model_m2=b"UnityFS m2 changed") - assert describe(stage, M1, None, live2d_env(store, data=other)).key == key + assert describe(stage, M1, None, live2d_env(depot, data=other)).key == key bumped = type("Bumped", (Live2DStage,), {"version": 2})() - assert describe(bumped, M1, None, live2d_env(store)).key != key + assert describe(bumped, M1, None, live2d_env(depot)).key != key def test_live2d_waits_for_unfetched_bundles_and_names_unreadable_ones(tmp_path): - stage, store = Live2DStage(), Store(tmp_path / "s") - env = live2d_env(store) + stage, depot = Live2DStage(), Depot(tmp_path / "s") + env = live2d_env(depot) env.facts["bundles"] = dict(env.facts["bundles"], tex=Pending("fetch:tex")) with pytest.raises(Pending, match="fetch:tex"): describe(stage, M1, None, env) @@ -148,17 +148,17 @@ def test_live2d_waits_for_unfetched_bundles_and_names_unreadable_ones(tmp_path): describe(stage, M1, None, env) -def stored_task(store, stage=None): - """The live2d.model task of M1 with every bundle in the store (store locators only).""" +def stored_task(depot, stage=None): + """The live2d.model task of M1 with every bundle in the depot (depot locators only).""" for body in contents().values(): - store.put(body) - return describe(stage or Live2DStage(), M1, None, live2d_env(store, locators=False)) + depot.put(body) + return describe(stage or Live2DStage(), M1, None, live2d_env(depot, locators=False)) def test_task_catalog_answers_from_the_task(tmp_path): - store = Store(tmp_path / "s") - t = stored_task(store) - cat = TaskCatalog.from_task(t, store) + depot = Depot(tmp_path / "s") + t = stored_task(depot) + cat = TaskCatalog.from_task(t, depot) assert cat.apk is not None and cat.has(M1) and not cat.has(M2) assert cat.keys("Character/") == [M1] and cat.keys("Other/") == [] assert cat.entry(M1)["internal_id"] == PREFAB and cat.entries_for(M2) == [] @@ -174,7 +174,7 @@ def test_task_catalog_answers_from_the_task(tmp_path): assert [ i.role for i in TaskCatalog( - "k", "Assets/k", modelstages._ordered(Task(Live2DStage.name, 1, "k", inputs=tuple(many))), store + "k", "Assets/k", modelstages._ordered(Task(Live2DStage.name, 1, "k", inputs=tuple(many))), depot ).bundles ] == ["bundle:1", "bundle:2", "bundle:10"] @@ -204,12 +204,12 @@ def test_live2d_run_stores_every_file_of_the_extractor(tmp_path, monkeypatch): calls = [] monkeypatch.setattr(live2d, "extract_model", fake_extract(calls)) - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") stage = Live2DStage() - t = stored_task(store, stage) - ex = execute(t, store, registry(stage)) + t = stored_task(depot, stage) + ex = execute(t, depot, registry(stage)) assert ex.status == "ran" and calls == [(M1, PREFAB)] - doc = store.result(t.key) + doc = depot.result(t.key) tid = "live2d.model:" + M1 got = {contract.parse_artifact_id(a["id"])[1]: a for a in doc["artifacts"]} assert sorted(got) == [ @@ -233,18 +233,18 @@ def test_live2d_run_stores_every_file_of_the_extractor(tmp_path, monkeypatch): "textures/texture_00-1a2b3c4d.png": ("live2d.texture", "png"), } order = ["monoscripts", "shared", "model_m1", "tex"] - assert store.read(got["m1.moc3"]["content"]["sha256"]) == b"MOC3" + b"".join(contents()[s] for s in order) + assert depot.read(got["m1.moc3"]["content"]["sha256"]) == b"MOC3" + b"".join(contents()[s] for s in order) assert got["m1.model3.json"]["semantics"]["facts"] == { "expressions": 1, "motions": 1, "name": "m1", "textures": ["textures/texture_00-1a2b3c4d.png"], } - assert execute(t, store, registry(stage)).status == "hit" - assert execute(t, store, registry(stage), force=True).status == "ran" # the same result again + assert execute(t, depot, registry(stage)).status == "hit" + assert execute(t, depot, registry(stage), force=True).status == "ran" # the same result again other = Task(t.stage, t.version, t.subject, t.params, {"live2d.extract_model": "unitypy-0/1"}, t.inputs, t.context) with pytest.raises(IncompatibleTask): - execute(other, store, registry(stage)) + execute(other, depot, registry(stage)) def test_the_model_key_and_libraries_are_those_of_the_extractors(): @@ -356,36 +356,36 @@ def spine_censuses(): } -def census_result(store, env, subject, doc, body): +def census_result(depot, env, subject, doc, body): inp = Input("bundle", contract.sha256(body), len(body), subject + ".bundle", ()) - t = describe(CensusStage(), subject, None, Env(store, {"bundles": {subject: inp}})) + t = describe(CensusStage(), subject, None, Env(depot, {"bundles": {subject: inp}})) rec = contract.artifact( contract.artifact_id(t.id, "census"), - store.add(contract.encode(doc), "json"), + depot.add(contract.encode(doc), "json"), contract.provenance(t), {"kind": "unity.census", "format": "json", "facts": census.facts(doc)}, ) - store.commit(contract.result(t, [rec], [])) + depot.commit(contract.result(t, [rec], [])) env.done(t.id, t.key) return inp -def spine_env(store, docs=None, data=None, selected=("other_d", "scene_e", "spine_a")): +def spine_env(depot, docs=None, data=None, selected=("other_d", "scene_e", "spine_a")): docs = docs or spine_censuses() data = data or {} - env = Env(store, {"selected": sorted(selected)}) + env = Env(depot, {"selected": sorted(selected)}) env.facts["bundles"] = { - s: census_result(store, env, s, d, data.get(s, b"UnityFS " + s.encode())) for s, d in sorted(docs.items()) + s: census_result(depot, env, s, d, data.get(s, b"UnityFS " + s.encode())) for s, d in sorted(docs.items()) } st = describe(link.ScriptsStage(), "all", None, env) - execute(st, store, registry(link.ScriptsStage())) + execute(st, depot, registry(link.ScriptsStage())) env.done(st.id, st.key) return env def test_spine_subjects_closures_and_context(tmp_path): - stage, store = SpineStage(), Store(tmp_path / "s") - env = spine_env(store) + stage, depot = SpineStage(), Depot(tmp_path / "s") + env = spine_env(depot) assert stage.subjects(env) == ["other_d:5", "scene_e.sharedAssets:7", "spine_a:10"] t = describe(stage, "spine_a:10", None, env) assert t.id == "spine.skeleton:spine_a:10" and t.atoms == SpineStage.ATOMS and t.params == {} @@ -409,16 +409,16 @@ def test_spine_subjects_closures_and_context(tmp_path): def test_spine_keys_follow_the_closure_contents_only(tmp_path): stage = SpineStage() - key = describe(stage, "spine_a:10", None, spine_env(Store(tmp_path / "a"))).key - env = spine_env(Store(tmp_path / "b"), data={"other_d": b"UnityFS other, changed"}) + key = describe(stage, "spine_a:10", None, spine_env(Depot(tmp_path / "a"))).key + env = spine_env(Depot(tmp_path / "b"), data={"other_d": b"UnityFS other, changed"}) assert describe(stage, "spine_a:10", None, env).key == key - env = spine_env(Store(tmp_path / "c"), data={"shader_c": b"UnityFS shader, changed"}) + env = spine_env(Depot(tmp_path / "c"), data={"shader_c": b"UnityFS shader, changed"}) assert describe(stage, "spine_a:10", None, env).key != key def test_spine_waits_for_censuses_and_the_script_table(tmp_path): - stage, store = SpineStage(), Store(tmp_path / "s") - env = spine_env(store) + stage, depot = SpineStage(), Depot(tmp_path / "s") + env = spine_env(depot) env.waiting("unity.census:tex_b") with pytest.raises(Pending, match="unity.census:tex_b"): stage.subjects(env) @@ -490,10 +490,10 @@ def fake_skeleton(pages=1): def test_spine_run_stores_the_files_of_the_skeleton(tmp_path, monkeypatch): from nnnotes.unity import reader as unity - stage, store = SpineStage(), Store(tmp_path / "s") - env = spine_env(store) + stage, depot = SpineStage(), Depot(tmp_path / "s") + env = spine_env(depot) for s in ("scripts", "shader_c", "spine_a", "tex_b"): - store.put(b"UnityFS " + s.encode()) + depot.put(b"UnityFS " + s.encode()) loaded = [] def load_closure(paths): @@ -503,16 +503,16 @@ def load_closure(paths): monkeypatch.setattr(unity, "load_closure", load_closure) sda = fake_skeleton() t = describe(stage, "spine_a:10", None, env) - assert execute(t, store, registry(stage)).status == "ran" + assert execute(t, depot, registry(stage)).status == "ran" assert loaded == [[b"UnityFS scripts", b"UnityFS shader_c", b"UnityFS spine_a", b"UnityFS tex_b"]] - doc = store.result(t.key) + doc = depot.result(t.key) got = {contract.parse_artifact_id(a["id"])[1]: a for a in doc["artifacts"]} assert {r: (a["semantics"]["kind"], a["content"]["ext"]) for r, a in got.items()} == { "chr.skel": ("spine.skeleton", "skel"), "chr.atlas": ("spine.atlas", "atlas"), "chr.png": ("spine.page", "png"), } - assert store.read(got["chr.skel"]["content"]["sha256"]) == b"\x01skel" + assert depot.read(got["chr.skel"]["content"]["sha256"]) == b"\x01skel" assert got["chr.skel"]["semantics"]["facts"]["atlases"] == ["chr.atlas"] (item,) = doc["items"] assert item == { @@ -523,7 +523,7 @@ def load_closure(paths): } sda = fake_skeleton(pages=2) # a writer error: a failed item, cached t2 = Task(t.stage, t.version, t.subject, {}, t.atoms, t.inputs[:3], t.context) # another key - assert execute(t2, store, registry(stage)).result_status == "partial" - doc2 = store.result(t2.key) + assert execute(t2, depot, registry(stage)).result_status == "partial" + doc2 = depot.result(t2.key) assert doc2["artifacts"] == [] and doc2["items"][0]["reason"]["code"] == "failed.export" assert "1 pages vs 2 materials" in doc2["items"][0]["reason"]["message"] diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index f4e9ad2..d0019dc 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -19,9 +19,9 @@ from nnnotes.pipeline.atoms import Impl, astc, estimate, gltf from nnnotes.pipeline.atoms.reader import UnityBundle from nnnotes.pipeline.contract import IncompatibleTask, Input, Task +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.objexport import CONVERTERS, CROP, EXPORT, ExportStage, SpriteCropStage, export_atoms from nnnotes.pipeline.stages import Env, describe, execute, registry -from nnnotes.pipeline.store import Store from tests.helpers.fakeunity import ( F32, NORMAL, @@ -573,9 +573,9 @@ def classes_of(name): return sorted({r.type.name for f in BUNDLES[name]().assets for r in f.objects.values()}) -def task_for(store, name, classes=None, params=None, scripts=None): +def task_for(depot, name, classes=None, params=None, scripts=None): data = f"fake:{name}".encode() - sha = store.put(data) + sha = depot.put(data) stage = ExportStage(FAKE_READER) p = stage.normalize(dict(params or {}, classes=classes)) return Task( @@ -584,15 +584,15 @@ def task_for(store, name, classes=None, params=None, scripts=None): name, p, export_atoms(classes_of(name), FAKE_READER.id), - (Input("bundle", sha, len(data), f"{name}.bundle", ({"kind": "store"},)),), + (Input("bundle", sha, len(data), f"{name}.bundle", ({"kind": "depot"},)),), {"scripts": dict(SCRIPTS if scripts is None else scripts)}, ) -def run(store, task): +def run(depot, task): stage = ExportStage(FAKE_READER) - execute(task, store, registry(stage), force=True) - return store.result(task.key) + execute(task, depot, registry(stage), force=True) + return depot.result(task.key) def by_object(doc): @@ -603,12 +603,12 @@ def art(doc, aid): return next(a for a in doc["artifacts"] if a["id"] == aid) -def load(store, doc, aid): - return store.read(art(doc, aid)["content"]["sha256"]) +def load(depot, doc, aid): + return depot.read(art(doc, aid)["content"]["sha256"]) -def obj(store, doc, aid): - return contract.loads(load(store, doc, aid)) +def obj(depot, doc, aid): + return contract.loads(load(depot, doc, aid)) def oid(pid, cab=CAB): @@ -617,14 +617,14 @@ def oid(pid, cab=CAB): @pytest.fixture def exported(tmp_path): - store = Store(tmp_path / "store") - task = task_for(store, "all") - return store, task, run(store, task) + depot = Depot(tmp_path / "depot") + task = task_for(depot, "all") + return depot, task, run(depot, task) # ---------------------------------------------------------------- items def test_every_object_has_exactly_one_item(exported): - store, task, doc = exported + depot, task, doc = exported objects = [contract.object_id(f.name, pid) for f in bundle_all().assets for pid in f.objects] items = by_object(doc) assert sorted(items) == sorted(objects) and len(doc["items"]) == len(objects) @@ -675,11 +675,11 @@ def test_every_object_has_exactly_one_item(exported): for a in doc["artifacts"]: owner, _role = contract.parse_artifact_id(a["id"]) assert ("object" in a["provenance"]) == (not owner.startswith("unity.export:")) - assert store.has(a["content"]["sha256"], a["content"]["size"]) + assert depot.has(a["content"]["sha256"], a["content"]["size"]) def test_provenance_facts_and_atoms(exported): - store, task, doc = exported + depot, task, doc = exported a = art(doc, f"{oid(10)}#image") assert a["provenance"]["object"] == { "file": CAB, @@ -706,8 +706,8 @@ def test_provenance_facts_and_atoms(exported): # ---------------------------------------------------------------- converters def test_textures(exported): - store, _task, doc = exported - png = load(store, doc, f"{oid(10)}#image") + depot, _task, doc = exported + png = load(depot, doc, f"{oid(10)}#image") img = Image.open(__import__("io").BytesIO(png)) assert img.size == (W, H) and img.mode == "RGBA" expected = Image.frombytes("RGBA", (W, H), TEX_PIXELS).transpose(Image.Transpose.FLIP_TOP_BOTTOM) @@ -725,15 +725,15 @@ def test_textures(exported): "exportedMips": [0], "mode": "RGBA", } - a = load(store, doc, f"{oid(15)}#image") + a = load(depot, doc, f"{oid(15)}#image") assert astc.parse_header(a) == {"block": (4, 4, 1), "size": (8, 8, 1)} and a[16:] == bytes(range(64)) assert art(doc, f"{oid(15)}#image")["content"]["ext"] == "astc" - empty = obj(store, doc, f"{oid(14)}#json") + empty = obj(depot, doc, f"{oid(14)}#json") assert empty["m_Width"] == 0 and empty["image data"] == {"$hex": ""} def test_sprites_same_bundle(exported): - store, _task, doc = exported + depot, _task, doc = exported from nnnotes.pipeline.atoms import png, sprite from nnnotes.pipeline.atoms.mesh import arrays from nnnotes.pipeline.atoms.reader import mesh_input_from_typetree @@ -754,11 +754,11 @@ def test_sprites_same_bundle(exported): 100.0, canvas={"rect": tt["m_Rect"], "offset": src["textureRectOffset"], "pivot": tt["m_Pivot"]}, ) - assert load(store, doc, f"{oid(pid)}#image") == png.encode(want) - meta = obj(store, doc, f"{oid(pid)}#meta") + assert load(depot, doc, f"{oid(pid)}#image") == png.encode(want) + meta = obj(depot, doc, f"{oid(pid)}#meta") assert meta["image"]["texture"] == {"$ref": {"file": CAB, "pathId": 10, "class": "Texture2D", "name": "tex"}} assert meta["sprite"]["m_Name"] == tt["m_Name"] - meta = obj(store, doc, f"{oid(12)}#meta") + meta = obj(depot, doc, f"{oid(12)}#meta") assert meta["image"]["source"] == "atlas" and meta["image"]["atlas"]["$ref"]["pathId"] == 20 assert meta["sprite"]["m_RD"]["m_VertexData"]["m_DataSize"]["$hex"] # small: inline refs = art(doc, f"{oid(12)}#image")["semantics"]["refs"] @@ -766,18 +766,18 @@ def test_sprites_same_bundle(exported): def test_sprite_with_its_atlas_elsewhere_gets_a_meta_only(exported): - store, _task, doc = exported + depot, _task, doc = exported assert by_object(doc)[oid(13)]["artifacts"] == [f"{oid(13)}#meta"] a = art(doc, f"{oid(13)}#meta") assert a["semantics"]["facts"] == {"crop": CROP, "atlas": f"{ATLAS_CAB}:99"} - meta = obj(store, doc, a["id"]) + meta = obj(depot, doc, a["id"]) assert meta["image"] == {"source": "atlas", "atlas": {"$ref": {"file": ATLAS_CAB, "pathId": 99}}, "stage": CROP} assert meta["object"]["name"] == "remote" and len(meta["mesh"]["positions"]) == 4 def test_mesh(exported): - store, _task, doc = exported - d, body = gltf.read(load(store, doc, f"{oid(30)}#mesh")) + depot, _task, doc = exported + d, body = gltf.read(load(depot, doc, f"{oid(30)}#mesh")) (prim,) = d["meshes"][0]["primitives"] assert set(prim["attributes"]) == {"POSITION", "NORMAL", "TEXCOORD_0"} and prim["mode"] == 4 pos = gltf.accessor_array(d, body, prim["attributes"]["POSITION"]) @@ -791,18 +791,18 @@ def test_mesh(exported): def test_text_and_fonts(exported): - store, _task, doc = exported + depot, _task, doc = exported a = art(doc, f"{oid(40)}#data") - assert load(store, doc, a["id"]) == b"hello\n" and a["content"]["ext"] == "txt" - assert load(store, doc, f"{oid(41)}#data") == b"\xff\xfe\x00bin" + assert load(depot, doc, a["id"]) == b"hello\n" and a["content"]["ext"] == "txt" + assert load(depot, doc, f"{oid(41)}#data") == b"\xff\xfe\x00bin" assert art(doc, f"{oid(41)}#data")["content"]["ext"] == "bin" f = art(doc, f"{oid(42)}#font") - assert load(store, doc, f["id"]) == _ttf() and f["content"]["ext"] == "ttf" + assert load(depot, doc, f["id"]) == _ttf() and f["content"]["ext"] == "ttf" def test_prefab(exported): - store, _task, doc = exported - p = obj(store, doc, f"{oid(60)}#prefab") + depot, _task, doc = exported + p = obj(depot, doc, f"{oid(60)}#prefab") assert p["name"] == "root" and [n["path"] for n in p["nodes"]] == ["root", "root/child"] root, child = p["nodes"] assert root["pathId"] == 60 and root["transformPathId"] == 61 and root["layer"] == 5 @@ -813,9 +813,9 @@ def test_prefab(exported): assert fields["tex"] == {"$ref": {"file": CAB, "pathId": 10, "class": "Texture2D", "name": "tex"}} assert fields["m_Script"] == {"$ref": {"file": "CAB-scripts", "pathId": 7}} assert math.isnan(fields["value"]) and fields["small"] == {"$hex": "616263"} - assert b'{"$float": "nan"}' in load(store, doc, f"{oid(60)}#prefab").replace(b"\n", b"").replace( + assert b'{"$float": "nan"}' in load(depot, doc, f"{oid(60)}#prefab").replace(b"\n", b"").replace( b" ", b"" - ) or b'"$float"' in load(store, doc, f"{oid(60)}#prefab") + ) or b'"$float"' in load(depot, doc, f"{oid(60)}#prefab") (c,) = child["components"] assert c["fields"]["up"] == {"transform": "root"} assert c["fields"]["comp"] == {"component": "MonoBehaviour", "gameObject": "root", "class": "Behaviour"} @@ -824,8 +824,8 @@ def test_prefab(exported): def test_scriptable_objects_blobs_and_strings(exported): - store, _task, doc = exported - m = obj(store, doc, f"{oid(70)}#json") + depot, _task, doc = exported + m = obj(depot, doc, f"{oid(70)}#json") assert m["$script"] == "A|Game|Local" and m["m_Name"] == "moc" blob = m["_bytes"]["$blob"] assert blob == {"sha256": contract.sha256(MOC), "size": len(MOC), "format": "moc3"} @@ -835,19 +835,19 @@ def test_scriptable_objects_blobs_and_strings(exported): "format": "moc3", "facts": {"field": "_bytes"}, } - assert load(store, doc, b["id"]) == MOC + assert load(depot, doc, b["id"]) == MOC assert m["text"] == {"$hex": b"caf".hex() + "c3"} assert by_object(doc)[oid(70)]["artifacts"] == [f"{oid(70)}#blob:_bytes", f"{oid(70)}#json"] - lost = obj(store, doc, f"{oid(71)}#json") + lost = obj(depot, doc, f"{oid(71)}#json") assert "$script" not in lost and lost["m_Script"] == {"$ref": {"file": "CAB-scripts", "pathId": 9}} def test_cue_sheets_held_in_the_bundle(exported): - store, _task, doc = exported + depot, _task, doc = exported from nnnotes.formats import cri a = art(doc, f"{oid(73)}#acb") - assert load(store, doc, a["id"]) == SPLIT_ACB == cri.split_acb(SPLIT_CHUNKS) + assert load(depot, doc, a["id"]) == SPLIT_ACB == cri.split_acb(SPLIT_CHUNKS) assert a["content"]["ext"] == "acb" and a["semantics"] == { "kind": "cri.acb", "format": "acb", @@ -855,25 +855,25 @@ def test_cue_sheets_held_in_the_bundle(exported): "refs": [{"rel": "chunk", "object": oid(44)}, {"rel": "chunk", "object": oid(45)}], } assert by_object(doc)[oid(73)]["artifacts"] == [f"{oid(73)}#acb", f"{oid(73)}#json"] - assert obj(store, doc, f"{oid(73)}#json")["$script"] == "Assembly-CSharp|Fwk.Sound|SplitAcbData" - assert load(store, doc, f"{oid(44)}#data") == SPLIT_CHUNKS[0] # the chunks stay exported + assert obj(depot, doc, f"{oid(73)}#json")["$script"] == "Assembly-CSharp|Fwk.Sound|SplitAcbData" + assert load(depot, doc, f"{oid(44)}#data") == SPLIT_CHUNKS[0] # the chunks stay exported e = art(doc, f"{oid(74)}#acb") - assert load(store, doc, e["id"]) == EMBEDDED_ACB + assert load(depot, doc, e["id"]) == EMBEDDED_ACB assert e["semantics"]["facts"] == {"cueSheet": "se_test", "layout": "embedded"} - m = obj(store, doc, f"{oid(74)}#json") + m = obj(depot, doc, f"{oid(74)}#json") ref = m["references"]["RefIds"][0] assert ref["type"] == {"class": IMPL[2], "ns": IMPL[1], "asm": IMPL[0]} assert ref["data"]["data"] == { "$blob": {"sha256": contract.sha256(EMBEDDED_ACB), "size": len(EMBEDDED_ACB), "format": "acb"} } assert by_object(doc)[oid(74)]["artifacts"] == [f"{oid(74)}#acb", f"{oid(74)}#json"] # no blob: artifact - small = run(store, task_for(store, "all", classes=["MonoBehaviour"], params={"blobMin": 1 << 20})) - assert load(store, small, f"{oid(74)}#acb") == EMBEDDED_ACB # whatever its size + small = run(depot, task_for(depot, "all", classes=["MonoBehaviour"], params={"blobMin": 1 << 20})) + assert load(depot, small, f"{oid(74)}#acb") == EMBEDDED_ACB # whatever its size -def movie_task(store, converter=True): +def movie_task(depot, converter=True): """The task of the movie bundle: converter cri.usm in its atoms, as uses() gives it for that bundle.""" - t = task_for(store, "movie", scripts=USM_SCRIPTS) + t = task_for(depot, "movie", scripts=USM_SCRIPTS) atoms = {**t.atoms, "cri.usm": "cri.usm/1"} if converter else t.atoms return Task(t.stage, t.version, t.subject, t.params, dict(sorted(atoms.items())), t.inputs, t.context) @@ -882,33 +882,33 @@ def test_movies_held_in_the_bundle(tmp_path): """A movie asset's USM bytes (CriSerializedBytesAssetImpl) are its `usm` artifact whatever their size; the cue sheet beside it keeps its `acb`. Without converter cri.usm the movie asset is exported as before (its bytes are not an ACB: the typetree with generic.error).""" - store = Store(tmp_path / "s") - doc = run(store, movie_task(store)) + depot = Depot(tmp_path / "s") + doc = run(depot, movie_task(depot)) m, a = "CAB-movie:2", "CAB-movie:3" usm = art(doc, f"{m}#usm") - assert load(store, doc, usm["id"]) == WARMUP and usm["content"]["ext"] == "usm" + assert load(depot, doc, usm["id"]) == WARMUP and usm["content"]["ext"] == "usm" assert usm["semantics"] == {"kind": "cri.usm", "format": "usm", "facts": {"movie": "warmup"}} - tree = obj(store, doc, f"{m}#json") + tree = obj(depot, doc, f"{m}#json") assert tree["$script"] == objexport.USM_ASSET_SCRIPT and tree["movieInfo"] == MOVIE_INFO assert tree["references"]["RefIds"][0]["data"]["data"] == { "$blob": {"sha256": contract.sha256(WARMUP), "size": len(WARMUP), "format": "usm"} } assert by_object(doc)[m]["status"] == "exported" and by_object(doc)[m]["artifacts"] == [f"{m}#json", f"{m}#usm"] - assert load(store, doc, f"{a}#acb") == EMBEDDED_ACB - old = run(store, movie_task(store, converter=False)) + assert load(depot, doc, f"{a}#acb") == EMBEDDED_ACB + old = run(depot, movie_task(depot, converter=False)) assert by_object(old)[m]["status"] == "generic" and by_object(old)[m]["reason"]["code"] == "generic.error" - assert f"{m}#usm" not in {x["id"] for x in old["artifacts"]} and load(store, old, f"{a}#acb") == EMBEDDED_ACB + assert f"{m}#usm" not in {x["id"] for x in old["artifacts"]} and load(depot, old, f"{a}#acb") == EMBEDDED_ACB def test_the_movie_converter_is_keyed_by_the_script_context(tmp_path): - store = Store(tmp_path / "s") - env = planning_env(store, ("all", "movie"), scripts=USM_SCRIPTS) + depot = Depot(tmp_path / "s") + env = planning_env(depot, ("all", "movie"), scripts=USM_SCRIPTS) stage = ExportStage(FAKE_READER) assert describe(stage, "movie", None, env).atoms["cri.usm"] == "cri.usm/1" assert "cri.usm" not in describe(stage, "all", None, env).atoms assert "cri.usm" not in describe(stage, "movie", {"classes": ["AssetBundle"]}, env).atoms assert "cri.usm" in stage.ATOMS - plain = planning_env(Store(tmp_path / "p"), ("movie",)) + plain = planning_env(Depot(tmp_path / "p"), ("movie",)) assert "cri.usm" not in describe(stage, "movie", None, plain).atoms @@ -949,7 +949,7 @@ def test_acb_layouts(tmp_path): def test_a_split_sheet_with_a_missing_chunk_gives_the_typetree(tmp_path, monkeypatch): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") real = objexport.acb_asset monkeypatch.setattr( objexport, @@ -958,21 +958,21 @@ def test_a_split_sheet_with_a_missing_chunk_gives_the_typetree(tmp_path, monkeyp dict(real(tt), chunks=[pptr(44), pptr(99, 1)]) if real(tt) and real(tt)["layout"] == "split" else real(tt) ), ) - doc = run(store, task_for(store, "all", classes=["MonoBehaviour"])) + doc = run(depot, task_for(depot, "all", classes=["MonoBehaviour"])) item = by_object(doc)[oid(73)] assert item["status"] == "generic" and item["reason"]["code"] == "generic.error" assert "is not a TextAsset of the bundle" in item["reason"]["message"] def test_shader(exported): - store, _task, doc = exported + depot, _task, doc = exported from nnnotes.formats import shader env = bundle_all() r = env.assets[0].objects[51] text, recs, codes = shader._render(r, r.tt, "Test/Unlit") # what `nnnotes shader` writes - assert load(store, doc, f"{oid(51)}#json") == text.encode("utf-8") - index = obj(store, doc, f"{oid(51)}#index") + assert load(depot, doc, f"{oid(51)}#json") == text.encode("utf-8") + index = obj(depot, doc, f"{oid(51)}#index") assert ( index["name"] == "Test/Unlit" and [{k: v for k, v in x.items() if k != "role"} for x in index["variants"]] == recs @@ -980,10 +980,10 @@ def test_shader(exported): assert [x["role"] for x in index["variants"]] == ["program:gles3/s0p0_vertex_0", "program:gles3/s0p0_fragment_0"] for x, code in zip(index["variants"], codes): a = art(doc, f"{oid(51)}#{x['role']}") - assert load(store, doc, a["id"]) == code and a["content"]["ext"] == "glsl" + assert load(depot, doc, a["id"]) == code and a["content"]["ext"] == "glsl" assert a["semantics"]["facts"]["keywords"] == x["keywords"] assert codes[0] == b"#ifdef VERTEX\nvoid main(){}\n#endif\n" and index["variants"][0]["keywords"] == ["KW_A"] - t = obj(store, doc, f"{oid(51)}#typetree") + t = obj(depot, doc, f"{oid(51)}#typetree") assert t["m_ParsedForm"]["m_Name"] == "Test/Unlit" and "$hex" in t["compressedBlob"] assert art(doc, f"{oid(51)}#json")["semantics"]["facts"] == { "name": "Test/Unlit", @@ -993,33 +993,33 @@ def test_shader(exported): def test_clip_bindings_name_scripts_of_other_bundles(exported): - store, _task, doc = exported - clip = obj(store, doc, f"{oid(96)}#json") + depot, _task, doc = exported + clip = obj(depot, doc, f"{oid(96)}#json") (b,) = clip["bindings"] assert b["typeID"] == 114 and b["class"] == "Fader" and b["path"] == "" assert census_mod.script_refs(census_doc(open_fake(b"fake:all"))) == sorted(SCRIPTS) + ["CAB-scripts:9"] def test_material_clip_controller(exported): - store, _task, doc = exported - mat = obj(store, doc, f"{oid(50)}#json") + depot, _task, doc = exported + mat = obj(depot, doc, f"{oid(50)}#json") assert mat["shader"] == {"$ref": {"file": ATLAS_CAB, "pathId": 5}} assert mat["textures"]["_MainTex"]["texture"]["$ref"]["pathId"] == 10 - assert mat["colors"]["_Color"]["g"] == INF and b"1e999" in load(store, doc, f"{oid(50)}#json") - clip = obj(store, doc, f"{oid(90)}#json") + assert mat["colors"]["_Color"]["g"] == INF and b"1e999" in load(depot, doc, f"{oid(50)}#json") + clip = obj(depot, doc, f"{oid(90)}#json") assert clip["clip"] == "clip" and clip["bindings"] == [ {"path": "", "typeID": 4, "class": "Transform", "attribute": "m_LocalPosition", "curves": 3} ] - ctl = obj(store, doc, f"{oid(92)}#json") + ctl = obj(depot, doc, f"{oid(92)}#json") assert ctl["clips"] == [{"$ref": {"file": CAB, "pathId": 90, "class": "AnimationClip", "name": "clip"}}] - assert obj(store, doc, f"{oid(95)}#json")["m_Speed"] == INF + assert obj(depot, doc, f"{oid(95)}#json")["m_Speed"] == INF def test_loose_documents(exported): - store, task, doc = exported - s = obj(store, doc, "unity.export:all#scripts.json") + depot, task, doc = exported + s = obj(depot, doc, "unity.export:all#scripts.json") assert list(s["scripts"]) == [oid(80)] and s["scripts"][oid(80)]["m_ClassName"] == "Local" - b = obj(store, doc, "unity.export:all#bundle.json") + b = obj(depot, doc, "unity.export:all#bundle.json") assert b["files"] == [ { "name": CAB, @@ -1039,7 +1039,7 @@ def test_loose_documents(exported): def test_two_runs_are_identical(tmp_path): - a, b = Store(tmp_path / "a"), Store(tmp_path / "b") + a, b = Depot(tmp_path / "a"), Depot(tmp_path / "b") ta, tb = task_for(a, "all"), task_for(b, "all") assert ta.key == tb.key da, db = run(a, ta), run(b, tb) @@ -1049,11 +1049,11 @@ def test_two_runs_are_identical(tmp_path): def test_key_follows_the_census_classes_and_params(tmp_path): - store = Store(tmp_path / "s") - t = task_for(store, "atlas") + depot = Depot(tmp_path / "s") + t = task_for(depot, "atlas") assert "mesh.glb" not in t.atoms and "mesh.arrays" in t.atoms # sprites use mesh.arrays assert {"tex.png", "sprite.png", "generic.json", "bundle.json"} <= set(t.atoms) - assert task_for(store, "atlas", params={"png": {"level": 3}}).key != t.key + assert task_for(depot, "atlas", params={"png": {"level": 3}}).key != t.key assert export_atoms(["Texture2D"]) == export_atoms(["Texture2D", "Texture2D"]) assert set(export_atoms(["Mesh"])) == {"reader", "sniff", "mesh.arrays", "gltf.write", "mesh.glb", "generic.json"} with pytest.raises(ValueError, match="png level"): @@ -1064,66 +1064,66 @@ def test_key_follows_the_census_classes_and_params(tmp_path): def test_png_level_changes_bytes_not_pixels(tmp_path): - store = Store(tmp_path / "s") - d6 = run(store, task_for(store, "all", classes=["Texture2D"])) - d0 = run(store, task_for(store, "all", classes=["Texture2D"], params={"png": {"level": 0}})) - p6, p0 = load(store, d6, f"{oid(10)}#image"), load(store, d0, f"{oid(10)}#image") + depot = Depot(tmp_path / "s") + d6 = run(depot, task_for(depot, "all", classes=["Texture2D"])) + d0 = run(depot, task_for(depot, "all", classes=["Texture2D"], params={"png": {"level": 0}})) + p6, p0 = load(depot, d6, f"{oid(10)}#image"), load(depot, d0, f"{oid(10)}#image") assert p6 != p0 io = __import__("io") assert Image.open(io.BytesIO(p6)).tobytes() == Image.open(io.BytesIO(p0)).tobytes() def test_blob_threshold(tmp_path): - store = Store(tmp_path / "s") - doc = run(store, task_for(store, "all", classes=["MonoBehaviour"], params={"blobMin": 1 << 20})) - m = obj(store, doc, f"{oid(70)}#json") + depot = Depot(tmp_path / "s") + doc = run(depot, task_for(depot, "all", classes=["MonoBehaviour"], params={"blobMin": 1 << 20})) + m = obj(depot, doc, f"{oid(70)}#json") assert m["_bytes"] == {"$hex": MOC.hex()} def test_class_selection(tmp_path): - store = Store(tmp_path / "s") - doc = run(store, task_for(store, "all", classes=["Sprite"])) + depot = Depot(tmp_path / "s") + doc = run(depot, task_for(depot, "all", classes=["Sprite"])) items = by_object(doc) assert set(items) == {oid(11), oid(12), oid(13)} # textures decoded, not exported assert all(i["status"] == "exported" for i in items.values()) - doc = run(store, task_for(store, "all", classes=["GameObject"])) + doc = run(depot, task_for(depot, "all", classes=["GameObject"])) assert set(by_object(doc)) == {oid(p) for p in (60, 61, 62, 63, 64, 65)} def test_scripts_come_from_the_context_only(tmp_path): - store = Store(tmp_path / "s") - doc = run(store, task_for(store, "all", classes=["MonoBehaviour"], scripts={})) + depot = Depot(tmp_path / "s") + doc = run(depot, task_for(depot, "all", classes=["MonoBehaviour"], scripts={})) items = by_object(doc) assert items[oid(70)]["reason"]["code"] == "script.unresolved" assert items[oid(62)]["reason"]["code"] == "script.unresolved" # no hierarchy: one by one def test_a_converter_error_gives_the_typetree(tmp_path, monkeypatch): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") real = objexport.RefExporter.conv_material def broken(self, info): raise KeyError("m_SavedProperties") monkeypatch.setattr(objexport.RefExporter, "conv_material", broken) - doc = run(store, task_for(store, "all", classes=["Material"])) + doc = run(depot, task_for(depot, "all", classes=["Material"])) item = by_object(doc)[oid(50)] assert item["status"] == "generic" and item["reason"] == { "code": "generic.error", "message": "KeyError: 'm_SavedProperties'", } - assert obj(store, doc, f"{oid(50)}#json")["m_Name"] == "mat" + assert obj(depot, doc, f"{oid(50)}#json")["m_Name"] == "mat" monkeypatch.setattr(objexport.RefExporter, "conv_material", real) def test_a_broken_hierarchy_gives_its_objects_one_by_one(tmp_path, monkeypatch): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") def broken(self, o): raise RuntimeError("component failed") monkeypatch.setattr(objexport.RefExporter, "component", broken) - doc = run(store, task_for(store, "all", classes=["GameObject"])) + doc = run(depot, task_for(depot, "all", classes=["GameObject"])) items = by_object(doc) assert {i["status"] for i in items.values()} == {"generic"} assert all(i["reason"]["code"] == "generic.error" for i in items.values()) @@ -1131,7 +1131,7 @@ def broken(self, o): def test_unreadable_objects_fail(tmp_path, monkeypatch): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") real = UnityBundle.typetree def typetree(self, o): @@ -1140,17 +1140,17 @@ def typetree(self, o): return real(self, o) monkeypatch.setattr(UnityBundle, "typetree", typetree) - doc = run(store, task_for(store, "all", classes=["Animator"])) + doc = run(depot, task_for(depot, "all", classes=["Animator"])) (item,) = doc["items"] assert item["status"] == "failed" and item["reason"]["code"] == "failed.read" and doc["status"] == "partial" def test_task_with_other_atoms_is_refused(tmp_path): - store = Store(tmp_path / "s") - t = task_for(store, "all") + depot = Depot(tmp_path / "s") + t = task_for(depot, "all") bad = Task(t.stage, t.version, t.subject, t.params, {**t.atoms, "tex.png": "tex.png/0"}, t.inputs, t.context) with pytest.raises(IncompatibleTask, match="tex.png"): - execute(bad, store, registry(ExportStage(FAKE_READER))) + execute(bad, depot, registry(ExportStage(FAKE_READER))) missing = Task( t.stage, t.version, @@ -1161,7 +1161,7 @@ def test_task_with_other_atoms_is_refused(tmp_path): t.context, ) with pytest.raises(IncompatibleTask, match="mesh.glb"): - execute(missing, store, registry(ExportStage(FAKE_READER))) + execute(missing, depot, registry(ExportStage(FAKE_READER))) # ---------------------------------------------------------------- describing tasks @@ -1206,57 +1206,57 @@ def ref(s): } -def committed(store, env, task, role, doc, kind): +def committed(depot, env, task, role, doc, kind): rec = contract.artifact( contract.artifact_id(task.id, role), - store.add(contract.encode(doc), "json"), + depot.add(contract.encode(doc), "json"), contract.provenance(task), {"kind": kind, "format": "json", "facts": {}}, ) - store.commit(contract.result(task, [rec], [])) + depot.commit(contract.result(task, [rec], [])) env.done(task.id, task.key) -def planning_env(store, names=("all", "atlas"), selected=None, scripts=SCRIPTS): +def planning_env(depot, names=("all", "atlas"), selected=None, scripts=SCRIPTS): names = tuple(sorted(set(names) | set(selected or ()))) bundles = {} for n in names: data = f"fake:{n}".encode() - bundles[n] = Input("bundle", store.put(data), len(data), f"{n}.bundle", ({"kind": "store"},)) - env = Env(store, {"bundles": bundles, "selected": sorted(selected if selected is not None else names)}) + bundles[n] = Input("bundle", depot.put(data), len(data), f"{n}.bundle", ({"kind": "depot"},)) + env = Env(depot, {"bundles": bundles, "selected": sorted(selected if selected is not None else names)}) for n in names: t = Task(census_mod.CensusStage.name, 1, n, {}, {}, (bundles[n],)) doc = census_doc(open_fake(f"fake:{n}".encode())) rec = contract.artifact( contract.artifact_id(t.id, "census"), - store.add(contract.encode(doc), "json"), + depot.add(contract.encode(doc), "json"), contract.provenance(t), {"kind": "unity.census", "format": "json", "facts": census_mod.facts(doc)}, ) - store.commit(contract.result(t, [rec], [])) + depot.commit(contract.result(t, [rec], [])) env.done(t.id, t.key) table = Task("link.scripts", 1, "all") - committed(store, env, table, "scripts", {"schema": "nnnotes.scripts/1", "scripts": scripts}, "link.scripts") + committed(depot, env, table, "scripts", {"schema": "nnnotes.scripts/1", "scripts": scripts}, "link.scripts") return env def test_describe_reads_the_census_and_the_script_table(tmp_path): - store = Store(tmp_path / "s") - env = planning_env(store) + depot = Depot(tmp_path / "s") + env = planning_env(depot) stage = ExportStage(FAKE_READER) assert stage.subjects(env) == ["all", "atlas"] # every closure bundle holding a SpriteAtlas joins a selection that holds sprites (a sprite reaches its # texture through its atlas' bundle); a selection without sprites stays as it is - assert stage.subjects(planning_env(Store(tmp_path / "t"), selected=["all"])) == ["all", "atlas"] - assert stage.subjects(planning_env(Store(tmp_path / "u"), selected=["plain"])) == ["plain"] + assert stage.subjects(planning_env(Depot(tmp_path / "t"), selected=["all"])) == ["all", "atlas"] + assert stage.subjects(planning_env(Depot(tmp_path / "u"), selected=["plain"])) == ["plain"] t = describe(stage, "all", None, env) assert t.atoms == export_atoms(classes_of("all"), FAKE_READER.id) - assert t.context == {"scripts": SCRIPTS} and t.key == task_for(store, "all").key + assert t.context == {"scripts": SCRIPTS} and t.key == task_for(depot, "all").key assert stage.depends("all", env) == ["unity.census:all", "link.scripts:all"] a = describe(stage, "atlas", None, env) assert a.context == {} and "mesh.glb" not in a.atoms assert t.cost.peak_bytes >= 72 << 20 - fresh = Env(store, env.facts) + fresh = Env(depot, env.facts) from nnnotes.pipeline.stages import Pending fresh.waiting("unity.census:all") @@ -1265,11 +1265,11 @@ def test_describe_reads_the_census_and_the_script_table(tmp_path): def test_class_selection_keeps_only_its_converters_in_the_key(tmp_path): - store = Store(tmp_path / "s") - env = planning_env(store) + depot = Depot(tmp_path / "s") + env = planning_env(depot) stage = ExportStage(FAKE_READER) full = describe(stage, "all", None, env) - assert full.key == describe(stage, "all", {"classes": None}, env).key == task_for(store, "all").key + assert full.key == describe(stage, "all", {"classes": None}, env).key == task_for(depot, "all").key meshes = describe(stage, "all", {"classes": ["Mesh"]}, env) assert meshes.atoms == export_atoms(["Mesh"], FAKE_READER.id) and meshes.params["classes"] == ["Mesh"] assert "tex.png" not in meshes.atoms and meshes.key != full.key @@ -1278,45 +1278,45 @@ def test_class_selection_keeps_only_its_converters_in_the_key(tmp_path): assert env.params == {} # set while describing only for classes in (["Mesh"], ["Sprite"], ["GameObject"], ["Material", "AnimationClip"]): t = describe(stage, "all", {"classes": classes}, env) - execute(t, store, registry(stage)) - doc = store.result(t.key) + execute(t, depot, registry(stage)) + doc = depot.result(t.key) assert doc["items"] and all(i["class"] in {*classes, "Transform", "MonoBehaviour"} for i in doc["items"]) -def crop_setup(store): - env = planning_env(store) +def crop_setup(depot): + env = planning_env(depot) stage = ExportStage(FAKE_READER) for n in ("all", "atlas"): t = describe(stage, n, None, env) - execute(t, store, registry(stage)) + execute(t, depot, registry(stage)) env.done(t.id, t.key) return env def test_sprite_crop_equals_the_same_sprite_cropped_in_its_atlas_bundle(tmp_path): - store = Store(tmp_path / "s") - env = crop_setup(store) + depot = Depot(tmp_path / "s") + env = crop_setup(depot) stage = SpriteCropStage() assert stage.subjects(env) == ["atlas:99"] t = describe(stage, "atlas:99", None, env) assert [i.role for i in t.inputs] == ["atlas", f"sprite:{oid(13)}", f"texture:{ATLAS_CAB}:98"] assert stage.depends("atlas:99", env) == ["unity.export:all", "unity.export:atlas"] assert set(t.atoms) == {"png.encode", "sprite.crop"} - execute(t, store, registry(stage)) - doc = store.result(t.key) + execute(t, depot, registry(stage)) + doc = depot.result(t.key) (item,) = doc["items"] assert item == {"object": oid(13), "status": "exported", "artifacts": [f"{oid(13)}#image"], "class": "Sprite"} a = doc["artifacts"][0] assert a["provenance"]["object"]["name"] == "remote" and a["provenance"]["object"]["file"] == CAB - local = store.result(env.key("unity.export:atlas")) + local = depot.result(env.key("unity.export:atlas")) same = art(local, f"{ATLAS_CAB}:97#image") assert a["content"] == same["content"] # the same bytes as a local crop - assert Image.open(__import__("io").BytesIO(store.read(a["content"]["sha256"]))).size == (8, 8) + assert Image.open(__import__("io").BytesIO(depot.read(a["content"]["sha256"]))).size == (8, 8) def test_sprite_crop_waits_for_exports(tmp_path): - store = Store(tmp_path / "s") - env = planning_env(store) + depot = Depot(tmp_path / "s") + env = planning_env(depot) env.waiting("unity.export:all") from nnnotes.pipeline.stages import Pending @@ -1377,7 +1377,7 @@ def export_golden(): classes the converter takes), and the whole bundle.""" def fixture(classes): - return lambda store: task_for(store, "all", classes=classes) + return lambda depot: task_for(depot, "all", classes=classes) fixtures = {name: fixture(classes) for name, classes in CONVERTER_CLASSES.items()} fixtures["all"] = fixture(None) @@ -1388,8 +1388,8 @@ def fixture(classes): def crop_golden(): """Provider of tests/golden/sprite.crop.json.""" - def fixture(store): - env = crop_setup(store) + def fixture(depot): + env = crop_setup(depot) return describe(SpriteCropStage(), "atlas:99", None, env) return {"stage": SpriteCropStage(), "fixtures": {"atlas": fixture}, "libraries": LIBRARIES} @@ -1416,7 +1416,7 @@ def changed(self, info): monkeypatch.setattr(objexport.RefExporter, "conv_font", changed) p = export_golden() - actual = golden(p["stage"], p["fixtures"], Store(tmp_path / "s"), p["libraries"]) + actual = golden(p["stage"], p["fixtures"], Depot(tmp_path / "s"), p["libraries"]) problems = golden_problems(expected, actual, p["stage"]) assert "unity.export: fixture font: output changed without a version bump" in problems assert "unity.export: fixture all: output changed without a version bump" in problems diff --git a/tests/pipeline/test_orchestrate.py b/tests/pipeline/test_orchestrate.py index a3e586a..a4cee60 100644 --- a/tests/pipeline/test_orchestrate.py +++ b/tests/pipeline/test_orchestrate.py @@ -7,10 +7,10 @@ from nnnotes.pipeline import contract from nnnotes.pipeline.contract import Cost, Task +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.orchestrate import Orchestrator, Scheduler, coverage from nnnotes.pipeline.plan import plan from nnnotes.pipeline.stages import execute -from nnnotes.pipeline.store import Store from tests.pipeline.test_stages import PIPELINE, toy_files, toy_stages TEXTS = { @@ -34,11 +34,11 @@ def tree(root: Path, *subs) -> dict: def orchestrate(tmp_path, name, texts, workers=0, stages=None, force=(), **kw): - store = Store(tmp_path / name) + depot = Depot(tmp_path / name) files = toy_files(tmp_path / "in", texts) - orch = Orchestrator(store, stages or toy_stages(), workers=workers, log=lambda m: None, **kw) + orch = Orchestrator(depot, stages or toy_stages(), workers=workers, log=lambda m: None, **kw) run = orch.run(PIPELINE, facts={"files": files}, context={"region": "test"}, selection=["all"], force=force) - return store, run, orch + return depot, run, orch def test_one_worker_and_four_workers_give_identical_stores(tmp_path): @@ -56,7 +56,7 @@ def test_one_worker_and_four_workers_give_identical_stores(tmp_path): def test_run_stage_task_by_task_equals_the_orchestrated_store(tmp_path): s0, _, _ = orchestrate(tmp_path, "orch", TEXTS) - s1 = Store(tmp_path / "tasks-store") + s1 = Depot(tmp_path / "tasks-depot") stages = toy_stages() facts = {"files": toy_files(tmp_path / "in", TEXTS)} rounds = 0 @@ -73,7 +73,7 @@ def test_run_stage_task_by_task_equals_the_orchestrated_store(tmp_path): def test_item_failures_are_cached_task_failures_are_not(tmp_path): texts = {"a": "x\nFAIL", "b": "CRASH", "c": "y"} - store, run, _ = orchestrate(tmp_path, "s", texts) + depot, run, _ = orchestrate(tmp_path, "s", texts) st = {t: v["status"] for t, v in run.tasks.items()} assert st == { "toy.src:a": "ran", @@ -92,7 +92,7 @@ def test_item_failures_are_cached_task_failures_are_not(tmp_path): ("toy.src:b", None, "task.error"), ] assert run.exit_code == 1 - total = store.result(run.tasks["toy.total:all"]["key"]) + total = depot.result(run.tasks["toy.total:all"]["key"]) assert [i[0] for i in total["keyParts"]["inputs"]] == ["count:a", "count:c"] _, again, _ = orchestrate(tmp_path, "s", texts) st = {t: v["status"] for t, v in again.tasks.items()} @@ -101,7 +101,7 @@ def test_item_failures_are_cached_task_failures_are_not(tmp_path): def test_a_dying_worker_fails_its_task_only(tmp_path): - store, run, orch = orchestrate(tmp_path, "s", {"a": "x", "d": "DIE", "c": "y"}, workers=2) + depot, run, orch = orchestrate(tmp_path, "s", {"a": "x", "d": "DIE", "c": "y"}, workers=2) assert run.tasks["toy.src:d"]["status"] == "failed" assert [f["code"] for f in run.failures() if f["task"] == "toy.src:d"] == ["task.worker_died"] assert run.tasks["toy.src:a"]["status"] == run.tasks["toy.src:c"]["status"] == "ran" @@ -110,13 +110,13 @@ def test_a_dying_worker_fails_its_task_only(tmp_path): def test_equal_content_under_two_subjects_is_stored_once(tmp_path): - store, run, _ = orchestrate(tmp_path, "s", {"a": "same", "b": "same"}, workers=2) + depot, run, _ = orchestrate(tmp_path, "s", {"a": "same", "b": "same"}, workers=2) assert all(v["status"] == "ran" for v in run.tasks.values()) assert run.tasks["toy.src:a"]["key"] != run.tasks["toy.src:b"]["key"] # the subject names artifacts - ra, rb = (store.result(run.tasks[f"toy.src:{s}"]["key"]) for s in "ab") + ra, rb = (depot.result(run.tasks[f"toy.src:{s}"]["key"]) for s in "ab") assert [a["id"] for a in rb["artifacts"]] == ["F-b:1#data", "toy.src:b#all.txt"] assert {a["content"]["sha256"] for a in ra["artifacts"]} == {a["content"]["sha256"] for a in rb["artifacts"]} - assert len(tree(store.root, "cas")) == 3 # SAME, the two equal counts, the total + assert len(tree(depot.root, "cas")) == 3 # SAME, the two equal counts, the total def test_no_more_workers_start_than_a_stage_has_tasks(tmp_path): @@ -206,7 +206,7 @@ def test_coverage_and_failures_documents(tmp_path): def test_unknown_or_misordered_stages_are_refused(tmp_path): - orch = Orchestrator(Store(tmp_path / "s"), toy_stages(), log=lambda m: None) + orch = Orchestrator(Depot(tmp_path / "s"), toy_stages(), log=lambda m: None) with pytest.raises(ValueError, match="unknown stages"): orch.run(["toy.nope"], facts={"files": {}}) with pytest.raises(ValueError, match="runs before toy.src"): @@ -218,12 +218,12 @@ def test_a_config_error_stops_the_run(tmp_path): from tests.pipeline.test_stages import Src class NeedsTool(Src): - def run(self, task, store): + def run(self, task, depot): raise ConfigError("setting paths.tool is not set") stages = toy_stages() stages["toy.src"] = NeedsTool() - orch = Orchestrator(Store(tmp_path / "s"), stages, log=lambda m: None) + orch = Orchestrator(Depot(tmp_path / "s"), stages, log=lambda m: None) with pytest.raises(ConfigError, match="paths.tool"): orch.run(PIPELINE, facts={"files": toy_files(tmp_path / "in", {"a": "x", "b": "y"})}) @@ -239,7 +239,7 @@ def subjects(self, env): return ["x"] stages = dict(toy_stages(), **{"toy.after": AfterAll()}) - orch = Orchestrator(Store(tmp_path / "s"), stages, log=lambda m: None) + orch = Orchestrator(Depot(tmp_path / "s"), stages, log=lambda m: None) run = orch.run(["toy.src", "toy.after"], facts={"files": toy_files(tmp_path / "in", {"a": "x", "b": "CRASH"})}) assert run.tasks["toy.after:*"]["status"] == "failed" assert [f["code"] for f in run.failures() if f["task"] == "toy.after:*"] == ["upstream.failed"] @@ -333,10 +333,10 @@ def test_compact_coverage_equals_a_reading_of_whole_items(): def test_the_reports_read_the_results_once_until_the_tasks_change(tmp_path): - store, run, _ = orchestrate(tmp_path, "s", {"a": "x\nFAIL\n?y\n", "b": "z"}) + depot, run, _ = orchestrate(tmp_path, "s", {"a": "x\nFAIL\n?y\n", "b": "z"}) reads = [] - real = store.result - store.result = lambda key: reads.append(key) or real(key) + real = depot.result + depot.result = lambda key: reads.append(key) or real(key) first = (run.summary(), run.failures(), run.coverage()) n = len(reads) assert n == sum(t["status"] != "failed" for t in run.tasks.values()) diff --git a/tests/pipeline/test_plan.py b/tests/pipeline/test_plan.py index 177e017..df9bbe5 100644 --- a/tests/pipeline/test_plan.py +++ b/tests/pipeline/test_plan.py @@ -4,22 +4,22 @@ from nnnotes.pipeline import contract from nnnotes.pipeline.contract import Task +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.orchestrate import Orchestrator from nnnotes.pipeline.plan import plan, why -from nnnotes.pipeline.store import Store from tests.pipeline.test_stages import PIPELINE, toy_files, toy_stages TEXTS = {"a": "alpha\nbeta", "b": "gamma", "c": "delta\n?x"} def setup(tmp_path, texts=TEXTS): - store = Store(tmp_path / "store") + depot = Depot(tmp_path / "depot") facts = {"files": toy_files(tmp_path / "in", texts)} - return store, facts + return depot, facts -def run(store, facts, stages=None, params=None, selection=("all",)): - r = Orchestrator(store, stages or toy_stages(), log=lambda m: None).run( +def run(depot, facts, stages=None, params=None, selection=("all",)): + r = Orchestrator(depot, stages or toy_stages(), log=lambda m: None).run( PIPELINE, params=params, facts=facts, selection=list(selection) ) return r.write() @@ -30,8 +30,8 @@ def nodes(p) -> dict: def test_a_cold_plan_runs_the_first_stage_and_knows_nothing_after_it(tmp_path): - store, facts = setup(tmp_path) - p = plan(toy_stages(), PIPELINE, store, facts=facts) + depot, facts = setup(tmp_path) + p = plan(toy_stages(), PIPELINE, depot, facts=facts) n = nodes(p) assert n["toy.src:a"] == ("run", [{"code": "new-input"}]) assert n["toy.count:a"] == ("unknown", [{"code": "pending", "task": "toy.src:a"}]) @@ -53,73 +53,73 @@ def test_a_cold_plan_runs_the_first_stage_and_knows_nothing_after_it(tmp_path): def test_unchanged_inputs_are_all_hits(tmp_path): - store, facts = setup(tmp_path) - prev = run(store, facts) - p = plan(toy_stages(), PIPELINE, store, facts=facts, previous=prev) + depot, facts = setup(tmp_path) + prev = run(depot, facts) + p = plan(toy_stages(), PIPELINE, depot, facts=facts, previous=prev) assert all(v == ("hit", [{"code": "unchanged"}]) for v in nodes(p).values()) assert p.check() == 0 and p.tasks == [] and p.emit(tmp_path / "none") == [] assert "7 tasks: 7 hits, 0 to run, 0 unknown" in p.text() def test_a_stage_bump_reruns_it_and_leaves_downstream_pending(tmp_path): - store, facts = setup(tmp_path) - prev = run(store, facts) + depot, facts = setup(tmp_path) + prev = run(depot, facts) bumped = toy_stages(Count=2) - n = nodes(plan(bumped, PIPELINE, store, facts=facts, previous=prev)) + n = nodes(plan(bumped, PIPELINE, depot, facts=facts, previous=prev)) assert n["toy.src:a"][0] == "hit" assert n["toy.count:a"] == ("run", [{"code": "stage-version", "old": 1, "new": 2}]) assert n["toy.total:all"][0] == "unknown" - after = run(store, facts, bumped) + after = run(depot, facts, bumped) ran = {t["id"] for t in after["tasks"] if t["status"] == "ran"} assert ran == {"toy.count:a", "toy.count:b", "toy.count:c", "toy.total:all"} - assert plan(bumped, PIPELINE, store, facts=facts, previous=after).check() == 0 + assert plan(bumped, PIPELINE, depot, facts=facts, previous=after).check() == 0 def test_parameter_atom_input_and_context_reasons(tmp_path): - store, facts = setup(tmp_path) - prev = run(store, facts) - n = nodes(plan(toy_stages(), PIPELINE, store, facts=facts, previous=prev, params={"toy.src": {"suffix": "!"}})) + depot, facts = setup(tmp_path) + prev = run(depot, facts) + n = nodes(plan(toy_stages(), PIPELINE, depot, facts=facts, previous=prev, params={"toy.src": {"suffix": "!"}})) assert n["toy.src:b"] == ("run", [{"code": "params-changed", "path": "/suffix", "old": "", "new": "!"}]) stages = toy_stages() stages["toy.src"].ATOMS = {"upper": "toy-upper/2"} - n = nodes(plan(stages, PIPELINE, store, facts=facts, previous=prev)) + n = nodes(plan(stages, PIPELINE, depot, facts=facts, previous=prev)) assert n["toy.src:a"] == ( "run", [{"code": "atom-changed", "atom": "upper", "old": "toy-upper/1", "new": "toy-upper/2"}], ) Path(facts["files"]["b"]).write_bytes(b"GAMMA!") - n = nodes(plan(toy_stages(), PIPELINE, store, facts=facts, previous=prev)) + n = nodes(plan(toy_stages(), PIPELINE, depot, facts=facts, previous=prev)) ((status, reasons),) = [n["toy.src:b"]] assert status == "run" and reasons[0]["code"] == "input-changed" and reasons[0]["role"] == "file" assert reasons[0]["new"] == contract.sha256(b"GAMMA!") and reasons[0]["name"] == "b.txt" assert n["toy.src:a"][0] == "hit" and n["toy.count:b"][0] == "unknown" - n = nodes(plan(toy_stages(), PIPELINE, store, facts=dict(facts, context={"c": "x"}), previous=prev)) + n = nodes(plan(toy_stages(), PIPELINE, depot, facts=dict(facts, context={"c": "x"}), previous=prev)) assert n["toy.src:c"] == ("run", [{"code": "context-changed", "entry": "c"}]) def test_new_failed_forced_and_cached_reasons(tmp_path): - store, facts = setup(tmp_path, {"a": "alpha", "b": "CRASH"}) - prev = run(store, facts) - n = nodes(plan(toy_stages(), PIPELINE, store, facts=facts, previous=prev)) + depot, facts = setup(tmp_path, {"a": "alpha", "b": "CRASH"}) + prev = run(depot, facts) + n = nodes(plan(toy_stages(), PIPELINE, depot, facts=facts, previous=prev)) assert n["toy.src:b"] == ("run", [{"code": "previous-failed"}]) assert n["toy.src:a"] == ("hit", [{"code": "unchanged"}]) - n = nodes(plan(toy_stages(), PIPELINE, store, facts=facts, previous=prev, force=["toy.src:a"])) + n = nodes(plan(toy_stages(), PIPELINE, depot, facts=facts, previous=prev, force=["toy.src:a"])) assert n["toy.src:a"] == ("run", [{"code": "forced"}]) facts["files"].update(toy_files(tmp_path / "more", {"z": "new"})) - n = nodes(plan(toy_stages(), PIPELINE, store, facts=facts, previous=prev)) + n = nodes(plan(toy_stages(), PIPELINE, depot, facts=facts, previous=prev)) assert n["toy.src:z"] == ("run", [{"code": "new-input"}]) - n = nodes(plan(toy_stages(), PIPELINE, store, facts=facts)) # no previous run: hits are cached + n = nodes(plan(toy_stages(), PIPELINE, depot, facts=facts)) # no previous run: hits are cached assert n["toy.src:a"] == ("hit", [{"code": "cached"}]) def test_the_latest_run_of_the_selection_is_the_default_previous(tmp_path): - store, facts = setup(tmp_path) - prev = run(store, facts, selection=["group:x"]) - assert store.latest_run(["group:x"])["id"] == prev["id"] - n = nodes(plan(toy_stages(), PIPELINE, store, facts=facts, previous=store.latest_run(["group:x"]))) + depot, facts = setup(tmp_path) + prev = run(depot, facts, selection=["group:x"]) + assert depot.latest_run(["group:x"])["id"] == prev["id"] + n = nodes(plan(toy_stages(), PIPELINE, depot, facts=facts, previous=depot.latest_run(["group:x"]))) assert n["toy.src:a"] == ("hit", [{"code": "unchanged"}]) @@ -139,8 +139,8 @@ def test_why_lists_every_difference(): def test_emitted_tasks_are_runnable_descriptions(tmp_path): - store, facts = setup(tmp_path) - p = plan(toy_stages(), PIPELINE, store, facts=facts) + depot, facts = setup(tmp_path) + p = plan(toy_stages(), PIPELINE, depot, facts=facts) files = p.emit(tmp_path / "tasks") assert [f.name for f in files] == sorted(f"{t.key}.json" for t in p.tasks) for f in files: @@ -149,11 +149,11 @@ def test_emitted_tasks_are_runnable_descriptions(tmp_path): def test_peak_estimate_oversize_and_layout_counts(tmp_path): - store, facts = setup(tmp_path, {"a": "x" * 100, "b": "y" * 10, "c": "z" * 5}) # peaks 1000, 100, 50 + depot, facts = setup(tmp_path, {"a": "x" * 100, "b": "y" * 10, "c": "z" * 5}) # peaks 1000, 100, 50 p = plan( toy_stages(), PIPELINE, - store, + depot, facts=facts, workers=2, memory=500, @@ -163,8 +163,8 @@ def test_peak_estimate_oversize_and_layout_counts(tmp_path): s = p.doc["summary"] assert s["peakBytes"] == 1000 and s["oversize"] == ["toy.src:a"] and s["budget"] == 500 assert "layout original: write 0 / remove 0 / keep 4" in p.text() - p = plan(toy_stages(), PIPELINE, store, facts=facts, workers=2, memory=None, worker_bytes=10) + p = plan(toy_stages(), PIPELINE, depot, facts=facts, workers=2, memory=None, worker_bytes=10) assert p.doc["summary"]["peakBytes"] == 1000 + 100 + 2 * 10 - run(store, facts) - p = plan(toy_stages(), PIPELINE, store, facts=facts, layouts={"cas": {"write": 1, "remove": 0, "keep": 3}}) + run(depot, facts) + p = plan(toy_stages(), PIPELINE, depot, facts=facts, layouts={"cas": {"write": 1, "remove": 0, "keep": 3}}) assert p.check() == 1 # nothing runs, but a file would be written diff --git a/tests/pipeline/test_stages.py b/tests/pipeline/test_stages.py index 3ebd6c4..8bbebea 100644 --- a/tests/pipeline/test_stages.py +++ b/tests/pipeline/test_stages.py @@ -14,8 +14,8 @@ from nnnotes.pipeline import contract from nnnotes.pipeline.contract import Cost, IncompatibleTask, Input +from nnnotes.pipeline.depot import Depot, DepotConflict, InputMissing from nnnotes.pipeline.stages import Env, Output, Pending, Stage, describe, execute, registry -from nnnotes.pipeline.store import InputMissing, Store, StoreConflict class Src(Stage): @@ -28,7 +28,7 @@ def subjects(self, env): def inputs(self, subject, env): p = Path(env.fact("files")[subject]) - sha, size = env.store.identify(p, "raw", p.name) + sha, size = env.depot.identify(p, "raw", p.name) return [Input("file", sha, size, p.name, ({"kind": "file", "path": str(p)},))] def context(self, subject, env): @@ -38,8 +38,8 @@ def context(self, subject, env): def estimate(self, subject, env, inputs): return Cost(inputs[0].size / 100, inputs[0].size * 10) - def run(self, task, store): - data = store.input_bytes(task.input("file")) + def run(self, task, depot): + data = depot.input_bytes(task.input("file")) if b"CRASH" in data: raise RuntimeError("crash requested") if b"DIE" in data: @@ -48,7 +48,7 @@ def run(self, task, store): arts = [ contract.artifact( whole, - store.add(data.upper() + task.params["suffix"].encode(), "txt"), + depot.add(data.upper() + task.params["suffix"].encode(), "txt"), contract.provenance(task), {"kind": "toy.text"}, ) @@ -68,7 +68,7 @@ def run(self, task, store): arts.append( contract.artifact( aid, - store.add(line.upper().encode(), "txt"), + depot.add(line.upper().encode(), "txt"), contract.provenance(task, obj=obj), {"kind": "toy.line"}, ) @@ -90,8 +90,8 @@ def inputs(self, subject, env): def depends(self, subject, env): return [f"toy.src:{subject}"] - def run(self, task, store): - text = store.input_bytes(task.input("text")).decode() + def run(self, task, depot): + text = depot.input_bytes(task.input("text")).decode() doc = {"lines": sum(1 for line in text.split("\n") if line and not line.startswith(("FAIL", "?")))} if self.version > 1: doc["chars"] = len(text) @@ -99,7 +99,7 @@ def run(self, task, store): return Output( [ contract.artifact( - aid, store.add(contract.encode(doc), "json"), contract.provenance(task), {"kind": "toy.count"} + aid, depot.add(contract.encode(doc), "json"), contract.provenance(task), {"kind": "toy.count"} ) ], [], @@ -121,14 +121,14 @@ def inputs(self, subject, env): def depends(self, subject, env): return [f"toy.count:{s}" for s in sorted(env.fact("files"))] - def run(self, task, store): - n = sum(json.loads(store.input_bytes(i))["lines"] for i in task.inputs) + def run(self, task, depot): + n = sum(json.loads(depot.input_bytes(i))["lines"] for i in task.inputs) aid = contract.artifact_id(task.id, "total.json") return Output( [ contract.artifact( aid, - store.add(contract.encode({"lines": n}), "json"), + depot.add(contract.encode({"lines": n}), "json"), contract.provenance(task), {"kind": "toy.total"}, ) @@ -164,8 +164,8 @@ def toy_files(root: Path, texts: dict) -> dict: # ---------------------------------------------------------------- tests def test_describe_fills_defaults_and_refuses_unknown_parameters(tmp_path): - store = Store(tmp_path / "store") - env = Env(store, {"files": toy_files(tmp_path / "in", {"a": "x\ny"})}) + depot = Depot(tmp_path / "depot") + env = Env(depot, {"files": toy_files(tmp_path / "in", {"a": "x\ny"})}) t = describe(Src(), "a", None, env) assert t.id == "toy.src:a" and t.params == {"suffix": ""} and t.atoms == {"upper": "toy-upper/1"} assert t.input("file").name == "a.txt" and t.cost.peak_bytes == 30 @@ -175,73 +175,73 @@ def test_describe_fills_defaults_and_refuses_unknown_parameters(tmp_path): def test_execute_commits_then_hits(tmp_path): - store = Store(tmp_path / "store") - env = Env(store, {"files": toy_files(tmp_path / "in", {"a": "one\n\nFAIL\n?q"})}) + depot = Depot(tmp_path / "depot") + env = Env(depot, {"files": toy_files(tmp_path / "in", {"a": "one\n\nFAIL\n?q"})}) t = describe(Src(), "a", None, env) - ex = execute(t, store, toy_stages()) + ex = execute(t, depot, toy_stages()) assert (ex.status, ex.result_status) == ("ran", "partial") and ex.cost["cpuSeconds"] >= 0 - doc = store.result(t.key) + doc = depot.result(t.key) assert [i["status"] for i in doc["items"]] == ["exported", "contained", "failed", "unsupported"] arts = {a["id"]: a for a in doc["artifacts"]} assert sorted(arts) == ["F-a:1#data", "toy.src:a#all.txt"] - assert store.read(arts["toy.src:a#all.txt"]["content"]["sha256"]) == b"ONE\n\nFAIL\n?Q" + assert depot.read(arts["toy.src:a#all.txt"]["content"]["sha256"]) == b"ONE\n\nFAIL\n?Q" assert "task" not in doc and doc["keyParts"] == t.key_parts() - assert execute(t, store, toy_stages()).status == "hit" - assert execute(t, store, toy_stages(), force=True).status == "ran" # identical output: accepted + assert execute(t, depot, toy_stages()).status == "hit" + assert execute(t, depot, toy_stages(), force=True).status == "ran" # identical output: accepted def test_execute_refuses_other_versions_and_unknown_stages(tmp_path): - store = Store(tmp_path / "store") - env = Env(store, {"files": toy_files(tmp_path / "in", {"a": "x"})}) + depot = Depot(tmp_path / "depot") + env = Env(depot, {"files": toy_files(tmp_path / "in", {"a": "x"})}) t = describe(Src(), "a", None, env) with pytest.raises(IncompatibleTask, match="version 2 here"): - execute(t, store, toy_stages(Src=2)) + execute(t, depot, toy_stages(Src=2)) with pytest.raises(IncompatibleTask, match="no stage"): - execute(t, store, {}) - assert store.result(t.key) is None + execute(t, depot, {}) + assert depot.result(t.key) is None def test_execute_verifies_inputs_and_caches_no_task_failure(tmp_path): - store = Store(tmp_path / "store") + depot = Depot(tmp_path / "depot") files = toy_files(tmp_path / "in", {"a": "x", "b": "CRASH"}) - env = Env(store, {"files": files}) + env = Env(depot, {"files": files}) ta, tb = describe(Src(), "a", None, env), describe(Src(), "b", None, env) with pytest.raises(RuntimeError, match="crash requested"): - execute(tb, store, toy_stages()) - assert store.result(tb.key) is None + execute(tb, depot, toy_stages()) + assert depot.result(tb.key) is None Path(files["a"]).write_bytes(b"y") # same size, other content: the input is refused with pytest.raises(InputMissing, match="other content"): - execute(ta, store, toy_stages()) - assert store.result(ta.key) is None + execute(ta, depot, toy_stages()) + assert depot.result(ta.key) is None def test_a_stage_whose_output_is_not_a_function_of_its_key_is_caught(tmp_path): class Clock(Src): n = 0 - def run(self, task, store): + def run(self, task, depot): Clock.n += 1 aid = contract.artifact_id(task.id, "all.txt") return Output( [ contract.artifact( - aid, store.add(str(Clock.n).encode(), "txt"), contract.provenance(task), {"kind": "toy.text"} + aid, depot.add(str(Clock.n).encode(), "txt"), contract.provenance(task), {"kind": "toy.text"} ) ], [], ) - store = Store(tmp_path / "store") - env = Env(store, {"files": toy_files(tmp_path / "in", {"a": "x"})}) + depot = Depot(tmp_path / "depot") + env = Env(depot, {"files": toy_files(tmp_path / "in", {"a": "x"})}) t = describe(Clock(), "a", None, env) - execute(t, store, {"toy.src": Clock()}) - with pytest.raises(StoreConflict): - execute(t, store, {"toy.src": Clock()}, force=True) + execute(t, depot, {"toy.src": Clock()}) + with pytest.raises(DepotConflict): + execute(t, depot, {"toy.src": Clock()}, force=True) def test_env_pending_and_failed_tasks(tmp_path): - store = Store(tmp_path / "store") - env = Env(store, {"files": {"a": "unused", "b": "unused"}}) + depot = Depot(tmp_path / "depot") + env = Env(depot, {"files": {"a": "unused", "b": "unused"}}) env.waiting("toy.src:a") env.waiting("toy.src:b", failed=True) with pytest.raises(Pending) as e: @@ -275,7 +275,7 @@ def uses(self, subject, env): want = env.params["classes"] return {k: v for k, v in self.ATOMS.items() if want is None or k in want} - env = Env(Store(tmp_path / "s")) + env = Env(Depot(tmp_path / "s")) assert describe(Selective(), "s", None, env).atoms == {"a": "x/1", "b": "y/1"} t = describe(Selective(), "s", {"classes": ["b"]}, env) assert isinstance(t, Task) and t.atoms == {"b": "y/1"} and env.params == {} @@ -289,7 +289,7 @@ class _Spawner(Stage): def subjects(self, env): return ["s"] - def run(self, task, store): + def run(self, task, depot): import subprocess import sys @@ -301,8 +301,8 @@ def test_the_cost_counts_the_child_processes(tmp_path): from nnnotes.pipeline.contract import Task from nnnotes.pipeline.stages import children_cpu - store = Store(tmp_path / "s") - ex = execute(Task("toy.spawn", 1, "s"), store, registry(_Spawner())) + depot = Depot(tmp_path / "s") + ex = execute(Task("toy.spawn", 1, "s"), depot, registry(_Spawner())) cost = ex.cost if children_cpu() is None: # the platform does not report child processes assert cost["childCpuSeconds"] is None diff --git a/tests/pipeline/test_views.py b/tests/pipeline/test_views.py index 3137335..dd9dda9 100644 --- a/tests/pipeline/test_views.py +++ b/tests/pipeline/test_views.py @@ -12,10 +12,10 @@ from nnnotes.pipeline import contract, views from nnnotes.pipeline.contract import Input +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.orchestrate import Orchestrator from nnnotes.pipeline.plan import plan from nnnotes.pipeline.stages import Env, Output, Pending, Stage, describe, execute, registry -from nnnotes.pipeline.store import Store from nnnotes.pipeline.views import AddressIndex, Obj, Recorder, ViewError, ViewStage from nnnotes.source import languages @@ -757,12 +757,12 @@ def subjects(self, env): return sorted(env.fact("catalogs")) def inputs(self, subject, env): - return [Input("doc", env.store.put(self.data), len(self.data), None, ({"kind": "store"},))] + return [Input("doc", env.depot.put(self.data), len(self.data), None, ({"kind": "depot"},))] - def run(self, task, store): + def run(self, task, depot): rec = contract.artifact( contract.artifact_id(task.id, "addresses"), - store.add(store.input_bytes(task.input("doc")), "json"), + depot.add(depot.input_bytes(task.input("doc")), "json"), contract.provenance(task), {"kind": "link.addresses", "format": "json"}, ) @@ -787,12 +787,12 @@ def write_master(d: Path, tables: dict) -> Path: return d -def env_with(store, doc: dict, master, **facts) -> Env: +def env_with(depot, doc: dict, master, **facts) -> Env: """An environment where link.addresses:main has run with the address table `doc`.""" link = FakeAddresses(doc) - env = Env(store, {"catalogs": {"main": []}, "master": master, **facts}) + env = Env(depot, {"catalogs": {"main": []}, "master": master, **facts}) t = describe(link, "main", None, env) - execute(t, store, {link.name: link}) + execute(t, depot, {link.name: link}) env.done(t.id, t.key) return env @@ -810,29 +810,29 @@ def test_one_stage_per_view(): def test_view_stage_subjects_and_dependencies(tmp_path): - store, mdir = Store(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) + depot, mdir = Depot(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) stage = ViewStage("cards", views.load_rules()) - assert stage.subjects(Env(store, {"catalogs": {"main": []}, "master": None})) == [] - assert stage.subjects(Env(store, {"catalogs": {"main": []}, "master": mdir, "views": ["stamps"]})) == [] - assert stage.subjects(Env(store, {"catalogs": {"main": [], "b": []}, "master": mdir, "views": ["cards"]})) == [ + assert stage.subjects(Env(depot, {"catalogs": {"main": []}, "master": None})) == [] + assert stage.subjects(Env(depot, {"catalogs": {"main": []}, "master": mdir, "views": ["stamps"]})) == [] + assert stage.subjects(Env(depot, {"catalogs": {"main": [], "b": []}, "master": mdir, "views": ["cards"]})) == [ "b", "main", ] - env = Env(store, {"catalogs": {"main": []}, "master": mdir}) + env = Env(depot, {"catalogs": {"main": []}, "master": mdir}) assert stage.subjects(env) == ["main"] and stage.depends("main", env) == ["link.addresses:main"] with pytest.raises(Pending) as e: describe(stage, "main", None, env) assert e.value.task == "link.addresses:main" (mdir / "MasterBand.json").unlink() with pytest.raises(ViewError, match="MasterBand is not in the master data"): - describe(stage, "main", None, env_with(store, addresses_doc(stage_entries()), mdir)) + describe(stage, "main", None, env_with(depot, addresses_doc(stage_entries()), mdir)) def test_view_stage_runs_on_its_inputs_alone(tmp_path): rules = views.load_rules() stage = ViewStage("cards", rules) mdir, doc = write_master(tmp_path / "m", STAGE_TABLES), addresses_doc(stage_entries()) - a = Store(tmp_path / "a") + a = Depot(tmp_path / "a") task = describe(stage, "main", None, env_with(a, doc, mdir)) assert (task.id, task.version, task.atoms, task.context) == ("view.cards:main", 1, {}, {}) assert [i.role for i in task.inputs] == [ @@ -851,26 +851,26 @@ def test_view_stage_runs_on_its_inputs_alone(tmp_path): assert view == views.build(rules, "cards", views.MasterTables(mdir), AddressIndex.from_addresses(doc)) assert res["artifacts"][0]["semantics"]["facts"]["rows"] == 3 assert res["artifacts"][0]["semantics"]["facts"]["gaps"] == 0 - # the task description runs in another store holding only its stored inputs: the same result - b = Store(tmp_path / "b") + # the task description runs in another depot holding only its stored inputs: the same result + b = Depot(tmp_path / "b") for i in task.inputs: - if i.locators[0]["kind"] == "store": + if i.locators[0]["kind"] == "depot": b.put(a.read(i.sha256)) again = contract.Task.from_json(json.loads(json.dumps(task.to_json()))) execute(again, b, {stage.name: ViewStage("cards", rules)}) assert contract.encode(b.result(task.key)) == contract.encode(res) # described again elsewhere: the same key; run again: the same output - assert describe(stage, "main", None, env_with(Store(tmp_path / "c"), doc, mdir)).key == task.key + assert describe(stage, "main", None, env_with(Depot(tmp_path / "c"), doc, mdir)).key == task.key assert execute(task, a, {stage.name: stage}, force=True).status == "ran" def test_view_keys_follow_only_their_own_inputs(tmp_path): - store, mdir = Store(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) + depot, mdir = Depot(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) rules = views.load_rules() entries = stage_entries() def keys(entries=entries, rules=rules): - env = env_with(store, addresses_doc(entries), mdir) + env = env_with(depot, addresses_doc(entries), mdir) return [describe(ViewStage(n, rules), "main", None, env).key for n in ("cards", "stamps")] cards, stamps = keys() @@ -905,20 +905,20 @@ def keys(entries=entries, rules=rules): def test_png_encoding_never_reaches_view_keys(tmp_path): - store, mdir = Store(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) + depot, mdir = Depot(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) stage = ViewStage("cards", views.load_rules()) assert "png" not in stage.PARAMS with pytest.raises(ValueError, match="unknown parameters png"): stage.normalize({"png": {"level": 9}}) - env = env_with(store, addresses_doc(stage_entries()), mdir) + env = env_with(depot, addresses_doc(stage_entries()), mdir) key = describe(stage, "main", None, env).key oid = card_entries()["MemberCard/10/member_full"][1].id for level in (6, 9): # exports of the objects with other PNG bytes t = contract.Task("unity.export", 1, "b", {"png": {"encoder": "pillow", "level": level}}) rec = contract.artifact( - f"{oid}#image", store.add(b"png %d" % level, "png"), contract.provenance(t), {"kind": "texture.image"} + f"{oid}#image", depot.add(b"png %d" % level, "png"), contract.provenance(t), {"kind": "texture.image"} ) - store.commit(contract.result(t, [rec], [contract.item(oid, "exported", artifacts=[rec["id"]])])) + depot.commit(contract.result(t, [rec], [contract.item(oid, "exported", artifacts=[rec["id"]])])) env.done(t.id, t.key) assert describe(stage, "main", None, env).key == key @@ -937,11 +937,11 @@ def artifact(self, aid): def test_derived_view_document_has_layout_paths(tmp_path): - store, mdir = Store(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) + depot, mdir = Depot(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) stage = ViewStage("cards", views.load_rules()) - task = describe(stage, "main", None, env_with(store, addresses_doc(stage_entries()), mdir)) - execute(task, store, {stage.name: stage}) - res = store.result(task.key) + task = describe(stage, "main", None, env_with(depot, addresses_doc(stage_entries()), mdir)) + execute(task, depot, {stage.name: stage}) + res = depot.result(task.key) full = next( x.id for x in card_entries()["MemberCard/10/member_full"] if x.name == "member_full" and x.cls == "Sprite" ) @@ -949,7 +949,7 @@ def test_derived_view_document_has_layout_paths(tmp_path): f"{full}#meta": "MemberCard/10/member_full[member_full].png.meta.json", f"{full}#image": "MemberCard/10/member_full[member_full].png", } - out = stage.derived(task.id, res, store, Paths(placed)) + out = stage.derived(task.id, res, depot, Paths(placed)) assert [p for p, _ in out] == ["views/cards.json"] doc = contract.loads(out[0][1]) assert out[0][1] == contract.encode(doc) and doc["layout"] == "original" @@ -960,7 +960,7 @@ def test_derived_view_document_has_layout_paths(tmp_path): assert "path" not in r10["roles"]["skill_sprite"] # missing-key: no object assert "path" not in r10["roles"]["movie"][0] # everything else is the view document - view = contract.loads(store.read(res["artifacts"][0]["content"]["sha256"])) + view = contract.loads(depot.read(res["artifacts"][0]["content"]["sha256"])) for row in doc["rows"]: for e in (x for v in row["roles"].values() for x in (v if isinstance(v, list) else [v])): e.pop("path", None), e.pop("files", None) @@ -970,20 +970,20 @@ def test_derived_view_document_has_layout_paths(tmp_path): def test_views_plan_and_run_after_the_address_table(tmp_path): - store, mdir = Store(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) + depot, mdir = Depot(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) vs = views.stages() reg = registry(FakeAddresses(addresses_doc(stage_entries())), *vs) pipeline = ["link.addresses"] + [s.name for s in vs] facts = {"catalogs": {"main": []}, "master": mdir, "views": ["cards", "stamps"]} - nodes = {n["id"]: n for n in plan(reg, pipeline, store, facts=facts).doc["nodes"]} + nodes = {n["id"]: n for n in plan(reg, pipeline, depot, facts=facts).doc["nodes"]} assert sorted(nodes) == ["link.addresses:main", "view.cards:main", "view.stamps:main"] assert nodes["view.cards:main"]["status"] == "unknown" assert nodes["view.cards:main"]["reasons"] == [{"code": "pending", "task": "link.addresses:main"}] - first = Orchestrator(store, reg, log=lambda m: None).run(pipeline, facts=facts) + first = Orchestrator(depot, reg, log=lambda m: None).run(pipeline, facts=facts) assert {t: v["status"] for t, v in first.tasks.items()} == dict.fromkeys(nodes, "ran") - second = Orchestrator(store, reg, log=lambda m: None).run(pipeline, facts=facts) + second = Orchestrator(depot, reg, log=lambda m: None).run(pipeline, facts=facts) assert {t: v["status"] for t, v in second.tasks.items()} == dict.fromkeys(nodes, "hit") - assert plan(reg, pipeline, store, facts=facts).check() == 0 + assert plan(reg, pipeline, depot, facts=facts).check() == 0 def test_views_import_neither_unitypy_nor_numpy(): diff --git a/tests/pipeline/test_voices.py b/tests/pipeline/test_voices.py index be7ab29..3c61e00 100644 --- a/tests/pipeline/test_voices.py +++ b/tests/pipeline/test_voices.py @@ -15,8 +15,8 @@ from nnnotes.formats import acb from nnnotes.pipeline import contract, voices from nnnotes.pipeline.contract import Input, Task +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry -from nnnotes.pipeline.store import Store from nnnotes.pipeline.views import AddressIndex, ViewError from nnnotes.source import languages from tests.pipeline.test_views import FakeAddresses, addresses_doc, write_master @@ -420,8 +420,8 @@ def audio_task(sheet: str, sheets=None) -> Task: return Task("cri.audio", 1, sha, {"flac": {"level": 8}}, {}, (Input("acb", sha, len(data)),)) -def audio_result(store, sheet: str, unsupported: bool = False, sheets=None) -> dict: - """A cri.audio result of a sheet, committed to `store` when given.""" +def audio_result(depot, sheet: str, unsupported: bool = False, sheets=None) -> dict: + """A cri.audio result of a sheet, committed to `depot` when given.""" sheets = sheets or SHEETS t = ( audio_task(sheet, sheets) @@ -442,8 +442,8 @@ def audio_result(store, sheet: str, unsupported: bool = False, sheets=None) -> d for n, name, file, samples in stream_files(sheet, sheets): data = f"{sheet}/{file}".encode() content = ( - store.add(data, "flac") - if store is not None + depot.add(data, "flac") + if depot is not None else {"sha256": contract.sha256(data), "size": len(data), "mediaType": "audio/flac", "ext": "flac"} ) arts.append( @@ -459,18 +459,18 @@ def audio_result(store, sheet: str, unsupported: bool = False, sheets=None) -> d ) ) res = contract.result(t, arts, [contract.item(t.id, "exported", artifacts=[a["id"] for a in arts], cls="ACB")]) - if store is not None: - store.commit(res) + if depot is not None: + depot.commit(res) return res -def audio(store=None, sheets=None) -> dict: +def audio(depot=None, sheets=None) -> dict: sheets = sheets or SHEETS out = { - s: voices.sheet_audio(audio_result(store, s, sheets=sheets), acb.cue_streams(sheet_acb(s, sheets))) + s: voices.sheet_audio(audio_result(depot, s, sheets=sheets), acb.cue_streams(sheet_acb(s, sheets))) for s in sheets } - out["VoiceBroken"] = voices.sheet_audio(audio_result(store, "VoiceBroken", unsupported=True), None) + out["VoiceBroken"] = voices.sheet_audio(audio_result(depot, "VoiceBroken", unsupported=True), None) return out @@ -696,7 +696,7 @@ def test_rules_are_checked(): # ---------------------------------------------------------------- the stage -def planning(store, tmp: Path, keys=KEYS, skip=(), sheets=None, tables=None, doc=None, results=None) -> Env: +def planning(depot, tmp: Path, keys=KEYS, skip=(), sheets=None, tables=None, doc=None, results=None) -> Env: """An environment where link.addresses:main has run (the catalog keys `keys`, or the address table `doc`) and the cri.audio tasks of the decoded sheets have results; the raw ACBs are files under their catalog keys. `results(env)`: commits further results (unity.export) before the environment is returned.""" @@ -709,7 +709,7 @@ def planning(store, tmp: Path, keys=KEYS, skip=(), sheets=None, tables=None, doc p = tmp / "raw" / stable.replace("/", "_") p.parent.mkdir(parents=True, exist_ok=True) p.write_bytes(data) - sha, size = store.identify(p, "raw", stable) + sha, size = depot.identify(p, "raw", stable) raw[stable] = Input("raw", sha, size, f"{stable}_{'0' * 32}", ({"kind": "file", "path": str(p)},)) deps[f"Cri/Sound/{sheet}"] = [stable] lid = {s: f"remote:{i}" for i, s in enumerate(sorted(raw))} @@ -720,13 +720,13 @@ def planning(store, tmp: Path, keys=KEYS, skip=(), sheets=None, tables=None, doc ] cat = {"rawFiles": [{"stable": s, "locations": [lid[s]]} for s in sorted(raw)], "locations": locations} link = FakeAddresses(doc or addresses_doc({k: [] for k in keys})) - env = Env(store, {"catalogs": {"main": []}, "master": mdir, "views": ["voices"], "raw": raw, "index": cat}) + env = Env(depot, {"catalogs": {"main": []}, "master": mdir, "views": ["voices"], "raw": raw, "index": cat}) t = describe(link, "main", None, env) - execute(t, store, {link.name: link}) + execute(t, depot, {link.name: link}) env.done(t.id, t.key) for sheet in sheets: if sheet not in skip: - res = audio_result(store, sheet, sheets=sheets) + res = audio_result(depot, sheet, sheets=sheets) env.done(audio_task(sheet, sheets).id, res["key"]) b = Task("cri.audio", 1, raw["cri_assets_cri/sound/voicebroken"].sha256, {}, {}, ()) broken = contract.result( @@ -738,7 +738,7 @@ def planning(store, tmp: Path, keys=KEYS, skip=(), sheets=None, tables=None, doc ) ], ) - store.commit(broken) + depot.commit(broken) env.done(b.id, broken["key"]) if results is not None: results(env) @@ -748,19 +748,19 @@ def planning(store, tmp: Path, keys=KEYS, skip=(), sheets=None, tables=None, doc def run_stage(env): stage = voices.VoiceStage() task = describe(stage, "main", None, env) - ex = execute(task, env.store, registry(stage)) - res = env.store.result(task.key) - return stage, task, ex, res, contract.loads(env.store.read(res["artifacts"][0]["content"]["sha256"])) + ex = execute(task, env.depot, registry(stage)) + res = env.depot.result(task.key) + return stage, task, ex, res, contract.loads(env.depot.read(res["artifacts"][0]["content"]["sha256"])) def test_stage_subjects_inputs_and_waits(tmp_path): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") stage = voices.VoiceStage() assert stage.name == "view.voices" and stage.version == 2 assert stage.after == ("link.addresses", "unity.export", "cri.audio") - assert stage.subjects(Env(store, {"catalogs": {"main": []}, "master": None})) == [] - assert stage.subjects(Env(store, {"catalogs": {"main": []}, "master": tmp_path, "views": ["cards"]})) == [] - env = planning(store, tmp_path) + assert stage.subjects(Env(depot, {"catalogs": {"main": []}, "master": None})) == [] + assert stage.subjects(Env(depot, {"catalogs": {"main": []}, "master": tmp_path, "views": ["cards"]})) == [] + env = planning(depot, tmp_path) assert stage.subjects(env) == ["main"] task = describe(stage, "main", None, env) roles = sorted(i.role for i in task.inputs) @@ -774,7 +774,7 @@ def test_stage_subjects_inputs_and_waits(tmp_path): "rules", "sheets", ] - sheets = contract.loads(store.read(task.input("sheets").sha256)) + sheets = contract.loads(depot.read(task.input("sheets").sha256)) assert sorted(sheets) == sorted([*SHEETS, "VoiceBroken"]) assert sheets["VoiceSystem_01"] == { "task": audio_task("VoiceSystem_01").id, @@ -782,7 +782,7 @@ def test_stage_subjects_inputs_and_waits(tmp_path): } assert sorted(task.context["audio"]) == sorted(stage.depends("main", env)[1:]) assert stage.depends("main", env)[0] == "link.addresses:main" - waiting = Env(store, env.facts) + waiting = Env(depot, env.facts) for tid in env.tasks("link.addresses"): waiting.done(tid, env.key(tid)) waiting.waiting(audio_task("VoiceSystem_01").id) @@ -791,12 +791,12 @@ def test_stage_subjects_inputs_and_waits(tmp_path): def test_stage_document_equals_the_index_of_its_inputs(tmp_path): - store = Store(tmp_path / "s") - env = planning(store, tmp_path) + depot = Depot(tmp_path / "s") + env = planning(depot, tmp_path) stage, task, ex, res, doc = run_stage(env) assert (ex.status, ex.result_status) == ("ran", "ok") and res["items"] == [] assert [a["id"] for a in res["artifacts"]] == ["view.voices:main#view"] - want = voices.build(voices.load_rules(), TABLES, index(), audio(store)) + want = voices.build(voices.load_rules(), TABLES, index(), audio(depot)) assert doc["rows"] == want["rows"] and doc["coverage"] == want["coverage"] assert sorted(doc["snapshot"]["tables"]) == voices.tables_of(stage.rules) assert doc["snapshot"]["sheets"]["VoiceBroken"]["status"] == "unsupported" @@ -809,20 +809,20 @@ def test_stage_document_equals_the_index_of_its_inputs(tmp_path): } # described again: the same key; run again: the same bytes assert describe(voices.VoiceStage(), "main", None, env).key == task.key - again = execute(task, store, registry(stage), force=True) - assert again.status == "ran" and contract.encode(store.result(task.key)) == contract.encode(res) + again = execute(task, depot, registry(stage), force=True) + assert again.status == "ran" and contract.encode(depot.result(task.key)) == contract.encode(res) # the task crosses to another process as JSON same = contract.Task.from_json(json.loads(json.dumps(task.to_json()))) assert same.key == task.key def test_stage_key_follows_the_audio_and_the_catalog(tmp_path): - store = Store(tmp_path / "s") - env = planning(store, tmp_path) + depot = Depot(tmp_path / "s") + env = planning(depot, tmp_path) key = describe(voices.VoiceStage(), "main", None, env).key - other = planning(Store(tmp_path / "t"), tmp_path / "t", keys=KEYS + ["Cri/Sound/VoiceGone"]) + other = planning(Depot(tmp_path / "t"), tmp_path / "t", keys=KEYS + ["Cri/Sound/VoiceGone"]) assert describe(voices.VoiceStage(), "main", None, other).key != key - fewer = planning(Store(tmp_path / "u"), tmp_path / "u", skip=("InitialVoice",)) + fewer = planning(Depot(tmp_path / "u"), tmp_path / "u", skip=("InitialVoice",)) k2 = describe(voices.VoiceStage(), "main", None, fewer).key assert k2 != key _, _, _, _, doc = run_stage(fewer) @@ -841,12 +841,12 @@ def objects(self, oid): def test_derived_document_has_the_stream_paths(tmp_path): - store = Store(tmp_path / "s") - env = planning(store, tmp_path) + depot = Depot(tmp_path / "s") + env = planning(depot, tmp_path) stage, task, _, res, doc = run_stage(env) tid = audio_task("VoiceSystem_01").id placed = {f"{tid}#Growth_Alpha_RankUp_01.flac": "Cri/Sound/VoiceSystem_01/Growth_Alpha_RankUp_01.flac"} - out = stage.derived(task.id, res, store, Paths(placed)) + out = stage.derived(task.id, res, depot, Paths(placed)) assert [p for p, _ in out] == ["views/voices.json"] d = contract.loads(out[0][1]) assert d["layout"] == "original" @@ -875,18 +875,18 @@ def keys(self, prefix=""): return [k for k in KEYS if k.startswith(prefix)] -def export_dir(tmp: Path, store) -> Path: +def export_dir(tmp: Path, depot) -> Path: """An export directory: views/voices.json with paths and the stream files at them.""" out = tmp / "out" placed = {} - doc = voices.build(voices.load_rules(), TABLES, index(), audio(store)) + doc = voices.build(voices.load_rules(), TABLES, index(), audio(depot)) for r in doc["rows"]: for s in (r.get("audio") or {}).get("streams", []): sheet = r["sound"]["sheet"] placed[s["artifact"]] = f"Cri/Sound/{sheet}/{s['file']}" f = out / placed[s["artifact"]] f.parent.mkdir(parents=True, exist_ok=True) - f.write_bytes(store.read(s["sha256"])) + f.write_bytes(depot.read(s["sha256"])) d = voices.with_paths(doc, Paths(placed)) (out / "views").mkdir(parents=True) (out / "views" / "voices.json").write_bytes(contract.encode(d)) diff --git a/tests/pipeline/test_voices_story.py b/tests/pipeline/test_voices_story.py index 5736017..8fd0dd3 100644 --- a/tests/pipeline/test_voices_story.py +++ b/tests/pipeline/test_voices_story.py @@ -11,8 +11,8 @@ from nnnotes.content import advcommand, storyvoices from nnnotes.pipeline import contract, voices from nnnotes.pipeline.contract import Task +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Pending, describe, execute, registry -from nnnotes.pipeline.store import Store from nnnotes.pipeline.views import AddressIndex, Obj, ViewError from nnnotes.source import languages from tests.pipeline.test_views import addresses_doc @@ -482,7 +482,7 @@ def test_story_rules_are_checked(): # ---------------------------------------------------------------- the stage -def export_results(store, env) -> dict: +def export_results(depot, env) -> dict: """unity.export results holding the exported episode parts, one bundle per episode: {asset: {part: oid}}.""" oids = {} for n, (asset, parts) in enumerate(sorted(EPISODES.items())): @@ -495,7 +495,7 @@ def export_results(store, env) -> dict: arts.append( contract.artifact( contract.artifact_id(oid, role), - store.add(data, "json" if part == "script" else "txt"), + depot.add(data, "json" if part == "script" else "txt"), contract.provenance(t), {"kind": "unity.object", "format": "json"}, ) @@ -503,7 +503,7 @@ def export_results(store, env) -> dict: items.append(contract.item(oid, "exported", artifacts=[arts[-1]["id"]])) oids.setdefault(asset, {})[part] = oid res = contract.result(t, arts, items) - store.commit(res) + depot.commit(res) env.done(t.id, res["key"]) return oids @@ -527,24 +527,24 @@ def address_doc() -> dict: return doc -def story_env(store, tmp_path): +def story_env(depot, tmp_path): return planning( - store, + depot, tmp_path, keys=catalog_keys(), sheets=SHEETS2, tables=tables(), doc=address_doc(), - results=lambda env: export_results(store, env), + results=lambda env: export_results(depot, env), ) def test_stage_reads_the_exported_episodes(tmp_path): - store = Store(tmp_path / "s") - env = story_env(store, tmp_path) + depot = Depot(tmp_path / "s") + env = story_env(depot, tmp_path) stage = voices.VoiceStage() task = describe(stage, "main", None, env) - eps = contract.loads(store.read(task.input("episodes").sha256)) + eps = contract.loads(depot.read(task.input("episodes").sha256)) assert sorted(eps) == sorted(EPISODES) assert sorted(eps["adv_script_x_02"]) == ["script", "sound"] x01 = eps["adv_script_x_01"]["script"] @@ -557,44 +557,44 @@ def test_stage_reads_the_exported_episodes(tmp_path): assert sorted(i.role for i in task.inputs if i.role.startswith("adv:")) == sorted(f"adv:{s}" for s in shas) deps = stage.depends("main", env) assert sorted(d for d in deps if d.startswith("unity.export:")) == sorted(f"unity.export:adv_{a}" for a in EPISODES) - ex = execute(task, store, registry(stage)) + ex = execute(task, depot, registry(stage)) assert (ex.status, ex.result_status) == ("ran", "ok") - res = store.result(task.key) - doc = contract.loads(store.read(res["artifacts"][0]["content"]["sha256"])) + res = depot.result(task.key) + doc = contract.loads(depot.read(res["artifacts"][0]["content"]["sha256"])) want = voices.build( - RULES, tables(), AddressIndex({k: [] for k in catalog_keys()}), audio(store, SHEETS2), story=story() + RULES, tables(), AddressIndex({k: [] for k in catalog_keys()}), audio(depot, SHEETS2), story=story() ) assert doc["rows"] == want["rows"] and doc["episodes"] == want["episodes"] assert doc["coverage"] == want["coverage"] assert doc["snapshot"]["episodes"] == task.input("episodes").sha256 assert STORY_SHEET in doc["snapshot"]["sheets"] assert describe(voices.VoiceStage(), "main", None, env).key == task.key - again = execute(task, store, registry(stage), force=True) - assert again.status == "ran" and contract.encode(store.result(task.key)) == contract.encode(res) + again = execute(task, depot, registry(stage), force=True) + assert again.status == "ran" and contract.encode(depot.result(task.key)) == contract.encode(res) def test_stage_waits_for_the_exports(tmp_path): - store = Store(tmp_path / "s") - env = story_env(store, tmp_path) + depot = Depot(tmp_path / "s") + env = story_env(depot, tmp_path) env.waiting("unity.export:adv_adv_script_x_01") with pytest.raises(Pending, match="unity.export"): describe(voices.VoiceStage(), "main", None, env) def test_stage_without_exports_has_no_story_rows(tmp_path): - store = Store(tmp_path / "s") - env = planning(store, tmp_path, keys=catalog_keys(), sheets=SHEETS2, tables=tables(), doc=address_doc()) + depot = Depot(tmp_path / "s") + env = planning(depot, tmp_path, keys=catalog_keys(), sheets=SHEETS2, tables=tables(), doc=address_doc()) stage = voices.VoiceStage() task = describe(stage, "main", None, env) assert not [i for i in task.inputs if i.role.startswith("adv:")] - execute(task, store, registry(stage)) - doc = contract.loads(store.read(store.result(task.key)["artifacts"][0]["content"]["sha256"])) + execute(task, depot, registry(stage)) + doc = contract.loads(depot.read(depot.result(task.key)["artifacts"][0]["content"]["sha256"])) assert story_rows(doc) == [] assert {e["status"] for e in doc["episodes"] if e["asset"] in EPISODES} == {"not-exported"} # ---------------------------------------------------------------- the command -def export_dir(tmp_path, store): +def export_dir(tmp_path, depot): out = tmp_path / "out" doc = build() placed = {} diff --git a/tests/site/test_live_options_web.py b/tests/site/test_live_options_web.py index c0c9db1..59381c3 100644 --- a/tests/site/test_live_options_web.py +++ b/tests/site/test_live_options_web.py @@ -161,7 +161,7 @@ def test_live_option_flag(tmp_path, capsys, monkeypatch): master = option_tables(write_master(tmp_path / "m")) monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", (tmp_path / "cdn").as_uri()) base = [ - "--cache", + "--depot", str(tmp_path / "c"), "--region", "zz", diff --git a/tests/site/test_pipeline.py b/tests/site/test_pipeline.py index fd8ea51..91b227d 100644 --- a/tests/site/test_pipeline.py +++ b/tests/site/test_pipeline.py @@ -151,15 +151,15 @@ def test_catalog_cached(tmp_path): cat = Catalog(synth.CatalogWriter().build([("k", synth.remote("b_1.bundle"), [])]), tmp_path) b = Bundle(1, synth.remote("b_1.bundle"), "b_1.bundle", True) - assert cat.cached(b) is None + assert cat.downloaded(b) is None (tmp_path / "bundles").mkdir() (tmp_path / "bundles" / "b_1.bundle").write_bytes(b"UnityFS\0x") - assert cat.cached(b) == tmp_path / "bundles" / "b_1.bundle" == cat.fetch(b) + assert cat.downloaded(b) == tmp_path / "bundles" / "b_1.bundle" == cat.fetch(b) e = {"internal_id": synth.remote("x/raw_1")} - assert cat.cached_raw(e) is None and cat.cached_raw({"internal_id": "Assets/x"}) is None + assert cat.downloaded_raw(e) is None and cat.downloaded_raw({"internal_id": "Assets/x"}) is None (tmp_path / "raw" / "asset" / "Android" / "x").mkdir(parents=True) (tmp_path / "raw" / "asset" / "Android" / "x" / "raw_1").write_bytes(b"@UTF") - assert cat.cached_raw(e) == cat.fetch_raw(e) + assert cat.downloaded_raw(e) == cat.fetch_raw(e) def test_pipeline_window_keeps_the_read_set_slots_busy(): diff --git a/tests/site/test_readsets.py b/tests/site/test_readsets.py index 3ba5b39..d24e4ee 100644 --- a/tests/site/test_readsets.py +++ b/tests/site/test_readsets.py @@ -456,7 +456,7 @@ def test_web_pair_that_no_region_has_is_a_usage_error(tmp_path, capsys, monkeypa monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", (tmp_path / "cdn").as_uri()) monkeypatch.setattr(web, "build", lambda *a, **k: pytest.fail("nothing may be built")) base = [ - "--cache", + "--depot", str(tmp_path / "c"), "--region", "zz", diff --git a/tests/source/test_catalog.py b/tests/source/test_catalog.py index 795ce92..0349c10 100644 --- a/tests/source/test_catalog.py +++ b/tests/source/test_catalog.py @@ -102,34 +102,34 @@ def make_apk(path): return path -def test_closure_fetch_and_cache(tmp_path): +def test_closure_fetch_and_download(tmp_path): cdn = make_cdn(tmp_path / "cdn") - cat = Catalog(remote_catalog(), tmp_path / "cache", cdn=cdn + "/", bundle_key=KEY) + cat = Catalog(remote_catalog(), tmp_path / "downloads", cdn=cdn + "/", bundle_key=KEY) assert cat.keys("Char/") == ["Char/A", "Char/B"] assert cat.has("Story/物語") and not cat.has("Char/C") assert sorted(b.name for b in cat.resolve("Char/A")) == ["a_01.bundle", "b_02.bundle"] paths = cat.fetch_key("Char/A") assert sorted(p.name for p in paths) == ["a_01.bundle", "b_02.bundle"] for p in paths: - assert p.parent == tmp_path / "cache" / "bundles" + assert p.parent == tmp_path / "downloads" / "bundles" assert p.read_bytes() == synth.fake_bundle(p.name) - # cached bundles are not fetched again + # downloaded bundles are not fetched again for p in (tmp_path / "cdn" / "asset" / "Android").iterdir(): p.unlink() - assert cat.fetch_key("Char/B") == [tmp_path / "cache" / "bundles" / "b_02.bundle"] + assert cat.fetch_key("Char/B") == [tmp_path / "downloads" / "bundles" / "b_02.bundle"] with pytest.raises(KeyError): cat.resolve("Char/C") def test_encrypted_bundle_needs_key(tmp_path): - cat = Catalog(remote_catalog(), tmp_path / "cache", cdn=make_cdn(tmp_path / "cdn")) + cat = Catalog(remote_catalog(), tmp_path / "downloads", cdn=make_cdn(tmp_path / "cdn")) with pytest.raises(RuntimeError, match="no bundle key"): cat.fetch_key("Char/B") def test_apk_catalog_is_merged(tmp_path): apk = make_apk(tmp_path / "base.apk") - cat = Catalog(remote_catalog(), tmp_path / "cache", cdn=make_cdn(tmp_path / "cdn"), bundle_key=KEY, apk=apk) + cat = Catalog(remote_catalog(), tmp_path / "downloads", cdn=make_cdn(tmp_path / "cdn"), bundle_key=KEY, apk=apk) assert "Shared/S" in cat.keys() (b,) = cat.resolve("Shared/S") assert not b.remote and b.offset >= APK_OFFSET_BASE @@ -141,7 +141,7 @@ def test_apk_catalog_is_merged(tmp_path): def test_apk_bundles_skipped_without_apk(tmp_path): - cat = Catalog(synth.CatalogWriter().build(APK_ENTRIES), tmp_path / "cache", bundle_key=KEY) + cat = Catalog(synth.CatalogWriter().build(APK_ENTRIES), tmp_path / "downloads", bundle_key=KEY) assert cat.fetch_key("Shared/S") == [] @@ -149,13 +149,13 @@ def test_load_downloads_catalog_once(tmp_path): cdn_root = tmp_path / "cdn" (cdn_root / "asset" / "Android").mkdir(parents=True) (cdn_root / "asset" / "Android" / "catalog_main_xx.bin").write_bytes(remote_catalog()) - cat = Catalog.load("xx", tmp_path / "cache", cdn=cdn_root.as_uri()) - assert (tmp_path / "cache" / "catalog_main_xx.bin").read_bytes() == remote_catalog() + cat = Catalog.load("xx", tmp_path / "downloads", cdn=cdn_root.as_uri()) + assert (tmp_path / "downloads" / "catalog_main_xx.bin").read_bytes() == remote_catalog() assert cat.keys("Char/") == ["Char/A", "Char/B"] (cdn_root / "asset" / "Android" / "catalog_main_xx.bin").unlink() - assert Catalog.load("xx", tmp_path / "cache").keys("Story/") == ["Story/物語"] + assert Catalog.load("xx", tmp_path / "downloads").keys("Story/") == ["Story/物語"] with pytest.raises(FileNotFoundError): - Catalog.load("yy", tmp_path / "cache") + Catalog.load("yy", tmp_path / "downloads") def test_downloads_keep_one_connection_per_thread(monkeypatch): diff --git a/tests/source/test_catalog_lazy.py b/tests/source/test_catalog_lazy.py index 60aec10..295dbbd 100644 --- a/tests/source/test_catalog_lazy.py +++ b/tests/source/test_catalog_lazy.py @@ -43,12 +43,12 @@ def f(): return f -def test_cached_bundles_read_no_settings(tmp_path): +def test_downloaded_bundles_read_no_settings(tmp_path): cdn = cdn_tree(tmp_path / "cdn") - first = Catalog(synth.CatalogWriter().build(ENTRIES), tmp_path / "cache", cdn=cdn, bundle_key=KEY) + first = Catalog(synth.CatalogWriter().build(ENTRIES), tmp_path / "downloads", cdn=cdn, bundle_key=KEY) (p,) = first.fetch_key("Char/A") again = Catalog( - synth.CatalogWriter().build(ENTRIES), tmp_path / "cache", cdn=never("cdn"), bundle_key=never("bundle key") + synth.CatalogWriter().build(ENTRIES), tmp_path / "downloads", cdn=never("cdn"), bundle_key=never("bundle key") ) assert again.fetch_key("Char/A") == [p] and again.fetch_key("Char/B") == [p] assert p.read_bytes() == synth.fake_bundle("a_01.bundle") @@ -59,7 +59,7 @@ def test_settings_are_read_once_when_a_download_needs_them(tmp_path): calls = [] cat = Catalog( synth.CatalogWriter().build(ENTRIES), - tmp_path / "cache", + tmp_path / "downloads", cdn=lambda: calls.append("cdn") or cdn, bundle_key=lambda: calls.append("key") or KEY, ) @@ -74,19 +74,23 @@ def test_settings_are_read_once_when_a_download_needs_them(tmp_path): def test_missing_setting_names_the_file_and_the_setting(tmp_path): cdn = cdn_tree(tmp_path / "cdn") cat = Catalog( - synth.CatalogWriter().build(ENTRIES), tmp_path / "cache", cdn=missing("servers.zz.cdn"), bundle_key=KEY + synth.CatalogWriter().build(ENTRIES), tmp_path / "downloads", cdn=missing("servers.zz.cdn"), bundle_key=KEY ) with pytest.raises(ConfigError) as e: cat.fetch_key("Char/A") - assert str(e.value) == "a_01.bundle is not in the cache: setting servers.zz.cdn is not set" - cat = Catalog(synth.CatalogWriter().build(ENTRIES), tmp_path / "cache", cdn=cdn, bundle_key=missing("bundle.key")) - with pytest.raises(ConfigError, match="^a_01.bundle is not in the cache: setting bundle.key is not set$"): + assert str(e.value) == "a_01.bundle is not downloaded: setting servers.zz.cdn is not set" + cat = Catalog( + synth.CatalogWriter().build(ENTRIES), tmp_path / "downloads", cdn=cdn, bundle_key=missing("bundle.key") + ) + with pytest.raises(ConfigError, match="^a_01.bundle is not downloaded: setting bundle.key is not set$"): cat.fetch_key("Char/A") def test_missing_key_fails_before_the_download(tmp_path): gone = (tmp_path / "no-such-cdn").as_uri() # a download would fail with URLError - cat = Catalog(synth.CatalogWriter().build(ENTRIES), tmp_path / "cache", cdn=gone, bundle_key=missing("bundle.key")) + cat = Catalog( + synth.CatalogWriter().build(ENTRIES), tmp_path / "downloads", cdn=gone, bundle_key=missing("bundle.key") + ) with pytest.raises(ConfigError, match="bundle.key"): cat.fetch_key("Char/A") @@ -107,7 +111,7 @@ def test_apk_bundles_read_the_key_only_when_encrypted(tmp_path): z.writestr("assets/aa/Android/p_04.bundle", synth.fake_bundle("p_04.bundle")) cat = Catalog( synth.CatalogWriter().build(ENTRIES), - tmp_path / "cache", + tmp_path / "downloads", cdn=never("cdn"), bundle_key=missing("bundle.key"), apk=apk, @@ -115,19 +119,19 @@ def test_apk_bundles_read_the_key_only_when_encrypted(tmp_path): (p,) = cat.fetch_key("Shared/P") # plain: no key assert p.read_bytes() == synth.fake_bundle("p_04.bundle") assert cat.apk_bundle("p_04").read_bytes() == synth.fake_bundle("p_04.bundle") - with pytest.raises(ConfigError, match="^s_03.bundle is not in the cache: setting bundle.key"): + with pytest.raises(ConfigError, match="^s_03.bundle is not downloaded: setting bundle.key"): cat.fetch_key("Shared/S") - with pytest.raises(ConfigError, match="^s_03.bundle is not in the cache"): + with pytest.raises(ConfigError, match="^s_03.bundle is not downloaded"): cat.apk_bundle("s_03") -def test_load_reads_the_cdn_only_for_an_uncached_catalog(tmp_path): +def test_load_reads_the_cdn_only_for_a_catalog_not_downloaded(tmp_path): cdn = cdn_tree(tmp_path / "cdn") - with pytest.raises(ConfigError, match="^catalog_main_xx.bin is not in the cache: setting servers.zz.cdn"): - Catalog.load("xx", tmp_path / "cache", cdn=missing("servers.zz.cdn")) + with pytest.raises(ConfigError, match="^catalog_main_xx.bin is not downloaded: setting servers.zz.cdn"): + Catalog.load("xx", tmp_path / "downloads", cdn=missing("servers.zz.cdn")) calls = [] - Catalog.load("xx", tmp_path / "cache", cdn=lambda: calls.append(1) or cdn) - cat = Catalog.load("xx", tmp_path / "cache", cdn=never("cdn"), bundle_key=never("bundle key")) + Catalog.load("xx", tmp_path / "downloads", cdn=lambda: calls.append(1) or cdn) + cat = Catalog.load("xx", tmp_path / "downloads", cdn=never("cdn"), bundle_key=never("bundle key")) assert calls == [1] and cat.keys("Char/") == ["Char/A", "Char/B"] @@ -186,10 +190,10 @@ def run(argv, capsys): return code, out, err -def test_pull_of_cached_bundles_needs_no_region_cdn_or_key(tmp_path, capsys, monkeypatch): +def test_pull_of_downloaded_bundles_needs_no_region_cdn_or_key(tmp_path, capsys, monkeypatch): catbin = tmp_path / "catalog_main_xx.bin" catbin.write_bytes(synth.CatalogWriter().build(ENTRIES)) - base = ["--catalog", str(catbin), "--cache", str(tmp_path / "cache")] + base = ["--catalog", str(catbin), "--depot", str(tmp_path / "depot")] monkeypatch.setenv("NNNOTES_BUNDLE_KEY", synth.BUNDLE_KEY.hex()) monkeypatch.setenv("NNNOTES_BUNDLE_NONCE_SEED", synth.BUNDLE_SEED.hex()) monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", cdn_tree(tmp_path / "cdn")) @@ -207,9 +211,9 @@ def test_pull_that_downloads_names_the_missing_setting_in_one_line(tmp_path, cap catbin.write_bytes(synth.CatalogWriter().build(ENTRIES)) monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", cdn_tree(tmp_path / "cdn")) code, out, err = run( - ["--catalog", str(catbin), "--cache", str(tmp_path / "cache"), "--region", "zz", "bundle", "pull", "Char/A"], + ["--catalog", str(catbin), "--depot", str(tmp_path / "depot"), "--region", "zz", "bundle", "pull", "Char/A"], capsys, ) assert code == 2 and out == "" and len(err.strip().splitlines()) == 1 - assert err.startswith("nnnotes: a_01.bundle is not in the cache: setting bundle.key") + assert err.startswith("nnnotes: a_01.bundle is not downloaded: setting bundle.key") assert "NNNOTES_BUNDLE_KEY" in err diff --git a/tests/source/test_catalog_versions.py b/tests/source/test_catalog_versions.py index aebfcc4..c0d56cf 100644 --- a/tests/source/test_catalog_versions.py +++ b/tests/source/test_catalog_versions.py @@ -33,7 +33,7 @@ def config(tmp_path, *, api=True, version=None, region="tw"): "servers": { region: {"cdn": "https://cdn.example/prod/" + region, **({"api": "https://api.example"} if api else {})} }, - "paths": {"cache": str(tmp_path / "cache")}, + "paths": {"depot": str(tmp_path / "depot")}, }, environ={}, ) @@ -43,9 +43,9 @@ def test_auto_discovery_ignores_stale_main_and_advances_resource_version(tmp_pat from nnnotes.site.webmodel import catalog_models cfg = config(tmp_path) - cache = cfg.require_path("paths", "cache") - cache.mkdir() - catalog.Catalog.cache_file("en", cache).write_bytes(binary("Old")) + downloads = Session(cfg).downloads() + downloads.mkdir(parents=True) + (downloads / catalog.Catalog.file_name("en")).write_bytes(binary("Old")) publication = gameapi.MasterVersion("master-one", "1.0.0.201") seen = [] @@ -71,20 +71,20 @@ def download(url): publication = gameapi.MasterVersion("master-two", "1.0.0.202") second = Session(cfg).catalog(bundles=False) assert second.has("master-two") and not second.has("master-one") - assert first.cache_dir != second.cache_dir + assert first.download_dir != second.download_dir assert all("catalog_main" not in request for request in seen) -def test_explicit_version_cache_is_offline_and_cdn_scoped(tmp_path, monkeypatch): +def test_explicit_version_download_is_offline_and_cdn_scoped(tmp_path, monkeypatch): cfg = config(tmp_path, version="1.0.0.201") monkeypatch.setattr(Session, "master_version", lambda *args: pytest.fail("pin called Version")) monkeypatch.setattr(catalog, "download", lambda url: binary("First")) first = Session(cfg).catalog(bundles=False) - monkeypatch.setattr(catalog, "download", lambda url: pytest.fail("cached pin downloaded")) + monkeypatch.setattr(catalog, "download", lambda url: pytest.fail("downloaded pin downloaded")) assert Session(cfg).catalog(bundles=False).has("First") monkeypatch.setattr(catalog, "download", lambda url: binary("Other region")) other = Session(config(tmp_path, version="1.0.0.201", region="en")).catalog(bundles=False) - assert other.has("Other region") and first.cache_dir != other.cache_dir + assert other.has("Other region") and first.download_dir != other.download_dir def test_explicit_catalog_file_stays_offline(tmp_path, monkeypatch): @@ -94,7 +94,7 @@ def test_explicit_catalog_file_stays_offline(tmp_path, monkeypatch): { "catalog": {"region": "tw", "version": "1.0.0.201"}, "servers": {"tw": {"api": "https://api.example", "catalog": str(path)}}, - "paths": {"cache": str(tmp_path / "cache")}, + "paths": {"depot": str(tmp_path / "depot")}, }, environ={}, ) @@ -104,9 +104,9 @@ def test_explicit_catalog_file_stays_offline(tmp_path, monkeypatch): def test_missing_versioned_catalog_does_not_fall_back_to_main(tmp_path, monkeypatch): cfg = config(tmp_path, version="1.0.0.201") - cache = cfg.require_path("paths", "cache") - cache.mkdir() - catalog.Catalog.cache_file("en", cache).write_bytes(binary("Old")) + downloads = Session(cfg).downloads() + downloads.mkdir(parents=True) + (downloads / catalog.Catalog.file_name("en")).write_bytes(binary("Old")) urls = [] def missing(url): @@ -117,7 +117,7 @@ def missing(url): with pytest.raises(urllib.error.HTTPError): Session(cfg).catalog(bundles=False) assert len(urls) == 1 and "catalog_1.0.0.201_en.bin" in urls[0] - assert not list((cache / "international").rglob("*.bin")) + assert not list((downloads / "international").rglob("*.bin")) def test_failed_discovery_does_not_claim_main_is_current(tmp_path, monkeypatch): @@ -156,7 +156,7 @@ def test_build_workers_inherit_the_version_pin(tmp_path, monkeypatch): monkeypatch.setenv("NNNOTES_SERVERS_TW_CATALOG_VERSION", "1.0.0.201") monkeypatch.setenv("NNNOTES_SERVERS_TW_CDN", "https://cdn.example/prod/tw") monkeypatch.setenv("NNNOTES_SERVERS_TW_API", "https://unavailable.example") - monkeypatch.setenv("NNNOTES_PATHS_CACHE", str(tmp_path / "cache")) + monkeypatch.setenv("NNNOTES_PATHS_DEPOT", str(tmp_path / "depot")) monkeypatch.setattr(catalog, "download", lambda url: binary(MODEL, EPISODE)) assert Session(Config.load()).catalog(bundles=False).has(MODEL) command = [ @@ -175,7 +175,7 @@ def test_invalid_version_cannot_be_used_as_a_path(tmp_path, version): with pytest.raises(ConfigError): Session(config(tmp_path, version=version)).catalog_version() with pytest.raises(ValueError): - catalog.Catalog.cache_file("en", tmp_path, version=version) + catalog.Catalog.file_name("en", version=version) with pytest.raises(ValueError): configfile.validate(configfile.resolve("catalog.version")[0], version) @@ -187,11 +187,11 @@ def test_empty_api_version_fails_before_download(tmp_path, monkeypatch): Session(config(tmp_path)).catalog(bundles=False) -def test_bad_download_is_not_installed_in_cache(tmp_path, monkeypatch): +def test_bad_download_is_not_installed(tmp_path, monkeypatch): monkeypatch.setattr(catalog, "download", lambda url: b"not a catalog") with pytest.raises(ValueError): Session(config(tmp_path, version="1.0.0.201")).catalog(bundles=False) - assert not list((tmp_path / "cache").rglob("*.bin")) + assert not list((tmp_path / "depot").rglob("*.bin")) def test_fetch_queries_version_before_downloading_and_labels_matching_bytes(tmp_path, monkeypatch, capsys): @@ -211,9 +211,9 @@ def fetch(cdn, language, *, version): monkeypatch.setattr(Session, "master_version", observe) monkeypatch.setattr(catalogdb, "fetch", fetch) - cli.main(["--region", "tw", "--language", "en", "--cache", str(tmp_path / "cache"), "catalog", "fetch"]) + cli.main(["--region", "tw", "--language", "en", "--depot", str(tmp_path / "depot"), "catalog", "fetch"]) assert events == ["version", "1.0.0.201"] - version = catalogdb.CatalogDB(tmp_path / "cache" / "store").versions()[0] + version = catalogdb.CatalogDB(tmp_path / "depot").versions()[0] assert version["resourceVersion"] == "1.0.0.201" assert version["remote"]["sha256"] == hashlib.sha256(data).hexdigest() assert "1.0.0.201" in capsys.readouterr().out @@ -235,7 +235,7 @@ def test_browser_uses_the_same_selector(tmp_path, monkeypatch): calls = [] monkeypatch.setattr(catalog, "download", lambda url: calls.append(url) or binary(MODEL)) handler = browse.Handler.__new__(browse.Handler) - handler.server = SimpleNamespace(catalogs={}, cache=tmp_path / "cache", bundle_key=None) + handler.server = SimpleNamespace(catalogs={}, downloads=tmp_path / "depot" / "downloads", bundle_key=None) handler.catalog(browse.Region("tw", "TW", cfg.cdn("tw"), ["en"], cfg), "en") assert len(calls) == 1 and calls[0].endswith("catalog_1.0.0.201_en.bin") diff --git a/tests/source/test_configfile.py b/tests/source/test_configfile.py index b77900e..acb20a2 100644 --- a/tests/source/test_configfile.py +++ b/tests/source/test_configfile.py @@ -79,7 +79,7 @@ def test_init_with_values(tmp_path, capsys): "--set", "catalog.region=en", "--set", - "paths.cache=cache", + "paths.depot=depot", ], capsys, ) @@ -102,7 +102,7 @@ def test_init_with_values(tmp_path, capsys): "catalog_version": "", } } # the example region table renamed - assert data["paths"]["cache"] == str((tmp_path / "cache").absolute()) + assert data["paths"]["depot"] == str((tmp_path / "depot").absolute()) assert "# CDN base URL of the region" in f.read_text(encoding="utf-8") # the comments stay if os.name != "nt": assert f.stat().st_mode & 0o777 == 0o600 @@ -240,7 +240,7 @@ def test_check_does_not_turn_jp_language_defaults_into_flags(tmp_path, capsys): def test_check_reports_states_not_values(tmp_path, capsys, monkeypatch): (tmp_path / "nnnotes.toml").write_text( f'[bundle]\nkey = "{KEY}"\n[master]\niv = "00"\n[catalog]\nregion = "xx"\n[servers.tw]\ncdn = "ftp://h"\n' - f'[paths]\napk = "missing.apk"\ncache = "c"\n[typo]\nkey = "1"\n', + f'[paths]\napk = "missing.apk"\ndepot = "c"\n[typo]\nkey = "1"\n', encoding="utf-8", ) monkeypatch.setenv("NNNOTES_BUNDLE_NONCE_SEED", "ab") @@ -256,7 +256,7 @@ def test_check_reports_states_not_values(tmp_path, capsys, monkeypatch): assert by["master.key"]["status"] == "unset" assert by["catalog.region"]["status"] == "invalid" assert by["servers.tw.cdn"]["status"] == "invalid" - assert by["paths.apk"]["status"] == "not found" and by["paths.cache"]["status"] == "ok" + assert by["paths.apk"]["status"] == "not found" and by["paths.depot"]["status"] == "ok" assert by["typo.key"]["status"] == "unknown" and by["NNNOTES_BUNDLE_KY"]["status"] == "unknown" assert doc["problems"] == 6 code, out, _ = run(["--region", "tw", "config", "check"], capsys) diff --git a/tests/source/test_jp.py b/tests/source/test_jp.py index f318fb2..60dc8ec 100644 --- a/tests/source/test_jp.py +++ b/tests/source/test_jp.py @@ -173,14 +173,14 @@ def test_authenticated_download_rotation_and_redaction(upstream): assert "Authorization" not in json.dumps(obs.source.to_dict()) -def test_truncated_download_is_not_cached(upstream, tmp_path): +def test_truncated_download_is_not_kept(upstream, tmp_path): client = jp.Client(upstream.cfg, "jp") source = client.observe().source upstream.content_length = len(upstream.body) + 5 cat = Catalog(cat_bytes(), tmp_path, source=source, client=client) with pytest.raises(gameapi.GameApiError, match="truncated"): cat.fetch_raw({"internal_id": jp.PLACEHOLDER + "audio/test.acb"}) - assert cat.cached_raw({"internal_id": jp.PLACEHOLDER + "audio/test.acb"}) is None + assert cat.downloaded_raw({"internal_id": jp.PLACEHOLDER + "audio/test.acb"}) is None def test_declared_download_limit_is_checked(upstream): @@ -246,16 +246,16 @@ def test_placeholder_rejects_unsafe_paths(path): location_kind(jp.PLACEHOLDER + path) -def test_catalog_cache_source_and_offline_replay(upstream, tmp_path, monkeypatch): +def test_catalog_download_source_and_offline_replay(upstream, tmp_path, monkeypatch): cfg = upstream.cfg - cfg._data["paths"] = {"cache": str(tmp_path / "cache")} + cfg._data["paths"] = {"depot": str(tmp_path / "depot")} cat = Session(cfg).catalog(bundles=False) - path = cat.cache_dir / "catalog_main.bin" + path = cat.download_dir / "catalog_main.bin" assert path.read_bytes() == upstream.body assert jp.read_source(path, upstream.body) == cat.source - assert cat.cache_dir != tmp_path / "cache" + assert cat.download_dir != tmp_path / "depot" / "downloads" assert cat.keys("Live/") == ["Live/MusicScore/0001/0001_00"] - record = catalogdb.CatalogDB(tmp_path / "store").add(upstream.body, source=cat.source.to_dict(), region="jp") + record = catalogdb.CatalogDB(tmp_path / "depot").add(upstream.body, source=cat.source.to_dict(), region="jp") assert upstream.credential not in json.dumps(record) cfg._data["paths"]["catalog"] = str(path) monkeypatch.setattr(jp.Client, "observe", lambda *a, **k: pytest.fail("offline read made an API call")) @@ -267,27 +267,27 @@ def test_catalog_cache_source_and_offline_replay(upstream, tmp_path, monkeypatch def test_catalogdb_source_identity_and_fetcher(upstream, tmp_path): cfg = upstream.cfg - cfg._data["paths"] = {"cache": str(tmp_path / "cache")} + cfg._data["paths"] = {"depot": str(tmp_path / "depot")} client = jp.Client(cfg, "jp") source = client.observe().source - db = catalogdb.CatalogDB(tmp_path / "store") + db = catalogdb.CatalogDB(tmp_path / "depot") old = db.add(cat_bytes(), source=source.to_dict(), region="jp") new_source = {**source.to_dict(), "hash": "c" * 32} new = db.add(cat_bytes(), source=new_source, region="jp") assert old["id"] != new["id"] and old["labels"] != new["labels"] assert old["remote"] == new["remote"] - fetcher = CatalogFetcher(tmp_path / "store", tmp_path / "cache", cfg) + fetcher = CatalogFetcher(tmp_path / "depot", cfg) cat, _ = fetcher.catalog(old["id"]) - assert cat.source == source and cat.cache_dir == source.cache_dir(tmp_path / "cache") + assert cat.source == source and cat.download_dir == source.download_dir(tmp_path / "depot" / "downloads") -def test_apk_update_with_same_asset_snapshot_does_not_reuse_embedded_cache(tmp_path): +def test_apk_update_with_same_asset_snapshot_does_not_reuse_embedded_downloads(tmp_path): from nnnotes.source.catalog import APK_AA_DIR, APK_CATALOG source = jp.Source("jp", "1.0", HASH, "https://cdn.invalid") apk = tmp_path / "base.apk" - cache = tmp_path / "cache" - db = catalogdb.CatalogDB(tmp_path / "store") + downloads = tmp_path / "depot" / "downloads" + db = catalogdb.CatalogDB(tmp_path / "depot") cfg = Config({"catalog": {"region": "jp"}, "paths": {"apk": str(apk)}}, environ={}) from nnnotes.cli.assets import Workspace @@ -298,14 +298,14 @@ def test_apk_update_with_same_asset_snapshot_does_not_reuse_embedded_cache(tmp_p ) content = b"UnityFS\0" + bytes([revision]) apk.write_bytes(zip_bytes({APK_CATALOG: local, APK_AA_DIR + "Android/local.bundle": content})) - cat = Catalog(cat_bytes(), source.cache_dir(cache), source=source, apk=apk) + cat = Catalog(cat_bytes(), source.download_dir(downloads), source=source, apk=apk) v = db.add(cat_bytes(), local, source=source.to_dict(), region="jp") workspace = Workspace.__new__(Workspace) workspace.version = v - rel = workspace._cache_rel("bundle", {"name": "local.bundle", "remote": False}, {}) + rel = workspace._download_rel("bundle", {"name": "local.bundle", "remote": False}, {}) assert cat.fetch(cat.resolve("Local")[0]).read_bytes() == content - assert (cache / rel).read_bytes() == content - fetcher = CatalogFetcher(tmp_path / "store", cache, cfg) + assert (downloads / rel).read_bytes() == content + fetcher = CatalogFetcher(tmp_path / "depot", cfg) replay, _ = fetcher.catalog(v["id"]) assert replay.fetch(replay.resolve("Local")[0]).read_bytes() == content assert len(db.versions()) == 2 @@ -329,7 +329,7 @@ def test_remote_replay_does_not_open_the_apk(upstream, tmp_path, apk_state, kind cfg = upstream.cfg cfg._data["bundle"] = {"key": "00" * 16, "nonce_seed": "11"} - cfg._data["paths"] = {"cache": str(tmp_path / "cache")} + cfg._data["paths"] = {"depot": str(tmp_path / "depot")} apk = tmp_path / "base.apk" if apk_state != "unset": cfg._data["paths"]["apk"] = str(apk) @@ -338,18 +338,18 @@ def test_remote_replay_does_not_open_the_apk(upstream, tmp_path, apk_state, kind elif apk_state == "broken": apk.write_bytes(b"not a zip") source = jp.Client(cfg, "jp").observe().source - db = catalogdb.CatalogDB(tmp_path / "store") + db = catalogdb.CatalogDB(tmp_path / "depot") record = db.add(cat_bytes(), local_catalog(), source=source.to_dict(), region="jp") entry = next(e for e in db.index(record)[kind] if e["remote"]) upstream.body = b"UnityFS\0synthetic resource" - fetcher = CatalogFetcher(tmp_path / "store", tmp_path / "cache", cfg) + fetcher = CatalogFetcher(tmp_path / "depot", cfg) assert fetcher.fetch_location(record["id"], entry["location"]).read_bytes() == upstream.body assert len(upstream.seen) == 1 @pytest.mark.parametrize("kind", ["bundles", "rawFiles"]) -@pytest.mark.parametrize("state", ["matching", "different", "no-catalog", "cached"]) -def test_local_replay_validates_only_on_cache_miss(tmp_path, kind, state): +@pytest.mark.parametrize("state", ["matching", "different", "no-catalog", "downloaded"]) +def test_local_replay_validates_only_when_not_downloaded(tmp_path, kind, state): from nnnotes.source.catalog import APK_AA_DIR, APK_CATALOG source = jp.Source("jp", "1.0", HASH, "https://cdn.invalid") @@ -359,20 +359,20 @@ def test_local_replay_validates_only_on_cache_miss(tmp_path, kind, state): members[APK_CATALOG] = local_catalog(2 if state == "different" else 1) apk.write_bytes(zip_bytes(members)) cfg = Config({"catalog": {"region": "jp"}, "paths": {"apk": str(apk)}}, environ={}) - db = catalogdb.CatalogDB(tmp_path / "store") + db = catalogdb.CatalogDB(tmp_path / "depot") record = db.add(cat_bytes(), local_catalog(), source=source.to_dict(), region="jp") entry = next(e for e in db.index(record)[kind] if not e["remote"]) - fetcher = CatalogFetcher(tmp_path / "store", tmp_path / "cache", cfg) + fetcher = CatalogFetcher(tmp_path / "depot", cfg) if state in ("different", "no-catalog"): with pytest.raises(ConfigError, match="APK set it was imported with"): fetcher.fetch_location(record["id"], entry["location"]) return expected = members[APK_AA_DIR + ("Android/local.bundle" if kind == "bundles" else "Android/local.acb")] assert fetcher.fetch_location(record["id"], entry["location"]).read_bytes() == expected - if state == "cached": + if state == "downloaded": apk.unlink() cfg._data["paths"].pop("apk") - replay = CatalogFetcher(tmp_path / "store", tmp_path / "cache", cfg) + replay = CatalogFetcher(tmp_path / "depot", cfg) assert replay.fetch_location(record["id"], entry["location"]).read_bytes() == expected @@ -522,16 +522,16 @@ def test_jp_master_rejects_missing_or_malformed_file_lists(tmp_path, files): assert not (tmp_path / "MasterManifest.json").exists() -def test_export_cache_locator_uses_source_namespace(tmp_path): +def test_export_download_locator_uses_source_namespace(tmp_path): from nnnotes.cli.assets import Workspace source = jp.Source("jp", "1.0.0.300", HASH, "https://cdn.invalid") workspace = Workspace.__new__(Workspace) workspace.version = {"source": source.to_dict()} - rel = workspace._cache_rel("bundle", {"name": "same.bundle"}, {}) - assert tmp_path / rel == source.cache_dir(tmp_path) / "bundles/same.bundle" - rel = workspace._cache_rel("raw", {}, {"internalId": jp.PLACEHOLDER + "audio/sample.acb"}) - assert tmp_path / rel == source.cache_dir(tmp_path) / "raw/audio/sample.acb" + rel = workspace._download_rel("bundle", {"name": "same.bundle"}, {}) + assert tmp_path / rel == source.download_dir(tmp_path) / "bundles/same.bundle" + rel = workspace._download_rel("raw", {}, {"internalId": jp.PLACEHOLDER + "audio/sample.acb"}) + assert tmp_path / rel == source.download_dir(tmp_path) / "raw/audio/sample.acb" def test_music_data_preserves_asset_hash(tmp_path): From 00f6e200dd20741b81acad5bb0e876aabaf34445 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 14 Oct 2026 21:55:46 +0800 Subject: [PATCH 02/91] refactor(scripts): use the depot in the atom evaluation script The evaluation script reads and writes a depot (`--base DEPOT`, `--depot D`) like the commands do. Co-Authored-By: Claude Opus 5.5 --- scripts/atom_eval.py | 78 ++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/scripts/atom_eval.py b/scripts/atom_eval.py index 327520e..ece713e 100644 --- a/scripts/atom_eval.py +++ b/scripts/atom_eval.py @@ -11,9 +11,9 @@ m1 DIR --atom NAME=SPEC ... conformance: every object of the checked classes through the reference and through each candidate; per check and class (or texture format): equal / differ / refused / ... - prepare DIR --base STORE census and link.scripts of the bundles into a base store (the tasks of + prepare DIR --base DEPOT census and link.scripts of the bundles into a base depot (the tasks of `m3` are described from it) - m3 DIR --base STORE --store S cost of a stage per bundle: wall, user + system CPU and peak resident + m3 DIR --base DEPOT --depot D cost of a stage per bundle: wall, user + system CPU and peak resident memory per task, in a worker pool (per-worker peak) or with --fresh in a new process per bundle (imports measured apart) imports [--module M ...] import cost of modules, each set in new processes @@ -1036,12 +1036,12 @@ def versions() -> dict: return out -# ---------------------------------------------------------------- prepare (base store) +# ---------------------------------------------------------------- prepare (base depot) def _identify(job): - from nnnotes.pipeline.store import Store + from nnnotes.pipeline.depot import Depot - root, store, name = job - s, size = Store(store).identify(Path(root) / name) + root, depot, name = job + s, size = Depot(depot).identify(Path(root) / name) return name, s, size @@ -1049,20 +1049,20 @@ def _census_task(task): from nnnotes.pipeline import census from nnnotes.pipeline.stages import execute - ex = execute(task, G["store"], {"unity.census": census.CensusStage()}) + ex = execute(task, G["depot"], {"unity.census": census.CensusStage()}) return task.subject, task.key, ex.status def cmd_prepare(a) -> dict: from nnnotes.pipeline import census, contract, link from nnnotes.pipeline.contract import Input + from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env, describe, execute - from nnnotes.pipeline.store import Store check_clean_env() root, base = Path(a.dir).resolve(), Path(a.base) items = bundle_list(root, a.bundles) - store = Store(base) + depot = Depot(base) t0 = time.perf_counter() ids = pool_map( _identify, [(str(root), str(base), n) for n, _ in sorted(items, key=lambda x: -x[1])], a.workers, maxtasks=None @@ -1074,14 +1074,14 @@ def cmd_prepare(a) -> dict: subject = stable[names[n]] inputs[subject] = Input("bundle", s, size, names[n], ({"kind": "file", "path": str(root / n)},)) subj_name[subject] = n - env = Env(store, {"bundles": inputs}) + env = Env(depot, {"bundles": inputs}) tasks = [describe(census.CensusStage(), s, None, env) for s in sorted(inputs, key=lambda s: -inputs[s].size)] - G["store"] = store + G["depot"] = depot res = pool_map(_census_task, tasks, a.workers) for t in tasks: env.done(t.id, t.key) st = describe(link.ScriptsStage(), "all", None, env) - execute(st, store, {"link.scripts": link.ScriptsStage()}) + execute(st, depot, {"link.scripts": link.ScriptsStage()}) doc = { "dir": str(root), "bundles": { @@ -1101,8 +1101,8 @@ def cmd_prepare(a) -> dict: # ---------------------------------------------------------------- m3 (stage cost) def _prepared_env(base: Path): from nnnotes.pipeline.contract import Input + from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env - from nnnotes.pipeline.store import Store prep = json.loads((base / PREPARED).read_text(encoding="utf-8")) root = Path(prep["dir"]) @@ -1116,16 +1116,16 @@ def _prepared_env(base: Path): ) for s, b in prep["bundles"].items() } - env = Env(Store(base), {"bundles": inputs}) + env = Env(Depot(base), {"bundles": inputs}) for s, k in prep["census"].items(): env.done(f"unity.census:{s}", k) env.done("link.scripts:all", prep["scripts"]) return prep, env -def _new_store(base: Path, path: Path): - """A run store that knows the base store's input identities (so inputs are not hashed again).""" - from nnnotes.pipeline.store import Store +def _new_depot(base: Path, path: Path): + """A run depot that knows the base depot's input identities (so inputs are not hashed again).""" + from nnnotes.pipeline.depot import Depot path.mkdir(parents=True, exist_ok=True) src = base / "inputs" @@ -1134,7 +1134,7 @@ def _new_store(base: Path, path: Path): for f in src.iterdir(): if f.is_file() and not (path / "inputs" / f.name).exists(): shutil.copyfile(f, path / "inputs" / f.name) - return Store(path) + return Depot(path) def _stage(name: str): @@ -1144,7 +1144,7 @@ def _stage(name: str): return {"unity.export": ExportStage, "unity.census": census.CensusStage}[name]() -def _run_task(task, store, stage_name: str) -> dict: +def _run_task(task, depot, stage_name: str) -> dict: from nnnotes.pipeline.stages import execute hwm = reset_hwm() @@ -1152,7 +1152,7 @@ def _run_task(task, store, stage_name: str) -> dict: t0 = time.perf_counter() rec = {"subject": task.subject, "key": task.key} try: - ex = execute(task, store, {stage_name: _stage(stage_name)}, force=True) + ex = execute(task, depot, {stage_name: _stage(stage_name)}, force=True) rec.update(status=ex.status, result=ex.result_status) except Exception as e: # noqa: BLE001 rec.update(status="error", error=message(e), trace=traceback.format_exc()[-1200:]) @@ -1169,7 +1169,7 @@ def _run_task(task, store, stage_name: str) -> dict: def _m3_worker(task) -> dict: - return _run_task(task, G["store"], G["stage"]) + return _run_task(task, G["depot"], G["stage"]) def _apply_atoms(atoms) -> None: @@ -1229,19 +1229,19 @@ def summarize_runs(recs: list) -> dict: def cmd_m3(a) -> dict: check_clean_env() - base, out_store = Path(a.base), Path(a.store) + base, out_depot = Path(a.base), Path(a.depot) atoms = parse_atoms(a.atom) params = parse_params(a.param) or None prep, env = _prepared_env(base) subjects = _subjects(prep, a.bundles) if a.fresh: - return m3_fresh(a, base, out_store, atoms, params, prep, subjects) + return m3_fresh(a, base, out_depot, atoms, params, prep, subjects) imp = _import_stage_modules() _apply_atoms(atoms) tasks = _describe_all(a.stage, subjects, params, env) tasks.sort(key=lambda t: -t.inputs[0].size) - store = _new_store(base, out_store) - G.update(store=store, stage=a.stage) + depot = _new_depot(base, out_depot) + G.update(depot=depot, stage=a.stage) log("m3", a.stage, len(tasks), "tasks,", a.workers, "workers, atoms", dict(atoms), "params", params) jsonl = Path(a.out).with_suffix(".jsonl") if a.out else None fo = open(jsonl, "w", encoding="utf-8", newline="\n") if jsonl else None @@ -1266,7 +1266,7 @@ def take(r): "maxtasks": a.maxtasks, "atoms": dict(atoms), "params": params, - "store": str(out_store.resolve()), + "depot": str(out_depot.resolve()), "taskAtoms": tasks[0].atoms if tasks else None, "wall": round(time.perf_counter() - t0, 2), "childrenCpu": round((c1.children_user - c0.children_user) + (c1.children_system - c0.children_system), 2), @@ -1281,18 +1281,18 @@ def take(r): return doc -def m3_fresh(a, base, out_store, atoms, params, prep, subjects) -> dict: +def m3_fresh(a, base, out_depot, atoms, params, prep, subjects) -> dict: """One new process per bundle: its wall, user + system CPU and peak RSS (wait4), and inside it the import time and the task's own cost.""" - _new_store(base, out_store) + _new_depot(base, out_depot) cmd = [ sys.executable, str(Path(__file__).resolve()), "m3-one", "--base", str(base), - "--store", - str(out_store), + "--depot", + str(out_depot), "--stage", a.stage, ] @@ -1306,7 +1306,7 @@ def m3_fresh(a, base, out_store, atoms, params, prep, subjects) -> dict: queue = sorted(subjects, key=lambda s: -sizes[s]) running: dict[int, tuple] = {} recs = [] - tmp = out_store / ".atom_eval" + tmp = out_depot / ".atom_eval" tmp.mkdir(exist_ok=True) jsonl = Path(a.out).with_suffix(".jsonl") if a.out else None fo = open(jsonl, "w", encoding="utf-8", newline="\n") if jsonl else None @@ -1362,7 +1362,7 @@ def m3_fresh(a, base, out_store, atoms, params, prep, subjects) -> dict: "workers": a.workers, "atoms": dict(atoms), "params": params, - "store": str(out_store.resolve()), + "depot": str(out_depot.resolve()), "wall": round(time.perf_counter() - t0, 2), "tasks": len(recs), "ok": len(ok), @@ -1400,9 +1400,9 @@ def cmd_m3_one(a) -> None: prep, env = _prepared_env(base) (task,) = _describe_all(a.stage, [a.subject], parse_params(a.param) or None, env) describe_cost = {"wall": round(time.perf_counter() - t0, 4), "cpu": round(time.process_time() - cd, 4)} - from nnnotes.pipeline.store import Store + from nnnotes.pipeline.depot import Depot - rec = _run_task(task, Store(a.store), a.stage) + rec = _run_task(task, Depot(a.depot), a.stage) rec.update(imports=imp, describe=describe_cost, processCpuInside=round(time.process_time() - c0, 4)) print(json.dumps(rec, ensure_ascii=False)) @@ -1525,10 +1525,10 @@ def _compare_job(job) -> dict: def cmd_compare(a) -> dict: - from nnnotes.pipeline.store import Store + from nnnotes.pipeline.depot import Depot ra, rb = (json.loads(Path(p).read_text(encoding="utf-8")) for p in (a.run_a, a.run_b)) - G.update(sa=Store(ra["store"]), sb=Store(rb["store"])) + G.update(sa=Depot(ra["depot"]), sb=Depot(rb["depot"])) ta, tb = ra["taskKeys"], rb["taskKeys"] jobs = [(s, ta.get(s), tb.get(s)) for s in sorted(ta.keys() | tb.keys())] total = {"counts": {}, "stats": {}, "examples": {}} @@ -1536,8 +1536,8 @@ def cmd_compare(a) -> dict: merge(total, r) doc = { "command": "compare", - "a": {k: ra.get(k) for k in ("atoms", "params", "store", "workers", "mode")}, - "b": {k: rb.get(k) for k in ("atoms", "params", "store", "workers", "mode")}, + "a": {k: ra.get(k) for k in ("atoms", "params", "depot", "workers", "mode")}, + "b": {k: rb.get(k) for k in ("atoms", "params", "depot", "workers", "mode")}, "subjects": len(jobs), **total, } @@ -1583,7 +1583,7 @@ def common(p, bundles=True): else: p.add_argument("--subject", required=True) p.add_argument("--base", required=True) - p.add_argument("--store", required=True) + p.add_argument("--depot", required=True) p.add_argument("--stage", default="unity.export", choices=("unity.export", "unity.census")) p.add_argument("--atom", action="append") p.add_argument("--param", action="append", help="stage parameter path=value, e.g. png.level=1") From 0b169fa08a5b792d9608db955537924685e948c9 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 14 Oct 2026 22:18:41 +0800 Subject: [PATCH 03/91] feat!: keep the depot's downloads as a cache bounded by [depot] downloads Every command that reads the depot holds a lease in /leases/ while it runs. When the last running command ends, the bundles and raw files of downloads/ used least recently are removed until the rest fit [depot] downloads: a size (500MB, 20GiB), all, or 0 (unset). Catalog files stay. The catalog marks a file as used when it hands it out, and a removed file is downloaded again when a command needs its bytes. A command that must run alone holds leases/.exclusive: trimming the downloads, and depot gc, which exits with an error while another command uses the depot. Commands that start meanwhile wait for it. bundle pull copies the decrypted bundles of its keys to -o DIR. BREAKING CHANGE: downloaded bundles and raw files are removed when the last command ends unless [depot] downloads keeps them, and bundle pull requires -o DIR. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- README.zh-CN.md | 2 +- docs/assets.md | 4 +- docs/cli.md | 3 +- docs/commands.md | 32 +++- docs/configuration.md | 3 +- docs/contracts.md | 9 +- docs/jp.md | 2 +- src/nnnotes/cli/__init__.py | 18 +- src/nnnotes/cli/assets.py | 5 +- src/nnnotes/cli/bundle.py | 36 +++- src/nnnotes/cli/depot.py | 10 +- src/nnnotes/core/downloads.py | 220 ++++++++++++++++++++++++ src/nnnotes/pipeline/depot.py | 2 + src/nnnotes/source/catalog.py | 36 ++-- src/nnnotes/source/configfile.py | 3 + src/nnnotes/source/nnnotes.example.toml | 5 + src/nnnotes/source/session.py | 9 + tests/cli/test_cli.py | 24 ++- tests/conftest.py | 4 +- tests/core/test_downloads.py | 139 +++++++++++++++ tests/pipeline/test_depot_gc.py | 22 +++ tests/source/test_catalog_lazy.py | 8 +- 23 files changed, 551 insertions(+), 47 deletions(-) create mode 100644 src/nnnotes/core/downloads.py create mode 100644 tests/core/test_downloads.py diff --git a/README.md b/README.md index d1836c4..ac6fe38 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ questions: `nnnotes config init --user --no-input --set SECTION.KEY=VALUE ...`. | `servers` | the game API's server list: regions with their CDN and API roots | | `catalog keys\|browse` | the addressable keys of a region's catalog; a local web page to browse catalogs and bundles | | `catalog versions\|import\|fetch\|diff` | catalog versions kept in the depot, and what changed between two | -| `bundle pull` | fetch and decrypt the bundle closure of keys into the depot | +| `bundle pull` | fetch the bundle closure of keys and copy the decrypted bundles to a directory | | `master version\|download\|decode` | the master data version a region serves, its download and its decoding to JSON | | `export assets` | every object of the selected bundles in common formats, through the content-addressed depot | | `export episode\|story\|live\|live2d\|spot\|room\|shader\|audio\|ui` | one subject: an ADV episode, a story, a live, a Live2D model, a spot, a room, shaders, a cue sheet, the UI library | diff --git a/README.zh-CN.md b/README.zh-CN.md index 0d49d32..332cfc3 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -46,7 +46,7 @@ nnnotes site build -o out/site --all --player path/to/ournotes-player | `servers` | 游戏 API 的服务器列表：各区服及其 CDN 与 API 根地址 | | `catalog keys\|browse` | 区服 catalog 的资源键；在本地网页中浏览 catalog 与资源包 | | `catalog versions\|import\|fetch\|diff` | depot 中保存的 catalog 版本，以及两个版本之间的变化 | -| `bundle pull` | 下载并解密资源键的依赖闭包到 depot | +| `bundle pull` | 下载资源键的依赖闭包，把解密后的 bundle 复制到指定目录 | | `master version\|download\|decode` | 区服当前的 masterdata 版本、下载，以及解码为 JSON | | `export assets` | 经内容寻址的 depot，把所选资源包中的每个对象导出为通用格式 | | `export episode\|story\|live\|live2d\|spot\|room\|shader\|audio\|ui` | 单个对象：ADV 剧情集、完整剧情、谱面、Live2D 模型、据点、房间、着色器、cue sheet、UI 库 | diff --git a/docs/assets.md b/docs/assets.md index f52f1e6..a5471a4 100644 --- a/docs/assets.md +++ b/docs/assets.md @@ -126,7 +126,9 @@ Removes what no kept run reaches. The kept runs are, for each selection, the `-- the latest run of a selection is always kept). A kept run reaches the results of its tasks, the contents those results name (their artifacts, and the inputs their task keys name when the depot holds them) and its layout manifests with the files they list. Every other run, result (with its measured cost) and content is removed. -`--dry-run` counts what would be removed. A layout made of hard links or clones keeps its files. +`--dry-run` counts what would be removed. A layout made of hard links or clones keeps its files. gc runs only when +no other command uses the depot (it exits with an error otherwise), and commands that start meanwhile wait for it. +The downloads are bounded separately, by `[depot] downloads` ([commands.md](commands.md#downloads)). ## Layouts diff --git a/docs/cli.md b/docs/cli.md index 7077399..abe2ac7 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -128,11 +128,12 @@ keys, bundles, raw files (and objects) that differ between two catalog versions ## nnnotes bundle pull -fetch the bundle closure of keys into the depot (prints the downloaded bundle files) +fetch the bundle closure of keys and copy the decrypted bundles to a directory (prints the bundle files) | Argument | Description | | --- | --- | | `KEY` ... | addressable keys (`catalog keys` lists them) | +| `-o`, `--out` `OUT` | directory the bundles are copied to, under their file names (required) | ## nnnotes master version diff --git a/docs/commands.md b/docs/commands.md index a6b8a37..fe4738b 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -40,8 +40,8 @@ setting's value. Details in [configuration.md](configuration.md#writing-the-conf ## Downloads -`bundle pull` and every extractor read the game files from the depot's `downloads/` directory (`[paths] depot`), -which keeps the files under their own names: +The commands read the game files from the depot's `downloads/` directory (`[paths] depot`), which keeps the files +under their own names: ``` /downloads/catalog_main_.bin the main catalog (explicit main or no API/version pin) @@ -52,8 +52,26 @@ which keeps the files under their own names: versioned catalog, bundles and raw files of one CDN and version ``` -Files already downloaded are not downloaded again. When `[paths] apk` is set, the APK's own catalog is merged with -the region's and bundles that ship inside the APK are read from it. +A file is downloaded when a command first needs it and read from `downloads/` while it is there. When `[paths] apk` +is set, the APK's own catalog is merged with the region's and bundles that ship inside the APK are read from it. + +The bundles and raw files are a cache bounded by `[depot] downloads`. Every command that reads the depot holds a +lease in `/leases/` while it runs; when the last running command ends, the bundles and raw files used least +recently are removed until the rest fit the budget: + +| `[depot] downloads` | kept once no command runs | +|---|---| +| unset or `0` | no bundles or raw files | +| a size, e.g. `20GiB` | the most recently used ones, up to that size | +| `all` | every one | + +Catalog files are kept. A removed file is downloaded again when a command needs its bytes. The depot records the +identity (sha256 and size) of every bundle and raw file it reads, so `export assets` plans with files whose name +carries the catalog's hash without downloading them again. + +Commands may run at the same time on one depot: no command removes downloads while another one holds a lease, and a +command that starts while one trims waits for it. The lease of a process that ended without releasing it is dropped +by the next command; a lease written on another host counts for a day. For international regions with a configured API, catalog-based commands discover `resource_version` first and download `catalog__.bin`. `catalog fetch` labels the bytes with the version used to @@ -83,11 +101,11 @@ it, decrypted; `bundles/` lists every bundle directly. ## bundle pull ``` -nnnotes bundle pull KEY [KEY ...] +nnnotes bundle pull KEY [KEY ...] -o DIR ``` -Fetches the bundle closure (the key's location and all its dependencies) of each key into the depot's downloads and -prints the path of every bundle. +Fetches the bundle closure (the key's location and all its dependencies) of each key, copies the decrypted bundles +to `DIR` under their file names, and prints the path of every copied bundle. ## servers diff --git a/docs/configuration.md b/docs/configuration.md index f0764aa..da9b42f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -59,7 +59,7 @@ No `config` command prints a setting's value. The config files these commands wr The global flags go before or after the command name: ```bash -nnnotes --config ~/nnnotes.toml --depot /data/nnnotes-depot bundle pull +nnnotes --config ~/nnnotes.toml --depot /data/nnnotes-depot bundle pull -o bundles NNNOTES_PATHS_MASTER=/data/master nnnotes export episode 10462 -o episode.json ``` @@ -110,6 +110,7 @@ exist when they are given. | `[paths] player` | `NNNOTES_PATHS_PLAYER` | `--player` (of `site build`) | ournotes-player checkout (built) or installed package | | `[paths] fonts.` | `NNNOTES_PATHS_FONTS_` (e.g. `NNNOTES_PATHS_FONTS_ZH_HANT`) | `--font =PATH` (of `site build`) | font file (OpenType or TrueType) the story text of a language is drawn with (`--fonts open`); also read for the other story languages whose game fonts fall back to that language's font | | `[paths] fonts.emoji` | `NNNOTES_PATHS_FONTS_EMOJI` | `--font emoji=PATH` (of `site build`) | colour emoji font (PNG bitmap glyphs: CBDT or sbix) the stories' emoji sprites are drawn from (`--fonts open`); Noto Color Emoji (SIL Open Font License 1.1) is the tested font. Optional: without it the sprites keep their layout with empty glyphs, listed in `ui/fonts.json` `coverage.sprites.missing` | +| `[depot] downloads` | `NNNOTES_DEPOT_DOWNLOADS` | — | how much of the depot's downloaded bundles and raw files to keep once no command uses the depot: a size (`500MB`, `20GiB`; units B, KB, MB, GB, TB and KiB, MiB, GiB, TiB), `all`, or `0` (unset) ([commands.md](commands.md#downloads)) | | `[export] link` | `NNNOTES_EXPORT_LINK` | `--link` (of `export assets`) | how `export assets` makes the layout files from the depot: `auto` (unset), `clone`, `hard` or `copy` ([assets.md](assets.md)) | Hex values are case-insensitive; a `0x` prefix and surrounding whitespace are ignored. All values are TOML strings diff --git a/docs/contracts.md b/docs/contracts.md index 930f5ab..13b86bb 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -235,7 +235,8 @@ described in [music-data.md](music-data.md). The depot is a directory: the downloaded game files, and the results of all stage work by content: ``` -/downloads/ the game files as downloaded, under their own names ([commands.md](commands.md#downloads)) +/downloads/ the game files as downloaded, under their own names; the bundles and raw files + are kept up to `[depot] downloads` ([commands.md](commands.md#downloads)) /cas/sha256/ab/ bytes by content id, written once (no extension: records carry it) /ac/ab/.json results by task key (written after the bytes they name) /inputs/.jsonl input identities: name -> sha256, size, with the path, size and mtime of the @@ -245,9 +246,11 @@ The depot is a directory: the downloaded game files, and the results of all stag /runs/.log.jsonl the log of the run's last execution /runs/selections/ the id of the latest run of a selection (key hash of the selection list) /costs/.json the measured cost of a task's last run +/leases/.json one per running command: {"host", "pid", "started"} +/leases/.exclusive held while a command runs alone (trimming the downloads, `depot gc`) ``` Every file is written to a temporary name and renamed into place, so concurrent writers never see or leave a partial file and an interrupted task leaves no result. `cas/` and `ac/` depend only on the tasks run: two runs of the -same tasks, with any number of workers, leave identical trees. `downloads/`, `inputs/`, `runs/*.log.jsonl` and -`costs/` are local and not deterministic. +same tasks, with any number of workers, leave identical trees. `downloads/`, `inputs/`, `runs/*.log.jsonl`, +`costs/` and `leases/` are local and not deterministic. diff --git a/docs/jp.md b/docs/jp.md index 63d0862..ecccac6 100644 --- a/docs/jp.md +++ b/docs/jp.md @@ -65,7 +65,7 @@ nnnotes --region jp master version nnnotes --region jp master download --latest -o work/jp-master-bin nnnotes master decode work/jp-master-bin -o work/jp-master nnnotes --region jp catalog keys --prefix Live/MusicScore/ --limit 10 -nnnotes --region jp bundle pull Live/MusicScore/0001/0001_00 +nnnotes --region jp bundle pull Live/MusicScore/0001/0001_00 -o bundles nnnotes --region jp catalog fetch nnnotes --region jp export assets --select key:Image/Jacket/jacket_temporary --views none -o out/jp-sample nnnotes --region jp music-data --decoded-master --no-bgm -o out/jp-music-data.json diff --git a/src/nnnotes/cli/__init__.py b/src/nnnotes/cli/__init__.py index 0aa5979..756c1dc 100644 --- a/src/nnnotes/cli/__init__.py +++ b/src/nnnotes/cli/__init__.py @@ -25,6 +25,7 @@ from __future__ import annotations import argparse +import contextlib import os import sys @@ -40,6 +41,7 @@ from .common import GLOBALS, Formatter, fill_globals, load_config GROUPS = (config, servers, catalog, bundle, master, export, apk, depot, voices, site, musicdata) +NO_DEPOT = {"config", "servers", "master", "apk"} # the groups whose commands never read the depot def build_parser() -> argparse.ArgumentParser: @@ -64,7 +66,8 @@ def main(argv=None): args = parse_args(argv) try: cfg = load_config(args) if args.reads_config else None - args.func(args, cfg) + with depot_lease(args, cfg) as args.lease: + args.func(args, cfg) except ConfigError as e: print(f"nnnotes: {e}", file=sys.stderr) sys.exit(2) @@ -72,5 +75,18 @@ def main(argv=None): sys.exit(f"nnnotes: {e}") +def depot_lease(args, cfg): + """The lease a command holds on the depot while it runs (core.downloads.lease: the last command to end trims the + downloads to `[depot] downloads`; `args.lease` is its file); none for the commands that never read the depot or + without `[paths] depot`.""" + if cfg is None or args.command in NO_DEPOT or cfg.path("paths", "depot") is None: + return contextlib.nullcontext() + from ..core.downloads import lease + from ..source.session import Session + + session = Session(cfg) + return lease(session.depot_root(), session.downloads_budget()) + + if __name__ == "__main__": main() diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index 55d7a7b..6c336d8 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -328,11 +328,14 @@ def fetch_location(self, vid: str, lid: str) -> Path: def _apk_file(self, internal_id: str, cat) -> Path: """A raw APK file through its snapshot's downloads; only a file not read yet needs the matching APK.""" from ..core.cache import write_atomic + from ..core.downloads import touch from ..source.catalog import APK_AA_DIR rel = apk_rel(internal_id) dst = cat.local_download_dir() / "raw" / rel - if not (dst.is_file() and dst.stat().st_size > 0): + if dst.is_file() and dst.stat().st_size > 0: + touch(dst) + else: if cat.apk is None: raise self.cfg.missing("paths", "apk") with ApkSet(cat.apk) as z: diff --git a/src/nnnotes/cli/bundle.py b/src/nnnotes/cli/bundle.py index 14b98a6..56375da 100644 --- a/src/nnnotes/cli/bundle.py +++ b/src/nnnotes/cli/bundle.py @@ -1,21 +1,49 @@ -"""`nnnotes bundle`: fetch bundles into the depot's downloads.""" +"""`nnnotes bundle`: copy the decrypted bundles of keys out of the depot's downloads.""" from __future__ import annotations +import os +import shutil +from pathlib import Path + +from ..core.cache import temp_path from ..source.session import Session -from .common import command, group, known_keys, report +from .common import command, group, known_keys, out_arg, report def cmd_pull(args, cfg): cat = Session(cfg).catalog() known_keys(args, cat, args.keys) - files = {key: [str(p) for p in cat.fetch_key(key)] for key in args.keys} + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + copied: dict[Path, Path] = {} + files = {} + for key in args.keys: + for p in (paths := cat.fetch_key(key)): + if p not in copied: + copied[p] = _copy(p, out / p.name) + files[key] = [str(copied[p]) for p in paths] report(args, {"keys": files}, lambda r: [p for ps in r["keys"].values() for p in ps]) +def _copy(src: Path, dst: Path) -> Path: + tmp = temp_path(dst) + try: + shutil.copyfile(src, tmp) + os.replace(tmp, dst) + except BaseException: + tmp.unlink(missing_ok=True) + raise + return dst + + def register(sub) -> None: bs = group(sub, "bundle", "the bundles of the catalog") c = command( - bs, "pull", cmd_pull, "fetch the bundle closure of keys into the depot (prints the downloaded bundle files)" + bs, + "pull", + cmd_pull, + "fetch the bundle closure of keys and copy the decrypted bundles to a directory (prints the bundle files)", ) c.add_argument("keys", nargs="+", metavar="KEY", help="addressable keys (`catalog keys` lists them)") + out_arg(c, "directory the bundles are copied to, under their file names") diff --git a/src/nnnotes/cli/depot.py b/src/nnnotes/cli/depot.py index 77f5d69..d7f5ba2 100644 --- a/src/nnnotes/cli/depot.py +++ b/src/nnnotes/cli/depot.py @@ -22,11 +22,19 @@ def cmd_verify(args, cfg): def cmd_gc(args, cfg): + from ..core.downloads import exclusive from ..pipeline.depot import Depot if args.keep is not None and args.keep < 1: usage(args, "--keep must be at least 1") - r = Depot(Session(cfg).depot_root()).gc(keep=args.keep, dry_run=args.dry_run) + root = Session(cfg).depot_root() + with exclusive(root, args.lease) as alone: + if not alone: + sys.exit( + f"nnnotes depot gc: another command uses the depot ({root / 'leases'}); " + "gc runs when no other command does" + ) + r = Depot(root).gc(keep=args.keep, dry_run=args.dry_run) report( args, r, diff --git a/src/nnnotes/core/downloads.py b/src/nnnotes/core/downloads.py new file mode 100644 index 0000000..732afcc --- /dev/null +++ b/src/nnnotes/core/downloads.py @@ -0,0 +1,220 @@ +"""The downloads of a depot as a cache bounded in size: what a command reads again stays while commands run, and the +bundles and raw files are trimmed to the budget when the last command using the depot ends. + + /downloads/ the game files as downloaded (source.catalog) + /leases/.json one per running command: {"host", "pid", "started"} + /leases/.exclusive held by a command that must be the only one (trimming, `depot gc`) + +A command holds a lease while it runs (`lease`). When it ends and no other lease is live (a lease of this host whose +process runs, or of another host younger than a day), the files under the `bundles/` and `raw/` directories of +downloads/ are removed, least recently used first (a file's modification time: the catalog sets it when it hands +the file out), until they fit the budget. Catalog files stay. A removed file is downloaded again when a command needs +it: tasks name their inputs by content and know where to fetch them. + +A command that must run alone (`exclusive`) creates `.exclusive` before it looks for other leases, and a command +that starts writes its lease before it waits for `.exclusive` to go: either the first sees the new lease and does +nothing, or the new command touches nothing until the first is done. +""" + +from __future__ import annotations + +import ctypes +import json +import os +import re +import secrets +import socket +import time +from collections.abc import Generator, Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +__all__ = ["evictable", "exclusive", "lease", "parse_budget", "trim"] + +FOREIGN_LEASE_SECONDS = 24 * 3600 # a lease of another host counts as live this long +EXCLUSIVE = ".exclusive" +TORN_SECONDS = 60 # an .exclusive without a complete record (its writer stopped mid-write) counts as live this long +_UNITS = {"": 1, "b": 1, "kb": 1000, "mb": 1000**2, "gb": 1000**3, "tb": 1000**4} +_UNITS.update({"kib": 1 << 10, "mib": 1 << 20, "gib": 1 << 30, "tib": 1 << 40}) +_SIZE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([a-z]*)\s*$", re.IGNORECASE) + + +def parse_budget(text: str | None) -> int | None: + """The size a budget text names: bytes (`0`, `500MB`, `20GiB`, `1.5 TB`), None for `all` (no limit); empty: 0.""" + if text is None or not text.strip(): + return 0 + if text.strip().lower() == "all": + return None + m = _SIZE.match(text) + unit = m.group(2).lower() if m else None + if m is None or unit not in _UNITS: + raise ValueError(f"{text!r} is not a size (e.g. 0, 500MB, 20GiB) or all") + return int(float(m.group(1)) * _UNITS[unit]) + + +def touch(path: Path) -> None: + """Mark a downloaded file as used now (its modification time).""" + try: + os.utime(path) + except OSError: + pass + + +def evictable(downloads: Path) -> Iterator[Path]: + """The bundles and raw files of a downloads directory: the files below a `bundles/` or `raw/` directory.""" + if not downloads.is_dir(): + return + for p in downloads.rglob("*"): + if p.is_file() and not p.name.endswith(".part"): + if {"bundles", "raw"} & set(p.relative_to(downloads).parts[:-1]): + yield p + + +def trim(downloads: Path, budget: int | None, *, dry_run: bool = False) -> dict[str, int]: + """Remove the least recently used bundles and raw files until the rest fit `budget` bytes (None: nothing). + -> {"kept", "removed", "bytesKept", "bytesRemoved"}.""" + files: list[tuple[int, int, Path]] = [] + for p in evictable(downloads): + try: + st = p.stat() + except OSError: + continue + files.append((st.st_mtime_ns, st.st_size, p)) + total = sum(f[1] for f in files) + removed = removed_bytes = 0 + if budget is not None: + for _, size, p in sorted(files, key=lambda f: (f[0], str(f[2]))): + if total - removed_bytes <= budget: + break + if not dry_run: + try: + p.unlink() + except OSError: # open elsewhere (Windows): kept + continue + removed += 1 + removed_bytes += size + if not dry_run and removed: + _prune_dirs(downloads) + return { + "kept": len(files) - removed, + "removed": removed, + "bytesKept": total - removed_bytes, + "bytesRemoved": removed_bytes, + } + + +def _prune_dirs(root: Path) -> None: + for d in sorted((p for p in root.rglob("*") if p.is_dir()), key=lambda p: len(p.parts), reverse=True): + try: + d.rmdir() # only an empty directory + except OSError: + pass + + +def _alive(pid: int) -> bool: + """Whether a process of this host runs (an unknown answer counts as running).""" + if pid <= 0: + return False + if os.name == "nt": + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) # pyright: ignore[reportAttributeAccessIssue] + handle = kernel32.OpenProcess(0x1000, False, pid) # PROCESS_QUERY_LIMITED_INFORMATION + if not handle: + return ctypes.get_last_error() == 5 # access denied: it exists + try: + code = ctypes.c_ulong() + if not kernel32.GetExitCodeProcess(handle, ctypes.byref(code)): + return True + return code.value == 259 # STILL_ACTIVE + finally: + kernel32.CloseHandle(handle) + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def _live(doc: dict[str, Any], host: str, now: float) -> bool: + if doc.get("host") == host: + return _alive(int(doc.get("pid") or 0)) + return now - float(doc.get("started") or 0) < FOREIGN_LEASE_SECONDS + + +def others_live(root: Path, own: Path | None = None) -> bool: + """Whether a lease other than `own` is live in the depot `root` (stale leases are removed on the way).""" + host, now = socket.gethostname(), time.time() + for p in sorted((root / "leases").glob("*.json")): + if own is not None and p == own: + continue + try: + doc = json.loads(p.read_bytes()) + except (OSError, ValueError): + continue + if _live(doc, host, now): + return True + p.unlink(missing_ok=True) + return False + + +def _record() -> str: + return json.dumps({"host": socket.gethostname(), "pid": os.getpid(), "started": time.time()}) + + +def _wait_alone(leases: Path) -> None: + """Return once no live command holds `.exclusive` (a stale one is removed).""" + p, host = leases / EXCLUSIVE, socket.gethostname() + while True: + try: + data, mtime = p.read_bytes(), p.stat().st_mtime + except FileNotFoundError: + return + try: + live = _live(json.loads(data), host, time.time()) + except ValueError: # being written + live = time.time() - mtime < TORN_SECONDS + if not live: + p.unlink(missing_ok=True) + return + time.sleep(0.1) + + +@contextmanager +def exclusive(root: Path, own: Path | None) -> Generator[bool]: + """Hold `.exclusive` of the depot `root` while the block runs: True when no command but the one holding `own` + holds a lease (a command that starts meanwhile waits for the block to end); False when another one does.""" + lock = root / "leases" / EXCLUSIVE + lock.parent.mkdir(parents=True, exist_ok=True) + try: + with open(lock, "x", encoding="utf-8") as f: + f.write(_record()) + except FileExistsError: # another command holds it + yield False + return + try: + yield not others_live(root, own) + finally: + lock.unlink(missing_ok=True) + + +@contextmanager +def lease(root: Path, budget: int | None) -> Generator[Path]: + """Hold a lease on the depot `root` while the block runs; at its end, when no other lease is live, trim the + downloads to `budget` (None: keep them all).""" + d = root / "leases" + d.mkdir(parents=True, exist_ok=True) + own = d / f"{socket.gethostname()}-{os.getpid()}-{secrets.token_hex(4)}.json" + own.write_text(_record(), encoding="utf-8") + try: + _wait_alone(d) + yield own + finally: + try: + if budget is not None: + with exclusive(root, own) as alone: + if alone: + trim(root / "downloads", budget) + finally: + own.unlink(missing_ok=True) diff --git a/src/nnnotes/pipeline/depot.py b/src/nnnotes/pipeline/depot.py index 027fae0..9005430 100644 --- a/src/nnnotes/pipeline/depot.py +++ b/src/nnnotes/pipeline/depot.py @@ -32,6 +32,7 @@ from pathlib import Path from ..core.cache import temp_path +from ..core.downloads import touch from . import contract from .contract import Input @@ -426,6 +427,7 @@ def open_input(self, inp: Input) -> Path: p = Path(self.fetch(inp, loc)) elif kind == "download": p = self.downloads / loc["path"] + touch(p) else: p = Path(loc["path"]) if not p.is_file() or p.stat().st_size != inp.size: diff --git a/src/nnnotes/source/catalog.py b/src/nnnotes/source/catalog.py index cd5979f..b0cde09 100644 --- a/src/nnnotes/source/catalog.py +++ b/src/nnnotes/source/catalog.py @@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, TypeAlias, TypeVar, cast from ..core.cache import write_atomic as _write_atomic +from ..core.downloads import touch from .addressables import ( REMOTE_PREFIX, BundleKey, @@ -341,22 +342,21 @@ def resolve(self, key: str) -> list[Bundle]: # --- fetch ------------------------------------------------------------- def downloaded(self, b: Bundle) -> Path | None: - """The file fetch(b) returns when the bundle is downloaded already, else None.""" + """The file fetch(b) returns when the bundle is downloaded already (marked as used now), else None.""" dst = (self.download_dir if b.remote else self.local_download_dir()) / "bundles" / b.name - return dst if dst.is_file() and dst.stat().st_size > 0 else None + return _used(dst) def downloaded_raw(self, e: Location) -> Path | None: - """The file fetch_raw(e) returns when it is downloaded already, else None.""" + """The file fetch_raw(e) returns when it is downloaded already (marked as used now), else None.""" rel = remote_path(e["internal_id"]) - dst = self.download_dir / "raw" / rel.lstrip("/") if rel is not None else None - return dst if dst is not None and dst.is_file() and dst.stat().st_size > 0 else None + return _used(self.download_dir / "raw" / rel.lstrip("/")) if rel is not None else None def fetch(self, b: Bundle) -> Path: - """Local path to the decrypted bundle (CDN download or APK read).""" + """Local path to the decrypted bundle (CDN download or APK read), marked as used now.""" dst = (self.download_dir if b.remote else self.local_download_dir()) / "bundles" / b.name - dst.parent.mkdir(parents=True, exist_ok=True) - if dst.exists() and dst.stat().st_size > 0: + if _used(dst) is not None: return dst + dst.parent.mkdir(parents=True, exist_ok=True) if b.remote: url = self._url(b.internal_id) key = self._bundle_key_for(b.name) # before the download: a missing key fails first @@ -410,14 +410,14 @@ def bundle_of(self, key: str) -> Bundle: raise KeyError(f"{key}: no bundle among its dependencies") def fetch_raw(self, e: Location) -> Path: - """A non-bundle CDN asset (e.g. CRI ACB/AWB data) as stored -- no decryption.""" + """A non-bundle CDN asset (e.g. CRI ACB/AWB data) as stored -- no decryption; marked as used now.""" iid = e["internal_id"] rel = remote_path(iid) if rel is None: raise ValueError(f"not a CDN asset: {iid}") dst = self.download_dir / "raw" / rel.lstrip("/") - dst.parent.mkdir(parents=True, exist_ok=True) - if not (dst.exists() and dst.stat().st_size > 0): + if _used(dst) is None: + dst.parent.mkdir(parents=True, exist_ok=True) _write_atomic(dst, self._download(self._url(iid))) return dst @@ -435,7 +435,7 @@ def apk_bundle(self, name_contains: str) -> Path: raise KeyError(f"{name_contains!r}: {len(names)} APK bundles match") name = names[0].rsplit("/", 1)[1] dst = self.local_download_dir() / "bundles" / name - if not (dst.exists() and dst.stat().st_size > 0): + if _used(dst) is None: dst.parent.mkdir(parents=True, exist_ok=True) self.check_apk(z) data = z.read(names[0]) @@ -494,6 +494,18 @@ def _urlopen(url: str, timeout: float) -> bytes: return r.read() +def _used(path: Path) -> Path | None: + """`path` when it is a downloaded file (not empty), marked as used now (the downloads are trimmed least recently + used first, core.downloads); else None.""" + try: + if path.stat().st_size > 0: + touch(path) + return path + except OSError: + pass + return None + + def _setting(v: Lazy[T], needed_by: str) -> T | None: """`v`, or what it returns when it is a function; its ConfigError names `needed_by` (what is not downloaded).""" if not callable(v): diff --git a/src/nnnotes/source/configfile.py b/src/nnnotes/source/configfile.py index 7577587..14321b5 100644 --- a/src/nnnotes/source/configfile.py +++ b/src/nnnotes/source/configfile.py @@ -19,6 +19,7 @@ from typing import Any, TypeAlias, cast from urllib.parse import urlsplit +from ..core.downloads import parse_budget from . import languages from .config import Config, ConfigError, check_catalog_version, env_name, template @@ -152,6 +153,8 @@ def validate(setting: Setting, value: Value) -> None: channel_target(value) elif k == ("export", "link") and value not in LINKS: raise ValueError(f"must be one of {', '.join(LINKS)}") + elif k == ("depot", "downloads"): + parse_budget(value) # ---------------------------------------------------------------- TOML text diff --git a/src/nnnotes/source/nnnotes.example.toml b/src/nnnotes/source/nnnotes.example.toml index 0221b2b..437beb1 100644 --- a/src/nnnotes/source/nnnotes.example.toml +++ b/src/nnnotes/source/nnnotes.example.toml @@ -77,6 +77,11 @@ node = "" # ournotes-player checkout (built) or installed package for `nnnotes site build` (flag --player) player = "" +[depot] +# how much of /downloads (decrypted bundles, raw CDN files) to keep once no command uses the depot: a size such +# as 20GiB or 500MB, or all; empty: 0 (the last command to end removes them; they are downloaded again when needed) +downloads = "" + [export] # how `nnnotes export assets` makes the layout files from the depot: auto (a clone where the file system makes them, else # a hard link to the read-only depot object, else a copy), clone, hard or copy; empty: auto (flag --link) diff --git a/src/nnnotes/source/session.py b/src/nnnotes/source/session.py index aba1841..6637767 100644 --- a/src/nnnotes/source/session.py +++ b/src/nnnotes/source/session.py @@ -9,6 +9,7 @@ from pathlib import Path +from ..core.downloads import parse_budget from . import gameapi from .addressables import BundleKey from .catalog import Catalog @@ -57,6 +58,14 @@ def downloads(self) -> Path: """The directory of the depot that holds the downloaded game files (its downloads/ directory).""" return self.depot_root() / "downloads" + def downloads_budget(self) -> int | None: + """`[depot] downloads`: the bytes of bundles and raw files the downloads keep once no command uses the depot + (None: all; unset: 0).""" + try: + return parse_budget(self.cfg.get("depot", "downloads")) + except ValueError as error: + raise ConfigError(f"setting depot.downloads: {error}") from None + def apk(self, region: str | None = None) -> Path: """`[paths] apk` of `region`, required.""" cfg = self.cfg.for_region(region) if region else self.cfg diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 0f85f43..9ec5b87 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -149,7 +149,7 @@ def test_catalog_listing(catalog_file, tmp_path, capsys): def test_missing_settings_exit_2_and_name_the_setting(catalog_file, tmp_path, cdn, capsys, monkeypatch): base = ["--catalog", str(catalog_file), "--depot", str(tmp_path / "depot")] - code, out, err = run(base + ["bundle", "pull", "Char/A"], capsys) + code, out, err = run(base + ["bundle", "pull", "Char/A", "-o", str(tmp_path / "b")], capsys) assert code == 2 and out == "" assert err.strip() == ( "nnnotes: a_01.bundle is not downloaded: setting catalog.region is not set: give it " @@ -157,13 +157,13 @@ def test_missing_settings_exit_2_and_name_the_setting(catalog_file, tmp_path, cd "NNNOTES_CATALOG_REGION or --region (no config file was found: `nnnotes config init` " "writes one to fill in)" ) - code, _, err = run(base + ["--region", "zz", "bundle", "pull", "Char/A"], capsys) + code, _, err = run(base + ["--region", "zz", "bundle", "pull", "Char/A", "-o", str(tmp_path / "b")], capsys) assert code == 2 and "servers.zz.cdn" in err and "NNNOTES_SERVERS_ZZ_CDN" in err monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", cdn) - code, _, err = run(base + ["--region", "zz", "bundle", "pull", "Char/A"], capsys) + code, _, err = run(base + ["--region", "zz", "bundle", "pull", "Char/A", "-o", str(tmp_path / "b")], capsys) assert code == 2 and "bundle.key" in err and "NNNOTES_BUNDLE_KEY" in err assert cdn not in err and len(err.strip().splitlines()) == 1 - code, _, err = run(["bundle", "pull", "Char/A"], capsys) + code, _, err = run(["bundle", "pull", "Char/A", "-o", str(tmp_path / "b")], capsys) assert code == 2 and "paths.depot" in err and "NNNOTES_PATHS_DEPOT" in err and "--depot" in err @@ -183,6 +183,8 @@ def test_malformed_setting_names_setting_not_value(catalog_file, tmp_path, cdn, "bundle", "pull", "Char/A", + "-o", + str(tmp_path / "b"), ], capsys, ) @@ -199,7 +201,8 @@ def test_missing_path_on_disk(tmp_path, capsys): assert code == 2 and "setting paths.catalog: file" in err -def test_pull_from_config_file(catalog_file, tmp_path, cdn, capsys): +def test_pull_from_config_file(catalog_file, tmp_path, cdn, capsys, monkeypatch): + monkeypatch.delenv("NNNOTES_DEPOT_DOWNLOADS") # the default budget: 0 conf = tmp_path / "conf" / "nnnotes.toml" conf.parent.mkdir() conf.write_text( @@ -209,12 +212,17 @@ def test_pull_from_config_file(catalog_file, tmp_path, cdn, capsys): f'[paths]\ncatalog = "../{catalog_file.name}"\ndepot = "depot"\n', encoding="utf-8", ) - code, out, err = run(["--config", str(conf), "bundle", "pull", "Char/A", "Char/B"], capsys) + code, out, err = run( + ["--config", str(conf), "bundle", "pull", "Char/A", "Char/B", "-o", str(tmp_path / "out")], capsys + ) assert code == 0, err - pulled = tmp_path / "conf" / "depot" / "downloads" / "bundles" / "a_01.bundle" + pulled = tmp_path / "out" / "a_01.bundle" assert pulled.read_bytes() == synth.fake_bundle("a_01.bundle") assert [Path(line).resolve() for line in out.splitlines()] == [pulled.resolve()] * 2 assert KEY_HEX not in out + err + downloads = tmp_path / "conf" / "depot" / "downloads" + assert not (downloads / "bundles").exists() # [depot] downloads unset: removed when the command ends + assert not list((tmp_path / "conf" / "depot" / "leases").iterdir()) def test_master_decode_command(tmp_path, capsys, monkeypatch): @@ -359,7 +367,7 @@ def data(catalog_file, tmp_path, cdn, monkeypatch): @pytest.mark.parametrize( "argv, message", [ - (["bundle", "pull", "Char/A", "Nope/1", "Nope/2"], "not a key of the catalog: Nope/1, Nope/2"), + (["bundle", "pull", "Char/A", "Nope/1", "Nope/2", "-o", "b"], "not a key of the catalog: Nope/1, Nope/2"), (["export", "room", "Nope/1", "-o", "r.glb"], "not a key of the catalog: Nope/1"), (["export", "shader", "--key", "Nope/1", "-o", "s"], "not a key of the catalog: Nope/1"), (["export", "shader", "--apk-bundle", "zz", "-o", "s"], "--apk-bundle 'zz': 0 APK bundles match"), diff --git a/tests/conftest.py b/tests/conftest.py index 1343c8d..21d61f4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,10 +12,12 @@ def isolated(tmp_path, monkeypatch): """Every test runs in its own directory (no ./nnnotes.toml), without NNNOTES_* variables of the caller, with an empty per-user config directory, with no active settings left over and with the caches at their defaults (empty, - no disk layer).""" + no disk layer). The tests place game files in the depot's downloads as a configuration that keeps them + (`[depot] downloads = all`); the tests of the downloads budget unset it.""" for k in list(os.environ): if k.startswith("NNNOTES_"): monkeypatch.delenv(k) + monkeypatch.setenv("NNNOTES_DEPOT_DOWNLOADS", "all") monkeypatch.setenv("APPDATA", str(tmp_path / "user-config")) monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "user-config")) monkeypatch.chdir(tmp_path) diff --git a/tests/core/test_downloads.py b/tests/core/test_downloads.py new file mode 100644 index 0000000..656645a --- /dev/null +++ b/tests/core/test_downloads.py @@ -0,0 +1,139 @@ +"""The downloads of a depot as a bounded cache: budgets, least-recently-used trimming, leases of running commands.""" + +import json +import os +import socket +import subprocess +import sys +import time + +import pytest + +from nnnotes.core import downloads + + +def put(root, rel, size, mtime): + p = root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(b"x" * size) + os.utime(p, ns=(mtime, mtime)) + return p + + +@pytest.mark.parametrize( + "text, size", + [ + (None, 0), + ("", 0), + ("0", 0), + ("all", None), + ("ALL", None), + ("123", 123), + ("500MB", 500_000_000), + ("1.5 gb", 1_500_000_000), + ("20GiB", 20 << 30), + ("2kib", 2048), + ], +) +def test_parse_budget(text, size): + assert downloads.parse_budget(text) == size + + +@pytest.mark.parametrize("text", ["-1", "lots", "5 parsecs", "1e9"]) +def test_parse_budget_names_the_forms(text): + with pytest.raises(ValueError, match="not a size"): + downloads.parse_budget(text) + + +def test_trim_removes_the_least_recently_used_bundles_and_raw_files(tmp_path): + d = tmp_path / "downloads" + old = put(d, "bundles/old.bundle", 100, 1_000_000_000) + mid = put(d, "international/abc/1.0/raw/cri/x.awb", 100, 2_000_000_000) + new = put(d, "jp/site/v/h/bundles/new.bundle", 100, 3_000_000_000) + catalog = put(d, "catalog_main_xx.bin", 1000, 0) + put(d, "bundles/torn.part", 10, 0) + + assert downloads.trim(d, 150, dry_run=True) == {"kept": 1, "removed": 2, "bytesKept": 100, "bytesRemoved": 200} + assert old.exists() and mid.exists() + assert downloads.trim(d, 150)["removed"] == 2 + assert not old.exists() and not mid.exists() and new.exists() and catalog.exists() + assert not (d / "international").exists() # emptied directories go too + assert downloads.trim(d, None) == {"kept": 1, "removed": 0, "bytesKept": 100, "bytesRemoved": 0} + assert downloads.trim(d, 0)["removed"] == 1 and catalog.exists() + + +def test_using_a_file_keeps_it_longest(tmp_path): + d = tmp_path / "downloads" + first = put(d, "bundles/a.bundle", 10, 1_000_000_000) + second = put(d, "bundles/b.bundle", 10, 2_000_000_000) + downloads.touch(first) + downloads.trim(d, 10) + assert first.exists() and not second.exists() + + +def test_the_last_command_to_end_trims(tmp_path): + f = put(tmp_path / "downloads", "bundles/a.bundle", 10, 1_000_000_000) + with downloads.lease(tmp_path, 0) as outer: + with downloads.lease(tmp_path, 0): + pass + assert f.exists() # the outer command still runs + assert [p.name for p in (tmp_path / "leases").iterdir()] == [outer.name] + assert not f.exists() and not list((tmp_path / "leases").iterdir()) + + +def test_no_budget_keeps_everything(tmp_path): + f = put(tmp_path / "downloads", "bundles/a.bundle", 10, 1_000_000_000) + with downloads.lease(tmp_path, None): + pass + assert f.exists() + + +def test_leases_of_ended_processes_are_dropped(tmp_path): + ended = subprocess.run([sys.executable, "-c", "import os; print(os.getpid())"], capture_output=True, text=True) + leases = tmp_path / "leases" + leases.mkdir() + host = socket.gethostname() + stale = leases / "stale.json" + stale.write_text(json.dumps({"host": host, "pid": int(ended.stdout), "started": time.time()})) + old = leases / "old.json" + old.write_text(json.dumps({"host": host + "-other", "pid": 1, "started": time.time() - 2 * 24 * 3600})) + f = put(tmp_path / "downloads", "bundles/a.bundle", 10, 1_000_000_000) + with downloads.lease(tmp_path, 0): + pass + assert not f.exists() and not list(leases.iterdir()) + foreign = leases / "foreign.json" + foreign.write_text(json.dumps({"host": host + "-other", "pid": 1, "started": time.time()})) + assert downloads.others_live(tmp_path) # a recent lease of another host counts + + +def test_a_command_waits_for_a_trim_under_way_and_drops_a_stale_one(tmp_path, monkeypatch): + leases = tmp_path / "leases" + leases.mkdir() + lock = leases / downloads.EXCLUSIVE + lock.write_text(json.dumps({"host": socket.gethostname(), "pid": os.getpid(), "started": time.time()})) + waited = [] + monkeypatch.setattr(downloads.time, "sleep", lambda s: waited.append(s) or lock.unlink()) + with downloads.lease(tmp_path, None): + assert waited and not lock.exists() # the trimming command ended + lock.write_text(json.dumps({"host": socket.gethostname(), "pid": 0, "started": time.time()})) + with downloads.lease(tmp_path, None): + assert not lock.exists() and len(waited) == 1 # its holder is gone + + +def test_a_trim_under_way_elsewhere_is_left_to_its_command(tmp_path): + f = put(tmp_path / "downloads", "bundles/a.bundle", 10, 1_000_000_000) + with downloads.lease(tmp_path, 0): + (tmp_path / "leases" / downloads.EXCLUSIVE).write_text("{") # another command runs alone + assert f.exists() + + +def test_exclusive_is_granted_to_the_only_command(tmp_path): + with downloads.lease(tmp_path, None) as own: + with downloads.exclusive(tmp_path, own) as alone: + assert alone and (tmp_path / "leases" / downloads.EXCLUSIVE).exists() + with downloads.exclusive(tmp_path, own) as again: + assert not again # held + with downloads.lease(tmp_path, None): + with downloads.exclusive(tmp_path, own) as alone: + assert not alone # another lease is live + assert not list((tmp_path / "leases").iterdir()) diff --git a/tests/pipeline/test_depot_gc.py b/tests/pipeline/test_depot_gc.py index 326cc6d..75df301 100644 --- a/tests/pipeline/test_depot_gc.py +++ b/tests/pipeline/test_depot_gc.py @@ -1,7 +1,13 @@ """The collection of the depot: what kept runs reach stays, the rest goes.""" +import json import os +import socket +import time +import pytest + +from nnnotes import cli from nnnotes.pipeline import contract from nnnotes.pipeline.contract import Input, Task from nnnotes.pipeline.depot import Depot @@ -72,3 +78,19 @@ def test_gc_keeps_the_files_of_a_layout_a_run_names(tmp_path): manifest = d.add(contract.encode({"files": [{"path": "x.json", "sha256": placed, "size": 14}]}), "json") write_run(d, ["group:a"], [], layouts={"original": manifest["sha256"]}) assert d.gc()["contents"] == {"kept": 2, "removed": 0} and d.has(placed) + + +def test_the_gc_command_runs_only_when_no_other_command_does(tmp_path, capsys): + d = Depot(tmp_path) + orphan = d.put(b"no run reaches it") + other = tmp_path / "leases" / "other.json" + other.parent.mkdir() + other.write_text(json.dumps({"host": socket.gethostname() + "-other", "pid": 1, "started": time.time()})) + argv = ["--depot", str(tmp_path), "depot", "gc", "--json"] + with pytest.raises(SystemExit, match="another command uses the depot"): + cli.main(argv) + assert d.path(orphan).exists() + other.unlink() + cli.main(argv) + assert json.loads(capsys.readouterr().out)["contents"]["removed"] == 1 and not d.path(orphan).exists() + assert not list((tmp_path / "leases").iterdir()) diff --git a/tests/source/test_catalog_lazy.py b/tests/source/test_catalog_lazy.py index 295dbbd..c061698 100644 --- a/tests/source/test_catalog_lazy.py +++ b/tests/source/test_catalog_lazy.py @@ -197,11 +197,12 @@ def test_pull_of_downloaded_bundles_needs_no_region_cdn_or_key(tmp_path, capsys, monkeypatch.setenv("NNNOTES_BUNDLE_KEY", synth.BUNDLE_KEY.hex()) monkeypatch.setenv("NNNOTES_BUNDLE_NONCE_SEED", synth.BUNDLE_SEED.hex()) monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", cdn_tree(tmp_path / "cdn")) - code, out, err = run(base + ["--region", "zz", "bundle", "pull", "Char/A"], capsys) + pull = ["bundle", "pull", "-o", str(tmp_path / "out")] + code, out, err = run(base + ["--region", "zz", *pull, "Char/A"], capsys) assert code == 0, err for name in ("NNNOTES_BUNDLE_KEY", "NNNOTES_BUNDLE_NONCE_SEED", "NNNOTES_SERVERS_ZZ_CDN"): monkeypatch.delenv(name) - code, again, err = run(base + ["bundle", "pull", "Char/A", "Char/B"], capsys) + code, again, err = run(base + [*pull, "Char/A", "Char/B"], capsys) assert code == 0, err assert again.splitlines() == out.splitlines() * 2 @@ -211,7 +212,8 @@ def test_pull_that_downloads_names_the_missing_setting_in_one_line(tmp_path, cap catbin.write_bytes(synth.CatalogWriter().build(ENTRIES)) monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", cdn_tree(tmp_path / "cdn")) code, out, err = run( - ["--catalog", str(catbin), "--depot", str(tmp_path / "depot"), "--region", "zz", "bundle", "pull", "Char/A"], + ["--catalog", str(catbin), "--depot", str(tmp_path / "depot"), "--region", "zz", "bundle", "pull", "Char/A"] + + ["-o", str(tmp_path / "out")], capsys, ) assert code == 2 and out == "" and len(err.strip().splitlines()) == 1 From 5b39baf7421cc7e3db7ffa4a34e42807cf68222a Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 14 Oct 2026 22:49:54 +0800 Subject: [PATCH 04/91] fix(depot): keep every input identity recorded by concurrent threads The depot records the identity of each input name and of each hashed file as one file, under inputs// and inputs/paths/, written through a temporary file and a rename. Identities recorded at the same time by the threads that fetch and hash inputs are all kept, so a depot that no longer holds a downloaded file plans its tasks from the recorded identity without downloading the file again. Co-Authored-By: Claude Opus 5.5 --- docs/contracts.md | 14 +++-- src/nnnotes/pipeline/depot.py | 110 +++++++++++++--------------------- tests/cli/test_assets.py | 16 ++++- tests/pipeline/test_depot.py | 26 ++++++-- 4 files changed, 88 insertions(+), 78 deletions(-) diff --git a/docs/contracts.md b/docs/contracts.md index 13b86bb..7a7bb3a 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -239,8 +239,12 @@ The depot is a directory: the downloaded game files, and the results of all stag are kept up to `[depot] downloads` ([commands.md](commands.md#downloads)) /cas/sha256/ab/ bytes by content id, written once (no extension: records carry it) /ac/ab/.json results by task key (written after the bytes they name) -/inputs/.jsonl input identities: name -> sha256, size, with the path, size and mtime of the - file that was hashed (bundles.jsonl, raw.jsonl, files.jsonl) +/inputs//ab/.json + the identity recorded for an input name (kind bundles, raw or files): + {"name", "sha256", "size"}; digest: the sha256 of the name +/inputs/paths/ab/.json + the identity of a file as last hashed, with its path, size and mtime (a file + seen unchanged is not hashed again); digest: the sha256 of the path /catalogs/ versioned catalogs /runs/.json run manifests /runs/.log.jsonl the log of the run's last execution @@ -251,6 +255,6 @@ The depot is a directory: the downloaded game files, and the results of all stag ``` Every file is written to a temporary name and renamed into place, so concurrent writers never see or leave a -partial file and an interrupted task leaves no result. `cas/` and `ac/` depend only on the tasks run: two runs of the -same tasks, with any number of workers, leave identical trees. `downloads/`, `inputs/`, `runs/*.log.jsonl`, -`costs/` and `leases/` are local and not deterministic. +partial file and an interrupted task leaves no result; a record that changes is one file, replaced whole. `cas/` and +`ac/` depend only on the tasks run: two runs of the same tasks, with any number of workers, leave identical trees. +`downloads/`, `inputs/`, `runs/*.log.jsonl`, `costs/` and `leases/` are local and not deterministic. diff --git a/src/nnnotes/pipeline/depot.py b/src/nnnotes/pipeline/depot.py index 9005430..86d0184 100644 --- a/src/nnnotes/pipeline/depot.py +++ b/src/nnnotes/pipeline/depot.py @@ -6,9 +6,13 @@ /cas/sha256/ab/ bytes by content id, write-once (no extension: the records carry it) /ac/ab/.json task results (nnnotes.result/1) by task key; written after every byte they name is in cas/, so a result present is a complete result (the commit) - /inputs/.jsonl input identities: name -> sha256, size, with the (path, size, mtime) of the - file that was hashed (a memo: a file seen unchanged is not hashed again); - bundles.jsonl, raw.jsonl, files.jsonl (files verified by path only) + /inputs//ab/.json + the identity recorded for an input name: {"name", "sha256", "size"} (kind: + bundles, raw or files; digest: the sha256 of the name) + /inputs/paths/ab/.json + the identity of a file as last hashed: {"path", "fileSize", "mtimeNs", + "sha256", "size"} (a memo: a file seen unchanged is not hashed again; + digest: the sha256 of the path) /catalogs/ versioned catalogs (catalogdb) /runs/.json run manifests (nnnotes.run/1), deterministic /runs/.log.jsonl the log of the run's last execution (timings, workers, errors) @@ -17,6 +21,7 @@ Every file is written through a temporary file and a rename, so concurrent writers (threads, processes, machines on a shared file system) never see or leave a partial file, and a crash before a result's entry leaves no false hit. +Records that change (input identities, the latest run of a selection) are one file each, replaced whole. The files of cas/ are read-only (a layout may hard-link them: an edit of such a file fails instead of changing the depot). Only costs/, runs/ and inputs/ hold non-deterministic data; cas/ and ac/ are a function of the tasks run. @@ -138,27 +143,10 @@ def write_doc(dst, doc) -> str: return h.hexdigest() -def _append(path: Path, line: str) -> None: - """Append one line with one write (O_APPEND): lines of concurrent writers do not interleave.""" - path.parent.mkdir(parents=True, exist_ok=True) - fd = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | getattr(os, "O_BINARY", 0), 0o666) - try: - os.write(fd, line.encode("utf-8")) - finally: - os.close(fd) - - -def _lines(path: Path): - """The JSON lines of a file; a torn or malformed line (a writer that died mid-line) is skipped.""" - try: - text = path.read_text(encoding="utf-8") - except OSError: - return - for line in text.splitlines(): - try: - yield json.loads(line) - except ValueError: - continue +def _record_path(directory: Path, key: str) -> Path: + """The file of a record by key (a name, a path) in a directory of records: ab/.json.""" + d = hashlib.sha256(key.encode("utf-8")).hexdigest() + return directory / d[:2] / f"{d}.json" class Depot: @@ -170,8 +158,8 @@ def __init__(self, root, *, fetch=None): self.fetch = fetch self.written = self.reused = 0 self._have: set[str] = set() # content ids this depot object wrote or found - self._memo: dict | None = None # (path, size, mtime_ns) -> (sha256, size) - self._names: dict[str, dict] = {} # kind -> {name: (sha256, size)} + self._hashed: dict[str, dict] = {} # path -> its record (inputs/paths) + self._names: dict[tuple[str, str], tuple[str, int]] = {} # (kind, name) -> (sha256, size) self._docs: dict[str, dict] = {} # the documents document() decoded last self._lock = threading.Lock() @@ -354,59 +342,47 @@ def same(p: Path) -> bool: return _write(dst, data, same) # ---------------------------------------------------------------- inputs - def _load_memo(self) -> dict: - if self._memo is None: - memo: dict = {} - for kind in INPUT_KINDS: - names = self._names.setdefault(kind, {}) - for d in _lines(self.root / "inputs" / f"{kind}.jsonl"): - try: - entry = (d["sha256"], int(d["size"])) - if d.get("path") is not None: - memo[(d["path"], int(d["fileSize"]), int(d["mtimeNs"]))] = entry - if d.get("name") is not None: - names[d["name"]] = entry - except (KeyError, TypeError, ValueError): - continue - self._memo = memo - return self._memo - def identify(self, path, kind: str = "files", name: str | None = None) -> tuple[str, int]: - """(sha256, size) of a file: from the memo when the file's path, size and mtime are unchanged, else hashed - and remembered (inputs/.jsonl, with `name`).""" + """(sha256, size) of a file: from its record when the file's path, size and mtime are unchanged, else hashed + and recorded (inputs/paths); with `name`, also recorded as the identity of that name (inputs/).""" if kind not in INPUT_KINDS: raise ValueError(f"unknown input kind {kind!r}") p = Path(path).resolve() st = p.stat() - memo_key = (str(p), st.st_size, st.st_mtime_ns) + key = str(p) with self._lock: - memo = self._load_memo() - hit = memo.get(memo_key) - if hit is not None and (name is None or self._names[kind].get(name) == hit): - return hit - if hit is None: + rec = self._hashed.get(key) + if rec is None: + data = _read(_record_path(self.root / "inputs" / "paths", key)) + rec = json.loads(data) if data is not None else None + if rec is None or (rec["fileSize"], rec["mtimeNs"]) != (st.st_size, st.st_mtime_ns): with open(p, "rb") as f: - hit = (hashlib.file_digest(f, "sha256").hexdigest(), st.st_size) - line = { - "name": name, - "sha256": hit[0], - "size": hit[1], - "path": str(p), - "fileSize": st.st_size, - "mtimeNs": st.st_mtime_ns, - } - _append(self.root / "inputs" / f"{kind}.jsonl", json.dumps(line, ensure_ascii=False, sort_keys=True) + "\n") + sha = hashlib.file_digest(f, "sha256").hexdigest() + rec = {"path": key, "fileSize": st.st_size, "mtimeNs": st.st_mtime_ns, "sha256": sha, "size": st.st_size} + _write(_record_path(self.root / "inputs" / "paths", key), contract.encode(rec)) with self._lock: - self._load_memo()[memo_key] = hit - if name is not None: - self._names[kind][name] = hit + self._hashed[key] = rec + hit = (rec["sha256"], rec["size"]) + if name is not None and self.named(kind, name) != hit: + doc = {"name": name, "sha256": hit[0], "size": hit[1]} + _write(_record_path(self.root / "inputs" / kind, name), contract.encode(doc)) + with self._lock: + self._names[(kind, name)] = hit return hit def named(self, kind: str, name: str) -> tuple[str, int] | None: - """The last identity recorded for an input name, or None.""" + """The identity recorded for an input name, or None.""" with self._lock: - self._load_memo() - return self._names.get(kind, {}).get(name) + hit = self._names.get((kind, name)) + if hit is None: + data = _read(_record_path(self.root / "inputs" / kind, name)) + if data is None: + return None + doc = json.loads(data) + hit = (doc["sha256"], int(doc["size"])) + with self._lock: + self._names[(kind, name)] = hit + return hit def open_input(self, inp: Input) -> Path: """A file holding the bytes of `inp`: its locators in order (none: the depot), each checked against the diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index 8778e07..f049175 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -374,6 +374,20 @@ def test_export_writes_layouts_reports_and_hits_the_second_time(tmp_path, capsys assert code == 0, out +def test_a_depot_that_keeps_no_downloads_hits_without_them(tmp_path, capsys, fake, monkeypatch): + monkeypatch.delenv("NNNOTES_DEPOT_DOWNLOADS") # the default budget: 0 + d = setup_data(tmp_path, depots=("depot",)) + code, out, err = export(capsys, tmp_path, d, "depot", "out") + assert code == 1, err + first = json.loads(out) + assert not (d["downloads"] / "bundles").exists() # removed when the command ended + code, out, err = export(capsys, tmp_path, d, "depot", "out") + assert out, err + again = json.loads(out) + assert code == 1 and again["tasks"]["ran"] == 0 and again["run"] == first["run"], err + assert again["layouts"]["original"]["write"] == 0 + + def test_export_is_the_same_with_worker_processes(tmp_path, capsys, fake): d = setup_data(tmp_path, depots=("s0", "s2")) export(capsys, tmp_path, d, "s0", "o0") @@ -404,7 +418,7 @@ def test_plan_emit_tasks_and_run_stage_give_the_exported_depot(tmp_path, capsys, d = setup_data(tmp_path, depots=("orch", "q")) export(capsys, tmp_path, d, "orch", "o") rounds = 0 - while True: + while rounds <= 4: tasks = tmp_path / f"round{rounds}" code, out, err = nn( capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "q", "--emit-tasks", tasks, "--json" diff --git a/tests/pipeline/test_depot.py b/tests/pipeline/test_depot.py index c81bb48..8f45324 100644 --- a/tests/pipeline/test_depot.py +++ b/tests/pipeline/test_depot.py @@ -1,6 +1,7 @@ """The content-addressed depot: write-once bytes, results as the commit point, input identities, runs.""" import hashlib +import json import multiprocessing as mp import os from pathlib import Path @@ -116,8 +117,8 @@ def test_concurrent_writers_across_processes(tmp_path): assert len(list((tmp_path / "ac").rglob("*.json"))) == 10 assert sum(p.is_file() for p in (tmp_path / "cas").rglob("*")) == 50 # 10 blobs, 40 shared assert not list(tmp_path.rglob("*.part")) - lines = (tmp_path / "inputs" / "files.jsonl").read_text(encoding="utf-8").splitlines() - assert lines and all(line.startswith("{") and line.endswith("}") for line in lines) + records = list((tmp_path / "inputs" / "paths").rglob("*.json")) + assert len(records) == 1 and json.loads(records[0].read_bytes())["path"] == str(Path(__file__).resolve()) def test_identify_memoizes_by_path_size_and_mtime(tmp_path, monkeypatch): @@ -135,9 +136,24 @@ def test_identify_memoizes_by_path_size_and_mtime(tmp_path, monkeypatch): f.write_bytes(b"abcd") os.utime(f, ns=(1, 1)) assert fresh.identify(f) == (contract.sha256(b"abcd"), 4) and calls == [1] - with open(tmp_path / "s" / "inputs" / "files.jsonl", "a", encoding="utf-8") as g: - g.write('{"torn": ') # a writer that died mid-line - assert Depot(tmp_path / "s").identify(f) == (contract.sha256(b"abcd"), 4) + assert Depot(tmp_path / "s").identify(f) == (contract.sha256(b"abcd"), 4) and calls == [1] + assert fresh.named("bundles", "in.bin") == (contract.sha256(b"abc"), 3) # a name keeps what it was recorded as + + +def test_identities_recorded_by_concurrent_threads_are_all_kept(tmp_path): + from concurrent.futures import ThreadPoolExecutor + + files = [] + for i in range(64): + f = tmp_path / "in" / f"b{i}.bundle" + f.parent.mkdir(exist_ok=True) + f.write_bytes(f"bundle {i}".encode()) + files.append(f) + s = Depot(tmp_path / "s") + with ThreadPoolExecutor(8) as pool: + list(pool.map(lambda f: s.identify(f, "bundles", f.name), files)) + fresh = Depot(tmp_path / "s") + assert all(fresh.named("bundles", f.name) == (contract.sha256(f.read_bytes()), f.stat().st_size) for f in files) def test_open_input_tries_locators_in_order(tmp_path): From cd6152b575f63f249fab48968968070ad40634ad Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 14 Oct 2026 23:08:15 +0800 Subject: [PATCH 05/91] feat(depot): share results, contents and runs through a remote [depot] remote gives the depot a remote shared by the machines that use it: s3:/// in S3-compatible object storage (the remote extra, through boto3) or file:///, with [depot] endpoint, access_key_id, secret_access_key and signing_region. The remote holds the same keys as the depot directory: cas/, ac/, inputs//, runs/. The depot directory is the remote's cache. A content, result, input identity or run it lacks is read from the remote and kept, and a content read from the remote is checked against its id. What the depot adds goes to the remote as well: the bytes of a result before the result, a run manifest before the pointer to it. A result that differs from the remote's is a conflict. The results of the tasks of the latest run read from the remote are known to be there without a request for each. depot gc --remote collects the remote by its runs; its results and contents younger than a day stay. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 4 +- docs/assets.md | 4 +- docs/cli.md | 1 + docs/configuration.md | 5 + docs/contracts.md | 28 ++- pyproject.toml | 2 + src/nnnotes/cli/assets.py | 8 +- src/nnnotes/cli/catalog.py | 4 +- src/nnnotes/cli/depot.py | 28 ++- src/nnnotes/pipeline/cristages.py | 2 +- src/nnnotes/pipeline/depot.py | 313 +++++++++++++++++++----- src/nnnotes/pipeline/layout.py | 4 +- src/nnnotes/pipeline/remote.py | 233 ++++++++++++++++++ src/nnnotes/source/configfile.py | 17 +- src/nnnotes/source/nnnotes.example.toml | 12 + tests/cli/test_assets.py | 15 ++ tests/pipeline/test_depot_gc.py | 11 + tests/pipeline/test_depot_remote.py | 173 +++++++++++++ tests/source/test_configfile.py | 34 ++- 19 files changed, 806 insertions(+), 92 deletions(-) create mode 100644 src/nnnotes/pipeline/remote.py create mode 100644 tests/pipeline/test_depot_remote.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b8bf00..f6893a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,7 @@ jobs: - name: Install (with the deck model and the fonts extra) run: | python -m pip install --upgrade pip - python -m pip install -e ".[test,deck,fonts]" + python -m pip install -e ".[test,deck,fonts,remote]" - name: Lint run: | python -m ruff check . @@ -55,7 +55,7 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - python -m pip install -e ".[test,deck,fonts]" + python -m pip install -e ".[test,deck,fonts,remote]" - name: Type check run: python -m pyright diff --git a/docs/assets.md b/docs/assets.md index a5471a4..1eeb842 100644 --- a/docs/assets.md +++ b/docs/assets.md @@ -126,7 +126,9 @@ Removes what no kept run reaches. The kept runs are, for each selection, the `-- the latest run of a selection is always kept). A kept run reaches the results of its tasks, the contents those results name (their artifacts, and the inputs their task keys name when the depot holds them) and its layout manifests with the files they list. Every other run, result (with its measured cost) and content is removed. -`--dry-run` counts what would be removed. A layout made of hard links or clones keeps its files. gc runs only when +`--dry-run` counts what would be removed. A layout made of hard links or clones keeps its files. `--remote` +collects the depot's remote the same way, by the runs on the remote; its results and contents younger than a day stay, +since a run on another machine may be writing them. gc runs only when no other command uses the depot (it exits with an error otherwise), and commands that start meanwhile wait for it. The downloads are bounded separately, by `[depot] downloads` ([commands.md](commands.md#downloads)). diff --git a/docs/cli.md b/docs/cli.md index abe2ac7..469c685 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -356,6 +356,7 @@ remove the runs, results and contents no kept run reaches | --- | --- | | `--keep` `N` | keep the N latest runs of each selection (default: every run; the latest run of a selection is kept) | | `--dry-run` | count what would be removed, remove nothing | +| `--remote` | also collect the depot's remote, by its runs (its results and contents younger than a day stay) | ## nnnotes voices list diff --git a/docs/configuration.md b/docs/configuration.md index da9b42f..532e9b9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -111,6 +111,11 @@ exist when they are given. | `[paths] fonts.` | `NNNOTES_PATHS_FONTS_` (e.g. `NNNOTES_PATHS_FONTS_ZH_HANT`) | `--font =PATH` (of `site build`) | font file (OpenType or TrueType) the story text of a language is drawn with (`--fonts open`); also read for the other story languages whose game fonts fall back to that language's font | | `[paths] fonts.emoji` | `NNNOTES_PATHS_FONTS_EMOJI` | `--font emoji=PATH` (of `site build`) | colour emoji font (PNG bitmap glyphs: CBDT or sbix) the stories' emoji sprites are drawn from (`--fonts open`); Noto Color Emoji (SIL Open Font License 1.1) is the tested font. Optional: without it the sprites keep their layout with empty glyphs, listed in `ui/fonts.json` `coverage.sprites.missing` | | `[depot] downloads` | `NNNOTES_DEPOT_DOWNLOADS` | — | how much of the depot's downloaded bundles and raw files to keep once no command uses the depot: a size (`500MB`, `20GiB`; units B, KB, MB, GB, TB and KiB, MiB, GiB, TiB), `all`, or `0` (unset) ([commands.md](commands.md#downloads)) | +| `[depot] remote` | `NNNOTES_DEPOT_REMOTE` | — | a remote of the depot shared by the machines that use it: `s3:///` (S3-compatible object storage; `pip install 'nnnotes[remote]'`) or `file:///` ([contracts.md](contracts.md#depot-remote)) | +| `[depot] endpoint` | `NNNOTES_DEPOT_ENDPOINT` | — | endpoint URL of the S3-compatible object storage; unset: AWS S3 | +| `[depot] access_key_id` | `NNNOTES_DEPOT_ACCESS_KEY_ID` | — | access key id of the object storage; unset: boto3 looks for credentials itself (`AWS_ACCESS_KEY_ID`, `~/.aws`) | +| `[depot] secret_access_key` | `NNNOTES_DEPOT_SECRET_ACCESS_KEY` | — | secret access key of the object storage | +| `[depot] signing_region` | `NNNOTES_DEPOT_SIGNING_REGION` | — | region the requests to the object storage are signed for; unset: `us-east-1` | | `[export] link` | `NNNOTES_EXPORT_LINK` | `--link` (of `export assets`) | how `export assets` makes the layout files from the depot: `auto` (unset), `clone`, `hard` or `copy` ([assets.md](assets.md)) | Hex values are case-insensitive; a `0x` prefix and surrounding whitespace are ignored. All values are TOML strings diff --git a/docs/contracts.md b/docs/contracts.md index 7a7bb3a..9a9cb65 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -256,5 +256,29 @@ The depot is a directory: the downloaded game files, and the results of all stag Every file is written to a temporary name and renamed into place, so concurrent writers never see or leave a partial file and an interrupted task leaves no result; a record that changes is one file, replaced whole. `cas/` and -`ac/` depend only on the tasks run: two runs of the same tasks, with any number of workers, leave identical trees. -`downloads/`, `inputs/`, `runs/*.log.jsonl`, `costs/` and `leases/` are local and not deterministic. +`ac/` depend only on the tasks run: two runs of the same tasks, with any number of workers, leave identical trees; +`inputs/`, `runs/`, `costs/` and `leases/` are not deterministic. + +### Depot remote + +`[depot] remote` gives the depot a remote shared by the machines that use it: `s3:///` in +S3-compatible object storage (`pip install 'nnnotes[remote]'`; path-style requests, so stores other than AWS answer +them) or `file:///`. The remote holds the same keys as the depot directory, below its prefix: + +``` +cas/sha256/ab/ contents +ac/ab/.json results +inputs//ab/.json input identities by name +runs/.json run manifests +runs/selections/ the id of the latest run of a selection +``` + +The depot directory is the remote's cache. A content, result, input identity or run the depot lacks is read from the +remote and kept; a content read from the remote is checked against its content id. What the depot adds goes to the +remote as well: the bytes a result names before the result, a run manifest before the pointer to it. A result that +differs from the remote's result of the same key is a conflict, as in the depot itself. The latest run of a selection +is the one any machine wrote last, and the results of its tasks are known to be on the remote without asking for each +of them. Downloads, file identities by path (`inputs/paths/`), catalog versions, measured costs, run logs and leases +stay in the depot directory. + +Every request names one object; only `depot gc --remote` lists the remote. diff --git a/pyproject.toml b/pyproject.toml index f91c8a6..231f7b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,8 @@ dynamic = ["version"] test = ["pytest>=8", "ruff>=0.15", "jsonschema>=4.18", "import-linter>=2.1", "pyright>=1.1.414"] fonts = ["fonttools>=4.50", "freetype-py>=2.5", "scipy>=1.13"] docs = ["mkdocs>=1.6,<2", "mkdocs-material>=9.5", "mkdocstrings[python]>=0.29"] +# a depot remote in S3-compatible object storage ([depot] remote = "s3://...") +remote = ["boto3>=1.35"] # the deck model of music-data: one nnnotes version carries one ournotes-deck release (.github/workflows/deck.yml moves # the pin when ournotes-deck publishes a newer one) deck = ["ournotes-sim==0.0.9"] diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index 6c336d8..0c4eef3 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -456,13 +456,13 @@ class Workspace: def __init__(self, args, cfg, log=_log): from ..pipeline.catalogdb import CatalogDB - from ..pipeline.depot import Depot + from ..pipeline.depot import open_depot self.args, self.cfg, self.log = args, cfg, log self.root = Session(cfg).depot_root() self.apk = cfg.path("paths", "apk") self.fetcher = CatalogFetcher(self.root, cfg) - self.depot = Depot(self.root, fetch=self.fetcher) + self.depot = open_depot(cfg, fetch=self.fetcher) self.db = CatalogDB(self.root) try: self.selection = parse_selection(getattr(args, "select", None)) @@ -1399,7 +1399,7 @@ def _task_files(items) -> list[Path]: def cmd_run_stage(args, cfg): - from ..pipeline.depot import Depot + from ..pipeline.depot import open_depot from ..pipeline.orchestrate import run_one problems, tasks = [], [] @@ -1425,7 +1425,7 @@ def cmd_run_stage(args, cfg): print(f"nnnotes: incompatible task: {m}", file=sys.stderr) sys.exit(EXIT_INCOMPATIBLE) root = Session(cfg).depot_root() - depot = Depot(root, fetch=CatalogFetcher(root, cfg) if args.fetch else None) + depot = open_depot(cfg, fetch=CatalogFetcher(root, cfg) if args.fetch else None) use(cfg) out, bad = [], 0 for t in tasks: diff --git a/src/nnnotes/cli/catalog.py b/src/nnnotes/cli/catalog.py index f1f6f14..67273b6 100644 --- a/src/nnnotes/cli/catalog.py +++ b/src/nnnotes/cli/catalog.py @@ -190,7 +190,7 @@ def cmd_fetch(args, cfg): def cmd_diff(args, cfg): from ..pipeline import catalogdb - from ..pipeline.depot import Depot + from ..pipeline.depot import open_depot from .assets import addresses_of db = _db(cfg) @@ -200,7 +200,7 @@ def cmd_diff(args, cfg): vs.append(db.get(ref)) except (KeyError, ValueError) as e: usage(args, f"{ref}: {e.args[0] if e.args else e}") - depot = Depot(Session(cfg).depot_root()) + depot = open_depot(cfg) a, b = vs d = catalogdb.diff(db.index(a), db.index(b), addresses_of(depot, a["id"]), addresses_of(depot, b["id"])) if args.json: diff --git a/src/nnnotes/cli/depot.py b/src/nnnotes/cli/depot.py index d7f5ba2..1e3a431 100644 --- a/src/nnnotes/cli/depot.py +++ b/src/nnnotes/cli/depot.py @@ -23,7 +23,7 @@ def cmd_verify(args, cfg): def cmd_gc(args, cfg): from ..core.downloads import exclusive - from ..pipeline.depot import Depot + from ..pipeline.depot import open_depot if args.keep is not None and args.keep < 1: usage(args, "--keep must be at least 1") @@ -34,17 +34,20 @@ def cmd_gc(args, cfg): f"nnnotes depot gc: another command uses the depot ({root / 'leases'}); " "gc runs when no other command does" ) - r = Depot(root).gc(keep=args.keep, dry_run=args.dry_run) - report( - args, - r, - lambda r: [ - f"{'would remove' if args.dry_run else 'removed'}: {r['runs']['removed']} runs, " + depot = open_depot(cfg) + if args.remote and depot.remote is None: + usage(args, "--remote: the depot has no remote (setting depot.remote)") + r = depot.gc(keep=args.keep, dry_run=args.dry_run, remote=args.remote) + + def lines(r: dict, where: str = "") -> list[str]: + return [ + f"{where}{'would remove' if args.dry_run else 'removed'}: {r['runs']['removed']} runs, " f"{r['results']['removed']} results, {r['contents']['removed']} contents " f"({r['bytesRemoved'] / 1e6:.1f} MB)", - f"kept: {r['runs']['kept']} runs, {r['results']['kept']} results, {r['contents']['kept']} contents", - ], - ) + f"{where}kept: {r['runs']['kept']} runs, {r['results']['kept']} results, {r['contents']['kept']} contents", + ] + + report(args, r, lambda r: lines(r) + (lines(r["remote"], "remote ") if "remote" in r else [])) def register(sub) -> None: @@ -81,3 +84,8 @@ def register(sub) -> None: help="keep the N latest runs of each selection (default: every run; the latest run of a selection is kept)", ) c.add_argument("--dry-run", action="store_true", help="count what would be removed, remove nothing") + c.add_argument( + "--remote", + action="store_true", + help="also collect the depot's remote, by its runs (its results and contents younger than a day stay)", + ) diff --git a/src/nnnotes/pipeline/cristages.py b/src/nnnotes/pipeline/cristages.py index 8d4d794..c80c897 100644 --- a/src/nnnotes/pipeline/cristages.py +++ b/src/nnnotes/pipeline/cristages.py @@ -92,7 +92,7 @@ def boot_input(depot, apk) -> Input | None: if known is None or not depot.has(*known): with ApkSet(apk) as z: data = z.read(BOOT_IN_APK) - known = depot.identify(depot.path(depot.put(data)), "files", name) + known = depot.identify(depot.local(depot.put(data)), "files", name) return Input("boot", known[0], known[1], "data.unity3d", ({"kind": "depot"},)) diff --git a/src/nnnotes/pipeline/depot.py b/src/nnnotes/pipeline/depot.py index 86d0184..0bd5b5b 100644 --- a/src/nnnotes/pipeline/depot.py +++ b/src/nnnotes/pipeline/depot.py @@ -25,6 +25,11 @@ The files of cas/ are read-only (a layout may hard-link them: an edit of such a file fails instead of changing the depot). Only costs/, runs/ and inputs/ hold non-deterministic data; cas/ and ac/ are a function of the tasks run. + +A depot may have a remote (pipeline.remote): object storage or a directory shared by the machines that use it, with +the same keys as the files above (cas/, ac/, inputs//, runs/). The local depot is its cache: a content, +result, input identity or run it lacks is read from the remote and kept, and what it adds goes to the remote as well +(the bytes of a result before the result). downloads/, inputs/paths/, catalogs/, costs/ and run logs stay local. """ from __future__ import annotations @@ -40,8 +45,11 @@ from ..core.downloads import touch from . import contract from .contract import Input +from .remote import Remote, open_remote INPUT_KINDS = ("bundles", "raw", "files") +REMOTE_GRACE = 24 * 3600 # `gc(remote=True)` keeps results and contents of the remote younger than this (seconds) +DONE = ("ran", "hit") # the statuses of a run's tasks whose results are committed class InputMissing(FileNotFoundError): @@ -151,26 +159,62 @@ def _record_path(directory: Path, key: str) -> Path: class Depot: """A depot directory. `fetch(input, locator) -> Path`: how a "catalog" locator is fetched (none: such locators - are skipped).""" + are skipped). `remote`: the depot's remote (None: a local depot).""" - def __init__(self, root, *, fetch=None): + def __init__(self, root, *, fetch=None, remote: Remote | None = None): self.root = Path(root) self.fetch = fetch + self.remote = remote self.written = self.reused = 0 - self._have: set[str] = set() # content ids this depot object wrote or found + self._have: set[str] = set() # content ids this depot object wrote or found, here or on the remote + self._sizes: dict[str, int] = {} # remote key -> size, of the objects known to be on the remote + self._listed: set[str] = set() # task keys whose results the remote holds (the latest runs read from it) self._hashed: dict[str, dict] = {} # path -> its record (inputs/paths) self._names: dict[tuple[str, str], tuple[str, int]] = {} # (kind, name) -> (sha256, size) self._docs: dict[str, dict] = {} # the documents document() decoded last self._lock = threading.Lock() def __getstate__(self): # a depot crosses to worker processes as its settings - return {"root": self.root, "fetch": self.fetch} + return {"root": self.root, "fetch": self.fetch, "remote": self.remote} def __setstate__(self, state): - self.__init__(state["root"], fetch=state["fetch"]) + self.__init__(state["root"], fetch=state["fetch"], remote=state["remote"]) def __repr__(self) -> str: - return f"Depot({str(self.root)!r})" + more = f", remote={self.remote.url!r}" if self.remote is not None else "" + return f"Depot({str(self.root)!r}{more})" + + def _key(self, p: Path) -> str: + """The remote key of a file of the depot: its path below the root.""" + return p.relative_to(self.root).as_posix() + + def _on_remote(self, key: str, size: int | None = None) -> bool: + """Whether the remote holds the object `key` (of `size`).""" + n = self._sizes.get(key) + if n is None: + n = self.remote.size(key) if self.remote is not None else None + if n is None: + return False + self._sizes[key] = n + return size is None or n == size + + def _push(self, sha: str, size: int) -> None: + """Copy a content of this depot to the remote unless the remote holds it.""" + if self.remote is not None: + p = self.path(sha) + key = self._key(p) + if not self._on_remote(key, size): + self.remote.put_file(key, p) + self._sizes[key] = size + + def _fetch_record(self, p: Path) -> bytes | None: + """The bytes of a record file of the depot, read from the remote (and kept) when only the remote holds it.""" + data = _read(p) + if data is None and self.remote is not None: + data = self.remote.get(self._key(p)) + if data is not None: + _write(p, data, lambda q: True) + return data @property def downloads(self) -> Path: @@ -189,30 +233,45 @@ def path(self, sha: str) -> Path: return self.root / "cas" / "sha256" / sha[:2] / sha def has(self, sha: str, size: int | None = None) -> bool: + """Whether the depot or its remote holds the content (of `size`).""" if sha in self._have: return True p = self.path(sha) try: n = p.stat().st_size except OSError: - return False - if size is not None and n != size: - return False + if not self._on_remote(self._key(p), size): + return False + else: + if size is not None and n != size: + return False self._have.add(sha) return True + def local(self, sha: str) -> Path: + """The file of a content in this depot, read from the remote first when only the remote holds it.""" + p = self.path(sha) + if self.remote is not None and not p.is_file(): + data = self.remote.get(self._key(p)) + if data is None: + raise FileNotFoundError(f"content {sha} is not in the depot or its remote") + if contract.sha256(data) != sha: + raise RuntimeError(f"content {sha} of the remote differs from its id") + _write(p, data, lambda q: q.stat().st_size == len(data), read_only=True) + return p + def put(self, data) -> str: """Store bytes; their content id. Write-once: bytes already stored are not written again.""" data = bytes(data) sha = contract.sha256(data) if self.has(sha, len(data)): self.reused += 1 - return sha - if _write(self.path(sha), data, lambda p: p.stat().st_size == len(data), read_only=True): + elif _write(self.path(sha), data, lambda p: p.stat().st_size == len(data), read_only=True): self.written += 1 else: self.reused += 1 self._have.add(sha) + self._push(sha, len(data)) return sha def put_file(self, src) -> tuple[str, int]: @@ -221,6 +280,7 @@ def put_file(self, src) -> tuple[str, int]: sha, size = self.identify(src) if self.has(sha, size): self.reused += 1 + self._push(sha, size) return sha, size dst = self.path(sha) dst.parent.mkdir(parents=True, exist_ok=True) @@ -241,6 +301,7 @@ def put_file(self, src) -> tuple[str, int]: if _replace(tmp, dst, lambda p: p.stat().st_size == size): self.written += 1 self._have.add(sha) + self._push(sha, size) return sha, size def add(self, data, ext: str, media: str | None = None) -> dict: @@ -277,11 +338,12 @@ def add_chunks(self, chunks, ext: str, media: str | None = None) -> dict: else: self.reused += 1 self._have.add(sha) + self._push(sha, size) ext = ext.lower().lstrip(".") return {"sha256": sha, "size": size, "mediaType": media or contract.media_type(ext), "ext": ext} def read(self, sha: str) -> bytes: - return self.path(sha).read_bytes() + return self.local(sha).read_bytes() def document(self, sha: str) -> dict: """The JSON document stored as `sha`, decoded; the last two decoded are kept (a catalog index is read at @@ -304,7 +366,7 @@ def result_path(self, key: str) -> Path: def result(self, key: str) -> dict | None: """The committed result of a task key, or None.""" - data = _read(self.result_path(key)) + data = self._fetch_record(self.result_path(key)) return None if data is None else contract.loads(data) def committed(self, key: str, task: str) -> dict: @@ -315,7 +377,8 @@ def committed(self, key: str, task: str) -> dict: return doc def has_result(self, key: str) -> bool: - return self.result_path(key).is_file() + p = self.result_path(key) + return p.is_file() or key in self._listed or self._on_remote(self._key(p)) def commit(self, doc: dict) -> bool: """Write a result entry, after checking that the bytes of every artifact it names are stored. Write-once: an @@ -326,6 +389,7 @@ def commit(self, doc: dict) -> bool: c = a["content"] if not self.has(c["sha256"], c["size"]): raise RuntimeError(f"result {doc['key']}: bytes of {a['id']} ({c['sha256']}) are not stored") + self._push(c["sha256"], c["size"]) data = contract.encode(doc) dst = self.result_path(doc["key"]) @@ -337,9 +401,16 @@ def same(p: Path) -> bool: raise DepotConflict(f"result {doc['key']} differs from the one in the depot ({dst})") return True - if same(dst): - return False - return _write(dst, data, same) + here = same(dst) + if self.remote is not None: + key = self._key(dst) + old = self.remote.get(key) + if old is None: + self.remote.put(key, data) + elif old != data: + raise DepotConflict(f"result {doc['key']} differs from the one in the depot's remote") + self._sizes[key] = len(data) + return False if here else _write(dst, data, same) # ---------------------------------------------------------------- inputs def identify(self, path, kind: str = "files", name: str | None = None) -> tuple[str, int]: @@ -364,8 +435,11 @@ def identify(self, path, kind: str = "files", name: str | None = None) -> tuple[ self._hashed[key] = rec hit = (rec["sha256"], rec["size"]) if name is not None and self.named(kind, name) != hit: - doc = {"name": name, "sha256": hit[0], "size": hit[1]} - _write(_record_path(self.root / "inputs" / kind, name), contract.encode(doc)) + doc = contract.encode({"name": name, "sha256": hit[0], "size": hit[1]}) + rec_path = _record_path(self.root / "inputs" / kind, name) + _write(rec_path, doc) + if self.remote is not None: + self.remote.put(self._key(rec_path), doc) with self._lock: self._names[(kind, name)] = hit return hit @@ -375,7 +449,7 @@ def named(self, kind: str, name: str) -> tuple[str, int] | None: with self._lock: hit = self._names.get((kind, name)) if hit is None: - data = _read(_record_path(self.root / "inputs" / kind, name)) + data = self._fetch_record(_record_path(self.root / "inputs" / kind, name)) if data is None: return None doc = json.loads(data) @@ -393,7 +467,7 @@ def open_input(self, inp: Input) -> Path: try: if kind == "depot": if self.has(inp.sha256, inp.size): - return self.path(inp.sha256) + return self.local(inp.sha256) tried.append("depot: absent") continue if kind == "catalog": @@ -494,25 +568,34 @@ def write_run(self, doc: dict, log: list[dict] | None = None) -> Path: if log is not None: text = "".join(json.dumps(line, ensure_ascii=False, sort_keys=True) + "\n" for line in log) write_file(p.with_name(f"{doc['id']}.log.jsonl"), text.encode("utf-8")) - write_file(self._selection_path(doc["selection"]), (doc["id"] + "\n").encode("ascii")) + pointer = self._selection_path(doc["selection"]) + write_file(pointer, (doc["id"] + "\n").encode("ascii")) + if self.remote is not None: + self.remote.put(self._key(p), contract.encode(doc)) + self.remote.put(self._key(pointer), (doc["id"] + "\n").encode("ascii")) return p def _selection_path(self, selection) -> Path: return self.root / "runs" / "selections" / contract.digest(list(selection)) def run(self, run_id: str) -> dict | None: - try: - return contract.loads(self.run_path(run_id).read_bytes()) - except OSError: - return None + data = self._fetch_record(self.run_path(run_id)) + return None if data is None else contract.loads(data) def latest_run(self, selection) -> dict | None: - """The latest run manifest written for `selection`, or None.""" - try: - rid = self._selection_path(selection).read_text(encoding="ascii").strip() - except OSError: - return None - return self.run(rid) + """The latest run manifest written for `selection` (by any machine of the remote), or None. The results of + the tasks of a run read from the remote are known to be on the remote (gc keeps them).""" + pointer = self._selection_path(selection) + data = self.remote.get(self._key(pointer)) if self.remote is not None else None + shared = data is not None + if data is None: + data = _read(pointer) + if data is None: + return None + doc = self.run(data.decode("ascii").strip()) + if shared and doc is not None: + self._listed.update(t["key"] for t in doc.get("tasks") or () if t.get("key") and t.get("status") in DONE) + return doc def write_cost(self, key: str, cost: dict) -> None: write_file(self.root / "costs" / f"{key}.json", contract.encode(cost)) @@ -524,53 +607,51 @@ def cost(self, key: str) -> dict | None: return None # ---------------------------------------------------------------- collection - def gc(self, *, keep: int | None = None, dry_run: bool = False) -> dict: + def gc( + self, *, keep: int | None = None, dry_run: bool = False, remote: bool = False, grace: float = REMOTE_GRACE + ) -> dict: """Remove what no kept run reaches. The kept runs: of each selection the `keep` latest by the time their manifest was written (None: every run), and always the run the selection names as its latest. A kept run reaches the results of its tasks, the contents those results name (their artifacts, and the inputs their key parts name when the depot holds them) and the layout manifests it names with the contents they list. - Removed: the other runs (manifest and log), results and their measured costs, and contents. `dry_run`: count - only. -> {"runs", "results", "contents": {"kept", "removed"}, "bytesRemoved"}.""" + Removed: the other runs (manifest and log), results and their measured costs, and contents. `remote`: the + remote's runs, results and contents too, by the runs of the remote; its results and contents younger than + `grace` seconds stay (a run of another machine may be writing them). `dry_run`: count only. + -> {"runs", "results", "contents": {"kept", "removed"}, "bytesRemoved"}, and "remote": the same.""" + out = self._gc_local(keep, dry_run) + if remote: + if self.remote is None: + raise ValueError("the depot has no remote") + out["remote"] = self._gc_remote(self.remote, keep, dry_run, grace) + return out + + def _gc_local(self, keep: int | None, dry_run: bool) -> dict: runs_dir = self.root / "runs" - manifests = sorted(p for p in runs_dir.glob("*.json") if p.is_file()) latest = set() for p in (runs_dir / "selections").glob("*"): try: latest.add(p.read_text(encoding="ascii").strip()) except OSError: continue - by_selection: dict[str, list[tuple[int, Path, dict]]] = {} - for p in manifests: + runs = [] + for p in sorted(p for p in runs_dir.glob("*.json") if p.is_file()): try: - doc = contract.loads(p.read_bytes()) + runs.append((p.stat().st_mtime_ns, p, contract.loads(p.read_bytes()))) except (OSError, ValueError): continue - sel = contract.digest(list(doc.get("selection") or ())) - by_selection.setdefault(sel, []).append((p.stat().st_mtime_ns, p, doc)) - kept, dropped = [], [] - for runs in by_selection.values(): - runs.sort(key=lambda r: r[0], reverse=True) - for i, (_, p, doc) in enumerate(runs): - (kept if keep is None or i < keep or doc.get("id") in latest else dropped).append((p, doc)) - keys: set[str] = set() - shas: set[str] = set() - for _, doc in kept: - for t in doc.get("tasks") or (): - if t.get("key"): - keys.add(t["key"]) - for sha in (doc.get("layouts") or {}).values(): - shas.add(sha) - if self.has(sha): - _collect_shas(contract.loads(self.read(sha)), shas) - for key in keys: - res = self.result(key) - if res is None: - continue - shas.update(a["content"]["sha256"] for a in res.get("artifacts", ())) - shas.update(sha for _, sha in (res.get("keyParts") or {}).get("inputs", ())) - removed_bytes = 0 + kept, dropped = _kept_runs(runs, latest, keep) + + def result(key: str) -> dict | None: + data = _read(self.result_path(key)) + return None if data is None else contract.loads(data) + + def document(sha: str) -> dict | None: + p = self.path(sha) + return contract.loads(p.read_bytes()) if p.is_file() else None + + keys, shas = _reach([doc for _, _, doc in kept], result, document) out: dict = {"runs": {"kept": len(kept), "removed": len(dropped)}} - for p, _ in dropped: + for _, p, _ in dropped: if not dry_run: p.unlink(missing_ok=True) p.with_name(p.name[: -len(".json")] + ".log.jsonl").unlink(missing_ok=True) @@ -585,7 +666,7 @@ def gc(self, *, keep: int | None = None, dry_run: bool = False) -> dict: p.unlink(missing_ok=True) (self.root / "costs" / f"{key}.json").unlink(missing_ok=True) out["results"] = {"kept": n_kept, "removed": n_removed} - n_kept = n_removed = 0 + n_kept = n_removed = removed_bytes = 0 for p in sorted((self.root / "cas" / "sha256").glob("*/*")): if p.name in shas: n_kept += 1 @@ -599,6 +680,108 @@ def gc(self, *, keep: int | None = None, dry_run: bool = False) -> dict: out["bytesRemoved"] = removed_bytes return out + def _gc_remote(self, remote: Remote, keep: int | None, dry_run: bool, grace: float) -> dict: + now = time.time() + latest, runs = set(), [] + for key, _, mtime in remote.list("runs/"): + data = remote.get(key) + if data is None: + continue + if key.startswith("runs/selections/"): + latest.add(data.decode("ascii").strip()) + elif key.endswith(".json"): + runs.append((mtime, key, contract.loads(data))) + kept, dropped = _kept_runs(runs, latest, keep) + keys, shas = _reach( + [doc for _, _, doc in kept], + self.result, + lambda sha: contract.loads(self.read(sha)) if self.has(sha) else None, + ) + out: dict = {"runs": {"kept": len(kept), "removed": len(dropped)}} + gone = [key for _, key, _ in dropped] + counts = {} + removed_bytes = 0 + for part, prefix in (("results", "ac/"), ("contents", "cas/sha256/")): + n_kept = n_removed = 0 + for key, size, mtime in remote.list(prefix): + name = key.rsplit("/", 1)[-1] + reached = name[: -len(".json")] in keys if part == "results" else name in shas + if reached or now - mtime < grace: + n_kept += 1 + continue + n_removed += 1 + gone.append(key) + if part == "contents": + removed_bytes += size + counts[part] = {"kept": n_kept, "removed": n_removed} + if not dry_run: + remote.delete(gone) + for key in gone: + self._sizes.pop(key, None) + self._have.clear() + self._listed.clear() + out.update(counts) + out["bytesRemoved"] = removed_bytes + return out + + +def open_depot(cfg, *, fetch=None) -> Depot: + """The depot of a configuration: `[paths] depot`, with the remote `[depot] remote` names (its `endpoint`, + `access_key_id`, `secret_access_key` and `signing_region`) when it is set.""" + from ..source.config import ConfigError + from ..source.session import Session + + root = Session(cfg).depot_root() + url = cfg.get("depot", "remote") + if not url: + return Depot(root, fetch=fetch) + try: + remote = open_remote( + url, + endpoint=cfg.get("depot", "endpoint"), + key_id=cfg.get("depot", "access_key_id"), + secret=cfg.get("depot", "secret_access_key"), + region=cfg.get("depot", "signing_region"), + ) + except ValueError as e: + raise ConfigError(f"setting depot.remote: {e}") from None + return Depot(root, fetch=fetch, remote=remote) + + +def _kept_runs(runs: list, latest: set[str], keep: int | None) -> tuple[list, list]: + """(kept, dropped) of runs (time, handle, manifest): of each selection the `keep` latest by time (None: every + run), and the runs `latest` names.""" + by_selection: dict[str, list] = {} + for r in runs: + by_selection.setdefault(contract.digest(list(r[2].get("selection") or ())), []).append(r) + kept, dropped = [], [] + for rs in by_selection.values(): + rs.sort(key=lambda r: r[0], reverse=True) + for i, r in enumerate(rs): + (kept if keep is None or i < keep or r[2].get("id") in latest else dropped).append(r) + return kept, dropped + + +def _reach(runs: list[dict], result, document) -> tuple[set[str], set[str]]: + """(task keys, content ids) that run manifests reach: their tasks' results with the contents those name, and + their layout manifests with the contents they list. `result(key)`, `document(sha)`: None when absent.""" + keys: set[str] = set() + shas: set[str] = set() + for doc in runs: + keys.update(t["key"] for t in doc.get("tasks") or () if t.get("key")) + for sha in (doc.get("layouts") or {}).values(): + shas.add(sha) + manifest = document(sha) + if manifest is not None: + _collect_shas(manifest, shas) + for key in keys: + res = result(key) + if res is None: + continue + shas.update(a["content"]["sha256"] for a in res.get("artifacts", ())) + shas.update(sha for _, sha in (res.get("keyParts") or {}).get("inputs", ())) + return keys, shas + def _collect_shas(doc, out: set[str]) -> None: """Add every "sha256" value of a document (a layout manifest names its files' contents so).""" diff --git a/src/nnnotes/pipeline/layout.py b/src/nnnotes/pipeline/layout.py index 1ef6591..cc9a3c2 100644 --- a/src/nnnotes/pipeline/layout.py +++ b/src/nnnotes/pipeline/layout.py @@ -367,8 +367,8 @@ def paths(entries_: list[dict]) -> dict[str, str]: # ---------------------------------------------------------------- writing def from_depot(depot): - """A byte source reading the depot's cas/.""" - return depot.path + """A byte source reading the depot's cas/ (a content only its remote holds is read from the remote first).""" + return depot.local def from_layout(directory): diff --git a/src/nnnotes/pipeline/remote.py b/src/nnnotes/pipeline/remote.py new file mode 100644 index 0000000..ffa50c6 --- /dev/null +++ b/src/nnnotes/pipeline/remote.py @@ -0,0 +1,233 @@ +"""The remote of a depot: a copy of its contents, results, input identities and runs in object storage, shared by the +machines that use it. + + /cas/sha256/ab/ contents + /ac/ab/.json results + /inputs//.json input identities by name ({"name", "sha256", "size"}; digest: the + sha256 of the name) + /runs/.json run manifests + /runs/selections/ the id of the latest run of a selection + +A remote is `s3:///` (S3-compatible object storage: path-style requests, checksums only where the +request needs them, so stores other than AWS answer them) or `file://` (a shared directory). Every +operation but `list` names one object, so a remote behind a gateway that caches listings answers them all; `list` is +used by `depot gc` only. Contents and results are written once: a write is skipped when the object is there with the +same size. The local depot is the remote's cache (pipeline.depot). +""" + +from __future__ import annotations + +import os +import threading +import urllib.parse +from collections.abc import Iterator +from pathlib import Path + +from ..core.cache import temp_path + +__all__ = ["Remote", "open_remote"] + + +class Remote: + """The operations of a remote, by object key (a relative path under the remote's root).""" + + url: str + + def get(self, key: str) -> bytes | None: + """The object's bytes, None when it does not exist.""" + raise NotImplementedError + + def size(self, key: str) -> int | None: + """The object's size, None when it does not exist.""" + raise NotImplementedError + + def put(self, key: str, data: bytes) -> None: + """Write (or replace) the object.""" + raise NotImplementedError + + def put_file(self, key: str, path: Path) -> None: + """Write (or replace) the object with a file's bytes.""" + self.put(key, Path(path).read_bytes()) + + def delete(self, keys: list[str]) -> None: + """Remove objects (missing ones are no error).""" + raise NotImplementedError + + def list(self, prefix: str) -> Iterator[tuple[str, int, float]]: + """(key, size, time it was written: seconds since the epoch) of every object whose key starts with + `prefix`.""" + raise NotImplementedError + + def __repr__(self) -> str: + return f"{type(self).__name__}({self.url!r})" + + +class FileRemote(Remote): + """A remote in a directory (`file://`).""" + + def __init__(self, url: str, root: Path): + self.url, self.root = url, Path(root) + + def __getstate__(self): + return {"url": self.url, "root": self.root} + + def __setstate__(self, state): + self.__init__(state["url"], state["root"]) + + def _path(self, key: str) -> Path: + parts = key.split("/") + if not key or any(p in ("", ".", "..") for p in parts): + raise ValueError(f"not an object key: {key!r}") + return self.root.joinpath(*parts) + + def get(self, key: str) -> bytes | None: + try: + return self._path(key).read_bytes() + except FileNotFoundError: + return None + + def size(self, key: str) -> int | None: + try: + return self._path(key).stat().st_size + except FileNotFoundError: + return None + + def put(self, key: str, data: bytes) -> None: + dst = self._path(key) + dst.parent.mkdir(parents=True, exist_ok=True) + tmp = temp_path(dst) + try: + with open(tmp, "xb") as f: + f.write(data) + os.replace(tmp, dst) + except BaseException: + tmp.unlink(missing_ok=True) + raise + + def delete(self, keys: list[str]) -> None: + for k in keys: + self._path(k).unlink(missing_ok=True) + + def list(self, prefix: str) -> Iterator[tuple[str, int, float]]: + if not self.root.is_dir(): + return + for p in sorted(self.root.rglob("*")): + if p.is_file() and not p.name.endswith(".part"): + key = p.relative_to(self.root).as_posix() + if key.startswith(prefix): + st = p.stat() + yield key, st.st_size, st.st_mtime + + +class S3Remote(Remote): + """A remote in S3-compatible object storage (`s3:///`), through boto3 (the `remote` extra).""" + + def __init__(self, url: str, endpoint: str | None, key_id: str | None, secret: str | None, region: str): + parts = urllib.parse.urlsplit(url) + if not parts.netloc: + raise ValueError("an s3:// remote names its bucket: s3:///") + self.url, self.bucket = url, parts.netloc + prefix = parts.path.strip("/") + self.prefix = f"{prefix}/" if prefix else "" + self.endpoint, self.key_id, self.secret, self.region = endpoint, key_id, secret, region + self._local = threading.local() + + def __getstate__(self): + return {k: getattr(self, k) for k in ("url", "endpoint", "key_id", "secret", "region")} + + def __setstate__(self, state): + self.__init__(state["url"], state["endpoint"], state["key_id"], state["secret"], state["region"]) + + def __repr__(self) -> str: # never the credentials + return f"S3Remote({self.url!r})" + + @property + def _s3(self): + """One client per thread (boto3 clients are not shared between threads here).""" + client = getattr(self._local, "client", None) + if client is None: + try: + import boto3 + from botocore.config import Config + except ImportError: + raise RuntimeError("an s3:// depot remote needs boto3: pip install 'nnnotes[remote]'") from None + config = Config( + s3={"addressing_style": "path"}, + retries={"max_attempts": 8, "mode": "adaptive"}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + ) + client = self._local.client = boto3.client( + "s3", + endpoint_url=self.endpoint or None, + aws_access_key_id=self.key_id or None, + aws_secret_access_key=self.secret or None, + region_name=self.region, + config=config, + ) + return client + + @staticmethod + def _missing(error) -> bool: + return error.response.get("Error", {}).get("Code") in ("404", "NoSuchKey", "NotFound") + + def get(self, key: str) -> bytes | None: + from botocore.exceptions import ClientError + + try: + return self._s3.get_object(Bucket=self.bucket, Key=self.prefix + key)["Body"].read() + except ClientError as e: + if self._missing(e): + return None + raise + + def size(self, key: str) -> int | None: + from botocore.exceptions import ClientError + + try: + return int(self._s3.head_object(Bucket=self.bucket, Key=self.prefix + key)["ContentLength"]) + except ClientError as e: + if self._missing(e): + return None + raise + + def put(self, key: str, data: bytes) -> None: + self._s3.put_object(Bucket=self.bucket, Key=self.prefix + key, Body=data) + + def put_file(self, key: str, path: Path) -> None: + self._s3.upload_file(str(path), self.bucket, self.prefix + key) + + def delete(self, keys: list[str]) -> None: + for i in range(0, len(keys), 1000): + batch = [{"Key": self.prefix + k} for k in keys[i : i + 1000]] + if batch: + self._s3.delete_objects(Bucket=self.bucket, Delete={"Objects": batch, "Quiet": True}) + + def list(self, prefix: str) -> Iterator[tuple[str, int, float]]: + pages = self._s3.get_paginator("list_objects_v2").paginate(Bucket=self.bucket, Prefix=self.prefix + prefix) + for page in pages: + for o in page.get("Contents", ()): + yield o["Key"][len(self.prefix) :], int(o["Size"]), o["LastModified"].timestamp() + + +def open_remote( + url: str, + *, + endpoint: str | None = None, + key_id: str | None = None, + secret: str | None = None, + region: str | None = None, +) -> Remote: + """The remote a URL names: `s3:///` (with the endpoint, credentials and signing region of the + object storage; region default us-east-1) or `file://`.""" + parts = urllib.parse.urlsplit(url) + if parts.scheme == "s3": + return S3Remote(url, endpoint, key_id, secret, region or "us-east-1") + if parts.scheme == "file": + path = urllib.parse.unquote(parts.path) + if parts.netloc or not path: # file://host/path is not a local directory + raise ValueError("a file:// remote names a local directory: file:///") + if os.name == "nt" and len(path) > 2 and path[0] == "/" and path[2] == ":": + path = path[1:] # file:///C:/x + return FileRemote(url, Path(path)) + raise ValueError("a depot remote is s3:/// or file:///") diff --git a/src/nnnotes/source/configfile.py b/src/nnnotes/source/configfile.py index 14321b5..4f4774b 100644 --- a/src/nnnotes/source/configfile.py +++ b/src/nnnotes/source/configfile.py @@ -23,7 +23,14 @@ from . import languages from .config import Config, ConfigError, check_catalog_version, env_name, template -SECRETS = {("bundle", "key"), ("bundle", "nonce_seed"), ("master", "key"), ("master", "iv")} +SECRETS = { + ("bundle", "key"), + ("bundle", "nonce_seed"), + ("master", "key"), + ("master", "iv"), + ("depot", "access_key_id"), + ("depot", "secret_access_key"), +} HEX_BYTES = {("bundle", "key"): 16, ("bundle", "nonce_seed"): None, ("master", "key"): 32, ("master", "iv"): 32} REGION = "servers." # the section of the per-region settings TEMPLATE_REGION = "tw" # the example region table of the template @@ -155,6 +162,14 @@ def validate(setting: Setting, value: Value) -> None: raise ValueError(f"must be one of {', '.join(LINKS)}") elif k == ("depot", "downloads"): parse_budget(value) + elif k == ("depot", "remote"): + u = urlsplit(value) + if not ((u.scheme == "s3" and u.netloc) or (u.scheme == "file" and not u.netloc and u.path)): + raise ValueError("must be s3:/// or file:///") + elif k == ("depot", "endpoint"): + u = urlsplit(value) + if u.scheme not in ("https", "http") or not u.netloc: + raise ValueError("must be an http(s):// URL") # ---------------------------------------------------------------- TOML text diff --git a/src/nnnotes/source/nnnotes.example.toml b/src/nnnotes/source/nnnotes.example.toml index 437beb1..350e687 100644 --- a/src/nnnotes/source/nnnotes.example.toml +++ b/src/nnnotes/source/nnnotes.example.toml @@ -81,6 +81,18 @@ player = "" # how much of /downloads (decrypted bundles, raw CDN files) to keep once no command uses the depot: a size such # as 20GiB or 500MB, or all; empty: 0 (the last command to end removes them; they are downloaded again when needed) downloads = "" +# a remote of the depot, shared by the machines that use it: s3:/// (S3-compatible object storage, +# through boto3: pip install 'nnnotes[remote]') or file:///; the depot reads the contents, results, input +# identities and runs it lacks from the remote and writes what it adds to it +remote = "" +# the endpoint URL of the S3-compatible object storage of the remote; empty: AWS S3 +endpoint = "" +# the access key id of the object storage; empty: boto3 looks for credentials itself (AWS_ACCESS_KEY_ID, ~/.aws) +access_key_id = "" +# the secret access key of the object storage +secret_access_key = "" +# the region requests to the object storage are signed for; empty: us-east-1 +signing_region = "" [export] # how `nnnotes export assets` makes the layout files from the depot: auto (a clone where the file system makes them, else diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index f049175..2e10549 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -388,6 +388,21 @@ def test_a_depot_that_keeps_no_downloads_hits_without_them(tmp_path, capsys, fak assert again["layouts"]["original"]["write"] == 0 +def test_a_second_depot_with_the_same_remote_hits_without_downloads(tmp_path, capsys, fake, monkeypatch): + monkeypatch.setenv("NNNOTES_DEPOT_REMOTE", (tmp_path / "remote").as_uri()) + d = setup_data(tmp_path, depots=("a",)) + code, out, err = export(capsys, tmp_path, d, "a", "out-a") + assert code == 1, err + first = json.loads(out) + code, out, err = export(capsys, tmp_path, d, "b", "out-b") # a depot with no downloads and no CDN + assert out, err + again = json.loads(out) + assert code == 1 and again["tasks"]["ran"] == 0 and again["run"] == first["run"], err + skip = [f"{assets.REPORTS}/run.json"] # the statuses: ran, then hit + assert tree(tmp_path / "out-b", skip) == tree(tmp_path / "out-a", skip) + assert not [p for p in (tmp_path / "b").glob("downloads/**/*") if p.is_file()] # nothing downloaded + + def test_export_is_the_same_with_worker_processes(tmp_path, capsys, fake): d = setup_data(tmp_path, depots=("s0", "s2")) export(capsys, tmp_path, d, "s0", "o0") diff --git a/tests/pipeline/test_depot_gc.py b/tests/pipeline/test_depot_gc.py index 75df301..7170dd6 100644 --- a/tests/pipeline/test_depot_gc.py +++ b/tests/pipeline/test_depot_gc.py @@ -94,3 +94,14 @@ def test_the_gc_command_runs_only_when_no_other_command_does(tmp_path, capsys): cli.main(argv) assert json.loads(capsys.readouterr().out)["contents"]["removed"] == 1 and not d.path(orphan).exists() assert not list((tmp_path / "leases").iterdir()) + + +def test_the_gc_command_collects_the_remote_with_remote(tmp_path, capsys, monkeypatch): + argv = ["--depot", str(tmp_path / "d"), "depot", "gc", "--remote", "--json"] + with pytest.raises(SystemExit): + cli.main(argv) + assert "the depot has no remote" in capsys.readouterr().err + monkeypatch.setenv("NNNOTES_DEPOT_REMOTE", (tmp_path / "remote").as_uri()) + cli.main(argv) + r = json.loads(capsys.readouterr().out) + assert r["remote"]["runs"] == {"kept": 0, "removed": 0} and r["contents"] == {"kept": 0, "removed": 0} diff --git a/tests/pipeline/test_depot_remote.py b/tests/pipeline/test_depot_remote.py new file mode 100644 index 0000000..fe85851 --- /dev/null +++ b/tests/pipeline/test_depot_remote.py @@ -0,0 +1,173 @@ +"""A depot with a remote: what the local depot writes goes to the remote too, and a depot that lacks something reads it +from the remote and keeps it.""" + +from datetime import UTC, datetime + +import pytest + +from nnnotes.pipeline import contract +from nnnotes.pipeline.contract import Input, Task +from nnnotes.pipeline.depot import Depot, DepotConflict +from nnnotes.pipeline.remote import FileRemote, open_remote + + +def commit(d: Depot, task: Task, payload: bytes) -> dict: + aid = contract.artifact_id(task.id, "data.bin") + art = contract.artifact(aid, d.add(payload, "bin"), contract.provenance(task), {"kind": "test"}) + doc = contract.result(task, [art], []) + d.commit(doc) + return doc + + +def run_doc(selection, tasks) -> dict: + return contract.run_doc( + context={"region": "r"}, + selection=selection, + params={}, + stages={"t.x": 1}, + tasks=[{"id": t.id, "key": t.key, "status": "ran"} for t in tasks], + layouts={}, + summary={}, + ) + + +def test_open_remote_names_a_directory_or_a_bucket(tmp_path): + r = open_remote(tmp_path.as_uri()) + assert isinstance(r, FileRemote) and r.root == tmp_path + s3 = open_remote("s3://bucket/some/prefix", endpoint="https://storage.example", key_id="KEYID", secret="SECRET") + assert (s3.bucket, s3.prefix) == ("bucket", "some/prefix/") + assert "KEYID" not in repr(s3) and "SECRET" not in repr(s3) + for bad in ("http://example/x", "s3:///x", "file://host/share"): + with pytest.raises(ValueError): + open_remote(bad) + + +def test_a_second_depot_finds_the_results_and_contents_of_the_first(tmp_path): + remote = open_remote((tmp_path / "remote").as_uri()) + first, second = Depot(tmp_path / "a", remote=remote), Depot(tmp_path / "b", remote=remote) + stored = first.put(b"a stored input") + task = Task("t.x", 1, "a", inputs=(Input("doc", stored, 14, None, ({"kind": "depot"},)),)) + doc = commit(first, task, b"payload") + run = run_doc(["group:a"], [task]) + first.write_run(run) + first.identify(first.path(stored), "bundles", "x_0123.bundle") + + art = doc["artifacts"][0]["content"]["sha256"] + assert not second.path(art).exists() and second.has(art, 7) and second.has_result(task.key) + assert second.result(task.key) == doc and second.read(art) == b"payload" + assert second.path(art).is_file() # kept locally + assert second.open_input(task.inputs[0]).read_bytes() == b"a stored input" + assert second.latest_run(["group:a"]) == run and second.run(run["id"]) == run + assert second.named("bundles", "x_0123.bundle") == (stored, 14) + assert second.named("bundles", "other.bundle") is None + + +class Counting(FileRemote): + """A directory remote that counts the objects it is asked the size of.""" + + def __init__(self, url, root): + super().__init__(url, root) + self.asked = [] + + def size(self, key): + self.asked.append(key) + return super().size(key) + + +def test_the_latest_run_of_the_remote_tells_which_results_it_holds(tmp_path): + remote = open_remote((tmp_path / "remote").as_uri()) + first = Depot(tmp_path / "a", remote=remote) + tasks = [Task("t.x", 1, n) for n in "abc"] + for t in tasks: + commit(first, t, t.subject.encode()) + first.write_run(run_doc(["group:a"], tasks)) + counting = Counting(remote.url, remote.root) + second = Depot(tmp_path / "b", remote=counting) + assert second.latest_run(["group:a"])["id"] == run_doc(["group:a"], tasks)["id"] + assert all(second.has_result(t.key) for t in tasks) and counting.asked == [] + assert not second.has_result(Task("t.x", 1, "z").key) and len(counting.asked) == 1 + + +def test_a_depot_crosses_processes_with_its_remote(tmp_path): + import pickle + + remote = open_remote((tmp_path / "remote").as_uri()) + d = pickle.loads(pickle.dumps(Depot(tmp_path / "a", remote=remote))) + assert isinstance(d.remote, FileRemote) and d.remote.root == remote.root + assert "remote=" in repr(d) + + +def test_contents_stored_before_the_remote_go_to_it_with_their_result(tmp_path): + task = Task("t.x", 1, "a") + local = Depot(tmp_path / "a") + aid = contract.artifact_id(task.id, "data.bin") + art = contract.artifact(aid, local.add(b"payload", "bin"), contract.provenance(task), {"kind": "test"}) + remote = open_remote((tmp_path / "remote").as_uri()) + Depot(tmp_path / "a", remote=remote).commit(contract.result(task, [art], [])) + assert Depot(tmp_path / "b", remote=remote).read(art["content"]["sha256"]) == b"payload" + + +def test_a_content_of_the_remote_is_checked_against_its_id(tmp_path): + remote = open_remote((tmp_path / "remote").as_uri()) + sha = Depot(tmp_path / "a", remote=remote).put(b"right") + remote.put(f"cas/sha256/{sha[:2]}/{sha}", b"wrong") + with pytest.raises(RuntimeError, match="differs from its id"): + Depot(tmp_path / "b", remote=remote).read(sha) + + +def test_s3_requests_name_the_bucket_and_the_prefixed_keys(): + pytest.importorskip("boto3") + from botocore.stub import Stubber + + r = open_remote("s3://bucket/depot", endpoint="https://storage.example", key_id="k", secret="s") + written = datetime(2026, 1, 2, tzinfo=UTC) + with Stubber(r._s3) as stub: + stub.add_response("head_object", {"ContentLength": 3}, {"Bucket": "bucket", "Key": "depot/ac/ab/x.json"}) + stub.add_client_error( + "head_object", + service_error_code="404", + http_status_code=404, + expected_params={"Bucket": "bucket", "Key": "depot/ac/ab/y.json"}, + ) + stub.add_response("put_object", {}, {"Bucket": "bucket", "Key": "depot/runs/r.json", "Body": b"{}"}) + stub.add_response( + "list_objects_v2", + { + "Contents": [{"Key": "depot/cas/sha256/ab/abc", "Size": 5, "LastModified": written}], + "IsTruncated": False, + }, + {"Bucket": "bucket", "Prefix": "depot/cas/"}, + ) + stub.add_response( + "delete_objects", + {}, + {"Bucket": "bucket", "Delete": {"Objects": [{"Key": "depot/cas/sha256/ab/abc"}], "Quiet": True}}, + ) + assert r.size("ac/ab/x.json") == 3 and r.size("ac/ab/y.json") is None + r.put("runs/r.json", b"{}") + assert list(r.list("cas/")) == [("cas/sha256/ab/abc", 5, written.timestamp())] + r.delete(["cas/sha256/ab/abc"]) + stub.assert_no_pending_responses() + + +def test_a_result_differing_from_the_remote_one_is_a_conflict(tmp_path): + remote = open_remote((tmp_path / "remote").as_uri()) + task = Task("t.x", 1, "a") + commit(Depot(tmp_path / "a", remote=remote), task, b"one") + with pytest.raises(DepotConflict): + commit(Depot(tmp_path / "b", remote=remote), task, b"two") + + +def test_gc_of_the_remote_keeps_what_its_kept_runs_reach(tmp_path): + remote = open_remote((tmp_path / "remote").as_uri()) + d = Depot(tmp_path / "a", remote=remote) + kept, gone = Task("t.x", 1, "a"), Task("t.x", 1, "b") + commit(d, kept, b"kept") + commit(d, gone, b"gone") + d.write_run(run_doc(["group:a"], [kept])) + assert Depot(tmp_path / "b", remote=remote).gc(remote=True)["remote"]["results"] == {"kept": 2, "removed": 0} + r = Depot(tmp_path / "b", remote=remote).gc(remote=True, grace=0) + assert r["remote"]["results"] == {"kept": 1, "removed": 1} and r["remote"]["contents"] == {"kept": 1, "removed": 1} + assert ( + remote.size(f"ac/{kept.key[:2]}/{kept.key}.json") and remote.size(f"ac/{gone.key[:2]}/{gone.key}.json") is None + ) diff --git a/tests/source/test_configfile.py b/tests/source/test_configfile.py index acb20a2..76fa57e 100644 --- a/tests/source/test_configfile.py +++ b/tests/source/test_configfile.py @@ -36,6 +36,29 @@ def leaves(table, section=""): yield f"{section}.{k}" if section else k, v +@pytest.mark.parametrize( + "name, value, ok", + [ + ("depot.remote", "s3://bucket/prefix", True), + ("depot.remote", "file:///srv/depot", True), + ("depot.remote", "s3:///prefix", False), + ("depot.remote", "https://bucket/prefix", False), + ("depot.remote", "file://host/share", False), + ("depot.endpoint", "https://storage.example", True), + ("depot.endpoint", "storage.example", False), + ("depot.downloads", "20GiB", True), + ("depot.downloads", "lots", False), + ], +) +def test_depot_settings_are_checked(name, value, ok): + setting, _ = configfile.resolve(name) + if ok: + configfile.validate(setting, value) + else: + with pytest.raises(ValueError): + configfile.validate(setting, value) + + def test_settings_are_the_documented_ones(): doc = (ROOT / "docs" / "configuration.md").read_text(encoding="utf-8") documented = set() @@ -54,7 +77,14 @@ def test_template_values_are_empty(): data = tomllib.loads(template().decode("utf-8")) assert all(v in ("", []) for _, v in leaves(data)) secrets = {f"{s.section}.{s.key}" for s in configfile.SETTINGS if s.secret} - assert secrets == {"bundle.key", "bundle.nonce_seed", "master.key", "master.iv"} + assert secrets == { + "bundle.key", + "bundle.nonce_seed", + "master.key", + "master.iv", + "depot.access_key_id", + "depot.secret_access_key", + } def test_resolve(): @@ -170,7 +200,7 @@ def ask_secret(p): got = {f"{s}.{k}": v for s, k, v in values} assert regions == ["tw", "kr"] assert got == {"bundle.key": KEY, "master.key": MKEY, "catalog.region": "tw", "catalog.language": "zh-Hant"} - assert len(secret_prompts) == 5 and all("(hidden)" in p for p in secret_prompts) + assert len(secret_prompts) == 7 and all("(hidden)" in p for p in secret_prompts) assert any(p.startswith("[servers.kr] cdn") for p in prompts) text = configfile.render(values, regions) assert set(tomllib.loads(text)["servers"]) == {"tw", "kr"} From 61143bce00eed7a2e3791ebac209fca7f7fcdef0 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 14 Oct 2026 23:42:35 +0800 Subject: [PATCH 06/91] refactor: move the atoms to the unity layer The atoms (texture decoding, PNG and ASTC encoding, sprite images, mesh arrays, glTF, file sniffing and the bundle reader) are the pure functions Unity objects are converted with, and live in nnnotes.unity.atoms. Object ids (nnnotes.unity.ids) and media types (nnnotes.core.media) sit below them; nnnotes.pipeline.contract keeps object_id, parse_object_id, media_type and MEDIA_TYPES among its names. Atom ids, and with them task keys, are unchanged. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 4 +- docs/stages.md | 4 +- scripts/atom_eval.py | 18 ++++---- scripts/atom_eval_pillow.py | 6 +-- scripts/atom_eval_unityrs.py | 18 ++++---- src/nnnotes/core/media.py | 34 +++++++++++++++ src/nnnotes/pipeline/census.py | 2 +- src/nnnotes/pipeline/contract.py | 43 ++----------------- src/nnnotes/pipeline/cristages.py | 2 +- src/nnnotes/pipeline/modelstages.py | 2 +- src/nnnotes/pipeline/objexport.py | 14 +++--- .../{pipeline => unity}/atoms/__init__.py | 0 src/nnnotes/{pipeline => unity}/atoms/astc.py | 0 src/nnnotes/{pipeline => unity}/atoms/gltf.py | 0 src/nnnotes/{pipeline => unity}/atoms/mesh.py | 0 src/nnnotes/{pipeline => unity}/atoms/png.py | 0 .../{pipeline => unity}/atoms/reader.py | 6 +-- .../{pipeline => unity}/atoms/sniff.py | 4 +- .../{pipeline => unity}/atoms/sprite.py | 0 .../{pipeline => unity}/atoms/texture.py | 0 src/nnnotes/unity/ids.py | 17 ++++++++ tests/pipeline/test_atom_eval.py | 6 +-- tests/pipeline/test_cristages.py | 2 +- tests/pipeline/test_objexport.py | 12 +++--- tests/{pipeline => unity}/test_atoms_mesh.py | 6 +-- .../{pipeline => unity}/test_atoms_sprite.py | 8 ++-- .../{pipeline => unity}/test_atoms_texture.py | 16 +++---- tests/{pipeline => unity}/test_sniff.py | 4 +- 28 files changed, 122 insertions(+), 106 deletions(-) create mode 100644 src/nnnotes/core/media.py rename src/nnnotes/{pipeline => unity}/atoms/__init__.py (100%) rename src/nnnotes/{pipeline => unity}/atoms/astc.py (100%) rename src/nnnotes/{pipeline => unity}/atoms/gltf.py (100%) rename src/nnnotes/{pipeline => unity}/atoms/mesh.py (100%) rename src/nnnotes/{pipeline => unity}/atoms/png.py (100%) rename src/nnnotes/{pipeline => unity}/atoms/reader.py (98%) rename src/nnnotes/{pipeline => unity}/atoms/sniff.py (99%) rename src/nnnotes/{pipeline => unity}/atoms/sprite.py (100%) rename src/nnnotes/{pipeline => unity}/atoms/texture.py (100%) create mode 100644 src/nnnotes/unity/ids.py rename tests/{pipeline => unity}/test_atoms_mesh.py (98%) rename tests/{pipeline => unity}/test_atoms_sprite.py (97%) rename tests/{pipeline => unity}/test_atoms_texture.py (94%) rename tests/{pipeline => unity}/test_sniff.py (97%) diff --git a/docs/architecture.md b/docs/architecture.md index 8b5646b..1aeb4a7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -12,10 +12,10 @@ subpackage of `nnnotes` and imports only the layers below it. | Products | `nnnotes.site`, `nnnotes.musicdata` | the ournotes-player site (charts, models, stories, indexes); the music data file and its deck statistics | | Pipeline | `nnnotes.pipeline` | stages, task descriptions and results, the content-addressed depot, plans, layouts, the catalog versions, the views of the master data | | Content | `nnnotes.content` | the game's subjects as exports: ADV episodes and stories, lives and scores, spots, rooms, the UI library | -| Unity objects | `nnnotes.unity` | bundle reading, the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | +| Unity objects | `nnnotes.unity` | bundle reading, object ids, the atoms (the pure functions objects are converted with: texture decoding, sprite images, mesh arrays, glTF, PNG, file sniffing), the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | | File formats | `nnnotes.formats` | formats read from bytes: CRI ACB/AWB and HCA, USM movies, CRI Lips data, shaders, motions | | Sources | `nnnotes.source` | the configuration, the APK, Addressables catalogs and bundles, the game API, master data, languages, `Session` | -| Core | `nnnotes.core` | the one JSON writer and the process caches | +| Core | `nnnotes.core` | the one JSON writer, media types, the process caches and the downloads cache | The order is a contract: the `layers` contract of [import-linter](https://import-linter.readthedocs.io/) in `pyproject.toml` fails the build when a module imports a layer above its own, and `nnnotes.site` and diff --git a/docs/stages.md b/docs/stages.md index 16d9e9f..fadf601 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -128,14 +128,14 @@ else found on PATH. The `boot` input is the APK's `assets/bin/Data/data.unity3d` ## Atoms -`nnnotes.pipeline.atoms.ATOMS[name]` is the reference implementation of an atom: its id, its function and its cost estimate. +`nnnotes.unity.atoms.ATOMS[name]` is the reference implementation of an atom: its id, its function and its cost estimate. The id names the libraries whose behaviour the output depends on, with their installed versions, and the revision of the nnnotes code (`pillow-12.1.1/1`); `nnnotes/` when no library's behaviour is involved. Task keys include the ids of the atoms a task uses, so upgrading a library or changing an atom re-runs exactly the tasks that use it. | Atom | Reference implementation | Id | Does | |---|---|---|---| -| `reader` | `nnnotes.pipeline.atoms.reader.open_bundle` (UnityPy) | `unitypy-/1` | one bundle loaded alone from its bytes: serialized files, objects, raw bytes, typetrees, container, externals, and the inputs of `texture.decode` and `mesh.arrays` (resource files are read from inside the bundle only) | +| `reader` | `nnnotes.unity.atoms.reader.open_bundle` (UnityPy) | `unitypy-/1` | one bundle loaded alone from its bytes: serialized files, objects, raw bytes, typetrees, container, externals, and the inputs of `texture.decode` and `mesh.arrays` (resource files are read from inside the bundle only) | | `texture.decode` | UnityPy `Texture2DConverter.parse_image_data` | `unitypy-+texture2ddecoder-+astc-encoder-py-+pillow-/1` | the first mip level, top row first, as UnityPy's `Texture2D.image` | | `png.encode` | Pillow | `pillow-/1` | PNG bytes; level 6 (the default) gives Pillow's default bytes, the ones the other commands write | | `astc.container` | nnnotes | `nnnotes/1` | the standard `.astc` file (16-byte header, first mip level's blocks) | diff --git a/scripts/atom_eval.py b/scripts/atom_eval.py index ece713e..1e23446 100644 --- a/scripts/atom_eval.py +++ b/scripts/atom_eval.py @@ -2,7 +2,7 @@ """Measure candidate implementations of the export atoms against the reference implementations (maintainer tool; not installed, not needed to use nnnotes). -`nnnotes.pipeline.atoms.ATOMS` holds one reference implementation per atom; `NNNOTES_ATOM_=module:attribute` names +`nnnotes.unity.atoms.ATOMS` holds one reference implementation per atom; `NNNOTES_ATOM_=module:attribute` names another one for evaluation (docs/stages.md, "Atoms"). This script gives the numbers a switch is decided on: sample DIR -o FILE a seeded stratified sample of a bundle directory: one bundle of every @@ -42,7 +42,7 @@ A candidate view is the reader candidate's result; the optional methods census(), text_bytes(obj), font_bytes(obj), sprite_image(obj) and the attribute typetree_form serve the checks that have no atom of their own. A candidate -raising NotImplementedError has no such facet ("absent"); `nnnotes.pipeline.atoms.Unsupported` is a refusal. +raising NotImplementedError has no such facet ("absent"); `nnnotes.unity.atoms.Unsupported` is a refusal. Statuses: equal, differ, refused (the candidate raised where the reference gave a result), refused.both, extra (a result where the reference refused), absent, reference-error. Output: one JSON document (-o); m1 and m3 also write @@ -133,7 +133,7 @@ def check_clean_env() -> None: def load_impl(spec: str): """The Impl named by module:attribute (an Impl, or a callable returning one).""" - from nnnotes.pipeline.atoms import Impl + from nnnotes.unity.atoms import Impl module, sep, attr = spec.partition(":") if not sep or not module or not attr: @@ -146,7 +146,7 @@ def load_impl(spec: str): def parse_atoms(items) -> list[tuple[str, str]]: - from nnnotes.pipeline.atoms import ATOMS + from nnnotes.unity.atoms import ATOMS out = [] for it in items or (): @@ -449,7 +449,7 @@ def cmd_sample(a) -> dict: # ---------------------------------------------------------------- m1 def m1_init(cfg: dict) -> None: - from nnnotes.pipeline.atoms import resolve + from nnnotes.unity.atoms import resolve G.clear() G.update(cfg) @@ -477,7 +477,7 @@ def _key(info) -> tuple: def _call(fn, *args): """(result, None) or (None, refusal) where refusal is the Unsupported code or the error's message.""" - from nnnotes.pipeline.atoms import Unsupported + from nnnotes.unity.atoms import Unsupported try: return fn(*args), None @@ -700,7 +700,7 @@ def hdr_reference(inp): import numpy as np from PIL import Image - from nnnotes.pipeline.atoms.texture import ASTC_BLOCKS + from nnnotes.unity.atoms.texture import ASTC_BLOCKS bx, by = ASTC_BLOCKS[int(inp.format)] w, h = int(inp.width), int(inp.height) @@ -723,7 +723,7 @@ def _hdr_library_image(inp): def check_textures(T: Tally, ref, cv, infos, cmap, checks) -> None: - from nnnotes.pipeline.atoms.texture import format_name, is_hdr_astc + from nnnotes.unity.atoms.texture import format_name, is_hdr_astc ref_dec, ref_png = G["ref"]["texture.decode"], G["ref"]["png.encode"] decs, pngs = G["cand"].get("texture.decode", []), G["cand"].get("png.encode", []) @@ -1173,7 +1173,7 @@ def _m3_worker(task) -> dict: def _apply_atoms(atoms) -> None: - from nnnotes.pipeline.atoms import env_name, resolve + from nnnotes.unity.atoms import env_name, resolve for name, spec in atoms: os.environ[env_name(name)] = spec diff --git a/scripts/atom_eval_pillow.py b/scripts/atom_eval_pillow.py index ec5d9f4..b5741b4 100644 --- a/scripts/atom_eval_pillow.py +++ b/scripts/atom_eval_pillow.py @@ -1,6 +1,6 @@ """Candidate `png.encode` implementations on Pillow, for scripts/atom_eval.py only. -The reference (`nnnotes.pipeline.atoms.png.encode`) writes at the level a task's `png.level` parameter gives (6 by +The reference (`nnnotes.unity.atoms.png.encode`) writes at the level a task's `png.level` parameter gives (6 by default). These fix the setting instead, so several can be measured side by side in one `atom_eval.py m1` run: level1, level3, level6, level9 Pillow at that zlib level @@ -15,8 +15,8 @@ from functools import cache from importlib import metadata -from nnnotes.pipeline.atoms import Impl -from nnnotes.pipeline.atoms import png as png_atom +from nnnotes.unity.atoms import Impl +from nnnotes.unity.atoms import png as png_atom def _pillow() -> str: diff --git a/scripts/atom_eval_unityrs.py b/scripts/atom_eval_unityrs.py index d403c35..e544f1a 100644 --- a/scripts/atom_eval_unityrs.py +++ b/scripts/atom_eval_unityrs.py @@ -27,13 +27,13 @@ from functools import cache from importlib import metadata -from nnnotes.pipeline.atoms import Impl, Unsupported -from nnnotes.pipeline.atoms import png as png_atom -from nnnotes.pipeline.atoms import reader as reader_atom -from nnnotes.pipeline.atoms import texture as texture_atom -from nnnotes.pipeline.atoms.mesh import MeshInput -from nnnotes.pipeline.atoms.reader import ObjectInfo, UnityBundle -from nnnotes.pipeline.atoms.texture import TextureInput +from nnnotes.unity.atoms import Impl, Unsupported +from nnnotes.unity.atoms import png as png_atom +from nnnotes.unity.atoms import reader as reader_atom +from nnnotes.unity.atoms import texture as texture_atom +from nnnotes.unity.atoms.mesh import MeshInput +from nnnotes.unity.atoms.reader import ObjectInfo, UnityBundle +from nnnotes.unity.atoms.texture import TextureInput FALLBACKS = {"png": 0} _KEEP = "unity_rs.image" @@ -390,7 +390,7 @@ def png_encode_fast() -> Impl: def _mesh(inp: MeshInput): import numpy as np - from nnnotes.pipeline.atoms.mesh import Channel, MeshArrays, Primitive + from nnnotes.unity.atoms.mesh import Channel, MeshArrays, Primitive src = getattr(inp, "source", None) if src is None: @@ -429,6 +429,6 @@ def _mesh(inp: MeshInput): @cache def mesh_arrays() -> Impl: - from nnnotes.pipeline.atoms import mesh as mesh_atom + from nnnotes.unity.atoms import mesh as mesh_atom return Impl(_id(_rs(), "obj"), _mesh, mesh_atom.cost) diff --git a/src/nnnotes/core/media.py b/src/nnnotes/core/media.py new file mode 100644 index 0000000..5e96322 --- /dev/null +++ b/src/nnnotes/core/media.py @@ -0,0 +1,34 @@ +"""Media types of file extensions.""" + +from __future__ import annotations + +__all__ = ["MEDIA_TYPES", "media_type"] + +MEDIA_TYPES = { + "png": "image/png", + "astc": "image/astc", + "jpg": "image/jpeg", + "json": "application/json", + "glb": "model/gltf-binary", + "flac": "audio/flac", + "wav": "audio/wav", + "ogg": "audio/ogg", + "mkv": "video/x-matroska", + "ivf": "video/x-ivf", + "adx": "audio/x-adx", + "ttf": "font/ttf", + "otf": "font/otf", + "ttc": "font/collection", + "woff": "font/woff", + "woff2": "font/woff2", + "txt": "text/plain", + "xml": "application/xml", + "csv": "text/csv", + "bin": "application/octet-stream", + "bytes": "application/octet-stream", +} + + +def media_type(ext: str) -> str: + """The media type of a file extension (case-insensitive); application/octet-stream when unknown.""" + return MEDIA_TYPES.get(ext.lower(), "application/octet-stream") diff --git a/src/nnnotes/pipeline/census.py b/src/nnnotes/pipeline/census.py index 2875e36..23a75cb 100644 --- a/src/nnnotes/pipeline/census.py +++ b/src/nnnotes/pipeline/census.py @@ -25,8 +25,8 @@ import re +from ..unity.atoms import impl_id from . import contract -from .atoms import impl_id from .contract import Cost, Input from .stages import Output, Pending, Stage diff --git a/src/nnnotes/pipeline/contract.py b/src/nnnotes/pipeline/contract.py index 8c958c8..68d6f26 100644 --- a/src/nnnotes/pipeline/contract.py +++ b/src/nnnotes/pipeline/contract.py @@ -35,6 +35,10 @@ from functools import cached_property from ..core import jsonio +from ..core.media import MEDIA_TYPES as MEDIA_TYPES +from ..core.media import media_type as media_type +from ..unity.ids import object_id as object_id +from ..unity.ids import parse_object_id as parse_object_id CANON = "canon/1" TASK = "nnnotes.task/1" @@ -59,29 +63,6 @@ LOCATOR_KINDS = ("depot", "download", "file", "catalog") # an object's primary artifact is the one whose role comes first here; its other artifacts are secondary PRIMARY_ROLES = ("image", "mesh", "audio", "video", "font", "data", "prefab", "json") -MEDIA_TYPES = { - "png": "image/png", - "astc": "image/astc", - "jpg": "image/jpeg", - "json": "application/json", - "glb": "model/gltf-binary", - "flac": "audio/flac", - "wav": "audio/wav", - "ogg": "audio/ogg", - "mkv": "video/x-matroska", - "ivf": "video/x-ivf", - "adx": "audio/x-adx", - "ttf": "font/ttf", - "otf": "font/otf", - "ttc": "font/collection", - "woff": "font/woff", - "woff2": "font/woff2", - "txt": "text/plain", - "xml": "application/xml", - "csv": "text/csv", - "bin": "application/octet-stream", - "bytes": "application/octet-stream", -} _SHA256 = re.compile(r"[0-9a-f]{64}") _EXT = re.compile(r"[0-9a-z]+") _BUNDLE_SUFFIX = re.compile(r"(_[0-9a-f]{32})?\.bundle$") @@ -181,18 +162,6 @@ def is_sha256(s) -> bool: # ---------------------------------------------------------------- identities -def object_id(file: str, path_id: int) -> str: - return f"{file}:{int(path_id)}" - - -def parse_object_id(oid: str) -> tuple[str, int]: - """(serialized file, pathId) of an object id.""" - file, sep, pid = oid.rpartition(":") - if not sep or not file: - raise ValueError(f"not an object id: {oid!r}") - return file, int(pid) - - def artifact_id(owner: str, role: str) -> str: """The id of the artifact `role` of `owner` (an object id, or a task id for artifacts of no object).""" if "#" in owner: @@ -262,10 +231,6 @@ def stable_address( return f"{bundle}:{int(path_id)}" -def media_type(ext: str) -> str: - return MEDIA_TYPES.get(ext.lower(), "application/octet-stream") - - # ---------------------------------------------------------------- tasks @dataclass(frozen=True) class Cost: diff --git a/src/nnnotes/pipeline/cristages.py b/src/nnnotes/pipeline/cristages.py index c80c897..fbb0098 100644 --- a/src/nnnotes/pipeline/cristages.py +++ b/src/nnnotes/pipeline/cristages.py @@ -44,8 +44,8 @@ from ..formats.encoders import FLAC_LEVEL from ..source.apkset import ApkSet +from ..unity.atoms import impl_id from . import contract -from .atoms import impl_id from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage diff --git a/src/nnnotes/pipeline/modelstages.py b/src/nnnotes/pipeline/modelstages.py index 1fadad3..290feda 100644 --- a/src/nnnotes/pipeline/modelstages.py +++ b/src/nnnotes/pipeline/modelstages.py @@ -35,8 +35,8 @@ from pathlib import Path from ..source.catalog import Bundle +from ..unity.atoms import impl_id from . import catalogdb, contract, link -from .atoms import impl_id from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage diff --git a/src/nnnotes/pipeline/objexport.py b/src/nnnotes/pipeline/objexport.py index a1fbb0d..c82db05 100644 --- a/src/nnnotes/pipeline/objexport.py +++ b/src/nnnotes/pipeline/objexport.py @@ -69,10 +69,10 @@ from types import SimpleNamespace from typing import TYPE_CHECKING, cast +from ..unity.atoms import ATOMS as ATOM_REGISTRY +from ..unity.atoms import Impl, Unsupported, atom_id, resolve from . import census as census_mod from . import contract, link -from .atoms import ATOMS as ATOM_REGISTRY -from .atoms import Impl, Unsupported, atom_id, resolve from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage @@ -948,7 +948,7 @@ def _texture_key(self, owner, pptr: dict) -> tuple: return k def conv_texture(self, info, holder: dict) -> list: - from .atoms.texture import ASTC_BLOCKS, format_name, is_hdr_astc + from ..unity.atoms.texture import ASTC_BLOCKS, format_name, is_hdr_astc key = (info.file, info.path_id) try: @@ -983,7 +983,7 @@ def conv_texture(self, info, holder: dict) -> list: def _decode_into(self, info, holder: dict) -> None: """The texture's image for its sprites (a task whose `classes` leave Texture2D out).""" - from .atoms.texture import format_name, is_hdr_astc + from ..unity.atoms.texture import format_name, is_hdr_astc try: inp = self.view.texture_input(info) @@ -997,7 +997,7 @@ def _decoded(self, key: tuple): return self.fns["texture.decode"](self.view.texture_input(self.info[key])) def conv_sprite(self, info, plan: dict, holder: dict) -> list: - from .atoms import sprite as sprite_atom + from ..unity.atoms import sprite as sprite_atom err = holder.get("error") if err is not None: @@ -1060,7 +1060,7 @@ def conv_sprite(self, info, plan: dict, holder: dict) -> list: return [image_id, meta_id] def conv_sprite_external(self, info, plan: dict) -> list: - from .atoms import sprite as sprite_atom + from ..unity.atoms import sprite as sprite_atom key = (info.file, info.path_id) m = self.fns["mesh.arrays"](self.view.mesh_input(info)) @@ -1700,7 +1700,7 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: def run(self, task, depot) -> Output: from PIL import Image - from .atoms import sprite as sprite_atom + from ..unity.atoms import sprite as sprite_atom _check_atoms(task) crop, encode = resolve("sprite.crop").fn, resolve("png.encode").fn diff --git a/src/nnnotes/pipeline/atoms/__init__.py b/src/nnnotes/unity/atoms/__init__.py similarity index 100% rename from src/nnnotes/pipeline/atoms/__init__.py rename to src/nnnotes/unity/atoms/__init__.py diff --git a/src/nnnotes/pipeline/atoms/astc.py b/src/nnnotes/unity/atoms/astc.py similarity index 100% rename from src/nnnotes/pipeline/atoms/astc.py rename to src/nnnotes/unity/atoms/astc.py diff --git a/src/nnnotes/pipeline/atoms/gltf.py b/src/nnnotes/unity/atoms/gltf.py similarity index 100% rename from src/nnnotes/pipeline/atoms/gltf.py rename to src/nnnotes/unity/atoms/gltf.py diff --git a/src/nnnotes/pipeline/atoms/mesh.py b/src/nnnotes/unity/atoms/mesh.py similarity index 100% rename from src/nnnotes/pipeline/atoms/mesh.py rename to src/nnnotes/unity/atoms/mesh.py diff --git a/src/nnnotes/pipeline/atoms/png.py b/src/nnnotes/unity/atoms/png.py similarity index 100% rename from src/nnnotes/pipeline/atoms/png.py rename to src/nnnotes/unity/atoms/png.py diff --git a/src/nnnotes/pipeline/atoms/reader.py b/src/nnnotes/unity/atoms/reader.py similarity index 98% rename from src/nnnotes/pipeline/atoms/reader.py rename to src/nnnotes/unity/atoms/reader.py index 8e55ffc..0cb56ce 100644 --- a/src/nnnotes/pipeline/atoms/reader.py +++ b/src/nnnotes/unity/atoms/reader.py @@ -17,7 +17,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Protocol -from .. import contract +from ..ids import object_id from . import estimate from .mesh import CHANNELS, RAW_LAYOUT_SINCE, MeshInput, Submesh from .texture import TextureInput @@ -37,7 +37,7 @@ class ObjectInfo: @property def id(self) -> str: - return contract.object_id(self.file, self.path_id) + return object_id(self.file, self.path_id) class BundleView(Protocol): @@ -308,7 +308,7 @@ def mesh_input(self, obj: ObjectInfo) -> MeshInput: def open_bundle(data: bytes) -> UnityBundle: """The BundleView of a bundle's bytes (decrypted UnityFS), loaded alone.""" - from ...unity.reader import load_bytes + from ..reader import load_bytes return UnityBundle(load_bytes(bytes(data))) diff --git a/src/nnnotes/pipeline/atoms/sniff.py b/src/nnnotes/unity/atoms/sniff.py similarity index 99% rename from src/nnnotes/pipeline/atoms/sniff.py rename to src/nnnotes/unity/atoms/sniff.py index 79289f4..f43f7d6 100644 --- a/src/nnnotes/pipeline/atoms/sniff.py +++ b/src/nnnotes/unity/atoms/sniff.py @@ -13,7 +13,7 @@ import re from dataclasses import dataclass -from .. import contract +from ...core.media import media_type from . import estimate @@ -99,7 +99,7 @@ class Sniffed: def _media(ext: str, fallback: str = "application/octet-stream") -> str: - m = contract.media_type(ext) + m = media_type(ext) return fallback if m == "application/octet-stream" else m diff --git a/src/nnnotes/pipeline/atoms/sprite.py b/src/nnnotes/unity/atoms/sprite.py similarity index 100% rename from src/nnnotes/pipeline/atoms/sprite.py rename to src/nnnotes/unity/atoms/sprite.py diff --git a/src/nnnotes/pipeline/atoms/texture.py b/src/nnnotes/unity/atoms/texture.py similarity index 100% rename from src/nnnotes/pipeline/atoms/texture.py rename to src/nnnotes/unity/atoms/texture.py diff --git a/src/nnnotes/unity/ids.py b/src/nnnotes/unity/ids.py new file mode 100644 index 0000000..ad9c3e5 --- /dev/null +++ b/src/nnnotes/unity/ids.py @@ -0,0 +1,17 @@ +"""Object ids: an object of a serialized file is `:`.""" + +from __future__ import annotations + +__all__ = ["object_id", "parse_object_id"] + + +def object_id(file: str, path_id: int) -> str: + return f"{file}:{int(path_id)}" + + +def parse_object_id(oid: str) -> tuple[str, int]: + """(serialized file, pathId) of an object id.""" + file, sep, pid = oid.rpartition(":") + if not sep or not file: + raise ValueError(f"not an object id: {oid!r}") + return file, int(pid) diff --git a/tests/pipeline/test_atom_eval.py b/tests/pipeline/test_atom_eval.py index 7537316..3e075f4 100644 --- a/tests/pipeline/test_atom_eval.py +++ b/tests/pipeline/test_atom_eval.py @@ -17,9 +17,9 @@ import atom_eval # noqa: E402 -from nnnotes.pipeline.atoms import Impl, resolve # noqa: E402 -from nnnotes.pipeline.atoms.mesh import Channel, MeshArrays, Primitive # noqa: E402 -from nnnotes.pipeline.atoms.reader import UnityBundle # noqa: E402 +from nnnotes.unity.atoms import Impl, resolve # noqa: E402 +from nnnotes.unity.atoms.mesh import Channel, MeshArrays, Primitive # noqa: E402 +from nnnotes.unity.atoms.reader import UnityBundle # noqa: E402 from tests.helpers.fakeunity import FakeEnvironment, FakeTexture2D, rgba_bytes # noqa: E402 RGBA32, RGB24 = 4, 3 diff --git a/tests/pipeline/test_cristages.py b/tests/pipeline/test_cristages.py index 6881be6..4f2c418 100644 --- a/tests/pipeline/test_cristages.py +++ b/tests/pipeline/test_cristages.py @@ -631,7 +631,7 @@ def run(self, task, depot): class GoldenMovie(MovieStage): @property def ATOMS(self) -> dict: - from nnnotes.pipeline.atoms import impl_id + from nnnotes.unity.atoms import impl_id return { "usm.demux": impl_id(("numpy",), cristages.USM_DEMUX_REVISION), diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index d0019dc..264ac09 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -16,12 +16,12 @@ from nnnotes.pipeline import census as census_mod from nnnotes.pipeline import contract, objexport -from nnnotes.pipeline.atoms import Impl, astc, estimate, gltf -from nnnotes.pipeline.atoms.reader import UnityBundle from nnnotes.pipeline.contract import IncompatibleTask, Input, Task from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.objexport import CONVERTERS, CROP, EXPORT, ExportStage, SpriteCropStage, export_atoms from nnnotes.pipeline.stages import Env, describe, execute, registry +from nnnotes.unity.atoms import Impl, astc, estimate, gltf +from nnnotes.unity.atoms.reader import UnityBundle from tests.helpers.fakeunity import ( F32, NORMAL, @@ -734,10 +734,10 @@ def test_textures(exported): def test_sprites_same_bundle(exported): depot, _task, doc = exported - from nnnotes.pipeline.atoms import png, sprite - from nnnotes.pipeline.atoms.mesh import arrays - from nnnotes.pipeline.atoms.reader import mesh_input_from_typetree - from nnnotes.pipeline.atoms.texture import TextureInput, decode + from nnnotes.unity.atoms import png, sprite + from nnnotes.unity.atoms.mesh import arrays + from nnnotes.unity.atoms.reader import mesh_input_from_typetree + from nnnotes.unity.atoms.texture import TextureInput, decode env = bundle_all() f = env.assets[0] diff --git a/tests/pipeline/test_atoms_mesh.py b/tests/unity/test_atoms_mesh.py similarity index 98% rename from tests/pipeline/test_atoms_mesh.py rename to tests/unity/test_atoms_mesh.py index 2e89bd2..f00fbf4 100644 --- a/tests/pipeline/test_atoms_mesh.py +++ b/tests/unity/test_atoms_mesh.py @@ -8,10 +8,10 @@ import pytest from UnityPy.helpers.MeshHelper import MeshHandler -from nnnotes.pipeline.atoms import ATOMS, Unsupported, gltf -from nnnotes.pipeline.atoms.mesh import MeshInput, arrays -from nnnotes.pipeline.atoms.reader import UnityBundle, handler_input, mesh_input_from_typetree from nnnotes.unity import reader as unity +from nnnotes.unity.atoms import ATOMS, Unsupported, gltf +from nnnotes.unity.atoms.mesh import MeshInput, arrays +from nnnotes.unity.atoms.reader import UnityBundle, handler_input, mesh_input_from_typetree from tests.helpers.fakeunity import ( BLENDINDICES, BLENDWEIGHT, diff --git a/tests/pipeline/test_atoms_sprite.py b/tests/unity/test_atoms_sprite.py similarity index 97% rename from tests/pipeline/test_atoms_sprite.py rename to tests/unity/test_atoms_sprite.py index 7adc803..e06f8e9 100644 --- a/tests/pipeline/test_atoms_sprite.py +++ b/tests/unity/test_atoms_sprite.py @@ -7,10 +7,10 @@ import pytest from UnityPy.export.SpriteHelper import get_image_from_sprite -from nnnotes.pipeline.atoms import ATOMS, Unsupported, sprite -from nnnotes.pipeline.atoms.mesh import arrays -from nnnotes.pipeline.atoms.reader import mesh_input_from_typetree -from nnnotes.pipeline.atoms.texture import TextureInput, decode +from nnnotes.unity.atoms import ATOMS, Unsupported, sprite +from nnnotes.unity.atoms.mesh import arrays +from nnnotes.unity.atoms.reader import mesh_input_from_typetree +from nnnotes.unity.atoms.texture import TextureInput, decode from tests.helpers.fakeunity import ( ANDROID, VERSION, diff --git a/tests/pipeline/test_atoms_texture.py b/tests/unity/test_atoms_texture.py similarity index 94% rename from tests/pipeline/test_atoms_texture.py rename to tests/unity/test_atoms_texture.py index 6d4f6dc..a87d90b 100644 --- a/tests/pipeline/test_atoms_texture.py +++ b/tests/unity/test_atoms_texture.py @@ -13,10 +13,10 @@ from PIL import Image from UnityPy.export.Texture2DConverter import parse_image_data -from nnnotes.pipeline.atoms import ATOMS, Impl, Unsupported, astc, env_name, ids, impl_id, png, resolve -from nnnotes.pipeline.atoms.reader import ObjectInfo, UnityBundle -from nnnotes.pipeline.atoms.texture import ASTC_BLOCKS, TextureInput, decode, is_hdr_astc from nnnotes.unity import export +from nnnotes.unity.atoms import ATOMS, Impl, Unsupported, astc, env_name, ids, impl_id, png, resolve +from nnnotes.unity.atoms.reader import ObjectInfo, UnityBundle +from nnnotes.unity.atoms.texture import ASTC_BLOCKS, TextureInput, decode, is_hdr_astc from tests.helpers.fakeunity import ANDROID, VERSION, FakeEnvironment, FakeTexture2D, rgba_bytes RGBA32, RGB24, ALPHA8, ARGB4444, YUY2 = 4, 3, 1, 2, 21 @@ -255,7 +255,7 @@ def test_every_atom_has_one_reference_with_versioned_id(): def test_evaluation_override(monkeypatch): assert env_name("png.encode") == "NNNOTES_ATOM_PNG_ENCODE" - monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "tests.pipeline.test_atoms_texture:REPLACEMENT") + monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "tests.unity.test_atoms_texture:REPLACEMENT") assert resolve("png.encode") is REPLACEMENT and ATOMS["png.encode"].fn(None) == b"replaced" assert ids("png.encode") == {"png.encode": "pillow-test/9"} # keys change with the override monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "no-colon") @@ -268,17 +268,17 @@ def test_evaluation_override(monkeypatch): def test_override_with_the_reference_id_is_refused(monkeypatch): - import tests.pipeline.test_atoms_texture as me + import tests.unity.test_atoms_texture as me monkeypatch.setattr(me, "SAME", Impl(ATOMS["png.encode"].id, png.encode, png.cost), raising=False) - monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "tests.pipeline.test_atoms_texture:SAME") + monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "tests.unity.test_atoms_texture:SAME") with pytest.raises(ValueError, match="reference's id"): resolve("png.encode") def test_listing_the_registry_loads_no_image_or_unity_library(): code = ( - "import sys, nnnotes.pipeline.atoms as a; a.ids(*a.ATOMS); " + "import sys, nnnotes.unity.atoms as a; a.ids(*a.ATOMS); " "print(sorted(m for m in ('UnityPy', 'PIL') if m in sys.modules))" ) out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True).stdout @@ -296,7 +296,7 @@ def test_every_atom_and_reason_code_is_documented(): assert documented == set(ATOMS) codes = set(re.findall(r"^\| `([a-zA-Z_.]+)` \|", doc.split("## Reason codes", 1)[1], re.M)) emitted = set() - for src in (root / "src" / "nnnotes" / "pipeline" / "atoms").glob("*.py"): + for src in (root / "src" / "nnnotes" / "unity" / "atoms").glob("*.py"): emitted |= set( re.findall( r"\"((?:empty|unsupported|partial|generic|no_data)\.[a-zA-Z_.]+)\"", src.read_text(encoding="utf-8") diff --git a/tests/pipeline/test_sniff.py b/tests/unity/test_sniff.py similarity index 97% rename from tests/pipeline/test_sniff.py rename to tests/unity/test_sniff.py index bc3df9b..1c9d70b 100644 --- a/tests/pipeline/test_sniff.py +++ b/tests/unity/test_sniff.py @@ -9,8 +9,8 @@ import pytest from nnnotes.pipeline import contract -from nnnotes.pipeline.atoms import ATOMS -from nnnotes.pipeline.atoms.sniff import MAGIC, Sniffed, sniff, spine_binary_version +from nnnotes.unity.atoms import ATOMS +from nnnotes.unity.atoms.sniff import MAGIC, Sniffed, sniff, spine_binary_version def sfnt(tag: bytes, tables: int = 3) -> bytes: From 9bca3e4c54e4ed179ae5e20543797b96b59d3ee8 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 00:47:54 +0800 Subject: [PATCH 07/91] refactor!: convert Unity objects with one converter library nnnotes.unity.convert holds the converters, one per Unity class: the bundle's object view (names, typetrees, scripts, the object JSON of a typetree) in objects.py and a module per group of classes. Artifacts go out through an emit function the caller gives, so the library knows nothing of tasks or the depot. unity.export runs the converters over one bundle and keeps the items, the fallbacks and the provenance. The typetree exporter takes its material and controller documents and the streamed clip decoding from the library. canon/1 and its hashes are nnnotes.core.canon; nnnotes.pipeline.contract keeps exporting them. clip.json/2 writes a clip's bindings as stored: pathCrc, attributeCrc, typeID with its class, customType, the pptr, int and serializeReference flags, and for a binding that names a script its reference and its script table entry. It resolves the attributes of Transform bindings, which need no hierarchy, and leaves paths and other attributes to the hierarchy that plays the clip. Events keep every field. The clip's name is `name`. controller.json/2 drops the `controller` field, which repeated `name`. BREAKING CHANGE: clip.json and controller.json documents have the forms above; the converter versions are clip.json/2 and controller.json/2. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 4 +- docs/contracts.md | 2 + docs/stages.md | 23 +- src/nnnotes/content/storyhost.py | 3 +- src/nnnotes/core/canon.py | 130 +++ src/nnnotes/pipeline/contract.py | 117 +- src/nnnotes/pipeline/objexport.py | 1412 +++--------------------- src/nnnotes/unity/convert/__init__.py | 144 +++ src/nnnotes/unity/convert/animation.py | 262 +++++ src/nnnotes/unity/convert/files.py | 46 + src/nnnotes/unity/convert/generic.py | 30 + src/nnnotes/unity/convert/material.py | 63 ++ src/nnnotes/unity/convert/mono.py | 111 ++ src/nnnotes/unity/convert/objects.py | 409 +++++++ src/nnnotes/unity/convert/prefab.py | 193 ++++ src/nnnotes/unity/convert/textures.py | 263 +++++ src/nnnotes/unity/export.py | 169 +-- tests/golden/unity.export.json | 64 +- tests/pipeline/test_objexport.py | 100 +- 19 files changed, 1939 insertions(+), 1606 deletions(-) create mode 100644 src/nnnotes/core/canon.py create mode 100644 src/nnnotes/unity/convert/__init__.py create mode 100644 src/nnnotes/unity/convert/animation.py create mode 100644 src/nnnotes/unity/convert/files.py create mode 100644 src/nnnotes/unity/convert/generic.py create mode 100644 src/nnnotes/unity/convert/material.py create mode 100644 src/nnnotes/unity/convert/mono.py create mode 100644 src/nnnotes/unity/convert/objects.py create mode 100644 src/nnnotes/unity/convert/prefab.py create mode 100644 src/nnnotes/unity/convert/textures.py diff --git a/docs/architecture.md b/docs/architecture.md index 1aeb4a7..cd89197 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -12,10 +12,10 @@ subpackage of `nnnotes` and imports only the layers below it. | Products | `nnnotes.site`, `nnnotes.musicdata` | the ournotes-player site (charts, models, stories, indexes); the music data file and its deck statistics | | Pipeline | `nnnotes.pipeline` | stages, task descriptions and results, the content-addressed depot, plans, layouts, the catalog versions, the views of the master data | | Content | `nnnotes.content` | the game's subjects as exports: ADV episodes and stories, lives and scores, spots, rooms, the UI library | -| Unity objects | `nnnotes.unity` | bundle reading, object ids, the atoms (the pure functions objects are converted with: texture decoding, sprite images, mesh arrays, glTF, PNG, file sniffing), the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | +| Unity objects | `nnnotes.unity` | bundle reading, object ids, the atoms (the pure functions objects are converted with: texture decoding, sprite images, mesh arrays, glTF, PNG, file sniffing), the converters (one per Unity class, `nnnotes.unity.convert`), the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | | File formats | `nnnotes.formats` | formats read from bytes: CRI ACB/AWB and HCA, USM movies, CRI Lips data, shaders, motions | | Sources | `nnnotes.source` | the configuration, the APK, Addressables catalogs and bundles, the game API, master data, languages, `Session` | -| Core | `nnnotes.core` | the one JSON writer, media types, the process caches and the downloads cache | +| Core | `nnnotes.core` | the one JSON writer, canonical JSON and its hashes, media types, the process caches and the downloads cache | The order is a contract: the `layers` contract of [import-linter](https://import-linter.readthedocs.io/) in `pyproject.toml` fails the build when a module imports a layer above its own, and `nnnotes.site` and diff --git a/docs/contracts.md b/docs/contracts.md index 9a9cb65..affcad8 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -17,6 +17,8 @@ stays). ## canon/1 +The canonical JSON (`nnnotes.core.canon`) of every document nnnotes writes: + - UTF-8, LF line endings, one trailing newline, non-ASCII characters as they are. - Contract documents (everything on this page): object keys sorted, indent 1. - Object documents (the JSON of one Unity object): the object's own field order, indent 1. diff --git a/docs/stages.md b/docs/stages.md index fadf601..bcd1f01 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -38,7 +38,9 @@ gzip input and Japanese remote asset placeholders). ## Converters (`unity.export`) -`unity.export` reads one bundle alone and gives every object of it exactly one item status: `exported`, +The converters are the library `nnnotes.unity.convert`, one module per group of classes; `unity.export` +(`nnnotes.pipeline.objexport`) runs them over one bundle read alone and gives every object of it exactly one item +status: `exported`, `contained` (part of another artifact), `generic` (typetree JSON because the specialized form could not be made; the reason says why), `unsupported` (a class the stage does not convert; its typetree JSON is written as well) or `failed` (not even the typetree JSON could be made). A converter's documented limit gives the generic typetree JSON @@ -52,8 +54,8 @@ with its reason code; any other error of a converter gives it with `generic.erro | GameObject (a root) | `prefab.json/1` | `prefab`: the hierarchy's nodes (path, path ids, transform) with their components' fields | | the objects below it are `contained`; a root of a scene's file names its scene | | Material | `material.json/1` | `json`: shader, keywords, textures, floats, colours | | | | Shader | `shader.json/1` | `json`: the parsed form (the bytes of `nnnotes export shader`'s `.json`), `program:/sp

__`: each compiled sub-program as stored (`.glsl`, `.metal`, `.vkprog`), `index`: the variants (platform, subshader, pass, stage, GPU program type, keywords, program role), `typetree`: the typetree JSON | | a shader without a parsed form: its typetree with `generic.error` | -| AnimationClip | `clip.json/1` | `json`: the Mecanim clip data with its bindings resolved in the bundle | | a legacy or compressed clip: `generic.clip.legacy` | -| AnimatorController | `controller.json/1` | `json`: layers, state machines, parameters | | an AnimatorOverrideController: `generic.controller.override` | +| AnimationClip | `clip.json/2` | `json`: the Mecanim clip data and its bindings as stored (see below) | | a legacy or compressed clip: `generic.clip.legacy` | +| AnimatorController | `controller.json/2` | `json`: name, clips (`$ref`), layers, state machines, parameters, default values | | an AnimatorOverrideController: `generic.controller.override` | | MonoBehaviour (not in a hierarchy) | `mono.json/2`, `cri.usm/1` | `json`: `$script` (assembly, namespace, class) and the typetree; a cue sheet asset also `acb`, a movie asset holding its USM also `usm` (see below) | `sniff` | the class from the task's script table | | TextAsset | `data/1` | `data`: the stored bytes | `sniff` | extension from the container path, else from the content | | Font | `font/1` | `font`: the font file | `sniff` | | @@ -77,6 +79,21 @@ Object JSON is canon/1 in the object's own field order: - a string whose stored bytes are not UTF-8 is `{"$hex": ""}`; - NaN is `{"$float": "nan"}`, infinities `1e999` / `-1e999`. +An AnimationClip's `json` holds `name`, `sampleRate`, `wrapMode`, `startTime`, `stopTime`, `loopTime`, +`cycleOffset`, `events` (with `objectReferenceParameter` as a reference), `bindings`, `streamed` (`curveCount`, and +`discreteCurveCount` when the clip stores it; `frames`: `[time, [[curve, c0, c1, c2, c3], ...]]`, a curve's value at +`t` being `((c0*x + c1)*x + c2)*x + c3` with `x = t - time` from its last key at or before `t`), `dense` +(`curveCount`, `frameCount`, `sampleRate`, `beginTime`, `samples`), `constant` and `pptrCurveMapping` (references). +Curve indices run over the streamed curves, then the dense ones, then the constant ones; each float binding takes +`curves` of them (1 when the binding has no `curves`) in `bindings` order. A binding is the clip's GenericBinding as +stored: `pathCrc` (crc32 of the animated object's path relative to the GameObject whose Animator plays the clip), +`attributeCrc`, `typeID` with its `class` when it is a known class id, `customType`, `pptr`, `int`, +`serializeReference`; a binding that names a script has `script` (the MonoScript reference) and `scriptClass` +(its entry in the task's script table, `null` when the table lacks it), and a Transform binding (`typeID` 4) of +position, rotation, scale or euler angles has `attribute` (`m_LocalPosition`, `m_LocalRotation`, `m_LocalScale`, +`localEulerAnglesRaw`) and `curves` (3, 4, 3, 3). Paths and the attributes of other components are named by the +hierarchy that plays the clip, which the clip alone does not give. + A cue sheet held in a bundle is a MonoBehaviour in one of two layouts; either gives the artifact `#acb` (kind `cri.acb`, facts: `cueSheet`, `layout`), which `cri.audio` decodes: diff --git a/src/nnnotes/content/storyhost.py b/src/nnnotes/content/storyhost.py index dc6cd33..9c51813 100644 --- a/src/nnnotes/content/storyhost.py +++ b/src/nnnotes/content/storyhost.py @@ -39,7 +39,8 @@ from ..formats import shader as shader_mod from ..source import languages, master from ..unity import fontsettings, textstyle, tmpfont -from ..unity.export import RECT_PROPS, Exporter, TexelPacker, streamed_frames +from ..unity.convert.animation import streamed_frames +from ..unity.export import RECT_PROPS, Exporter, TexelPacker from ..unity.reader import SceneGraph, deref, mesh_arrays, referent, script_class from . import advui, room, spot, storyfonts diff --git a/src/nnnotes/core/canon.py b/src/nnnotes/core/canon.py new file mode 100644 index 0000000..96b1266 --- /dev/null +++ b/src/nnnotes/core/canon.py @@ -0,0 +1,130 @@ +"""canon/1, the canonical JSON of nnnotes' documents, and the hashes taken of JSON values and bytes. + +canon/1 UTF-8, LF, one trailing newline, non-ASCII as is, indent 1. Contract documents: keys sorted. + Object documents: the object's own field order. Non-finite floats as `1e999` / `-1e999` + (jsonio), NaN as {"$float": "nan"}. +content id sha256 of the bytes (hex). +key hash sha256 of the compact canonical text (ASCII only, keys sorted) of a JSON value. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Iterator +from typing import Any, cast + +from . import jsonio + +__all__ = [ + "CANON", + "NAN", + "digest", + "dumps", + "encode", + "encode_chunks", + "is_sha256", + "key_text", + "loads", + "sha256", +] + +CANON = "canon/1" +NAN = {"$float": "nan"} +_SHA256 = re.compile(r"[0-9a-f]{64}") + + +def _tag_nan(v: Any) -> Any: + """`v` with every NaN replaced by {"$float": "nan"}; containers are copied only where something changed.""" + if isinstance(v, float): + return dict(NAN) if v != v else v + if isinstance(v, dict): + d = cast(dict[Any, Any], v) + out: dict[Any, Any] | None = None + for k, x in d.items(): + y = _tag_nan(x) + if y is not x: + if out is None: + out = dict(d) + out[k] = y + return d if out is None else out + if isinstance(v, (list, tuple)): + s = cast(list[Any] | tuple[Any, ...], v) + items: list[Any] | None = None + for i, x in enumerate(s): + y = _tag_nan(x) + if y is not x: + if items is None: + items = list(s) + items[i] = y + return s if items is None else items + return v + + +def _text(obj: Any, **kw: Any) -> str: + try: + return jsonio.dumps(obj, **kw) + except ValueError: + tagged = _tag_nan(obj) + if tagged is obj: + raise # not a NaN: the writer's own error + return jsonio.dumps(tagged, **kw) + + +def dumps(obj: Any, *, sort_keys: bool = True) -> str: + """The canon/1 text of a document: keys sorted (`sort_keys=False` keeps the document's order, for object + documents), indent 1, non-ASCII as is, one trailing newline.""" + return _text(obj, indent=1, ensure_ascii=False, sort_keys=sort_keys) + "\n" + + +def encode(obj: Any, *, sort_keys: bool = True) -> bytes: + """dumps() as UTF-8 bytes.""" + return dumps(obj, sort_keys=sort_keys).encode("utf-8") + + +def encode_chunks(obj: Any, size: int = 1 << 20) -> Iterator[bytes]: + """encode(obj) as byte strings of about `size` bytes, made as they are written (a large document's text is + never whole in memory, nor the encoder's list of its pieces). Raises ValueError, possibly after some chunks, + for a document with a non-finite number: encode() writes those.""" + enc = json.JSONEncoder(indent=1, ensure_ascii=False, sort_keys=True, allow_nan=False) + buf: list[str] = [] + n = 0 + for piece in enc.iterencode(obj): + buf.append(piece) + n += len(piece) + if n >= size: + yield "".join(buf).encode("utf-8") + buf, n = [], 0 + buf.append("\n") + yield "".join(buf).encode("utf-8") + + +def _untag(d: dict[str, Any]) -> Any: + return float("nan") if d == NAN else d + + +def loads(text: str | bytes | bytearray) -> Any: + """A canon/1 (or any JSON) document: {"$float": "nan"} reads back as NaN, 1e999 as infinity.""" + if isinstance(text, (bytes, bytearray)): + text = bytes(text).decode("utf-8") + return json.loads(text, object_hook=_untag) + + +def key_text(value: Any) -> str: + """The compact canonical text a key hash is taken of: keys sorted, ASCII only, no whitespace.""" + return _text(value, separators=(",", ":"), ensure_ascii=True, sort_keys=True) + + +def digest(value: Any) -> str: + """The key hash of a JSON value (sha256 hex of key_text).""" + return hashlib.sha256(key_text(value).encode("ascii")).hexdigest() + + +def sha256(data: bytes | bytearray | memoryview) -> str: + """The content id of bytes.""" + return hashlib.sha256(data).hexdigest() + + +def is_sha256(s: object) -> bool: + return isinstance(s, str) and _SHA256.fullmatch(s) is not None diff --git a/src/nnnotes/pipeline/contract.py b/src/nnnotes/pipeline/contract.py index 68d6f26..69eb785 100644 --- a/src/nnnotes/pipeline/contract.py +++ b/src/nnnotes/pipeline/contract.py @@ -2,12 +2,8 @@ Everything here is a public format (docs/contracts.md, JSON Schemas in nnnotes.schemas): - canon/1 UTF-8, LF, one trailing newline, non-ASCII as is. Contract documents: keys sorted, indent 1. - Object documents (typetree JSON): the object's own field order, indent 1. Non-finite floats as - `1e999` / `-1e999` (jsonio), NaN as {"$float": "nan"}. - content id sha256 of the bytes (hex). - key hash sha256 of the compact canonical text (ASCII only, keys sorted) of a JSON value. - object id ":" (CAB-2f1e...:-4153...; built-in files keep their name: + canon/1 the canonical JSON of every document (nnnotes.core.canon, with the content id and the key hash) + object id ":" (CAB-2f1e...:-4153...; built-in files keep their name: "unity default resources:10"). A bundle's serialized file name changes with its content, so an object id names one content version; stable_address names the object across versions. artifact id "#" for an object's artifact, ":#" for any other. @@ -28,19 +24,25 @@ from __future__ import annotations -import hashlib -import json import re from dataclasses import dataclass, field from functools import cached_property -from ..core import jsonio +from ..core.canon import CANON as CANON +from ..core.canon import NAN as NAN +from ..core.canon import digest as digest +from ..core.canon import dumps as dumps +from ..core.canon import encode as encode +from ..core.canon import encode_chunks as encode_chunks +from ..core.canon import is_sha256 as is_sha256 +from ..core.canon import key_text as key_text +from ..core.canon import loads as loads +from ..core.canon import sha256 as sha256 from ..core.media import MEDIA_TYPES as MEDIA_TYPES from ..core.media import media_type as media_type from ..unity.ids import object_id as object_id from ..unity.ids import parse_object_id as parse_object_id -CANON = "canon/1" TASK = "nnnotes.task/1" ARTIFACT = "nnnotes.artifact/1" RESULT = "nnnotes.result/1" @@ -54,7 +56,6 @@ CENSUS = "nnnotes.census/1" VIEW = "nnnotes.view/1" -NAN = {"$float": "nan"} # item statuses of a result: every object a task covers has exactly one ITEM_STATUSES = ("exported", "contained", "generic", "unsupported", "failed") RESULT_STATUSES = ("ok", "partial") @@ -63,104 +64,10 @@ LOCATOR_KINDS = ("depot", "download", "file", "catalog") # an object's primary artifact is the one whose role comes first here; its other artifacts are secondary PRIMARY_ROLES = ("image", "mesh", "audio", "video", "font", "data", "prefab", "json") -_SHA256 = re.compile(r"[0-9a-f]{64}") _EXT = re.compile(r"[0-9a-z]+") _BUNDLE_SUFFIX = re.compile(r"(_[0-9a-f]{32})?\.bundle$") -# ---------------------------------------------------------------- canonical JSON -def _tag_nan(v): - """`v` with every NaN replaced by {"$float": "nan"}; containers are copied only where something changed.""" - if isinstance(v, float): - return dict(NAN) if v != v else v - if isinstance(v, dict): - out = None - for k, x in v.items(): - y = _tag_nan(x) - if y is not x: - if out is None: - out = dict(v) - out[k] = y - return v if out is None else out - if isinstance(v, (list, tuple)): - out = None - for i, x in enumerate(v): - y = _tag_nan(x) - if y is not x: - if out is None: - out = list(v) - out[i] = y - return v if out is None else out - return v - - -def _text(obj, **kw) -> str: - try: - return jsonio.dumps(obj, **kw) - except ValueError: - tagged = _tag_nan(obj) - if tagged is obj: - raise # not a NaN: the writer's own error - return jsonio.dumps(tagged, **kw) - - -def dumps(obj, *, sort_keys: bool = True) -> str: - """The canon/1 text of a document: keys sorted (`sort_keys=False` keeps the document's order, for object - documents), indent 1, non-ASCII as is, one trailing newline.""" - return _text(obj, indent=1, ensure_ascii=False, sort_keys=sort_keys) + "\n" - - -def encode(obj, *, sort_keys: bool = True) -> bytes: - """dumps() as UTF-8 bytes.""" - return dumps(obj, sort_keys=sort_keys).encode("utf-8") - - -def encode_chunks(obj, size: int = 1 << 20): - """encode(obj) as byte strings of about `size` bytes, made as they are written (a large document's text is - never whole in memory, nor the encoder's list of its pieces). Raises ValueError, possibly after some chunks, - for a document with a non-finite number: encode() writes those.""" - enc = json.JSONEncoder(indent=1, ensure_ascii=False, sort_keys=True, allow_nan=False) - buf, n = [], 0 - for piece in enc.iterencode(obj): - buf.append(piece) - n += len(piece) - if n >= size: - yield "".join(buf).encode("utf-8") - buf, n = [], 0 - buf.append("\n") - yield "".join(buf).encode("utf-8") - - -def _untag(d: dict): - return float("nan") if d == NAN else d - - -def loads(text): - """A canon/1 (or any JSON) document: {"$float": "nan"} reads back as NaN, 1e999 as infinity.""" - if isinstance(text, (bytes, bytearray)): - text = bytes(text).decode("utf-8") - return json.loads(text, object_hook=_untag) - - -def key_text(value) -> str: - """The compact canonical text a key hash is taken of: keys sorted, ASCII only, no whitespace.""" - return _text(value, separators=(",", ":"), ensure_ascii=True, sort_keys=True) - - -def digest(value) -> str: - """The key hash of a JSON value (sha256 hex of key_text).""" - return hashlib.sha256(key_text(value).encode("ascii")).hexdigest() - - -def sha256(data) -> str: - """The content id of bytes.""" - return hashlib.sha256(data).hexdigest() - - -def is_sha256(s) -> bool: - return isinstance(s, str) and _SHA256.fullmatch(s) is not None - - # ---------------------------------------------------------------- identities def artifact_id(owner: str, role: str) -> str: """The id of the artifact `role` of `owner` (an object id, or a task id for artifacts of no object).""" diff --git a/src/nnnotes/pipeline/objexport.py b/src/nnnotes/pipeline/objexport.py index c82db05..f17904d 100644 --- a/src/nnnotes/pipeline/objexport.py +++ b/src/nnnotes/pipeline/objexport.py @@ -1,56 +1,18 @@ """Stages `unity.export` (one bundle -> an artifact per object) and `sprite.crop` (the images of the sprites whose atlas is in another bundle). -unity.export reads one bundle alone (atom `reader`) and gives every object exactly one item (contract.item): -`exported` (its artifacts), `contained` (part of another artifact: the objects of a hierarchy in its root's prefab, -the MonoScripts in `scripts.json`, the AssetBundle objects in `bundle.json`), `generic` (the object's typetree JSON, -with the reason its converter gave for not making the specialized form), `unsupported` (a class this stage does not -convert; its typetree JSON is written all the same) or `failed` (not even the typetree JSON could be made). The -converters (CONVERTERS; each has its own version, and a bundle's task key includes only the converters of the classes -its census lists): - - Texture2D tex.png `image`: PNG of the first mip level (HDR ASTC: the .astc file) - Sprite sprite.png `image`: PNG on the sprite's rect; `meta`: JSON - Mesh mesh.glb `mesh`: binary glTF in the mesh's local space - GameObject (a root) prefab.json `prefab`: the hierarchy below it with its components - Material material.json `json` - Shader shader.json `json`: the parsed form as `nnnotes export shader` writes it; - `program:/sp

__`: each compiled sub-program as - stored; `index`: the - variants (platform, pass, stage, GPU program type, keywords); `typetree` - AnimationClip clip.json `json` - AnimatorController controller.json `json` - MonoBehaviour mono.json `json` (a MonoBehaviour that is not part of a hierarchy); a cue sheet asset - also `acb`: its ACB (kind cri.acb, the cri.audio stage decodes it); with - converter cri.usm, a movie asset also `usm`: its USM (kind cri.usm, the - cri.movie stage converts it) - TextAsset data `data`: the stored bytes - Font font `font`: the font file - MonoScript scripts.json "#scripts.json": every MonoScript of the bundle - AssetBundle bundle.json "#bundle.json": serialized files, externals, AssetBundle objects - any other class generic.json `json`: the object's typetree - -Object JSON (canon/1 in the object's own field order): -- a reference is {"$ref": {"file", "pathId"}}, with "class" and "name" when the target is in the bundle; a null - reference is null. Inside a prefab, a GameObject, Transform or component of the same prefab is named by its path: - {"gameObject": path}, {"transform": path}, {"component": class, "gameObject": path}; -- a byte array (vector of UInt8 / SInt8 / char, TypelessData) shorter than `blobMin` bytes is {"$hex": "..."}; a - longer one is {"$blob": {"sha256", "size", "format"}} with its bytes in the artifact - "#blob:" (field names and array indices joined by "/", the format from atom `sniff`); -- a string whose stored bytes are not UTF-8 is {"$hex": ""}; -- NaN is {"$float": "nan"}, infinities 1e999 / -1e999. - -A cue sheet held in a bundle is a MonoBehaviour in one of two layouts (cri.py): a SplitAcbData (`_cueSheetName`, -`_chunks`: TextAssets of this bundle, joined and unmasked into the ACB) or a CriWare.Assets asset named like the sheet -whose `implementation` managed reference is a CriSerializedBytesAssetImpl (its `data.data` bytes are the ACB: the -field's {"$blob"} names the `acb` artifact, whatever its size). The artifact's facts name the cue sheet. -A movie asset (CriWare.Assets CriManaUsmAsset) whose `implementation` is a CriSerializedBytesAssetImpl holds its USM -the same way: with converter `cri.usm` (in the key of a bundle whose script context names that class) its -`data.data` bytes are the `usm` artifact, whatever their size, the facts naming the asset. - -A converter's documented limit (atoms.Unsupported) gives the typetree JSON with its reason code (status `generic`); -any other error of a converter gives it with `generic.error` and the error's message. The reason codes are listed in -docs/stages.md. +unity.export reads one bundle alone (atom `reader`) and runs the converters of nnnotes.unity.convert over it, giving +every object exactly one item (contract.item): `exported` (its artifacts), `contained` (part of another artifact: +the objects of a hierarchy in its root's prefab, the MonoScripts in `scripts.json`, the AssetBundle objects in +`bundle.json`), `generic` (the object's typetree JSON, with the reason its converter gave for not making the +specialized form), `unsupported` (a class this stage does not convert; its typetree JSON is written all the same) or +`failed` (not even the typetree JSON could be made). A converter's documented limit (atoms.Unsupported) gives the +typetree JSON with its reason code; any other error of a converter gives it with `generic.error` and the error's +message. The reason codes are listed in docs/stages.md. A bundle's task key includes only the converters of the +classes its census lists. + +A texture is converted before the sprites cut from it (they read its decoded image) and a hierarchy before the +MonoBehaviours outside it; the objects no converter took get their typetree JSON last. sprite.crop: a sprite whose SpriteAtlas is in another bundle gets its `meta` from unity.export (with the sprite's mesh and object facts); one sprite.crop task per atlas then reads the atlas' JSON, the images of its textures and the @@ -61,539 +23,74 @@ from __future__ import annotations import io -import json -import re from collections import defaultdict -from functools import cache -from pathlib import Path -from types import SimpleNamespace -from typing import TYPE_CHECKING, cast from ..unity.atoms import ATOMS as ATOM_REGISTRY from ..unity.atoms import Impl, Unsupported, atom_id, resolve +from ..unity.convert import ( + BASE_ATOMS, + CLASS_CONVERTERS, + CONVERTER_ATOMS, + CONVERTERS, + UNSUPPORTED_CLASSES, + USM_ASSET_SCRIPT, + USM_CONVERTER, + animation, + converter_id, + export_atoms, + files, + generic, + material, + mono, + prefab, + textures, +) +from ..unity.convert.objects import BundleObjects, ReadError, encode_object, label, message, ref_object_id from . import census as census_mod from . import contract, link from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage -if TYPE_CHECKING: # the exporter's base; exporter_type combines them when UnityPy is loaded - from ..unity.export import Exporter as _ExporterBase - from ..unity.reader import SceneGraph -else: - _ExporterBase = object - EXPORT, CROP, CENSUS, SCRIPTS = "unity.export", "sprite.crop", "unity.census", "link.scripts" SCRIPTS_TASK = contract.task_id(SCRIPTS, "all") - -# converter -> version -CONVERTERS = { - "tex.png": 1, - "sprite.png": 1, - "mesh.glb": 1, - "prefab.json": 1, - "material.json": 1, - "clip.json": 1, - "controller.json": 1, - "mono.json": 2, - "data": 1, - "font": 1, - "scripts.json": 1, - "bundle.json": 1, - "shader.json": 1, - "generic.json": 1, - "cri.usm": 1, -} -# class -> the converters its objects may use (every class may fall back to generic.json) -CLASS_CONVERTERS = { - "Texture2D": ("tex.png",), - "Sprite": ("sprite.png",), - "Mesh": ("mesh.glb",), - "GameObject": ("prefab.json",), - "Material": ("material.json",), - "AnimationClip": ("clip.json",), - "AnimatorController": ("controller.json",), - "AnimatorOverrideController": ("controller.json",), - "MonoBehaviour": ("mono.json",), - "TextAsset": ("data",), - "Font": ("font",), - "MonoScript": ("scripts.json",), - "AssetBundle": ("bundle.json",), - "Shader": ("shader.json",), -} -ALWAYS = ("generic.json",) -BASE_ATOMS = ("reader", "sniff") -CONVERTER_ATOMS = { - "tex.png": ("texture.decode", "png.encode", "astc.container"), - "sprite.png": ("texture.decode", "png.encode", "sprite.crop", "mesh.arrays"), - "mesh.glb": ("mesh.arrays", "gltf.write"), -} -UNSUPPORTED_CLASSES = ("AudioClip", "VideoClip", "MovieTexture") -LATE_CLASSES = ("AnimationClip", "AnimatorController", "AnimatorOverrideController") -FONT_FORMATS = ("ttf", "otf", "ttc", "woff", "woff2") -# every reason code this module gives (the atoms give more; docs/stages.md lists them all) -REASONS = ( - "generic.error", - "generic.clip.legacy", - "generic.controller.override", - "script.unresolved", - "script.missing", - "no_data.sprite", - "no_data.font", - "unsupported.sprite.external_texture", - "unsupported.cri.awb_external", - "failed.read", - "failed.export", - *(f"unsupported.class.{c}" for c in UNSUPPORTED_CLASSES), -) -ACB_KIND = "cri.acb" -SPLIT_ACB, EMBEDDED_ACB = "split", "embedded" -EMBEDDED_ACB_IMPL = "CriSerializedBytesAssetImpl" -# a movie asset holding its USM: converter cri.usm, only in the key of a bundle whose script context names the class -USM_KIND, USM_CONVERTER = "cri.usm", "cri.usm" -USM_ASSET_SCRIPT = "CriMw.CriWare.Assets.Runtime|CriWare.Assets|CriManaUsmAsset" - -_BYTE_TYPES = frozenset({"UInt8", "SInt8", "char"}) -_SCALARS = (int, float, str, bool, type(None)) -_ADDRESS = re.compile(r"0x[0-9a-fA-F]{6,}") -_EXT = re.compile(r"[0-9a-z]+") -_NO_SCRIPT = "\0none" # a MonoBehaviour whose m_Script is null ATOM_NAMES = frozenset(ATOM_REGISTRY) +# class -> (converter, function), in the order they run after the textures and sprites +_SIMPLE = ( + ("Mesh", "mesh.glb", files.mesh), + ("TextAsset", "data", files.text), + ("Font", "font", files.font), + ("Material", "material.json", material.material), + ("Shader", "shader.json", material.shader), + ("AnimationClip", "clip.json", animation.clip), + ("AnimatorController", "controller.json", animation.controller), + ("AnimatorOverrideController", "controller.json", animation.controller), +) -def converter_id(name: str) -> str: - return f"{name}/{CONVERTERS[name]}" - - -def converters_for(classes) -> list[str]: - """The converters a bundle holding objects of `classes` may use.""" - out: set[str] = set(ALWAYS) - for c in classes: - out.update(CLASS_CONVERTERS.get(c, ())) - return sorted(out) - - -def export_atoms(classes, reader_id: str | None = None) -> dict: - """The `atoms` part of a unity.export task for a bundle holding `classes`: the atoms and converters it uses.""" - convs = converters_for(classes) - names = set(BASE_ATOMS) - for c in convs: - names.update(CONVERTER_ATOMS.get(c, ())) - out = {n: (reader_id if n == "reader" and reader_id else atom_id(n)) for n in names} - out.update((c, converter_id(c)) for c in convs) - return dict(sorted(out.items())) - - -def _message(e: BaseException) -> str: - return _label(_ADDRESS.sub("0x?", f"{type(e).__name__}: {e}")[:500]) - - -def _label(s): - """A name for a record: the string, with bytes that are not UTF-8 shown as U+FFFD.""" - if not isinstance(s, str) or s.isascii(): - return s - try: - s.encode("utf-8") - return s - except UnicodeEncodeError: - return s.encode("utf-8", "surrogateescape").decode("utf-8", "replace") - - -def _stored(v): - """`v` with every string that is not UTF-8 (read with surrogateescape) as {"$hex": its stored bytes}.""" - if isinstance(v, str): - try: - v.encode("utf-8") - return v - except UnicodeEncodeError: - return {"$hex": v.encode("utf-8", "surrogateescape").hex()} - if isinstance(v, dict): - return {_label(k): _stored(x) for k, x in v.items()} - if isinstance(v, (list, tuple)): - return [_stored(x) for x in v] - return v - - -def encode_object(doc) -> bytes: - """canon/1 bytes of an object document (its own key order).""" - try: - return contract.encode(doc, sort_keys=False) - except UnicodeEncodeError: - return contract.encode(_stored(doc), sort_keys=False) - - -def _canon(v) -> str: - return json.dumps(v, sort_keys=True, ensure_ascii=True) - - -def _join(path: str, name: str) -> str: - return f"{path}/{name}" if path else name - - -def _file_name(path: str) -> str: - return path.replace("\\", "/").rsplit("/", 1)[-1] - - -def _container_ext(path: str | None) -> str | None: - if not path: - return None - last = path.rsplit("/", 1)[-1] - if "." not in last: - return None - ext = last.rsplit(".", 1)[1].lower() - return ext if _EXT.fullmatch(ext) else None - - -def acb_asset(tt: dict) -> dict | None: - """Which cue sheet layout a MonoBehaviour's typetree holds (cri.py): {"layout": "split", "cueSheet", "chunks"} - for a SplitAcbData, {"layout": "embedded", "cueSheet", "ref": index in references.RefIds, "impl"} for an asset - whose `implementation` is a CriSerializedBytesAssetImpl; None for any other. SplitAcbData first.""" - sheet = tt.get("_cueSheetName") - if isinstance(sheet, str) and isinstance(tt.get("_chunks"), list): - return {"layout": SPLIT_ACB, "cueSheet": sheet, "chunks": tt["_chunks"]} - ref = embedded_bytes(tt) - return ( - None if ref is None else {"layout": EMBEDDED_ACB, "cueSheet": tt.get("m_Name"), "ref": ref[0], "impl": ref[1]} - ) - - -def embedded_bytes(tt: dict) -> tuple[int, dict] | None: - """(index in references.RefIds, the reference) of a CriWare.Assets asset's `implementation` when it is a - CriSerializedBytesAssetImpl (bytes held in the asset), else None.""" - impl = tt.get("implementation") - if not isinstance(impl, dict) or "rid" not in impl: - return None - ids = ((tt.get("references") or {}).get("RefIds")) or [] - for i, r in enumerate(ids): - typ = r.get("type") if isinstance(r, dict) else None - if isinstance(typ, dict) and r.get("rid") == impl["rid"] and typ.get("class") == EMBEDDED_ACB_IMPL: - return i, r - return None - - -def _ref_type_node(ref: dict, assets_file): - """The type tree of a managed reference's data (UnityPy's get_ref_type_node over the file's ref types).""" - typ = ref.get("type") or {} - cls = typ.get("class") if isinstance(typ, dict) else getattr(typ, "class", None) - if not cls: - return None - ns = typ.get("ns") if isinstance(typ, dict) else typ.ns - asm = typ.get("asm") if isinstance(typ, dict) else typ.asm - for rt in getattr(assets_file, "ref_types", None) or (): - if rt.m_ClassName == cls and rt.m_NameSpace == ns and rt.m_AssemblyName == asm: - return rt.node - raise ValueError(f"referenced type {asm}|{ns}|{cls} is not among the file's reference types") - - -class _ReadError(Exception): - """An object's typetree could not be read.""" - - -class _Script: - """A MonoScript outside the bundle, known from the task's script table: what Exporter code reads of it.""" - - type = SimpleNamespace(name="MonoScript", value=115) - - def __init__(self, file: str, path_id: int, entry: str): - self.assets_file = SimpleNamespace(name=file) - self.path_id = path_id - asm, ns, cls = (entry.split("|", 2) + ["", ""])[:3] - self._ms = SimpleNamespace(m_AssemblyName=asm, m_Namespace=ns, m_ClassName=cls, m_Name=cls) - - def read(self): - return self._ms - - -# ---------------------------------------------------------------- the exporter -class RefExporter(_ExporterBase): - """unity.export on one bundle. Combined with export.Exporter (exporter_type), whose material, clip and controller - code runs here in reference mode: `ref` gives $ref values, `deref` resolves inside the bundle only (a MonoScript - of another bundle from the script table), and MonoBehaviour classes come from the task's script table.""" +# ---------------------------------------------------------------- one bundle +class BundleRun: + """unity.export on one bundle: the converters over its objects (the task's `classes` only), one item per + object, the artifacts in the depot.""" def __init__(self, view, task, depot, fns: dict): - # no catalog: the objects come from the task's bundle view - super().__init__(None, Path("."), class_of=self._script_name) # pyright: ignore[reportArgumentType] - self.view, self.task, self.depot, self.fns = view, task, depot, fns - self.blob_min = int(task.params["blobMin"]) - self.png_level = int(task.params["png"]["level"]) - self.scripts = dict(task.context.get("scripts", {})) - self.infos = view.objects() - self.info = {(i.file, i.path_id): i for i in self.infos} - self.readers = {k: view.reader(i) for k, i in self.info.items()} - self._objects = self.readers - self.files = view.files() - self.ext_files = {f: [_file_name(p) for p in view.externals(f)] for f in self.files} - self.versions = {f: view.unity_version(f) for f in self.files} - self.containers: dict[tuple, str] = {} - for path, file, pid in view.container(): - k = (file, int(pid)) - if k not in self.containers or path < self.containers[k]: - self.containers[k] = path + self.task, self.depot = task, depot + self.ctx = BundleObjects( + view, + fns, + scripts=task.context.get("scripts", {}), + blob_min=int(task.params["blobMin"]), + png_level=int(task.params["png"]["level"]), + emit=self._emit, + ) + classes = task.params.get("classes") + self.selected = None if classes is None else frozenset(classes) + self.usm_assets = USM_CONVERTER in task.atoms self.items: dict[str, dict] = {} self.artifacts: list[dict] = [] self.emitted: set[str] = set() - self.names: dict[tuple, str | None] = {} - self.script_entry: dict[tuple, str | None] = {} - self.special_memo: dict[int, tuple] = {} - self.children_memo: dict[int, tuple] = {} - self.heads: dict[int, tuple] = {} - self.atlas_entries: dict[tuple, list] = {} - self.graphs: dict[str, SceneGraph] = {} - self.member: dict[tuple, tuple] = {} - self.comp_go: dict[tuple, int] = {} - self.roots: list[tuple] = [] - self.scene_of: dict[str, str] = {} - self.go_clash = False - self.current: tuple | None = None self.staged: list[dict] = [] - self.blob_ids: list[str] = [] self.conv = "generic.json" - self.acb_field: tuple | None = None # (object key, field path, facts) of the embedded ACB being walked - self.usm_field: tuple | None = None # the same for the USM of a movie asset - self.usm_assets = USM_CONVERTER in task.atoms self.atoms_of: dict[str, dict] = {} - self.unresolved = 0 - classes = task.params.get("classes") - self.selected = None if classes is None else frozenset(classes) - self.has_clips = any(i.class_name == "AnimationClip" for i in self.infos) and self._selected("AnimationClip") - - # ---------------------------------------------------------------- identities, names, scripts - @staticmethod - def _key(o) -> tuple: - return (o.assets_file.name, int(o.path_id)) - - def _file_of(self, owner_file: str, fid: int): - if fid == 0: - return owner_file - ext = self.ext_files.get(owner_file, ()) - return ext[fid - 1] if 0 < fid <= len(ext) else None - - def _target(self, owner, pptr: dict): - """(file, pathId) a PPtr of `owner` names; None for a null reference or an unknown file index.""" - pid = int(pptr.get("m_PathID", 0) or 0) - if not pid: - return None - file = self._file_of(owner.assets_file.name, int(pptr.get("m_FileID", 0) or 0)) - return None if file is None else (file, pid) - - def _peek(self, r) -> str | None: - """An object's m_Name, read with its type tree cut after that field.""" - node = r._get_typetree_node() - hit = self.heads.get(id(node)) - if hit is None or hit[0] is not node: - names = [c.m_Name for c in node.m_Children] - cut = None - if "m_Name" in names: - from UnityPy.helpers.TypeTreeNode import TypeTreeNode - - cut = TypeTreeNode( - node.m_Level, - node.m_Type, - node.m_Name, - node.m_ByteSize, - node.m_Version, - m_MetaFlag=node.m_MetaFlag, - m_Children=node.m_Children[: names.index("m_Name") + 1], - ) - hit = self.heads[id(node)] = (node, cut) - if hit[1] is None: - return None - name = r.read_typetree(hit[1], check_read=False).get("m_Name") - return name if isinstance(name, str) else None - - def _name_of(self, key: tuple) -> str | None: - if key not in self.names: - r = self.readers.get(key) - name = None - if r is not None: - try: - name = self._peek(r) - except Exception: - name = None - self.names[key] = name - return self.names[key] - - def _tt(self, key: tuple) -> dict: - try: - tt = self.view.typetree(self.info[key]) - except Exception as e: - raise _ReadError(_message(e)) from None - name = tt.get("m_Name") - self.names[key] = name if isinstance(name, str) else None - return tt - - def _note_script(self, key: tuple, tt: dict): - s = tt.get("m_Script") - if not isinstance(s, dict) or not s.get("m_PathID"): - entry = _NO_SCRIPT - else: - file = self._file_of(key[0], int(s.get("m_FileID", 0) or 0)) - entry = None if file is None else self.scripts.get(contract.object_id(file, int(s["m_PathID"]))) - self.script_entry[key] = entry - return entry - - def _script_name(self, o) -> str | None: - """The class name of a MonoBehaviour from the task's script table (None when unknown).""" - key = self._key(o) - if key not in self.script_entry: - self._note_script(key, self._tt(key)) - e = self.script_entry[key] - return e.rsplit("|", 1)[-1] if e and e != _NO_SCRIPT else None - - # ---------------------------------------------------------------- references and values - def _dref(self, key: tuple) -> dict: - file, pid = key - d = {"file": _label(file), "pathId": pid} - r = self.readers.get(key) - if r is not None: - d["class"] = r.type.name - name = self._name_of(key) - if name is not None: - d["name"] = _label(name) - return {"$ref": d} - - def _local(self, key: tuple) -> dict: - file, pid = key - o, g = self.readers[key], self.graphs[file] - t = o.type.name - if t == "GameObject": - return {"gameObject": g.path(g.tf_of_go[pid])} - if t in ("Transform", "RectTransform"): - return {"transform": g.path(pid)} - d = {"component": t, "gameObject": g.path(g.tf_of_go[self.comp_go[key]])} - if t == "MonoBehaviour": - d["class"] = self._script_name(o) - return d - - def ref(self, owner, pptr: dict): - pid = int(pptr.get("m_PathID", 0) or 0) - if not pid: - return None - fid = int(pptr.get("m_FileID", 0) or 0) - file = self._file_of(owner.assets_file.name, fid) - if file is None: - return {"$ref": {"fileId": fid, "pathId": pid}} - key = (file, pid) - if self.current is not None and self.member.get(key) == self.current: - return self._local(key) - return self._dref(key) - - def deref(self, owner, pptr: dict): - key = self._target(owner, pptr) - if key is None: - return None - r = self.readers.get(key) - if r is not None: - return r - entry = self.scripts.get(contract.object_id(*key)) - return None if entry is None else _Script(key[0], key[1], entry) - - def graph(self, o): - return self.graphs[o.assets_file.name] - - def value(self, owner, v): - if isinstance(v, dict): - if v.keys() == {"m_FileID", "m_PathID"}: - return self.ref(owner, v) - return {k: self.value(owner, x) for k, x in v.items()} - if isinstance(v, (list, tuple)): - if all(type(x) in _SCALARS for x in v): - return list(v) - return [self.value(owner, x) for x in v] - if isinstance(v, (bytes, bytearray)): - return {"$hex": bytes(v).hex()} - return v - - def _child(self, node, name: str): - m = self.children_memo.get(id(node)) - if m is None or m[0] is not node: - m = self.children_memo[id(node)] = (node, {c.m_Name: c for c in node.m_Children}) - return m[1].get(name) - - def _special(self, node) -> bool: - """Whether values of `node` need rewriting: references, byte arrays, managed references below it.""" - hit = self.special_memo.get(id(node)) - if hit is not None and hit[0] is node: - return hit[1] - t, ch = node.m_Type, node.m_Children - if t.startswith("PPtr<") or t in ("TypelessData", "ReferencedObject", "ManagedReferencesRegistry"): - r = True - elif t == "string" or not ch: - r = False - elif ch[0].m_Type == "Array" and len(ch[0].m_Children) > 1: - sub = ch[0].m_Children[1] - r = sub.m_Type in _BYTE_TYPES or self._special(sub) - else: - r = any(self._special(c) for c in ch) - self.special_memo[id(node)] = (node, r) - return r - - def tree(self, owner, tt: dict) -> dict: - """The object JSON of `owner`'s typetree `tt` (read with owner's type tree).""" - return cast(dict, self._walk(owner, owner._get_typetree_node(), tt, "")) - - def _walk(self, owner, node, v, path: str): - if not self._special(node): - return v - t = node.m_Type - if t.startswith("PPtr<"): - return self.ref(owner, v) if isinstance(v, dict) else v - if t == "TypelessData": - return self._bytes_value(owner, v, path) - ch = node.m_Children - if t == "ReferencedObject": - out = {} - for k, x in v.items(): - c = self._child(node, k) - if c is not None and c.m_Type == "ReferencedObjectData": - rn = _ref_type_node(v, owner.assets_file) - out[k] = x if rn is None else self._walk(owner, rn, x, _join(path, k)) - else: - out[k] = x if c is None else self._walk(owner, c, x, _join(path, k)) - return out - if t != "string" and ch and ch[0].m_Type == "Array": - sub = ch[0].m_Children[1] - if sub.m_Type in _BYTE_TYPES: - return self._bytes_value(owner, v, path, sub.m_Type == "SInt8") - return [self._walk(owner, sub, x, _join(path, str(i))) for i, x in enumerate(v)] - if t == "pair": - return [self._walk(owner, ch[0], v[0], _join(path, "0")), self._walk(owner, ch[1], v[1], _join(path, "1"))] - if not isinstance(v, dict): - return v - out = {} - for k, x in v.items(): - c = self._child(node, k) - out[k] = x if c is None else self._walk(owner, c, x, _join(path, k)) - return out - - def _bytes_value(self, owner, v, path: str, signed: bool = False) -> dict: - if isinstance(v, (bytes, bytearray, memoryview)): - data = bytes(v) - elif signed: - import numpy as np - - data = np.asarray(v, np.int16).astype(np.uint8).tobytes() - else: - data = bytes(v) - if self.acb_field is not None and self.acb_field[:2] == (self._key(owner), path): - from ..formats import cri - - if data[:4] != cri.ACB_SIGNATURE: - raise ValueError(f"{path}: the cue sheet data is not an ACB (@UTF): {data[:4].hex()}") - aid, content = self._emit(self._key(owner), "acb", data, "acb", ACB_KIND, "acb", self.acb_field[2]) - self.blob_ids.append(aid) - return {"$blob": {"sha256": content["sha256"], "size": content["size"], "format": "acb"}} - if self.usm_field is not None and self.usm_field[:2] == (self._key(owner), path): - if data[:4] != b"CRID": - raise ValueError(f"{path}: the movie data is not a USM (CRID): {data[:4].hex()}") - aid, content = self._emit(self._key(owner), "usm", data, "usm", USM_KIND, "usm", self.usm_field[2]) - self.blob_ids.append(aid) - return {"$blob": {"sha256": content["sha256"], "size": content["size"], "format": "usm"}} - if len(data) < self.blob_min: - return {"$hex": data.hex()} - s = self.fns["sniff"](data) - aid, content = self._emit(self._key(owner), "blob:" + path, data, s.ext, "blob", s.format, {"field": path}) - self.blob_ids.append(aid) - return {"$blob": {"sha256": content["sha256"], "size": content["size"], "format": s.format}} # ---------------------------------------------------------------- artifacts and items def _atoms(self, conv: str) -> dict: @@ -603,20 +100,6 @@ def _atoms(self, conv: str) -> dict: a = self.atoms_of[conv] = {n: self.task.atoms[n] for n in sorted(names) if n in self.task.atoms} return a - def _facts(self, key: tuple) -> dict: - i = self.info[key] - d = {"file": _label(i.file), "pathId": i.path_id, "classId": i.class_id, "class": i.class_name} - name = self._name_of(key) - if name is not None: - d["name"] = _label(name) - if i.type_hash: - d["typeHash"] = i.type_hash - d["unityVersion"] = self.versions.get(i.file) - c = self.containers.get(key) - if c: - d["container"] = _label(c) - return d - def _emit( self, key, @@ -636,7 +119,7 @@ def _emit( if aid in self.emitted or any(a["id"] == aid for a in self.staged): raise ValueError(f"artifact {aid} made twice") content = self.depot.add(data, ext) - prov = contract.provenance(self.task, atoms=self._atoms(self.conv), obj=None if owner else self._facts(key)) + prov = contract.provenance(self.task, atoms=self._atoms(self.conv), obj=None if owner else self.ctx.facts(key)) sem = {"kind": kind, "format": fmt, "facts": facts} if issues: sem["issues"] = [{"code": c, "message": m} for c, m in issues] @@ -646,14 +129,27 @@ def _emit( return aid, content def _begin(self, conv: str) -> None: - self.staged, self.blob_ids, self.conv = [], [], conv + self.staged, self.conv = [], conv - def _commit(self, info, status: str, ids, why: dict | None = None) -> None: - for a in self.staged: - self.emitted.add(a["id"]) + def _publish(self) -> list[str]: + """Keep the staged artifacts; returns their ids.""" + ids = [a["id"] for a in self.staged] + self.emitted.update(ids) self.artifacts.extend(self.staged) self.staged = [] - self.items[info.id] = contract.item(info.id, status, artifacts=list(ids), why=why, cls=info.class_name) + return ids + + def _commit(self, info, status: str, why: dict | None = None) -> None: + """The item of `info`: the artifacts its conversion staged.""" + ids = self._publish() + self.items[info.id] = contract.item(info.id, status, artifacts=ids, why=why, cls=info.class_name) + + def _failed(self, info, e: Exception, prefix: str = "") -> None: + self._begin("generic.json") + what = "failed.read" if isinstance(e, ReadError) else "failed.export" + detail = str(e) if isinstance(e, ReadError) else message(e) + why = contract.reason(what, label(prefix + detail)) + self.items[info.id] = contract.item(info.id, "failed", why=why, cls=info.class_name) def _need(self, conv: str) -> None: if conv not in self.task.atoms: @@ -661,90 +157,62 @@ def _need(self, conv: str) -> None: f"task {self.task.id}: converter {conv} is not in the task (its census lacks the class)" ) - def _convert(self, info, conv: str, fn, *args) -> bool: + def _convert(self, info, conv: str, fn, *args, **kw) -> bool: """Run a converter on one object: its artifacts (exported), else the typetree JSON with the reason.""" if info.id in self.items: return False self._need(conv) self._begin(conv) try: - ids = fn(info, *args) + fn(self.ctx, info, *args, **kw) except Unsupported as e: self._fallback(info, e.code, e.message) return False except Exception as e: - self._fallback(info, "generic.error", _message(e)) + self._fallback(info, "generic.error", message(e)) return False - self._commit(info, "exported", [*ids, *self.blob_ids]) + self._commit(info, "exported") return True - def _fallback(self, info, code: str, message: str, status: str = "generic") -> None: + def _fallback(self, info, code: str, why: str, status: str = "generic") -> None: self._begin("generic.json") try: - aid = self.conv_generic(info)[0] + generic.generic(self.ctx, info) except Exception as e: - self._begin("generic.json") - what = "failed.read" if isinstance(e, _ReadError) else "failed.export" - detail = str(e) if isinstance(e, _ReadError) else _message(e) - self.items[info.id] = contract.item( - info.id, - "failed", - why=contract.reason(what, _label(f"{code}: {message}; typetree JSON: {detail}")), - cls=info.class_name, - ) + self._failed(info, e, f"{code}: {why}; typetree JSON: ") return - self._commit(info, status, [aid, *self.blob_ids], contract.reason(code, _label(message))) + self._commit(info, status, contract.reason(code, label(why))) def _generic(self, info) -> None: """An object of a class without a specialized converter: its typetree JSON.""" - if info.id in self.items: - return self._begin("generic.json") try: - ids = self.conv_generic(info) + generic.generic(self.ctx, info) except Exception as e: - self._begin("generic.json") - what = "failed.read" if isinstance(e, _ReadError) else "failed.export" - self.items[info.id] = contract.item( - info.id, - "failed", - why=contract.reason(what, str(e) if isinstance(e, _ReadError) else _message(e)), - cls=info.class_name, - ) + self._failed(info, e) return - self._commit(info, "exported", [*ids, *self.blob_ids]) + self._commit(info, "exported") def _selected(self, cls: str) -> bool: return self.selected is None or cls in self.selected def _of(self, cls: str) -> list: """The objects of `cls` this task converts (none when the task's `classes` leave the class out).""" - if not self._selected(cls): - return [] - return [i for i in self.infos if i.class_name == cls] + return self.ctx.of(cls) if self._selected(cls) else [] # ---------------------------------------------------------------- the bundle - def run_bundle(self) -> tuple[list, list]: - self._index() + def run(self) -> tuple[list, list]: self._textures_sprites() - for i in self._of("Mesh"): - self._convert(i, "mesh.glb", self.conv_mesh) - for i in self._of("TextAsset"): - self._convert(i, "data", self.conv_text) - for i in self._of("Font"): - self._convert(i, "font", self.conv_font) - for i in self._of("Material"): - self._convert(i, "material.json", self.conv_material) - for i in self._of("Shader"): - self._convert(i, "shader.json", self.conv_shader) + for cls, conv, fn in _SIMPLE: + for i in self._of(cls): + self._convert(i, conv, fn) self._prefabs() for i in self._of("MonoBehaviour"): - if i.id not in self.items: - self._convert(i, "mono.json", self.conv_mono) - self._loose("AssetBundle", "bundle.json") - self._loose("MonoScript", "scripts.json") - for i in self.infos: - if i.id in self.items or i.class_name in LATE_CLASSES or not self._selected(i.class_name): + self._convert(i, "mono.json", mono.mono, usm_assets=self.usm_assets) + for conv, (cls, kind) in generic.LOOSE.items(): + self._loose(cls, conv, kind) + for i in self.ctx.infos: + if i.id in self.items or not self._selected(i.class_name): continue if i.class_name in UNSUPPORTED_CLASSES: self._fallback( @@ -752,599 +220,69 @@ def run_bundle(self) -> tuple[list, list]: ) else: self._generic(i) - for i in self._of("AnimationClip"): - self._convert(i, "clip.json", self.conv_clip) - for cls in ("AnimatorController", "AnimatorOverrideController"): - for i in self._of(cls): - self._convert(i, "controller.json", self.conv_controller) - for i in self.infos: - if i.id not in self.items and self._selected(i.class_name): - self.items[i.id] = contract.item( - i.id, - "failed", - why=contract.reason("failed.export", "no converter took the object"), - cls=i.class_name, - ) return self.artifacts, list(self.items.values()) - def _index(self) -> None: - """Hierarchies per serialized file (their roots, the objects each root's prefab holds), scene files.""" - from ..unity.reader import SceneGraph - - by_file = defaultdict(list) - for i in self.infos: - by_file[i.file].append(i) - for f in self.files: - objs = [ - self.readers[(i.file, i.path_id)] - for i in by_file[f] - if i.class_name in ("GameObject", "Transform", "RectTransform") - ] - try: - g = SceneGraph(SimpleNamespace(objects=objs)) - except Exception: - continue # the file's objects are exported one by one - self.graphs[f] = g - if by_file[f]: - self._graph_of_file[id(self.readers[(f, by_file[f][0].path_id)].assets_file)] = g - for go_pid in g.go: - if self._go_file.get(go_pid, f) != f: - self.go_clash = True - self._go_file[go_pid] = f - for tf_pid, t in g.tf.items(): - father = int(t["m_Father"]["m_PathID"] or 0) - if father and father in g.tf: - continue - go_pid = int(t["m_GameObject"]["m_PathID"] or 0) - members, comps, error = self._subtree(f, g, tf_pid) - self.roots.append((f, tf_pid, go_pid, members, error)) - if error is None: - for k in members: - self.member[k] = (f, go_pid) - self.comp_go.update(comps) - for i in self._of("AssetBundle"): - try: - tt = self._tt((i.file, i.path_id)) - except _ReadError: - continue - for pair in tt.get("m_SceneHashes") or (): - scene, cab = pair[0], str(pair[1]).lower() - for f in self.files: - if cab and f.lower().startswith(cab): - self.scene_of.setdefault(f, scene) - - def _subtree(self, f: str, g, root_tf: int): - """(member keys in hierarchy order, {component key: GameObject pathId}, error) of the hierarchy at root_tf.""" - members, comps, seen, stack = [], {}, set(), [root_tf] - try: - while stack: - tf = stack.pop() - if tf in seen: - raise ValueError(f"transform {tf} appears twice in the hierarchy") - if tf not in g.tf: - raise ValueError(f"child transform {tf} is not in the file") - seen.add(tf) - t = g.tf[tf] - go_pid = int(t["m_GameObject"]["m_PathID"] or 0) - if go_pid not in g.go: - raise ValueError(f"transform {tf}: GameObject {go_pid} is not in the file") - keys = [(f, go_pid), (f, tf)] - for c in g.go[go_pid]["m_Component"]: - p = c["component"] - pid = int(p["m_PathID"] or 0) - if not pid or pid == tf: - continue - if int(p.get("m_FileID", 0) or 0) or (f, pid) not in self.readers: - raise ValueError(f"GameObject {go_pid}: component {pid} is not in the file") - keys.append((f, pid)) - comps[(f, pid)] = go_pid - for k in keys: - if k in self.member: - raise ValueError(f"object {k[1]} belongs to two hierarchies") - members.extend(keys) - stack.extend(reversed([int(ch["m_PathID"]) for ch in t["m_Children"]])) - except Exception as e: - return members, comps, _message(e) - return members, comps, None - - # ---------------------------------------------------------------- textures and sprites def _textures_sprites(self) -> None: - waiting = defaultdict(list) + waiting, atlases = defaultdict(list), {} for s in self._of("Sprite"): self._need("sprite.png") self._begin("sprite.png") try: - plan = self._sprite_plan((s.file, s.path_id)) + plan = textures.sprite_plan(self.ctx, (s.file, s.path_id), atlases) except Unsupported as e: self._fallback(s, e.code, e.message) continue except Exception as e: - self._fallback(s, "generic.error", _message(e)) + self._fallback(s, "generic.error", message(e)) continue if plan["external"]: - self._convert(s, "sprite.png", self.conv_sprite_external, plan) + self._convert(s, "sprite.png", textures.sprite_external, plan, CROP) else: waiting[plan["texture"]].append((s, plan)) export_textures = self._selected("Texture2D") - for t in self.infos: - if t.class_name != "Texture2D": - continue + for t in self.ctx.of("Texture2D"): sprites = waiting.pop((t.file, t.path_id), ()) if not sprites and not export_textures: continue holder: dict = {} if export_textures: - self._convert(t, "tex.png", self.conv_texture, holder) + self._convert(t, "tex.png", textures.texture, holder) else: - self._decode_into(t, holder) + textures.decode_into(self.ctx, t, holder) for s, plan in sprites: - self._convert(s, "sprite.png", self.conv_sprite, plan, holder) + self._convert(s, "sprite.png", textures.sprite, plan, holder) holder.clear() for tkey, rest in waiting.items(): for s, _plan in rest: self._fallback(s, "no_data.sprite", f"texture {contract.object_id(*tkey)} is not decoded") - def _atlas_entry(self, akey: tuple, rd_key) -> dict: - m = self.atlas_entries.get(akey) - if m is None: - at = self._tt(akey) - m = self.atlas_entries[akey] = [(_canon(k), d) for k, d in at.get("m_RenderDataMap") or ()] - want = _canon(rd_key) - for k, d in m: - if k == want: - return d - raise ValueError(f"atlas {contract.object_id(*akey)} has no render data for the sprite's key") - - def _sprite_plan(self, key: tuple) -> dict: - """Where a sprite's image comes from (UnityPy's get_image_from_sprite): its SpriteAtlas' render data entry - when it names an atlas (an atlas of another bundle: the sprite.crop stage), else the atlas of its file - named by its first atlas tag, else its own render data.""" - r = self.readers[key] - tt = self._tt(key) - atlas_key = None - sa = tt.get("m_SpriteAtlas") - if isinstance(sa, dict) and sa.get("m_PathID"): - akey = self._target(r, sa) - if akey is None: - raise ValueError("m_SpriteAtlas names an unknown file") - if akey not in self.readers: - return {"tt": tt, "external": True, "atlasKey": akey} - atlas_key = akey - elif tt.get("m_AtlasTags"): - tag = tt["m_AtlasTags"][0] - for i in self.infos: - if i.file == key[0] and i.class_name == "SpriteAtlas" and self._name_of((i.file, i.path_id)) == tag: - atlas_key = (i.file, i.path_id) - break - if atlas_key is not None: - src, owner = self._atlas_entry(atlas_key, tt["m_RenderDataKey"]), self.readers[atlas_key] - else: - src, owner = tt["m_RD"], r - tex = src.get("texture") or {} - if not tex.get("m_PathID"): - raise Unsupported("no_data.sprite", "the sprite's render data names no texture") - tkey = self._texture_key(owner, tex) - alpha = src.get("alphaTexture") or {} - akey = self._texture_key(owner, alpha) if alpha.get("m_PathID") else None - return { - "tt": tt, - "external": False, - "src": src, - "owner": owner, - "atlasKey": atlas_key, - "texture": tkey, - "alpha": akey, - } - - def _texture_key(self, owner, pptr: dict) -> tuple: - k = self._target(owner, pptr) - if k is None or k not in self.info: - raise Unsupported( - "unsupported.sprite.external_texture", - f"texture {pptr.get('m_FileID')}:{pptr.get('m_PathID')} is outside the bundle", - ) - if self.info[k].class_name != "Texture2D": - raise Unsupported("no_data.sprite", f"the texture reference names a {self.info[k].class_name}") - return k - - def conv_texture(self, info, holder: dict) -> list: - from ..unity.atoms.texture import ASTC_BLOCKS, format_name, is_hdr_astc - - key = (info.file, info.path_id) - try: - tt = self._tt(key) - inp = self.view.texture_input(info) - except Exception as e: - holder["error"] = e - raise - fmt, w, h = int(inp.format), int(inp.width), int(inp.height) - facts = { - "width": w, - "height": h, - "textureFormat": format_name(fmt), - "mipCount": int(tt.get("m_MipCount", 1) or 1), - "exportedMips": [0], - } - if w and h and is_hdr_astc(fmt): - holder["error"] = Unsupported( - "unsupported.texture.format", f"{format_name(fmt)} (HDR ASTC; the texture is exported as .astc)" - ) - data = self.fns["astc.container"](inp.data, w, h, ASTC_BLOCKS[fmt]) - return [self._emit(key, "image", data, "astc", "texture.image", "astc", facts)[0]] - try: - img = self.fns["texture.decode"](inp) - except Exception as e: - holder["error"] = e - raise - holder["image"] = img - facts["mode"] = img.mode - data = self.fns["png.encode"](img, self.png_level) - return [self._emit(key, "image", data, "png", "texture.image", "png", facts)[0]] - - def _decode_into(self, info, holder: dict) -> None: - """The texture's image for its sprites (a task whose `classes` leave Texture2D out).""" - from ..unity.atoms.texture import format_name, is_hdr_astc - - try: - inp = self.view.texture_input(info) - if inp.width and inp.height and is_hdr_astc(inp.format): - raise Unsupported("unsupported.texture.format", f"{format_name(inp.format)} (HDR ASTC)") - holder["image"] = self.fns["texture.decode"](inp) - except Exception as e: - holder["error"] = e - - def _decoded(self, key: tuple): - return self.fns["texture.decode"](self.view.texture_input(self.info[key])) - - def conv_sprite(self, info, plan: dict, holder: dict) -> list: - from ..unity.atoms import sprite as sprite_atom - - err = holder.get("error") - if err is not None: - if isinstance(err, Unsupported): - raise Unsupported(err.code, f"texture: {err.message}") - raise RuntimeError(f"texture: {_message(err)}") - if "image" not in holder: - raise RuntimeError("texture image not decoded") - key = (info.file, info.path_id) - r, tt, src = self.readers[key], plan["tt"], plan["src"] - alpha = self._decoded(plan["alpha"]) if plan["alpha"] else None - raw = int(src["settingsRaw"]) - st = sprite_atom.settings(raw) - mesh = self.fns["mesh.arrays"](self.view.mesh_input(info)) if st.packing_mode == sprite_atom.TIGHT else None - downscale = float(src.get("downscaleMultiplier", 1.0)) - img = self.fns["sprite.crop"]( - holder["image"], - src["textureRect"], - raw, - mesh, - float(tt["m_PixelsToUnits"]), - alpha, - downscale=downscale, - canvas={"rect": tt["m_Rect"], "offset": src["textureRectOffset"], "pivot": tt["m_Pivot"]}, - ) - data = self.fns["png.encode"](img, self.png_level) - tex_oid = contract.object_id(*plan["texture"]) - refs = [{"rel": "texture", "object": tex_oid}] - if plan["atlasKey"] is not None: - refs.append({"rel": "atlas", "object": contract.object_id(*plan["atlasKey"])}) - image_id = self._emit( - key, - "image", - data, - "png", - "sprite.image", - "png", - {"width": img.width, "height": img.height, "mode": img.mode}, - refs=refs, - )[0] - meta = { - "sprite": self.tree(r, tt), - "image": { - "source": "atlas" if plan["atlasKey"] is not None else "renderData", - "atlas": None if plan["atlasKey"] is None else self._dref(plan["atlasKey"]), - "texture": self._dref(plan["texture"]), - "alphaTexture": None if plan["alpha"] is None else self._dref(plan["alpha"]), - "textureRect": src["textureRect"], - "textureRectOffset": src["textureRectOffset"], - "settingsRaw": raw, - "packed": st.packed, - "packingMode": st.packing_mode, - "packingRotation": st.packing_rotation, - "downscaleMultiplier": downscale, - "width": img.width, - "height": img.height, - }, - } - meta_id = self._emit(key, "meta", encode_object(meta), "json", "sprite.meta", "json", {}, refs=refs)[0] - return [image_id, meta_id] - - def conv_sprite_external(self, info, plan: dict) -> list: - from ..unity.atoms import sprite as sprite_atom - - key = (info.file, info.path_id) - m = self.fns["mesh.arrays"](self.view.mesh_input(info)) - mesh = sprite_atom.mesh_values(m) if "position" in m.channels else None - atlas_oid = contract.object_id(*plan["atlasKey"]) - meta = { - "object": self._facts(key), - "sprite": self.tree(self.readers[key], plan["tt"]), - "image": {"source": "atlas", "atlas": self._dref(plan["atlasKey"]), "stage": CROP}, - "mesh": mesh, - } - return [ - self._emit( - key, - "meta", - encode_object(meta), - "json", - "sprite.meta", - "json", - {"crop": CROP, "atlas": atlas_oid}, - refs=[{"rel": "atlas", "object": atlas_oid}], - )[0] - ] - - # ---------------------------------------------------------------- other converters - def conv_mesh(self, info) -> list: - key = (info.file, info.path_id) - m = self.fns["mesh.arrays"](self.view.mesh_input(info)) - glb = self.fns["gltf.write"]([m]) - facts = { - "vertexCount": int(m.vertex_count), - "submeshes": len(m.submeshes), - "channels": list(m.channels), - "skinned": m.skin is not None, - } - return [self._emit(key, "mesh", glb, "glb", "mesh.gltf", "glb", facts, issues=m.issues)[0]] - - def conv_text(self, info) -> list: - key = (info.file, info.path_id) - s = self._tt(key).get("m_Script", "") - data = s.encode("utf-8", "surrogateescape") if isinstance(s, str) else bytes(s or b"") - container = self.containers.get(key) - sn = self.fns["sniff"](data, container) - ext = _container_ext(container) or sn.ext - return [self._emit(key, "data", data, ext, "text", sn.format, {"size": len(data)})[0]] - - def conv_font(self, info) -> list: - key = (info.file, info.path_id) - fd = self._tt(key).get("m_FontData") or b"" - data = bytes(fd) - if not data: - raise Unsupported("no_data.font", "the font has no font data") - sn = self.fns["sniff"](data) - ext = sn.ext if sn.format in FONT_FORMATS else (_container_ext(self.containers.get(key)) or sn.ext) - return [self._emit(key, "font", data, ext, "font", sn.format, {"size": len(data)})[0]] - - def conv_material(self, info) -> list: - key = (info.file, info.path_id) - doc = self.material(self.readers[key]) - return [self._emit(key, "json", encode_object(doc), "json", "material", "json", {})[0]] - - def conv_shader(self, info) -> list: - """The parsed form (the bytes `nnnotes export shader` writes as .json), every compiled sub-program as - stored (`program:/sp

__`), the variant index and the typetree JSON.""" - from ..formats import shader - - key = (info.file, info.path_id) - r, tt = self.readers[key], self._tt(key) - name = (tt.get("m_ParsedForm") or {}).get("m_Name") or f"shader_{info.path_id}" - text, recs, codes = shader._render(r, tt, name) - variants, ids = [], [] - for rec, code in zip(recs, codes): - fn = rec["file"].rsplit("/", 1)[-1] - stem, ext = fn.rsplit(".", 1) - role = f"program:{rec['platform']}/{stem}" - facts = {k: rec[k] for k in ("platform", "subShader", "pass", "stage", "type", "keywords")} - ids.append(self._emit(key, role, code, ext, "shader.program", ext, facts)[0]) - variants.append({**rec, "role": role}) - platforms = sorted({v["platform"] for v in variants}) - facts = {"name": _label(name), "variants": len(variants), "platforms": platforms} - ids.append(self._emit(key, "json", text.encode("utf-8"), "json", "shader", "json", facts)[0]) - ids.append( - self._emit( - key, - "index", - encode_object({"name": name, "variants": variants}), - "json", - "shader.index", - "json", - {"variants": len(variants)}, - )[0] - ) - ids.append(self._emit(key, "typetree", encode_object(self.tree(r, tt)), "json", "unity.object", "json", {})[0]) - return ids - - def conv_clip(self, info) -> list: - key = (info.file, info.path_id) - if self.go_clash: - raise RuntimeError("GameObject path ids repeat across the bundle's serialized files") - try: - doc = self.clip(self.readers[key]) - except NotImplementedError as e: - if "legacy/compressed" in str(e): - raise Unsupported("generic.clip.legacy", str(e)) from None - raise - return [self._emit(key, "json", encode_object(doc), "json", "animation.clip", "json", {})[0]] - - def conv_controller(self, info) -> list: - key = (info.file, info.path_id) - if info.class_name != "AnimatorController": - raise Unsupported("generic.controller.override", f"{info.class_name} is exported as its typetree") - doc = self.controller(self.readers[key]) - return [self._emit(key, "json", encode_object(doc), "json", "animator.controller", "json", {})[0]] - - def conv_mono(self, info) -> list: - key = (info.file, info.path_id) - tt = self._tt(key) - entry = self._note_script(key, tt) - if entry == _NO_SCRIPT: - raise Unsupported("script.missing", "m_Script is null") - if entry is None: - s = tt.get("m_Script") or {} - raise Unsupported( - "script.unresolved", f"script {s.get('m_FileID')}:{s.get('m_PathID')} is not in the task's script table" - ) - movie = embedded_bytes(tt) if self.usm_assets and entry == USM_ASSET_SCRIPT else None - acb = acb_asset(tt) if movie is None else None - if movie is not None: - self.usm_field = (key, f"references/RefIds/{movie[0]}/data/data", {"movie": _label(tt.get("m_Name"))}) - if acb is not None and acb["layout"] == EMBEDDED_ACB: - if (tt.get("awb") or {}).get("m_PathID"): - raise Unsupported( - "unsupported.cri.awb_external", f"cue sheet {acb['cueSheet']}: its AWB is another object" - ) - path = f"references/RefIds/{acb['ref']}/data/data" - self.acb_field = (key, path, {"cueSheet": _label(acb["cueSheet"]), "layout": EMBEDDED_ACB}) - try: - doc = {"$script": entry, **self.tree(self.readers[key], tt)} - finally: - self.acb_field = self.usm_field = None - ids = [self._emit(key, "json", encode_object(doc), "json", "mono.json", "json", {"script": entry})[0]] - if movie is not None and not any(a["id"] == contract.artifact_id(info.id, "usm") for a in self.staged): - raise ValueError(f"movie {tt.get('m_Name')}: no USM bytes in its {EMBEDDED_ACB_IMPL}") - if ( - acb is not None - and acb["layout"] == EMBEDDED_ACB - and not any(a["id"] == contract.artifact_id(info.id, "acb") for a in self.staged) - ): - raise ValueError(f"cue sheet {acb['cueSheet']}: no ACB bytes in its {EMBEDDED_ACB_IMPL}") - if acb is not None and acb["layout"] == SPLIT_ACB: - ids.append(self._split_acb(key, acb)) - return ids - - def _split_acb(self, key: tuple, acb: dict) -> str: - """The `acb` artifact of a SplitAcbData: its chunk TextAssets (of this bundle) joined and unmasked.""" - from ..formats import cri - - owner, parts, chunks = self.readers[key], [], [] - for c in acb["chunks"]: - k = self._target(owner, c) if isinstance(c, dict) else None - if k is None or k not in self.info or self.info[k].class_name != "TextAsset": - raise ValueError(f"cue sheet {acb['cueSheet']}: chunk {c} is not a TextAsset of the bundle") - parts.append(cri.text_bytes(self._tt(k).get("m_Script", ""))) - chunks.append(k) - data = cri.split_acb(parts, acb["cueSheet"]) - facts = {"cueSheet": _label(acb["cueSheet"]), "layout": SPLIT_ACB, "chunks": len(parts)} - return self._emit( - key, - "acb", - data, - "acb", - ACB_KIND, - "acb", - facts, - refs=[{"rel": "chunk", "object": contract.object_id(*k)} for k in chunks], - )[0] - - def conv_generic(self, info) -> list: - key = (info.file, info.path_id) - doc = self.tree(self.readers[key], self._tt(key)) - return [self._emit(key, "json", encode_object(doc), "json", "unity.object", "json", {})[0]] - - def component(self, o) -> dict: - key = self._key(o) - tt = self._tt(key) - out = {"type": o.type.name, "pathId": key[1]} - if o.type.name == "MonoBehaviour": - entry = self._note_script(key, tt) - out["script"] = entry if entry and entry != _NO_SCRIPT else None - if entry is None: - self.unresolved += 1 - if self.has_clips: - self._register_names(o, tt) - body = self.tree(o, tt) - out["fields"] = {k: v for k, v in body.items() if k != "m_GameObject"} - return out - - def conv_prefab(self, info, root_tf: int, members: list) -> list: - key = (info.file, info.path_id) - g = self.graphs[info.file] - self.unresolved = 0 - nodes, stack = [], [root_tf] - while stack: - tf = stack.pop() - t = g.tf[tf] - go_pid = int(t["m_GameObject"]["m_PathID"]) - go = g.go[go_pid] - comps = [] - for c in go["m_Component"]: - pid = int(c["component"]["m_PathID"] or 0) - if pid and pid != tf: - comps.append(self.component(self.readers[(info.file, pid)])) - node = { - "path": g.path(tf), - "name": go["m_Name"], - "pathId": go_pid, - "transformPathId": tf, - "active": bool(go["m_IsActive"]), - "layer": go["m_Layer"], - "tag": go.get("m_Tag"), - "localPosition": t["m_LocalPosition"], - "localRotation": t["m_LocalRotation"], - "localScale": t["m_LocalScale"], - } - if "m_AnchorMin" in t: - node["rect"] = { - k: t[k] for k in ("m_AnchorMin", "m_AnchorMax", "m_AnchoredPosition", "m_SizeDelta", "m_Pivot") - } - node["components"] = comps - nodes.append(node) - stack.extend(reversed([int(ch["m_PathID"]) for ch in t["m_Children"]])) - doc = {"name": nodes[0]["name"]} - if info.file in self.scene_of: - doc["scene"] = self.scene_of[info.file] - doc["nodes"] = nodes - issues = ( - [("script.unresolved", f"{self.unresolved} MonoBehaviours whose script is not in the task's script table")] - if self.unresolved - else [] - ) - return [ - self._emit( - key, - "prefab", - encode_object(doc), - "json", - "prefab", - "json", - {"nodes": len(nodes), "objects": len(members)}, - issues=issues, - )[0] - ] - def _prefabs(self) -> None: if not self._selected("GameObject"): return - for f, tf_pid, go_pid, members, error in self.roots: - gkey = (f, go_pid) - info = self.info.get(gkey) - if error is not None or info is None or info.class_name != "GameObject": - why = error or f"transform {tf_pid}: no GameObject" - for k in members: - if k in self.info and self.info[k].id not in self.items: - self._fallback(self.info[k], "generic.error", f"hierarchy at transform {tf_pid}: {why}") + h = prefab.hierarchies(self.ctx) + info_of = self.ctx.info + for root in h.roots: + info = info_of.get((root.file, root.game_object)) + if root.error is not None or info is None or info.class_name != "GameObject": + why = root.error or f"transform {root.transform}: no GameObject" + for k in root.members: + if k in info_of and info_of[k].id not in self.items: + self._fallback(info_of[k], "generic.error", f"hierarchy at transform {root.transform}: {why}") continue - self.current = gkey - try: - ok = self._convert(info, "prefab.json", self.conv_prefab, tf_pid, members) - finally: - self.current = None - if ok: + if self._convert(info, "prefab.json", prefab.prefab, h, root): within = contract.artifact_id(info.id, "prefab") - for k in members: - i = self.info[k] + for k in root.members: + i = info_of[k] if i.id != info.id: self.items[i.id] = contract.item(i.id, "contained", within=within, cls=i.class_name) else: why = self.items[info.id].get("reason") or {"code": "generic.error", "message": ""} - for k in members: - i = self.info[k] + for k in root.members: + i = info_of[k] if i.id not in self.items: self._fallback(i, why["code"], f"hierarchy of {info.id}: {why['message']}") - def _loose(self, cls: str, conv: str) -> None: + def _loose(self, cls: str, conv: str, kind: str) -> None: """Objects of `cls` in one task-level artifact "#" (contained in it).""" infos = [i for i in self._of(cls) if i.id not in self.items] if not infos: @@ -1353,51 +291,25 @@ def _loose(self, cls: str, conv: str) -> None: self._begin(conv) entries, done, failed = {}, [], [] for i in infos: - mark = (len(self.staged), len(self.blob_ids)) + mark = len(self.staged) key = (i.file, i.path_id) try: - entries[i.id] = self.tree(self.readers[key], self._tt(key)) + entries[i.id] = self.ctx.tree(self.ctx.readers[key], self.ctx.tt(key)) done.append(i) except Exception as e: - del self.staged[mark[0] :], self.blob_ids[mark[1] :] + del self.staged[mark:] failed.append((i, e)) if done: - if conv == "bundle.json": - doc = { - "files": [ - {"name": f, "unityVersion": self.versions.get(f), "externals": list(self.view.externals(f))} - for f in self.files - ], - "assetBundles": entries, - } - kind = "unity.bundle" - else: - doc = {"scripts": entries} - kind = "unity.scripts" - aid = self._emit( - None, conv, encode_object(doc), "json", kind, "json", {"objects": len(done)}, owner=self.task.id - )[0] - for a in self.staged: - self.emitted.add(a["id"]) - self.artifacts.extend(self.staged) - self.staged = [] + doc = encode_object(generic.loose_doc(self.ctx, conv, entries)) + aid = self._emit(None, conv, doc, "json", kind, "json", {"objects": len(done)}, owner=self.task.id)[0] + self._publish() for i in done: self.items[i.id] = contract.item(i.id, "contained", within=aid, cls=i.class_name) for i, e in failed: - if isinstance(e, _ReadError): - self.items[i.id] = contract.item( - i.id, "failed", why=contract.reason("failed.read", str(e)), cls=i.class_name - ) + if isinstance(e, ReadError): + self._failed(i, e) else: - self._fallback(i, "generic.error", _message(e)) - - -@cache -def exporter_type(): - """RefExporter combined with export.Exporter (imported here: it loads UnityPy).""" - from ..unity.export import Exporter - - return type("ObjectExporter", (RefExporter, Exporter), {}) + self._fallback(i, "generic.error", message(e)) # ---------------------------------------------------------------- stages @@ -1556,7 +468,7 @@ def run(self, task, depot) -> Output: _check_atoms(task, reader) view = reader.fn(depot.input_bytes(task.input("bundle"))) fns = {n: resolve(n).fn for n in task.atoms if n in ATOM_NAMES and n != "reader"} - arts, items = exporter_type()(view, task, depot, fns).run_bundle() + arts, items = BundleRun(view, task, depot, fns).run() return Output(arts, items) @@ -1573,13 +485,6 @@ def _store_input(role: str, record: dict) -> Input: return Input(role, c["sha256"], c["size"], None, ({"kind": "depot"},)) -def _pptr_oid(v) -> str | None: - """The object id of a {"$ref": {...}} value (None for null or an unresolved file).""" - if not isinstance(v, dict) or "$ref" not in v or "file" not in v["$ref"]: - return None - return contract.object_id(v["$ref"]["file"], v["$ref"]["pathId"]) - - class SpriteCropStage(Stage): """sprite.crop: the images of the sprites whose SpriteAtlas is in another bundle. One task per atlas, subject ":"; inputs: "atlas" (the atlas' JSON), "sprite: list[Input]: idx, atlas, reqs = self._subject(subject, env) rec = idx["atlases"][atlas][1] ins = [_store_input("atlas", rec)] - entries = _entries(self._docs.get(env.depot, rec["content"]["sha256"])) + entries = textures.atlas_entries(self._docs.get(env.depot, rec["content"]["sha256"])) needed = set() for owner, _tid, meta in reqs: ins.append(_store_input("sprite:" + owner, meta)) doc = self._docs.get(env.depot, meta["content"]["sha256"]) - entry = _entry(entries, doc["sprite"].get("m_RenderDataKey")) + entry = textures.atlas_entry(entries, doc["sprite"].get("m_RenderDataKey")) if entry is not None: - needed.update(t for t in (_pptr_oid(entry.get("texture")), _pptr_oid(entry.get("alphaTexture"))) if t) + needed.update( + t for t in (ref_object_id(entry.get("texture")), ref_object_id(entry.get("alphaTexture"))) if t + ) ins += [_store_input("texture:" + t, idx["textures"][t][1]) for t in sorted(needed) if t in idx["textures"]] return ins @@ -1700,12 +607,10 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: def run(self, task, depot) -> Output: from PIL import Image - from ..unity.atoms import sprite as sprite_atom - _check_atoms(task) crop, encode = resolve("sprite.crop").fn, resolve("png.encode").fn level = int(task.params["png"]["level"]) - entries = _entries(contract.loads(depot.input_bytes(task.input("atlas")))) + entries = textures.atlas_entries(contract.loads(depot.input_bytes(task.input("atlas")))) images = {} def image(oid: str): @@ -1726,7 +631,7 @@ def image(oid: str): oid = inp.role[len("sprite:") :] meta = contract.loads(depot.input_bytes(inp)) try: - img = _crop_meta(meta, entries, image, crop, sprite_atom) + img = textures.crop_meta(meta, entries, image, crop) aid = contract.artifact_id(oid, "image") arts.append( contract.artifact( @@ -1743,50 +648,13 @@ def image(oid: str): items.append(contract.item(oid, "exported", artifacts=[aid], cls="Sprite")) except Unsupported as e: items.append( - contract.item(oid, "unsupported", why=contract.reason(e.code, _label(e.message)), cls="Sprite") + contract.item(oid, "unsupported", why=contract.reason(e.code, label(e.message)), cls="Sprite") ) except Exception as e: items.append( - contract.item(oid, "failed", why=contract.reason("failed.export", _message(e)), cls="Sprite") + contract.item(oid, "failed", why=contract.reason("failed.export", message(e)), cls="Sprite") ) return Output(arts, items) -def _entries(atlas_doc: dict) -> list: - return [(_canon(k), d) for k, d in atlas_doc.get("m_RenderDataMap") or ()] - - -def _entry(entries: list, key): - want = _canon(key) - return next((d for k, d in entries if k == want), None) - - -def _crop_meta(meta: dict, entries: list, image, crop, sprite_atom): - """A cross-bundle sprite's image from its meta, its atlas' render data entry and the atlas texture's image.""" - s = meta["sprite"] - entry = _entry(entries, s.get("m_RenderDataKey")) - if entry is None: - raise ValueError("the atlas has no render data for the sprite's key") - tex = _pptr_oid(entry.get("texture")) - if tex is None: - raise Unsupported("no_data.sprite", "the atlas entry names no texture") - alpha_oid = _pptr_oid(entry.get("alphaTexture")) - raw = int(entry["settingsRaw"]) - mesh = None - if sprite_atom.settings(raw).packing_mode == sprite_atom.TIGHT: - if not meta.get("mesh"): - raise Unsupported("unsupported.sprite.tight_mask", "tightly packed sprite without a mesh") - mesh = sprite_atom.mesh_from_values(meta["mesh"]) - return crop( - image(tex), - entry["textureRect"], - raw, - mesh, - float(s["m_PixelsToUnits"]), - None if alpha_oid is None else image(alpha_oid), - downscale=float(entry.get("downscaleMultiplier", 1.0)), - canvas={"rect": s["m_Rect"], "offset": entry["textureRectOffset"], "pivot": s["m_Pivot"]}, - ) - - STAGES = (ExportStage, SpriteCropStage) diff --git a/src/nnnotes/unity/convert/__init__.py b/src/nnnotes/unity/convert/__init__.py new file mode 100644 index 0000000..1020360 --- /dev/null +++ b/src/nnnotes/unity/convert/__init__.py @@ -0,0 +1,144 @@ +"""Converters: each Unity class has one, turning an object of a bundle into artifacts. + +A converter reads one bundle alone through its `BundleObjects` (objects.py) and writes its artifacts through the +`emit` function the caller gives it; the caller (the unity.export stage) keeps the items, the depot and the +provenance. Each converter carries a version (CONVERTERS); a task's key includes the converters of the classes its +bundle holds, with the atoms they use (CONVERTER_ATOMS). + + Texture2D tex.png `image`: PNG of the first mip level (HDR ASTC: the .astc file) + Sprite sprite.png `image`: PNG on the sprite's rect; `meta`: JSON + Mesh mesh.glb `mesh`: binary glTF in the mesh's local space + GameObject (a root) prefab.json `prefab`: the hierarchy below it with its components + Material material.json `json` + Shader shader.json `json`: the parsed form as `nnnotes export shader` writes it; + `program:/sp

__`: each compiled sub-program as + stored; `index`: the variants (platform, pass, stage, GPU program type, + keywords); `typetree` + AnimationClip clip.json `json`: the Mecanim clip data and its bindings as stored + AnimatorController controller.json `json` + MonoBehaviour mono.json `json` (a MonoBehaviour that is not part of a hierarchy); a cue sheet asset + also `acb`: its ACB (kind cri.acb); with converter cri.usm, a movie asset + also `usm`: its USM (kind cri.usm) + TextAsset data `data`: the stored bytes + Font font `font`: the font file + MonoScript scripts.json "#scripts.json": every MonoScript of the bundle + AssetBundle bundle.json "#bundle.json": serialized files, externals, AssetBundle objects + any other class generic.json `json`: the object's typetree + +Object JSON (canon/1 in the object's own field order): +- a reference is {"$ref": {"file", "pathId"}}, with "class" and "name" when the target is in the bundle; a null + reference is null. Inside a prefab, a GameObject, Transform or component of the same prefab is named by its path: + {"gameObject": path}, {"transform": path}, {"component": class, "gameObject": path}; +- a byte array (vector of UInt8 / SInt8 / char, TypelessData) shorter than `blobMin` bytes is {"$hex": "..."}; a + longer one is {"$blob": {"sha256", "size", "format"}} with its bytes in the artifact + "#blob:" (field names and array indices joined by "/", the format from atom `sniff`); +- a string whose stored bytes are not UTF-8 is {"$hex": ""}; +- NaN is {"$float": "nan"}, infinities 1e999 / -1e999. + +A converter's documented limit raises atoms.Unsupported with a reason code; the reason codes are listed in +docs/stages.md. +""" + +from __future__ import annotations + +from ..atoms import atom_id + +__all__ = [ + "ALWAYS", + "BASE_ATOMS", + "CLASS_CONVERTERS", + "CONVERTERS", + "CONVERTER_ATOMS", + "REASONS", + "UNSUPPORTED_CLASSES", + "USM_ASSET_SCRIPT", + "USM_CONVERTER", + "converter_id", + "converters_for", + "export_atoms", +] + +# converter -> version +CONVERTERS = { + "tex.png": 1, + "sprite.png": 1, + "mesh.glb": 1, + "prefab.json": 1, + "material.json": 1, + "clip.json": 2, + "controller.json": 2, + "mono.json": 2, + "data": 1, + "font": 1, + "scripts.json": 1, + "bundle.json": 1, + "shader.json": 1, + "generic.json": 1, + "cri.usm": 1, +} +# class -> the converters its objects may use (every class may fall back to generic.json) +CLASS_CONVERTERS = { + "Texture2D": ("tex.png",), + "Sprite": ("sprite.png",), + "Mesh": ("mesh.glb",), + "GameObject": ("prefab.json",), + "Material": ("material.json",), + "AnimationClip": ("clip.json",), + "AnimatorController": ("controller.json",), + "AnimatorOverrideController": ("controller.json",), + "MonoBehaviour": ("mono.json",), + "TextAsset": ("data",), + "Font": ("font",), + "MonoScript": ("scripts.json",), + "AssetBundle": ("bundle.json",), + "Shader": ("shader.json",), +} +ALWAYS = ("generic.json",) +BASE_ATOMS = ("reader", "sniff") +CONVERTER_ATOMS = { + "tex.png": ("texture.decode", "png.encode", "astc.container"), + "sprite.png": ("texture.decode", "png.encode", "sprite.crop", "mesh.arrays"), + "mesh.glb": ("mesh.arrays", "gltf.write"), +} +UNSUPPORTED_CLASSES = ("AudioClip", "VideoClip", "MovieTexture") +# a movie asset holding its USM: converter cri.usm, only in the key of a bundle whose script context names the class +USM_CONVERTER = "cri.usm" +USM_ASSET_SCRIPT = "CriMw.CriWare.Assets.Runtime|CriWare.Assets|CriManaUsmAsset" +# every reason code the converters and their caller give (the atoms give more; docs/stages.md lists them all) +REASONS = ( + "generic.error", + "generic.clip.legacy", + "generic.controller.override", + "script.unresolved", + "script.missing", + "no_data.sprite", + "no_data.font", + "unsupported.sprite.external_texture", + "unsupported.cri.awb_external", + "failed.read", + "failed.export", + *(f"unsupported.class.{c}" for c in UNSUPPORTED_CLASSES), +) + + +def converter_id(name: str) -> str: + return f"{name}/{CONVERTERS[name]}" + + +def converters_for(classes) -> list[str]: + """The converters a bundle holding objects of `classes` may use.""" + out: set[str] = set(ALWAYS) + for c in classes: + out.update(CLASS_CONVERTERS.get(c, ())) + return sorted(out) + + +def export_atoms(classes, reader_id: str | None = None) -> dict: + """The atoms and converters a bundle holding `classes` uses (the `atoms` of its unity.export task).""" + convs = converters_for(classes) + names = set(BASE_ATOMS) + for c in convs: + names.update(CONVERTER_ATOMS.get(c, ())) + out = {n: (reader_id if n == "reader" and reader_id else atom_id(n)) for n in names} + out.update((c, converter_id(c)) for c in convs) + return dict(sorted(out.items())) diff --git a/src/nnnotes/unity/convert/animation.py b/src/nnnotes/unity/convert/animation.py new file mode 100644 index 0000000..2f93a2a --- /dev/null +++ b/src/nnnotes/unity/convert/animation.py @@ -0,0 +1,262 @@ +"""AnimationClip (clip.json) and AnimatorController (controller.json). + +A clip's document holds the Mecanim data the Animator samples and its bindings as stored. A binding names what it +animates by crc32 values (`pathCrc`: the path of the animated object relative to the GameObject whose Animator plays +the clip; `attributeCrc`: the animated property), which only the hierarchy that plays the clip resolves; the document +resolves what needs no hierarchy: the class of the bound component, the script of a MonoBehaviour binding, and the +attributes of a Transform binding. + +Curve indices run over the streamed curves, then the dense ones, then the constant ones; each float binding (not +`pptr`) takes `curves` of them in `bindings` order (1 when the binding has no `curves`). A pptr binding indexes +`pptrCurveMapping` with its streamed values. +""" + +from __future__ import annotations + +import struct +from collections.abc import Callable + +from ..atoms import Unsupported +from .objects import BundleObjects, encode_object + +__all__ = [ + "CLASS_ID", + "FLT_MAX", + "TRANSFORM_ATTR", + "clip", + "clip_doc", + "controller", + "controller_doc", + "streamed_frames", + "transition", +] + +FLT_MAX = 3.4028234663852886e38 +# Unity class IDs (GenericBinding.typeID) -> class name +CLASS_ID = { + 1: "GameObject", + 4: "Transform", + 23: "MeshRenderer", + 95: "Animator", + 198: "ParticleSystem", + 199: "ParticleSystemRenderer", + 212: "SpriteRenderer", + 224: "RectTransform", + 225: "CanvasGroup", + 114: "MonoBehaviour", + 137: "SkinnedMeshRenderer", + 120: "LineRenderer", + 96: "TrailRenderer", + 331: "SpriteMask", + 222: "CanvasRenderer", +} +# Transform binding attribute -> (property, curve count) (GenericBinding, typeID 4) +TRANSFORM_ATTR = { + 1: ("m_LocalPosition", 3), + 2: ("m_LocalRotation", 4), + 3: ("m_LocalScale", 3), + 4: ("localEulerAnglesRaw", 3), +} + + +def streamed_frames(words: list[int]) -> list[list]: + """Mecanim StreamedClip data (uint32 words) -> [[time, [[curve, c0, c1, c2, c3], ...]], ...]. + + A frame holds the cubic segments that start at its time; a curve's value at t + comes from its last key at or before t: ((c0*x + c1)*x + c2)*x + c3, x = t - time. + The stream usually opens with a frame at -FLT_MAX (every curve, constant); clips + whose curves all start at t >= 0 open later. It closes with an empty frame at +inf, + which is dropped here (JSON has no infinity). An empty stream (no streamed + curves) gives no frames. + """ + if not words: + return [] + buf = struct.pack(f"<{len(words)}I", *words) + frames, i = [], 0 + while i < len(buf): + t, n = struct.unpack_from(" dict: + tid = b["typeID"] + d = {"pathCrc": b["path"], "attributeCrc": b["attribute"], "typeID": tid} + if tid in CLASS_ID: + d["class"] = CLASS_ID[tid] + d.update( + customType=b["customType"], + pptr=bool(b["isPPtrCurve"]), + int=bool(b["isIntCurve"]), + serializeReference=bool(b["isSerializeReferenceCurve"]), + ) + if b["script"].get("m_PathID"): + d["script"] = ref(b["script"]) + d["scriptClass"] = script(b["script"]) + if tid == 4 and b["attribute"] in TRANSFORM_ATTR and not d["pptr"]: + d["attribute"], d["curves"] = TRANSFORM_ATTR[b["attribute"]] + return d + + +def clip_doc(tt: dict, ref: Callable[[dict], object], value, script: Callable[[dict], str | None]) -> dict: + """An AnimationClip's document: name, sample rate, wrap mode, the muscle clip's time range, loop and cycle + offset, its events, its bindings, its streamed (cubic keys by frame), dense and constant curves and its + pptr curve mapping. `ref(pptr)`: the value of a reference; `value(v)`: the JSON of a typetree value (an event); + `script(pptr)`: the script table entry of a MonoScript reference (None when unknown). Unsupported for a legacy + or compressed clip.""" + if tt["m_Legacy"] or tt["m_Compressed"]: + raise Unsupported("generic.clip.legacy", f"clip {tt['m_Name']}: legacy/compressed") + mc = tt["m_MuscleClip"] + data = mc["m_Clip"]["data"] + st, de = data["m_StreamedClip"], data["m_DenseClip"] + cbc = tt["m_ClipBindingConstant"] + streamed = {"curveCount": st["curveCount"]} + if "discreteCurveCount" in st: + streamed["discreteCurveCount"] = st["discreteCurveCount"] + streamed["frames"] = streamed_frames(st["data"]) + return { + "name": tt["m_Name"], + "sampleRate": tt["m_SampleRate"], + "wrapMode": tt["m_WrapMode"], + "startTime": mc["m_StartTime"], + "stopTime": mc["m_StopTime"], + "loopTime": bool(mc["m_LoopTime"]), + "cycleOffset": mc["m_CycleOffset"], + "events": [value(e) for e in tt["m_Events"]], + "bindings": [_binding(b, ref, script) for b in cbc["genericBindings"]], + "streamed": streamed, + "dense": { + "curveCount": de["m_CurveCount"], + "frameCount": de["m_FrameCount"], + "sampleRate": de["m_SampleRate"], + "beginTime": de["m_BeginTime"], + "samples": list(de["m_SampleArray"]), + }, + "constant": list(data["m_ConstantClip"]["data"]), + "pptrCurveMapping": [ref(p) for p in cbc.get("pptrCurveMapping", [])], + } + + +def clip(ctx: BundleObjects, info) -> None: + """`json`: clip_doc with references as {"$ref"}.""" + key = (info.file, info.path_id) + r = ctx.readers[key] + doc = clip_doc( + ctx.tt(key), + lambda p: ctx.ref(r, p), + lambda v: ctx.value(r, v), + lambda p: ctx.script_entry(key[0], p), + ) + ctx.emit(key, "json", encode_object(doc), "json", "animation.clip", "json", {}) + + +# ---------------------------------------------------------------- controllers +def transition(t: dict, tos: dict) -> dict: + """A state machine transition: destination state, name, duration, offset, exit time, flags, conditions.""" + return { + "destination": t["m_DestinationState"], + "name": tos.get(t["m_ID"], t["m_ID"]), + "duration": t["m_TransitionDuration"], + "offset": t["m_TransitionOffset"], + "exitTime": t["m_ExitTime"], + "hasExitTime": t["m_HasExitTime"], + "fixedDuration": t["m_HasFixedDuration"], + "canTransitionToSelf": t["m_CanTransitionToSelf"], + "conditions": [ + { + "mode": x["data"]["m_ConditionMode"], + "event": tos.get(x["data"]["m_EventID"], x["data"]["m_EventID"]), + "threshold": x["data"]["m_EventThreshold"], + "exitTime": x["data"]["m_ExitTime"], + } + for x in t["m_ConditionConstantArray"] + ], + } + + +def controller_doc(tt: dict, ref: Callable[[dict], object], value) -> dict: + """An AnimatorController's name, clips, layers, state machines (states with speed, loop, blend trees, + transitions), parameters and default values. Names are looked up in its TOS table. `ref(pptr)`: the value of a + clip reference, taken first and in order; `value(v)`: the JSON of the default values.""" + tos = dict(tt["m_TOS"]) + clips = [ref(c) for c in tt["m_AnimationClips"]] + c = tt["m_Controller"] + machines = [] + for sm in c["m_StateMachineArray"]: + sm = sm["data"] + states = [] + for st in sm["m_StateConstantArray"]: + st = st["data"] + trees = [] + for bt in st["m_BlendTreeConstantArray"]: + trees.append( + [ + { + "clip": n["data"]["m_ClipID"], + "duration": n["data"]["m_Duration"], + "blendType": n["data"]["m_BlendType"], + "children": n["data"]["m_ChildIndices"], + } + for n in bt["data"]["m_NodeArray"] + ] + ) + states.append( + { + "name": tos.get(st["m_NameID"], st["m_NameID"]), + "path": tos.get(st["m_FullPathID"]), + "speed": st["m_Speed"], + "cycleOffset": st["m_CycleOffset"], + "loop": st["m_Loop"], + "writeDefaultValues": st["m_WriteDefaultValues"], + "mirror": st["m_Mirror"], + "speedParam": tos.get(st["m_SpeedParamID"], st["m_SpeedParamID"]), + "timeParam": tos.get(st["m_TimeParamID"], st["m_TimeParamID"]), + "blendTrees": trees, + "transitions": [transition(t["data"], tos) for t in st["m_TransitionConstantArray"]], + } + ) + machines.append( + { + "defaultState": sm["m_DefaultState"], + "states": states, + "anyStateTransitions": [transition(t["data"], tos) for t in sm["m_AnyStateTransitionConstantArray"]], + } + ) + layers = [ + { + "name": tos.get(layer["data"]["m_Binding"]), + "stateMachine": layer["data"]["m_StateMachineIndex"], + "defaultWeight": layer["data"]["m_DefaultWeight"], + "blending": layer["data"]["(int&)m_LayerBlendingMode"], + } + for layer in c["m_LayerArray"] + ] + vals = c["m_Values"]["data"]["m_ValueArray"] if "data" in c["m_Values"] else [] + params = [{"name": tos.get(v["m_ID"], v["m_ID"]), "type": v["m_Type"], "index": v["m_Index"]} for v in vals] + return { + "name": tt["m_Name"], + "clips": clips, + "layers": layers, + "stateMachines": machines, + "parameters": params, + "defaultValues": value(c["m_DefaultValues"]), + } + + +def controller(ctx: BundleObjects, info) -> None: + """`json`: controller_doc with the clips as {"$ref"}. Unsupported for an AnimatorOverrideController.""" + key = (info.file, info.path_id) + if info.class_name != "AnimatorController": + raise Unsupported("generic.controller.override", f"{info.class_name} is exported as its typetree") + r = ctx.readers[key] + doc = controller_doc(ctx.tt(key), lambda p: ctx.ref(r, p), lambda v: ctx.value(r, v)) + ctx.emit(key, "json", encode_object(doc), "json", "animator.controller", "json", {}) diff --git a/src/nnnotes/unity/convert/files.py b/src/nnnotes/unity/convert/files.py new file mode 100644 index 0000000..3c0269e --- /dev/null +++ b/src/nnnotes/unity/convert/files.py @@ -0,0 +1,46 @@ +"""Mesh (mesh.glb), TextAsset (data) and Font (font).""" + +from __future__ import annotations + +from ..atoms import Unsupported +from .objects import BundleObjects, container_ext + +__all__ = ["FONT_FORMATS", "font", "mesh", "text"] + +FONT_FORMATS = ("ttf", "otf", "ttc", "woff", "woff2") + + +def mesh(ctx: BundleObjects, info) -> None: + """`mesh`: binary glTF in the mesh's local space; the limits of its conversion as issues.""" + key = (info.file, info.path_id) + m = ctx.fns["mesh.arrays"](ctx.view.mesh_input(info)) + glb = ctx.fns["gltf.write"]([m]) + facts = { + "vertexCount": int(m.vertex_count), + "submeshes": len(m.submeshes), + "channels": list(m.channels), + "skinned": m.skin is not None, + } + ctx.emit(key, "mesh", glb, "glb", "mesh.gltf", "glb", facts, issues=m.issues) + + +def text(ctx: BundleObjects, info) -> None: + """`data`: the stored bytes, the extension from the container path, else from the content.""" + key = (info.file, info.path_id) + s = ctx.tt(key).get("m_Script", "") + data = s.encode("utf-8", "surrogateescape") if isinstance(s, str) else bytes(s or b"") + container = ctx.containers.get(key) + sn = ctx.fns["sniff"](data, container) + ext = container_ext(container) or sn.ext + ctx.emit(key, "data", data, ext, "text", sn.format, {"size": len(data)}) + + +def font(ctx: BundleObjects, info) -> None: + """`font`: the font file.""" + key = (info.file, info.path_id) + data = bytes(ctx.tt(key).get("m_FontData") or b"") + if not data: + raise Unsupported("no_data.font", "the font has no font data") + sn = ctx.fns["sniff"](data) + ext = sn.ext if sn.format in FONT_FORMATS else (container_ext(ctx.containers.get(key)) or sn.ext) + ctx.emit(key, "font", data, ext, "font", sn.format, {"size": len(data)}) diff --git a/src/nnnotes/unity/convert/generic.py b/src/nnnotes/unity/convert/generic.py new file mode 100644 index 0000000..7199f98 --- /dev/null +++ b/src/nnnotes/unity/convert/generic.py @@ -0,0 +1,30 @@ +"""Objects without a converter of their own (generic.json: the typetree JSON), and the bundle's AssetBundle objects +(bundle.json) and MonoScripts (scripts.json), which go into one document each.""" + +from __future__ import annotations + +from .objects import BundleObjects, encode_object + +__all__ = ["LOOSE", "generic", "loose_doc"] + +# converter of the objects that go into one document -> (class, kind of the document) +LOOSE = {"bundle.json": ("AssetBundle", "unity.bundle"), "scripts.json": ("MonoScript", "unity.scripts")} + + +def generic(ctx: BundleObjects, info) -> None: + """`json`: the object's typetree JSON.""" + key = (info.file, info.path_id) + doc = ctx.tree(ctx.readers[key], ctx.tt(key)) + ctx.emit(key, "json", encode_object(doc), "json", "unity.object", "json", {}) + + +def loose_doc(ctx: BundleObjects, conv: str, entries: dict) -> dict: + """The document of converter `conv` (LOOSE) for `entries` ({object id: typetree JSON}): bundle.json also lists + the bundle's serialized files with their Unity versions and externals.""" + if conv == "bundle.json": + files = [ + {"name": f, "unityVersion": ctx.versions.get(f), "externals": list(ctx.view.externals(f))} + for f in ctx.files + ] + return {"files": files, "assetBundles": entries} + return {"scripts": entries} diff --git a/src/nnnotes/unity/convert/material.py b/src/nnnotes/unity/convert/material.py new file mode 100644 index 0000000..fef22f5 --- /dev/null +++ b/src/nnnotes/unity/convert/material.py @@ -0,0 +1,63 @@ +"""Material (material.json) and Shader (shader.json).""" + +from __future__ import annotations + +from collections.abc import Callable + +from .objects import BundleObjects, encode_object, label + +__all__ = ["material", "material_doc", "shader"] + + +def material_doc(tt: dict, ref: Callable[[dict], object]) -> dict: + """A Material's shader, keywords, render queue, tags, disabled passes and saved properties (textures with their + scale and offset, ints, floats, colours); `ref(pptr)` gives the value of a reference (the shader, textures).""" + sp = tt["m_SavedProperties"] + return { + "material": tt["m_Name"], + "shader": ref(tt["m_Shader"]), + "keywords": tt.get("m_ValidKeywords", []), + "invalidKeywords": tt.get("m_InvalidKeywords", []), + "renderQueue": tt.get("m_CustomRenderQueue"), + "tags": dict(tt.get("stringTagMap", [])), + "disabledPasses": tt.get("disabledShaderPasses", []), + "textures": { + k: {"texture": ref(v["m_Texture"]), "scale": v["m_Scale"], "offset": v["m_Offset"]} + for k, v in sp["m_TexEnvs"] + }, + "ints": dict(sp.get("m_Ints", [])), + "floats": dict(sp.get("m_Floats", [])), + "colors": dict(sp.get("m_Colors", [])), + } + + +def material(ctx: BundleObjects, info) -> None: + """`json`: material_doc with references as {"$ref"}.""" + key = (info.file, info.path_id) + r = ctx.readers[key] + doc = material_doc(ctx.tt(key), lambda p: ctx.ref(r, p)) + ctx.emit(key, "json", encode_object(doc), "json", "material", "json", {}) + + +def shader(ctx: BundleObjects, info) -> None: + """The parsed form (the bytes `nnnotes export shader` writes as .json), every compiled sub-program as + stored (`program:/sp

__`), the variant index and the typetree JSON.""" + from ...formats import shader as shader_format + + key = (info.file, info.path_id) + r, tt = ctx.readers[key], ctx.tt(key) + name = (tt.get("m_ParsedForm") or {}).get("m_Name") or f"shader_{info.path_id}" + text, recs, codes = shader_format._render(r, tt, name) + variants = [] + for rec, code in zip(recs, codes): + stem, ext = rec["file"].rsplit("/", 1)[-1].rsplit(".", 1) + role = f"program:{rec['platform']}/{stem}" + facts = {k: rec[k] for k in ("platform", "subShader", "pass", "stage", "type", "keywords")} + ctx.emit(key, role, code, ext, "shader.program", ext, facts) + variants.append({**rec, "role": role}) + platforms = sorted({v["platform"] for v in variants}) + facts = {"name": label(name), "variants": len(variants), "platforms": platforms} + ctx.emit(key, "json", text.encode("utf-8"), "json", "shader", "json", facts) + index = encode_object({"name": name, "variants": variants}) + ctx.emit(key, "index", index, "json", "shader.index", "json", {"variants": len(variants)}) + ctx.emit(key, "typetree", encode_object(ctx.tree(r, tt)), "json", "unity.object", "json", {}) diff --git a/src/nnnotes/unity/convert/mono.py b/src/nnnotes/unity/convert/mono.py new file mode 100644 index 0000000..8d3a639 --- /dev/null +++ b/src/nnnotes/unity/convert/mono.py @@ -0,0 +1,111 @@ +"""MonoBehaviour (mono.json), with the cue sheets (ACB) and movies (USM) its assets hold. + +A cue sheet held in a bundle is a MonoBehaviour in one of two layouts (formats/cri.py): a SplitAcbData +(`_cueSheetName`, `_chunks`: TextAssets of this bundle, joined and unmasked into the ACB) or a CriWare.Assets asset +named like the sheet whose `implementation` managed reference is a CriSerializedBytesAssetImpl (its `data.data` +bytes are the ACB: the field's {"$blob"} names the `acb` artifact, whatever its size). The artifact's facts name the +cue sheet. A movie asset (CriWare.Assets CriManaUsmAsset) whose `implementation` is a CriSerializedBytesAssetImpl +holds its USM the same way; with converter cri.usm its `data.data` bytes are the `usm` artifact, whatever their size, +the facts naming the asset. +""" + +from __future__ import annotations + +from ..atoms import Unsupported +from ..ids import object_id +from . import USM_ASSET_SCRIPT +from .objects import NO_SCRIPT, BundleObjects, Capture, encode_object, label + +__all__ = ["ACB_KIND", "EMBEDDED_ACB", "SPLIT_ACB", "USM_KIND", "acb_asset", "embedded_bytes", "mono"] + +ACB_KIND, USM_KIND = "cri.acb", "cri.usm" +SPLIT_ACB, EMBEDDED_ACB = "split", "embedded" +EMBEDDED_ACB_IMPL = "CriSerializedBytesAssetImpl" + + +def acb_asset(tt: dict) -> dict | None: + """Which cue sheet layout a MonoBehaviour's typetree holds (cri.py): {"layout": "split", "cueSheet", "chunks"} + for a SplitAcbData, {"layout": "embedded", "cueSheet", "ref": index in references.RefIds, "impl"} for an asset + whose `implementation` is a CriSerializedBytesAssetImpl; None for any other. SplitAcbData first.""" + sheet = tt.get("_cueSheetName") + if isinstance(sheet, str) and isinstance(tt.get("_chunks"), list): + return {"layout": SPLIT_ACB, "cueSheet": sheet, "chunks": tt["_chunks"]} + ref = embedded_bytes(tt) + return ( + None if ref is None else {"layout": EMBEDDED_ACB, "cueSheet": tt.get("m_Name"), "ref": ref[0], "impl": ref[1]} + ) + + +def embedded_bytes(tt: dict) -> tuple[int, dict] | None: + """(index in references.RefIds, the reference) of a CriWare.Assets asset's `implementation` when it is a + CriSerializedBytesAssetImpl (bytes held in the asset), else None.""" + impl = tt.get("implementation") + if not isinstance(impl, dict) or "rid" not in impl: + return None + ids = ((tt.get("references") or {}).get("RefIds")) or [] + for i, r in enumerate(ids): + typ = r.get("type") if isinstance(r, dict) else None + if isinstance(typ, dict) and r.get("rid") == impl["rid"] and typ.get("class") == EMBEDDED_ACB_IMPL: + return i, r + return None + + +def mono(ctx: BundleObjects, info, *, usm_assets: bool) -> None: + """`json`: {"$script": the script table entry, ...the typetree JSON}; a cue sheet asset also `acb`, and with + `usm_assets` (converter cri.usm) a movie asset holding its USM also `usm`.""" + key = (info.file, info.path_id) + tt = ctx.tt(key) + entry = ctx.mono_script(key, tt) + if entry == NO_SCRIPT: + raise Unsupported("script.missing", "m_Script is null") + if entry is None: + s = tt.get("m_Script") or {} + raise Unsupported( + "script.unresolved", f"script {s.get('m_FileID')}:{s.get('m_PathID')} is not in the task's script table" + ) + movie = embedded_bytes(tt) if usm_assets and entry == USM_ASSET_SCRIPT else None + acb = acb_asset(tt) if movie is None else None + captures = {} + if movie is not None: + captures[f"references/RefIds/{movie[0]}/data/data"] = Capture( + "usm", + USM_KIND, + b"CRID", + "the movie data is not a USM (CRID)", + f"movie {tt.get('m_Name')}: no USM bytes in its {EMBEDDED_ACB_IMPL}", + {"movie": label(tt.get("m_Name"))}, + ) + if acb is not None and acb["layout"] == EMBEDDED_ACB: + if (tt.get("awb") or {}).get("m_PathID"): + raise Unsupported("unsupported.cri.awb_external", f"cue sheet {acb['cueSheet']}: its AWB is another object") + from ...formats import cri + + captures[f"references/RefIds/{acb['ref']}/data/data"] = Capture( + "acb", + ACB_KIND, + cri.ACB_SIGNATURE, + "the cue sheet data is not an ACB (@UTF)", + f"cue sheet {acb['cueSheet']}: no ACB bytes in its {EMBEDDED_ACB_IMPL}", + {"cueSheet": label(acb["cueSheet"]), "layout": EMBEDDED_ACB}, + ) + doc = {"$script": entry, **ctx.tree(ctx.readers[key], tt, captures=captures)} + ctx.emit(key, "json", encode_object(doc), "json", "mono.json", "json", {"script": entry}) + if acb is not None and acb["layout"] == SPLIT_ACB: + _split_acb(ctx, key, acb) + + +def _split_acb(ctx: BundleObjects, key: tuple, acb: dict) -> None: + """The `acb` artifact of a SplitAcbData: its chunk TextAssets (of this bundle) joined and unmasked.""" + from ...formats import cri + + owner, parts, chunks = ctx.readers[key], [], [] + for c in acb["chunks"]: + k = ctx.target(owner, c) if isinstance(c, dict) else None + if k is None or k not in ctx.info or ctx.info[k].class_name != "TextAsset": + raise ValueError(f"cue sheet {acb['cueSheet']}: chunk {c} is not a TextAsset of the bundle") + parts.append(cri.text_bytes(ctx.tt(k).get("m_Script", ""))) + chunks.append(k) + data = cri.split_acb(parts, acb["cueSheet"]) + facts = {"cueSheet": label(acb["cueSheet"]), "layout": SPLIT_ACB, "chunks": len(parts)} + refs = [{"rel": "chunk", "object": object_id(*k)} for k in chunks] + ctx.emit(key, "acb", data, "acb", ACB_KIND, "acb", facts, refs=refs) diff --git a/src/nnnotes/unity/convert/objects.py b/src/nnnotes/unity/convert/objects.py new file mode 100644 index 0000000..e92c2ce --- /dev/null +++ b/src/nnnotes/unity/convert/objects.py @@ -0,0 +1,409 @@ +"""One bundle as the converters read it: its objects and their names, typetrees and scripts, and the object JSON +of a typetree (references, byte arrays, strings that are not UTF-8; see the package documentation). + +Artifacts go out through `emit(key, role, data, ext, kind, format, facts, *, issues=(), refs=(), owner=None)`, which +the caller gives: it stores the bytes and records the artifact "#" (owner: the object `key`, else the +task-level name `owner`) and returns (artifact id, content record). +""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import cast + +from ...core.canon import encode +from ..ids import object_id + +__all__ = [ + "NO_SCRIPT", + "BundleObjects", + "Capture", + "ReadError", + "container_ext", + "encode_object", + "label", + "message", + "ref_object_id", +] + +NO_SCRIPT = "\0none" # the script entry of a MonoBehaviour whose m_Script is null +_BYTE_TYPES = frozenset({"UInt8", "SInt8", "char"}) +_SCALARS = (int, float, str, bool, type(None)) +_ADDRESS = re.compile(r"0x[0-9a-fA-F]{6,}") +_EXT = re.compile(r"[0-9a-z]+") + +Emit = Callable[..., tuple[str, dict]] + + +def message(e: BaseException) -> str: + """A deterministic message of an error for a record: no memory addresses, at most 500 characters.""" + return label(_ADDRESS.sub("0x?", f"{type(e).__name__}: {e}")[:500]) + + +def label(s): + """A name for a record: the string, with bytes that are not UTF-8 shown as U+FFFD.""" + if not isinstance(s, str) or s.isascii(): + return s + try: + s.encode("utf-8") + return s + except UnicodeEncodeError: + return s.encode("utf-8", "surrogateescape").decode("utf-8", "replace") + + +def _stored(v): + """`v` with every string that is not UTF-8 (read with surrogateescape) as {"$hex": its stored bytes}.""" + if isinstance(v, str): + try: + v.encode("utf-8") + return v + except UnicodeEncodeError: + return {"$hex": v.encode("utf-8", "surrogateescape").hex()} + if isinstance(v, dict): + return {label(k): _stored(x) for k, x in v.items()} + if isinstance(v, (list, tuple)): + return [_stored(x) for x in v] + return v + + +def encode_object(doc) -> bytes: + """canon/1 bytes of an object document (its own key order).""" + try: + return encode(doc, sort_keys=False) + except UnicodeEncodeError: + return encode(_stored(doc), sort_keys=False) + + +def ref_object_id(v) -> str | None: + """The object id of a {"$ref": {...}} value (None for null or an unresolved file).""" + if not isinstance(v, dict) or "$ref" not in v or "file" not in v["$ref"]: + return None + return object_id(v["$ref"]["file"], v["$ref"]["pathId"]) + + +def container_ext(path: str | None) -> str | None: + """The extension of a container path's last part (lower case letters and digits), None when it has none.""" + if not path: + return None + last = path.rsplit("/", 1)[-1] + if "." not in last: + return None + ext = last.rsplit(".", 1)[1].lower() + return ext if _EXT.fullmatch(ext) else None + + +def _join(path: str, name: str) -> str: + return f"{path}/{name}" if path else name + + +def _file_name(path: str) -> str: + return path.replace("\\", "/").rsplit("/", 1)[-1] + + +def _ref_type_node(ref: dict, assets_file): + """The type tree of a managed reference's data (UnityPy's get_ref_type_node over the file's ref types).""" + typ = ref.get("type") or {} + cls = typ.get("class") if isinstance(typ, dict) else getattr(typ, "class", None) + if not cls: + return None + ns = typ.get("ns") if isinstance(typ, dict) else typ.ns + asm = typ.get("asm") if isinstance(typ, dict) else typ.asm + for rt in getattr(assets_file, "ref_types", None) or (): + if rt.m_ClassName == cls and rt.m_NameSpace == ns and rt.m_AssemblyName == asm: + return rt.node + raise ValueError(f"referenced type {asm}|{ns}|{cls} is not among the file's reference types") + + +class ReadError(Exception): + """An object's typetree could not be read.""" + + +@dataclass(frozen=True) +class Capture: + """A byte array of the object being walked that is an artifact of its own whatever its size: role `role` (also + its extension and format), kind `kind`, facts `facts`. Its bytes start with `signature` (else the error + `mismatch`); `missing` is the error when the walk does not reach it.""" + + role: str + kind: str + signature: bytes + mismatch: str + missing: str + facts: dict = field(default_factory=dict) + + +class BundleObjects: + """The objects of one bundle (`view`, a reader atom's BundleView). `fns`: the atom functions by name; `scripts`: + the script table entries the bundle's objects use ({MonoScript object id: "assembly|namespace|class"}); + `blob_min`: byte arrays of this size or more are blobs; `png_level`: the zlib level of PNG files.""" + + def __init__(self, view, fns: dict, *, scripts: dict, blob_min: int, png_level: int, emit: Emit): + self.view, self.fns, self.emit = view, fns, emit + self.scripts, self.blob_min, self.png_level = dict(scripts), int(blob_min), int(png_level) + self.infos = view.objects() + self.info = {(i.file, i.path_id): i for i in self.infos} + self.readers = {k: view.reader(i) for k, i in self.info.items()} + self.files = view.files() + self.ext_files = {f: [_file_name(p) for p in view.externals(f)] for f in self.files} + self.versions = {f: view.unity_version(f) for f in self.files} + self.containers: dict[tuple, str] = {} + for path, file, pid in view.container(): + k = (file, int(pid)) + if k not in self.containers or path < self.containers[k]: + self.containers[k] = path + self.names: dict[tuple, str | None] = {} + self._mono_scripts: dict[tuple, str | None] = {} + self._special_memo: dict[int, tuple] = {} + self._children_memo: dict[int, tuple] = {} + self._heads: dict[int, tuple] = {} + self._captures: dict[str, Capture] = {} + self._local: Callable[[tuple], dict | None] | None = None + + # ---------------------------------------------------------------- identities and names + @staticmethod + def key(o) -> tuple: + return (o.assets_file.name, int(o.path_id)) + + def of(self, cls: str) -> list: + """The objects of class `cls`.""" + return [i for i in self.infos if i.class_name == cls] + + def file_of(self, owner_file: str, fid: int) -> str | None: + if fid == 0: + return owner_file + ext = self.ext_files.get(owner_file, ()) + return ext[fid - 1] if 0 < fid <= len(ext) else None + + def target(self, owner, pptr: dict) -> tuple | None: + """(file, pathId) a PPtr of `owner` names; None for a null reference or an unknown file index.""" + pid = int(pptr.get("m_PathID", 0) or 0) + if not pid: + return None + file = self.file_of(owner.assets_file.name, int(pptr.get("m_FileID", 0) or 0)) + return None if file is None else (file, pid) + + def _peek(self, r) -> str | None: + """An object's m_Name, read with its type tree cut after that field.""" + node = r._get_typetree_node() + hit = self._heads.get(id(node)) + if hit is None or hit[0] is not node: + names = [c.m_Name for c in node.m_Children] + cut = None + if "m_Name" in names: + from UnityPy.helpers.TypeTreeNode import TypeTreeNode + + cut = TypeTreeNode( + node.m_Level, + node.m_Type, + node.m_Name, + node.m_ByteSize, + node.m_Version, + m_MetaFlag=node.m_MetaFlag, + m_Children=node.m_Children[: names.index("m_Name") + 1], + ) + hit = self._heads[id(node)] = (node, cut) + if hit[1] is None: + return None + name = r.read_typetree(hit[1], check_read=False).get("m_Name") + return name if isinstance(name, str) else None + + def name_of(self, key: tuple) -> str | None: + if key not in self.names: + r = self.readers.get(key) + name = None + if r is not None: + try: + name = self._peek(r) + except Exception: + name = None + self.names[key] = name + return self.names[key] + + def tt(self, key: tuple) -> dict: + """The object's typetree (ReadError when it cannot be read).""" + try: + tt = self.view.typetree(self.info[key]) + except Exception as e: + raise ReadError(message(e)) from None + name = tt.get("m_Name") + self.names[key] = name if isinstance(name, str) else None + return tt + + def facts(self, key: tuple) -> dict: + """The object facts of an artifact's provenance: file, path id, class, name, type hash, Unity version, + container path.""" + i = self.info[key] + d = {"file": label(i.file), "pathId": i.path_id, "classId": i.class_id, "class": i.class_name} + name = self.name_of(key) + if name is not None: + d["name"] = label(name) + if i.type_hash: + d["typeHash"] = i.type_hash + d["unityVersion"] = self.versions.get(i.file) + c = self.containers.get(key) + if c: + d["container"] = label(c) + return d + + # ---------------------------------------------------------------- scripts + def script_entry(self, owner_file: str, pptr) -> str | None: + """The script table entry of the MonoScript `pptr` (read from an object of `owner_file`) names; None for a + null reference or a script the table lacks.""" + if not isinstance(pptr, dict) or not pptr.get("m_PathID"): + return None + file = self.file_of(owner_file, int(pptr.get("m_FileID", 0) or 0)) + return None if file is None else self.scripts.get(object_id(file, int(pptr["m_PathID"]))) + + def mono_script(self, key: tuple, tt: dict) -> str | None: + """The script entry of a MonoBehaviour: NO_SCRIPT when its m_Script is null, None when the table lacks it.""" + s = tt.get("m_Script") + entry = NO_SCRIPT if not isinstance(s, dict) or not s.get("m_PathID") else self.script_entry(key[0], s) + self._mono_scripts[key] = entry + return entry + + def script_class(self, key: tuple) -> str | None: + """The class name of a MonoBehaviour from the script table (None when unknown).""" + if key not in self._mono_scripts: + self.mono_script(key, self.tt(key)) + e = self._mono_scripts[key] + return e.rsplit("|", 1)[-1] if e and e != NO_SCRIPT else None + + # ---------------------------------------------------------------- references and values + def dref(self, key: tuple) -> dict: + """{"$ref"} of an object, with its class and name when it is in the bundle.""" + file, pid = key + d = {"file": label(file), "pathId": pid} + r = self.readers.get(key) + if r is not None: + d["class"] = r.type.name + name = self.name_of(key) + if name is not None: + d["name"] = label(name) + return {"$ref": d} + + def ref(self, owner, pptr: dict): + """The value of a PPtr read from `owner`'s typetree: null, {"$ref"}, or a prefab's local name of the object + while a prefab is walked.""" + pid = int(pptr.get("m_PathID", 0) or 0) + if not pid: + return None + fid = int(pptr.get("m_FileID", 0) or 0) + file = self.file_of(owner.assets_file.name, fid) + if file is None: + return {"$ref": {"fileId": fid, "pathId": pid}} + key = (file, pid) + if self._local is not None: + local = self._local(key) + if local is not None: + return local + return self.dref(key) + + def value(self, owner, v): + """The JSON of a value read from `owner`'s typetree without its type tree: PPtrs as references, bytes as + {"$hex"}.""" + if isinstance(v, dict): + if v.keys() == {"m_FileID", "m_PathID"}: + return self.ref(owner, v) + return {k: self.value(owner, x) for k, x in v.items()} + if isinstance(v, (list, tuple)): + if all(type(x) in _SCALARS for x in v): + return list(v) + return [self.value(owner, x) for x in v] + if isinstance(v, (bytes, bytearray)): + return {"$hex": bytes(v).hex()} + return v + + def tree(self, owner, tt: dict, *, captures: dict[str, Capture] | None = None, local=None) -> dict: + """The object JSON of `owner`'s typetree `tt` (read with owner's type tree). `captures`: {field path: + Capture}, byte arrays emitted as artifacts of their own (every one must be found); `local(key)`: the local + name of an object of the prefab being walked (None for any other).""" + self._captures, self._local = dict(captures or {}), local + try: + doc = cast(dict, self._walk(owner, owner._get_typetree_node(), tt, "")) + left = self._captures + finally: + self._captures, self._local = {}, None + for c in left.values(): + raise ValueError(c.missing) + return doc + + def _child(self, node, name: str): + m = self._children_memo.get(id(node)) + if m is None or m[0] is not node: + m = self._children_memo[id(node)] = (node, {c.m_Name: c for c in node.m_Children}) + return m[1].get(name) + + def _special(self, node) -> bool: + """Whether values of `node` need rewriting: references, byte arrays, managed references below it.""" + hit = self._special_memo.get(id(node)) + if hit is not None and hit[0] is node: + return hit[1] + t, ch = node.m_Type, node.m_Children + if t.startswith("PPtr<") or t in ("TypelessData", "ReferencedObject", "ManagedReferencesRegistry"): + r = True + elif t == "string" or not ch: + r = False + elif ch[0].m_Type == "Array" and len(ch[0].m_Children) > 1: + sub = ch[0].m_Children[1] + r = sub.m_Type in _BYTE_TYPES or self._special(sub) + else: + r = any(self._special(c) for c in ch) + self._special_memo[id(node)] = (node, r) + return r + + def _walk(self, owner, node, v, path: str): + if not self._special(node): + return v + t = node.m_Type + if t.startswith("PPtr<"): + return self.ref(owner, v) if isinstance(v, dict) else v + if t == "TypelessData": + return self._bytes_value(owner, v, path) + ch = node.m_Children + if t == "ReferencedObject": + out = {} + for k, x in v.items(): + c = self._child(node, k) + if c is not None and c.m_Type == "ReferencedObjectData": + rn = _ref_type_node(v, owner.assets_file) + out[k] = x if rn is None else self._walk(owner, rn, x, _join(path, k)) + else: + out[k] = x if c is None else self._walk(owner, c, x, _join(path, k)) + return out + if t != "string" and ch and ch[0].m_Type == "Array": + sub = ch[0].m_Children[1] + if sub.m_Type in _BYTE_TYPES: + return self._bytes_value(owner, v, path, sub.m_Type == "SInt8") + return [self._walk(owner, sub, x, _join(path, str(i))) for i, x in enumerate(v)] + if t == "pair": + return [self._walk(owner, ch[0], v[0], _join(path, "0")), self._walk(owner, ch[1], v[1], _join(path, "1"))] + if not isinstance(v, dict): + return v + out = {} + for k, x in v.items(): + c = self._child(node, k) + out[k] = x if c is None else self._walk(owner, c, x, _join(path, k)) + return out + + def _bytes_value(self, owner, v, path: str, signed: bool = False) -> dict: + if isinstance(v, (bytes, bytearray, memoryview)): + data = bytes(v) + elif signed: + import numpy as np + + data = np.asarray(v, np.int16).astype(np.uint8).tobytes() + else: + data = bytes(v) + c = self._captures.pop(path, None) + if c is not None: + if not data.startswith(c.signature): + raise ValueError(f"{path}: {c.mismatch}: {data[:4].hex()}") + _aid, content = self.emit(self.key(owner), c.role, data, c.role, c.kind, c.role, c.facts) + return {"$blob": {"sha256": content["sha256"], "size": content["size"], "format": c.role}} + if len(data) < self.blob_min: + return {"$hex": data.hex()} + s = self.fns["sniff"](data) + _aid, content = self.emit(self.key(owner), "blob:" + path, data, s.ext, "blob", s.format, {"field": path}) + return {"$blob": {"sha256": content["sha256"], "size": content["size"], "format": s.format}} diff --git a/src/nnnotes/unity/convert/prefab.py b/src/nnnotes/unity/convert/prefab.py new file mode 100644 index 0000000..0dd4e1c --- /dev/null +++ b/src/nnnotes/unity/convert/prefab.py @@ -0,0 +1,193 @@ +"""GameObject hierarchies (prefab.json): a root GameObject's prefab holds the hierarchy below it with its components. + +`hierarchies` finds, per serialized file, the roots and the objects each root's hierarchy holds (its GameObjects, +Transforms and components); inside a prefab those objects are named by their paths. +""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass, field +from types import SimpleNamespace + +from .objects import NO_SCRIPT, BundleObjects, ReadError, encode_object, message + +__all__ = ["Hierarchies", "Root", "hierarchies", "prefab"] + + +@dataclass +class Root: + """A hierarchy's root: its file, Transform and GameObject path ids, the keys of the objects it holds in + hierarchy order, and why it could not be walked (None when it could).""" + + file: str + transform: int + game_object: int + members: list + error: str | None + + +@dataclass +class Hierarchies: + graphs: dict = field(default_factory=dict) # serialized file -> SceneGraph + roots: list[Root] = field(default_factory=list) + member: dict = field(default_factory=dict) # object key -> the key of its root's GameObject + comp_go: dict = field(default_factory=dict) # component key -> its GameObject's path id + scene_of: dict = field(default_factory=dict) # serialized file -> the scene it holds + + def local(self, ctx: BundleObjects, key: tuple, root: tuple) -> dict | None: + """The local name of an object of the hierarchy at `root` (None for any other object).""" + if self.member.get(key) != root: + return None + file, pid = key + o, g = ctx.readers[key], self.graphs[file] + t = o.type.name + if t == "GameObject": + return {"gameObject": g.path(g.tf_of_go[pid])} + if t in ("Transform", "RectTransform"): + return {"transform": g.path(pid)} + d = {"component": t, "gameObject": g.path(g.tf_of_go[self.comp_go[key]])} + if t == "MonoBehaviour": + d["class"] = ctx.script_class(key) + return d + + +def hierarchies(ctx: BundleObjects) -> Hierarchies: + """The hierarchies of every serialized file and the scenes of the files a scene bundle holds.""" + from ..reader import SceneGraph + + h = Hierarchies() + by_file = defaultdict(list) + for i in ctx.infos: + by_file[i.file].append(i) + for f in ctx.files: + objs = [ + ctx.readers[(i.file, i.path_id)] + for i in by_file[f] + if i.class_name in ("GameObject", "Transform", "RectTransform") + ] + try: + g = SceneGraph(SimpleNamespace(objects=objs)) + except Exception: + continue # the file's objects are exported one by one + h.graphs[f] = g + for tf_pid, t in g.tf.items(): + father = int(t["m_Father"]["m_PathID"] or 0) + if father and father in g.tf: + continue + go_pid = int(t["m_GameObject"]["m_PathID"] or 0) + members, comps, error = _subtree(ctx, h, f, g, tf_pid) + h.roots.append(Root(f, tf_pid, go_pid, members, error)) + if error is None: + for k in members: + h.member[k] = (f, go_pid) + h.comp_go.update(comps) + for i in ctx.of("AssetBundle"): + try: + tt = ctx.tt((i.file, i.path_id)) + except ReadError: + continue + for pair in tt.get("m_SceneHashes") or (): + scene, cab = pair[0], str(pair[1]).lower() + for f in ctx.files: + if cab and f.lower().startswith(cab): + h.scene_of.setdefault(f, scene) + return h + + +def _subtree(ctx: BundleObjects, h: Hierarchies, f: str, g, root_tf: int): + """(member keys in hierarchy order, {component key: GameObject pathId}, error) of the hierarchy at root_tf.""" + members, comps, seen, stack = [], {}, set(), [root_tf] + try: + while stack: + tf = stack.pop() + if tf in seen: + raise ValueError(f"transform {tf} appears twice in the hierarchy") + if tf not in g.tf: + raise ValueError(f"child transform {tf} is not in the file") + seen.add(tf) + t = g.tf[tf] + go_pid = int(t["m_GameObject"]["m_PathID"] or 0) + if go_pid not in g.go: + raise ValueError(f"transform {tf}: GameObject {go_pid} is not in the file") + keys = [(f, go_pid), (f, tf)] + for c in g.go[go_pid]["m_Component"]: + p = c["component"] + pid = int(p["m_PathID"] or 0) + if not pid or pid == tf: + continue + if int(p.get("m_FileID", 0) or 0) or (f, pid) not in ctx.readers: + raise ValueError(f"GameObject {go_pid}: component {pid} is not in the file") + keys.append((f, pid)) + comps[(f, pid)] = go_pid + for k in keys: + if k in h.member: + raise ValueError(f"object {k[1]} belongs to two hierarchies") + members.extend(keys) + stack.extend(reversed([int(ch["m_PathID"]) for ch in t["m_Children"]])) + except Exception as e: + return members, comps, message(e) + return members, comps, None + + +def _component(ctx: BundleObjects, h: Hierarchies, o, root: tuple, unresolved: list) -> dict: + key = ctx.key(o) + tt = ctx.tt(key) + out = {"type": o.type.name, "pathId": key[1]} + if o.type.name == "MonoBehaviour": + entry = ctx.mono_script(key, tt) + out["script"] = entry if entry and entry != NO_SCRIPT else None + if entry is None: + unresolved.append(key) + body = ctx.tree(o, tt, local=lambda k: h.local(ctx, k, root)) + out["fields"] = {k: v for k, v in body.items() if k != "m_GameObject"} + return out + + +def prefab(ctx: BundleObjects, info, h: Hierarchies, root: Root) -> None: + """`prefab`: {"name", "scene" (a root of a scene's file), "nodes"}: every node of the hierarchy in hierarchy + order (path, name, path ids, active, layer, tag, local transform, rect, components with their fields).""" + key = (info.file, info.path_id) + g = h.graphs[info.file] + unresolved: list = [] + nodes, stack = [], [root.transform] + while stack: + tf = stack.pop() + t = g.tf[tf] + go_pid = int(t["m_GameObject"]["m_PathID"]) + go = g.go[go_pid] + comps = [] + for c in go["m_Component"]: + pid = int(c["component"]["m_PathID"] or 0) + if pid and pid != tf: + comps.append(_component(ctx, h, ctx.readers[(info.file, pid)], key, unresolved)) + node = { + "path": g.path(tf), + "name": go["m_Name"], + "pathId": go_pid, + "transformPathId": tf, + "active": bool(go["m_IsActive"]), + "layer": go["m_Layer"], + "tag": go.get("m_Tag"), + "localPosition": t["m_LocalPosition"], + "localRotation": t["m_LocalRotation"], + "localScale": t["m_LocalScale"], + } + if "m_AnchorMin" in t: + node["rect"] = { + k: t[k] for k in ("m_AnchorMin", "m_AnchorMax", "m_AnchoredPosition", "m_SizeDelta", "m_Pivot") + } + node["components"] = comps + nodes.append(node) + stack.extend(reversed([int(ch["m_PathID"]) for ch in t["m_Children"]])) + doc = {"name": nodes[0]["name"]} + if info.file in h.scene_of: + doc["scene"] = h.scene_of[info.file] + doc["nodes"] = nodes + issues = ( + [("script.unresolved", f"{len(unresolved)} MonoBehaviours whose script is not in the task's script table")] + if unresolved + else [] + ) + facts = {"nodes": len(nodes), "objects": len(root.members)} + ctx.emit(key, "prefab", encode_object(doc), "json", "prefab", "json", facts, issues=issues) diff --git a/src/nnnotes/unity/convert/textures.py b/src/nnnotes/unity/convert/textures.py new file mode 100644 index 0000000..2cb7ec3 --- /dev/null +++ b/src/nnnotes/unity/convert/textures.py @@ -0,0 +1,263 @@ +"""Texture2D (tex.png) and Sprite (sprite.png). + +A texture's image is decoded once per bundle: `texture` (or `decode_into`, when the task leaves textures out) puts it +in a holder that the sprites cut from it read. Where a sprite's image comes from is its plan (`sprite_plan`). +""" + +from __future__ import annotations + +import json + +from ..atoms import Unsupported +from ..ids import object_id +from .objects import BundleObjects, encode_object, message, ref_object_id + +__all__ = [ + "atlas_entries", + "atlas_entry", + "crop_meta", + "decode_into", + "sprite", + "sprite_external", + "sprite_plan", + "texture", +] + + +def _canon(v) -> str: + return json.dumps(v, sort_keys=True, ensure_ascii=True) + + +def texture(ctx: BundleObjects, info, holder: dict) -> None: + """`image`: PNG of the first mip level (HDR ASTC: the .astc file). The decoded image (or the error) goes into + `holder` for the texture's sprites.""" + from ..atoms.texture import ASTC_BLOCKS, format_name, is_hdr_astc + + key = (info.file, info.path_id) + try: + tt = ctx.tt(key) + inp = ctx.view.texture_input(info) + except Exception as e: + holder["error"] = e + raise + fmt, w, h = int(inp.format), int(inp.width), int(inp.height) + facts = { + "width": w, + "height": h, + "textureFormat": format_name(fmt), + "mipCount": int(tt.get("m_MipCount", 1) or 1), + "exportedMips": [0], + } + if w and h and is_hdr_astc(fmt): + holder["error"] = Unsupported( + "unsupported.texture.format", f"{format_name(fmt)} (HDR ASTC; the texture is exported as .astc)" + ) + data = ctx.fns["astc.container"](inp.data, w, h, ASTC_BLOCKS[fmt]) + ctx.emit(key, "image", data, "astc", "texture.image", "astc", facts) + return + try: + img = ctx.fns["texture.decode"](inp) + except Exception as e: + holder["error"] = e + raise + holder["image"] = img + facts["mode"] = img.mode + data = ctx.fns["png.encode"](img, ctx.png_level) + ctx.emit(key, "image", data, "png", "texture.image", "png", facts) + + +def decode_into(ctx: BundleObjects, info, holder: dict) -> None: + """The texture's image for its sprites only (a task whose classes leave Texture2D out).""" + from ..atoms.texture import format_name, is_hdr_astc + + try: + inp = ctx.view.texture_input(info) + if inp.width and inp.height and is_hdr_astc(inp.format): + raise Unsupported("unsupported.texture.format", f"{format_name(inp.format)} (HDR ASTC)") + holder["image"] = ctx.fns["texture.decode"](inp) + except Exception as e: + holder["error"] = e + + +def _atlas_entry(ctx: BundleObjects, memo: dict, akey: tuple, rd_key) -> dict: + m = memo.get(akey) + if m is None: + m = memo[akey] = [(_canon(k), d) for k, d in ctx.tt(akey).get("m_RenderDataMap") or ()] + want = _canon(rd_key) + for k, d in m: + if k == want: + return d + raise ValueError(f"atlas {object_id(*akey)} has no render data for the sprite's key") + + +def _texture_key(ctx: BundleObjects, owner, pptr: dict) -> tuple: + k = ctx.target(owner, pptr) + if k is None or k not in ctx.info: + raise Unsupported( + "unsupported.sprite.external_texture", + f"texture {pptr.get('m_FileID')}:{pptr.get('m_PathID')} is outside the bundle", + ) + if ctx.info[k].class_name != "Texture2D": + raise Unsupported("no_data.sprite", f"the texture reference names a {ctx.info[k].class_name}") + return k + + +def sprite_plan(ctx: BundleObjects, key: tuple, atlases: dict) -> dict: + """Where a sprite's image comes from (UnityPy's get_image_from_sprite): its SpriteAtlas' render data entry when + it names an atlas ("external": an atlas of another bundle), else the atlas of its file named by its first atlas + tag, else its own render data. `atlases`: a memo of the atlases' render data maps, kept across the sprites.""" + r = ctx.readers[key] + tt = ctx.tt(key) + atlas_key = None + sa = tt.get("m_SpriteAtlas") + if isinstance(sa, dict) and sa.get("m_PathID"): + akey = ctx.target(r, sa) + if akey is None: + raise ValueError("m_SpriteAtlas names an unknown file") + if akey not in ctx.readers: + return {"tt": tt, "external": True, "atlasKey": akey} + atlas_key = akey + elif tt.get("m_AtlasTags"): + tag = tt["m_AtlasTags"][0] + for i in ctx.infos: + if i.file == key[0] and i.class_name == "SpriteAtlas" and ctx.name_of((i.file, i.path_id)) == tag: + atlas_key = (i.file, i.path_id) + break + if atlas_key is not None: + src, owner = _atlas_entry(ctx, atlases, atlas_key, tt["m_RenderDataKey"]), ctx.readers[atlas_key] + else: + src, owner = tt["m_RD"], r + tex = src.get("texture") or {} + if not tex.get("m_PathID"): + raise Unsupported("no_data.sprite", "the sprite's render data names no texture") + tkey = _texture_key(ctx, owner, tex) + alpha = src.get("alphaTexture") or {} + akey = _texture_key(ctx, owner, alpha) if alpha.get("m_PathID") else None + return { + "tt": tt, + "external": False, + "src": src, + "owner": owner, + "atlasKey": atlas_key, + "texture": tkey, + "alpha": akey, + } + + +def sprite(ctx: BundleObjects, info, plan: dict, holder: dict) -> None: + """`image`: PNG on the sprite's rect cut from its texture's image in `holder`; `meta`: the sprite's JSON and + where its image came from.""" + from ..atoms import sprite as sprite_atom + + err = holder.get("error") + if err is not None: + if isinstance(err, Unsupported): + raise Unsupported(err.code, f"texture: {err.message}") + raise RuntimeError(f"texture: {message(err)}") + if "image" not in holder: + raise RuntimeError("texture image not decoded") + key = (info.file, info.path_id) + r, tt, src = ctx.readers[key], plan["tt"], plan["src"] + alpha = ctx.fns["texture.decode"](ctx.view.texture_input(ctx.info[plan["alpha"]])) if plan["alpha"] else None + raw = int(src["settingsRaw"]) + st = sprite_atom.settings(raw) + mesh = ctx.fns["mesh.arrays"](ctx.view.mesh_input(info)) if st.packing_mode == sprite_atom.TIGHT else None + downscale = float(src.get("downscaleMultiplier", 1.0)) + img = ctx.fns["sprite.crop"]( + holder["image"], + src["textureRect"], + raw, + mesh, + float(tt["m_PixelsToUnits"]), + alpha, + downscale=downscale, + canvas={"rect": tt["m_Rect"], "offset": src["textureRectOffset"], "pivot": tt["m_Pivot"]}, + ) + data = ctx.fns["png.encode"](img, ctx.png_level) + refs = [{"rel": "texture", "object": object_id(*plan["texture"])}] + if plan["atlasKey"] is not None: + refs.append({"rel": "atlas", "object": object_id(*plan["atlasKey"])}) + facts = {"width": img.width, "height": img.height, "mode": img.mode} + ctx.emit(key, "image", data, "png", "sprite.image", "png", facts, refs=refs) + meta = { + "sprite": ctx.tree(r, tt), + "image": { + "source": "atlas" if plan["atlasKey"] is not None else "renderData", + "atlas": None if plan["atlasKey"] is None else ctx.dref(plan["atlasKey"]), + "texture": ctx.dref(plan["texture"]), + "alphaTexture": None if plan["alpha"] is None else ctx.dref(plan["alpha"]), + "textureRect": src["textureRect"], + "textureRectOffset": src["textureRectOffset"], + "settingsRaw": raw, + "packed": st.packed, + "packingMode": st.packing_mode, + "packingRotation": st.packing_rotation, + "downscaleMultiplier": downscale, + "width": img.width, + "height": img.height, + }, + } + ctx.emit(key, "meta", encode_object(meta), "json", "sprite.meta", "json", {}, refs=refs) + + +def sprite_external(ctx: BundleObjects, info, plan: dict, crop: str) -> None: + """`meta` of a sprite whose SpriteAtlas is in another bundle: its JSON, mesh and object facts, for the stage + `crop` that cuts its image from the atlas texture.""" + from ..atoms import sprite as sprite_atom + + key = (info.file, info.path_id) + m = ctx.fns["mesh.arrays"](ctx.view.mesh_input(info)) + mesh = sprite_atom.mesh_values(m) if "position" in m.channels else None + atlas_oid = object_id(*plan["atlasKey"]) + meta = { + "object": ctx.facts(key), + "sprite": ctx.tree(ctx.readers[key], plan["tt"]), + "image": {"source": "atlas", "atlas": ctx.dref(plan["atlasKey"]), "stage": crop}, + "mesh": mesh, + } + facts = {"crop": crop, "atlas": atlas_oid} + refs = [{"rel": "atlas", "object": atlas_oid}] + ctx.emit(key, "meta", encode_object(meta), "json", "sprite.meta", "json", facts, refs=refs) + + +# ---------------------------------------------------------------- sprites cut from an atlas of another bundle +def atlas_entries(atlas_doc: dict) -> list: + """The render data entries of a SpriteAtlas' JSON, keyed for atlas_entry.""" + return [(_canon(k), d) for k, d in atlas_doc.get("m_RenderDataMap") or ()] + + +def atlas_entry(entries: list, key): + """The entry of atlas_entries under a sprite's m_RenderDataKey (None when the atlas has none).""" + want = _canon(key) + return next((d for k, d in entries if k == want), None) + + +def crop_meta(meta: dict, entries: list, image, crop): + """The image of a sprite whose atlas is in another bundle, from its `meta` (sprite_external), its atlas' + render data entries and `image(object id)`, the decoded image of an atlas texture; `crop`: atom sprite.crop.""" + from ..atoms import sprite as sprite_atom + + s = meta["sprite"] + entry = atlas_entry(entries, s.get("m_RenderDataKey")) + if entry is None: + raise ValueError("the atlas has no render data for the sprite's key") + tex = ref_object_id(entry.get("texture")) + if tex is None: + raise Unsupported("no_data.sprite", "the atlas entry names no texture") + alpha_oid = ref_object_id(entry.get("alphaTexture")) + raw = int(entry["settingsRaw"]) + mesh = None + if sprite_atom.settings(raw).packing_mode == sprite_atom.TIGHT: + if not meta.get("mesh"): + raise Unsupported("unsupported.sprite.tight_mask", "tightly packed sprite without a mesh") + mesh = sprite_atom.mesh_from_values(meta["mesh"]) + return crop( + image(tex), + entry["textureRect"], + raw, + mesh, + float(s["m_PixelsToUnits"]), + None if alpha_oid is None else image(alpha_oid), + downscale=float(entry.get("downscaleMultiplier", 1.0)), + canvas={"rect": s["m_Rect"], "offset": entry["textureRectOffset"], "pivot": s["m_Pivot"]}, + ) diff --git a/src/nnnotes/unity/export.py b/src/nnnotes/unity/export.py index 3e77412..4b0eb68 100644 --- a/src/nnnotes/unity/export.py +++ b/src/nnnotes/unity/export.py @@ -26,7 +26,6 @@ import json import math import re -import struct import zlib from pathlib import Path from typing import Literal @@ -38,6 +37,8 @@ from ..core import cache from ..source.catalog import Catalog +from .convert.animation import CLASS_ID, TRANSFORM_ATTR, controller_doc, streamed_frames +from .convert.material import material_doc from .reader import ( DEFAULT_RESOURCES, SceneGraph, @@ -119,37 +120,6 @@ def _crc(s: str) -> int: return zlib.crc32(s.encode("utf-8")) -FLT_MAX = 3.4028234663852886e38 - - -def streamed_frames(words: list[int]) -> list[list]: - """Mecanim StreamedClip data (uint32 words) -> [[time, [[curve, c0, c1, c2, c3], ...]], ...]. - - A frame holds the cubic segments that start at its time; a curve's value at t - comes from its last key at or before t: ((c0*x + c1)*x + c2)*x + c3, x = t - time. - The stream usually opens with a frame at -FLT_MAX (every curve, constant); clips - whose curves all start at t >= 0 open later. It closes with an empty frame at +inf, - which is dropped here (JSON has no infinity). An empty stream (no streamed - curves) gives no frames. - """ - if not words: - return [] - buf = struct.pack(f"<{len(words)}I", *words) - frames, i = [], 0 - while i < len(buf): - t, n = struct.unpack_from(" list[str]: """Every field path of a typetree ("m_Color", "m_Color.r", "m_Enabled", ...); lists are leaves.""" out = [] @@ -162,31 +132,6 @@ def flat_names(tt, prefix: str = "") -> list[str]: # --- clip bindings ------------------------------------------------------------ -# Unity class IDs (GenericBinding.typeID) -> class name -CLASS_ID = { - 1: "GameObject", - 4: "Transform", - 23: "MeshRenderer", - 95: "Animator", - 198: "ParticleSystem", - 199: "ParticleSystemRenderer", - 212: "SpriteRenderer", - 224: "RectTransform", - 225: "CanvasGroup", - 114: "MonoBehaviour", - 137: "SkinnedMeshRenderer", - 120: "LineRenderer", - 96: "TrailRenderer", - 331: "SpriteMask", - 222: "CanvasRenderer", -} -# Transform binding attribute -> (property, curve count) (GenericBinding, typeID 4) -TRANSFORM_ATTR = { - 1: ("m_LocalPosition", 3), - 2: ("m_LocalRotation", 4), - 3: ("m_LocalScale", 3), - 4: ("localEulerAnglesRaw", 3), -} # Animatable property names a binding can name (attribute = crc32 of the name) that are not field paths of a # component the exporter has seen: common component properties, RectTransform driven fields, and material # properties ("material.[.]", renderer material property blocks). @@ -824,24 +769,7 @@ def packed_sprite(self, ref: str) -> dict: } def material(self, o) -> dict: - tt = o.read_typetree() - sp = tt["m_SavedProperties"] - m = { - "material": tt["m_Name"], - "shader": self.ref(o, tt["m_Shader"]), - "keywords": tt.get("m_ValidKeywords", []), - "invalidKeywords": tt.get("m_InvalidKeywords", []), - "renderQueue": tt.get("m_CustomRenderQueue"), - "tags": dict(tt.get("stringTagMap", [])), - "disabledPasses": tt.get("disabledShaderPasses", []), - "textures": { - k: {"texture": self.ref(o, v["m_Texture"]), "scale": v["m_Scale"], "offset": v["m_Offset"]} - for k, v in sp["m_TexEnvs"] - }, - "ints": dict(sp.get("m_Ints", [])), - "floats": dict(sp.get("m_Floats", [])), - "colors": dict(sp.get("m_Colors", [])), - } + m = material_doc(o.read_typetree(), lambda p: self.ref(o, p)) self.material_props.update(m["floats"]) self.material_props.update(m["colors"]) return m @@ -1505,103 +1433,16 @@ def controller(self, o) -> dict: if k in self._done: return self._done[k] rec = self.controller_record(o) - tt, tos = rec["tt"], rec["tos"] registry = self.clip_format == "livenotes" - cid = _rid(o, tt["m_Name"]) if registry else tt["m_Name"] + cid = _rid(o, rec["name"]) if registry else rec["name"] ref = {"controller": cid} self._done[k] = ref - clips = [self.ref(o, c) for c in tt["m_AnimationClips"]] - c = tt["m_Controller"] - machines = [] - for sm in c["m_StateMachineArray"]: - sm = sm["data"] - states = [] - for st in sm["m_StateConstantArray"]: - st = st["data"] - trees = [] - for bt in st["m_BlendTreeConstantArray"]: - nodes = bt["data"]["m_NodeArray"] - trees.append( - [ - { - "clip": n["data"]["m_ClipID"], - "duration": n["data"]["m_Duration"], - "blendType": n["data"]["m_BlendType"], - "children": n["data"]["m_ChildIndices"], - } - for n in nodes - ] - ) - states.append( - { - "name": tos.get(st["m_NameID"], st["m_NameID"]), - "path": tos.get(st["m_FullPathID"]), - "speed": st["m_Speed"], - "cycleOffset": st["m_CycleOffset"], - "loop": st["m_Loop"], - "writeDefaultValues": st["m_WriteDefaultValues"], - "mirror": st["m_Mirror"], - "speedParam": tos.get(st["m_SpeedParamID"], st["m_SpeedParamID"]), - "timeParam": tos.get(st["m_TimeParamID"], st["m_TimeParamID"]), - "blendTrees": trees, - "transitions": [self._transition(t["data"], tos) for t in st["m_TransitionConstantArray"]], - } - ) - machines.append( - { - "defaultState": sm["m_DefaultState"], - "states": states, - "anyStateTransitions": [ - self._transition(t["data"], tos) for t in sm["m_AnyStateTransitionConstantArray"] - ], - } - ) - layers = [ - { - "name": tos.get(layer["data"]["m_Binding"]), - "stateMachine": layer["data"]["m_StateMachineIndex"], - "defaultWeight": layer["data"]["m_DefaultWeight"], - "blending": layer["data"]["(int&)m_LayerBlendingMode"], - } - for layer in c["m_LayerArray"] - ] - vals = c["m_Values"]["data"]["m_ValueArray"] if "data" in c["m_Values"] else [] - params = [{"name": tos.get(v["m_ID"], v["m_ID"]), "type": v["m_Type"], "index": v["m_Index"]} for v in vals] - body = { - "name": tt["m_Name"], - "clips": clips, - "layers": layers, - "stateMachines": machines, - "parameters": params, - "defaultValues": self.value(o, c["m_DefaultValues"]), - } + body = controller_doc(rec["tt"], lambda p: self.ref(o, p), lambda v: self.value(o, v)) if registry: self.controllers[cid] = body return ref return {**ref, **body} - @staticmethod - def _transition(t: dict, tos: dict) -> dict: - return { - "destination": t["m_DestinationState"], - "name": tos.get(t["m_ID"], t["m_ID"]), - "duration": t["m_TransitionDuration"], - "offset": t["m_TransitionOffset"], - "exitTime": t["m_ExitTime"], - "hasExitTime": t["m_HasExitTime"], - "fixedDuration": t["m_HasFixedDuration"], - "canTransitionToSelf": t["m_CanTransitionToSelf"], - "conditions": [ - { - "mode": x["data"]["m_ConditionMode"], - "event": tos.get(x["data"]["m_EventID"], x["data"]["m_EventID"]), - "threshold": x["data"]["m_EventThreshold"], - "exitTime": x["data"]["m_ExitTime"], - } - for x in t["m_ConditionConstantArray"] - ], - } - def controller_states(self, o) -> dict: """A single-layer AnimatorController without transitions, blend trees or parameter-driven states as its state list (name, speed, loop, cycle offset, clip name) -- what the ADV UI runtime plays.""" diff --git a/tests/golden/unity.export.json b/tests/golden/unity.export.json index b1bb240..96e8707 100644 --- a/tests/golden/unity.export.json +++ b/tests/golden/unity.export.json @@ -2,8 +2,8 @@ "atoms": { "astc.container": "nnnotes/1", "bundle.json": "bundle.json/1", - "clip.json": "clip.json/1", - "controller.json": "controller.json/1", + "clip.json": "clip.json/2", + "controller.json": "controller.json/2", "cri.usm": "cri.usm/1", "data": "data/1", "font": "font/1", @@ -26,68 +26,68 @@ }, "fixtures": { "all": { - "key": "1ab3c28765bcdf65aee5d4014e5502bca2fbb325d13d78b4d2514c6648f6b8bb", - "result": "a222a665e46c2de7c249ed3f7114ba0a57eae4949d1d0e57d15c0179ee7849ae" + "key": "d3be85097dda7b39fb780c7b8d4560dbf88e85901372cc2256c2478b9d2ba1ef", + "result": "13f069ec2e00b913f537b8434f235b1232b5f875aebe1c3f40c39ff4e3274bdc" }, "bundle.json": { - "key": "3bcdd4f9cab872b42263dd6e82002617784859c5290d10e53e9ff49a1d38211d", - "result": "34acf8e3a63b7577e35a8d8d369da5dac4503122ff47f53e4927f3703996318c" + "key": "c81a9154d13d953e581bbd4cffe0590ad296a296ff6eeda9398aa8d394b44325", + "result": "cbaaf45561972f2004fa74352dc3adcc96c2baabadb56d85a49add792921fe9c" }, "clip.json": { - "key": "d5f28f12fc88b4eeaccf39b4d1f48943b539e2ab8daec1b0fe7ede78ff06a1cd", - "result": "c25e23fccc2dc59971751d729de1e383ef59f313efc8f5af1d22d07d40bafcc2" + "key": "07f85633bae3127fe11950188284b9325066133b4d86eaf61c3414b3c622fa77", + "result": "f133a83bce862cef316e2ea2907790b073eb7928911150bed8225529534df177" }, "controller.json": { - "key": "c8f44677dc63238367b1fe4bdf5a10d323f614c6b0b3e1b8d40dbc2042cfc161", - "result": "146083127e3ad68d0107f83d907812ad705c7e3142487145267fb3fcef0a3042" + "key": "0488d21f6759dc95639a3d1e643980472452fbd4b04c3d5623942a1095b5e4a5", + "result": "c1ca4ec4b97bb18b24a8effc3a0faa4f5dc5a6e3724eeba8368dd0505f59235a" }, "cri.usm": { "key": "8d1d45652a6a631253b3bad7f096af988fcc3894831a8fe5e7ec90b939bc3427", "result": "ca67e2f4878b562b24cb6ca374c469ee2085a2c0259245adcae05cba4f9095b1" }, "data": { - "key": "5505c5d504c4408e8ff8eca82b9d456f2b04ff341438a24fda41583af0878596", - "result": "e4ed537df09c81586d8ab3ab8083a6b93d00aef96f84def21b76a150319b1246" + "key": "62b78ee012b053bfb1ee11245b07cc58428f13bd0fb1b5de221b7b0a22372654", + "result": "7ca1098d52be75092183689a236f8d0b876edd68684e32b691364bad93ca89bd" }, "font": { - "key": "70e013a625db99cc6414a28bb0caf8eeaa42106fec646d3e163f695c754dc0b6", - "result": "9bfff8507fe1c84fd2ed0e3917359b0e1f2cb2f45480bcb7ff821068f81e0e28" + "key": "7b6638e13d5a7771ca82970c82ff10e8e815e9ae11e022f608829ca7e6ff2a11", + "result": "bdbbe3b89be967d95eb2cfb1e116773b149d4ba521551807030627e8d57c5cca" }, "generic.json": { - "key": "1f543920892cdce08fd39d1133bb30755d138fbcae9433eea9b8a04620c35992", - "result": "f3ccf1e5e0587527cad6424a6a1af9c4a6444ed8bd53828a79b68f436199b3be" + "key": "56d22e25eec789a5b67a0e73ccd1db2edc9a1b50f23e7bc367ae26e7c1410509", + "result": "f404a37e961ab0a92b773317a65eff12caa0a2a8816d5f954a329bb967d1d638" }, "material.json": { - "key": "be21160354b26585d75bcbc18cde6879d5c9fc978312ba69be25a7d6b4b8db74", - "result": "63064631044454034a3a0699d91ac14124901e05a00a4cb23a9c47797d1c31ad" + "key": "413ce336e41f3966b370858050aa0ba522b35862338b6762d78747b9d36940c5", + "result": "246599600a2de4bae74b7b13dbc83ebf7856e1d13a1e1ee35b8274706eae2068" }, "mesh.glb": { - "key": "2ea0f78651c24712f6ac5c00cccce8ce3d45404242c425add45f2a6f14212031", - "result": "c156f4d378f410220ab3b3534f3880840a9eef585729900d0dc49c1c313e5f67" + "key": "d73226629446825538e9f52f7139470cc2ee3d06480cbf480d8cb8876b98e491", + "result": "bb4ce49483164483c3729736c92f82e5b48db83f4ed67c57890a30b976ec1ed0" }, "mono.json": { - "key": "b322bbe2a6e231bd8acd1330f48be73d2bde9727178abcb0bbc35feca8b22d75", - "result": "d8851cfc61b16159672f67907717a209f4fc1fdf84149b5b585de8d4b5130f8d" + "key": "3da938e46ad6126b00d2b4efd67416bbc3b7983dfc39ec18a92da5dae1cbd5d6", + "result": "c20cf16802c538e51cd327facc43a3401da0ca67bf5f1adaa7a3554ff1d56954" }, "prefab.json": { - "key": "dd93b4b9200a6579bdd097ef5d35975a0f9696a2506fa665cba18e15c19f1435", - "result": "5ad430bcff12dc77a6f164547ff87962f9f7b04837783e689cb24068520be62e" + "key": "c0e4016cd8ccad7ef8870c61a8a4be8271c2997683b4d868aa99a9763a5bdee9", + "result": "7512a5b00635e586c8a0e8777897b7f2287a293f39552af78a6ab8d8d6c65b69" }, "scripts.json": { - "key": "8508cf2e260e7612201b0d1d868f3ff2958ed25caac9813f5f1722c558a39166", - "result": "1ed7832da5508c45f072f24dbb8cfa00e16b938d9d90efead9e14850f6efb094" + "key": "82609d6edbc52764a5f491b6abe2e8bf1f1383336549058d588dbed6ca72f1fe", + "result": "48743bb2322d8e81d31607cab08986e7e6b7b921543b25067fac0deb875d0478" }, "shader.json": { - "key": "07cc0530d8e40e292c085f790c6a8df5aa6c9e139d054d3a348a132b789c97c2", - "result": "9e954c70fbaa039e3a820c36a2e3399af7f1772d60947912f09ac5053fae4104" + "key": "355da22724a98e77ccf12c9f7d532fd6e3c8cd23fd34ea4d3c536136a36c2bca", + "result": "5404f4143a35b8efd83ac2f09f9ab2c4c52782db76c4c9c3637afa6b7d4bc2fb" }, "sprite.png": { - "key": "f8f642cd3213a5602ebdb9cef5776276d885f029487d070d95140191a2172f41", - "result": "12a4c52632c57cb230249be878083425358e3c8b80dfa111cd8b4d864514ea50" + "key": "0ed6598b9a398b409e57f47465f5b7f5106a4ea7f6327771c6272fa4e3df4617", + "result": "50a216c09f9a3479eacd619bc857b27ee84345ac251d5b73c0a0493246646aae" }, "tex.png": { - "key": "af044eb0b8d9efecf3fd490f96c7ca1eca15074a44b7f883a5d45cbd278e513d", - "result": "f28504d49a5b6b95be8cb0b94cdd7b23151e32d31764fab72b6570acb94cddfc" + "key": "03e26506c9a6187554a6047f78e9ecded3c470632550fee0067cc992f6823077", + "result": "9ef852eca9d54b96138aab99e835507d7c27f25d921b2741610d0cd0f18f9d8c" } }, "libraries": { diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index 264ac09..79cab58 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -18,10 +18,11 @@ from nnnotes.pipeline import contract, objexport from nnnotes.pipeline.contract import IncompatibleTask, Input, Task from nnnotes.pipeline.depot import Depot -from nnnotes.pipeline.objexport import CONVERTERS, CROP, EXPORT, ExportStage, SpriteCropStage, export_atoms +from nnnotes.pipeline.objexport import CROP, EXPORT, ExportStage, SpriteCropStage from nnnotes.pipeline.stages import Env, describe, execute, registry from nnnotes.unity.atoms import Impl, astc, estimate, gltf from nnnotes.unity.atoms.reader import UnityBundle +from nnnotes.unity.convert import CONVERTERS, REASONS, USM_ASSET_SCRIPT, export_atoms, files, material, mono, prefab from tests.helpers.fakeunity import ( F32, NORMAL, @@ -544,7 +545,7 @@ def bundle_plain() -> FakeEnvironment: # a movie asset holding its USM (script 13), beside a cue sheet asset -USM_SCRIPTS = {**SCRIPTS, "CAB-scripts:13": objexport.USM_ASSET_SCRIPT} +USM_SCRIPTS = {**SCRIPTS, "CAB-scripts:13": USM_ASSET_SCRIPT} WARMUP = b"CRID" + bytes(range(256)) * 2 MOVIE_INFO = {"numAlphaStreams": 0, "width": 64, "height": 64, "codecType": 1, "numAudioStreams": 0} @@ -889,7 +890,7 @@ def test_movies_held_in_the_bundle(tmp_path): assert load(depot, doc, usm["id"]) == WARMUP and usm["content"]["ext"] == "usm" assert usm["semantics"] == {"kind": "cri.usm", "format": "usm", "facts": {"movie": "warmup"}} tree = obj(depot, doc, f"{m}#json") - assert tree["$script"] == objexport.USM_ASSET_SCRIPT and tree["movieInfo"] == MOVIE_INFO + assert tree["$script"] == USM_ASSET_SCRIPT and tree["movieInfo"] == MOVIE_INFO assert tree["references"]["RefIds"][0]["data"]["data"] == { "$blob": {"sha256": contract.sha256(WARMUP), "size": len(WARMUP), "format": "usm"} } @@ -915,7 +916,7 @@ def test_the_movie_converter_is_keyed_by_the_script_context(tmp_path): def test_acb_layouts(tmp_path): from nnnotes.formats import cri - assert objexport.acb_asset({"_cueSheetName": "x", "_chunks": []})["layout"] == "split" + assert mono.acb_asset({"_cueSheetName": "x", "_chunks": []})["layout"] == "split" t = { "m_Name": "s", "implementation": {"rid": 5}, @@ -926,18 +927,18 @@ def test_acb_layouts(tmp_path): ] }, } - assert objexport.acb_asset(t) == { + assert mono.acb_asset(t) == { "layout": "embedded", "cueSheet": "s", "ref": 1, "impl": t["references"]["RefIds"][1], } assert ( - objexport.acb_asset( + mono.acb_asset( {"implementation": {"rid": 5}, "references": {"RefIds": [{"rid": 5, "type": {"class": "Other"}}]}} ) is None - and objexport.acb_asset({"m_Name": "x"}) is None + and mono.acb_asset({"m_Name": "x"}) is None ) assert cri.embedded_acb(t, t["references"]["RefIds"][1]) == b"@UTF" with pytest.raises(NotImplementedError, match="external AWB"): @@ -950,9 +951,9 @@ def test_acb_layouts(tmp_path): def test_a_split_sheet_with_a_missing_chunk_gives_the_typetree(tmp_path, monkeypatch): depot = Depot(tmp_path / "s") - real = objexport.acb_asset + real = mono.acb_asset monkeypatch.setattr( - objexport, + mono, "acb_asset", lambda tt: ( dict(real(tt), chunks=[pptr(44), pptr(99, 1)]) if real(tt) and real(tt)["layout"] == "split" else real(tt) @@ -996,7 +997,18 @@ def test_clip_bindings_name_scripts_of_other_bundles(exported): depot, _task, doc = exported clip = obj(depot, doc, f"{oid(96)}#json") (b,) = clip["bindings"] - assert b["typeID"] == 114 and b["class"] == "Fader" and b["path"] == "" + assert b == { + "pathCrc": 0, + "attributeCrc": 77, + "typeID": 114, + "class": "MonoBehaviour", + "customType": 0, + "pptr": False, + "int": False, + "serializeReference": False, + "script": {"$ref": {"file": "CAB-scripts", "pathId": 12}}, + "scriptClass": "Assembly-CSharp|Game|Fader", + } assert census_mod.script_refs(census_doc(open_fake(b"fake:all"))) == sorted(SCRIPTS) + ["CAB-scripts:9"] @@ -1007,10 +1019,38 @@ def test_material_clip_controller(exported): assert mat["textures"]["_MainTex"]["texture"]["$ref"]["pathId"] == 10 assert mat["colors"]["_Color"]["g"] == INF and b"1e999" in load(depot, doc, f"{oid(50)}#json") clip = obj(depot, doc, f"{oid(90)}#json") - assert clip["clip"] == "clip" and clip["bindings"] == [ - {"path": "", "typeID": 4, "class": "Transform", "attribute": "m_LocalPosition", "curves": 3} + assert list(clip) == [ + "name", + "sampleRate", + "wrapMode", + "startTime", + "stopTime", + "loopTime", + "cycleOffset", + "events", + "bindings", + "streamed", + "dense", + "constant", + "pptrCurveMapping", + ] + assert clip["name"] == "clip" and clip["constant"] == [0.5, 1.0, 2.0] and clip["streamed"]["frames"] == [] + assert clip["bindings"] == [ + { + "pathCrc": 0, + "attributeCrc": 1, + "typeID": 4, + "class": "Transform", + "customType": 0, + "pptr": False, + "int": False, + "serializeReference": False, + "attribute": "m_LocalPosition", + "curves": 3, + } ] ctl = obj(depot, doc, f"{oid(92)}#json") + assert list(ctl) == ["name", "clips", "layers", "stateMachines", "parameters", "defaultValues"] assert ctl["clips"] == [{"$ref": {"file": CAB, "pathId": 90, "class": "AnimationClip", "name": "clip"}}] assert obj(depot, doc, f"{oid(95)}#json")["m_Speed"] == INF @@ -1100,12 +1140,11 @@ def test_scripts_come_from_the_context_only(tmp_path): def test_a_converter_error_gives_the_typetree(tmp_path, monkeypatch): depot = Depot(tmp_path / "s") - real = objexport.RefExporter.conv_material - def broken(self, info): + def broken(tt, ref): raise KeyError("m_SavedProperties") - monkeypatch.setattr(objexport.RefExporter, "conv_material", broken) + monkeypatch.setattr(material, "material_doc", broken) doc = run(depot, task_for(depot, "all", classes=["Material"])) item = by_object(doc)[oid(50)] assert item["status"] == "generic" and item["reason"] == { @@ -1113,16 +1152,15 @@ def broken(self, info): "message": "KeyError: 'm_SavedProperties'", } assert obj(depot, doc, f"{oid(50)}#json")["m_Name"] == "mat" - monkeypatch.setattr(objexport.RefExporter, "conv_material", real) def test_a_broken_hierarchy_gives_its_objects_one_by_one(tmp_path, monkeypatch): depot = Depot(tmp_path / "s") - def broken(self, o): + def broken(*args): raise RuntimeError("component failed") - monkeypatch.setattr(objexport.RefExporter, "component", broken) + monkeypatch.setattr(prefab, "_component", broken) doc = run(depot, task_for(depot, "all", classes=["GameObject"])) items = by_object(doc) assert {i["status"] for i in items.values()} == {"generic"} @@ -1337,9 +1375,14 @@ def test_modules_import_without_unitypy(): def test_every_reason_code_and_converter_is_documented(): doc = (ROOT / "docs" / "stages.md").read_text(encoding="utf-8") documented = set(re.findall(r"^\| `([a-z_]+(?:\.[A-Za-z_]+)+)` \|", doc, re.M)) - pipeline = ROOT / "src" / "nnnotes" / "pipeline" - sources = [pipeline / "objexport.py", *sorted((pipeline / "atoms").glob("*.py"))] - raised = set(objexport.REASONS) + src = ROOT / "src" / "nnnotes" + sources = [ + src / "pipeline" / "objexport.py", + *sorted((src / "unity" / "convert").glob("*.py")), + *sorted((src / "unity" / "atoms").glob("*.py")), + ] + assert len(sources) > 10 + raised = set(REASONS) for p in sources: text = p.read_text(encoding="utf-8") raised |= set(re.findall(r'Unsupported\(\s*"([^"]+)"', text)) @@ -1407,14 +1450,17 @@ def test_a_converter_change_without_a_version_bump_fails_its_golden(tmp_path, mo expected = contract.loads((ROOT / "tests" / "golden" / "unity.export.json").read_bytes()) if library_versions(expected["libraries"]) != expected["libraries"]: pytest.skip("golden record of other library versions") - real = objexport.RefExporter.conv_font - def changed(self, info): - ids = real(self, info) - self.staged[-1]["semantics"]["facts"]["extra"] = 1 - return ids + def changed(ctx, info): + emit = ctx.emit + ctx.emit = lambda *a, **k: emit(*a[:6], {**a[6], "extra": 1}, *a[7:], **k) + try: + files.font(ctx, info) + finally: + ctx.emit = emit - monkeypatch.setattr(objexport.RefExporter, "conv_font", changed) + simple = tuple((cls, conv, changed if conv == "font" else fn) for cls, conv, fn in objexport._SIMPLE) + monkeypatch.setattr(objexport, "_SIMPLE", simple) p = export_golden() actual = golden(p["stage"], p["fixtures"], Depot(tmp_path / "s"), p["libraries"]) problems = golden_problems(expected, actual, p["stage"]) From 133f731be68523bb83b87bc4fd1ebde8f8b71f7e Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 01:16:28 +0800 Subject: [PATCH 08/91] feat(export)!: give every object a json document and list its references Every object a converter takes has a `json` artifact or is contained in another object's. A Texture2D, Mesh, TextAsset or Font adds its typetree JSON without the bytes its other artifact holds (the pixels, the vertex, index and compressed mesh data, the stored text, the font file). A sprite's document and a prefab's are their `json`. A sprite's json holds its decoded geometry (positions, triangles, stored texture coordinates) and the render data its image was cut from: the sprite's own or its atlas' entry, with texture, rects, uvTransform, downscale and settings. Each item lists in `refs` the ids of the objects its documents reference, other bundles' objects included and the objects of its own prefab left out, so the objects reachable from an object follow from the items alone. The original layout writes a secondary artifact whose role is its extension at `.` (a texture's json at x.png.json). BREAKING CHANGE: the `meta` and `prefab` roles are `json`; the converters are tex.png/2, sprite.png/2, mesh.glb/2, prefab.json/2, data/2 and font/2. Co-Authored-By: Claude Opus 5.5 --- docs/assets.md | 3 +- docs/contracts.md | 5 +- docs/stages.md | 26 ++++++--- src/nnnotes/pipeline/contract.py | 15 ++++- src/nnnotes/pipeline/layout.py | 6 +- src/nnnotes/pipeline/objexport.py | 19 +++++-- src/nnnotes/schemas/result.schema.json | 9 +++ src/nnnotes/unity/convert/__init__.py | 19 ++++--- src/nnnotes/unity/convert/files.py | 31 ++++++++--- src/nnnotes/unity/convert/objects.py | 23 +++++++- src/nnnotes/unity/convert/prefab.py | 4 +- src/nnnotes/unity/convert/textures.py | 65 +++++++++++++++------- tests/golden/unity.export.json | 74 ++++++++++++------------- tests/pipeline/test_layout.py | 7 ++- tests/pipeline/test_objexport.py | 77 ++++++++++++++++++++++---- 15 files changed, 273 insertions(+), 110 deletions(-) diff --git a/docs/assets.md b/docs/assets.md index 1eeb842..7060085 100644 --- a/docs/assets.md +++ b/docs/assets.md @@ -147,7 +147,8 @@ no longer wanted, never touches other files, and continues an interrupted write - other objects under the same container path: `stem[].`, the Addressables sub-object form (`…/member_full[square].png`); - objects without a container path: `_bundles///~.`; -- secondary artifacts beside the primary one: `..` (`….meta.json`, `….blob.m_Data.moc3`); +- secondary artifacts beside the primary one: `..` (`….blob.m_Data.moc3`), `.` for a + role that is its extension (`….png.json`, the `json` of a texture or sprite); - a container path present in two bundles: `stem@.ext` for each; - names Windows refuses, reserved device names and trailing dots are escaped as `%XX`; paths that coincide ignoring case get `~<8 hex>` and are listed in the layout report; diff --git a/docs/contracts.md b/docs/contracts.md index affcad8..389a28b 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -109,7 +109,8 @@ One output file ([artifact.schema.json](https://github.com/MetaSekaiLab/nnnotes/ `ext` is lower-case letters and digits. The bytes are stored once, by content id, whatever their names. Roles: an object's primary artifact is the one whose role comes first in `image`, `mesh`, `audio`, `video`, `font`, -`data`, `prefab`, `json` (then by role name); its other artifacts (`meta`, `blob:`, …) are secondary. +`data`, `json` (then by role name); its other artifacts (`json` beside an image, `blob:`, …) are +secondary. ## Result: nnnotes.result/1 @@ -120,7 +121,7 @@ The result of a task, written after the bytes of all its artifacts are stored: i {"schema": "nnnotes.result/1", "key": "…", "keyParts": {…}, "status": "ok", "artifacts": [], "items": [{"object": "CAB-…:1", "class": "Texture2D", "status": "exported", "artifacts": ["CAB-…:1#image"]}, - {"object": "CAB-…:2", "class": "Transform", "status": "contained", "in": "CAB-…:9#prefab"}, + {"object": "CAB-…:2", "class": "Transform", "status": "contained", "in": "CAB-…:9#json"}, {"object": "CAB-…:3", "class": "Texture2D", "status": "unsupported", "reason": {"code": "empty.texture", "message": "…"}}]} ``` diff --git a/docs/stages.md b/docs/stages.md index bcd1f01..f0473c6 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -48,21 +48,33 @@ with its reason code; any other error of a converter gives it with `generic.erro | Class | Converter | Artifacts | Atoms | Notes | |---|---|---|---|---| -| Texture2D | `tex.png/1` | `image`: PNG of the first mip level (HDR ASTC: `.astc`) | `texture.decode`, `png.encode`, `astc.container` | further mip levels as facts; a texture without pixels: `empty.texture` | -| Sprite | `sprite.png/1` | `image`: PNG on the sprite's rect (packing undone, tight mask applied), `meta`: JSON | `texture.decode`, `sprite.crop`, `png.encode`, `mesh.arrays` | the texture is decoded once per task; a sprite whose SpriteAtlas is in another bundle gets its `meta` here (with its mesh and object facts) and its `image` from `sprite.crop` | -| Mesh | `mesh.glb/1` | `mesh`: binary glTF in the mesh's local space | `mesh.arrays`, `gltf.write` | see `gltf.write` below; limits of the skin and blend shapes as issues of the artifact | -| GameObject (a root) | `prefab.json/1` | `prefab`: the hierarchy's nodes (path, path ids, transform) with their components' fields | | the objects below it are `contained`; a root of a scene's file names its scene | +| Texture2D | `tex.png/2` | `image`: PNG of the first mip level (HDR ASTC: `.astc`), `json`: the typetree without `image data` (settings, colour space, mip count, stream data) | `texture.decode`, `png.encode`, `astc.container` | further mip levels as facts; a texture without pixels: `empty.texture` | +| Sprite | `sprite.png/2` | `image`: PNG on the sprite's rect (packing undone, tight mask applied), `json`: the typetree, geometry, render data and image (see below) | `texture.decode`, `sprite.crop`, `png.encode`, `mesh.arrays` | the texture is decoded once per task; a sprite whose SpriteAtlas is in another bundle gets its `json` here (with its geometry and object facts) and its `image` from `sprite.crop` | +| Mesh | `mesh.glb/2` | `mesh`: binary glTF in the mesh's local space, `json`: the typetree without the vertex, index and compressed mesh data | `mesh.arrays`, `gltf.write` | see `gltf.write` below; limits of the skin and blend shapes as issues of the artifact | +| GameObject (a root) | `prefab.json/2` | `json`: the hierarchy's nodes (path, path ids, transform) with their components' fields | | the objects below it are `contained`; a root of a scene's file names its scene | | Material | `material.json/1` | `json`: shader, keywords, textures, floats, colours | | | | Shader | `shader.json/1` | `json`: the parsed form (the bytes of `nnnotes export shader`'s `.json`), `program:/sp

__`: each compiled sub-program as stored (`.glsl`, `.metal`, `.vkprog`), `index`: the variants (platform, subshader, pass, stage, GPU program type, keywords, program role), `typetree`: the typetree JSON | | a shader without a parsed form: its typetree with `generic.error` | | AnimationClip | `clip.json/2` | `json`: the Mecanim clip data and its bindings as stored (see below) | | a legacy or compressed clip: `generic.clip.legacy` | | AnimatorController | `controller.json/2` | `json`: name, clips (`$ref`), layers, state machines, parameters, default values | | an AnimatorOverrideController: `generic.controller.override` | | MonoBehaviour (not in a hierarchy) | `mono.json/2`, `cri.usm/1` | `json`: `$script` (assembly, namespace, class) and the typetree; a cue sheet asset also `acb`, a movie asset holding its USM also `usm` (see below) | `sniff` | the class from the task's script table | -| TextAsset | `data/1` | `data`: the stored bytes | `sniff` | extension from the container path, else from the content | -| Font | `font/1` | `font`: the font file | `sniff` | | +| TextAsset | `data/2` | `data`: the stored bytes, `json`: the typetree without `m_Script` | `sniff` | extension from the container path, else from the content | +| Font | `font/2` | `font`: the font file, `json`: the typetree without `m_FontData` | `sniff` | | | MonoScript | `scripts.json/1` | `#scripts.json`: every MonoScript of the bundle | | | | AssetBundle | `bundle.json/1` | `#bundle.json`: serialized files, externals, the AssetBundle objects | | | | any other class | `generic.json/1` | `json`: the typetree | `sniff` | | +Every object a converter takes has a `json` artifact or is contained in another object's. Each item lists in +`refs` the objects its documents reference (by object id, other bundles' objects included, the objects of its own +prefab left out): the objects reachable from an object follow from the items alone. + +A sprite's `json` holds `sprite` (its typetree), `mesh` (its geometry decoded: `positions` in the sprite's units, +`submeshes` of triangles, `uv0` when stored), `renderData` (`source`: `renderData` for the sprite's own render +data, `atlas` for its SpriteAtlas' entry; `atlas`; and that render data's `texture`, `alphaTexture`, +`secondaryTextures`, `textureRect`, `textureRectOffset`, `atlasRectOffset`, `uvTransform`, `downscaleMultiplier`, +`settingsRaw`) and `image` (`packed`, `packingMode`, `packingRotation`, `width`, `height`). The texture coordinate +of a position `p` is `((p.x * uvTransform.x + uvTransform.y) / width, (p.y * uvTransform.z + uvTransform.w) / +height)` of the render data's texture. + A task's key includes the atoms `reader` and `sniff`, the converter `generic.json`, and the converters (with their atoms) of the classes the bundle's census lists, so a converter's version bump runs again exactly the bundles holding its classes. `classes` (a list of class names) limits a task to the objects of those classes and the objects their @@ -110,7 +122,7 @@ the same way: its `data.data` bytes, whatever their size, are the artifact `:`). Its inputs are the atlas' JSON, the `meta` of each such sprite and the images of the +subject>:`). Its inputs are the atlas' JSON, the `json` of each such sprite and the images of the atlas textures they use; it writes `#image` with the same bytes `unity.export` writes for a sprite whose atlas is in its own bundle. diff --git a/src/nnnotes/pipeline/contract.py b/src/nnnotes/pipeline/contract.py index 69eb785..26fa343 100644 --- a/src/nnnotes/pipeline/contract.py +++ b/src/nnnotes/pipeline/contract.py @@ -63,7 +63,7 @@ # locator kinds of a task input: where the bytes can be read (never part of the key) LOCATOR_KINDS = ("depot", "download", "file", "catalog") # an object's primary artifact is the one whose role comes first here; its other artifacts are secondary -PRIMARY_ROLES = ("image", "mesh", "audio", "video", "font", "data", "prefab", "json") +PRIMARY_ROLES = ("image", "mesh", "audio", "video", "font", "data", "json") _EXT = re.compile(r"[0-9a-z]+") _BUNDLE_SUFFIX = re.compile(r"(_[0-9a-f]{32})?\.bundle$") @@ -324,11 +324,18 @@ def reason(code: str, message: str = "") -> dict: def item( - obj: str, status: str, *, artifacts=(), within: str | None = None, why: dict | None = None, cls: str | None = None + obj: str, + status: str, + *, + artifacts=(), + within: str | None = None, + why: dict | None = None, + cls: str | None = None, + refs=(), ) -> dict: """The outcome for one object of a task. exported: its artifacts; contained: `within` (the artifact it is part of); generic: its artifacts and why the specialized form was not made; unsupported: why (and artifacts, if - any); failed: why.""" + any); failed: why. `refs`: the ids of the other objects its documents reference.""" if status not in ITEM_STATUSES: raise ValueError(f"item {obj}: unknown status {status!r}") need_art = status in ("exported", "generic") @@ -348,6 +355,8 @@ def item( d["artifacts"] = sorted(artifacts) if within is not None: d["in"] = within + if refs: + d["refs"] = sorted(set(refs)) if why is not None: d["reason"] = why return d diff --git a/src/nnnotes/pipeline/layout.py b/src/nnnotes/pipeline/layout.py index cc9a3c2..3fbb13d 100644 --- a/src/nnnotes/pipeline/layout.py +++ b/src/nnnotes/pipeline/layout.py @@ -11,7 +11,8 @@ `~` (stem[name]~123.png) - an object's primary artifact (contract.PRIMARY_ROLES) is written at that path, its extension appended unless the path already ends with it (x.prefab -> x.prefab.json, y.png -> y.png); its other artifacts at - `..` (':' and '/' of the role as '.') + `..` (':' and '/' of the role as '.'), `.` for a role that is its extension + (y.png -> y.png.json) - objects without a container: `//~.`, root = the root of the object's serialized file when one is given (`object_roots`: the export command roots every object of a bundle at `_bundles/`, whichever task made the artifact), else the source's root; under a root whose @@ -301,7 +302,8 @@ def _object_files(base: list[str], records: list[dict], append_ext: bool) -> lis segs = base[:-1] + [f"{base[-1]}.{ext}"] if append_ext else _with_ext(base, ext) else: role = contract.parse_artifact_id(r["id"])[1] - segs = base[:-1] + [f"{base[-1]}.{_role_segment(role)}.{ext}"] + name = f"{base[-1]}.{ext}" if role == ext else f"{base[-1]}.{_role_segment(role)}.{ext}" + segs = base[:-1] + [name] out.append((segs, _entry("", r))) return out diff --git a/src/nnnotes/pipeline/objexport.py b/src/nnnotes/pipeline/objexport.py index f17904d..182e638 100644 --- a/src/nnnotes/pipeline/objexport.py +++ b/src/nnnotes/pipeline/objexport.py @@ -130,6 +130,7 @@ def _emit( def _begin(self, conv: str) -> None: self.staged, self.conv = [], conv + self.ctx.seen.clear() def _publish(self) -> list[str]: """Keep the staged artifacts; returns their ids.""" @@ -139,10 +140,15 @@ def _publish(self) -> list[str]: self.staged = [] return ids + def _refs(self, info) -> list[str]: + """The objects the documents of `info` made since the last _begin reference.""" + return sorted(self.ctx.seen - {info.id}) + def _commit(self, info, status: str, why: dict | None = None) -> None: - """The item of `info`: the artifacts its conversion staged.""" + """The item of `info`: the artifacts its conversion staged and the objects they reference.""" + refs = self._refs(info) ids = self._publish() - self.items[info.id] = contract.item(info.id, status, artifacts=ids, why=why, cls=info.class_name) + self.items[info.id] = contract.item(info.id, status, artifacts=ids, why=why, cls=info.class_name, refs=refs) def _failed(self, info, e: Exception, prefix: str = "") -> None: self._begin("generic.json") @@ -270,7 +276,7 @@ def _prefabs(self) -> None: self._fallback(info_of[k], "generic.error", f"hierarchy at transform {root.transform}: {why}") continue if self._convert(info, "prefab.json", prefab.prefab, h, root): - within = contract.artifact_id(info.id, "prefab") + within = contract.artifact_id(info.id, "json") for k in root.members: i = info_of[k] if i.id != info.id: @@ -293,9 +299,10 @@ def _loose(self, cls: str, conv: str, kind: str) -> None: for i in infos: mark = len(self.staged) key = (i.file, i.path_id) + self.ctx.seen.clear() try: entries[i.id] = self.ctx.tree(self.ctx.readers[key], self.ctx.tt(key)) - done.append(i) + done.append((i, self._refs(i))) except Exception as e: del self.staged[mark:] failed.append((i, e)) @@ -303,8 +310,8 @@ def _loose(self, cls: str, conv: str, kind: str) -> None: doc = encode_object(generic.loose_doc(self.ctx, conv, entries)) aid = self._emit(None, conv, doc, "json", kind, "json", {"objects": len(done)}, owner=self.task.id)[0] self._publish() - for i in done: - self.items[i.id] = contract.item(i.id, "contained", within=aid, cls=i.class_name) + for i, refs in done: + self.items[i.id] = contract.item(i.id, "contained", within=aid, cls=i.class_name, refs=refs) for i, e in failed: if isinstance(e, ReadError): self._failed(i, e) diff --git a/src/nnnotes/schemas/result.schema.json b/src/nnnotes/schemas/result.schema.json index 96557fa..541d034 100644 --- a/src/nnnotes/schemas/result.schema.json +++ b/src/nnnotes/schemas/result.schema.json @@ -68,6 +68,15 @@ "in": { "$ref": "urn:nnnotes:schema:common#/$defs/artifactId" }, + "refs": { + "description": "The ids of the other objects the object's documents reference, other bundles' objects included.", + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, "reason": { "$ref": "urn:nnnotes:schema:common#/$defs/reason" } diff --git a/src/nnnotes/unity/convert/__init__.py b/src/nnnotes/unity/convert/__init__.py index 1020360..759d8ab 100644 --- a/src/nnnotes/unity/convert/__init__.py +++ b/src/nnnotes/unity/convert/__init__.py @@ -6,9 +6,9 @@ bundle holds, with the atoms they use (CONVERTER_ATOMS). Texture2D tex.png `image`: PNG of the first mip level (HDR ASTC: the .astc file) - Sprite sprite.png `image`: PNG on the sprite's rect; `meta`: JSON + Sprite sprite.png `image`: PNG on the sprite's rect; `json`: typetree, geometry, render data Mesh mesh.glb `mesh`: binary glTF in the mesh's local space - GameObject (a root) prefab.json `prefab`: the hierarchy below it with its components + GameObject (a root) prefab.json `json`: the hierarchy below it with its components Material material.json `json` Shader shader.json `json`: the parsed form as `nnnotes export shader` writes it; `program:/sp

__`: each compiled sub-program as @@ -25,6 +25,9 @@ AssetBundle bundle.json "#bundle.json": serialized files, externals, AssetBundle objects any other class generic.json `json`: the object's typetree +Every object a converter takes has a `json` artifact (or is contained in one): the converter's document, or the +typetree JSON (a Texture2D, Mesh, TextAsset or Font without the bytes its other artifact holds). + Object JSON (canon/1 in the object's own field order): - a reference is {"$ref": {"file", "pathId"}}, with "class" and "name" when the target is in the bundle; a null reference is null. Inside a prefab, a GameObject, Transform or component of the same prefab is named by its path: @@ -60,16 +63,16 @@ # converter -> version CONVERTERS = { - "tex.png": 1, - "sprite.png": 1, - "mesh.glb": 1, - "prefab.json": 1, + "tex.png": 2, + "sprite.png": 2, + "mesh.glb": 2, + "prefab.json": 2, "material.json": 1, "clip.json": 2, "controller.json": 2, "mono.json": 2, - "data": 1, - "font": 1, + "data": 2, + "font": 2, "scripts.json": 1, "bundle.json": 1, "shader.json": 1, diff --git a/src/nnnotes/unity/convert/files.py b/src/nnnotes/unity/convert/files.py index 3c0269e..fe1509b 100644 --- a/src/nnnotes/unity/convert/files.py +++ b/src/nnnotes/unity/convert/files.py @@ -1,17 +1,26 @@ -"""Mesh (mesh.glb), TextAsset (data) and Font (font).""" +"""Mesh (mesh.glb), TextAsset (data) and Font (font). Each also has `json`: its typetree JSON without the bytes its +other artifact holds.""" from __future__ import annotations from ..atoms import Unsupported -from .objects import BundleObjects, container_ext +from .objects import BundleObjects, container_ext, encode_object, omit -__all__ = ["FONT_FORMATS", "font", "mesh", "text"] +__all__ = ["FONT_FORMATS", "MESH_DATA", "font", "mesh", "text"] FONT_FORMATS = ("ttf", "otf", "ttc", "woff", "woff2") +# the fields of a Mesh whose data its glTF holds decoded: vertices, indices, the compressed form +MESH_DATA = ("m_VertexData/m_DataSize", "m_IndexBuffer", "m_CompressedMesh") + + +def _json(ctx: BundleObjects, key: tuple, tt: dict, *data: str) -> None: + doc = ctx.tree(ctx.readers[key], omit(tt, *data)) + ctx.emit(key, "json", encode_object(doc), "json", "unity.object", "json", {}) def mesh(ctx: BundleObjects, info) -> None: - """`mesh`: binary glTF in the mesh's local space; the limits of its conversion as issues.""" + """`mesh`: binary glTF in the mesh's local space, the limits of its conversion as issues; `json` without the + vertex, index and compressed mesh data (MESH_DATA).""" key = (info.file, info.path_id) m = ctx.fns["mesh.arrays"](ctx.view.mesh_input(info)) glb = ctx.fns["gltf.write"]([m]) @@ -22,25 +31,31 @@ def mesh(ctx: BundleObjects, info) -> None: "skinned": m.skin is not None, } ctx.emit(key, "mesh", glb, "glb", "mesh.gltf", "glb", facts, issues=m.issues) + _json(ctx, key, ctx.tt(key), *MESH_DATA) def text(ctx: BundleObjects, info) -> None: - """`data`: the stored bytes, the extension from the container path, else from the content.""" + """`data`: the stored bytes, the extension from the container path, else from the content; `json` without + them (`m_Script`).""" key = (info.file, info.path_id) - s = ctx.tt(key).get("m_Script", "") + tt = ctx.tt(key) + s = tt.get("m_Script", "") data = s.encode("utf-8", "surrogateescape") if isinstance(s, str) else bytes(s or b"") container = ctx.containers.get(key) sn = ctx.fns["sniff"](data, container) ext = container_ext(container) or sn.ext ctx.emit(key, "data", data, ext, "text", sn.format, {"size": len(data)}) + _json(ctx, key, tt, "m_Script") def font(ctx: BundleObjects, info) -> None: - """`font`: the font file.""" + """`font`: the font file; `json` without it (`m_FontData`).""" key = (info.file, info.path_id) - data = bytes(ctx.tt(key).get("m_FontData") or b"") + tt = ctx.tt(key) + data = bytes(tt.get("m_FontData") or b"") if not data: raise Unsupported("no_data.font", "the font has no font data") sn = ctx.fns["sniff"](data) ext = sn.ext if sn.format in FONT_FORMATS else (container_ext(ctx.containers.get(key)) or sn.ext) ctx.emit(key, "font", data, ext, "font", sn.format, {"size": len(data)}) + _json(ctx, key, tt, "m_FontData") diff --git a/src/nnnotes/unity/convert/objects.py b/src/nnnotes/unity/convert/objects.py index e92c2ce..b75a0d1 100644 --- a/src/nnnotes/unity/convert/objects.py +++ b/src/nnnotes/unity/convert/objects.py @@ -4,6 +4,9 @@ Artifacts go out through `emit(key, role, data, ext, kind, format, facts, *, issues=(), refs=(), owner=None)`, which the caller gives: it stores the bytes and records the artifact "#" (owner: the object `key`, else the task-level name `owner`) and returns (artifact id, content record). + +Every object a document names by {"$ref"} is recorded in `seen` (by object id, other bundles' objects included): +the objects an object's documents reference, which the caller clears before converting an object. """ from __future__ import annotations @@ -25,6 +28,7 @@ "encode_object", "label", "message", + "omit", "ref_object_id", ] @@ -76,6 +80,21 @@ def encode_object(doc) -> bytes: return encode(_stored(doc), sort_keys=False) +def omit(tt: dict, *paths: str) -> dict: + """`tt` without the fields at `paths` (field names joined by "/"); the dictionaries on the way are copied.""" + out = dict(tt) + for path in paths: + head, _, rest = path.partition("/") + if head not in out: + continue + if rest: + if isinstance(out[head], dict): + out[head] = omit(out[head], rest) + else: + del out[head] + return out + + def ref_object_id(v) -> str | None: """The object id of a {"$ref": {...}} value (None for null or an unresolved file).""" if not isinstance(v, dict) or "$ref" not in v or "file" not in v["$ref"]: @@ -160,6 +179,7 @@ def __init__(self, view, fns: dict, *, scripts: dict, blob_min: int, png_level: self._heads: dict[int, tuple] = {} self._captures: dict[str, Capture] = {} self._local: Callable[[tuple], dict | None] | None = None + self.seen: set[str] = set() # ---------------------------------------------------------------- identities and names @staticmethod @@ -272,8 +292,9 @@ def script_class(self, key: tuple) -> str | None: # ---------------------------------------------------------------- references and values def dref(self, key: tuple) -> dict: - """{"$ref"} of an object, with its class and name when it is in the bundle.""" + """{"$ref"} of an object, with its class and name when it is in the bundle (recorded in `seen`).""" file, pid = key + self.seen.add(object_id(file, pid)) d = {"file": label(file), "pathId": pid} r = self.readers.get(key) if r is not None: diff --git a/src/nnnotes/unity/convert/prefab.py b/src/nnnotes/unity/convert/prefab.py index 0dd4e1c..845719e 100644 --- a/src/nnnotes/unity/convert/prefab.py +++ b/src/nnnotes/unity/convert/prefab.py @@ -145,7 +145,7 @@ def _component(ctx: BundleObjects, h: Hierarchies, o, root: tuple, unresolved: l def prefab(ctx: BundleObjects, info, h: Hierarchies, root: Root) -> None: - """`prefab`: {"name", "scene" (a root of a scene's file), "nodes"}: every node of the hierarchy in hierarchy + """`json`: {"name", "scene" (a root of a scene's file), "nodes"}: every node of the hierarchy in hierarchy order (path, name, path ids, active, layer, tag, local transform, rect, components with their fields).""" key = (info.file, info.path_id) g = h.graphs[info.file] @@ -190,4 +190,4 @@ def prefab(ctx: BundleObjects, info, h: Hierarchies, root: Root) -> None: else [] ) facts = {"nodes": len(nodes), "objects": len(root.members)} - ctx.emit(key, "prefab", encode_object(doc), "json", "prefab", "json", facts, issues=issues) + ctx.emit(key, "json", encode_object(doc), "json", "prefab", "json", facts, issues=issues) diff --git a/src/nnnotes/unity/convert/textures.py b/src/nnnotes/unity/convert/textures.py index 2cb7ec3..ed05194 100644 --- a/src/nnnotes/unity/convert/textures.py +++ b/src/nnnotes/unity/convert/textures.py @@ -2,6 +2,12 @@ A texture's image is decoded once per bundle: `texture` (or `decode_into`, when the task leaves textures out) puts it in a holder that the sprites cut from it read. Where a sprite's image comes from is its plan (`sprite_plan`). + +A sprite's `json` holds its typetree JSON (`sprite`), its decoded geometry (`mesh`: positions in the sprite's units, +triangles per submesh, texture coordinates when stored; a texture coordinate of a position p is +((p.x * uvTransform.x + uvTransform.y) / width, (p.y * uvTransform.z + uvTransform.w) / height) of the render data's +texture), the render data its image was cut from (`renderData`: the sprite's own or its atlas' entry for it) and +the image (`image`). """ from __future__ import annotations @@ -10,7 +16,7 @@ from ..atoms import Unsupported from ..ids import object_id -from .objects import BundleObjects, encode_object, message, ref_object_id +from .objects import BundleObjects, encode_object, message, omit, ref_object_id __all__ = [ "atlas_entries", @@ -24,13 +30,27 @@ ] +# the fields of a sprite's render data (SpriteRenderData, or a SpriteAtlas entry) that say how it is drawn +RENDER_FIELDS = ( + "texture", + "alphaTexture", + "secondaryTextures", + "textureRect", + "textureRectOffset", + "atlasRectOffset", + "uvTransform", + "downscaleMultiplier", + "settingsRaw", +) + + def _canon(v) -> str: return json.dumps(v, sort_keys=True, ensure_ascii=True) def texture(ctx: BundleObjects, info, holder: dict) -> None: - """`image`: PNG of the first mip level (HDR ASTC: the .astc file). The decoded image (or the error) goes into - `holder` for the texture's sprites.""" + """`image`: PNG of the first mip level (HDR ASTC: the .astc file); `json`: the typetree JSON without the pixels + (`image data`). The decoded image (or the error) goes into `holder` for the texture's sprites.""" from ..atoms.texture import ASTC_BLOCKS, format_name, is_hdr_astc key = (info.file, info.path_id) @@ -54,6 +74,7 @@ def texture(ctx: BundleObjects, info, holder: dict) -> None: ) data = ctx.fns["astc.container"](inp.data, w, h, ASTC_BLOCKS[fmt]) ctx.emit(key, "image", data, "astc", "texture.image", "astc", facts) + _texture_json(ctx, key, tt) return try: img = ctx.fns["texture.decode"](inp) @@ -64,6 +85,12 @@ def texture(ctx: BundleObjects, info, holder: dict) -> None: facts["mode"] = img.mode data = ctx.fns["png.encode"](img, ctx.png_level) ctx.emit(key, "image", data, "png", "texture.image", "png", facts) + _texture_json(ctx, key, tt) + + +def _texture_json(ctx: BundleObjects, key: tuple, tt: dict) -> None: + doc = ctx.tree(ctx.readers[key], omit(tt, "image data")) + ctx.emit(key, "json", encode_object(doc), "json", "unity.object", "json", {}) def decode_into(ctx: BundleObjects, info, holder: dict) -> None: @@ -145,8 +172,8 @@ def sprite_plan(ctx: BundleObjects, key: tuple, atlases: dict) -> dict: def sprite(ctx: BundleObjects, info, plan: dict, holder: dict) -> None: - """`image`: PNG on the sprite's rect cut from its texture's image in `holder`; `meta`: the sprite's JSON and - where its image came from.""" + """`image`: PNG on the sprite's rect cut from its texture's image in `holder`; `json`: the sprite's typetree + JSON, geometry, render data and image (see the module documentation).""" from ..atoms import sprite as sprite_atom err = holder.get("error") @@ -161,7 +188,8 @@ def sprite(ctx: BundleObjects, info, plan: dict, holder: dict) -> None: alpha = ctx.fns["texture.decode"](ctx.view.texture_input(ctx.info[plan["alpha"]])) if plan["alpha"] else None raw = int(src["settingsRaw"]) st = sprite_atom.settings(raw) - mesh = ctx.fns["mesh.arrays"](ctx.view.mesh_input(info)) if st.packing_mode == sprite_atom.TIGHT else None + m = ctx.fns["mesh.arrays"](ctx.view.mesh_input(info)) + mesh = m if st.packing_mode == sprite_atom.TIGHT else None downscale = float(src.get("downscaleMultiplier", 1.0)) img = ctx.fns["sprite.crop"]( holder["image"], @@ -179,30 +207,29 @@ def sprite(ctx: BundleObjects, info, plan: dict, holder: dict) -> None: refs.append({"rel": "atlas", "object": object_id(*plan["atlasKey"])}) facts = {"width": img.width, "height": img.height, "mode": img.mode} ctx.emit(key, "image", data, "png", "sprite.image", "png", facts, refs=refs) - meta = { + owner = plan["owner"] + doc = { "sprite": ctx.tree(r, tt), - "image": { + "mesh": sprite_atom.mesh_values(m) if "position" in m.channels else None, + "renderData": { "source": "atlas" if plan["atlasKey"] is not None else "renderData", "atlas": None if plan["atlasKey"] is None else ctx.dref(plan["atlasKey"]), - "texture": ctx.dref(plan["texture"]), - "alphaTexture": None if plan["alpha"] is None else ctx.dref(plan["alpha"]), - "textureRect": src["textureRect"], - "textureRectOffset": src["textureRectOffset"], - "settingsRaw": raw, + **{k: ctx.value(owner, src[k]) for k in RENDER_FIELDS if k in src}, + }, + "image": { "packed": st.packed, "packingMode": st.packing_mode, "packingRotation": st.packing_rotation, - "downscaleMultiplier": downscale, "width": img.width, "height": img.height, }, } - ctx.emit(key, "meta", encode_object(meta), "json", "sprite.meta", "json", {}, refs=refs) + ctx.emit(key, "json", encode_object(doc), "json", "sprite.meta", "json", {}, refs=refs) def sprite_external(ctx: BundleObjects, info, plan: dict, crop: str) -> None: - """`meta` of a sprite whose SpriteAtlas is in another bundle: its JSON, mesh and object facts, for the stage - `crop` that cuts its image from the atlas texture.""" + """`json` of a sprite whose SpriteAtlas is in another bundle: its typetree JSON, geometry and object facts, for + the stage `crop` that cuts its image from the atlas texture.""" from ..atoms import sprite as sprite_atom key = (info.file, info.path_id) @@ -217,7 +244,7 @@ def sprite_external(ctx: BundleObjects, info, plan: dict, crop: str) -> None: } facts = {"crop": crop, "atlas": atlas_oid} refs = [{"rel": "atlas", "object": atlas_oid}] - ctx.emit(key, "meta", encode_object(meta), "json", "sprite.meta", "json", facts, refs=refs) + ctx.emit(key, "json", encode_object(meta), "json", "sprite.meta", "json", facts, refs=refs) # ---------------------------------------------------------------- sprites cut from an atlas of another bundle @@ -233,7 +260,7 @@ def atlas_entry(entries: list, key): def crop_meta(meta: dict, entries: list, image, crop): - """The image of a sprite whose atlas is in another bundle, from its `meta` (sprite_external), its atlas' + """The image of a sprite whose atlas is in another bundle, from its `json` (sprite_external), its atlas' render data entries and `image(object id)`, the decoded image of an atlas texture; `crop`: atom sprite.crop.""" from ..atoms import sprite as sprite_atom diff --git a/tests/golden/unity.export.json b/tests/golden/unity.export.json index 96e8707..d310ee3 100644 --- a/tests/golden/unity.export.json +++ b/tests/golden/unity.export.json @@ -5,89 +5,89 @@ "clip.json": "clip.json/2", "controller.json": "controller.json/2", "cri.usm": "cri.usm/1", - "data": "data/1", - "font": "font/1", + "data": "data/2", + "font": "font/2", "generic.json": "generic.json/1", "gltf.write": "numpy-2.4.6/1", "material.json": "material.json/1", "mesh.arrays": "numpy-2.4.6/1", - "mesh.glb": "mesh.glb/1", + "mesh.glb": "mesh.glb/2", "mono.json": "mono.json/2", "png.encode": "pillow-12.3.0/1", - "prefab.json": "prefab.json/1", + "prefab.json": "prefab.json/2", "reader": "fakeunity/1", "scripts.json": "scripts.json/1", "shader.json": "shader.json/1", "sniff": "nnnotes/1", "sprite.crop": "pillow-12.3.0+numpy-2.4.6/1", - "sprite.png": "sprite.png/1", - "tex.png": "tex.png/1", + "sprite.png": "sprite.png/2", + "tex.png": "tex.png/2", "texture.decode": "unitypy-1.25.3+texture2ddecoder-1.0.6+astc-encoder-py-0.1.12+pillow-12.3.0/1" }, "fixtures": { "all": { - "key": "d3be85097dda7b39fb780c7b8d4560dbf88e85901372cc2256c2478b9d2ba1ef", - "result": "13f069ec2e00b913f537b8434f235b1232b5f875aebe1c3f40c39ff4e3274bdc" + "key": "10585d61586a4362e7e5a3c8857a9b583fac363abb5874e760d767fde9cb7d30", + "result": "3bd7b57ee922de132411cb8d027a00421fed5c5a595cdb3eef3e3cb5982b7374" }, "bundle.json": { - "key": "c81a9154d13d953e581bbd4cffe0590ad296a296ff6eeda9398aa8d394b44325", - "result": "cbaaf45561972f2004fa74352dc3adcc96c2baabadb56d85a49add792921fe9c" + "key": "fb99b80d44ff61bf4b90ea87e40a632c0f34372ddebd60dd00f74368c5f6b94d", + "result": "169b9bab48a8cb6423de878430b6291bf56999a5d640e9893dc2a1eeda9b93f9" }, "clip.json": { - "key": "07f85633bae3127fe11950188284b9325066133b4d86eaf61c3414b3c622fa77", - "result": "f133a83bce862cef316e2ea2907790b073eb7928911150bed8225529534df177" + "key": "535034c7ce7875a3ea19ebadbb4d12bca912621d38982cd47097cb1c66ff62c5", + "result": "fad3849534a5ad3892f968f567c88efa426088232df6e26a8f646757592fa135" }, "controller.json": { - "key": "0488d21f6759dc95639a3d1e643980472452fbd4b04c3d5623942a1095b5e4a5", - "result": "c1ca4ec4b97bb18b24a8effc3a0faa4f5dc5a6e3724eeba8368dd0505f59235a" + "key": "522f6cefe4102041b197c0feda95de6668a7e88ab548b0d682ef26b492079fd1", + "result": "f17bc2432ad36b87f17e3fffec59eab429a803d23f0bc63db018ba2d99a847db" }, "cri.usm": { "key": "8d1d45652a6a631253b3bad7f096af988fcc3894831a8fe5e7ec90b939bc3427", - "result": "ca67e2f4878b562b24cb6ca374c469ee2085a2c0259245adcae05cba4f9095b1" + "result": "9a0ca28ce190fc8b5c03fc108b657825c1361a05b416894d2f9c55402cae62fd" }, "data": { - "key": "62b78ee012b053bfb1ee11245b07cc58428f13bd0fb1b5de221b7b0a22372654", - "result": "7ca1098d52be75092183689a236f8d0b876edd68684e32b691364bad93ca89bd" + "key": "98f3c28baedd007e88d9a48adccdad2a1970e63df6e487917cf1c6823feb651d", + "result": "a2c0972bc1c388038dedb9b70f4a8a6d67b58dfca0faac3c438610df125c7234" }, "font": { - "key": "7b6638e13d5a7771ca82970c82ff10e8e815e9ae11e022f608829ca7e6ff2a11", - "result": "bdbbe3b89be967d95eb2cfb1e116773b149d4ba521551807030627e8d57c5cca" + "key": "c1a50c8ae58a104313d93a7296464812f98c82d8c9aa516589b9d6e5cb89bffa", + "result": "d18d119de09ed000abca438357e72951a18d576c2b26b76932adf9621f67d2bd" }, "generic.json": { - "key": "56d22e25eec789a5b67a0e73ccd1db2edc9a1b50f23e7bc367ae26e7c1410509", - "result": "f404a37e961ab0a92b773317a65eff12caa0a2a8816d5f954a329bb967d1d638" + "key": "3f111bf015cc23af61fb457ad94be8c3479259d2e8ec2388390580161179afde", + "result": "a79ea2404c179705d54adb9c08f917acedd070bffccc01ba72da3f68af522cd4" }, "material.json": { - "key": "413ce336e41f3966b370858050aa0ba522b35862338b6762d78747b9d36940c5", - "result": "246599600a2de4bae74b7b13dbc83ebf7856e1d13a1e1ee35b8274706eae2068" + "key": "fd0603efb48c4582fa8664efcc44be7e238dc6168846dd5528ac6ec350732903", + "result": "373545644b49d9047bae346a78186d7bd0b1b5158d17a393b42050e505209e32" }, "mesh.glb": { - "key": "d73226629446825538e9f52f7139470cc2ee3d06480cbf480d8cb8876b98e491", - "result": "bb4ce49483164483c3729736c92f82e5b48db83f4ed67c57890a30b976ec1ed0" + "key": "d14c16a5c18145cc039a4568b4c8561b0120644c2de1e1abd1dd1be1f17917f0", + "result": "650489a1a9d4e52ae66bc44a101ad5c127e9aa836c8d69e4c08c662e70c73b47" }, "mono.json": { - "key": "3da938e46ad6126b00d2b4efd67416bbc3b7983dfc39ec18a92da5dae1cbd5d6", - "result": "c20cf16802c538e51cd327facc43a3401da0ca67bf5f1adaa7a3554ff1d56954" + "key": "5820ec6ca51a697d9ad2f3ed1ee5c8a849fd8f813b053f335b86b1b95082abd8", + "result": "cdabce4d56148f552c87738a9208fd36d0cc581590d7e5eb92f8e5b522de2266" }, "prefab.json": { - "key": "c0e4016cd8ccad7ef8870c61a8a4be8271c2997683b4d868aa99a9763a5bdee9", - "result": "7512a5b00635e586c8a0e8777897b7f2287a293f39552af78a6ab8d8d6c65b69" + "key": "8994f1ef92388d443797f11c65a61a6346acf01ff0dc8bfbd36dc2fb2d2d23d2", + "result": "368986bcb1df65ea7fb906e01fe49b814c35e41212a079cdfcc5b70d5ed06559" }, "scripts.json": { - "key": "82609d6edbc52764a5f491b6abe2e8bf1f1383336549058d588dbed6ca72f1fe", - "result": "48743bb2322d8e81d31607cab08986e7e6b7b921543b25067fac0deb875d0478" + "key": "aafda4c0fe111458f3b48e85b8f60d0e9234dd250590591966a8ac3305d99ef9", + "result": "a2644c8fe5f8198b3771374f10ad01b91a9b111d74aff79b85a9b9b2e071e2c5" }, "shader.json": { - "key": "355da22724a98e77ccf12c9f7d532fd6e3c8cd23fd34ea4d3c536136a36c2bca", - "result": "5404f4143a35b8efd83ac2f09f9ab2c4c52782db76c4c9c3637afa6b7d4bc2fb" + "key": "a71aa982ce0da9929eaf721a1db903ac0565e1232c94a1d451cdf818f2c59f31", + "result": "fef75615b20f6dc056a4934606d1f094159e219b87482b5efae95c407d0f6d55" }, "sprite.png": { - "key": "0ed6598b9a398b409e57f47465f5b7f5106a4ea7f6327771c6272fa4e3df4617", - "result": "50a216c09f9a3479eacd619bc857b27ee84345ac251d5b73c0a0493246646aae" + "key": "d42d38c45711b2de94fc4509b0dd8d919d01d123091f09d54e177df7e4cf751d", + "result": "5c6d4968cbc56a1eefb9c8c94b380487f9876f591e33acbef40bb735edbb70b9" }, "tex.png": { - "key": "03e26506c9a6187554a6047f78e9ecded3c470632550fee0067cc992f6823077", - "result": "9ef852eca9d54b96138aab99e835507d7c27f25d921b2741610d0cd0f18f9d8c" + "key": "a5af40387bb2c9fe6ba31415af4e4028163eab23bd4114ad4384b57159ae1318", + "result": "54d91ffc00400f162f677f9e1525e34698b33dc439560b4372dffcd09a8dd73b" } }, "libraries": { diff --git a/tests/pipeline/test_layout.py b/tests/pipeline/test_layout.py index b4a8655..7ef6086 100644 --- a/tests/pipeline/test_layout.py +++ b/tests/pipeline/test_layout.py @@ -72,7 +72,7 @@ def test_objects_of_one_container(tmp_path): "Sprite", "square", "assets/x/member_full.png", - {"image": png("s1"), "meta": (b"{}", "json")}, + {"image": png("s1"), "json": (b"{}", "json"), "meta": (b"[]", "json")}, ), ("CAB-1", 12, "Sprite", "member_full", "assets/x/member_full.png", {"image": png("s2")}), ("CAB-1", 21, "Sprite", "dup", "assets/x/member_full.png", {"image": png("d1")}), @@ -83,7 +83,7 @@ def test_objects_of_one_container(tmp_path): "GameObject", "foo", "assets/p/foo.prefab", - {"prefab": (b"{}\n", "json"), "blob:m_Data/Array": (b"MOC3", "moc3")}, + {"json": (b"{}\n", "json"), "blob:m_Data/Array": (b"MOC3", "moc3")}, ), ("CAB-1", 40, "Texture2D", "hdr", "assets/x/hdr.png", {"image": (b"\x13\xab\xa1\x5c", "astc")}), ], @@ -92,11 +92,12 @@ def test_objects_of_one_container(tmp_path): assert p == { "CAB-1:10#image": "assets/x/member_full.png", "CAB-1:11#image": "assets/x/member_full[square].png", + "CAB-1:11#json": "assets/x/member_full[square].png.json", "CAB-1:11#meta": "assets/x/member_full[square].png.meta.json", "CAB-1:12#image": "assets/x/member_full[member_full].png", "CAB-1:21#image": "assets/x/member_full[dup]~21.png", "CAB-1:22#image": "assets/x/member_full[dup]~22.png", - "CAB-1:30#prefab": "assets/p/foo.prefab.json", + "CAB-1:30#json": "assets/p/foo.prefab.json", "CAB-1:30#blob:m_Data/Array": "assets/p/foo.prefab.blob.m_Data.Array.moc3", "CAB-1:40#image": "assets/x/hdr.png.astc", } diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index 79cab58..ddee38c 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -669,7 +669,7 @@ def test_every_object_has_exactly_one_item(exported): oid(74): ("exported", None), oid(51): ("exported", None), } - assert items[oid(61)]["in"] == f"{oid(60)}#prefab" and items[oid(80)]["in"] == "unity.export:all#scripts.json" + assert items[oid(61)]["in"] == f"{oid(60)}#json" and items[oid(80)]["in"] == "unity.export:all#scripts.json" assert items[oid(1)]["in"] == "unity.export:all#bundle.json" assert items[oid(94)]["artifacts"] == [f"{oid(94)}#json"] assert doc["status"] == "ok" and all(i["class"] for i in doc["items"]) @@ -702,7 +702,7 @@ def test_provenance_facts_and_atoms(exported): assert a["provenance"]["inputs"] == [{"role": "bundle", "sha256": task.inputs[0].sha256}] g = art(doc, f"{oid(95)}#json") assert set(g["provenance"]["atoms"]) == {"reader", "sniff", "generic.json"} - assert task.atoms["tex.png"] == "tex.png/1" and task.atoms["reader"] == "fakeunity/1" + assert task.atoms["tex.png"] == "tex.png/2" and task.atoms["reader"] == "fakeunity/1" # ---------------------------------------------------------------- converters @@ -756,11 +756,27 @@ def test_sprites_same_bundle(exported): canvas={"rect": tt["m_Rect"], "offset": src["textureRectOffset"], "pivot": tt["m_Pivot"]}, ) assert load(depot, doc, f"{oid(pid)}#image") == png.encode(want) - meta = obj(depot, doc, f"{oid(pid)}#meta") - assert meta["image"]["texture"] == {"$ref": {"file": CAB, "pathId": 10, "class": "Texture2D", "name": "tex"}} + meta = obj(depot, doc, f"{oid(pid)}#json") + rd = meta["renderData"] + assert rd["texture"] == {"$ref": {"file": CAB, "pathId": 10, "class": "Texture2D", "name": "tex"}} + assert rd["textureRect"] == src["textureRect"] and rd["uvTransform"] == src["uvTransform"] + assert rd["settingsRaw"] == src["settingsRaw"] and rd["alphaTexture"] is None assert meta["sprite"]["m_Name"] == tt["m_Name"] - meta = obj(depot, doc, f"{oid(12)}#meta") - assert meta["image"]["source"] == "atlas" and meta["image"]["atlas"]["$ref"]["pathId"] == 20 + pos, idx = _sprite_mesh(tt["m_Rect"]) + assert meta["mesh"]["positions"] == np.asarray(pos, np.float32).astype(float).tolist() + assert meta["mesh"]["submeshes"] == [np.asarray(idx).reshape(-1, 3).tolist()] + assert meta["image"] == { + "packed": bool(src["settingsRaw"] & 1), + "packingMode": (src["settingsRaw"] >> 1) & 1, + "packingRotation": (src["settingsRaw"] >> 2) & 0xF, + "width": int(tt["m_Rect"]["width"]), + "height": int(tt["m_Rect"]["height"]), + } + assert by_object(doc)[oid(pid)]["refs"] == [oid(10)] + ([oid(20)] if pid == 12 else []) + meta = obj(depot, doc, f"{oid(11)}#json") + assert meta["renderData"]["source"] == "renderData" and meta["renderData"]["atlas"] is None + meta = obj(depot, doc, f"{oid(12)}#json") + assert meta["renderData"]["source"] == "atlas" and meta["renderData"]["atlas"]["$ref"]["pathId"] == 20 assert meta["sprite"]["m_RD"]["m_VertexData"]["m_DataSize"]["$hex"] # small: inline refs = art(doc, f"{oid(12)}#image")["semantics"]["refs"] assert refs == [{"rel": "texture", "object": oid(10)}, {"rel": "atlas", "object": oid(20)}] @@ -768,8 +784,8 @@ def test_sprites_same_bundle(exported): def test_sprite_with_its_atlas_elsewhere_gets_a_meta_only(exported): depot, _task, doc = exported - assert by_object(doc)[oid(13)]["artifacts"] == [f"{oid(13)}#meta"] - a = art(doc, f"{oid(13)}#meta") + assert by_object(doc)[oid(13)]["artifacts"] == [f"{oid(13)}#json"] + a = art(doc, f"{oid(13)}#json") assert a["semantics"]["facts"] == {"crop": CROP, "atlas": f"{ATLAS_CAB}:99"} meta = obj(depot, doc, a["id"]) assert meta["image"] == {"source": "atlas", "atlas": {"$ref": {"file": ATLAS_CAB, "pathId": 99}}, "stage": CROP} @@ -801,9 +817,48 @@ def test_text_and_fonts(exported): assert load(depot, doc, f["id"]) == _ttf() and f["content"]["ext"] == "ttf" +def test_every_converted_object_has_its_json(exported): + depot, _task, doc = exported + tex = obj(depot, doc, f"{oid(10)}#json") + assert "image data" not in tex and tex["m_Width"] == W and tex["m_TextureFormat"] == 4 + assert art(doc, f"{oid(10)}#json")["semantics"] == {"kind": "unity.object", "format": "json", "facts": {}} + assert obj(depot, doc, f"{oid(15)}#json")["m_MipCount"] == 2 # HDR ASTC: the .astc image and the json + mesh = obj(depot, doc, f"{oid(30)}#json") + assert "m_DataSize" not in mesh["m_VertexData"] and "m_IndexBuffer" not in mesh + assert mesh["m_VertexData"]["m_VertexCount"] == 3 and mesh["m_Name"] == "tri" + text = obj(depot, doc, f"{oid(40)}#json") + assert text == {"m_Name": "readme"} + font = obj(depot, doc, f"{oid(42)}#json") + assert font == {"m_Name": "font"} + items = by_object(doc) + for pid in (10, 11, 12, 15, 30, 40, 41, 42, 44, 45, 50, 60, 70, 73, 74, 90, 92, 95, 96): + assert f"{oid(pid)}#json" in items[oid(pid)]["artifacts"], pid + assert all( + i["status"] != "exported" or any(a.endswith("#json") for a in i["artifacts"]) or i["object"] == oid(13) + for i in doc["items"] + if i["class"] not in ("AssetBundle", "MonoScript") + ) + + +def test_items_name_the_objects_they_reference(exported): + _depot, _task, doc = exported + items = by_object(doc) + assert items[oid(50)]["refs"] == [oid(10), f"{ATLAS_CAB}:5"] # the material's shader and texture + # a prefab: the objects its components reference outside it, other bundles' included; its own objects are named + # by their paths + assert items[oid(60)]["refs"] == [oid(10), "CAB-scripts:3", "CAB-scripts:7", "CAB-scripts:8"] + assert "refs" not in items[oid(61)] and "refs" not in items[oid(63)] # contained in the prefab + assert items[oid(1)]["refs"] == [oid(10), oid(11), oid(40), oid(60)] # the AssetBundle's container + assert items[oid(92)]["refs"] == [oid(90)] # the controller's clip + assert items[oid(96)]["refs"] == ["CAB-scripts:12"] # a clip binding's script + assert items[oid(13)]["refs"] == [f"{ATLAS_CAB}:99"] # a sprite's atlas in another bundle + assert items[oid(70)]["refs"] == [oid(80)] # its script + assert "refs" not in items[oid(30)] and "refs" not in items[oid(40)] + + def test_prefab(exported): depot, _task, doc = exported - p = obj(depot, doc, f"{oid(60)}#prefab") + p = obj(depot, doc, f"{oid(60)}#json") assert p["name"] == "root" and [n["path"] for n in p["nodes"]] == ["root", "root/child"] root, child = p["nodes"] assert root["pathId"] == 60 and root["transformPathId"] == 61 and root["layer"] == 5 @@ -814,9 +869,9 @@ def test_prefab(exported): assert fields["tex"] == {"$ref": {"file": CAB, "pathId": 10, "class": "Texture2D", "name": "tex"}} assert fields["m_Script"] == {"$ref": {"file": "CAB-scripts", "pathId": 7}} assert math.isnan(fields["value"]) and fields["small"] == {"$hex": "616263"} - assert b'{"$float": "nan"}' in load(depot, doc, f"{oid(60)}#prefab").replace(b"\n", b"").replace( + assert b'{"$float": "nan"}' in load(depot, doc, f"{oid(60)}#json").replace(b"\n", b"").replace( b" ", b"" - ) or b'"$float"' in load(depot, doc, f"{oid(60)}#prefab") + ) or b'"$float"' in load(depot, doc, f"{oid(60)}#json") (c,) = child["components"] assert c["fields"]["up"] == {"transform": "root"} assert c["fields"]["comp"] == {"component": "MonoBehaviour", "gameObject": "root", "class": "Behaviour"} From 7fd3f0e4e60a413fb5bf02f7f6adaee12341ac60 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 02:19:29 +0800 Subject: [PATCH 09/91] feat(export)!: census and export the APK's built-in Unity files With `[paths] apk` set, the player's own Unity files under `assets/bin/Data/` of the APK are subjects of unity.census and unity.export: `apk/data.unity3d`, `apk/datapack.unity3d` and `apk/Resources/unity default resources`. The census always covers them; everything and `--select bundle:apk/*` export them. A serialized file outside a bundle is loaded under the name references into it use, so object ids such as `unity_builtin_extra:12` and `unity default resources:10` name objects these tasks export, and link.addresses maps their files to their subjects. These files store no type trees. MonoBehaviours are read with the type trees nnnotes ships for the game's classes (nnnotes.unity.scripttypes), checked against the stored type hash and the object's size; the export task's context carries the trees its objects use. A MonoBehaviour without one gives its header with `unsupported.typetree`, and PlayerSettings gives its fields up to `m_ActiveColorSpace`, the part its type tree reads right. generic.json is version 2. The CRI stages take their `boot` input from the built-in inputs. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 2 +- docs/configuration.md | 4 +- docs/stages.md | 38 ++++++- scripts/gen_typetrees.py | 14 +-- src/nnnotes/cli/assets.py | 44 +++++--- src/nnnotes/pipeline/builtins.py | 71 +++++++++++++ src/nnnotes/pipeline/census.py | 20 ++-- src/nnnotes/pipeline/cristages.py | 33 ++---- src/nnnotes/pipeline/link.py | 47 +++++---- src/nnnotes/pipeline/objexport.py | 62 +++++++++-- src/nnnotes/unity/atoms/reader.py | 18 +++- src/nnnotes/unity/convert/__init__.py | 7 +- src/nnnotes/unity/convert/generic.py | 8 +- src/nnnotes/unity/convert/objects.py | 141 +++++++++++++++++++++----- src/nnnotes/unity/convert/prefab.py | 17 ++-- src/nnnotes/unity/player.py | 56 +--------- src/nnnotes/unity/reader.py | 22 ++-- src/nnnotes/unity/scripttypes.py | 98 ++++++++++++++++++ tests/cli/test_assets.py | 85 ++++++++++++++-- tests/golden/unity.export.json | 66 ++++++------ tests/helpers/fakeunity.py | 31 +++++- tests/pipeline/test_builtins.py | 47 +++++++++ tests/pipeline/test_census.py | 11 +- tests/pipeline/test_cristages.py | 34 +++---- tests/pipeline/test_objexport.py | 128 +++++++++++++++++++++-- tests/unity/test_typetrees.py | 13 +-- 26 files changed, 844 insertions(+), 273 deletions(-) create mode 100644 src/nnnotes/pipeline/builtins.py create mode 100644 src/nnnotes/unity/scripttypes.py create mode 100644 tests/pipeline/test_builtins.py diff --git a/docs/architecture.md b/docs/architecture.md index cd89197..c55d9c0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -12,7 +12,7 @@ subpackage of `nnnotes` and imports only the layers below it. | Products | `nnnotes.site`, `nnnotes.musicdata` | the ournotes-player site (charts, models, stories, indexes); the music data file and its deck statistics | | Pipeline | `nnnotes.pipeline` | stages, task descriptions and results, the content-addressed depot, plans, layouts, the catalog versions, the views of the master data | | Content | `nnnotes.content` | the game's subjects as exports: ADV episodes and stories, lives and scores, spots, rooms, the UI library | -| Unity objects | `nnnotes.unity` | bundle reading, object ids, the atoms (the pure functions objects are converted with: texture decoding, sprite images, mesh arrays, glTF, PNG, file sniffing), the converters (one per Unity class, `nnnotes.unity.convert`), the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | +| Unity objects | `nnnotes.unity` | bundle reading, object ids, the atoms (the pure functions objects are converted with: texture decoding, sprite images, mesh arrays, glTF, PNG, file sniffing), the converters (one per Unity class, `nnnotes.unity.convert`), the type trees shipped for the player's script classes (`nnnotes.unity.scripttypes`), the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | | File formats | `nnnotes.formats` | formats read from bytes: CRI ACB/AWB and HCA, USM movies, CRI Lips data, shaders, motions | | Sources | `nnnotes.source` | the configuration, the APK, Addressables catalogs and bundles, the game API, master data, languages, `Session` | | Core | `nnnotes.core` | the one JSON writer, canonical JSON and its hashes, media types, the process caches and the downloads cache | diff --git a/docs/configuration.md b/docs/configuration.md index 532e9b9..9aedab4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -103,7 +103,7 @@ exist when they are given. | `[paths] catalog` | `NNNOTES_PATHS_CATALOG` | `--catalog` | a catalog `.bin` file to read instead of discovering/downloading a catalog; explicit files stay usable offline | | `[paths] depot` | `NNNOTES_PATHS_DEPOT` | `--depot` | depot directory (created when missing): the downloaded game files, the results of the stages, catalog versions and runs ([commands.md](commands.md#downloads), [contracts.md](contracts.md#depot)) | | `[paths] master` | `NNNOTES_PATHS_MASTER` | `--master` | decoded master data directory, one `

.json` per table; the flag overrides `[servers.] master` | -| `[paths] apk` | `NNNOTES_PATHS_APK` | `--apk` | the game's `base.apk` (the story export reads the CRI Lips library from it, or from `split_config.arm64_v8a.apk` next to it) | +| `[paths] apk` | `NNNOTES_PATHS_APK` | `--apk` | the game's `base.apk` (the story export reads the CRI Lips library from it, or from `split_config.arm64_v8a.apk` next to it; the asset export its built-in Unity files) | | `[paths] ffmpeg` | `NNNOTES_PATHS_FFMPEG` | `--ffmpeg` | `ffmpeg` executable; unset: `ffmpeg` on `PATH` | | `[paths] vgmstream` | `NNNOTES_PATHS_VGMSTREAM` | `--vgmstream` | `vgmstream-cli` executable; unset: `vgmstream-cli` on `PATH` | | `[paths] node` | `NNNOTES_PATHS_NODE` | `--node` | Node.js executable; unset: `node` on `PATH` | @@ -183,7 +183,7 @@ story text); it is not a setting. | `site build --live2d` / `--all-live2d` | catalog (bundles from the CDN of the site's first region), `[paths] apk`, `[paths] player`; not `node`; master data only for the model names (optional: without it `models.json` has no names) | | `site build --player-only` / `--reingest-json` | `[paths] player` | | `music-data` | catalog, `[master] key` + `iv` (not with `--decoded-master`); `--master-files` also `[catalog] region`; `--apk-master` also `[paths] apk`; `--decoded-master` also `[catalog] region` and master data with its `MasterManifest.json`; `[paths] apk` (optional otherwise) for the client version | -| `export assets`, `depot plan` | catalog (bundles are fetched into the depot); `[paths] apk` for the bundles inside the APK (without it they are reported as `source.absent`); master data for `--views`; with `--stored-catalog` a catalog version of the depot instead of the current catalog | +| `export assets`, `depot plan` | catalog (bundles are fetched into the depot); `[paths] apk` for the bundles inside the APK (without it they are reported as `source.absent`) and its built-in Unity files ([stages.md](stages.md#the-apks-built-in-files)); master data for `--views`; with `--stored-catalog` a catalog version of the depot instead of the current catalog | | `depot run` | the depot; `--fetch` also what fetching needs (region, `cdn`, bundle key, `[paths] apk`) | | `catalog versions` / `import` / `diff`, `depot verify` | the depot; `import` reads the APK's catalog from `[paths] apk` when it is set | | `catalog fetch` | the depot, `[catalog] region` and `language`, that region's `cdn`; its `api` (optional) for the resource version label | diff --git a/docs/stages.md b/docs/stages.md index f0473c6..c6746a8 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -19,15 +19,15 @@ gzip input and Japanese remote asset placeholders). | Stage | Subject | Inputs | Outputs | |---|---|---|---| | `catalog.index` | a catalog (remote + APK) | the catalog files | every location (with the bundle hash, crc and size), bundles and raw remote files | -| `unity.census` | a bundle | the bundle | serialized files, externals, objects (file, path id, class, size, type hash, name, script, the scripts an AnimationClip's bindings name), container, scene hashes, dependencies, script fields | +| `unity.census` | a bundle, or a built-in file of the APK | the bundle | serialized files, externals, objects (file, path id, class, size, type hash, name, script, the scripts an AnimationClip's bindings name), container, scene hashes, dependencies, script fields | | `link.scripts` | global | the censuses holding MonoScripts | serialized file and path id -> assembly, namespace, class | -| `unity.export` | a bundle | the bundle, the script entries its MonoBehaviours use | the objects' artifacts and one item per object | +| `unity.export` | a bundle, or a built-in file of the APK | the bundle, the script entries its MonoBehaviours use (and, for files without type trees, the type trees nnnotes ships for their classes) | the objects' artifacts and one item per object | | `sprite.crop` | a SpriteAtlas | the atlas' JSON, the images of its textures, the metas of the sprites of other bundles it packs | the sprites' images | | `cri.audio` | ACB content | `acb` (a raw file of the catalog, or the `acb` artifact of a cue sheet held in a bundle), `awb` (an AWB the catalog pairs with it), `boot` (the game's boot data: the key's source) | FLAC per stream, `cues.json`, `streams.json` | | `cri.movie` | USM content | `usm` (a raw file of the catalog, or the `usm` artifact of a movie held in a bundle), `boot` | every stream as stored (video, alpha, audio, subtitles), `movie.mkv` (or `movie.webm`) | | `live2d.model` | a Live2D model key (`Character/Live2D///model/`) whose bundle is selected | `bundle:`: the bundles of the key's dependency closure in the order the extractor loads them (the order is part of the key); context `internalId` (the key's location); atom `live2d.extract_model` (the extractor of `nnnotes export live2d`, run unchanged) | `live2d.model:#`: every file `nnnotes export live2d` writes for the model, `` relative to the model's directory; placed under `/` | | `spine.skeleton` | a SkeletonDataAsset of a selected bundle, by its stable address `:` (in a bundle of several serialized files the file's tag follows the bundle name, as in `link.addresses`) | `bundle:`: the bundles holding the serialized files the asset's file reaches through its externals; context `object` (the asset's object id); atom `spot.write_skeleton` (the Spine writer of `nnnotes export spot`) | `spine.skeleton:#`: the skeleton (`.json` or `.skel`), its atlases (`.atlas`) and their page images, the bytes `nnnotes export spot` writes into `spine/`; one item for the asset | -| `link.addresses` | global | catalog index, censuses | key -> objects, serialized file -> bundle, bundle -> keys | +| `link.addresses` | global | catalog index, censuses (the built-in files' too) | key -> objects, serialized file -> bundle or built-in file, bundle -> keys | | `link.artifacts` | global | the above, the results | object id -> artifact ids | | `view.` | a catalog (`main`) | `master:
` per master table it reads, `rules` (the view's rule and the resolvers it uses), `addresses` (the part of the address table it read: the keys looked up and the prefixes listed); no parameters, atoms or context | `view.:#view` (facts: rows, entries, gaps, unreferenced) | | `view.voices` | a catalog (`main`) | `master:
` per master table its rules read, `rules`, `addresses`, `sheets` (each cue sheet the `cri.audio` results are known by that a voice names or the rules' prefixes list: its task and ACB content id), `acb:` (the ACB of each decoded one), `episodes` (per story episode and part, the `unity.export` artifact holding the object its key names), `adv:` (those contents); context `audio` (the keys of those `cri.audio` results); runs after `unity.export` and `cri.audio` | `view.voices:#view` (facts: rows, entries, gaps, unreferenced streams) | @@ -61,7 +61,7 @@ with its reason code; any other error of a converter gives it with `generic.erro | Font | `font/2` | `font`: the font file, `json`: the typetree without `m_FontData` | `sniff` | | | MonoScript | `scripts.json/1` | `#scripts.json`: every MonoScript of the bundle | | | | AssetBundle | `bundle.json/1` | `#bundle.json`: serialized files, externals, the AssetBundle objects | | | -| any other class | `generic.json/1` | `json`: the typetree | `sniff` | | +| any other class | `generic.json/2` | `json`: the typetree | `sniff` | an object that can be read only in part (see below): that part, `unsupported.typetree` | Every object a converter takes has a `json` artifact or is contained in another object's. Each item lists in `refs` the objects its documents reference (by object id, other bundles' objects included, the objects of its own @@ -75,6 +75,32 @@ data, `atlas` for its SpriteAtlas' entry; `atlas`; and that render data's `textu of a position `p` is `((p.x * uvTransform.x + uvTransform.y) / width, (p.y * uvTransform.z + uvTransform.w) / height)` of the render data's texture. +### The APK's built-in files + +With `[paths] apk` set, the player's own Unity files under `assets/bin/Data/` of the APK are subjects of +`unity.census` and `unity.export` like the bundles, named `apk/`: + +| Subject | Holds | +|---|---| +| `apk/data.unity3d` | `globalgamemanagers` (player, graphics and quality settings, the resource table), `globalgamemanagers.assets` (render pipeline assets, renderer data and features, post-process data, shaders), `level0`, `resources.assets`, `sharedassets0.assets`, `unity_builtin_extra` (built-in shaders) | +| `apk/datapack.unity3d` | `resources.assets` in split builds | +| `apk/Resources/unity default resources` | the serialized file of the engine's default meshes, textures, shaders and font | + +Their objects have the ids of any serialized file's (`unity_builtin_extra:12`, `unity default resources:10`), so a +bundle's reference into them names an object these tasks export, and `link.addresses` maps their files to their +subjects. Census always covers them; everything and `--select bundle:apk/*` export them. + +These files store no type trees. Objects of the engine's classes are read with the type trees UnityPy has for the +file's Unity version; MonoBehaviours with the type trees nnnotes ships for the game's classes it reads there +(`nnnotes.unity.scripttypes`, generated by `scripts/gen_typetrees.py`): the render pipeline assets, renderer data, +renderer features, post-process data and the game's settings objects. A shipped type tree is used only when the type +hash the file stores is the one it was made for and its read consumes exactly the object's bytes; the task's +context holds the shipped type trees its MonoBehaviours use, so a new shipped tree runs those tasks again. A +MonoBehaviour without one is `unsupported.typetree` with its header (`m_GameObject`, `m_Enabled`, `m_Script`, +`m_Name`) as its typetree JSON; inside a hierarchy its component keeps that header and the prefab records the +issue. PlayerSettings, whose type tree reads the stored layout right up to `m_ActiveColorSpace` (by its stored +type hash), gives the fields up to it. + A task's key includes the atoms `reader` and `sniff`, the converter `generic.json`, and the converters (with their atoms) of the classes the bundle's census lists, so a converter's version bump runs again exactly the bundles holding its classes. `classes` (a list of class names) limits a task to the objects of those classes and the objects their @@ -153,7 +179,8 @@ and multi-audio paths are tested on synthetic movies only. The ids of the external tools are `-+sha256./` (the version as `vgmstream-cli -V` / `ffmpeg -version` report it): another build of a tool runs the tasks that use it again, and a task described for another build is refused as incompatible. The tools are `[paths] vgmstream` and `[paths] ffmpeg`, -else found on PATH. The `boot` input is the APK's `assets/bin/Data/data.unity3d` (`[paths] apk`), stored once. +else found on PATH. The `boot` input is the APK's `assets/bin/Data/data.unity3d` (`[paths] apk`), the input of +the `apk/data.unity3d` tasks. ## Atoms @@ -242,6 +269,7 @@ The codes of items that are `generic`, `unsupported` or `failed`, and of limits | `no_data.sprite` | a sprite whose render data names no texture, or whose texture has no image | | `unsupported.sprite.external_texture` | a sprite whose render data texture (not an atlas) is in another bundle | | `generic.error` | a converter failed on the object (the message says how); its typetree JSON instead | +| `unsupported.typetree` | an object of a file without type trees read in part: a MonoBehaviour without a shipped type tree of its class (its header), a class whose type tree fits only up to a field (those fields) | | `failed.read` | the object's typetree could not be read | | `failed.export` | the object's typetree JSON could not be made | | `source.absent` | a catalog entry whose file is not there (an APK entry without a file) | diff --git a/scripts/gen_typetrees.py b/scripts/gen_typetrees.py index 50e626e..6ce8f75 100644 --- a/scripts/gen_typetrees.py +++ b/scripts/gen_typetrees.py @@ -3,10 +3,11 @@ nnnotes reads a few MonoBehaviours of the APK's boot data (`assets/bin/Data/data.unity3d`): the URP pipeline assets, renderer data, renderer features and post-process data, DOTween and TextMeshPro settings, and the sound -and app-config defaults. IL2CPP player builds strip script typetrees from that data, so nnnotes ships the -typetrees of exactly these classes (CLASSES below) in `src/nnnotes/unity/typetrees/.json`, one file per -Unity version. Each entry holds the typetree nodes as [level, type, name, meta flag] rows and the type hash the -serialized file records for the class; the file also records the game version it was generated from. +and app-config defaults (the `apk/data.unity3d` tasks of `unity.export`, and PlayerData). IL2CPP player builds +strip script typetrees from that data, so nnnotes ships the typetrees of exactly these classes (CLASSES below) in +`src/nnnotes/unity/typetrees/.json`, one file per Unity version (read by nnnotes.unity.scripttypes). +Each entry holds the typetree nodes as [level, type, name, meta flag] rows and the type hash the serialized file +records for the class; the file also records the game version it was generated from. The rows have the node layout Unity itself serializes for a MonoBehaviour (as in the typetrees of asset bundles built with them): root type `MonoBehaviour`; a `string` is `string / Array / int size / char data`; built-in @@ -34,7 +35,7 @@ not read back stops the script without writing anything. 3. When a new game version keeps the Unity version, the file is replaced; a new Unity version adds a file. Keep the older files while the game versions they describe are still in use. -4. When code starts reading another MonoBehaviour class with `PlayerData.mono`, add the class to CLASSES and +4. When code starts reading another MonoBehaviour class of these files, add the class to CLASSES and regenerate. """ @@ -47,7 +48,8 @@ from UnityPy.helpers.TypeTreeGenerator import TypeTreeGenerator -from nnnotes.unity.player import TYPETREE_FORMAT, PlayerData, dump_typetrees, typetree_key, typetree_node +from nnnotes.unity.player import PlayerData +from nnnotes.unity.scripttypes import TYPETREE_FORMAT, dump_typetrees, typetree_key, typetree_node # (assembly, namespace, class) as the MonoScript objects name them CLASSES = ( diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index 0c4eef3..6580554 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -19,7 +19,8 @@ catalogs {"main": [Input remote, Input apk]}: the catalog version's files in /catalogs/ index the catalog index document (nnnotes.catalog-index/1) bundles {stable bundle name: Input "bundle"} of the selected bundles and their dependencies (a bundle whose - bytes are not known yet: a Pending "fetch:") + bytes are not known yet: a Pending "fetch:"), and of the APK's built-in files + (`apk/data.unity3d`, `apk/Resources/unity default resources`: pipeline.builtins) selected the selected bundles' stable names, sorted (the subjects of the export stages) raw {stable raw file name: Input "raw"} of the selected raw files (only with a stage that reads them; export fetches them while the stages before run, and reading the fact waits for that) @@ -31,7 +32,8 @@ stable name up to `_assets_`, else up to its first `_`), `key:PREFIX` the bundles and raw files the catalog keys under PREFIX load (their dependency closure), `bundle:GLOB` bundles and raw files whose stable name matches. The census also covers the bundles the selected ones depend on (the dependencies of the keys they hold: a key's first -dependency is its own bundle), so script and atlas references resolve. +dependency is its own bundle), so script and atlas references resolve. With `[paths] apk` the census always covers +the APK's built-in files; everything and `bundle:GLOB` (`bundle:apk/*`) select them. Task cost estimates (scheduling only: never part of a key) are each stage's estimate scaled per stage by the costs measured in earlier runs (/costs/calibration.json, updated after every export). @@ -193,14 +195,16 @@ class Selection: raw: list # stable names of the selected raw files -def select(index: dict, selection: list[str]) -> Selection: - """The bundles and raw files a selection names in a catalog index (module documentation).""" +def select(index: dict, selection: list[str], builtin=()) -> Selection: + """The bundles and raw files a selection names in a catalog index, and of the built-in subjects `builtin` + (module documentation).""" from ..pipeline import catalogdb bundles = sorted(b["stable"] for b in index["bundles"]) raws = sorted(r["stable"] for r in index["rawFiles"]) + builtin = sorted(builtin) if selection == ["all"]: - return Selection(bundles, bundles, raws) + return Selection(sorted(bundles + builtin), sorted(bundles + builtin), raws) files = catalogdb.file_refs(index) reach = catalogdb.closures(index) chosen: set = set() @@ -223,8 +227,10 @@ def select(index: dict, selection: list[str]) -> Selection: and files.get(loc["dependencies"][0]) in chosen ): deps |= reach[loc["id"]] + globs = [v for k, _, v in (s.partition(":") for s in selection) if k == "bundle"] + own = [s for s in builtin if any(fnmatch.fnmatchcase(s, g) for g in globs)] pick = lambda s, kind: sorted(n for k, n in s if k == kind) # noqa: E731 - return Selection(pick(chosen, "bundle"), pick(deps, "bundle"), pick(chosen, "raw")) + return Selection(sorted(pick(chosen, "bundle") + own), sorted(pick(deps, "bundle") + builtin), pick(chosen, "raw")) # ---------------------------------------------------------------- settings @@ -669,11 +675,18 @@ def facts(self, pipeline: list[str], fetch: bool, background: bool = False) -> d """The planning facts (module documentation). `fetch`: fetch the files whose content is not known yet (else they are Pending); what cannot be read is in problems() (after wait()). `background`: the raw files are fetched while the caller goes on (the fact waits for them when it is read).""" + from ..pipeline import builtins + index = self.index() - sel = select(index, self.selection) + try: + builtin = builtins.inputs(self.depot, self.apk) + except (OSError, zipfile.BadZipFile): + raise ConfigError(f"setting paths.apk: {self.apk} is not a readable APK") from None + sel = select(index, self.selection, builtin) self._problems = [] bundles = {b["stable"]: b for b in index["bundles"]} - inputs = self._inputs("bundle", [bundles[s] for s in sel.census], fetch) + inputs = self._inputs("bundle", [bundles[s] for s in sel.census if s not in builtin], fetch) + inputs = inputs.plus(builtin) sel.selected = [s for s in sel.selected if s in inputs] # files that cannot be read: problems() facts = { "catalogs": self.catalogs_fact(), @@ -696,7 +709,7 @@ def facts(self, pipeline: list[str], fetch: bool, background: bool = False) -> d facts["raw"] = Later(self._later) else: facts["raw"] = job[0](*job[1:]) - facts["boot"] = boot_input(self.depot, self.apk) + facts["boot"] = boot_input(builtin) self.selected = sel self._facts, self._names = facts, {} return facts @@ -1048,6 +1061,10 @@ def waiting(self) -> list[str]: """The subjects not fetched yet.""" return [k for k, v in self._items.items() if isinstance(v, Pending)] + def plus(self, items: dict) -> Inputs: + """These inputs and `items` ({subject: Input}).""" + return Inputs({**self._items, **items}) + class Later(Mapping): """A mapping a background job makes (Future): reading it waits for the job.""" @@ -1138,7 +1155,7 @@ def _write_report(d: Path, name: str, doc) -> Path: def cmd_export(args, cfg): - from ..pipeline import layout + from ..pipeline import builtins, layout from ..pipeline.orchestrate import Orchestrator, failure try: @@ -1176,9 +1193,12 @@ def cmd_export(args, cfg): ws.run_index() facts = ws.facts(pipeline, fetch=True, background=True) sel = ws.selected + own = sum(1 for s in sel.census if builtins.is_builtin(s)) + picked = sum(1 for s in sel.selected if builtins.is_builtin(s)) ws.log( - f"{len(sel.selected)} bundles selected ({len(sel.census)} with their dependencies), " - f"{len(sel.raw)} raw files" + f"{len(sel.selected) - picked} bundles selected ({len(sel.census) - own} with their dependencies), " + + (f"{picked} of the APK's {own} built-in files, " if own else "") + + f"{len(sel.raw)} raw files" + (" (fetched while the bundle stages run)" if isinstance(facts["raw"], Later) else "") + f"; {sum(p['kind'] == 'bundle' for p in ws.problems())} bundles not readable" ) diff --git a/src/nnnotes/pipeline/builtins.py b/src/nnnotes/pipeline/builtins.py new file mode 100644 index 0000000..81edc98 --- /dev/null +++ b/src/nnnotes/pipeline/builtins.py @@ -0,0 +1,71 @@ +"""The game's own Unity files in its APK as object sources. + +The player reads files under `assets/bin/Data/` of the APK: `data.unity3d` (a bundle of the global managers, +globalgamemanagers.assets, the first scene level0, resources.assets, sharedassets0.assets and unity_builtin_extra), +`datapack.unity3d` in split builds, and the serialized file `Resources/unity default resources` (which bundles name +`Library/unity default resources`). Each of them the APK holds is a subject of unity.census and unity.export: +`apk/` (`apk/data.unity3d`, `apk/Resources/unity default resources`). Their objects +have the ids of any serialized file's (`unity_builtin_extra:12`, `unity default resources:10`): a serialized file +outside a bundle takes the last part of its path as its name (serialized_name), as references into it name it. + +inputs(depot, apk) gives them as Inputs (role "bundle"), each stored in the depot once; for an APK file the set is +recorded by the APK's content id (the APK is hashed once, depot.identify memo), so a later call reads nothing of it. +""" + +from __future__ import annotations + +from pathlib import Path + +from ..source.apkset import ApkSet +from . import contract +from .contract import Input + +__all__ = ["DATA", "DATA_DIR", "MEMBERS", "PREFIX", "inputs", "is_builtin", "serialized_name"] + +DATA_DIR = "assets/bin/Data/" +MEMBERS = ("data.unity3d", "datapack.unity3d", "Resources/unity default resources") +PREFIX = "apk/" +DATA = PREFIX + "data.unity3d" # the boot data (the CRI stages read the HCA keycode from it) + + +def is_builtin(subject: str) -> bool: + return subject.startswith(PREFIX) + + +def serialized_name(subject: str) -> str | None: + """The name a built-in subject's file is loaded under: the last part of its path (a serialized file outside a + bundle is named so); None for a catalog bundle.""" + return subject.rsplit("/", 1)[-1] if is_builtin(subject) else None + + +def _read(depot, apk: Path) -> dict: + """{member: {sha256, size}} of the built-in files an APK holds, their bytes stored in `depot`.""" + out = {} + with ApkSet(apk) as z: + names = set(z.namelist()) + for m in MEMBERS: + if DATA_DIR + m in names: + data = z.read(DATA_DIR + m) + out[m] = {"sha256": depot.put(data), "size": len(data)} + return out + + +def inputs(depot, apk) -> dict[str, Input]: + """{subject: Input "bundle"} of the built-in files of an APK (a file, an APK set or a directory; see the module + documentation), sorted; {} without an APK.""" + if apk is None: + return {} + apk = Path(apk) + if apk.is_dir(): + found = _read(depot, apk) + else: + name = f"builtins:{depot.identify(apk)[0]}" + known = depot.named("files", name) + found = contract.loads(depot.read(known[0])) if known is not None and depot.has(*known) else None + if found is None or not all(depot.has(r["sha256"], r["size"]) for r in found.values()): + found = _read(depot, apk) + depot.identify(depot.local(depot.put(contract.encode(found))), "files", name) + return { + PREFIX + m: Input("bundle", r["sha256"], int(r["size"]), m.rsplit("/", 1)[-1], ({"kind": "depot"},)) + for m, r in sorted(found.items()) + } diff --git a/src/nnnotes/pipeline/census.py b/src/nnnotes/pipeline/census.py index 23a75cb..7aa1d50 100644 --- a/src/nnnotes/pipeline/census.py +++ b/src/nnnotes/pipeline/census.py @@ -18,7 +18,8 @@ classes objects per class; `objects` the total errors [{object, message}] of the objects whose header could not be read -A census is a function of the bundle's bytes: it holds no file name, time or path. +A census is a function of the bundle's bytes: it holds no file name, time or path (a serialized file outside a +bundle, one of the APK's built-in files, has the name its subject gives: builtins.serialized_name). """ from __future__ import annotations @@ -26,7 +27,7 @@ import re from ..unity.atoms import impl_id -from . import contract +from . import builtins, contract from .contract import Cost, Input from .stages import Output, Pending, Stage @@ -249,11 +250,11 @@ def head(node, fields=HEAD_FIELDS): } -def census_file(path) -> dict: - """The census of a bundle file (decrypted UnityFS).""" +def census_file(path, name: str | None = None) -> dict: + """The census of a bundle file (decrypted UnityFS), or of a serialized file named `name`.""" from ..unity import reader as unity - env = unity.load(path) + env = unity.load(path, name) try: return census_env(env) finally: @@ -305,9 +306,10 @@ def bundle_subjects(index: dict) -> dict[str, dict]: # ---------------------------------------------------------------- stage class CensusStage(Stage): """unity.census: one bundle -> its census. Subjects and inputs come from the fact "bundles": {subject: Input - (role "bundle": the decrypted bundle's sha256, size, name, locators)}; a bundle not fetched yet is Pending - (raised by the fact, or given as a Pending / None). Artifact "#census". The census - reads the bundle with UnityPy: its atom is the reader's ("reader", the UnityPy version).""" + (role "bundle": the decrypted bundle's sha256, size, name, locators)}, the APK's built-in files included + (builtins); a bundle not fetched yet is Pending (raised by the fact, or given as a Pending / None). Artifact + "#census". The census reads the bundle with UnityPy: its atom is the reader's ("reader", the UnityPy + version).""" name = "unity.census" version = 2 @@ -331,7 +333,7 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: return Cost(0.002 + size / 40e6, (32 << 20) + 12 * size) def run(self, task, depot) -> Output: - doc = census_file(depot.open_input(task.input("bundle"))) + doc = census_file(depot.open_input(task.input("bundle")), builtins.serialized_name(task.subject)) content = depot.add(contract.encode(doc), "json") rec = contract.artifact( contract.artifact_id(task.id, "census"), diff --git a/src/nnnotes/pipeline/cristages.py b/src/nnnotes/pipeline/cristages.py index fbb0098..fd7f1b9 100644 --- a/src/nnnotes/pipeline/cristages.py +++ b/src/nnnotes/pipeline/cristages.py @@ -43,9 +43,8 @@ from pathlib import Path from ..formats.encoders import FLAC_LEVEL -from ..source.apkset import ApkSet from ..unity.atoms import impl_id -from . import contract +from . import builtins, contract from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage @@ -53,7 +52,6 @@ ACB_MAGIC, AWB_MAGIC, USM_MAGIC = b"@UTF", b"AFS2", b"CRID" ACB_KIND = "cri.acb" # semantics kind of the ACB artifacts of unity.export USM_KIND = "cri.usm" # semantics kind of the USM artifacts of unity.export -BOOT_IN_APK = "assets/bin/Data/data.unity3d" ACB_NAME = "sheet" # the stem of the ACB file vgmstream reads (names come from its cues) TOOL_REVISION = 1 FORMATS = ("mkv", "webm") @@ -70,30 +68,11 @@ # ---------------------------------------------------------------- inputs shared by the stages -def boot_input(depot, apk) -> Input | None: - """The input "boot" of the CRI stages: the boot data of an APK (`assets/bin/Data/data.unity3d`, where the HCA - keycode is read), stored in `depot` once; the APK is hashed once (depot.identify memo) and its boot data is - remembered by the APK's content id. A path that is not an .apk / .zip is the boot data itself. None without an - APK.""" - if apk is None: - return None - apk = Path(apk) - if not apk.is_dir() and apk.suffix.lower() not in (".apk", ".zip", ".apks", ".xapk"): - sha, size = depot.identify(apk) - return Input("boot", sha, size, apk.name, ({"kind": "file", "path": str(apk.resolve())},)) - if apk.is_dir(): - with ApkSet(apk) as z: - data = z.read(BOOT_IN_APK) - sha = depot.put(data) - return Input("boot", sha, len(data), "data.unity3d", ({"kind": "depot"},)) - apk_sha, _ = depot.identify(apk) - name = f"boot:{apk_sha}" - known = depot.named("files", name) - if known is None or not depot.has(*known): - with ApkSet(apk) as z: - data = z.read(BOOT_IN_APK) - known = depot.identify(depot.local(depot.put(data)), "files", name) - return Input("boot", known[0], known[1], "data.unity3d", ({"kind": "depot"},)) +def boot_input(builtin: dict) -> Input | None: + """The input "boot" of the CRI stages: the APK's `assets/bin/Data/data.unity3d`, where the HCA keycode is read, + of its built-in files (`builtin`, builtins.inputs); None when they lack it (no APK).""" + inp = builtin.get(builtins.DATA) + return None if inp is None else Input("boot", inp.sha256, inp.size, inp.name, inp.locators) def _boot(env) -> Input: diff --git a/src/nnnotes/pipeline/link.py b/src/nnnotes/pipeline/link.py index 9a6ea97..88363db 100644 --- a/src/nnnotes/pipeline/link.py +++ b/src/nnnotes/pipeline/link.py @@ -6,10 +6,11 @@ consumes. link.addresses (nnnotes.addresses/1), one task per catalog pair: the catalog index and the censuses of its bundles -(by stable bundle name) -> +(by stable bundle name) and of the APK's built-in files (subjects `apk/...`, pipeline.builtins) -> bundles {stable name: {name, files, assetBundleName, keys}}: the file name in this catalog version, its - serialized files, the AssetBundle name the census found and the keys located in it - files {serialized file: stable bundle name} (CAB -> bundle) + serialized files, the AssetBundle name the census found and the keys located in it; a built-in file + is listed by its subject, named by its path in the APK's `assets/bin/Data/`, without keys + files {serialized file: stable bundle name} (CAB -> bundle; `unity_builtin_extra` -> `apk/data.unity3d`) keys {primary key: [{location, type, internalId, bundle | raw, objects}]}: an asset location's bundle is the dependency whose container holds its internal id (compared ignoring case); `objects` the object ids the container path names (null when a dependency bundle has no census) @@ -22,7 +23,8 @@ its extension names, else the first by file and path id: the object the `original` layout puts at the path), "/[]" for another object under it with a name no other object under it has, else ":". In a bundle of several serialized files (their path ids may coincide) the path id form names -the file after the bundle: the part of the file name after its CAB- prefix ("scene_x.sharedAssets:12"). +the file after the bundle: the part of the file name after its CAB- prefix ("scene_x.sharedAssets:12"), or +"|" ("apk/data.unity3d|level0:3"). Objects that no container path names are not listed; `address_of` gives any object's address. link.artifacts (nnnotes.artifacts/1), one global task: the results of the stages that export objects (unity.export, @@ -41,7 +43,8 @@ import json import re -from . import catalogdb, census, contract +from ..unity import scripttypes +from . import builtins, catalogdb, census, contract from .contract import Cost, Input from .layout import EXT_CLASSES from .stages import Output, Pending, Stage @@ -55,11 +58,8 @@ # ---------------------------------------------------------------- scripts def script_entry(s: dict) -> str: - """ "assembly|namespace|class" of a census MonoScript record.""" - asm = s.get("assembly") or "" - if asm.endswith(".dll"): - asm = asm[:-4] - return f"{asm}|{s.get('namespace') or ''}|{s.get('class') or ''}" + """ "assembly|namespace|class" of a census MonoScript record (scripttypes.script_entry).""" + return scripttypes.script_entry(s.get("assembly"), s.get("namespace"), s.get("class")) def scripts(censuses) -> dict: @@ -81,12 +81,13 @@ def script_context(table: dict, refs) -> dict: # ---------------------------------------------------------------- addresses def file_tags(files) -> dict[str, str]: """{serialized file: the tag its path id addresses carry after the bundle name}: "" in a bundle of one file, - else the file name after its CAB- prefix ("" / ".sharedAssets"); "|" where those coincide.""" + else the file name after its CAB- prefix ("" / ".sharedAssets"); "|" where those coincide or a file + has no such prefix.""" files = list(files) if len(files) <= 1: return {f: "" for f in files} tags = {f: _CAB.sub("", f) for f in files} - if len(set(tags.values())) < len(tags): + if len(set(tags.values())) < len(tags) or not all(_CAB.match(f) for f in files): tags = {f: "|" + f for f in files} return tags @@ -128,9 +129,10 @@ def cls(e): def addresses(index: dict, censuses) -> dict: - """The address table (nnnotes.addresses/1) of a catalog index and the censuses of its bundles: {stable bundle - name: census document}, or (stable name, census document) pairs (read one at a time: only the container - entries and the objects they name are kept). Censuses of names the index does not list are ignored.""" + """The address table (nnnotes.addresses/1) of a catalog index and the censuses of its bundles and of the APK's + built-in files: {stable bundle name or built-in subject: census document}, or (name, census document) pairs + (read one at a time: only the container entries and the objects they name are kept). Censuses of other names + the index does not list are ignored.""" files_of = catalogdb.file_refs(index) listed = {b["stable"]: b for b in index["bundles"]} bundles, owners, info, groups, tags_of = {}, {}, {}, {}, {} @@ -138,6 +140,8 @@ def addresses(index: dict, censuses) -> dict: pairs = sorted(censuses.items()) if isinstance(censuses, dict) else censuses for stable, doc in pairs: b = listed.get(stable) + if b is None and builtins.is_builtin(stable): + b = {"name": stable[len(builtins.PREFIX) :]} if b is None: continue names = [f["name"] for f in doc["files"]] @@ -242,8 +246,12 @@ def reverse(doc: dict) -> dict[str, str]: # ---------------------------------------------------------------- stages +def _subject(tid: str) -> str: + return contract.parse_task_id(tid, census.CensusStage.name)[1] + + def _census_input(env, tid: str) -> Input: - return env.input_of(tid, "census", as_role="census:" + contract.parse_task_id(tid, census.CensusStage.name)[1]) + return env.input_of(tid, "census", as_role="census:" + _subject(tid)) def _read(depot, task, role: str) -> dict: @@ -289,9 +297,9 @@ def run(self, task, depot) -> Output: class AddressesStage(Stage): """link.addresses: a catalog index (input "index", the result of catalog.index:) and the censuses of its - bundles (inputs "census:") -> the address table. Subjects: those of catalog.index (the fact - "catalogs"). Waits for census tasks of its bundles that have not run; skips failed ones. Artifact - "link.addresses:#addresses".""" + bundles and of the APK's built-in files (inputs "census:") -> the address table. + Subjects: those of catalog.index (the fact "catalogs"). Waits for census tasks of its bundles that have not run; + skips failed ones. Artifact "link.addresses:#addresses".""" name = "link.addresses" version = 1 @@ -305,6 +313,7 @@ def inputs(self, subject: str, env) -> list[Input]: idx = env.input_of(tid, "index") stables = {b["stable"] for b in env.depot.document(idx.sha256)["bundles"]} wanted = {contract.task_id(census.CensusStage.name, s) for s in stables} + wanted |= {t for t in env.tasks(census.CensusStage.name) if builtins.is_builtin(_subject(t))} pending = [t for t in env.pending(census.CensusStage.name) if t in wanted] if pending: raise Pending(pending[0]) diff --git a/src/nnnotes/pipeline/objexport.py b/src/nnnotes/pipeline/objexport.py index 182e638..e06d5c8 100644 --- a/src/nnnotes/pipeline/objexport.py +++ b/src/nnnotes/pipeline/objexport.py @@ -24,7 +24,9 @@ import io from collections import defaultdict +from functools import cache +from ..unity import scripttypes from ..unity.atoms import ATOMS as ATOM_REGISTRY from ..unity.atoms import Impl, Unsupported, atom_id, resolve from ..unity.convert import ( @@ -45,9 +47,17 @@ prefab, textures, ) -from ..unity.convert.objects import BundleObjects, ReadError, encode_object, label, message, ref_object_id +from ..unity.convert.objects import ( + TYPETREE, + BundleObjects, + ReadError, + encode_object, + label, + message, + ref_object_id, +) +from . import builtins, contract, link from . import census as census_mod -from . import contract, link from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage @@ -81,6 +91,7 @@ def __init__(self, view, task, depot, fns: dict): blob_min=int(task.params["blobMin"]), png_level=int(task.params["png"]["level"]), emit=self._emit, + typetrees=task.context.get("typetrees", {}), ) classes = task.params.get("classes") self.selected = None if classes is None else frozenset(classes) @@ -190,14 +201,18 @@ def _fallback(self, info, code: str, why: str, status: str = "generic") -> None: self._commit(info, status, contract.reason(code, label(why))) def _generic(self, info) -> None: - """An object of a class without a specialized converter: its typetree JSON.""" + """An object of a class without a specialized converter: its typetree JSON (`generic` when it could be read + only in part).""" self._begin("generic.json") try: - generic.generic(self.ctx, info) + why = generic.generic(self.ctx, info) except Exception as e: self._failed(info, e) return - self._commit(info, "exported") + if why is None: + self._commit(info, "exported") + else: + self._commit(info, "generic", contract.reason(TYPETREE, label(why))) def _selected(self, cls: str) -> bool: return self.selected is None or cls in self.selected @@ -371,7 +386,9 @@ class ExportStage(Stage): "bundles" {subject: Input (role "bundle")}, as unity.census. It reads the census of its bundle (unity.census:) for the classes (the converters and atoms of its key) and the MonoScripts its objects reference, and the script table (link.scripts:all) for the entries of those: its context - {"scripts": {object id: "assembly|namespace|class"}}. Parameters: `blobMin` (bytes; byte arrays of this size or + {"scripts": {object id: "assembly|namespace|class"}}, and for the MonoBehaviours of files that store no type trees + (the APK's built-in files) the type trees nnnotes ships for their classes: "typetrees" {script entry: + {"typeHash", "nodes"}}. Parameters: `blobMin` (bytes; byte arrays of this size or more are blobs), `png.level` (zlib level of the PNG files). `reader`: another implementation of the reader atom (its id enters the keys).""" @@ -449,11 +466,16 @@ def uses(self, subject: str, env) -> dict: return atoms def context(self, subject: str, env) -> dict: - refs = census_mod.script_refs(self.census(subject, env)) + doc = self.census(subject, env) + refs = census_mod.script_refs(doc) if not refs: return {} table = self._docs.get(env.depot, env.input_of(SCRIPTS_TASK, "scripts").sha256)["scripts"] - return {"scripts": link.script_context(table, refs)} + out = {"scripts": link.script_context(table, refs)} + types = _script_types(doc, out["scripts"]) + if types: + out["typetrees"] = types + return out def depends(self, subject: str, env) -> list[str]: return [contract.task_id(CENSUS, subject), SCRIPTS_TASK] @@ -473,12 +495,34 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: def run(self, task, depot) -> Output: reader = self.reader if self.reader is not None else resolve("reader") _check_atoms(task, reader) - view = reader.fn(depot.input_bytes(task.input("bundle"))) + view = reader.fn(depot.input_bytes(task.input("bundle")), builtins.serialized_name(task.subject)) fns = {n: resolve(n).fn for n in task.atoms if n in ATOM_NAMES and n != "reader"} arts, items = BundleRun(view, task, depot, fns).run() return Output(arts, items) +@cache +def _embedded(unity_version: str) -> dict: + return scripttypes.script_types(unity_version) + + +def _script_types(doc: dict, scripts: dict) -> dict: + """{script entry: {"typeHash", "nodes"}} of the embedded type trees (nnnotes.unity.scripttypes, of each file's + Unity version) of the classes of the MonoBehaviours in the census' files that store no type trees ({} when it has + none).""" + out = {} + for f in doc["files"]: + if f["typetree"]: + continue + types = _embedded(f["unityVersion"]) + for o in f["objects"]: + s = o.get("script") if o["class"] == "MonoBehaviour" else None + entry = scripts.get(contract.object_id(s["file"], s["pathId"])) if s and s.get("file") else None + if entry in types and out.setdefault(entry, types[entry]) != types[entry]: + raise ValueError(f"{entry}: files of several Unity versions with different type trees of it") + return dict(sorted(out.items())) + + def _census_classes(env, subject: str) -> dict: """{class: objects} of a bundle's census ({} when its census has no result).""" try: diff --git a/src/nnnotes/unity/atoms/reader.py b/src/nnnotes/unity/atoms/reader.py index 0cb56ce..2f4cf9e 100644 --- a/src/nnnotes/unity/atoms/reader.py +++ b/src/nnnotes/unity/atoms/reader.py @@ -2,7 +2,8 @@ `BundleView` is what a stage reads a bundle through; `UnityBundle` is the reference implementation (UnityPy, the bundle loaded alone from its bytes: references into other bundles are not followed, resource files inside the -bundle -- `.resS` -- are read). Objects are named by `ObjectInfo` (serialized file name and path id), never by +bundle -- `.resS` -- are read). A serialized file outside a bundle (one of the APK's built-in files) is read the same +way, under the name it is given. Objects are named by `ObjectInfo` (serialized file name and path id), never by library objects, so another implementation can stand in. `texture_input` and `mesh_input` collect exactly what `texture.decode` and `mesh.arrays` need: a Texture2D's image @@ -69,6 +70,11 @@ def unity_version(self, file: str) -> str: """The Unity version a serialized file was written by.""" ... + def stores_typetrees(self, file: str) -> bool: + """Whether a serialized file stores the type trees of its objects (a player build's own files do not: its + script classes read only their header).""" + ... + def texture_input(self, obj: ObjectInfo) -> TextureInput: ... def mesh_input(self, obj: ObjectInfo) -> MeshInput: ... @@ -266,6 +272,9 @@ def unity_version(self, file: str) -> str: v = getattr(f, "unity_version", None) return v if isinstance(v, str) else ".".join(str(x) for x in f.version) + def stores_typetrees(self, file: str) -> bool: + return bool(getattr(self._files[file], "_enable_type_tree", True)) + def texture_input(self, obj: ObjectInfo) -> TextureInput: r = self._reader(obj) tex = r.read() @@ -306,11 +315,12 @@ def mesh_input(self, obj: ObjectInfo) -> MeshInput: ) -def open_bundle(data: bytes) -> UnityBundle: - """The BundleView of a bundle's bytes (decrypted UnityFS), loaded alone.""" +def open_bundle(data: bytes, name: str | None = None) -> UnityBundle: + """The BundleView of a bundle's bytes (decrypted UnityFS), loaded alone; of a serialized file's bytes with its + `name`.""" from ..reader import load_bytes - return UnityBundle(load_bytes(bytes(data))) + return UnityBundle(load_bytes(bytes(data), name)) def cost(facts: dict) -> dict: diff --git a/src/nnnotes/unity/convert/__init__.py b/src/nnnotes/unity/convert/__init__.py index 759d8ab..ce85c0c 100644 --- a/src/nnnotes/unity/convert/__init__.py +++ b/src/nnnotes/unity/convert/__init__.py @@ -38,6 +38,10 @@ - a string whose stored bytes are not UTF-8 is {"$hex": ""}; - NaN is {"$float": "nan"}, infinities 1e999 / -1e999. +A file of the player itself (the APK's built-in files) stores no type trees: its MonoBehaviours are read with the +type trees nnnotes ships for their classes (nnnotes.unity.scripttypes), and an object that can be read only in part +gives that part with `unsupported.typetree` (objects.py). + A converter's documented limit raises atoms.Unsupported with a reason code; the reason codes are listed in docs/stages.md. """ @@ -76,7 +80,7 @@ "scripts.json": 1, "bundle.json": 1, "shader.json": 1, - "generic.json": 1, + "generic.json": 2, "cri.usm": 1, } # class -> the converters its objects may use (every class may fall back to generic.json) @@ -118,6 +122,7 @@ "no_data.font", "unsupported.sprite.external_texture", "unsupported.cri.awb_external", + "unsupported.typetree", "failed.read", "failed.export", *(f"unsupported.class.{c}" for c in UNSUPPORTED_CLASSES), diff --git a/src/nnnotes/unity/convert/generic.py b/src/nnnotes/unity/convert/generic.py index 7199f98..c69e4b2 100644 --- a/src/nnnotes/unity/convert/generic.py +++ b/src/nnnotes/unity/convert/generic.py @@ -11,11 +11,13 @@ LOOSE = {"bundle.json": ("AssetBundle", "unity.bundle"), "scripts.json": ("MonoScript", "unity.scripts")} -def generic(ctx: BundleObjects, info) -> None: - """`json`: the object's typetree JSON.""" +def generic(ctx: BundleObjects, info) -> str | None: + """`json`: the object's typetree JSON. Returns why when the typetree could be read only in part (its part is + the JSON: BundleObjects.partial), else None.""" key = (info.file, info.path_id) - doc = ctx.tree(ctx.readers[key], ctx.tt(key)) + doc = ctx.tree(ctx.readers[key], ctx.tt(key, whole=False)) ctx.emit(key, "json", encode_object(doc), "json", "unity.object", "json", {}) + return ctx.partial.get(key) def loose_doc(ctx: BundleObjects, conv: str, entries: dict) -> dict: diff --git a/src/nnnotes/unity/convert/objects.py b/src/nnnotes/unity/convert/objects.py index b75a0d1..dc3a44b 100644 --- a/src/nnnotes/unity/convert/objects.py +++ b/src/nnnotes/unity/convert/objects.py @@ -7,6 +7,14 @@ Every object a document names by {"$ref"} is recorded in `seen` (by object id, other bundles' objects included): the objects an object's documents reference, which the caller clears before converting an object. + +A file that stores no type trees (a player build's own files) gives its MonoBehaviours only their header; they are +read with the task's type tree of their script class (`typetrees`, nnnotes.unity.scripttypes), checked against the +type hash the file stores and the object's size. One without such a tree is `unsupported.typetree` (TYPETREE): its +header is still read. Its other objects are read with the type trees of their classes UnityPy has for the file's +Unity version; for a class whose tree reads the stored layout right only up to a field (PREFIXES, by the type hash +the file stores) the typetree JSON is the fields up to it, also `unsupported.typetree`. The objects read in part are +in `partial`, with the reason. """ from __future__ import annotations @@ -17,10 +25,14 @@ from typing import cast from ...core.canon import encode +from ..atoms import Unsupported from ..ids import object_id +from ..scripttypes import typetree_node __all__ = [ "NO_SCRIPT", + "PREFIXES", + "TYPETREE", "BundleObjects", "Capture", "ReadError", @@ -33,6 +45,10 @@ ] NO_SCRIPT = "\0none" # the script entry of a MonoBehaviour whose m_Script is null +TYPETREE = "unsupported.typetree" # an object of a file without type trees read in part +# classes whose type tree (UnityPy's, for the file's Unity version) reads the stored layout right only up to a field, +# by the type hash the file stores: {(class, type hash): the last field read right} +PREFIXES = {("PlayerSettings", "936fd1913a4f0b5da2ce7cadb0ce9666"): "m_ActiveColorSpace"} _BYTE_TYPES = frozenset({"UInt8", "SInt8", "char"}) _SCALARS = (int, float, str, bool, type(None)) _ADDRESS = re.compile(r"0x[0-9a-fA-F]{6,}") @@ -135,6 +151,24 @@ def _ref_type_node(ref: dict, assets_file): raise ValueError(f"referenced type {asm}|{ns}|{cls} is not among the file's reference types") +def _cut(node, last: str): + """A type tree cut after its field `last` (None when it has no such field).""" + names = [c.m_Name for c in node.m_Children] + if last not in names: + return None + from UnityPy.helpers.TypeTreeNode import TypeTreeNode + + return TypeTreeNode( + node.m_Level, + node.m_Type, + node.m_Name, + node.m_ByteSize, + node.m_Version, + m_MetaFlag=node.m_MetaFlag, + m_Children=node.m_Children[: names.index(last) + 1], + ) + + class ReadError(Exception): """An object's typetree could not be read.""" @@ -156,17 +190,33 @@ class Capture: class BundleObjects: """The objects of one bundle (`view`, a reader atom's BundleView). `fns`: the atom functions by name; `scripts`: the script table entries the bundle's objects use ({MonoScript object id: "assembly|namespace|class"}); - `blob_min`: byte arrays of this size or more are blobs; `png_level`: the zlib level of PNG files.""" - - def __init__(self, view, fns: dict, *, scripts: dict, blob_min: int, png_level: int, emit: Emit): + `typetrees`: the type trees of script classes for the files that store none ({script entry: {"typeHash", + "nodes"}}); `blob_min`: byte arrays of this size or more are blobs; `png_level`: the zlib level of PNG files.""" + + def __init__( + self, + view, + fns: dict, + *, + scripts: dict, + blob_min: int, + png_level: int, + emit: Emit, + typetrees: dict | None = None, + ): self.view, self.fns, self.emit = view, fns, emit self.scripts, self.blob_min, self.png_level = dict(scripts), int(blob_min), int(png_level) + self.typetrees = dict(typetrees or {}) self.infos = view.objects() self.info = {(i.file, i.path_id): i for i in self.infos} self.readers = {k: view.reader(i) for k, i in self.info.items()} self.files = view.files() self.ext_files = {f: [_file_name(p) for p in view.externals(f)] for f in self.files} self.versions = {f: view.unity_version(f) for f in self.files} + self.stripped = frozenset(f for f in self.files if not view.stores_typetrees(f)) + self.partial: dict[tuple, str] = {} # objects read in part -> why + self._nodes: dict[tuple, object] = {} # objects read with a type tree of `typetrees` -> its node + self._script_nodes: dict[str, object] = {} self.containers: dict[tuple, str] = {} for path, file, pid in view.container(): k = (file, int(pid)) @@ -209,21 +259,7 @@ def _peek(self, r) -> str | None: node = r._get_typetree_node() hit = self._heads.get(id(node)) if hit is None or hit[0] is not node: - names = [c.m_Name for c in node.m_Children] - cut = None - if "m_Name" in names: - from UnityPy.helpers.TypeTreeNode import TypeTreeNode - - cut = TypeTreeNode( - node.m_Level, - node.m_Type, - node.m_Name, - node.m_ByteSize, - node.m_Version, - m_MetaFlag=node.m_MetaFlag, - m_Children=node.m_Children[: names.index("m_Name") + 1], - ) - hit = self._heads[id(node)] = (node, cut) + hit = self._heads[id(node)] = (node, _cut(node, "m_Name")) if hit[1] is None: return None name = r.read_typetree(hit[1], check_read=False).get("m_Name") @@ -241,16 +277,72 @@ def name_of(self, key: tuple) -> str | None: self.names[key] = name return self.names[key] - def tt(self, key: tuple) -> dict: - """The object's typetree (ReadError when it cannot be read).""" + def tt(self, key: tuple, *, whole: bool = True) -> dict: + """The object's typetree (ReadError when it cannot be read). An object of a file that stores no type trees + that can be read only in part (see the module documentation) raises Unsupported(TYPETREE); with `whole` False + it gives that part (and is in `partial`).""" + info = self.info[key] + if info.class_name == "MonoBehaviour" and info.file in self.stripped: + return self._script_tt(key, info, whole) try: - tt = self.view.typetree(self.info[key]) + tt = self.view.typetree(info) except Exception as e: - raise ReadError(message(e)) from None + last = PREFIXES.get((info.class_name, info.type_hash)) if info.file in self.stripped else None + if last is None: + raise ReadError(message(e)) from None + why = f"{info.class_name}: {message(e)}; read up to {last}, which its type tree reads right" + if whole: + raise Unsupported(TYPETREE, why) from None + tt = self._prefix(self.readers[key], last) + self.partial[key] = why name = tt.get("m_Name") self.names[key] = name if isinstance(name, str) else None return tt + def _script_tt(self, key: tuple, info, whole: bool) -> dict: + r = self.readers[key] + try: + head = r.read_typetree(check_read=False) # the class's own tree: the header + except Exception as e: + raise ReadError(message(e)) from None + entry = self.mono_script(key, head) + t = self.typetrees.get(entry) if entry is not None else None + if entry is None or entry == NO_SCRIPT: + s = head.get("m_Script") or {} + why = f"script {s.get('m_FileID')}:{s.get('m_PathID')} is unknown and the file stores no type trees" + elif t is None: + why = f"the file stores no type trees and the task has none for {entry}" + elif t["typeHash"] != info.type_hash: + why = f"{entry}: the file's type hash {info.type_hash} is not the type tree's {t['typeHash']}" + else: + node = self._script_nodes.get(entry) + if node is None: + node = self._script_nodes[entry] = typetree_node(t["nodes"]) + try: + tt = r.read_typetree(node, check_read=True) + except Exception as e: + why = f"{entry}: the type tree does not fit the object ({message(e)})" + else: + self._nodes[key] = node + self.partial.pop(key, None) + name = tt.get("m_Name") + self.names[key] = name if isinstance(name, str) else None + return tt + if whole: + raise Unsupported(TYPETREE, why) + self.partial[key] = why + return head + + def _prefix(self, r, last: str) -> dict: + """An object's typetree up to the field `last` (its class's tree cut after it).""" + cut = _cut(r._get_typetree_node(), last) + if cut is None: + raise ReadError(f"its type tree has no field {last}") + try: + return r.read_typetree(cut, check_read=False) + except Exception as e: + raise ReadError(message(e)) from None + def facts(self, key: tuple) -> dict: """The object facts of an artifact's provenance: file, path id, class, name, type hash, Unity version, container path.""" @@ -286,7 +378,7 @@ def mono_script(self, key: tuple, tt: dict) -> str | None: def script_class(self, key: tuple) -> str | None: """The class name of a MonoBehaviour from the script table (None when unknown).""" if key not in self._mono_scripts: - self.mono_script(key, self.tt(key)) + self.mono_script(key, self.tt(key, whole=False)) e = self._mono_scripts[key] return e.rsplit("|", 1)[-1] if e and e != NO_SCRIPT else None @@ -342,7 +434,8 @@ def tree(self, owner, tt: dict, *, captures: dict[str, Capture] | None = None, l name of an object of the prefab being walked (None for any other).""" self._captures, self._local = dict(captures or {}), local try: - doc = cast(dict, self._walk(owner, owner._get_typetree_node(), tt, "")) + node = self._nodes.get(self.key(owner)) or owner._get_typetree_node() + doc = cast(dict, self._walk(owner, node, tt, "")) left = self._captures finally: self._captures, self._local = {}, None diff --git a/src/nnnotes/unity/convert/prefab.py b/src/nnnotes/unity/convert/prefab.py index 845719e..1610952 100644 --- a/src/nnnotes/unity/convert/prefab.py +++ b/src/nnnotes/unity/convert/prefab.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field from types import SimpleNamespace -from .objects import NO_SCRIPT, BundleObjects, ReadError, encode_object, message +from .objects import NO_SCRIPT, TYPETREE, BundleObjects, ReadError, encode_object, message __all__ = ["Hierarchies", "Root", "hierarchies", "prefab"] @@ -132,7 +132,7 @@ def _subtree(ctx: BundleObjects, h: Hierarchies, f: str, g, root_tf: int): def _component(ctx: BundleObjects, h: Hierarchies, o, root: tuple, unresolved: list) -> dict: key = ctx.key(o) - tt = ctx.tt(key) + tt = ctx.tt(key, whole=False) out = {"type": o.type.name, "pathId": key[1]} if o.type.name == "MonoBehaviour": entry = ctx.mono_script(key, tt) @@ -184,10 +184,13 @@ def prefab(ctx: BundleObjects, info, h: Hierarchies, root: Root) -> None: if info.file in h.scene_of: doc["scene"] = h.scene_of[info.file] doc["nodes"] = nodes - issues = ( - [("script.unresolved", f"{len(unresolved)} MonoBehaviours whose script is not in the task's script table")] - if unresolved - else [] - ) + issues = [] + if unresolved: + issues.append( + ("script.unresolved", f"{len(unresolved)} MonoBehaviours whose script is not in the task's script table") + ) + partial = sum(1 for k in root.members if k in ctx.partial) + if partial: + issues.append((TYPETREE, f"{partial} components read in part (their file stores no type tree that fits)")) facts = {"nodes": len(nodes), "objects": len(root.members)} ctx.emit(key, "json", encode_object(doc), "json", "prefab", "json", facts, issues=issues) diff --git a/src/nnnotes/unity/player.py b/src/nnnotes/unity/player.py index 21c36ad..10638f7 100644 --- a/src/nnnotes/unity/player.py +++ b/src/nnnotes/unity/player.py @@ -3,10 +3,8 @@ `assets/bin/Data/data.unity3d` holds the global managers (graphics, quality, player settings) and `globalgamemanagers.assets` with the URP pipeline assets, renderer data, renderer features and post-process data. IL2CPP player builds -strip script typetrees there, so MonoBehaviours are read with typetrees that -ship with nnnotes: `typetrees/.json`, generated from the game's -managed assembly stubs by `scripts/gen_typetrees.py`. Each entry records the -type hash the serialized file stores for its class; a class without an entry, +strip script typetrees there, so MonoBehaviours are read with the typetrees +that ship with nnnotes (nnnotes.unity.scripttypes); a class without an entry, a different type hash or a read that does not consume exactly the object's serialized size raises UnsupportedVersion. `unity default resources` (built-in meshes) is read from the APK as well. @@ -16,10 +14,7 @@ import copy import io -import json -import re import struct -from importlib import resources from pathlib import Path from typing import cast @@ -30,63 +25,18 @@ from ..source.apkset import MANIFEST_IN_APK, ApkSet, manifest_version_name from ..source.config import ConfigError from .reader import DEFAULT_RESOURCES, deref, external_path, is_pptr +from .scripttypes import load_typetrees, typetree_key, typetree_node DATA_IN_APK = "assets/bin/Data/data.unity3d" DEFAULT_RESOURCES_IN_APK = "assets/bin/Data/Resources/unity default resources" COLOR_SPACE = {0: "Gamma", 1: "Linear"} HEADER = ("m_GameObject", "m_Script", "m_Enabled") -TYPETREE_FORMAT = 1 class UnsupportedVersion(ConfigError): """The APK's boot data has a MonoBehaviour class that nnnotes has no matching typetree for.""" -# ---------------------------------------------------------------- embedded typetrees -def typetree_key(assembly: str, namespace: str, cls: str) -> str: - """Key of a class in a typetree file: `assembly|namespace|class`, as its MonoScript names them.""" - return f"{assembly}|{namespace}|{cls}" - - -def typetree_file(unity_version: str): - """The embedded typetree file of a Unity version (an importlib.resources Traversable; it may not exist).""" - return resources.files("nnnotes.unity").joinpath("typetrees", f"{unity_version}.json") - - -def load_typetrees(unity_version: str) -> dict | None: - """The embedded typetrees of a Unity version, or None when nnnotes has none for it.""" - if not re.fullmatch(r"[0-9A-Za-z.]+", unity_version or ""): - return None - f = typetree_file(unity_version) - if not f.is_file(): - return None - doc = json.loads(f.read_text(encoding="utf-8")) - if doc.get("format") != TYPETREE_FORMAT or doc.get("unityVersion") != unity_version: - raise ValueError(f"typetree file {unity_version}.json: unexpected format or Unity version") - return doc - - -def typetree_node(rows: list) -> TypeTreeNode: - """TypeTreeNode from [level, type, name, meta flag] rows (byte size and version 0, as generated).""" - return TypeTreeNode.from_list([TypeTreeNode(lv, ty, nm, 0, 0, m_MetaFlag=mf) for lv, ty, nm, mf in rows]) - - -def dump_typetrees(doc: dict) -> str: - """Text of a typetree file: sorted keys, one class per line, LF.""" - head = {k: v for k, v in doc.items() if k != "classes"} - lines = [json.dumps(head, sort_keys=True, ensure_ascii=False, separators=(",", ":"))[:-1] + ',"classes":{'] - items = sorted(doc["classes"].items()) - for i, (k, v) in enumerate(items): - lines.append( - json.dumps(k, ensure_ascii=False) - + ":" - + json.dumps(v, sort_keys=True, ensure_ascii=False, separators=(",", ":")) - + ("," if i + 1 < len(items) else "") - ) - lines.append("}}") - return "\n".join(lines) + "\n" - - class PlayerData: def __init__(self, apk: Path): with ApkSet(apk) as z: diff --git a/src/nnnotes/unity/reader.py b/src/nnnotes/unity/reader.py index 2c640b8..fd600e4 100644 --- a/src/nnnotes/unity/reader.py +++ b/src/nnnotes/unity/reader.py @@ -92,12 +92,22 @@ def parse_blob(cls, reader, version): _install_blob_memo() -def load(path: str | Path) -> UnityPy.environment.Environment: - return UnityPy.load(str(path)) - - -def load_bytes(data: bytes) -> UnityPy.environment.Environment: - return UnityPy.load(io.BytesIO(data)) +def load(path: str | Path, name: str | None = None) -> UnityPy.environment.Environment: + """The environment of a bundle file or a serialized file (load_bytes).""" + if name is None: + return UnityPy.load(str(path)) + return load_bytes(Path(path).read_bytes(), name) + + +def load_bytes(data: bytes, name: str | None = None) -> UnityPy.environment.Environment: + """The environment of a bundle's bytes, or of a serialized file's: `name` names a serialized file outside a bundle + (`unity default resources`; references into it name it so), which UnityPy would name after the stream. A + bundle's files keep their own names.""" + if name is None: + return UnityPy.load(io.BytesIO(data)) + env = UnityPy.Environment() + env.load_file(io.BytesIO(data), name=name) + return env _closures = threading.local() diff --git a/src/nnnotes/unity/scripttypes.py b/src/nnnotes/unity/scripttypes.py new file mode 100644 index 0000000..2f7d7f3 --- /dev/null +++ b/src/nnnotes/unity/scripttypes.py @@ -0,0 +1,98 @@ +"""The type trees of game script classes that ship with nnnotes. + +IL2CPP player builds strip the type trees of script classes from the player's own files (`assets/bin/Data/` of the +APK: the global managers, the URP pipeline assets, renderer data, renderer features and post-process data), so a +MonoBehaviour there reads only its header. nnnotes ships the type trees of the classes it reads from those files in +`typetrees/.json`, generated from the game's managed assembly stubs by `scripts/gen_typetrees.py`: + + {"format": 1, "unityVersion", "gameVersion", "classes": {"assembly|namespace|class": {"typeHash", "nodes"}}} + +A class is keyed by the names its MonoScript stores (typetree_key); `nodes` are [level, type, name, meta flag] rows +of Unity's layout and `typeHash` the hex type hash the serialized file stores for the class. A tree fits an object +when the type hashes are equal and its read consumes exactly the object's serialized size. + +Nothing here imports UnityPy until typetree_node is called. +""" + +from __future__ import annotations + +import json +import re +from importlib import resources + +__all__ = [ + "TYPETREE_FORMAT", + "dump_typetrees", + "load_typetrees", + "script_entry", + "script_types", + "typetree_file", + "typetree_key", + "typetree_node", +] + +TYPETREE_FORMAT = 1 + + +def typetree_key(assembly: str, namespace: str, cls: str) -> str: + """Key of a class in a typetree file: `assembly|namespace|class`, as its MonoScript names them.""" + return f"{assembly}|{namespace}|{cls}" + + +def script_entry(assembly: str | None, namespace: str | None, cls: str | None) -> str: + """The script table entry of a class: "assembly|namespace|class", the assembly without `.dll`, an empty + namespace empty.""" + asm = assembly or "" + if asm.endswith(".dll"): + asm = asm[:-4] + return f"{asm}|{namespace or ''}|{cls or ''}" + + +def typetree_file(unity_version: str): + """The embedded typetree file of a Unity version (an importlib.resources Traversable; it may not exist).""" + return resources.files("nnnotes.unity").joinpath("typetrees", f"{unity_version}.json") + + +def load_typetrees(unity_version: str) -> dict | None: + """The embedded typetrees of a Unity version, or None when nnnotes has none for it.""" + if not re.fullmatch(r"[0-9A-Za-z.]+", unity_version or ""): + return None + f = typetree_file(unity_version) + if not f.is_file(): + return None + doc = json.loads(f.read_text(encoding="utf-8")) + if doc.get("format") != TYPETREE_FORMAT or doc.get("unityVersion") != unity_version: + raise ValueError(f"typetree file {unity_version}.json: unexpected format or Unity version") + return doc + + +def script_types(unity_version: str) -> dict[str, dict]: + """{script table entry (script_entry): {"typeHash", "nodes"}} of the embedded typetrees of a Unity version ({} + when nnnotes has none for it).""" + doc = load_typetrees(unity_version) + if doc is None: + return {} + return {script_entry(*k.split("|", 2)): v for k, v in sorted(doc["classes"].items())} + + +def typetree_node(rows: list): + """UnityPy TypeTreeNode from [level, type, name, meta flag] rows (byte size and version 0, as generated).""" + from UnityPy.helpers.TypeTreeNode import TypeTreeNode + + return TypeTreeNode.from_list([TypeTreeNode(lv, ty, nm, 0, 0, m_MetaFlag=mf) for lv, ty, nm, mf in rows]) + + +def dump_typetrees(doc: dict) -> str: + """Text of a typetree file: sorted keys, one class per line, LF.""" + head = {k: v for k, v in doc.items() if k != "classes"} + lines = [json.dumps(head, sort_keys=True, ensure_ascii=False, separators=(",", ":"))[:-1] + ',"classes":{'] + items = sorted(doc["classes"].items()) + for i, (k, v) in enumerate(items): + lines.append( + json.dumps(k, ensure_ascii=False) + + ":" + + json.dumps(v, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + + ("," if i + 1 < len(items) else "") + ) + lines.append("}}") + return "\n".join(lines) + "\n" diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index 2e10549..85388f5 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -123,6 +123,25 @@ def bundle_bytes(doc: dict) -> bytes: return b"UnityFS\0" + contract.encode(doc) +def builtin_census() -> dict: + """The census of a player's data.unity3d: files without type trees and AssetBundle objects.""" + doc = cdoc("globalgamemanagers.assets", [obj(2, "Shader", "Blit")]) + head = dict(doc["files"][0], typetree=False) + files = [head, dict(head, name="unity_builtin_extra", objects=[obj(7, "Shader", "UI_Default")])] + return dict(doc, files=files, assetBundles=[], classes={"Shader": 2}, objects=2) + + +def builtin_apk(path, extra=()) -> Path: + """An APK holding the player's data.unity3d (builtin_census) and `extra` members {name: bytes}.""" + import zipfile + + with zipfile.ZipFile(path, "w") as z: + z.writestr("assets/bin/Data/data.unity3d", bundle_bytes(builtin_census())) + for n, data in dict(extra).items(): + z.writestr(n, data) + return path + + def catalog_bytes(extra=()) -> bytes: remote = {n: synth.remote(NAMES[n]) for n in NAMES} entries = [(NAMES[n], remote[n], []) for n in ("card_1", "card_2", "shared", "bg_x")] + list(extra) @@ -199,7 +218,7 @@ def run(self, task, depot): scripts = contract.loads(depot.input_bytes(task.input("scripts")))["scripts"] if any(o["class"] == "Crash" for f in doc["files"] for o in f["objects"]): raise ConfigError("setting paths.tool is not set") - container = {(e["file"], e["pathId"]): e["path"] for e in doc["assetBundles"][0]["container"]} + container = {(e["file"], e["pathId"]): e["path"] for b in doc["assetBundles"] for e in b["container"]} only = task.params["classes"] arts, items, first = [], [], None for f in doc["files"]: @@ -330,6 +349,60 @@ def test_select_adds_the_dependencies_to_the_census(): assert assets.select(idx, ["all"]).selected == sorted(stable.values()) +def test_select_always_census_the_built_in_files(): + from nnnotes.pipeline.catalogdb import index + + idx = index(catalog_bytes()) + stable = {n: contract.stable_bundle_name(NAMES[n]) for n in NAMES} + own = ["apk/data.unity3d", "apk/Resources/unity default resources"] + s = assets.select(idx, ["all"], own) + assert s.selected == s.census == sorted([*stable.values(), *own]) + s = assets.select(idx, ["group:bg"], own) + assert s.selected == [stable["bg_x"]] and s.census == sorted([stable["bg_x"], *own]) + s = assets.select(idx, ["bundle:apk/data*"], own) + assert s.selected == ["apk/data.unity3d"] and s.census == sorted(own) + + +def test_the_built_in_files_of_the_apk_are_exported(tmp_path, capsys, fake, monkeypatch): + monkeypatch.setattr("nnnotes.pipeline.catalogdb.apk_version", lambda apk: None) + d = setup_data(tmp_path, censuses(broken=False)) + apk = builtin_apk(tmp_path / "base.apk", {"assets/aa/catalog.bin": synth.CatalogWriter().build([])}) + base = [*flags(d), "--apk", apk, "depot", "plan", "--depot", tmp_path / "s", "--json"] + code, out, err = nn(capsys, *base, "--select", "group:bg") + ids = {n["id"] for n in json.loads(out)["nodes"]} + assert "unity.census:apk/data.unity3d" in ids and "unity.export:apk/data.unity3d" not in ids + code, out, err = nn( + capsys, + *flags(d), + "--apk", + apk, + "export", + "assets", + "-o", + tmp_path / "o", + "--depot", + tmp_path / "s", + "--workers", + "0", + ) + assert code == 0, err + root = tmp_path / "o" / "_bundles" / "apk" / "data.unity3d" + assert (root / "unity_builtin_extra" / "Shader" / "UI_Default~7.json").is_file() + assert (root / "globalgamemanagers.assets" / "Shader" / "Blit~2.json").is_file() + run = contract.loads((tmp_path / "o" / assets.REPORTS / "run.json").read_bytes()) + depot = Depot(tmp_path / "s") + key = {t["id"]: t["key"] for t in run["tasks"]}["link.addresses:main"] + rec = depot.result(key)["artifacts"][0] + table = contract.loads(depot.read(rec["content"]["sha256"])) + assert table["files"]["unity_builtin_extra"] == table["files"]["globalgamemanagers.assets"] == "apk/data.unity3d" + assert table["bundles"]["apk/data.unity3d"] == { + "name": "data.unity3d", + "files": ["globalgamemanagers.assets", "unity_builtin_extra"], + "assetBundleName": None, + "keys": [], + } + + # ---------------------------------------------------------------- export def test_export_writes_layouts_reports_and_hits_the_second_time(tmp_path, capsys, fake): d = setup_data(tmp_path, depots=("depot",)) @@ -1196,7 +1269,6 @@ def run(self, task, depot): def raw_apk(tmp_path, monkeypatch): """An APK with one raw file (a CRI sheet of the APK's catalog) and the CRI stage stand-in; -> (apk, name).""" - import zipfile monkeypatch.setattr( assets, @@ -1205,11 +1277,10 @@ def raw_apk(tmp_path, monkeypatch): ) monkeypatch.setattr("nnnotes.pipeline.catalogdb.apk_version", lambda apk: None) name = "cri_assets_embcri/sound/initialse_" + "ab" * 16 - apk = tmp_path / "base.apk" - with zipfile.ZipFile(apk, "w") as z: - z.writestr("assets/aa/catalog.bin", synth.CatalogWriter().build([("Cri/Initial/se", synth.local(name), [])])) - z.writestr("assets/aa/Android/" + name, b"@UTF sheet") - z.writestr("assets/bin/Data/data.unity3d", b"boot") + catalog = synth.CatalogWriter().build([("Cri/Initial/se", synth.local(name), [])]) + apk = builtin_apk( + tmp_path / "base.apk", {"assets/aa/catalog.bin": catalog, "assets/aa/Android/" + name: b"@UTF sheet"} + ) return apk, name diff --git a/tests/golden/unity.export.json b/tests/golden/unity.export.json index d310ee3..434f76a 100644 --- a/tests/golden/unity.export.json +++ b/tests/golden/unity.export.json @@ -7,7 +7,7 @@ "cri.usm": "cri.usm/1", "data": "data/2", "font": "font/2", - "generic.json": "generic.json/1", + "generic.json": "generic.json/2", "gltf.write": "numpy-2.4.6/1", "material.json": "material.json/1", "mesh.arrays": "numpy-2.4.6/1", @@ -26,68 +26,68 @@ }, "fixtures": { "all": { - "key": "10585d61586a4362e7e5a3c8857a9b583fac363abb5874e760d767fde9cb7d30", - "result": "3bd7b57ee922de132411cb8d027a00421fed5c5a595cdb3eef3e3cb5982b7374" + "key": "bfa791aaeaa8744197413dca21b95285355376a687dcb4a5b511410b4761bd28", + "result": "d9f811e5ca222529d22d1c5e2ec9cd89328d0229eac92b7e01d0cb59dbc56077" }, "bundle.json": { - "key": "fb99b80d44ff61bf4b90ea87e40a632c0f34372ddebd60dd00f74368c5f6b94d", - "result": "169b9bab48a8cb6423de878430b6291bf56999a5d640e9893dc2a1eeda9b93f9" + "key": "019bd61d3b8989c2f59289354d72d3857ed37cab1d90316359e6702ebab8a515", + "result": "f1516133579ff990af232a7b6d8b6303104c59997d20905f16aa280403d3a09e" }, "clip.json": { - "key": "535034c7ce7875a3ea19ebadbb4d12bca912621d38982cd47097cb1c66ff62c5", - "result": "fad3849534a5ad3892f968f567c88efa426088232df6e26a8f646757592fa135" + "key": "a9f5fb2a87fbcb197a5bc37f1c520b7b5a4a6fd5b5ad38b1198e9f8601dc223b", + "result": "6448026702f2239970f8041906335b3a2302a4463536d2f0394daa36906d9c79" }, "controller.json": { - "key": "522f6cefe4102041b197c0feda95de6668a7e88ab548b0d682ef26b492079fd1", - "result": "f17bc2432ad36b87f17e3fffec59eab429a803d23f0bc63db018ba2d99a847db" + "key": "472e6426b3af184e4fb17ccc0a147bd7b6fc9e5fc49f32d0ab2c2d79238417e1", + "result": "c43f0d2de335f10063d20f2d208b54cf7da37863860441a22781bce35f0cb1cc" }, "cri.usm": { - "key": "8d1d45652a6a631253b3bad7f096af988fcc3894831a8fe5e7ec90b939bc3427", - "result": "9a0ca28ce190fc8b5c03fc108b657825c1361a05b416894d2f9c55402cae62fd" + "key": "72cba01cb4fe8fa95d052a2fd17380eaacdd722dc0e5fc802bce8a301560e7a0", + "result": "9fecf00c65642c4ebb635839f79d9001cf4308b367711c2acaf82fa53326cff5" }, "data": { - "key": "98f3c28baedd007e88d9a48adccdad2a1970e63df6e487917cf1c6823feb651d", - "result": "a2c0972bc1c388038dedb9b70f4a8a6d67b58dfca0faac3c438610df125c7234" + "key": "1b44691876cbf5e28032ac2858d9616a8445e2357c543291e2806506c8c0a0b4", + "result": "cdaf30aeb760eef66ec41e029940a800af4d72b0ca0c4545c9c911b6c94fd331" }, "font": { - "key": "c1a50c8ae58a104313d93a7296464812f98c82d8c9aa516589b9d6e5cb89bffa", - "result": "d18d119de09ed000abca438357e72951a18d576c2b26b76932adf9621f67d2bd" + "key": "7d5549079bee9a0778d9d698545f21bfd0fea5466e0efa160661cfb527a83ea9", + "result": "d58abbc2a1b9990fadca993ac0502dc2d586b79e4947bfd1b6fb0a9f7bbc41b0" }, "generic.json": { - "key": "3f111bf015cc23af61fb457ad94be8c3479259d2e8ec2388390580161179afde", - "result": "a79ea2404c179705d54adb9c08f917acedd070bffccc01ba72da3f68af522cd4" + "key": "22deb0da7906ad444ca282fb2a8f993106268f748f2fab35357c049e5b020410", + "result": "a41f325a63c894bfe54aba013c1ee7aa6d674fddd49715c5e3898b7f40e6559c" }, "material.json": { - "key": "fd0603efb48c4582fa8664efcc44be7e238dc6168846dd5528ac6ec350732903", - "result": "373545644b49d9047bae346a78186d7bd0b1b5158d17a393b42050e505209e32" + "key": "b8b07211d9dbc578b3541a0e59653b01506a211426cc70b6e0aa2e82a948167e", + "result": "09085e48baebbda4a78a076e5dc66eeb19ae1f53710ec9f9b3553bb86dfdefde" }, "mesh.glb": { - "key": "d14c16a5c18145cc039a4568b4c8561b0120644c2de1e1abd1dd1be1f17917f0", - "result": "650489a1a9d4e52ae66bc44a101ad5c127e9aa836c8d69e4c08c662e70c73b47" + "key": "1c9b9a873c782e16650ef40cc0a41c17d37da55e05da6d44f47c488cc12d5027", + "result": "58304a1fd371109dbc38f99cd578ad8238453e769ae8c9d8562e1e187e2df1bc" }, "mono.json": { - "key": "5820ec6ca51a697d9ad2f3ed1ee5c8a849fd8f813b053f335b86b1b95082abd8", - "result": "cdabce4d56148f552c87738a9208fd36d0cc581590d7e5eb92f8e5b522de2266" + "key": "f2109d87be61d8cf5b0b527326d2cdc474785104091c55e770a18b31cd967b23", + "result": "02bc065f80513aad7e9afc21704d0f9591e0f110c9ff6fd818f357fe139686f8" }, "prefab.json": { - "key": "8994f1ef92388d443797f11c65a61a6346acf01ff0dc8bfbd36dc2fb2d2d23d2", - "result": "368986bcb1df65ea7fb906e01fe49b814c35e41212a079cdfcc5b70d5ed06559" + "key": "583e77337b68385457bb1f7ec789c943d69729706adc18b47d59354bd04fcf94", + "result": "3751c24516ea904f342e272234e57357b4c5d03ef6e90637c5c8bffbbca78d2a" }, "scripts.json": { - "key": "aafda4c0fe111458f3b48e85b8f60d0e9234dd250590591966a8ac3305d99ef9", - "result": "a2644c8fe5f8198b3771374f10ad01b91a9b111d74aff79b85a9b9b2e071e2c5" + "key": "6e6a37fb9135463601b78a8a5f699996221242585bcc31c63863158ca0a3f9dd", + "result": "9e9b03c721f1f163ed6e81c202b767a7af84be95e662d4d3b5a9b8d46bfe3392" }, "shader.json": { - "key": "a71aa982ce0da9929eaf721a1db903ac0565e1232c94a1d451cdf818f2c59f31", - "result": "fef75615b20f6dc056a4934606d1f094159e219b87482b5efae95c407d0f6d55" + "key": "f3c2c279ab2ed59a70932b19111c4f248593ec3ca9e0677b9402c38cb8505d03", + "result": "dac4f3d4f225dd3659ac78ab0b344daaa64e1a0a6f7dfe759f3ab49ffe3b3c43" }, "sprite.png": { - "key": "d42d38c45711b2de94fc4509b0dd8d919d01d123091f09d54e177df7e4cf751d", - "result": "5c6d4968cbc56a1eefb9c8c94b380487f9876f591e33acbef40bb735edbb70b9" + "key": "d35fe9220f41f10d16b8506f48f31774e9303a8ce6e7b2deba3eb2b26edc56d9", + "result": "ebe1ea68a2fb3289366624724ec4a9335ff6e42156ce4003c7669106e3981482" }, "tex.png": { - "key": "a5af40387bb2c9fe6ba31415af4e4028163eab23bd4114ad4384b57159ae1318", - "result": "54d91ffc00400f162f677f9e1525e34698b33dc439560b4372dffcd09a8dd73b" + "key": "1408f5b984cbcc1c74d23e3c6ef518288f2d112b6b2e4007aecb7f811ddfc809", + "result": "fec1a912d37045add008ea5c12941ea9c6e9eeb0cc0adb6b4d5be81c89ac26eb" } }, "libraries": { diff --git a/tests/helpers/fakeunity.py b/tests/helpers/fakeunity.py index 269cc1e..90fbbfd 100644 --- a/tests/helpers/fakeunity.py +++ b/tests/helpers/fakeunity.py @@ -62,6 +62,7 @@ "Animator": 95, "MonoBehaviour": 114, "MonoScript": 115, + "PlayerSettings": 129, "Font": 128, "AssetBundle": 142, "AnimatorController": 91, @@ -501,6 +502,9 @@ def node_of( raise TypeError(f"{name}: no node for {type(value).__name__}") +HEADER_FIELDS = ("m_GameObject", "m_Enabled", "m_Script", "m_Name") # a MonoBehaviour's own fields + + class FakeObjectReader: def __init__( self, @@ -515,8 +519,10 @@ def __init__( platform: int = ANDROID, node=None, bytes_fields=(), + short_read: bool = False, ): self.assets_file, self.path_id, self.tt, self.obj, self.raw = assets_file, path_id, tt, obj, raw + self.short_read = short_read # its class's tree reads fewer bytes than it has: a checked whole read fails self.class_id = CLASS_IDS[class_name] self.type = SimpleNamespace(name=class_name, value=self.class_id) self.byte_size = len(raw) @@ -525,18 +531,30 @@ def __init__( self.typetree_reads = 0 self._node, self._bytes_fields = node, tuple(bytes_fields) + def _stripped(self) -> bool: + """A MonoBehaviour of a file that stores no type trees: its class's own tree reads its header only.""" + return self.type.name == "MonoBehaviour" and not self.assets_file._enable_type_tree + def read_typetree(self, nodes=None, wrap: bool = False, check_read: bool = True): - """The typetree (a copy); with a cut node (UnityPy's head reads) only the fields the node names.""" + """The typetree (a copy); with a cut node (UnityPy's head reads) only the fields the node names. A stripped + MonoBehaviour read with a tree that does not name all its fields fails the size check.""" self.typetree_reads += 1 tt = copy.deepcopy(self.tt) + if nodes is None and self._stripped(): + nodes = self._get_typetree_node() + if nodes is None and check_read and self.short_read: + raise ValueError("Expected to read 996 bytes, but only read 992 bytes") if nodes is not None: names = [c.m_Name for c in nodes.m_Children] + if check_read and self._stripped() and set(tt) - set(names): + raise ValueError(f"Expected to read {len(self.raw) or 100} bytes, but only read 16 bytes") tt = {k: v for k, v in tt.items() if k in names} return tt def _get_typetree_node(self, node=None): if self._node is None: - self._node = node_of(self.tt, "Base", self.type.name, self._bytes_fields) + tt = {k: v for k, v in self.tt.items() if k in HEADER_FIELDS} if self._stripped() else self.tt + self._node = node_of(tt, "Base", self.type.name, self._bytes_fields) return self._node def peek_name(self): @@ -560,8 +578,11 @@ def read_bytes(self, n: int) -> bytes: class FakeSerializedFile: - def __init__(self, name: str, environment, externals=(), version=VERSION, unity_version="6000.3.12f1"): + def __init__( + self, name: str, environment, externals=(), version=VERSION, unity_version="6000.3.12f1", typetrees=True + ): self.name, self.environment, self.version, self.unity_version = name, environment, version, unity_version + self._enable_type_tree = typetrees self.objects: dict[int, FakeObjectReader] = {} self.externals = [SimpleNamespace(path=p) for p in externals] self.container = SimpleNamespace(container=[]) @@ -585,8 +606,8 @@ def __init__(self): self.resources: dict[str, bytes] = {} self.lookups: list[str] = [] - def file(self, name: str, externals=()) -> FakeSerializedFile: - f = FakeSerializedFile(name, self, externals) + def file(self, name: str, externals=(), typetrees: bool = True) -> FakeSerializedFile: + f = FakeSerializedFile(name, self, externals, typetrees=typetrees) self.assets.append(f) return f diff --git a/tests/pipeline/test_builtins.py b/tests/pipeline/test_builtins.py new file mode 100644 index 0000000..be9adec --- /dev/null +++ b/tests/pipeline/test_builtins.py @@ -0,0 +1,47 @@ +import zipfile +from unittest import mock + +from nnnotes.pipeline import builtins, contract +from nnnotes.pipeline.contract import Input +from nnnotes.pipeline.depot import Depot + +DATA = b"UnityFS data" +DEFAULTS = b"\0\0\0\0 a serialized file" + + +def apk(path, members): + with zipfile.ZipFile(path, "w") as z: + z.writestr("classes.dex", b"dex") + for m, data in members.items(): + z.writestr(builtins.DATA_DIR + m, data) + return path + + +def test_the_built_in_files_of_an_apk_are_subjects(tmp_path): + depot = Depot(tmp_path / "s") + a = apk(tmp_path / "base.apk", {"data.unity3d": DATA, "Resources/unity default resources": DEFAULTS}) + got = builtins.inputs(depot, a) + assert got == { + "apk/Resources/unity default resources": Input( + "bundle", contract.sha256(DEFAULTS), len(DEFAULTS), "unity default resources", ({"kind": "depot"},) + ), + "apk/data.unity3d": Input("bundle", contract.sha256(DATA), len(DATA), "data.unity3d", ({"kind": "depot"},)), + } + assert depot.read(contract.sha256(DEFAULTS)) == DEFAULTS + with mock.patch.object(builtins, "ApkSet", side_effect=AssertionError("read again")): + assert builtins.inputs(Depot(tmp_path / "s"), a) == got # remembered by the APK's content + assert builtins.inputs(depot, None) == {} + + +def test_an_apk_directory_and_its_members(tmp_path): + d = tmp_path / "apk" + d.mkdir() + apk(d / "base.apk", {"data.unity3d": DATA, "datapack.unity3d": b"UnityFS pack"}) + assert sorted(builtins.inputs(Depot(tmp_path / "s"), d)) == ["apk/data.unity3d", "apk/datapack.unity3d"] + + +def test_serialized_names(): + assert builtins.serialized_name("apk/Resources/unity default resources") == "unity default resources" + assert builtins.serialized_name("apk/data.unity3d") == "data.unity3d" + assert builtins.serialized_name("card_1") is None + assert builtins.is_builtin("apk/data.unity3d") and not builtins.is_builtin("card_1") diff --git a/tests/pipeline/test_census.py b/tests/pipeline/test_census.py index 933a34b..d215e85 100644 --- a/tests/pipeline/test_census.py +++ b/tests/pipeline/test_census.py @@ -295,8 +295,8 @@ def test_census_stage(tmp_path, monkeypatch): bundle.write_bytes(b"UnityFS\0 not really") loaded = [] - def load(path): - loaded.append(str(path)) + def load(path, name=None): + loaded.append((str(path), name)) return scene_bundle()[0] monkeypatch.setattr(unity, "load", load) @@ -316,10 +316,15 @@ def load(path): assert rec["id"] == "unity.census:x#census" and rec["semantics"]["facts"]["objects"] == 7 assert rec["provenance"]["inputs"] == [{"role": "bundle", "sha256": sha}] assert contract.loads(depot.read(rec["content"]["sha256"])) == census.census_env(scene_bundle()[0]) - assert loaded == [str(bundle)] + assert loaded == [(str(bundle), None)] # a catalog bundle: its files keep their names assert execute(task, depot, registry(stage)).status == "hit" and len(loaded) == 1 with pytest.raises(ValueError, match="expected 'bundle'"): describe(stage, "x", None, Env(depot, {"bundles": {"x": Input("other", sha, size)}})) for waiting in (None, Pending("fetch:x_1.bundle")): with pytest.raises(Pending, match="fetch:x"): describe(stage, "x", None, Env(depot, {"bundles": {"x": waiting}})) + # a serialized file of the APK outside a bundle is loaded under the name references into it use + own = "apk/Resources/unity default resources" + task = describe(stage, own, None, Env(depot, {"bundles": {own: inp}})) + assert execute(task, depot, registry(stage)).status == "ran" + assert loaded[-1] == (str(bundle), "unity default resources") diff --git a/tests/pipeline/test_cristages.py b/tests/pipeline/test_cristages.py index 4f2c418..97b24f5 100644 --- a/tests/pipeline/test_cristages.py +++ b/tests/pipeline/test_cristages.py @@ -15,7 +15,7 @@ from nnnotes.core import cache from nnnotes.formats import cri, crikey -from nnnotes.pipeline import contract, cristages +from nnnotes.pipeline import builtins, contract, cristages from nnnotes.pipeline.contract import IncompatibleTask, Input, Task from nnnotes.pipeline.cristages import AUDIO, AudioStage, MovieStage from nnnotes.pipeline.depot import Depot @@ -106,14 +106,19 @@ def catalog_index(keys: dict, bundles=()) -> dict: } +def boot_input(depot, tmp) -> Input: + apk = tmp / "base.apk" + with zipfile.ZipFile(apk, "w") as z: + z.writestr(builtins.DATA_DIR + "data.unity3d", b"UnityFS boot data") + return cristages.boot_input(builtins.inputs(depot, apk)) + + def planning(depot, tmp, files: dict, keys=None, boot=True, bundles=()) -> Env: raw = {s: file_input(depot, tmp, s, d) for s, d in files.items()} - b = tmp / "data.unity3d" - b.write_bytes(b"UnityFS boot data") facts = { "raw": raw, "index": catalog_index(keys or {}, bundles), - "boot": cristages.boot_input(depot, b) if boot else None, + "boot": boot_input(depot, tmp) if boot else None, } return Env(depot, facts) @@ -163,28 +168,17 @@ def load(depot, doc, aid): # ---------------------------------------------------------------- the key's source -def test_boot_input_from_an_apk_is_stored_once(tmp_path): +def test_boot_input_is_the_data_of_the_apk(tmp_path): depot = Depot(tmp_path / "s") - apk = tmp_path / "base.apk" - with zipfile.ZipFile(apk, "w") as z: - z.writestr(cristages.BOOT_IN_APK, b"UnityFS boot") - z.writestr("classes.dex", b"dex") - a = cristages.boot_input(depot, apk) + a = boot_input(depot, tmp_path) assert (a.role, a.sha256, a.size, a.name, a.locators) == ( "boot", - contract.sha256(b"UnityFS boot"), - 12, + contract.sha256(b"UnityFS boot data"), + 17, "data.unity3d", ({"kind": "depot"},), ) - assert depot.read(a.sha256) == b"UnityFS boot" - with mock.patch.object(zipfile, "ZipFile", side_effect=AssertionError("read again")): - assert cristages.boot_input(Depot(tmp_path / "s"), apk) == a # remembered by the APK's content - d = tmp_path / "data.unity3d" - d.write_bytes(b"UnityFS other") - b = cristages.boot_input(depot, d) - assert b.sha256 == contract.sha256(b"UnityFS other") and b.locators[0]["kind"] == "file" - assert cristages.boot_input(depot, None) is None + assert cristages.boot_input({}) is None def test_hca_key_is_read_once_per_content(tmp_path): diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index ddee38c..6689c94 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -560,10 +560,62 @@ def bundle_movie() -> FakeEnvironment: return env -BUNDLES = {"all": bundle_all, "atlas": bundle_atlas, "plain": bundle_plain, "movie": bundle_movie} +# a player's own file: no type trees; MonoBehaviours whose classes have a shipped type tree (one of another type +# hash), and none +PLAYER = "globalgamemanagers.assets" +PLAYER_HASH, OTHER_HASH = "a1" * 16, "b2" * 16 +SETTINGS, IMAGE, CHANGED = "Fwk|Fwk.Sound|SoundVolumeSettings", "UnityEngine.UI|UnityEngine.UI|Image", "Game|Game|Moved" +PLAYER_SCRIPTS = {f"{PLAYER}:50": SETTINGS, f"{PLAYER}:51": IMAGE, f"{PLAYER}:52": CHANGED} +SETTINGS_ROWS = [ + [0, "MonoBehaviour", "Base", 0], + [1, "PPtr", "m_GameObject", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + [1, "UInt8", "m_Enabled", 0x4000], + [1, "PPtr", "m_Script", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + [1, "string", "m_Name", 0], + [1, "PPtr", "target", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + [1, "float", "volume", 0], +] +PLAYER_TYPETREES = { + SETTINGS: {"typeHash": PLAYER_HASH, "nodes": SETTINGS_ROWS}, + CHANGED: {"typeHash": OTHER_HASH, "nodes": SETTINGS_ROWS}, +} + + +def bundle_player() -> FakeEnvironment: + env = FakeEnvironment() + f = env.file(PLAYER, externals=["Library/unity default resources"], typetrees=False) + h = bytes.fromhex(PLAYER_HASH) + settings = {"productGUID": "00", "m_ActiveColorSpace": 1, "m_MTRendering": True} + f.add(1, "PlayerSettings", settings, type_hash=bytes.fromhex("936fd1913a4f0b5da2ce7cadb0ce9666"), short_read=True) + f.add(6, "PlayerSettings", settings, type_hash=h, short_read=True) + f.add(2, "MonoBehaviour", _mono(0, pptr(50), "volume", target=pptr(10, 1), volume=0.5), type_hash=h) + f.add(3, "MonoBehaviour", _mono(20, pptr(51), "", m_Sprite=pptr(10, 1)), type_hash=h) + f.add(4, "MonoBehaviour", _mono(0, pptr(52), "moved", target=pptr(0), volume=1.0), type_hash=h) + f.add(5, "MonoBehaviour", _mono(0, pptr(51), "image", m_Sprite=pptr(10, 1)), type_hash=h) + f.add(20, "GameObject", _go("Canvas", [21, 3])) + f.add(21, "Transform", _transform(20)) + for pid, entry in ((50, SETTINGS), (51, IMAGE), (52, CHANGED)): + asm, ns, cls = entry.split("|") + f.add(pid, "MonoScript", {"m_Name": cls, "m_ClassName": cls, "m_Namespace": ns, "m_AssemblyName": asm}) + return env -def open_fake(data: bytes) -> UnityBundle: +BUNDLES = { + "all": bundle_all, + "atlas": bundle_atlas, + "plain": bundle_plain, + "movie": bundle_movie, + "player": bundle_player, +} + + +def open_fake(data: bytes, name: str | None = None) -> UnityBundle: return UnityBundle(BUNDLES[bytes(data).decode().removeprefix("fake:")]()) @@ -574,7 +626,7 @@ def classes_of(name): return sorted({r.type.name for f in BUNDLES[name]().assets for r in f.objects.values()}) -def task_for(depot, name, classes=None, params=None, scripts=None): +def task_for(depot, name, classes=None, params=None, scripts=None, typetrees=None): data = f"fake:{name}".encode() sha = depot.put(data) stage = ExportStage(FAKE_READER) @@ -586,7 +638,7 @@ def task_for(depot, name, classes=None, params=None, scripts=None): p, export_atoms(classes_of(name), FAKE_READER.id), (Input("bundle", sha, len(data), f"{name}.bundle", ({"kind": "depot"},)),), - {"scripts": dict(SCRIPTS if scripts is None else scripts)}, + {"scripts": dict(SCRIPTS if scripts is None else scripts), **({"typetrees": typetrees} if typetrees else {})}, ) @@ -1268,7 +1320,7 @@ def census_doc(view: UnityBundle) -> dict: continue classes[o.class_name] = classes.get(o.class_name, 0) + 1 rec = {"pathId": o.path_id, "classId": o.class_id, "class": o.class_name} - tt = view.typetree(o) + tt = view.reader(o).read_typetree(check_read=False) # a header read, as the census' ext = view.externals(f) def ref(s): @@ -1286,7 +1338,9 @@ def ref(s): if bindings: rec["bindingScripts"] = [ref(b) for b in bindings] objs.append(rec) - files.append({"name": f, "objects": objs}) + files.append( + {"name": f, "unityVersion": view.unity_version(f), "typetree": view.stores_typetrees(f), "objects": objs} + ) return { "schema": contract.CENSUS, "files": files, @@ -1357,6 +1411,68 @@ def test_describe_reads_the_census_and_the_script_table(tmp_path): describe(stage, "all", None, fresh) +def test_files_without_type_trees_read_their_scripts_with_the_shipped_ones(tmp_path): + depot = Depot(tmp_path / "s") + doc = run(depot, task_for(depot, "player", scripts=PLAYER_SCRIPTS, typetrees=PLAYER_TYPETREES)) + items = by_object(doc) + + def p(pid): + return oid(pid, PLAYER) + + def script(pid, cls): + return {"$ref": {"file": PLAYER, "pathId": pid, "class": "MonoScript", "name": cls}} + + assert items[p(2)]["status"] == "exported" + assert obj(depot, doc, f"{p(2)}#json") == { + "$script": SETTINGS, + "m_GameObject": None, + "m_Enabled": 1, + "m_Script": script(50, "SoundVolumeSettings"), + "m_Name": "volume", + "target": {"$ref": {"file": "unity default resources", "pathId": 10}}, + "volume": 0.5, + } + assert items[p(2)]["refs"] == [p(50), "unity default resources:10"] + # a class without a shipped type tree, or with one of another type hash: its header, and the reason + for pid, why in ((5, f"has none for {IMAGE}"), (4, f"type hash {PLAYER_HASH} is not the type tree's")): + assert items[p(pid)]["status"] == "generic" + assert items[p(pid)]["reason"]["code"] == "unsupported.typetree" and why in items[p(pid)]["reason"]["message"] + assert obj(depot, doc, f"{p(5)}#json") == { + "m_GameObject": None, + "m_Enabled": 1, + "m_Script": script(51, "Image"), + "m_Name": "image", + } + # a hierarchy keeps its components' headers and says so + assert art(doc, f"{p(20)}#json")["semantics"]["issues"] == [ + { + "code": "unsupported.typetree", + "message": "1 components read in part (their file stores no type tree that fits)", + } + ] + # a class whose type tree reads the stored layout right up to a known field (by its type hash): those fields + assert items[p(1)]["status"] == "generic" and items[p(1)]["reason"]["code"] == "unsupported.typetree" + assert "read up to m_ActiveColorSpace" in items[p(1)]["reason"]["message"] + assert obj(depot, doc, f"{p(1)}#json") == {"productGUID": "00", "m_ActiveColorSpace": 1} + assert items[p(6)]["status"] == "failed" and items[p(6)]["reason"]["code"] == "failed.read" + (component,) = obj(depot, doc, f"{p(20)}#json")["nodes"][0]["components"] + assert component["script"] == IMAGE and set(component["fields"]) == {"m_Enabled", "m_Script", "m_Name"} + assert items[p(3)]["status"] == "contained" + + +def test_the_context_holds_the_shipped_type_trees_of_its_scripts(tmp_path): + from nnnotes.unity import scripttypes + + shipped = scripttypes.load_typetrees("6000.3.12f1")["classes"]["DOTween.dll|DG.Tweening.Core|DOTweenSettings"] + entry = "DOTween|DG.Tweening.Core|DOTweenSettings" # as the script table names it + env = planning_env(Depot(tmp_path / "s"), names=("player",), scripts={**PLAYER_SCRIPTS, f"{PLAYER}:50": entry}) + t = describe(ExportStage(FAKE_READER), "player", None, env) + assert t.context["typetrees"] == {entry: shipped} + assert ( + "typetrees" not in describe(ExportStage(FAKE_READER), "all", None, planning_env(Depot(tmp_path / "t"))).context + ) + + def test_class_selection_keeps_only_its_converters_in_the_key(tmp_path): depot = Depot(tmp_path / "s") env = planning_env(depot) diff --git a/tests/unity/test_typetrees.py b/tests/unity/test_typetrees.py index e62edc1..ce285e6 100644 --- a/tests/unity/test_typetrees.py +++ b/tests/unity/test_typetrees.py @@ -12,17 +12,8 @@ from nnnotes import cli from nnnotes.source.config import ConfigError -from nnnotes.unity.player import ( - HEADER, - TYPETREE_FORMAT, - PlayerData, - UnsupportedVersion, - dump_typetrees, - load_typetrees, - manifest_version_name, - typetree_key, - typetree_node, -) +from nnnotes.unity.player import HEADER, PlayerData, UnsupportedVersion, manifest_version_name +from nnnotes.unity.scripttypes import TYPETREE_FORMAT, dump_typetrees, load_typetrees, typetree_key, typetree_node from tests.helpers import synth TYPETREE_DIR = resources.files("nnnotes.unity").joinpath("typetrees") From 8b97704969e1d746d111d97b539da554501c300c Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 02:50:48 +0800 Subject: [PATCH 10/91] feat(census)!: list what every object references and the hierarchy holding it The census (document nnnotes.census/2, stage version 3) reads every object's typetree and records two new fields per object: `refs`, the object ids its references name, and `unit`, the root GameObject of the hierarchy whose prefab holds it. The objects reachable from a request therefore follow from censuses alone, before anything is converted; they are a superset of the references in the unity.export documents. Shipped MonoBehaviour type trees are now selected by the type hash the file stores for an object. The census of a built-in file of the APK reads its MonoBehaviours with them and names them in its key (atom `typetrees`). BREAKING CHANGE: census documents are nnnotes.census/2, and the unity.export context keys shipped type trees by type hash. Co-Authored-By: Claude Opus 5.5 --- docs/contracts.md | 2 +- docs/schemas.md | 2 +- docs/stages.md | 20 +- src/nnnotes/pipeline/census.py | 203 ++++++++------ src/nnnotes/pipeline/contract.py | 2 +- src/nnnotes/pipeline/objexport.py | 36 +-- src/nnnotes/schemas/census.schema.json | 26 +- src/nnnotes/unity/atoms/reader.py | 4 + src/nnnotes/unity/convert/objects.py | 82 ++++-- src/nnnotes/unity/scripttypes.py | 36 ++- tests/helpers/fakeunity.py | 21 +- tests/pipeline/test_census.py | 366 ++++++++++++------------- tests/pipeline/test_objexport.py | 37 +-- 13 files changed, 480 insertions(+), 357 deletions(-) diff --git a/docs/contracts.md b/docs/contracts.md index 389a28b..5c5569b 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -218,7 +218,7 @@ Documents written by particular stages (artifacts of no object: their ids are `< | Document | Stage | Schema | |---|---|---| | `nnnotes.catalog-index/1` | `catalog.index` | [catalog-index.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalog-index.schema.json) | -| `nnnotes.census/1` | `unity.census` | [census.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/census.schema.json) | +| `nnnotes.census/2` | `unity.census` | [census.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/census.schema.json) | | `nnnotes.scripts/1` | `link.scripts` | [scripts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/scripts.schema.json) | | `nnnotes.addresses/1` | `link.addresses` | [addresses.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/addresses.schema.json) | | `nnnotes.artifacts/1` | `link.artifacts` | [artifacts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/artifacts.schema.json) | diff --git a/docs/schemas.md b/docs/schemas.md index 04a7d89..f0930e7 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -20,7 +20,7 @@ schemas.load("plan") # the schema as a dict | `catalog-diff` | `nnnotes.catalog-diff/1` | the difference between two catalog versions (`catalog diff`) | | `catalog-index` | `nnnotes.catalog-index/1` | the index of a catalog pair: remote and APK catalogs, every location in catalog order | | `catalogs` | `nnnotes.catalogs/1` | the catalog versions of a depot (`catalog versions`) | -| `census` | `nnnotes.census/1` | the census of one bundle: serialized files, type trees and objects, read without decoding | +| `census` | `nnnotes.census/2` | the census of one bundle: serialized files, type trees, objects and what they reference, read without decoding | | `common` | | definitions shared by the other schemas | | `cost` | `nnnotes.cost/1` | the measured cost of a task's last run | | `coverage` | `nnnotes.coverage/1` | the coverage of a run per class and per reason code | diff --git a/docs/stages.md b/docs/stages.md index c6746a8..a350faf 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -12,16 +12,17 @@ The asset export is built from three layers: ## Stages -Every stage is at version 1, except `unity.census` (version 2: it lists the scripts animation clip bindings name) +Every stage is at version 1, except `unity.census` (version 3: it lists what every object references and the +hierarchy holding it) and `cri.movie` (version 2: every stream kind, codec and channel of a USM), and `catalog.index` (version 2: gzip input and Japanese remote asset placeholders). | Stage | Subject | Inputs | Outputs | |---|---|---|---| | `catalog.index` | a catalog (remote + APK) | the catalog files | every location (with the bundle hash, crc and size), bundles and raw remote files | -| `unity.census` | a bundle, or a built-in file of the APK | the bundle | serialized files, externals, objects (file, path id, class, size, type hash, name, script, the scripts an AnimationClip's bindings name), container, scene hashes, dependencies, script fields | +| `unity.census` | a bundle, or a built-in file of the APK | the bundle | serialized files, externals, objects (file, path id, class, size, type hash, name, script, the scripts an AnimationClip's bindings name, the objects it references, the hierarchy holding it), container, scene hashes, dependencies, script fields | | `link.scripts` | global | the censuses holding MonoScripts | serialized file and path id -> assembly, namespace, class | -| `unity.export` | a bundle, or a built-in file of the APK | the bundle, the script entries its MonoBehaviours use (and, for files without type trees, the type trees nnnotes ships for their classes) | the objects' artifacts and one item per object | +| `unity.export` | a bundle, or a built-in file of the APK | the bundle, the script entries its MonoBehaviours use (and, for files without type trees, the type trees nnnotes ships for their type hashes) | the objects' artifacts and one item per object | | `sprite.crop` | a SpriteAtlas | the atlas' JSON, the images of its textures, the metas of the sprites of other bundles it packs | the sprites' images | | `cri.audio` | ACB content | `acb` (a raw file of the catalog, or the `acb` artifact of a cue sheet held in a bundle), `awb` (an AWB the catalog pairs with it), `boot` (the game's boot data: the key's source) | FLAC per stream, `cues.json`, `streams.json` | | `cri.movie` | USM content | `usm` (a raw file of the catalog, or the `usm` artifact of a movie held in a bundle), `boot` | every stream as stored (video, alpha, audio, subtitles), `movie.mkv` (or `movie.webm`) | @@ -65,7 +66,9 @@ with its reason code; any other error of a converter gives it with `generic.erro Every object a converter takes has a `json` artifact or is contained in another object's. Each item lists in `refs` the objects its documents reference (by object id, other bundles' objects included, the objects of its own -prefab left out): the objects reachable from an object follow from the items alone. +prefab left out): the objects reachable from an object follow from the items alone. The census of a bundle lists the +same before anything is converted, as a superset: per object `refs`, every object its typetree references, and +`unit`, the root of the hierarchy whose prefab holds it. A sprite's `json` holds `sprite` (its typetree), `mesh` (its geometry decoded: `positions` in the sprite's units, `submeshes` of triangles, `uv0` when stored), `renderData` (`source`: `renderData` for the sprite's own render @@ -93,10 +96,11 @@ subjects. Census always covers them; everything and `--select bundle:apk/*` expo These files store no type trees. Objects of the engine's classes are read with the type trees UnityPy has for the file's Unity version; MonoBehaviours with the type trees nnnotes ships for the game's classes it reads there (`nnnotes.unity.scripttypes`, generated by `scripts/gen_typetrees.py`): the render pipeline assets, renderer data, -renderer features, post-process data and the game's settings objects. A shipped type tree is used only when the type -hash the file stores is the one it was made for and its read consumes exactly the object's bytes; the task's -context holds the shipped type trees its MonoBehaviours use, so a new shipped tree runs those tasks again. A -MonoBehaviour without one is `unsupported.typetree` with its header (`m_GameObject`, `m_Enabled`, `m_Script`, +renderer features, post-process data and the game's settings objects. A MonoBehaviour takes the shipped type tree +made for the type hash the file stores for it (the hash of its class's serialized layout), and the read must consume +exactly the object's bytes. The export task's context holds the shipped type trees its MonoBehaviours use, and the +census of a built-in file has the shipped type trees' id among its atoms, so a new shipped tree runs those tasks +again. A MonoBehaviour without one is `unsupported.typetree` with its header (`m_GameObject`, `m_Enabled`, `m_Script`, `m_Name`) as its typetree JSON; inside a hierarchy its component keeps that header and the prefab records the issue. PlayerSettings, whose type tree reads the stored layout right up to `m_ActiveColorSpace` (by its stored type hash), gives the fields up to it. diff --git a/src/nnnotes/pipeline/census.py b/src/nnnotes/pipeline/census.py index 7aa1d50..c7b43c3 100644 --- a/src/nnnotes/pipeline/census.py +++ b/src/nnnotes/pipeline/census.py @@ -1,22 +1,31 @@ -"""The census of a bundle (stage unity.census, document nnnotes.census/1): what a bundle holds, read without -decoding any object. +"""The census of a bundle (stage unity.census, document nnnotes.census/2): what a bundle holds and what its +objects reference, read without decoding any object. files the serialized files, sorted by name, each with its Unity version, platform, whether it embeds type trees, its externals ({file, path, guid, type}; `file` is the name an object id uses: the last component of the path, e.g. CAB-... or "unity default resources") and its objects sorted by - path id: {pathId, classId, class, byteSize, typeHash, name, script}. `name` is the object's m_Name - when its type has one (read without the fields after it), `script` the {file, pathId} of a + path id: {pathId, classId, class, byteSize, typeHash, name, script, bindingScripts, refs, unit}. + `name` is the object's m_Name when its type has one, `script` the {file, pathId} of a MonoBehaviour's m_Script (null for a null reference), `bindingScripts` the {file, pathId} of the MonoScripts an AnimationClip's bindings name (MonoBehaviour fields it animates; sorted, only when - there are any). Objects are grouped per file: the files of one bundle may use the same path ids - (a scene bundle's scene and its shared assets). + there are any). `refs` the object ids every reference in its typetree names (sorted, itself left + out; only when there are any), `unit` the object id of the root GameObject of the hierarchy whose + prefab holds it (unity.export converts the objects of a hierarchy together; only for the objects + of a hierarchy other than its root). Objects are grouped per file: the files of one bundle may use + the same path ids (a scene bundle's scene and its shared assets). resources the other files of the bundle (.resS / .resource): {name, size} assetBundles the AssetBundle objects: {file, pathId, name, assetBundleName, dependencies, isStreamedSceneAssetBundle, sceneHashes [[scene, hash]], container [{path, file, pathId, preloadIndex, preloadSize}] (stored order)} scripts the MonoScript objects: {file, pathId, assembly, namespace, class, name} classes objects per class; `objects` the total - errors [{object, message}] of the objects whose header could not be read + errors [{object, message}] of the objects whose typetree could not be read (their record has no name, + script or refs) + +The objects reachable from an object follow from `refs` and `unit` across censuses: they are a superset of what the +documents of unity.export reference (their items' `refs`), so a request's closure computed from censuses holds every +object its products name. A file without type trees (the APK's built-in files) gives a MonoBehaviour's fields only +with a type tree nnnotes ships for its type hash (`shipped`), else its header. A census is a function of the bundle's bytes: it holds no file name, time or path (a serialized file outside a bundle, one of the APK's built-in files, has the name its subject gives: builtins.serialized_name). @@ -26,14 +35,13 @@ import re +from ..unity import scripttypes from ..unity.atoms import impl_id from . import builtins, contract -from .contract import Cost, Input +from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage _ADDRESS = re.compile(r"0x[0-9a-fA-F]{6,}") -HEAD_FIELDS = ("m_Name", "m_Script") -CLIP_FIELDS = ("m_Name", "m_ClipBindingConstant") # an AnimationClip is read up to its bindings def _ext_file(path: str) -> str: @@ -46,7 +54,7 @@ def _message(e: BaseException) -> str: def _serialized(env) -> tuple[list, list]: - """(serialized files, [name, size] of the other files) of a loaded environment, walking nested bundles.""" + """(serialized files, {name, size} of the other files) of a loaded environment, walking nested bundles.""" files, resources = [], [] def walk(container): @@ -70,11 +78,10 @@ def walk(container): class _Refs: - """PPtr resolution within one serialized file.""" + """PPtr resolution within one serialized file: its name and the names of its externals.""" - def __init__(self, sf): - self.name = sf.name - self.externals = [_ext_file(x.path) for x in sf.externals] + def __init__(self, name: str, externals: list[str]): + self.name, self.externals = name, list(externals) def file(self, file_id: int) -> str | None: if file_id == 0: @@ -143,89 +150,80 @@ def _binding_scripts(d: dict, refs: _Refs) -> list: return [out[k] for k in sorted(out)] -def census_env(env) -> dict: - """The census document of a loaded UnityPy environment holding one bundle.""" - from UnityPy.helpers.TypeTreeNode import TypeTreeNode - - heads: dict[tuple, tuple] = {} - - def head(node, fields=HEAD_FIELDS): - """The type tree cut after the last of its `fields` (None when it has none of them).""" - hit = heads.get((id(node), fields)) - if hit is not None and hit[0] is node: - return hit[1] - names = [c.m_Name for c in node.m_Children] - last = max((names.index(n) for n in fields if n in names), default=None) - cut = None - if last is not None: - cut = TypeTreeNode( - node.m_Level, - node.m_Type, - node.m_Name, - node.m_ByteSize, - node.m_Version, - m_MetaFlag=node.m_MetaFlag, - m_Children=node.m_Children[: last + 1], - ) - heads[(id(node), fields)] = (node, cut) - return cut - - sfiles, resources = _serialized(env) +def census_env(env, shipped: bool = False) -> dict: + """The census document of a loaded UnityPy environment holding one bundle (or one serialized file). `shipped`: + the MonoBehaviours of files without type trees are read with the type trees nnnotes ships + (nnnotes.unity.scripttypes) for their type hashes.""" + from ..unity.atoms.reader import UnityBundle + from ..unity.convert import prefab + from ..unity.convert.objects import BundleObjects, ReadError + + _, resources = _serialized(env) + view = UnityBundle(env) + trees: dict = {} + if shipped: + for f in view.files(): + if not view.stores_typetrees(f): + trees.update(scripttypes.script_types(view.unity_version(f))) + ctx = BundleObjects(view, {}, scripts={}, blob_min=1, png_level=0, emit=_no_emit, typetrees=trees) + units = _units(ctx, prefab.hierarchies(ctx)) + by_file: dict[str, list] = {} + for i in ctx.infos: + by_file.setdefault(i.file, []).append(i) files, bundles, scripts, errors, classes = [], [], [], [], {} - for sf in sfiles: - refs = _Refs(sf) + for name in sorted(by_file): + sf = view.serialized_file(name) + refs = _Refs(name, ctx.ext_files[name]) objects = [] - for pid in sorted(sf.objects): - o = sf.objects[pid] - try: - cls = o.type.name - except Exception: - cls = f"Class{o.class_id}" - st = getattr(o, "serialized_type", None) - th = getattr(st, "old_type_hash", None) if st is not None else None + for i in sorted(by_file[name], key=lambda i: i.path_id): + key = (i.file, i.path_id) rec = { - "pathId": int(pid), - "classId": int(o.class_id), - "class": cls, - "byteSize": int(o.byte_size), - "typeHash": th.hex() if th else None, + "pathId": i.path_id, + "classId": i.class_id, + "class": i.class_name, + "byteSize": i.byte_size, + "typeHash": i.type_hash, } - classes[cls] = classes.get(cls, 0) + 1 + classes[i.class_name] = classes.get(i.class_name, 0) + 1 try: - node = o._get_typetree_node() - cut = head(node, CLIP_FIELDS if cls == "AnimationClip" else HEAD_FIELDS) - d = o.read_typetree(cut, check_read=False) if cut is not None else {} - if "m_Name" in d: + d = ctx.tt(key, whole=False) + found = ctx.references(key, d) + except Exception as e: + why = str(e) if isinstance(e, ReadError) else _message(e) + errors.append({"object": contract.object_id(*key), "message": why}) + else: + if isinstance(d.get("m_Name"), str): rec["name"] = d["m_Name"] if "m_Script" in d: rec["script"] = refs.ref(d["m_Script"]) - if cls == "AnimationClip": + if i.class_name == "AnimationClip": scripts_ = _binding_scripts(d, refs) if scripts_: rec["bindingScripts"] = scripts_ - if cls == "AssetBundle": - bundles.append(_asset_bundle(o.read_typetree(), refs, (sf.name, int(pid)))) - elif cls == "MonoScript": - m = o.read_typetree() + if found: + rec["refs"] = sorted(contract.object_id(*k) for k in found) + if i.class_name == "AssetBundle": + bundles.append(_asset_bundle(d, refs, key)) + elif i.class_name == "MonoScript": scripts.append( { - "file": sf.name, - "pathId": int(pid), - "assembly": m.get("m_AssemblyName"), - "namespace": m.get("m_Namespace"), - "class": m.get("m_ClassName"), - "name": m.get("m_Name"), + "file": name, + "pathId": i.path_id, + "assembly": d.get("m_AssemblyName"), + "namespace": d.get("m_Namespace"), + "class": d.get("m_ClassName"), + "name": d.get("m_Name"), } ) - except Exception as e: - errors.append({"object": contract.object_id(sf.name, pid), "message": _message(e)}) + if key in units: + rec["unit"] = units[key] objects.append(rec) files.append( { - "name": sf.name, - "unityVersion": getattr(sf, "unity_version", None), + "name": name, + "unityVersion": view.unity_version(name), "platform": int(getattr(sf, "_m_target_platform", getattr(sf, "target_platform", 0)) or 0), - "typetree": bool(getattr(sf, "_enable_type_tree", True)), + "typetree": view.stores_typetrees(name), "externals": [ { "file": _ext_file(x.path), @@ -250,13 +248,34 @@ def head(node, fields=HEAD_FIELDS): } -def census_file(path, name: str | None = None) -> dict: - """The census of a bundle file (decrypted UnityFS), or of a serialized file named `name`.""" +def _no_emit(*args, **kw): + raise RuntimeError("a census writes no artifact") + + +def _units(ctx, h) -> dict[tuple, str]: + """{object key: the object id of the root GameObject of the hierarchy it belongs to} of the objects of each + hierarchy the converters export as one prefab (prefab.hierarchies: walked without error, rooted at a GameObject), + the root left out.""" + out = {} + for root in h.roots: + go = (root.file, root.game_object) + info = ctx.info.get(go) + if root.error is not None or info is None or info.class_name != "GameObject": + continue + oid = contract.object_id(*go) + for k in root.members: + if k != go: + out[k] = oid + return out + + +def census_file(path, name: str | None = None, shipped: bool = False) -> dict: + """The census of a bundle file (decrypted UnityFS), or of a serialized file named `name` (census_env).""" from ..unity import reader as unity env = unity.load(path, name) try: - return census_env(env) + return census_env(env, shipped) finally: del env @@ -309,10 +328,11 @@ class CensusStage(Stage): (role "bundle": the decrypted bundle's sha256, size, name, locators)}, the APK's built-in files included (builtins); a bundle not fetched yet is Pending (raised by the fact, or given as a Pending / None). Artifact "#census". The census reads the bundle with UnityPy: its atom is the reader's ("reader", the UnityPy - version).""" + version); a built-in file's also the shipped type trees ("typetrees", scripttypes.types_id) it reads its + MonoBehaviours with.""" name = "unity.census" - version = 2 + version = 3 ATOMS = {"reader": impl_id(("UnityPy",), 1)} def subjects(self, env) -> list[str]: @@ -328,12 +348,23 @@ def inputs(self, subject: str, env) -> list[Input]: raise ValueError(f"unity.census {subject}: input role {inp.role!r}, expected 'bundle'") return [inp] + def uses(self, subject: str, env) -> dict: + if builtins.is_builtin(subject): + return {**self.ATOMS, "typetrees": scripttypes.types_id()} + return dict(self.ATOMS) + def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs) - return Cost(0.002 + size / 40e6, (32 << 20) + 12 * size) + return Cost(0.002 + size / 20e6, (32 << 20) + 14 * size) def run(self, task, depot) -> Output: - doc = census_file(depot.open_input(task.input("bundle")), builtins.serialized_name(task.subject)) + if task.atoms.get("typetrees", scripttypes.types_id()) != scripttypes.types_id(): + raise IncompatibleTask(f"task {task.id}: typetrees are {scripttypes.types_id()} here") + doc = census_file( + depot.open_input(task.input("bundle")), + builtins.serialized_name(task.subject), + shipped="typetrees" in task.atoms, + ) content = depot.add(contract.encode(doc), "json") rec = contract.artifact( contract.artifact_id(task.id, "census"), diff --git a/src/nnnotes/pipeline/contract.py b/src/nnnotes/pipeline/contract.py index 26fa343..97ff7f6 100644 --- a/src/nnnotes/pipeline/contract.py +++ b/src/nnnotes/pipeline/contract.py @@ -53,7 +53,7 @@ PLAN = "nnnotes.plan/1" COST = "nnnotes.cost/1" CATALOG_INDEX = "nnnotes.catalog-index/1" -CENSUS = "nnnotes.census/1" +CENSUS = "nnnotes.census/2" VIEW = "nnnotes.view/1" # item statuses of a result: every object a task covers has exactly one diff --git a/src/nnnotes/pipeline/objexport.py b/src/nnnotes/pipeline/objexport.py index e06d5c8..0c40d9f 100644 --- a/src/nnnotes/pipeline/objexport.py +++ b/src/nnnotes/pipeline/objexport.py @@ -24,7 +24,6 @@ import io from collections import defaultdict -from functools import cache from ..unity import scripttypes from ..unity.atoms import ATOMS as ATOM_REGISTRY @@ -387,8 +386,8 @@ class ExportStage(Stage): (unity.census:) for the classes (the converters and atoms of its key) and the MonoScripts its objects reference, and the script table (link.scripts:all) for the entries of those: its context {"scripts": {object id: "assembly|namespace|class"}}, and for the MonoBehaviours of files that store no type trees - (the APK's built-in files) the type trees nnnotes ships for their classes: "typetrees" {script entry: - {"typeHash", "nodes"}}. Parameters: `blobMin` (bytes; byte arrays of this size or + (the APK's built-in files) the type trees nnnotes ships for their type hashes: "typetrees" {type hash: nodes}. + Parameters: `blobMin` (bytes; byte arrays of this size or more are blobs), `png.level` (zlib level of the PNG files). `reader`: another implementation of the reader atom (its id enters the keys).""" @@ -467,12 +466,12 @@ def uses(self, subject: str, env) -> dict: def context(self, subject: str, env) -> dict: doc = self.census(subject, env) + out = {} refs = census_mod.script_refs(doc) - if not refs: - return {} - table = self._docs.get(env.depot, env.input_of(SCRIPTS_TASK, "scripts").sha256)["scripts"] - out = {"scripts": link.script_context(table, refs)} - types = _script_types(doc, out["scripts"]) + if refs: + table = self._docs.get(env.depot, env.input_of(SCRIPTS_TASK, "scripts").sha256)["scripts"] + out["scripts"] = link.script_context(table, refs) + types = _script_types(doc) if types: out["typetrees"] = types return out @@ -501,25 +500,18 @@ def run(self, task, depot) -> Output: return Output(arts, items) -@cache -def _embedded(unity_version: str) -> dict: - return scripttypes.script_types(unity_version) - - -def _script_types(doc: dict, scripts: dict) -> dict: - """{script entry: {"typeHash", "nodes"}} of the embedded type trees (nnnotes.unity.scripttypes, of each file's - Unity version) of the classes of the MonoBehaviours in the census' files that store no type trees ({} when it has - none).""" +def _script_types(doc: dict) -> dict: + """{type hash: nodes} of the embedded type trees (nnnotes.unity.scripttypes, of each file's Unity version) of the + type hashes of the MonoBehaviours in the census' files that store no type trees ({} when it has none).""" out = {} for f in doc["files"]: if f["typetree"]: continue - types = _embedded(f["unityVersion"]) + types = scripttypes.script_types(f["unityVersion"] or "") for o in f["objects"]: - s = o.get("script") if o["class"] == "MonoBehaviour" else None - entry = scripts.get(contract.object_id(s["file"], s["pathId"])) if s and s.get("file") else None - if entry in types and out.setdefault(entry, types[entry]) != types[entry]: - raise ValueError(f"{entry}: files of several Unity versions with different type trees of it") + h = o["typeHash"] if o["class"] == "MonoBehaviour" else None + if h in types: + out[h] = types[h] return dict(sorted(out.items())) diff --git a/src/nnnotes/schemas/census.schema.json b/src/nnnotes/schemas/census.schema.json index f195134..a1c0354 100644 --- a/src/nnnotes/schemas/census.schema.json +++ b/src/nnnotes/schemas/census.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "urn:nnnotes:schema:census", - "title": "nnnotes.census/1", - "description": "The census of one bundle (stage unity.census), read without decoding any object: its serialized files by name (Unity version, platform, type trees, externals, objects by path id with class, size, type hash, m_Name and m_Script), the other files of the bundle, its AssetBundle objects (container in stored order, dependencies, scene hashes), its MonoScripts, objects per class and the objects whose header could not be read. A function of the bundle's bytes.", + "title": "nnnotes.census/2", + "description": "The census of one bundle (stage unity.census), read without decoding any object: its serialized files by name (Unity version, platform, type trees, externals, objects by path id with class, size, type hash, m_Name, m_Script, the objects they reference and the hierarchy holding them), the other files of the bundle, its AssetBundle objects (container in stored order, dependencies, scene hashes), its MonoScripts, objects per class and the objects whose typetree could not be read. A function of the bundle's bytes.", "type": "object", "required": [ "schema", @@ -16,7 +16,7 @@ ], "properties": { "schema": { - "const": "nnnotes.census/1" + "const": "nnnotes.census/2" }, "files": { "type": "array", @@ -151,7 +151,7 @@ ] }, "bindingScripts": { - "description": "An AnimationClip's MonoScripts named by its bindings (the MonoBehaviour fields it animates), sorted; present only when there are any (stage unity.census version 2).", + "description": "An AnimationClip's MonoScripts named by its bindings (the MonoBehaviour fields it animates), sorted; present only when there are any.", "type": "array", "items": { "anyOf": [ @@ -188,6 +188,21 @@ ] }, "minItems": 1 + }, + "refs": { + "description": "The object ids every reference in the object's typetree names, sorted, itself left out; present only when there are any.", + "type": "array", + "items": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "minItems": 1, + "uniqueItems": true + }, + "unit": { + "description": "The object id of the root GameObject of the hierarchy whose prefab holds the object (the objects of a hierarchy are converted together); present only for the objects of a hierarchy other than its root.", + "type": "string", + "pattern": "^.+:-?[0-9]+$" } } } @@ -387,7 +402,8 @@ "type": "string" } } - } + }, + "description": "The objects whose typetree could not be read: their records have no name, script or refs." } } } diff --git a/src/nnnotes/unity/atoms/reader.py b/src/nnnotes/unity/atoms/reader.py index 2f4cf9e..4d4f8e2 100644 --- a/src/nnnotes/unity/atoms/reader.py +++ b/src/nnnotes/unity/atoms/reader.py @@ -242,6 +242,10 @@ def reader(self, obj: ObjectInfo): """The library's object reader of `obj` (UnityPy's ObjectReader), for code that works on library objects.""" return self._readers[(obj.file, obj.path_id)] + def serialized_file(self, file: str): + """The library's serialized file (UnityPy's SerializedFile) named `file`.""" + return self._files[file] + def files(self) -> list[str]: return list(self._files) diff --git a/src/nnnotes/unity/convert/objects.py b/src/nnnotes/unity/convert/objects.py index dc3a44b..d6383cf 100644 --- a/src/nnnotes/unity/convert/objects.py +++ b/src/nnnotes/unity/convert/objects.py @@ -6,11 +6,12 @@ task-level name `owner`) and returns (artifact id, content record). Every object a document names by {"$ref"} is recorded in `seen` (by object id, other bundles' objects included): -the objects an object's documents reference, which the caller clears before converting an object. +the objects an object's documents reference, which the caller clears before converting an object. `references(key)` +gives the objects an object's typetree references, read as tt(key, whole=False) reads it. A file that stores no type trees (a player build's own files) gives its MonoBehaviours only their header; they are -read with the task's type tree of their script class (`typetrees`, nnnotes.unity.scripttypes), checked against the -type hash the file stores and the object's size. One without such a tree is `unsupported.typetree` (TYPETREE): its +read with the given type tree of the type hash the file stores for them (`typetrees`, nnnotes.unity.scripttypes), +which must consume exactly the object's bytes. One without such a tree is `unsupported.typetree` (TYPETREE): its header is still read. Its other objects are read with the type trees of their classes UnityPy has for the file's Unity version; for a class whose tree reads the stored layout right only up to a field (PREFIXES, by the type hash the file stores) the typetree JSON is the fields up to it, also `unsupported.typetree`. The objects read in part are @@ -190,8 +191,8 @@ class Capture: class BundleObjects: """The objects of one bundle (`view`, a reader atom's BundleView). `fns`: the atom functions by name; `scripts`: the script table entries the bundle's objects use ({MonoScript object id: "assembly|namespace|class"}); - `typetrees`: the type trees of script classes for the files that store none ({script entry: {"typeHash", - "nodes"}}); `blob_min`: byte arrays of this size or more are blobs; `png_level`: the zlib level of PNG files.""" + `typetrees`: the type trees of script classes for the files that store none ({type hash: nodes}); `blob_min`: + byte arrays of this size or more are blobs; `png_level`: the zlib level of PNG files.""" def __init__( self, @@ -216,7 +217,7 @@ def __init__( self.stripped = frozenset(f for f in self.files if not view.stores_typetrees(f)) self.partial: dict[tuple, str] = {} # objects read in part -> why self._nodes: dict[tuple, object] = {} # objects read with a type tree of `typetrees` -> its node - self._script_nodes: dict[str, object] = {} + self._script_nodes: dict[str, object] = {} # type hash -> node self.containers: dict[tuple, str] = {} for path, file, pid in view.container(): k = (file, int(pid)) @@ -306,22 +307,18 @@ def _script_tt(self, key: tuple, info, whole: bool) -> dict: except Exception as e: raise ReadError(message(e)) from None entry = self.mono_script(key, head) - t = self.typetrees.get(entry) if entry is not None else None - if entry is None or entry == NO_SCRIPT: - s = head.get("m_Script") or {} - why = f"script {s.get('m_FileID')}:{s.get('m_PathID')} is unknown and the file stores no type trees" - elif t is None: - why = f"the file stores no type trees and the task has none for {entry}" - elif t["typeHash"] != info.type_hash: - why = f"{entry}: the file's type hash {info.type_hash} is not the type tree's {t['typeHash']}" + what = entry if entry and entry != NO_SCRIPT else "its script" + rows = self.typetrees.get(info.type_hash) if info.type_hash else None + if rows is None: + why = f"the file stores no type trees and none is given for {what} (type hash {info.type_hash})" else: - node = self._script_nodes.get(entry) + node = self._script_nodes.get(info.type_hash) if node is None: - node = self._script_nodes[entry] = typetree_node(t["nodes"]) + node = self._script_nodes[info.type_hash] = typetree_node(rows) try: tt = r.read_typetree(node, check_read=True) except Exception as e: - why = f"{entry}: the type tree does not fit the object ({message(e)})" + why = f"the type tree of {what} does not fit the object ({message(e)})" else: self._nodes[key] = node self.partial.pop(key, None) @@ -333,6 +330,57 @@ def _script_tt(self, key: tuple, info, whole: bool) -> dict: self.partial[key] = why return head + def references(self, key: tuple, tt: dict | None = None) -> set[tuple]: + """The (file, pathId) of every object the object's typetree references (`tt`, as tt(key, whole=False) reads + it; read here when not given), itself and references with an unknown file index left out (ReadError when it + cannot be read).""" + r = self.readers[key] + tt = self.tt(key, whole=False) if tt is None else tt + out: set[tuple] = set() + self._pptrs(r, self._nodes.get(key) or r._get_typetree_node(), tt, out) + out.discard(key) + return out + + def _pptrs(self, owner, node, v, out: set) -> None: + """Collect the targets of the references in `v` (read with `node`), as _walk finds them.""" + if not self._special(node): + return + t = node.m_Type + if t.startswith("PPtr<"): + if isinstance(v, dict): + k = self.target(owner, v) + if k is not None: + out.add(k) + return + if t == "TypelessData" or not isinstance(v, (dict, list, tuple)): + return + ch = node.m_Children + if t == "ReferencedObject" and isinstance(v, dict): + for k, x in v.items(): + c = self._child(node, k) + if c is not None and c.m_Type == "ReferencedObjectData": + rn = _ref_type_node(v, owner.assets_file) + if rn is not None: + self._pptrs(owner, rn, x, out) + elif c is not None: + self._pptrs(owner, c, x, out) + return + if t != "string" and ch and ch[0].m_Type == "Array": + sub = ch[0].m_Children[1] + if sub.m_Type not in _BYTE_TYPES: + for x in v: + self._pptrs(owner, sub, x, out) + return + if t == "pair": + self._pptrs(owner, ch[0], v[0], out) + self._pptrs(owner, ch[1], v[1], out) + return + if isinstance(v, dict): + for k, x in v.items(): + c = self._child(node, k) + if c is not None: + self._pptrs(owner, c, x, out) + def _prefix(self, r, last: str) -> dict: """An object's typetree up to the field `last` (its class's tree cut after it).""" cut = _cut(r._get_typetree_node(), last) diff --git a/src/nnnotes/unity/scripttypes.py b/src/nnnotes/unity/scripttypes.py index 2f7d7f3..a2bf469 100644 --- a/src/nnnotes/unity/scripttypes.py +++ b/src/nnnotes/unity/scripttypes.py @@ -8,16 +8,20 @@ {"format": 1, "unityVersion", "gameVersion", "classes": {"assembly|namespace|class": {"typeHash", "nodes"}}} A class is keyed by the names its MonoScript stores (typetree_key); `nodes` are [level, type, name, meta flag] rows -of Unity's layout and `typeHash` the hex type hash the serialized file stores for the class. A tree fits an object -when the type hashes are equal and its read consumes exactly the object's serialized size. +of Unity's layout and `typeHash` the hex type hash the serialized file stores for the class. The type hash is the +hash of the class's serialized layout, so a stored object picks its tree by it alone (script_types); a tree fits the +object when its read consumes exactly the object's serialized size. `types_id()` names the shipped trees (the files' +contents), for the keys of the tasks that read with them. Nothing here imports UnityPy until typetree_node is called. """ from __future__ import annotations +import hashlib import json import re +from functools import cache from importlib import resources __all__ = [ @@ -26,6 +30,7 @@ "load_typetrees", "script_entry", "script_types", + "types_id", "typetree_file", "typetree_key", "typetree_node", @@ -66,13 +71,28 @@ def load_typetrees(unity_version: str) -> dict | None: return doc -def script_types(unity_version: str) -> dict[str, dict]: - """{script table entry (script_entry): {"typeHash", "nodes"}} of the embedded typetrees of a Unity version ({} - when nnnotes has none for it).""" +@cache +def script_types(unity_version: str) -> dict[str, list]: + """{type hash: nodes} of the embedded typetrees of a Unity version ({} when nnnotes has none for it). Shared: + callers must not change it.""" doc = load_typetrees(unity_version) - if doc is None: - return {} - return {script_entry(*k.split("|", 2)): v for k, v in sorted(doc["classes"].items())} + out: dict[str, list] = {} + for k, v in sorted((doc or {"classes": {}})["classes"].items()): + if out.setdefault(v["typeHash"], v["nodes"]) != v["nodes"]: + raise ValueError(f"typetree file {unity_version}.json: {k} and another class share a type hash") + return out + + +@cache +def types_id() -> str: + """The id of the shipped typetrees: the digest of every typetree file's name and bytes.""" + h = hashlib.sha256() + folder = resources.files("nnnotes.unity").joinpath("typetrees") + for f in sorted((f for f in folder.iterdir() if f.name.endswith(".json")), key=lambda f: f.name): + data = f.read_bytes() + h.update(f"{f.name}/{len(data)}/".encode()) + h.update(data) + return f"typetrees-{h.hexdigest()[:16]}/1" def typetree_node(rows: list): diff --git a/tests/helpers/fakeunity.py b/tests/helpers/fakeunity.py index 90fbbfd..49089c1 100644 --- a/tests/helpers/fakeunity.py +++ b/tests/helpers/fakeunity.py @@ -520,12 +520,15 @@ def __init__( node=None, bytes_fields=(), short_read: bool = False, + broken: bool = False, + byte_size: int | None = None, ): self.assets_file, self.path_id, self.tt, self.obj, self.raw = assets_file, path_id, tt, obj, raw self.short_read = short_read # its class's tree reads fewer bytes than it has: a checked whole read fails + self.broken = broken # no read succeeds self.class_id = CLASS_IDS[class_name] self.type = SimpleNamespace(name=class_name, value=self.class_id) - self.byte_size = len(raw) + self.byte_size = len(raw) if byte_size is None else byte_size self.serialized_type = SimpleNamespace(old_type_hash=type_hash) self.version, self.platform = version, platform self.typetree_reads = 0 @@ -539,6 +542,8 @@ def read_typetree(self, nodes=None, wrap: bool = False, check_read: bool = True) """The typetree (a copy); with a cut node (UnityPy's head reads) only the fields the node names. A stripped MonoBehaviour read with a tree that does not name all its fields fails the size check.""" self.typetree_reads += 1 + if self.broken: + raise ValueError(f"cannot read ") tt = copy.deepcopy(self.tt) if nodes is None and self._stripped(): nodes = self._get_typetree_node() @@ -583,8 +588,13 @@ def __init__( ): self.name, self.environment, self.version, self.unity_version = name, environment, version, unity_version self._enable_type_tree = typetrees + self._m_target_platform = ANDROID self.objects: dict[int, FakeObjectReader] = {} - self.externals = [SimpleNamespace(path=p) for p in externals] + # an external: its path, or (path, guid bytes, type) + self.externals = [ + SimpleNamespace(path=p) if isinstance(p, str) else SimpleNamespace(path=p[0], guid=p[1], type=p[2]) + for p in externals + ] self.container = SimpleNamespace(container=[]) self.ref_types: list = [] @@ -614,3 +624,10 @@ def file(self, name: str, externals=(), typetrees: bool = True) -> FakeSerialize def get_cab(self, name: str): self.lookups.append(name) return FakeResource(self.resources[name]) if name in self.resources else None + + @property + def files(self) -> dict: + """The loaded files as UnityPy lists them: one bundle holding the serialized files and resource files.""" + inner: dict = {f.name: f for f in self.assets} + inner.update({n: SimpleNamespace(Length=len(d)) for n, d in self.resources.items()}) + return {"fake.bundle": SimpleNamespace(files=inner)} diff --git a/tests/pipeline/test_census.py b/tests/pipeline/test_census.py index d215e85..8a6b73a 100644 --- a/tests/pipeline/test_census.py +++ b/tests/pipeline/test_census.py @@ -1,191 +1,148 @@ -from types import SimpleNamespace - import pytest -from UnityPy.helpers.TypeTreeNode import TypeTreeNode from nnnotes.pipeline import census, contract from nnnotes.pipeline.contract import Input from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry from nnnotes.unity import reader as unity +from nnnotes.unity import scripttypes +from tests.helpers.fakeunity import FakeEnvironment, pptr SCRIPTS = "CAB-5c1e0000000000000000000000000000" +MAIN = "CAB-aaaa0000000000000000000000000000" +SHARED = MAIN + ".sharedAssets" +HASH = bytes.fromhex("01" * 16) +# a MonoBehaviour's tree in Unity's layout: its header, a texture reference and a float +ROWS = [ + [0, "MonoBehaviour", "Base", 0], + [1, "PPtr", "m_GameObject", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + [1, "UInt8", "m_Enabled", 0x4000], + [1, "PPtr", "m_Script", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + [1, "string", "m_Name", 0], + [1, "PPtr", "target", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + [1, "float", "volume", 0], +] + + +def tf(go, children=(), father=0): + return { + "m_GameObject": pptr(go), + "m_LocalRotation": {"x": 0.0, "y": 0.0, "z": 0.0, "w": 1.0}, + "m_LocalPosition": {"x": 0.0, "y": 0.0, "z": 0.0}, + "m_LocalScale": {"x": 1.0, "y": 1.0, "z": 1.0}, + "m_Children": [pptr(c) for c in children], + "m_Father": pptr(father), + } -def node(cls: str, *fields: str) -> TypeTreeNode: - return TypeTreeNode(0, cls, "Base", -1, 1, m_Children=[TypeTreeNode(1, "x", f, -1, 0) for f in fields]) - - -class Obj: - """An object as UnityPy reads it: header reads get the fields of the node they are given.""" +def go(name, comps): + return {"m_Component": [{"component": pptr(c)} for c in comps], "m_Layer": 0, "m_Name": name, "m_IsActive": 1} - def __init__(self, pid, class_id, cls, fields: dict, *, size=100, type_hash=b"\x01" * 16, broken=False): - self.path_id, self.class_id, self.byte_size = pid, class_id, size - self.type = SimpleNamespace(name=cls) - self.serialized_type = SimpleNamespace(old_type_hash=type_hash) - self.fields, self.broken, self.reads = fields, broken, [] - self.node = node(cls, *fields) - def _get_typetree_node(self): - return self.node +def mono(go_pid, script, name="", **fields): + return {"m_GameObject": pptr(go_pid), "m_Enabled": 1, "m_Script": script, "m_Name": name, **fields} - def read_typetree(self, nodes=None, wrap=False, check_read=True): - if self.broken: - raise ValueError(f"cannot read ") - names = [c.m_Name for c in (nodes or self.node).m_Children] - self.reads.append(names) - return {k: self.fields[k] for k in names} +def asset_bundle(): + def entry(index, size, asset): + return {"preloadIndex": index, "preloadSize": size, "asset": asset} -def ext(path, guid=b"", typ=0): - return SimpleNamespace(path=path, guid=guid, type=typ) + return { + "m_Name": "scene_x", + "m_PreloadTable": [], + "m_Container": [ + ("Assets/Scenes/X.unity", entry(0, 0, pptr(0))), + ("Assets/Tex/tex.png", entry(0, 1, pptr(-5))), + ("Assets/Tex/tex.png", entry(1, 1, pptr(3, 2))), + ], + "m_MainAsset": {"preloadIndex": 0, "preloadSize": 0, "asset": pptr(0)}, + "m_RuntimeCompatibility": 1, + "m_AssetBundleName": "0123_abcd.bundle", + "m_Dependencies": ["dep_1.bundle"], + "m_IsStreamedSceneAssetBundle": True, + "m_SceneHashes": [("Assets/Scenes/X.unity", "ffee")], + } -def sfile(name, objs, externals=()): - return SimpleNamespace( - name=name, - objects={o.path_id: o for o in objs}, - externals=list(externals), - unity_version="6000.0.0f1", - _m_target_platform=13, - _enable_type_tree=True, +def scene_bundle() -> FakeEnvironment: + """A scene bundle: two serialized files sharing path ids, a resource file, a hierarchy (a root GameObject with a + child and MonoBehaviours, one of them without a script), a texture and the AssetBundle object, an unreadable + object.""" + env = FakeEnvironment() + scene = env.file(MAIN, externals=["archive:/" + SCRIPTS + "/" + SCRIPTS, "archive:/" + SHARED + "/" + SHARED]) + scene.add(1, "GameObject", go("root", [2, 7])) + scene.add(2, "Transform", tf(1, [4])) + scene.add(3, "GameObject", go("child", [4, 8])) + scene.add(4, "Transform", tf(3, father=2)) + scene.add(7, "MonoBehaviour", mono(1, pptr(42, 1), "", tex=pptr(-5, 2), big=[0] * 10), type_hash=HASH) + scene.add(8, "MonoBehaviour", mono(3, pptr(0), "n", other=pptr(9, 7)), type_hash=HASH) + scene.add(11, "Material", {"m_Name": "x"}, broken=True) + shared = env.file( + SHARED, + externals=[ + ("archive:/" + SCRIPTS + "/" + SCRIPTS, b"", 0), + ("Library/unity default resources", b"\x0f" * 16, 0), + ], ) - - -def ptr(fid, pid): - return {"m_FileID": fid, "m_PathID": pid} - - -def scene_bundle(): - """A bundle with two serialized files sharing path ids, a resource file, MonoBehaviours with and without a - script, an AssetBundle object and an unreadable object.""" - main = "CAB-aaaa0000000000000000000000000000" - mb = Obj( - 7, - 114, - "MonoBehaviour", - {"m_GameObject": ptr(0, 3), "m_Enabled": 1, "m_Script": ptr(1, 42), "m_Name": "", "big": [0] * 10}, - ) - null_mb = Obj( - 8, 114, "MonoBehaviour", {"m_GameObject": ptr(0, 3), "m_Enabled": 1, "m_Script": ptr(0, 0), "m_Name": "n"} - ) - tex = Obj(-5, 28, "Texture2D", {"m_Name": "tex", "m_Width": 4, "image data": b"\0" * 64}, size=500) - ab = Obj( - 1, - 142, - "AssetBundle", - { - "m_Name": "scene_x", - "m_PreloadTable": [], - "m_Container": [ - ("Assets/Scenes/X.unity", {"preloadIndex": 0, "preloadSize": 0, "asset": ptr(0, 0)}), - ("Assets/Tex/tex.png", {"preloadIndex": 0, "preloadSize": 1, "asset": ptr(0, -5)}), - ("Assets/Tex/tex.png", {"preloadIndex": 1, "preloadSize": 1, "asset": ptr(2, 3)}), - ], - "m_MainAsset": {}, - "m_RuntimeCompatibility": 1, - "m_AssetBundleName": "0123_abcd.bundle", - "m_Dependencies": ["dep_1.bundle"], - "m_IsStreamedSceneAssetBundle": True, - "m_SceneHashes": [("Assets/Scenes/X.unity", "ffee")], - }, - ) - tf = Obj(3, 4, "Transform", {"m_GameObject": ptr(0, 9), "m_LocalPosition": {}}, type_hash=None) - bad = Obj(11, 1, "GameObject", {"m_Component": [], "m_Name": "x"}, broken=True) - shared = sfile( - main + ".sharedAssets", - [tex, ab, Obj(3, 1, "GameObject", {"m_Component": [], "m_Layer": 0, "m_Name": "go", "m_IsActive": 1})], - [ext("archive:/" + SCRIPTS + "/" + SCRIPTS), ext("Library/unity default resources", b"\x0f" * 16)], - ) - scene = sfile(main, [tf, mb, null_mb, bad], [ext("archive:/" + SCRIPTS + "/" + SCRIPTS)]) - res = SimpleNamespace(Length=1234) - bundle = SimpleNamespace(files={scene.name: scene, main + ".resS": res, shared.name: shared}) - return SimpleNamespace(files={"x.bundle": bundle}), {"mb": mb, "tex": tex, "tf": tf} - - -def scripts_bundle(): - s = [ - Obj( - 42, - 115, - "MonoScript", - { - "m_Name": "Widget", - "m_ExecutionOrder": 0, - "m_ClassName": "Widget", - "m_Namespace": "Game.UI", - "m_AssemblyName": "Game.dll", - }, - ), - Obj( - 43, - 115, - "MonoScript", - { - "m_Name": "Top", - "m_ExecutionOrder": 0, - "m_ClassName": "Top", - "m_Namespace": "", - "m_AssemblyName": "Assembly-CSharp.dll", - }, - ), - ] - return SimpleNamespace(files={"s.bundle": SimpleNamespace(files={SCRIPTS: sfile(SCRIPTS, s)})}) + shared.add(-5, "Texture2D", {"m_Name": "tex", "m_Width": 4, "image data": b"\0" * 64}, byte_size=500) + shared.add(1, "AssetBundle", asset_bundle()) + shared.add(3, "GameObject", go("go", [])) + env.resources[MAIN + ".resS"] = b"\0" * 1234 + return env def test_census_of_a_scene_bundle(): - env, objs = scene_bundle() - doc = census.census_env(env) - main = "CAB-aaaa0000000000000000000000000000" - assert doc["schema"] == contract.CENSUS - assert [f["name"] for f in doc["files"]] == [main, main + ".sharedAssets"] - assert doc["resources"] == [{"name": main + ".resS", "size": 1234}] + doc = census.census_env(scene_bundle()) + assert doc["schema"] == contract.CENSUS == "nnnotes.census/2" + assert [f["name"] for f in doc["files"]] == [MAIN, SHARED] + assert doc["resources"] == [{"name": MAIN + ".resS", "size": 1234}] scene, shared = doc["files"] - assert [o["pathId"] for o in scene["objects"]] == [3, 7, 8, 11] + assert [o["pathId"] for o in scene["objects"]] == [1, 2, 3, 4, 7, 8, 11] assert [o["pathId"] for o in shared["objects"]] == [-5, 1, 3] # path id 3 in both files - assert scene["externals"] == [ - {"file": SCRIPTS, "path": "archive:/" + SCRIPTS + "/" + SCRIPTS, "guid": None, "type": 0} - ] + assert scene["externals"][0] == { + "file": SCRIPTS, + "path": "archive:/" + SCRIPTS + "/" + SCRIPTS, + "guid": None, + "type": None, + } assert shared["externals"][1] == { "file": "unity default resources", "path": "Library/unity default resources", "guid": "0f" * 16, "type": 0, } - assert (scene["unityVersion"], scene["platform"], scene["typetree"]) == ("6000.0.0f1", 13, True) - tf, mb, null_mb, bad = scene["objects"] - assert tf == {"pathId": 3, "classId": 4, "class": "Transform", "byteSize": 100, "typeHash": None} - assert mb == { + assert (scene["unityVersion"], scene["platform"], scene["typetree"]) == ("6000.3.12f1", 13, True) + objs = {o["pathId"]: o for o in scene["objects"]} + assert objs[7] == { "pathId": 7, "classId": 114, "class": "MonoBehaviour", - "byteSize": 100, + "byteSize": 0, "typeHash": "01" * 16, "name": "", "script": {"file": SCRIPTS, "pathId": 42}, + "refs": sorted([f"{MAIN}:1", f"{SHARED}:-5", f"{SCRIPTS}:42"]), # its GameObject, the texture, its script + "unit": f"{MAIN}:1", } - assert null_mb["script"] is None and null_mb["name"] == "n" - assert "name" not in bad and doc["errors"] == [ - {"object": main + ":11", "message": "ValueError: cannot read "} - ] + assert objs[8]["script"] is None and objs[8]["name"] == "n" + assert objs[8]["refs"] == [f"{MAIN}:3"] # a reference with an unknown file index is left out + assert "name" not in objs[11] and "refs" not in objs[11] + assert doc["errors"] == [{"object": MAIN + ":11", "message": "ValueError: cannot read "}] tex = shared["objects"][0] - assert tex["name"] == "tex" and tex["byteSize"] == 500 - # headers only: a MonoBehaviour is read up to m_Name, a texture up to m_Name, a transform not at all - assert objs["mb"].reads == [["m_GameObject", "m_Enabled", "m_Script", "m_Name"]] - assert objs["tex"].reads == [["m_Name"]] and objs["tf"].reads == [] + assert tex["name"] == "tex" and tex["byteSize"] == 500 and "refs" not in tex (ab,) = doc["assetBundles"] - assert ab["file"] == main + ".sharedAssets" and ab["pathId"] == 1 and ab["assetBundleName"] == "0123_abcd.bundle" + assert ab["file"] == SHARED and ab["pathId"] == 1 and ab["assetBundleName"] == "0123_abcd.bundle" assert ab["dependencies"] == ["dep_1.bundle"] and ab["isStreamedSceneAssetBundle"] is True assert ab["sceneHashes"] == [["Assets/Scenes/X.unity", "ffee"]] assert ab["container"] == [ {"path": "Assets/Scenes/X.unity", "file": None, "pathId": None, "preloadIndex": 0, "preloadSize": 0}, - { - "path": "Assets/Tex/tex.png", - "file": main + ".sharedAssets", - "pathId": -5, - "preloadIndex": 0, - "preloadSize": 1, - }, + {"path": "Assets/Tex/tex.png", "file": SHARED, "pathId": -5, "preloadIndex": 0, "preloadSize": 1}, { "path": "Assets/Tex/tex.png", "file": "unity default resources", @@ -194,21 +151,38 @@ def test_census_of_a_scene_bundle(): "preloadSize": 1, }, ] + assert shared["objects"][1]["refs"] == [f"{SHARED}:-5", "unity default resources:3"] assert census.container(doc) == ab["container"] - assert doc["classes"] == {"AssetBundle": 1, "GameObject": 2, "MonoBehaviour": 2, "Texture2D": 1, "Transform": 1} - assert doc["objects"] == 7 and doc["scripts"] == [] + assert doc["classes"] == { + "AssetBundle": 1, + "GameObject": 3, + "Material": 1, + "MonoBehaviour": 2, + "Texture2D": 1, + "Transform": 2, + } + assert doc["objects"] == 10 and doc["scripts"] == [] assert census.script_refs(doc) == [SCRIPTS + ":42"] - assert [(f, o["pathId"]) for f, o in census.objects(doc)][:2] == [(main, 3), (main, 7)] + assert [(f, o["pathId"]) for f, o in census.objects(doc)][:2] == [(MAIN, 1), (MAIN, 2)] assert census.facts(doc) == { "files": 2, - "objects": 7, + "objects": 10, "classes": doc["classes"], "scripts": 0, "scriptRefs": 1, "errors": 1, } - again, _ = scene_bundle() - assert contract.encode(census.census_env(again)) == contract.encode(doc) # deterministic + assert contract.encode(census.census_env(scene_bundle())) == contract.encode(doc) # deterministic + + +def test_the_objects_of_a_hierarchy_name_its_root(): + doc = census.census_env(scene_bundle()) + objs = {o["pathId"]: o for o in doc["files"][0]["objects"]} + assert "unit" not in objs[1] # the root GameObject is its own unit + assert {pid: objs[pid].get("unit") for pid in (2, 3, 4, 7, 8)} == dict.fromkeys((2, 3, 4, 7, 8), f"{MAIN}:1") + assert objs[2]["refs"] == [f"{MAIN}:1", f"{MAIN}:4"] # its GameObject and its child + assert objs[1]["refs"] == [f"{MAIN}:2", f"{MAIN}:7"] + assert "unit" not in objs[11] and "unit" not in doc["files"][1]["objects"][2] # outside any hierarchy def test_scripts_named_by_clip_bindings(): @@ -217,51 +191,43 @@ def test_scripts_named_by_clip_bindings(): def binding(tid, script): return {"path": 1, "attribute": 2, "script": script, "typeID": tid, "customType": 0, "isPPtrCurve": 0} - clip = Obj( + env = FakeEnvironment() + f = env.file(main, externals=["archive:/" + SCRIPTS + "/" + SCRIPTS]) + bindings = [ + binding(114, pptr(42, 1)), + binding(4, pptr(0)), + binding(114, pptr(42, 1)), + binding(114, pptr(6)), + binding(114, pptr(1, 5)), + ] + f.add( 5, - 74, "AnimationClip", - { - "m_Name": "fade", - "m_Legacy": False, - "m_MuscleClip": {}, - "m_ClipBindingConstant": { - "genericBindings": [ - binding(114, ptr(1, 42)), - binding(4, ptr(0, 0)), - binding(114, ptr(1, 42)), - binding(114, ptr(0, 6)), - binding(114, ptr(5, 1)), - ], - "pptrCurveMapping": [], - }, - "m_Events": [], - }, - ) - plain = Obj( - 6, - 74, - "AnimationClip", - {"m_Name": "move", "m_ClipBindingConstant": {"genericBindings": [binding(4, ptr(0, 0))]}, "m_Events": []}, + {"m_Name": "fade", "m_ClipBindingConstant": {"genericBindings": bindings, "pptrCurveMapping": []}}, ) - local = Obj(7, 115, "MonoScript", {"m_Name": "L", "m_ClassName": "L", "m_Namespace": "", "m_AssemblyName": "A.dll"}) - f = sfile(main, [clip, plain, local], [ext("archive:/" + SCRIPTS + "/" + SCRIPTS)]) - doc = census.census_env(SimpleNamespace(files={"c.bundle": SimpleNamespace(files={main: f})})) + f.add(6, "AnimationClip", {"m_Name": "move", "m_ClipBindingConstant": {"genericBindings": [binding(4, pptr(0))]}}) + f.add(7, "MonoScript", {"m_Name": "L", "m_ClassName": "L", "m_Namespace": "", "m_AssemblyName": "A.dll"}) + doc = census.census_env(env) rec, other, _ = doc["files"][0]["objects"] assert rec["bindingScripts"] == [ {"file": SCRIPTS, "pathId": 42}, {"file": main, "pathId": 6}, {"fileId": 5, "pathId": 1}, ] + assert rec["refs"] == [f"{SCRIPTS}:42", f"{main}:6"] assert rec["name"] == "fade" and "bindingScripts" not in other and "script" not in rec - assert clip.reads == [["m_Name", "m_Legacy", "m_MuscleClip", "m_ClipBindingConstant"]] # not m_Events assert census.script_refs(doc) == [SCRIPTS + ":42", main + ":6"] assert census.facts(doc)["scriptRefs"] == 2 - assert census.CensusStage.version == 2 def test_monoscripts_and_unknown_file_index(): - doc = census.census_env(scripts_bundle()) + env = FakeEnvironment() + f = env.file(SCRIPTS) + widget = {"m_Name": "Widget", "m_ClassName": "Widget", "m_Namespace": "Game.UI", "m_AssemblyName": "Game.dll"} + f.add(42, "MonoScript", widget) + top = {"m_Name": "Top", "m_ClassName": "Top", "m_Namespace": "", "m_AssemblyName": "Assembly-CSharp.dll"} + f.add(43, "MonoScript", top) + doc = census.census_env(env) assert doc["scripts"] == [ { "file": SCRIPTS, @@ -280,9 +246,29 @@ def test_monoscripts_and_unknown_file_index(): "name": "Top", }, ] - refs = census._Refs(sfile("CAB-1", [])) - assert refs.ref(ptr(3, 9)) == {"fileId": 3, "pathId": 9} and refs.ref(ptr(0, 0)) is None - assert refs.ref(None) is None and refs.ref(ptr(0, 5)) == {"file": "CAB-1", "pathId": 5} + refs = census._Refs("CAB-1", []) + assert refs.ref(pptr(9, 3)) == {"fileId": 3, "pathId": 9} and refs.ref(pptr(0)) is None + assert refs.ref(None) is None and refs.ref(pptr(5)) == {"file": "CAB-1", "pathId": 5} + + +def player_file() -> FakeEnvironment: + """A player's own file without type trees: a MonoBehaviour that references a built-in texture.""" + env = FakeEnvironment() + f = env.file("globalgamemanagers.assets", externals=["Library/unity default resources"], typetrees=False) + f.add(50, "MonoScript", {"m_Name": "S", "m_ClassName": "S", "m_Namespace": "", "m_AssemblyName": "A.dll"}) + f.add(2, "MonoBehaviour", mono(0, pptr(50), "volume", target=pptr(10, 1), volume=0.5), type_hash=HASH) + return env + + +def test_files_without_type_trees_read_with_the_shipped_ones(monkeypatch): + header = ["globalgamemanagers.assets:50"] + for trees, refs in (({HASH.hex(): ROWS}, [*header, "unity default resources:10"]), ({}, header)): + monkeypatch.setattr(scripttypes, "script_types", lambda version, trees=trees: trees) + rec = census.census_env(player_file(), shipped=True)["files"][0]["objects"][0] + assert rec["name"] == "volume" and rec["script"] == {"file": "globalgamemanagers.assets", "pathId": 50} + assert rec["refs"] == refs # without a shipped tree, the header's + monkeypatch.setattr(scripttypes, "script_types", lambda version: {HASH.hex(): ROWS}) + assert census.census_env(player_file())["files"][0]["objects"][0]["refs"] == header # not asked to ship def test_bundle_subjects(): @@ -297,7 +283,7 @@ def test_census_stage(tmp_path, monkeypatch): def load(path, name=None): loaded.append((str(path), name)) - return scene_bundle()[0] + return scene_bundle() monkeypatch.setattr(unity, "load", load) depot = Depot(tmp_path / "depot") @@ -307,15 +293,15 @@ def load(path, name=None): env = Env(depot, {"bundles": {"x": inp}}) assert stage.subjects(env) == ["x"] task = describe(stage, "x", None, env) - assert task.id == "unity.census:x" and task.atoms == census.CensusStage.ATOMS + assert task.id == "unity.census:x" and task.atoms == census.CensusStage.ATOMS and task.version == 3 assert task.atoms["reader"].startswith("unitypy-") assert task.cost.peak_bytes > size assert execute(task, depot, registry(stage)).status == "ran" doc = depot.result(task.key) (rec,) = doc["artifacts"] - assert rec["id"] == "unity.census:x#census" and rec["semantics"]["facts"]["objects"] == 7 + assert rec["id"] == "unity.census:x#census" and rec["semantics"]["facts"]["objects"] == 10 assert rec["provenance"]["inputs"] == [{"role": "bundle", "sha256": sha}] - assert contract.loads(depot.read(rec["content"]["sha256"])) == census.census_env(scene_bundle()[0]) + assert contract.loads(depot.read(rec["content"]["sha256"])) == census.census_env(scene_bundle()) assert loaded == [(str(bundle), None)] # a catalog bundle: its files keep their names assert execute(task, depot, registry(stage)).status == "hit" and len(loaded) == 1 with pytest.raises(ValueError, match="expected 'bundle'"): @@ -323,8 +309,10 @@ def load(path, name=None): for waiting in (None, Pending("fetch:x_1.bundle")): with pytest.raises(Pending, match="fetch:x"): describe(stage, "x", None, Env(depot, {"bundles": {"x": waiting}})) - # a serialized file of the APK outside a bundle is loaded under the name references into it use + # a serialized file of the APK outside a bundle is loaded under the name references into it use, and read with the + # shipped type trees, which its key names own = "apk/Resources/unity default resources" task = describe(stage, own, None, Env(depot, {"bundles": {own: inp}})) + assert task.atoms == {**census.CensusStage.ATOMS, "typetrees": scripttypes.types_id()} assert execute(task, depot, registry(stage)).status == "ran" assert loaded[-1] == (str(bundle), "unity default resources") diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index 6689c94..f2a1879 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -560,10 +560,10 @@ def bundle_movie() -> FakeEnvironment: return env -# a player's own file: no type trees; MonoBehaviours whose classes have a shipped type tree (one of another type -# hash), and none +# a player's own file: no type trees; MonoBehaviours whose type hashes have a shipped type tree (one that does not +# fit the object), and none PLAYER = "globalgamemanagers.assets" -PLAYER_HASH, OTHER_HASH = "a1" * 16, "b2" * 16 +PLAYER_HASH, OTHER_HASH, IMAGE_HASH = "a1" * 16, "b2" * 16, "c3" * 16 SETTINGS, IMAGE, CHANGED = "Fwk|Fwk.Sound|SoundVolumeSettings", "UnityEngine.UI|UnityEngine.UI|Image", "Game|Game|Moved" PLAYER_SCRIPTS = {f"{PLAYER}:50": SETTINGS, f"{PLAYER}:51": IMAGE, f"{PLAYER}:52": CHANGED} SETTINGS_ROWS = [ @@ -581,10 +581,7 @@ def bundle_movie() -> FakeEnvironment: [2, "SInt64", "m_PathID", 0], [1, "float", "volume", 0], ] -PLAYER_TYPETREES = { - SETTINGS: {"typeHash": PLAYER_HASH, "nodes": SETTINGS_ROWS}, - CHANGED: {"typeHash": OTHER_HASH, "nodes": SETTINGS_ROWS}, -} +PLAYER_TYPETREES = {PLAYER_HASH: SETTINGS_ROWS, OTHER_HASH: SETTINGS_ROWS[:-1]} # the second lacks a field def bundle_player() -> FakeEnvironment: @@ -595,9 +592,10 @@ def bundle_player() -> FakeEnvironment: f.add(1, "PlayerSettings", settings, type_hash=bytes.fromhex("936fd1913a4f0b5da2ce7cadb0ce9666"), short_read=True) f.add(6, "PlayerSettings", settings, type_hash=h, short_read=True) f.add(2, "MonoBehaviour", _mono(0, pptr(50), "volume", target=pptr(10, 1), volume=0.5), type_hash=h) - f.add(3, "MonoBehaviour", _mono(20, pptr(51), "", m_Sprite=pptr(10, 1)), type_hash=h) - f.add(4, "MonoBehaviour", _mono(0, pptr(52), "moved", target=pptr(0), volume=1.0), type_hash=h) - f.add(5, "MonoBehaviour", _mono(0, pptr(51), "image", m_Sprite=pptr(10, 1)), type_hash=h) + image, other = bytes.fromhex(IMAGE_HASH), bytes.fromhex(OTHER_HASH) + f.add(3, "MonoBehaviour", _mono(20, pptr(51), "", m_Sprite=pptr(10, 1)), type_hash=image) + f.add(4, "MonoBehaviour", _mono(0, pptr(52), "moved", target=pptr(0), volume=1.0), type_hash=other) + f.add(5, "MonoBehaviour", _mono(0, pptr(51), "image", m_Sprite=pptr(10, 1)), type_hash=image) f.add(20, "GameObject", _go("Canvas", [21, 3])) f.add(21, "Transform", _transform(20)) for pid, entry in ((50, SETTINGS), (51, IMAGE), (52, CHANGED)): @@ -1319,7 +1317,7 @@ def census_doc(view: UnityBundle) -> dict: if o.file != f: continue classes[o.class_name] = classes.get(o.class_name, 0) + 1 - rec = {"pathId": o.path_id, "classId": o.class_id, "class": o.class_name} + rec = {"pathId": o.path_id, "classId": o.class_id, "class": o.class_name, "typeHash": o.type_hash} tt = view.reader(o).read_typetree(check_read=False) # a header read, as the census' ext = view.externals(f) @@ -1433,8 +1431,11 @@ def script(pid, cls): "volume": 0.5, } assert items[p(2)]["refs"] == [p(50), "unity default resources:10"] - # a class without a shipped type tree, or with one of another type hash: its header, and the reason - for pid, why in ((5, f"has none for {IMAGE}"), (4, f"type hash {PLAYER_HASH} is not the type tree's")): + # a type hash without a shipped type tree, or with one that does not fit the object: its header, and the reason + for pid, why in ( + (5, f"none is given for {IMAGE} (type hash {IMAGE_HASH})"), + (4, f"type tree of {CHANGED} does not fit"), + ): assert items[p(pid)]["status"] == "generic" assert items[p(pid)]["reason"]["code"] == "unsupported.typetree" and why in items[p(pid)]["reason"]["message"] assert obj(depot, doc, f"{p(5)}#json") == { @@ -1460,14 +1461,16 @@ def script(pid, cls): assert items[p(3)]["status"] == "contained" -def test_the_context_holds_the_shipped_type_trees_of_its_scripts(tmp_path): +def test_the_context_holds_the_shipped_type_trees_of_its_scripts(tmp_path, monkeypatch): from nnnotes.unity import scripttypes shipped = scripttypes.load_typetrees("6000.3.12f1")["classes"]["DOTween.dll|DG.Tweening.Core|DOTweenSettings"] - entry = "DOTween|DG.Tweening.Core|DOTweenSettings" # as the script table names it - env = planning_env(Depot(tmp_path / "s"), names=("player",), scripts={**PLAYER_SCRIPTS, f"{PLAYER}:50": entry}) + assert scripttypes.script_types("6000.3.12f1")[shipped["typeHash"]] == shipped["nodes"] + trees = {**PLAYER_TYPETREES, shipped["typeHash"]: shipped["nodes"]} + monkeypatch.setattr(scripttypes, "script_types", lambda version: trees) + env = planning_env(Depot(tmp_path / "s"), names=("player",), scripts=PLAYER_SCRIPTS) t = describe(ExportStage(FAKE_READER), "player", None, env) - assert t.context["typetrees"] == {entry: shipped} + assert t.context["typetrees"] == PLAYER_TYPETREES # the trees of its MonoBehaviours' type hashes assert ( "typetrees" not in describe(ExportStage(FAKE_READER), "all", None, planning_env(Depot(tmp_path / "t"))).context ) From edc9ee431589ddd2cc672ac4b8c37f6078749b51 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 03:06:08 +0800 Subject: [PATCH 11/91] feat(orchestrate): run the tasks of a batch as one job in one worker A stage can now name a batch per task (Stage.batch). The tasks of a stage that share a batch form one job: one worker runs them one after another, so what they share (such as a loaded bundle) is loaded once, and calls Stage.end_batch after the last of them to drop it. A task without a batch is a job alone. Scheduling, memory admission and the plan's estimates work on jobs: a job's CPU is the sum of its tasks', its peak the largest of theirs. When a worker dies inside a job, the task it was running fails and the job's remaining tasks run in another worker. The run log names each task's job. Co-Authored-By: Claude Opus 5.5 --- docs/assets.md | 17 +- src/nnnotes/pipeline/orchestrate.py | 278 ++++++++++++++++++---------- src/nnnotes/pipeline/plan.py | 13 +- src/nnnotes/pipeline/stages.py | 8 + tests/pipeline/test_orchestrate.py | 127 +++++++++++-- tests/pipeline/test_plan.py | 10 +- 6 files changed, 328 insertions(+), 125 deletions(-) diff --git a/docs/assets.md b/docs/assets.md index 7060085..c94bb36 100644 --- a/docs/assets.md +++ b/docs/assets.md @@ -58,12 +58,15 @@ stages that need those results (atlas sprites, views), writes the layouts and th (CRI audio and movies) not downloaded yet are fetched while the bundle stages run. The census also covers the bundles the selected ones depend on, so script classes and atlas textures in other bundles resolve. -Tasks run in spawned worker processes that stay alive between tasks (no more of them start than a stage has tasks to -run). The longest tasks (by estimated CPU time) start first; a task starts only while the estimated peak memory of -the running tasks and its own fits the budget, and a task estimated above the budget runs alone. A worker is -replaced after 500 tasks or when its resident memory after a task is above 3 GiB, or above 40 % of its part of the -memory budget (the budget divided by the workers) when that is less, but not below 256 MiB: a worker keeps much of -its largest task's memory, which the budget does not count (`--explain` prints the threshold). The estimates come +Tasks run in spawned worker processes that stay alive between tasks. The tasks of a stage that read the same +loaded data (a stage's batch, such as one bundle) form one job: one worker runs them one after another and loads the +data once. Every other task is a job alone. No more workers start than a stage has jobs to run. The longest jobs (by +estimated CPU time, the sum over their tasks) start first; a job starts only while the estimated peak memory of the +running jobs and its own (the largest of its tasks') fits the budget, and a job estimated above the budget runs +alone. A worker is replaced after a job once it has run 500 tasks, or when its resident memory at the end of a job +is above 3 GiB, or above 40 % of its part of the memory budget (the budget divided by the workers) when that is +less, but not below 256 MiB: a worker keeps much of its largest task's memory, which the budget does not count +(`--explain` prints the threshold). The estimates come from each stage's cost model, scaled per stage by the costs measured in earlier runs of the depot (`/costs/calibration.json`, updated after every export; a task's measured peak counts from its worker's memory at its start). None of this changes an output: a run with one worker and a run with many write the same depot @@ -71,7 +74,7 @@ and the same files. A failed object is reported with its task's result and stays failed until something in its key changes. A task that fails as a whole (an exception, an input that cannot be read, a worker that dies) leaves no result and runs again -next time. +next time; when a worker dies, the tasks of its job after the failed one run in another worker. ### depot plan diff --git a/src/nnnotes/pipeline/orchestrate.py b/src/nnnotes/pipeline/orchestrate.py index 6d54736..8006c1b 100644 --- a/src/nnnotes/pipeline/orchestrate.py +++ b/src/nnnotes/pipeline/orchestrate.py @@ -4,17 +4,23 @@ the caller plus the keys of the tasks already done), so every input is known when a task is described. Per stage: a task whose result is in the depot is a hit; the others run in parallel, then the next stage starts. -The pool: up to `workers` spawned processes (no more than a stage has tasks to run), long-lived (each imports -what its stages need once), one task at a time each. Tasks start in order of decreasing estimated CPU (longest -first), a task only when the estimated peak memory of the running tasks plus its own fits `memory` less -`worker_bytes` per worker (what the processes hold before any task); a task estimated above that budget runs alone -and is reported. A worker is replaced after `recycle_tasks` tasks, or when its resident memory after a task is -above `recycle_rss`; the workers are stopped together at the end. `workers=0` runs the tasks in this process, one -after another. +Jobs: the tasks of a stage that share a batch (Stage.batch, e.g. the tasks reading one bundle) form one job, run +one after another by one process, which loads what they share once and drops it at the job's end +(Stage.end_batch); a task without a batch is a job alone. A job's estimate is the sum of its tasks' CPU and the +largest of their peaks. + +The pool: up to `workers` spawned processes (no more than a stage has jobs to run), long-lived (each imports +what its stages need once), one job at a time each. Jobs start in order of decreasing estimated CPU (longest +first), a job only when the estimated peak memory of the running jobs plus its own fits `memory` less +`worker_bytes` per worker (what the processes hold before any task); a job estimated above that budget runs alone +and is reported. A worker is replaced after a job once it has run `recycle_tasks` tasks, or when its resident memory +at the job's end is above `recycle_rss`; the workers are stopped together at the end. `workers=0` runs the jobs in +this process, one after another. Failures: a failed object is an item of its task's result, cached with it. An exception out of a stage, an input that cannot be read, or a worker that dies fails the task: nothing of it is cached, the run records it (failures(), the run log) and goes on; a task that reads the result of a failed task fails as `upstream.failed`. +The tasks of a job after the one its worker died in run in another worker. A ConfigError (a setting or tool the run needs) stops the run: no new task starts, and run() raises it once the running tasks have ended (their results stay valid). Stage results do not depend on the number of workers, their order or their timing: a run with one worker and a run @@ -143,6 +149,8 @@ def run_one(task: Task, depot, stages: dict, force: bool) -> dict: def _worker_main(conn, depot, stages: dict, initializer, initargs) -> None: + """Run jobs ([task description], [force], batch) as they come: one reply per task, in order (none after a + ConfigError), then {"end": True, "rss"} once the batch is dropped; None stops the worker.""" if initializer is not None: initializer(depot, *initargs) while True: @@ -152,13 +160,20 @@ def _worker_main(conn, depot, stages: dict, initializer, initargs) -> None: return if msg is None: return - doc, force = msg - try: - task = Task.from_json(doc) - except Exception as e: - conn.send({"ok": False, "code": "task.incompatible", "message": str(e), "trace": "", "rss": rss()}) - continue - conn.send(run_one(task, depot, stages, force)) + docs, forces, batch = msg + for doc, force in zip(docs, forces): + try: + task = Task.from_json(doc) + except Exception as e: + reply = {"ok": False, "code": "task.incompatible", "message": str(e), "trace": "", "rss": rss()} + else: + reply = run_one(task, depot, stages, force) + conn.send(reply) + if reply.get("code") == "config": + break + if batch is not None: + stages[batch[0]].end_batch(batch[1]) + conn.send({"end": True, "rss": rss()}) class _Worker: @@ -173,12 +188,13 @@ def __init__(self, ctx, n: int, depot, stages, initializer, initargs): ) self.proc.start() child.close() - self.task: Task | None = None - self.done = 0 + self.job: Job | None = None + self.replied = 0 # replies received for the tasks of the job + self.done = 0 # tasks run - def send(self, task: Task, force: bool) -> None: - self.task = task - self.conn.send((task.to_json(), force)) + def send(self, job: Job, forces: list[bool]) -> None: + self.job, self.replied = job, 0 + self.conn.send(([t.to_json() for t in job.tasks], forces, job.batch)) def ask_stop(self) -> None: try: @@ -200,40 +216,86 @@ def stop(self, wait_s: float = 10.0) -> None: # ---------------------------------------------------------------- scheduling +@dataclass +class Job: + """Tasks one process runs one after another: those of one batch (`batch`: (stage name, Stage.batch)), or a + task alone. Its estimate: the sum of its tasks' CPU, the largest of their peaks (one runs at a time).""" + + tasks: list + batch: tuple | None = None + cpu: float = field(init=False) + peak: int = field(init=False) + + def __post_init__(self): + self.cpu = sum(_cpu(t) for t in self.tasks) + self.peak = max(_peak(t) for t in self.tasks) + + @property + def id(self) -> str: + return self.tasks[0].id + + +def jobs(tasks, stages: dict) -> list[Job]: + """The jobs of `tasks`: one per batch of a stage (its tasks by id), one per task without a batch.""" + alone, batches = [], {} + for t in sorted(tasks, key=lambda t: t.id): + b = stages[t.stage].batch(t) + if b is None: + alone.append(Job([t])) + else: + batches.setdefault((t.stage, b), []).append(t) + return alone + [Job(ts, k) for k, ts in batches.items()] + + class Scheduler: - """The start order of a stage's tasks: decreasing estimated CPU (then id), admitted while the estimated peak - memory of the running tasks plus the next one fits `budget` (bytes; None: no limit). A task estimated above the - budget waits until nothing runs, then runs alone; no task behind it in the order starts before it.""" + """The start order of a stage's jobs: decreasing estimated CPU (then id), admitted while the estimated peak + memory of the running jobs plus the next one fits `budget` (bytes; None: no limit). A job estimated above the + budget waits until nothing runs, then runs alone; no job behind it in the order starts before it.""" - def __init__(self, tasks, budget: int | None): - self.queue = sorted(tasks, key=lambda t: (-_cpu(t), t.id)) + def __init__(self, jobs, budget: int | None): + self.queue = sorted(jobs, key=self._order) self.budget = budget self.running: dict[str, int] = {} - self.oversize: list[str] = [] + self.oversize: list[str] = [] # task ids + self.stopped = False + + @staticmethod + def _order(j: Job) -> tuple: + return (-j.cpu, j.id) def used(self) -> int: return sum(self.running.values()) - def next(self) -> Task | None: + def next(self) -> Job | None: used = self.used() - for i, t in enumerate(self.queue): - need = _peak(t) - if self.budget is None or used + need <= self.budget: + for i, j in enumerate(self.queue): + if self.budget is None or used + j.peak <= self.budget: return self._take(i) - if need > self.budget: + if j.peak > self.budget: if self.running: return None - self.oversize.append(t.id) + self.oversize += [t.id for t in j.tasks] return self._take(i) return None - def _take(self, i: int) -> Task: - t = self.queue.pop(i) - self.running[t.id] = _peak(t) - return t + def _take(self, i: int) -> Job: + j = self.queue.pop(i) + self.running[j.id] = j.peak + return j + + def finished(self, jid: str) -> None: + self.running.pop(jid, None) + + def put(self, job: Job) -> None: + """Queue `job` (the rest of a job whose worker died) in its place in the order; nothing once stopped.""" + if not self.stopped: + self.queue.append(job) + self.queue.sort(key=self._order) - def finished(self, tid: str) -> None: - self.running.pop(tid, None) + def stop(self) -> None: + """Start nothing more.""" + self.queue.clear() + self.stopped = True def _cpu(t: Task) -> float: @@ -594,22 +656,26 @@ def _fail(run: Run, env: Env, tid: str, key, stage: str, code: str, message: str # ---------------------------------------------------------------- execution def _execute(self, tasks: list[Task], force, run: Run) -> dict[str, dict]: - """Run `tasks`; {task id: reply}.""" + """Run `tasks`; {task id: reply} (none for a task not started because the run stopped).""" if not tasks: return {} - sched = Scheduler(tasks, self.task_budget()) + sched = Scheduler(jobs(tasks, self.stages), self.task_budget()) replies: dict[str, dict] = {} if self.workers == 0: while True: - t = sched.next() - if t is None: + j = sched.next() + if j is None: break - self._record(run, t, sched, None) - replies[t.id] = run_one(t, self.depot, self.stages, force(t)) - self._done(run, t, replies[t.id], None) - sched.finished(t.id) - if replies[t.id].get("code") == "config": - sched.queue.clear() + self._record(run, j, sched, None) + for t in j.tasks: + replies[t.id] = run_one(t, self.depot, self.stages, force(t)) + self._done(run, t, replies[t.id], None) + if replies[t.id].get("code") == "config": + sched.stop() + break + if j.batch is not None: + self.stages[j.batch[0]].end_batch(j.batch[1]) + sched.finished(j.id) else: self._pool_run(sched, force, replies, run) run.oversize += sched.oversize @@ -620,59 +686,77 @@ def _spawn(self) -> _Worker: return _Worker(mp.get_context("spawn"), self._started, self.depot, self.stages, self.initializer, self.initargs) def _pool_run(self, sched: Scheduler, force, replies: dict, run: Run) -> None: - while len(self._pool) < min(self.workers, len(sched.queue)): # no more processes than tasks + while len(self._pool) < min(self.workers, len(sched.queue)): # no more processes than jobs self._pool.append(self._spawn()) try: while sched.queue or sched.running: for i in range(len(self._pool)): - if self._pool[i].task is None: - t = sched.next() - if t is None: + if self._pool[i].job is None: + j = sched.next() + if j is None: break if not self._pool[i].proc.is_alive(): # died while idle self._replace(self._pool[i]) w = self._pool[i] - self._record(run, t, sched, w) - w.send(t, force(t)) - busy = [w for w in self._pool if w.task is not None] + self._record(run, j, sched, w) + w.send(j, [force(t) for t in j.tasks]) + busy = [w for w in self._pool if w.job is not None] ready = wait([w.conn for w in busy] + [w.proc.sentinel for w in busy]) for w in busy: - if w.conn not in ready and w.proc.sentinel not in ready: - continue - t = w.task - assert t is not None # a busy worker has its task - reply = None - try: - if w.conn.poll(): - reply = w.conn.recv() - except (EOFError, OSError): - reply = None - if reply is None: - w.proc.join(5) - reply = { - "ok": False, - "code": "task.worker_died", - "message": f"worker exited with code {w.proc.exitcode} while running the task", - "trace": "", - "rss": None, - } - replies[t.id] = reply - self._done(run, t, reply, w) - sched.finished(t.id) - if reply.get("code") == "config": - sched.queue.clear() - w.task = None - w.done += 1 - if not w.proc.is_alive() or reply.get("code") == "task.worker_died" or self._recycle(w, reply): - self._replace(w) + if w.conn in ready or w.proc.sentinel in ready: + self._collect(w, sched, replies, run) except BaseException: self.close(kill=True) raise - def _recycle(self, w: _Worker, reply: dict) -> bool: + def _collect(self, w: _Worker, sched: Scheduler, replies: dict, run: Run) -> None: + """Take what worker `w` sent: the replies to its job's tasks, in order, then the job's end. A worker that + died fails the task it was running; the job's tasks after it are queued again.""" + job = w.job + assert job is not None # a busy worker has its job + while True: + try: + if not w.conn.poll(): + if w.proc.is_alive(): + return # more to come + break + msg = w.conn.recv() + except (EOFError, OSError): + break + if msg.get("end"): + sched.finished(job.id) + w.job = None + w.done += w.replied + if not w.proc.is_alive() or self._recycle(w, msg.get("rss")): + self._replace(w) + return + t = job.tasks[w.replied] + w.replied += 1 + replies[t.id] = msg + self._done(run, t, msg, w) + if msg.get("code") == "config": + sched.stop() + w.proc.join(5) + if w.replied < len(job.tasks): + t = job.tasks[w.replied] + replies[t.id] = { + "ok": False, + "code": "task.worker_died", + "message": f"worker exited with code {w.proc.exitcode} while running the task", + "trace": "", + "rss": None, + } + self._done(run, t, replies[t.id], w) + rest = job.tasks[w.replied + 1 :] + if rest: + sched.put(Job(rest, job.batch)) + sched.finished(job.id) + w.job = None + self._replace(w) + + def _recycle(self, w: _Worker, r: int | None) -> bool: if self.recycle_tasks and w.done >= self.recycle_tasks: return True - r = reply.get("rss") return bool(self.recycle_rss and r is not None and r > self.recycle_rss) def _replace(self, w: _Worker) -> None: @@ -683,18 +767,20 @@ def _replace(self, w: _Worker) -> None: w.conn.close() self._pool[self._pool.index(w)] = self._spawn() - def _record(self, run: Run, t: Task, sched: Scheduler, w) -> None: - run.log.append( - { - "event": "start", - "task": t.id, - "key": t.key, - "worker": None if w is None else w.n, - "estimate": t.cost.to_json() if t.cost is not None else None, - "runningBytes": sched.used() - _peak(t), - "budget": sched.budget, - } - ) + def _record(self, run: Run, j: Job, sched: Scheduler, w) -> None: + for t in j.tasks: + run.log.append( + { + "event": "start", + "task": t.id, + "key": t.key, + "job": j.id, + "worker": None if w is None else w.n, + "estimate": t.cost.to_json() if t.cost is not None else None, + "runningBytes": sched.used() - j.peak, + "budget": sched.budget, + } + ) def _done(self, run: Run, t: Task, reply: dict, w) -> None: e = { diff --git a/src/nnnotes/pipeline/plan.py b/src/nnnotes/pipeline/plan.py index 1cba6ee..e5184ea 100644 --- a/src/nnnotes/pipeline/plan.py +++ b/src/nnnotes/pipeline/plan.py @@ -28,7 +28,7 @@ from . import contract from .depot import write_file -from .orchestrate import WORKER_BYTES +from .orchestrate import WORKER_BYTES, jobs from .stages import Env, Pending, Stage, describe @@ -217,7 +217,7 @@ def plan( "cost": t.cost.to_json() if t.cost is not None else None, } ) - return Plan(_doc(nodes, edges, runnable, workers, memory, worker_bytes, layouts or {}), runnable) + return Plan(_doc(nodes, edges, jobs(runnable, stages), workers, memory, worker_bytes, layouts or {}), runnable) def _run_reasons(t, old: dict | None, depot, forced: bool) -> list[dict]: @@ -241,21 +241,22 @@ def _run_reasons(t, old: dict | None, depot, forced: bool) -> list[dict]: def _doc( - nodes: list, edges: set, runnable: list, workers: int, memory: int | None, worker_bytes: int, layouts: dict + nodes: list, edges: set, todo: list, workers: int, memory: int | None, worker_bytes: int, layouts: dict ) -> dict: + """The plan document; `todo`: the jobs (orchestrate.jobs) of the runnable tasks.""" count = {s: sum(n["status"] == s for n in nodes) for s in ("hit", "run", "unknown")} - peaks = sorted((t.cost.peak_bytes if t.cost else 0 for t in runnable), reverse=True) + peaks = sorted((j.peak for j in todo), reverse=True) procs = max(1, workers) budget = None if memory is None else max(0, memory - procs * worker_bytes) peak = sum(peaks[:procs]) if budget is not None and peaks: peak = min(peak, max(budget, peaks[0])) peak += procs * worker_bytes if peaks else 0 - oversize = sorted({t.id for t in runnable if budget is not None and t.cost and t.cost.peak_bytes > budget}) + oversize = sorted(t.id for j in todo if budget is not None and j.peak > budget for t in j.tasks) summary = { "tasks": len(nodes), **count, - "cpuSeconds": round(sum(t.cost.cpu_seconds for t in runnable if t.cost), 3), + "cpuSeconds": round(sum(j.cpu for j in todo), 3), "peakBytes": int(peak), "budget": memory, "oversize": oversize, diff --git a/src/nnnotes/pipeline/stages.py b/src/nnnotes/pipeline/stages.py index ced2e4b..1435c28 100644 --- a/src/nnnotes/pipeline/stages.py +++ b/src/nnnotes/pipeline/stages.py @@ -102,6 +102,14 @@ def estimate(self, subject: str, env: Env, inputs: list[Input]) -> Cost: def run(self, task: Task, depot) -> Output: raise NotImplementedError + def batch(self, task: Task) -> str | None: + """The batch of a task to run: the tasks of a stage with the same batch run one after another in one process + (a job of the orchestrator), so what they share is loaded once. None: the task runs alone.""" + return None + + def end_batch(self, batch: str) -> None: + """The process ran the last task of `batch`: drop what the stage kept for it.""" + def registry(*stages: Stage) -> dict[str, Stage]: """{name: stage}; names must be unique.""" diff --git a/tests/pipeline/test_orchestrate.py b/tests/pipeline/test_orchestrate.py index a4cee60..47c2914 100644 --- a/tests/pipeline/test_orchestrate.py +++ b/tests/pipeline/test_orchestrate.py @@ -1,6 +1,7 @@ """The local orchestrator (toy stages of test_stages; spawned worker processes where the pool matters).""" import hashlib +import os from pathlib import Path import pytest @@ -8,10 +9,10 @@ from nnnotes.pipeline import contract from nnnotes.pipeline.contract import Cost, Task from nnnotes.pipeline.depot import Depot -from nnnotes.pipeline.orchestrate import Orchestrator, Scheduler, coverage +from nnnotes.pipeline.orchestrate import Job, Orchestrator, Scheduler, coverage, jobs from nnnotes.pipeline.plan import plan -from nnnotes.pipeline.stages import execute -from tests.pipeline.test_stages import PIPELINE, toy_files, toy_stages +from nnnotes.pipeline.stages import execute, registry +from tests.pipeline.test_stages import PIPELINE, Count, Src, Total, toy_files, toy_stages TEXTS = { "a": "alpha\nbeta\n\nFAIL here\n?what", @@ -144,30 +145,126 @@ def test_a_bumped_stage_reruns_itself_and_what_reads_it(tmp_path): assert {t for t, v in st.items() if v == "ran"} == {f"toy.count:{s}" for s in TEXTS} | {"toy.total:all"} +# ---------------------------------------------------------------- batches +class Batched(Src): + """toy.src whose tasks are batched by the first letter of their subject; every process logs to /.log + what it loads, runs and drops.""" + + def __init__(self, log: Path): + self.log, self.loaded = str(log), None + + def batch(self, task): + return task.subject[0] + + def run(self, task, depot): + if self.loaded != task.subject[0]: + self.loaded = task.subject[0] + self._note(f"load {self.loaded}") + self._note(f"run {task.subject}") + return super().run(task, depot) + + def end_batch(self, batch): + self._note(f"drop {batch}") + self.loaded = None + + def _note(self, line: str) -> None: + with open(Path(self.log) / f"{os.getpid()}.log", "a", encoding="utf-8") as f: + f.write(line + "\n") + + +def batched(tmp_path, name, texts, workers): + log = tmp_path / f"{name}-log" + log.mkdir() + stages = registry(Batched(log), Count(), Total()) + depot, run, orch = orchestrate(tmp_path, name, texts, workers=workers, stages=stages) + return depot, run, orch, [" ".join(f.read_text(encoding="utf-8").split()) for f in sorted(log.iterdir())] + + +BATCHED = {"a1": "x", "a2": "y\nz", "a3": "w", "b1": "v", "b2": "FAIL"} + + +def test_the_tasks_of_a_batch_run_in_one_process_which_drops_it_after_the_last(tmp_path): + plain, _, _ = orchestrate(tmp_path, "plain", BATCHED) + for workers in (0, 2): + depot, run, orch, logs = batched(tmp_path, f"w{workers}", BATCHED, workers) + assert tree(depot.root, "cas", "ac") == tree(plain.root, "cas", "ac") # batches change no result + assert all(v["status"] == "ran" for v in run.tasks.values()) + a, b = "load a run a1 run a2 run a3 drop a", "load b run b1 run b2 drop b" + if workers: + assert sorted(logs) == [a, b] and orch._started == 2 + else: + assert logs == [f"{a} {b}"] # equal estimates: by id + jobs_of = {e["task"]: e["job"] for e in run.log if e["event"] == "start" and e["task"].startswith("toy.src")} + assert jobs_of == {f"toy.src:{s}": f"toy.src:{s[0]}1" for s in BATCHED} + + +def test_a_worker_dying_in_a_job_fails_its_task_and_the_rest_runs_in_another(tmp_path): + texts = {"a1": "x", "a2": "DIE", "a3": "y", "a4": "z", "b1": "v"} + _, run, orch, logs = batched(tmp_path, "s", texts, workers=1) + st = {t: v["status"] for t, v in run.tasks.items() if t.startswith("toy.src")} + assert st == { + "toy.src:a1": "ran", + "toy.src:a2": "failed", + "toy.src:a3": "ran", + "toy.src:a4": "ran", + "toy.src:b1": "ran", + } + assert [f["code"] for f in run.failures() if f["task"] == "toy.src:a2"] == ["task.worker_died"] + assert sorted(logs) == ["load a run a1 run a2", "load a run a3 run a4 drop a load b run b1 drop b"] + assert orch._started >= 2 + + +def test_jobs_group_a_stages_batches(): + def task(stage, name, cpu, peak): + return Task(stage, 1, name, cost=Cost(cpu, peak)) + + class Firsts(Src): + def batch(self, task): + return task.subject[0] if task.subject != "a9" else None + + stages = registry(Firsts(), Count()) + ts = [task("toy.src", s, 1.0, p) for s, p in (("a1", 10), ("a2", 30), ("b1", 5), ("a9", 7))] + ts.append(task("toy.count", "a1", 2.0, 1)) + js = jobs(reversed(ts), stages) + assert [([t.subject for t in j.tasks], j.batch) for j in js] == [ + (["a1"], None), + (["a9"], None), + (["a1", "a2"], ("toy.src", "a")), + (["b1"], ("toy.src", "b")), + ] + assert (js[2].id, js[2].cpu, js[2].peak) == ("toy.src:a1", 2.0, 30) + + # ---------------------------------------------------------------- admission -def toy_task(name, cpu, peak): - return Task("toy.src", 1, name, cost=Cost(cpu, peak)) +def toy_job(name, cpu, peak): + return Job([Task("toy.src", 1, name, cost=Cost(cpu, peak))]) def test_scheduler_orders_longest_first_and_admits_within_the_budget(): - s = Scheduler([toy_task("a", 1, 40), toy_task("b", 5, 70), toy_task("c", 3, 30), toy_task("d", 3, 20)], 100) - assert s.next().subject == "b" # longest first - assert s.next().subject == "c" # 70 + 30 fits + s = Scheduler([toy_job("a", 1, 40), toy_job("b", 5, 70), toy_job("c", 3, 30), toy_job("d", 3, 20)], 100) + assert s.next().id == "toy.src:b" # longest first + assert s.next().id == "toy.src:c" # 70 + 30 fits assert s.next() is None # d (20) would pass 100 s.finished("toy.src:b") - assert [s.next().subject, s.next().subject] == ["d", "a"] + assert [s.next().id, s.next().id] == ["toy.src:d", "toy.src:a"] assert s.next() is None and s.used() == 90 and s.oversize == [] + s.put(toy_job("e", 9, 1)) + assert s.next().id == "toy.src:e" # queued again in its place + s.stop() + s.put(toy_job("f", 1, 1)) + assert s.next() is None and s.queue == [] -def test_scheduler_runs_an_oversize_task_alone(): - s = Scheduler([toy_task("big", 1, 500), toy_task("x", 9, 10), toy_task("y", 0.5, 10)], 100) - assert s.next().subject == "x" +def test_scheduler_runs_an_oversize_job_alone(): + big = Job([Task("toy.src", 1, n, cost=Cost(0.5, 500)) for n in ("big1", "big2")]) + s = Scheduler([big, toy_job("x", 9, 10), toy_job("y", 0.5, 10)], 100) + assert s.next().id == "toy.src:x" assert s.next() is None # big waits for the pool to drain; y must not pass it s.finished("toy.src:x") - assert s.next().subject == "big" and s.oversize == ["toy.src:big"] + assert s.next() is big and s.oversize == ["toy.src:big1", "toy.src:big2"] assert s.next() is None # nothing joins it - s.finished("toy.src:big") - assert s.next().subject == "y" + s.finished("toy.src:big1") + assert s.next().id == "toy.src:y" def test_pool_admission_respects_the_budget(tmp_path): diff --git a/tests/pipeline/test_plan.py b/tests/pipeline/test_plan.py index df9bbe5..76cb105 100644 --- a/tests/pipeline/test_plan.py +++ b/tests/pipeline/test_plan.py @@ -7,7 +7,8 @@ from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.orchestrate import Orchestrator from nnnotes.pipeline.plan import plan, why -from tests.pipeline.test_stages import PIPELINE, toy_files, toy_stages +from nnnotes.pipeline.stages import registry +from tests.pipeline.test_stages import PIPELINE, Count, Src, Total, toy_files, toy_stages TEXTS = {"a": "alpha\nbeta", "b": "gamma", "c": "delta\n?x"} @@ -165,6 +166,13 @@ def test_peak_estimate_oversize_and_layout_counts(tmp_path): assert "layout original: write 0 / remove 0 / keep 4" in p.text() p = plan(toy_stages(), PIPELINE, depot, facts=facts, workers=2, memory=None, worker_bytes=10) assert p.doc["summary"]["peakBytes"] == 1000 + 100 + 2 * 10 + + class OneBatch(Src): + def batch(self, task): + return "all" + + p = plan(registry(OneBatch(), Count(), Total()), PIPELINE, depot, facts=facts, workers=2, worker_bytes=10) + assert p.doc["summary"]["peakBytes"] == 1000 + 2 * 10 # one job: its tasks run one after another run(depot, facts) p = plan(toy_stages(), PIPELINE, depot, facts=facts, layouts={"cas": {"write": 1, "remove": 0, "keep": 3}}) assert p.check() == 1 # nothing runs, but a file would be written From 1c3dff87135898d373e1326e34786141358b9e31 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 04:01:45 +0800 Subject: [PATCH 12/91] feat(export)!: run unity.export once per unit instead of once per bundle unity.export (version 2) runs one task per unit: a prefab hierarchy (a root GameObject with the objects its prefab holds) or an object outside any hierarchy, named by the stable address of its root object, `:`. The unit table (nnnotes.pipeline.units) comes from the censuses: a run exports the units of the selected bundles and every unit their objects reference, and `--only-class` selects units by the class of their root object instead of entering the task keys. - The tasks of one bundle form one batch: a worker loads the bundle once and shares its object index, hierarchies, atlases and last decoded textures across them. A unit's result does not depend on the other units of the batch or their order. - A task key covers the bundle, the unit's root, the converters of its classes and the script entries and shipped type trees its objects use, so a converter or script change runs again only the units holding it. - MonoScript and AssetBundle objects get the typetree JSON like any other class. - The original layout roots objects without a container path at `_bundles/` from the unit table; cri.audio, cri.movie, sprite.crop and view.voices find bundle-held content through it. BREAKING CHANGE: unity.export task ids are `unity.export::`. The bundle-level artifacts `scripts.json` and `bundle.json`, the link.artifacts stage and the nnnotes.artifacts/1 schema are removed. Co-Authored-By: Claude Opus 5.5 --- docs/assets.md | 8 +- docs/cli.md | 4 +- docs/contracts.md | 17 +- docs/schemas.md | 1 - docs/stages.md | 55 ++- src/nnnotes/cli/assets.py | 81 +-- src/nnnotes/pipeline/cristages.py | 32 +- src/nnnotes/pipeline/link.py | 130 +---- src/nnnotes/pipeline/objexport.py | 574 +++++++++++----------- src/nnnotes/pipeline/units.py | 198 ++++++++ src/nnnotes/pipeline/voices.py | 28 +- src/nnnotes/schemas/artifacts.schema.json | 57 --- src/nnnotes/unity/convert/__init__.py | 16 +- src/nnnotes/unity/convert/generic.py | 20 +- src/nnnotes/unity/convert/objects.py | 59 ++- src/nnnotes/unity/convert/textures.py | 31 +- tests/cli/test_assets.py | 132 +++-- tests/golden/unity.export.json | 68 +-- tests/helpers/units.py | 76 +++ tests/pipeline/test_cristages.py | 47 +- tests/pipeline/test_link.py | 75 --- tests/pipeline/test_objexport.py | 520 ++++++++++---------- tests/pipeline/test_voices_story.py | 38 +- 23 files changed, 1145 insertions(+), 1122 deletions(-) create mode 100644 src/nnnotes/pipeline/units.py delete mode 100644 src/nnnotes/schemas/artifacts.schema.json create mode 100644 tests/helpers/units.py diff --git a/docs/assets.md b/docs/assets.md index c94bb36..96e04b3 100644 --- a/docs/assets.md +++ b/docs/assets.md @@ -45,7 +45,7 @@ nnnotes depot verify [--quick] [--workers N] | `--stored-catalog` | a catalog version of the depot (`catalog versions`) instead of the current catalog | | `--png-level N` | zlib level 0-9 of the PNG files (default 6); part of the task keys | | `--flac-level N` | FLAC compression level 0-12 of the CRI audio and the movies' audio (default 8; every level decodes to the same samples; 12 writes files 0.2-1.3 % smaller in about five times the encoding time); part of the task keys | -| `--only-class C,...` | export only objects of these Unity classes; part of the task keys | +| `--only-class C,...` | export only the units whose root object is of these Unity classes (a prefab by `GameObject`, [stages.md](stages.md#units)) | | `--link` | how the layout files are made from the depot (below): `auto` (default; `[export] link`), `clone`, `hard` or `copy` | | `--strict` | exit 1 when a view has gaps: a required role whose address names no catalog key or no fitting sub-object (empty values and roles that do not apply are not gaps); in `voices`, a voice whose cue sheet has no catalog key or whose decoded sheet lacks its cue, and a story episode without a catalog key | | `--dry-run` | print the plan and stop | @@ -53,10 +53,12 @@ nnnotes depot verify [--quick] [--workers N] A run resolves the context (region, language, catalog version) and indexes the catalog, identifies the selected bundles (the sha256 of each decrypted bundle, remembered by path, size and modification time; bundles not -downloaded yet are fetched into the depot's `downloads/`), takes the census of each bundle, links the scripts and addresses, exports the bundles, runs the +downloaded yet are fetched into the depot's `downloads/`), takes the census of each bundle, links the scripts and +addresses, exports the units (one task per hierarchy or object outside any, [stages.md](stages.md#units)), runs the stages that need those results (atlas sprites, views), writes the layouts and the reports. The selected raw files (CRI audio and movies) not downloaded yet are fetched while the bundle stages run. The census also covers -the bundles the selected ones depend on, so script classes and atlas textures in other bundles resolve. +the bundles the selected ones depend on, so script classes and atlas textures in other bundles resolve, and the +export covers the units of those bundles that the selected ones reference. Tasks run in spawned worker processes that stay alive between tasks. The tasks of a stage that read the same loaded data (a stage's batch, such as one bundle) form one job: one worker runs them one after another and loads the diff --git a/docs/cli.md b/docs/cli.md index 469c685..ec38edb 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -174,7 +174,7 @@ every object of the selected bundles -> common formats (the depot and its layout | `--stored-catalog` `LABEL\|SHA` | a catalog version of the depot (`catalog versions`; default: the current catalog) | | `--png-level` `N` | PNG compression level 0-9 (changes the output) | | `--flac-level` `N` | FLAC compression level 0-12 of the CRI audio (default 8; every level decodes to the same samples; changes the files) | -| `--only-class` `C,...` | export only objects of these classes | +| `--only-class` `C,...` | export only the units whose root object is of these classes | | `--workers` `N` | worker processes (default: the CPUs this process may use; 0: this process) | | `--memory` `GiB` | memory budget of the running tasks and workers (default: 80 % of the physical memory) | | `--fetch-workers` `N` | parallel downloads (default 8) | @@ -317,7 +317,7 @@ what `export assets` would run and why, without running it (--json: nnnotes.plan | `--stored-catalog` `LABEL\|SHA` | a catalog version of the depot (`catalog versions`; default: the current catalog) | | `--png-level` `N` | PNG compression level 0-9 (changes the output) | | `--flac-level` `N` | FLAC compression level 0-12 of the CRI audio (default 8; every level decodes to the same samples; changes the files) | -| `--only-class` `C,...` | export only objects of these classes | +| `--only-class` `C,...` | export only the units whose root object is of these classes | | `--workers` `N` | worker processes (default: the CPUs this process may use; 0: this process) | | `--memory` `GiB` | memory budget of the running tasks and workers (default: 80 % of the physical memory) | | `--fetch-workers` `N` | parallel downloads (default 8) | diff --git a/docs/contracts.md b/docs/contracts.md index 5c5569b..369222e 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -36,14 +36,15 @@ The canonical JSON (`nnnotes.core.canon`) of every document nnnotes writes: | object | `:` | `CAB-2f1e…:-4153…`, `unity default resources:10` | | artifact of an object | `#` | `CAB-2f1e…:-4153…#image` | | artifact of no object | `:#` | `cri.audio:#bgm_x.flac` | -| task | `:` | `unity.export:membercard_assets_membercard_100101` | +| task | `:` | `unity.census:membercard_assets_membercard_100101`, `unity.export:membercard_assets_membercard_100101:-4153…` | | stable bundle name | the bundle file name without `_<32 hex>` and `.bundle` | `membercard_assets_membercard_100101` | | stable address | `/` (`[]` for a sub-object), else `:` | `membercard_assets_membercard_100101/assets/…/member_full.png` | Stage names contain no `:` or `#`; an artifact's owner contains no `#` (its role may). A bundle's serialized file names change whenever its content changes, while its path ids stay: an object id names one content version of an object, and the stable address finds the same object in another catalog version. Bundle stages use the stable -bundle name as their subject, so a task keeps its id across versions while its key follows the content. When two +bundle name as their subject and object stages the `:` address of their object, so a task +keeps its id across versions while its key follows the content. When two bundles of a catalog have the same stable name, both keep their full file names. ## Task: nnnotes.task/1 @@ -52,13 +53,14 @@ One stage applied to one subject, with everything needed to run it ([task.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/task.schema.json)): ```json -{"schema": "nnnotes.task/1", "id": "unity.export:membercard_assets_membercard_100101", - "stage": {"name": "unity.export", "version": 1}, - "params": {"png": {"encoder": "pillow", "level": 6}, "blobMin": 4096}, +{"schema": "nnnotes.task/1", "id": "unity.export:membercard_assets_membercard_100101:-4153…", + "stage": {"name": "unity.export", "version": 2}, + "params": {"blobMin": 4096, "png": {"level": 6}}, "atoms": {"png.encode": "pillow-12.3.0/1", "texture.decode": "unitypy-1.25.3/1"}, "inputs": [{"role": "bundle", "sha256": "…", "size": 123456, "name": "membercard_…_3fa…c2.bundle", "locators": [{"kind": "depot"}, {"kind": "download", "path": "bundles/membercard_…_3fa…c2.bundle"}]}], - "context": {"scripts": {"CAB-…:12345": "Assembly-CSharp|Fwk.Sound|SplitAcbData"}}, + "context": {"unit": "CAB-2f1e…:-4153…", "bundle": "membercard_assets_membercard_100101", + "scripts": {"CAB-…:12345": "Assembly-CSharp|Fwk.Sound|SplitAcbData"}}, "key": "…", "cost": {"cpuSeconds": 1.2, "peakBytes": 450000000}} ``` @@ -97,7 +99,7 @@ One output file ([artifact.schema.json](https://github.com/MetaSekaiLab/nnnotes/ ```json {"id": "CAB-…:-4153…#image", "content": {"sha256": "…", "size": 1234, "mediaType": "image/png", "ext": "png"}, - "provenance": {"stage": {"name": "unity.export", "version": 1, "params": ""}, + "provenance": {"stage": {"name": "unity.export", "version": 2, "params": ""}, "atoms": {"texture.decode": "…", "png.encode": "…"}, "inputs": [{"role": "bundle", "sha256": "…"}], "object": {"file": "CAB-…", "pathId": -4153, "classId": 28, "class": "Texture2D", @@ -221,7 +223,6 @@ Documents written by particular stages (artifacts of no object: their ids are `< | `nnnotes.census/2` | `unity.census` | [census.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/census.schema.json) | | `nnnotes.scripts/1` | `link.scripts` | [scripts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/scripts.schema.json) | | `nnnotes.addresses/1` | `link.addresses` | [addresses.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/addresses.schema.json) | -| `nnnotes.artifacts/1` | `link.artifacts` | [artifacts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/artifacts.schema.json) | | `nnnotes.view/1` | `view.` | [view.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/view.schema.json) | The catalog versions of a depot, `nnnotes.catalogs/1` ([catalogs.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalogs.schema.json), diff --git a/docs/schemas.md b/docs/schemas.md index f0930e7..0fa3405 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -16,7 +16,6 @@ schemas.load("plan") # the schema as a dict |---|---|---| | `addresses` | `nnnotes.addresses/1` | the address table of a catalog version and the censuses of its bundles | | `artifact` | `nnnotes.artifact/1` | the content, provenance and semantics of one output file | -| `artifacts` | `nnnotes.artifacts/1` | the artifact table: the artifacts of every object, with content id, size and extension | | `catalog-diff` | `nnnotes.catalog-diff/1` | the difference between two catalog versions (`catalog diff`) | | `catalog-index` | `nnnotes.catalog-index/1` | the index of a catalog pair: remote and APK catalogs, every location in catalog order | | `catalogs` | `nnnotes.catalogs/1` | the catalog versions of a depot (`catalog versions`) | diff --git a/docs/stages.md b/docs/stages.md index a350faf..7d46629 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -2,34 +2,32 @@ The asset export is built from three layers: -- **Stages** apply one unit of work to one subject (a bundle, a content id, a model key, a view). Each has a name and +- **Stages** apply one unit of work to one subject (a bundle, an object, a content id, a model key, a view). Each has a name and a version; a task is a stage applied to one subject, and its key covers exactly what its output depends on ([contracts.md](contracts.md)). - **Converters** turn the objects of one class inside `unity.export` into artifacts. Each carries its own version; a - bundle's task includes only the converters of the classes the bundle holds. + task includes only the converters of the classes its objects have. - **Atoms** are the pure functions stages and converters are made of: no file or network IO, no threads, the same output for the same input. Each atom has exactly one reference implementation. ## Stages Every stage is at version 1, except `unity.census` (version 3: it lists what every object references and the -hierarchy holding it) -and `cri.movie` (version 2: every stream kind, codec and channel of a USM), and `catalog.index` (version 2: -gzip input and Japanese remote asset placeholders). +hierarchy holding it), `unity.export` (version 2: one task per unit), `cri.movie` (version 2: every stream kind, +codec and channel of a USM) and `catalog.index` (version 2: gzip input and Japanese remote asset placeholders). | Stage | Subject | Inputs | Outputs | |---|---|---|---| | `catalog.index` | a catalog (remote + APK) | the catalog files | every location (with the bundle hash, crc and size), bundles and raw remote files | | `unity.census` | a bundle, or a built-in file of the APK | the bundle | serialized files, externals, objects (file, path id, class, size, type hash, name, script, the scripts an AnimationClip's bindings name, the objects it references, the hierarchy holding it), container, scene hashes, dependencies, script fields | | `link.scripts` | global | the censuses holding MonoScripts | serialized file and path id -> assembly, namespace, class | -| `unity.export` | a bundle, or a built-in file of the APK | the bundle, the script entries its MonoBehaviours use (and, for files without type trees, the type trees nnnotes ships for their type hashes) | the objects' artifacts and one item per object | +| `unity.export` | a unit of a bundle or of a built-in file of the APK (below), by the stable address of its root object | the bundle; context `unit` (the root's object id), `bundle`, the script entries its objects use and, for files without type trees, the type trees nnnotes ships for their type hashes | its objects' artifacts and one item per object | | `sprite.crop` | a SpriteAtlas | the atlas' JSON, the images of its textures, the metas of the sprites of other bundles it packs | the sprites' images | | `cri.audio` | ACB content | `acb` (a raw file of the catalog, or the `acb` artifact of a cue sheet held in a bundle), `awb` (an AWB the catalog pairs with it), `boot` (the game's boot data: the key's source) | FLAC per stream, `cues.json`, `streams.json` | | `cri.movie` | USM content | `usm` (a raw file of the catalog, or the `usm` artifact of a movie held in a bundle), `boot` | every stream as stored (video, alpha, audio, subtitles), `movie.mkv` (or `movie.webm`) | | `live2d.model` | a Live2D model key (`Character/Live2D///model/`) whose bundle is selected | `bundle:`: the bundles of the key's dependency closure in the order the extractor loads them (the order is part of the key); context `internalId` (the key's location); atom `live2d.extract_model` (the extractor of `nnnotes export live2d`, run unchanged) | `live2d.model:#`: every file `nnnotes export live2d` writes for the model, `` relative to the model's directory; placed under `/` | | `spine.skeleton` | a SkeletonDataAsset of a selected bundle, by its stable address `:` (in a bundle of several serialized files the file's tag follows the bundle name, as in `link.addresses`) | `bundle:`: the bundles holding the serialized files the asset's file reaches through its externals; context `object` (the asset's object id); atom `spot.write_skeleton` (the Spine writer of `nnnotes export spot`) | `spine.skeleton:#`: the skeleton (`.json` or `.skel`), its atlases (`.atlas`) and their page images, the bytes `nnnotes export spot` writes into `spine/`; one item for the asset | | `link.addresses` | global | catalog index, censuses (the built-in files' too) | key -> objects, serialized file -> bundle or built-in file, bundle -> keys | -| `link.artifacts` | global | the above, the results | object id -> artifact ids | | `view.` | a catalog (`main`) | `master:
` per master table it reads, `rules` (the view's rule and the resolvers it uses), `addresses` (the part of the address table it read: the keys looked up and the prefixes listed); no parameters, atoms or context | `view.:#view` (facts: rows, entries, gaps, unreferenced) | | `view.voices` | a catalog (`main`) | `master:
` per master table its rules read, `rules`, `addresses`, `sheets` (each cue sheet the `cri.audio` results are known by that a voice names or the rules' prefixes list: its task and ACB content id), `acb:` (the ACB of each decoded one), `episodes` (per story episode and part, the `unity.export` artifact holding the object its key names), `adv:` (those contents); context `audio` (the keys of those `cri.audio` results); runs after `unity.export` and `cri.audio` | `view.voices:#view` (facts: rows, entries, gaps, unreferenced streams) | @@ -40,8 +38,8 @@ gzip input and Japanese remote asset placeholders). ## Converters (`unity.export`) The converters are the library `nnnotes.unity.convert`, one module per group of classes; `unity.export` -(`nnnotes.pipeline.objexport`) runs them over one bundle read alone and gives every object of it exactly one item -status: `exported`, +(`nnnotes.pipeline.objexport`) runs them over one unit of a bundle read alone and gives every object of the unit +exactly one item status: `exported`, `contained` (part of another artifact), `generic` (typetree JSON because the specialized form could not be made; the reason says why), `unsupported` (a class the stage does not convert; its typetree JSON is written as well) or `failed` (not even the typetree JSON could be made). A converter's documented limit gives the generic typetree JSON @@ -50,7 +48,7 @@ with its reason code; any other error of a converter gives it with `generic.erro | Class | Converter | Artifacts | Atoms | Notes | |---|---|---|---|---| | Texture2D | `tex.png/2` | `image`: PNG of the first mip level (HDR ASTC: `.astc`), `json`: the typetree without `image data` (settings, colour space, mip count, stream data) | `texture.decode`, `png.encode`, `astc.container` | further mip levels as facts; a texture without pixels: `empty.texture` | -| Sprite | `sprite.png/2` | `image`: PNG on the sprite's rect (packing undone, tight mask applied), `json`: the typetree, geometry, render data and image (see below) | `texture.decode`, `sprite.crop`, `png.encode`, `mesh.arrays` | the texture is decoded once per task; a sprite whose SpriteAtlas is in another bundle gets its `json` here (with its geometry and object facts) and its `image` from `sprite.crop` | +| Sprite | `sprite.png/2` | `image`: PNG on the sprite's rect (packing undone, tight mask applied), `json`: the typetree, geometry, render data and image (see below) | `texture.decode`, `sprite.crop`, `png.encode`, `mesh.arrays` | the tasks of a bundle share the textures it decoded last; a sprite whose SpriteAtlas is in another bundle gets its `json` here (with its geometry and object facts) and its `image` from `sprite.crop` | | Mesh | `mesh.glb/2` | `mesh`: binary glTF in the mesh's local space, `json`: the typetree without the vertex, index and compressed mesh data | `mesh.arrays`, `gltf.write` | see `gltf.write` below; limits of the skin and blend shapes as issues of the artifact | | GameObject (a root) | `prefab.json/2` | `json`: the hierarchy's nodes (path, path ids, transform) with their components' fields | | the objects below it are `contained`; a root of a scene's file names its scene | | Material | `material.json/1` | `json`: shader, keywords, textures, floats, colours | | | @@ -60,9 +58,7 @@ with its reason code; any other error of a converter gives it with `generic.erro | MonoBehaviour (not in a hierarchy) | `mono.json/2`, `cri.usm/1` | `json`: `$script` (assembly, namespace, class) and the typetree; a cue sheet asset also `acb`, a movie asset holding its USM also `usm` (see below) | `sniff` | the class from the task's script table | | TextAsset | `data/2` | `data`: the stored bytes, `json`: the typetree without `m_Script` | `sniff` | extension from the container path, else from the content | | Font | `font/2` | `font`: the font file, `json`: the typetree without `m_FontData` | `sniff` | | -| MonoScript | `scripts.json/1` | `#scripts.json`: every MonoScript of the bundle | | | -| AssetBundle | `bundle.json/1` | `#bundle.json`: serialized files, externals, the AssetBundle objects | | | -| any other class | `generic.json/2` | `json`: the typetree | `sniff` | an object that can be read only in part (see below): that part, `unsupported.typetree` | +| any other class (a MonoScript, an AssetBundle, ...) | `generic.json/2` | `json`: the typetree | `sniff` | an object that can be read only in part (see below): that part, `unsupported.typetree` | Every object a converter takes has a `json` artifact or is contained in another object's. Each item lists in `refs` the objects its documents reference (by object id, other bundles' objects included, the objects of its own @@ -105,10 +101,25 @@ again. A MonoBehaviour without one is `unsupported.typetree` with its header (`m issue. PlayerSettings, whose type tree reads the stored layout right up to `m_ActiveColorSpace` (by its stored type hash), gives the fields up to it. -A task's key includes the atoms `reader` and `sniff`, the converter `generic.json`, and the converters (with their -atoms) of the classes the bundle's census lists, so a converter's version bump runs again exactly the bundles holding -its classes. `classes` (a list of class names) limits a task to the objects of those classes and the objects their -hierarchies contain. +### Units + +A unit is what one `unity.export` task converts: a hierarchy (a root GameObject with the GameObjects, Transforms and +components of its prefab: the objects whose census record names it as their `unit`) or an object outside any +hierarchy. Its subject is the stable address of its root object, `:` (in a bundle of +several serialized files the file's tag follows the bundle name, as in `link.addresses`): path ids stay when a +bundle's content changes, so a unit keeps its task across catalog versions. + +A run exports the units holding an object of the selected bundles and every unit reachable from those through what +their objects reference (the census `refs`, across the censuses of the run); with `--only-class` only the units whose +root object is of those classes. The selection is not part of any key: a unit's task is the same whichever request +reaches it. + +A task's key includes the atoms `reader` and `sniff`, the converter `generic.json`, the converters (with their +atoms) of the classes of the unit's objects, its bundle, its root and the script entries and shipped type trees its +objects use, so a converter's version bump runs again exactly the units holding its classes. The tasks of one bundle +are one batch: a worker loads the bundle once for them and keeps what depends on the bundle alone (the index of its +objects, its hierarchies, its atlases and the textures it decoded last) until the batch ends. A unit's result +depends on neither the other units of the batch nor their order. Object JSON is canon/1 in the object's own field order: @@ -139,7 +150,7 @@ hierarchy that plays the clip, which the clip alone does not give. A cue sheet held in a bundle is a MonoBehaviour in one of two layouts; either gives the artifact `#acb` (kind `cri.acb`, facts: `cueSheet`, `layout`), which `cri.audio` decodes: -- a SplitAcbData (fields `_cueSheetName`, `_chunks`): its chunk TextAssets (of the same bundle, exported as well) joined +- a SplitAcbData (fields `_cueSheetName`, `_chunks`): its chunk TextAssets (of the same bundle, units it references) joined in order, every byte XORed with the one mask that gives the ACB signature `@UTF` (layout `split`; its refs name the chunks); - an asset named like the sheet whose `implementation` managed reference is a CriSerializedBytesAssetImpl: its @@ -148,11 +159,11 @@ A cue sheet held in a bundle is a MonoBehaviour in one of two layouts; either gi A movie asset (CriWare.Assets `CriManaUsmAsset`) whose `implementation` is a CriSerializedBytesAssetImpl holds its USM the same way: its `data.data` bytes, whatever their size, are the artifact `#usm` (kind `cri.usm`, facts: -`movie`, the asset's name), which `cri.movie` converts. Converter `cri.usm/1` does this; it is in the key of a bundle -only when the bundle's script context names that class, so no other bundle's key changes with it. +`movie`, the asset's name), which `cri.movie` converts. Converter `cri.usm/1` does this; it is in the key of a unit +only when the scripts of its objects name that class, so no other unit's key changes with it. -`sprite.crop` makes one task per SpriteAtlas that sprites of other bundles use (subject `:`). Its inputs are the atlas' JSON, the `json` of each such sprite and the images of the +`sprite.crop` makes one task per SpriteAtlas that sprites of other bundles use (subject: the `unity.export` +subject of the atlas' unit). Its inputs are the atlas' JSON, the `json` of each such sprite and the images of the atlas textures they use; it writes `#image` with the same bytes `unity.export` writes for a sprite whose atlas is in its own bundle. diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index 6580554..81d2ad4 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -104,13 +104,12 @@ def matches(self, stage: str) -> bool: StageSource("unity.census", "nnnotes.pipeline.census:CensusStage"), StageSource("link.scripts", "nnnotes.pipeline.link:ScriptsStage"), StageSource("link.addresses", "nnnotes.pipeline.link:AddressesStage"), - StageSource("unity.export", "nnnotes.pipeline.objexport:ExportStage", output=True, root="_bundles/{subject}"), + StageSource("unity.export", "nnnotes.pipeline.objexport:ExportStage", output=True), StageSource("sprite.crop", "nnnotes.pipeline.objexport:SpriteCropStage", output=True), StageSource("cri.audio", "nnnotes.pipeline.cristages:AudioStage", output=True), StageSource("cri.movie", "nnnotes.pipeline.cristages:MovieStage", output=True), StageSource("live2d.model", "nnnotes.pipeline.modelstages:Live2DStage", output=True), StageSource("spine.skeleton", "nnnotes.pipeline.modelstages:SpineStage", output=True), - StageSource("link.artifacts", "nnnotes.pipeline.link:ArtifactsStage"), StageSource("view.", "nnnotes.pipeline.views:stages", output=True), StageSource("view.voices", "nnnotes.pipeline.voices:VoiceStage", output=True), ) @@ -640,18 +639,6 @@ def params(self, pipeline: list[str]) -> dict: usage(self.args, "--flac-level: no stage of the pipeline encodes FLAC") for n in takers: out.setdefault(n, {})["flac"] = {**stages[n].PARAMS["flac"], "level": level} - only = getattr(self.args, "only_class", None) - if only: - classes = sorted({c.strip() for c in only.split(",") if c.strip()}) - takers = [n for n in pipeline if "classes" in stages[n].PARAMS] - if not classes or not takers: - usage( - self.args, - "--only-class: " - + ("no class given" if not classes else "no stage of the pipeline selects classes"), - ) - for n in takers: - out.setdefault(n, {})["classes"] = classes for n, p in out.items(): try: stages[n].normalize(p) @@ -659,6 +646,16 @@ def params(self, pipeline: list[str]) -> dict: usage(self.args, str(e)) return out + def classes(self) -> list[str] | None: + """The classes of --only-class (the root objects of the units to export), None for every class.""" + only = getattr(self.args, "only_class", None) + if only is None: + return None + classes = sorted({c.strip() for c in only.split(",") if c.strip()}) + if not classes: + usage(self.args, "--only-class: no class given") + return classes + def workers(self) -> int: w = getattr(self.args, "workers", None) return max(0, w) if w is not None else usable_cpus() @@ -695,6 +692,7 @@ def facts(self, pipeline: list[str], fetch: bool, background: bool = False) -> d "master": self.master() if any(n.startswith("view.") for n in pipeline) else None, "views": sorted(n[5:] for n in pipeline if n.startswith("view.")), "bundles": inputs, + "classes": self.classes(), "raw": Inputs({}), } if any(n in pipeline for n in RAW_STAGES): @@ -924,12 +922,12 @@ def layout_entries(self, name: str, placed: LayoutInput, *, store_derived: bool if name != "original": return layout.entries(name, sources) derives = lambda s: hasattr(stages[contract.parse_task_id(s.task)[0]], "derived") # noqa: E731 - roots = {} # every object of a bundle under its export task's root - for s in sources: - if s.task.startswith(EXPORT_STAGE + ":") and s.root is not None: - for f in s.result["files"]: - roots.setdefault(f, s.root) - entries, report = layout.entries(name, [s for s in sources if not derives(s)], case=case, object_roots=roots) + table = self.units() + files = table.files if table is not None else {} + roots = {f: f"_bundles/{b}" for f, b in files.items()} # every object of a bundle under one root + entries, report = layout.entries( + name, [s for s in sources if not derives(s)], case=case, bundles=files, object_roots=roots + ) report["names"] = named extra = self._derived(sources, entries, store_derived, placed.objects) taken = {e["path"].casefold() for e in entries} @@ -943,6 +941,22 @@ def done(self, tasks: dict) -> None: """The tasks whose results the layouts place, {task id: key} (after a run: every task with a result).""" self._done, self._names = dict(tasks), {} + def units(self): + """The unit table of the request (pipeline.units) for the tasks given to done(), a census without a result + counted as failed; None when the run has no censuses.""" + from ..pipeline import units + + env = Env(self.depot, self._facts) + for tid, key in sorted(self._done.items()): + env.done(tid, key) + census = [contract.task_id("unity.census", b) for b in self._facts.get("bundles") or ()] + if not any(t in self._done for t in census): + return None + for t in census: + if t not in self._done: + env.waiting(t, failed=True) + return units.table(env) + def names(self, stage_name: str) -> dict: """{subject: catalog names} of a stage that names its tasks' files (stage.names(env): the CRI stages root a content at each catalog name that refers to it, bundle-held content through the unity.export results; @@ -1014,11 +1028,7 @@ def add(self, tid: str, doc: dict) -> None: if obj: slim["provenance"] = {"object": obj} arts.append(slim) - files = {} - if tid.startswith(EXPORT_STAGE + ":"): - for it in doc["items"]: - files.setdefault(contract.parse_object_id(it["object"])[0], None) - self.results.append((tid, {"artifacts": arts, "files": list(files)})) + self.results.append((tid, {"artifacts": arts})) class _Asked: @@ -1118,23 +1128,19 @@ def parse_layouts(spec: str) -> list[str]: # ---------------------------------------------------------------- export class Totals: """What export reports of the results besides the run's counts, from its one reading of each (Run.scan): the - census totals of `subjects` ({class: objects}, the census artifacts' facts) and the results of the view tasks - (small; their master data tables and gaps).""" + census totals (`classes`: {class: objects} of the requested units, pipeline.units) and the results of the view + tasks (small; their master data tables and gaps).""" - def __init__(self, subjects): - self.want = {contract.task_id("unity.census", s) for s in subjects} - self.classes: Counter = Counter() + def __init__(self, classes: dict | None): + self.classes = None if classes is None else dict(sorted(classes.items())) self.views: list[tuple[str, dict]] = [] def add(self, tid: str, doc: dict) -> None: - if tid in self.want: - for a in doc["artifacts"]: - self.classes.update(a["semantics"].get("facts", {}).get("classes", {})) if tid.startswith("view."): self.views.append((tid, doc)) - def census(self) -> dict[str, int]: - return dict(sorted(self.classes.items())) + def census(self) -> dict[str, int] | None: + return self.classes def view_gaps(results) -> int: @@ -1237,7 +1243,8 @@ def cmd_export(args, cfg): ws.done(done) output = {tid for tid in done if ws._output(contract.parse_task_id(tid)[0])} placed = ws.layout_input({tid: done[tid] for tid in output}) - totals = Totals(sel.selected) + table = ws.units() + totals = Totals(table.classes if table is not None else None) def visit(tid, doc): totals.add(tid, doc) @@ -1531,7 +1538,7 @@ def run_args(c) -> None: help="FLAC compression level 0-12 of the CRI audio (default 8; every level decodes to the same " "samples; changes the files)", ) - c.add_argument("--only-class", metavar="C,...", help="export only objects of these classes") + c.add_argument("--only-class", metavar="C,...", help="export only the units whose root object is of these classes") c.add_argument( "--workers", type=nonnegative, diff --git a/src/nnnotes/pipeline/cristages.py b/src/nnnotes/pipeline/cristages.py index fd7f1b9..077e5ce 100644 --- a/src/nnnotes/pipeline/cristages.py +++ b/src/nnnotes/pipeline/cristages.py @@ -44,7 +44,7 @@ from ..formats.encoders import FLAC_LEVEL from ..unity.atoms import impl_id -from . import builtins, contract +from . import builtins, contract, units from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage @@ -306,10 +306,7 @@ def normalize(self, params: dict | None) -> dict: def index(self, env) -> dict: """{content id: {acb, awb, depends, names}} of the raw ACBs and the ACB artifacts of the unity.export results.""" - pending = env.pending(EXPORT) - if pending: - raise Pending(pending[0]) - tids = env.tasks(EXPORT) + tids = _holders(env) raw = _raw(env) cat = self.catalog(env) sig = (tuple((s, i.sha256) for s, (i, _m) in raw.items()), tuple(env.key(t) for t in tids), id(cat)) @@ -426,6 +423,24 @@ def run(self, task, depot) -> Output: return self._item(task, arts) +def _holders(env) -> list[str]: + """The unity.export tasks with a result whose unit is a MonoBehaviour outside any hierarchy (the units that may + hold an ACB or a USM); Pending while one has not run.""" + pending = env.pending(EXPORT) + if pending: + raise Pending(pending[0]) + tids = env.tasks(EXPORT) + if not tids: + return [] + table = units.table(env) + out = [] + for tid in tids: + u = table.units.get(contract.parse_task_id(tid, EXPORT)[1]) + if u is not None and u.cls == "MonoBehaviour": + out.append(tid) + return out + + def _export_artifacts(depot, key: str, kind: str) -> list[dict]: """The artifact records of semantics kind `kind` (cri.acb, cri.usm) of a unity.export result (a result without the kind is not parsed).""" @@ -480,10 +495,7 @@ def normalize(self, params: dict | None) -> dict: def index(self, env) -> dict: """{content id: {usm, depends, names}} of the raw USMs and the USM artifacts of the unity.export results.""" - pending = env.pending(EXPORT) - if pending: - raise Pending(pending[0]) - tids = env.tasks(EXPORT) + tids = _holders(env) raw = _raw(env) cat = self.catalog(env) sig = (tuple((s, i.sha256) for s, (i, _m) in raw.items()), tuple(env.key(t) for t in tids), id(cat)) @@ -499,7 +511,7 @@ def entry(inp: Input) -> dict: e = entry(Input("usm", inp.sha256, inp.size, inp.name, inp.locators)) e["names"] |= cat.keys.get(stable) or {stable} for tid in tids: - bundle = contract.parse_task_id(tid, EXPORT)[1] + bundle = units.table(env).unit(contract.parse_task_id(tid, EXPORT)[1]).bundle for a in _export_artifacts(env.depot, env.key(tid), USM_KIND): c = a["content"] e = entry(Input("usm", c["sha256"], c["size"], None, ({"kind": "depot"},))) diff --git a/src/nnnotes/pipeline/link.py b/src/nnnotes/pipeline/link.py index 88363db..bf41aa9 100644 --- a/src/nnnotes/pipeline/link.py +++ b/src/nnnotes/pipeline/link.py @@ -27,20 +27,10 @@ "|" ("apk/data.unity3d|level0:3"). Objects that no container path names are not listed; `address_of` gives any object's address. -link.artifacts (nnnotes.artifacts/1), one global task: the results of the stages that export objects (unity.export, -sprite.crop) -> where each object went: - objects {object id: [artifact ids]} of the objects with artifacts (exported, generic, unsupported with some) - contained {object id: the artifact it is part of} - artifacts {artifact id: {sha256, size, ext}} -Its context names the results it reads ({"results": {task id: key}}); results are functions of their keys, so the -table is a function of its key. With a layout manifest (its entries' ids) it gives every object's files. The task -reads the results one at a time and writes the table's text entry by entry (encode_artifacts), so its memory is the -table's entries, not the results nor the whole text. """ from __future__ import annotations -import json import re from ..unity import scripttypes @@ -51,8 +41,6 @@ SCRIPTS = "nnnotes.scripts/1" ADDRESSES = "nnnotes.addresses/1" -ARTIFACTS = "nnnotes.artifacts/1" -EXPORT_STAGES = ("unity.export", "sprite.crop") _CAB = re.compile(r"^CAB-[0-9a-f]+") @@ -92,7 +80,9 @@ def file_tags(files) -> dict[str, str]: return tags -def _path_address(stable: str, tags: dict, file: str, pid: int) -> str: +def path_address(stable: str, tags: dict, file: str, pid: int) -> str: + """The path-id stable address of object `pid` of serialized file `file` of bundle `stable` (`tags`: file_tags of + the bundle's files).""" tag = tags.get(file, "|" + file) if tags else "" return contract.stable_address(stable + tag, path_id=pid) @@ -124,7 +114,7 @@ def cls(e): if name and len(same) == 1: out.append((contract.stable_address(stable, path, name=name), e)) else: - out.append((_path_address(stable, tags, e["file"], e["pathId"]), e)) + out.append((path_address(stable, tags, e["file"], e["pathId"]), e)) return out @@ -233,7 +223,7 @@ def address_of(doc: dict, oid: str, container_objects: dict | None = None) -> st stable = doc["files"].get(file) if stable is None: return None - return _path_address(stable, file_tags(doc["bundles"][stable]["files"]), file, pid) + return path_address(stable, file_tags(doc["bundles"][stable]["files"]), file, pid) def reverse(doc: dict) -> dict[str, str]: @@ -343,113 +333,3 @@ def run(self, task, depot) -> Output: {"kind": "link.addresses", "format": "json", "facts": facts}, ) return Output([rec], []) - - -def artifact_table(results) -> tuple[dict, dict, dict]: - """The parts of the artifact table of results [(task id, result)], each result read once and not kept: - ({object id: set of artifact ids}, {object id: the artifact it is part of}, {artifact id: {sha256, size, ext}}), - unsorted. The ids an object is contained in repeat (every part of a prefab names it): one string each.""" - objects, contained, arts, same = {}, {}, {}, {} - for _, doc in results: - for a in doc["artifacts"]: - c = a["content"] - arts[a["id"]] = {"sha256": c["sha256"], "size": c["size"], "ext": c["ext"]} - for it in doc["items"]: - if it.get("artifacts"): - objects.setdefault(it["object"], set()).update(it["artifacts"]) - elif "in" in it: - contained[it["object"]] = same.setdefault(it["in"], it["in"]) - return objects, contained, arts - - -def artifacts(results) -> dict: - """The artifact table (nnnotes.artifacts/1) of results [(task id, result)].""" - objects, contained, arts = artifact_table(results) - return { - "schema": ARTIFACTS, - "objects": {o: sorted(v) for o, v in sorted(objects.items())}, - "contained": dict(sorted(contained.items())), - "artifacts": dict(sorted(arts.items())), - } - - -def encode_artifacts(objects: dict, contained: dict, arts: dict, chunk: int = 1 << 20): - """contract.encode(artifacts(...)) of the parts artifact_table gives, as byte strings of about `chunk` bytes: - the canonical text (keys sorted, indent 1, non-ASCII as is, one trailing newline) written entry by entry, so the - whole document is never built.""" - - def text(v) -> str: - return json.dumps(v, indent=1, ensure_ascii=False, sort_keys=True) - - buf, size = ["{"], 0 - tables = ( - ("artifacts", arts, arts.get), - ("contained", contained, contained.get), - ("objects", objects, lambda k: sorted(objects[k])), - ) - for i, (name, table, value) in enumerate(tables): - buf.append(("\n " if i == 0 else ",\n ") + text(name) + ": ") - if not table: - buf.append("{}") - continue - sep = "{\n " - for k in sorted(table): - s = sep + text(k) + ": " + text(value(k)).replace("\n", "\n ") - buf.append(s) - size += len(s) - sep = ",\n " - if size >= chunk: - yield "".join(buf).encode("utf-8") - buf, size = [], 0 - buf.append("\n }") - buf.append(",\n " + text("schema") + ": " + text(ARTIFACTS) + "\n}\n") - yield "".join(buf).encode("utf-8") - - -class ArtifactsStage(Stage): - """link.artifacts: the results of the export stages (EXPORT_STAGES, those of the pipeline) -> the artifact table. - One subject ("all"); no inputs, context {"results": {task id: key}}. Waits for export tasks that have not run; - skips failed ones. Artifact "link.artifacts:all#artifacts".""" - - name = "link.artifacts" - version = 1 - after = ("unity.export",) - - def subjects(self, env) -> list[str]: - return ["all"] - - def _tasks(self, env) -> list[str]: - for stage in EXPORT_STAGES: - pending = env.pending(stage) - if pending: - raise Pending(pending[0]) - return [t for stage in EXPORT_STAGES for t in env.tasks(stage)] - - def depends(self, subject: str, env) -> list[str]: - return self._tasks(env) - - def context(self, subject: str, env) -> dict: - return {"results": {t: env.key(t) for t in self._tasks(env)}} - - def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: - n = len(self._tasks(env)) - return Cost(0.05 + 0.002 * n, (60 << 20) + 200_000 * n) - - def run(self, task, depot) -> Output: - def results(): - for tid, key in sorted(task.context.get("results", {}).items()): - doc = depot.result(key) - if doc is None: - raise FileNotFoundError(f"result of {tid} ({key}) is not in the depot") - yield tid, doc - - objects, contained, arts = artifact_table(results()) - content = depot.add_chunks(encode_artifacts(objects, contained, arts), "json") - facts = {"objects": len(objects), "contained": len(contained), "artifacts": len(arts)} - rec = contract.artifact( - contract.artifact_id(task.id, "artifacts"), - content, - contract.provenance(task), - {"kind": "link.artifacts", "format": "json", "facts": facts}, - ) - return Output([rec], []) diff --git a/src/nnnotes/pipeline/objexport.py b/src/nnnotes/pipeline/objexport.py index 0c40d9f..1644ff1 100644 --- a/src/nnnotes/pipeline/objexport.py +++ b/src/nnnotes/pipeline/objexport.py @@ -1,29 +1,33 @@ -"""Stages `unity.export` (one bundle -> an artifact per object) and `sprite.crop` (the images of the sprites whose atlas -is in another bundle). - -unity.export reads one bundle alone (atom `reader`) and runs the converters of nnnotes.unity.convert over it, giving -every object exactly one item (contract.item): `exported` (its artifacts), `contained` (part of another artifact: -the objects of a hierarchy in its root's prefab, the MonoScripts in `scripts.json`, the AssetBundle objects in -`bundle.json`), `generic` (the object's typetree JSON, with the reason its converter gave for not making the -specialized form), `unsupported` (a class this stage does not convert; its typetree JSON is written all the same) or -`failed` (not even the typetree JSON could be made). A converter's documented limit (atoms.Unsupported) gives the -typetree JSON with its reason code; any other error of a converter gives it with `generic.error` and the error's -message. The reason codes are listed in docs/stages.md. A bundle's task key includes only the converters of the -classes its census lists. - -A texture is converted before the sprites cut from it (they read its decoded image) and a hierarchy before the -MonoBehaviours outside it; the objects no converter took get their typetree JSON last. - -sprite.crop: a sprite whose SpriteAtlas is in another bundle gets its `meta` from unity.export (with the sprite's -mesh and object facts); one sprite.crop task per atlas then reads the atlas' JSON, the images of its textures and the -metas, and makes "#image" exactly as unity.export makes the image of a sprite whose atlas is in its -own bundle. +"""Stages `unity.export` (one unit of a bundle -> the artifacts of its objects) and `sprite.crop` (the images of the +sprites whose atlas is in another bundle). + +unity.export: one task per unit (pipeline.units: a hierarchy, or an object outside any hierarchy) of a request. A +task reads its bundle alone (atom `reader`) and runs the converters of nnnotes.unity.convert over its unit, giving +every object of the unit exactly one item (contract.item): `exported` (its artifacts), `contained` (an object of a +hierarchy, in its root's prefab), `generic` (the object's typetree JSON, with the reason its converter gave for not +making the specialized form), `unsupported` (a class this stage does not convert; its typetree JSON is written all +the same) or `failed` (not even the typetree JSON could be made). A converter's documented limit +(atoms.Unsupported) gives the typetree JSON with its reason code; any other error of a converter gives it with +`generic.error` and the error's message. The reason codes are listed in docs/stages.md. A task's key holds the +bundle, the unit's root object, the converters of the classes of its objects and the script entries and shipped +type trees they use, so a unit converts again only when one of those changes. + +The tasks of one bundle are one batch (Stage.batch): a worker loads the bundle once for them and keeps, until the +batch ends, what depends on the bundle alone: the index of its objects, its hierarchies, its atlases' render data +and the last textures it decoded. A unit's result depends on neither the other units of its batch nor their order. +A sprite cut from a texture of its bundle reads the texture's decoded image; a hierarchy that cannot be walked +gives each of its objects the typetree JSON with the reason. + +sprite.crop: a sprite whose SpriteAtlas is in another bundle gets its `json` from unity.export (with the sprite's +mesh and object facts) and no image; one sprite.crop task per atlas then reads the atlas' JSON, the images of its +textures and those sprites' documents, and makes "#image" exactly as unity.export makes the image +of a sprite whose atlas is in its own bundle. """ from __future__ import annotations import io -from collections import defaultdict +from collections import OrderedDict, defaultdict from ..unity import scripttypes from ..unity.atoms import ATOMS as ATOM_REGISTRY @@ -48,22 +52,21 @@ ) from ..unity.convert.objects import ( TYPETREE, + BundleIndex, BundleObjects, ReadError, - encode_object, label, message, ref_object_id, ) -from . import builtins, contract, link -from . import census as census_mod +from . import builtins, contract, link, units from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage EXPORT, CROP, CENSUS, SCRIPTS = "unity.export", "sprite.crop", "unity.census", "link.scripts" SCRIPTS_TASK = contract.task_id(SCRIPTS, "all") ATOM_NAMES = frozenset(ATOM_REGISTRY) -# class -> (converter, function), in the order they run after the textures and sprites +# class -> (converter, function) of the classes converted by one function alone _SIMPLE = ( ("Mesh", "mesh.glb", files.mesh), ("TextAsset", "data", files.text), @@ -74,26 +77,80 @@ ("AnimatorController", "controller.json", animation.controller), ("AnimatorOverrideController", "controller.json", animation.controller), ) - - -# ---------------------------------------------------------------- one bundle -class BundleRun: - """unity.export on one bundle: the converters over its objects (the task's `classes` only), one item per - object, the artifacts in the depot.""" - - def __init__(self, view, task, depot, fns: dict): - self.task, self.depot = task, depot +DECODED_PIXELS = 1 << 25 # the decoded textures a batch keeps: the last ones, up to this many pixels + + +# ---------------------------------------------------------------- one bundle in a worker +class _Bundle: + """What the tasks of one bundle share in a worker: the bundle's view and BundleIndex, its hierarchies (valid + roots by GameObject key, and the objects of the hierarchies that cannot be walked with the reason), its atlases' + render data and the last decoded textures.""" + + def __init__(self, sha: str, view): + self.sha, self.view = sha, view + self.index = BundleIndex(view) + self.atlases: dict = {} + self._h = None + self.roots: dict[tuple, prefab.Root] = {} + self.broken: dict[tuple, str] = {} + self._decoded: OrderedDict = OrderedDict() + self._pixels = 0 + + def hierarchies(self, ctx: BundleObjects): + if self._h is None: + h = prefab.hierarchies(ctx) + for root in h.roots: + go = (root.file, root.game_object) + info = self.index.info.get(go) + if root.error is None and info is not None and info.class_name == "GameObject": + self.roots[go] = root + for root in h.roots: + if (root.file, root.game_object) in self.roots: + continue + why = root.error or f"transform {root.transform}: no GameObject" + for k in root.members: + if k in self.index.info and k not in h.member: + self.broken.setdefault(k, f"hierarchy at transform {root.transform}: {why}") + self._h = h + return self._h + + def decoded(self, ctx: BundleObjects, info) -> dict: + """A texture's decoded image or the error (textures.decode_into), kept while it is among the last + DECODED_PIXELS pixels decoded.""" + k = (info.file, info.path_id) + holder = self._decoded.get(k) + if holder is not None: + self._decoded.move_to_end(k) + return holder + holder = {} + textures.decode_into(ctx, info, holder) + img = holder.get("image") + self._decoded[k] = holder + self._pixels += img.width * img.height if img is not None else 0 + while self._pixels > DECODED_PIXELS and len(self._decoded) > 1: + _, old = self._decoded.popitem(last=False) + img = old.get("image") + self._pixels -= img.width * img.height if img is not None else 0 + return holder + + +# ---------------------------------------------------------------- one unit +class UnitRun: + """unity.export on one unit: its converters over its objects, one item per object, the artifacts in the + depot.""" + + def __init__(self, bundle: _Bundle, task, depot, fns: dict): + self.task, self.depot, self.bundle = task, depot, bundle self.ctx = BundleObjects( - view, + bundle.view, fns, scripts=task.context.get("scripts", {}), blob_min=int(task.params["blobMin"]), png_level=int(task.params["png"]["level"]), emit=self._emit, typetrees=task.context.get("typetrees", {}), + index=bundle.index, ) - classes = task.params.get("classes") - self.selected = None if classes is None else frozenset(classes) self.usm_assets = USM_CONVERTER in task.atoms self.items: dict[str, dict] = {} self.artifacts: list[dict] = [] @@ -175,8 +232,6 @@ def _need(self, conv: str) -> None: def _convert(self, info, conv: str, fn, *args, **kw) -> bool: """Run a converter on one object: its artifacts (exported), else the typetree JSON with the reason.""" - if info.id in self.items: - return False self._need(conv) self._begin(conv) try: @@ -213,124 +268,61 @@ def _generic(self, info) -> None: else: self._commit(info, "generic", contract.reason(TYPETREE, label(why))) - def _selected(self, cls: str) -> bool: - return self.selected is None or cls in self.selected - - def _of(self, cls: str) -> list: - """The objects of `cls` this task converts (none when the task's `classes` leave the class out).""" - return self.ctx.of(cls) if self._selected(cls) else [] - - # ---------------------------------------------------------------- the bundle + # ---------------------------------------------------------------- the unit def run(self) -> tuple[list, list]: - self._textures_sprites() - for cls, conv, fn in _SIMPLE: - for i in self._of(cls): - self._convert(i, conv, fn) - self._prefabs() - for i in self._of("MonoBehaviour"): - self._convert(i, "mono.json", mono.mono, usm_assets=self.usm_assets) - for conv, (cls, kind) in generic.LOOSE.items(): - self._loose(cls, conv, kind) - for i in self.ctx.infos: - if i.id in self.items or not self._selected(i.class_name): - continue - if i.class_name in UNSUPPORTED_CLASSES: - self._fallback( - i, f"unsupported.class.{i.class_name}", f"{i.class_name} is not converted", status="unsupported" - ) - else: - self._generic(i) + key = contract.parse_object_id(self.task.context["unit"]) + info = self.ctx.info.get(key) + if info is None: + raise IncompatibleTask(f"task {self.task.id}: object {self.task.context['unit']} is not in the bundle") + cls = info.class_name + self.bundle.hierarchies(self.ctx) + simple = {c: (conv, fn) for c, conv, fn in _SIMPLE} + if key in self.bundle.broken: + self._fallback(info, "generic.error", self.bundle.broken[key]) + elif key in self.bundle.roots: + self._prefab(info, self.bundle.roots[key]) + elif cls == "Sprite": + self._sprite(info) + elif cls == "Texture2D": + self._convert(info, "tex.png", textures.texture, self.bundle.decoded(self.ctx, info)) + elif cls in simple: + self._convert(info, *simple[cls]) + elif cls == "MonoBehaviour": + self._convert(info, "mono.json", mono.mono, usm_assets=self.usm_assets) + elif cls in UNSUPPORTED_CLASSES: + self._fallback(info, f"unsupported.class.{cls}", f"{cls} is not converted", status="unsupported") + else: + self._generic(info) return self.artifacts, list(self.items.values()) - def _textures_sprites(self) -> None: - waiting, atlases = defaultdict(list), {} - for s in self._of("Sprite"): - self._need("sprite.png") - self._begin("sprite.png") - try: - plan = textures.sprite_plan(self.ctx, (s.file, s.path_id), atlases) - except Unsupported as e: - self._fallback(s, e.code, e.message) - continue - except Exception as e: - self._fallback(s, "generic.error", message(e)) - continue - if plan["external"]: - self._convert(s, "sprite.png", textures.sprite_external, plan, CROP) - else: - waiting[plan["texture"]].append((s, plan)) - export_textures = self._selected("Texture2D") - for t in self.ctx.of("Texture2D"): - sprites = waiting.pop((t.file, t.path_id), ()) - if not sprites and not export_textures: - continue - holder: dict = {} - if export_textures: - self._convert(t, "tex.png", textures.texture, holder) - else: - textures.decode_into(self.ctx, t, holder) - for s, plan in sprites: - self._convert(s, "sprite.png", textures.sprite, plan, holder) - holder.clear() - for tkey, rest in waiting.items(): - for s, _plan in rest: - self._fallback(s, "no_data.sprite", f"texture {contract.object_id(*tkey)} is not decoded") - - def _prefabs(self) -> None: - if not self._selected("GameObject"): + def _sprite(self, info) -> None: + self._need("sprite.png") + self._begin("sprite.png") + try: + plan = textures.sprite_plan(self.ctx, (info.file, info.path_id), self.bundle.atlases) + except Unsupported as e: + self._fallback(info, e.code, e.message) return - h = prefab.hierarchies(self.ctx) - info_of = self.ctx.info - for root in h.roots: - info = info_of.get((root.file, root.game_object)) - if root.error is not None or info is None or info.class_name != "GameObject": - why = root.error or f"transform {root.transform}: no GameObject" - for k in root.members: - if k in info_of and info_of[k].id not in self.items: - self._fallback(info_of[k], "generic.error", f"hierarchy at transform {root.transform}: {why}") - continue - if self._convert(info, "prefab.json", prefab.prefab, h, root): - within = contract.artifact_id(info.id, "json") - for k in root.members: - i = info_of[k] - if i.id != info.id: - self.items[i.id] = contract.item(i.id, "contained", within=within, cls=i.class_name) - else: - why = self.items[info.id].get("reason") or {"code": "generic.error", "message": ""} - for k in root.members: - i = info_of[k] - if i.id not in self.items: - self._fallback(i, why["code"], f"hierarchy of {info.id}: {why['message']}") - - def _loose(self, cls: str, conv: str, kind: str) -> None: - """Objects of `cls` in one task-level artifact "#" (contained in it).""" - infos = [i for i in self._of(cls) if i.id not in self.items] - if not infos: + except Exception as e: + self._fallback(info, "generic.error", message(e)) return - self._need(conv) - self._begin(conv) - entries, done, failed = {}, [], [] - for i in infos: - mark = len(self.staged) - key = (i.file, i.path_id) - self.ctx.seen.clear() - try: - entries[i.id] = self.ctx.tree(self.ctx.readers[key], self.ctx.tt(key)) - done.append((i, self._refs(i))) - except Exception as e: - del self.staged[mark:] - failed.append((i, e)) - if done: - doc = encode_object(generic.loose_doc(self.ctx, conv, entries)) - aid = self._emit(None, conv, doc, "json", kind, "json", {"objects": len(done)}, owner=self.task.id)[0] - self._publish() - for i, refs in done: - self.items[i.id] = contract.item(i.id, "contained", within=aid, cls=i.class_name, refs=refs) - for i, e in failed: - if isinstance(e, ReadError): - self._failed(i, e) - else: - self._fallback(i, "generic.error", message(e)) + if plan["external"]: + self._convert(info, "sprite.png", textures.sprite_external, plan, CROP) + else: + holder = self.bundle.decoded(self.ctx, self.ctx.info[plan["texture"]]) + self._convert(info, "sprite.png", textures.sprite, plan, holder) + + def _prefab(self, info, root: prefab.Root) -> None: + h = self.bundle.hierarchies(self.ctx) + members = [self.ctx.info[k] for k in root.members if k != (info.file, info.path_id)] + if self._convert(info, "prefab.json", prefab.prefab, h, root): + within = contract.artifact_id(info.id, "json") + for i in members: + self.items[i.id] = contract.item(i.id, "contained", within=within, cls=i.class_name) + return + why = self.items[info.id].get("reason") or {"code": "generic.error", "message": ""} + for i in members: + self._fallback(i, why["code"], f"hierarchy of {info.id}: {why['message']}") # ---------------------------------------------------------------- stages @@ -361,7 +353,7 @@ def _check_atoms(task, reader: Impl | None = None) -> None: class _Docs: - """A few JSON documents of the depot by content id (the planning reads each census several times).""" + """A few JSON documents of the depot by content id (the planning reads some several times).""" def __init__(self, size: int = 64): self.size, self.docs = size, {} @@ -377,29 +369,27 @@ def get(self, depot, sha: str) -> dict: class ExportStage(Stage): - """unity.export: one bundle -> an artifact per object, one item per object (see the module documentation). - - Subjects: the selected bundles (the fact "selected", stable bundle names) and, when a selected bundle holds - sprites, the bundles of the fact "bundles" (the selection's closure) that hold a SpriteAtlas: a sprite reaches its - atlas texture through the atlas' bundle, whose export makes the texture's image (sprite.crop reads it). Inputs: - "bundles" {subject: Input (role "bundle")}, as unity.census. It reads the census of its bundle - (unity.census:) for the classes (the converters and atoms of its key) and the MonoScripts its objects - reference, and the script table (link.scripts:all) for the entries of those: its context - {"scripts": {object id: "assembly|namespace|class"}}, and for the MonoBehaviours of files that store no type trees - (the APK's built-in files) the type trees nnnotes ships for their type hashes: "typetrees" {type hash: nodes}. - Parameters: `blobMin` (bytes; byte arrays of this size or - more are blobs), `png.level` (zlib level of the PNG files). `reader`: another implementation of the reader atom - (its id enters the keys).""" + """unity.export: one unit -> the artifacts of its objects, one item per object (see the module documentation). + + Subjects: the units of the request (pipeline.units.table: the facts "selected", "bundles" and "classes", and + the censuses of the bundles). Inputs: the unit's bundle ("bundles" {stable name: Input (role "bundle")}, as + unity.census). It reads the script table (link.scripts:all) for the entries of the MonoScripts its objects + name. Context: "unit" (the root object's id), "bundle" (the stable bundle name), "scripts" {object id: + "assembly|namespace|class"} and, for the MonoBehaviours of files that store no type trees (the APK's built-in + files), "typetrees" {type hash: nodes}: the type trees nnnotes ships for their type hashes. Parameters: + `blobMin` (bytes; byte arrays of this size or more are blobs), `png.level` (zlib level of the PNG files). + `reader`: another implementation of the reader atom (its id enters the keys). Batch: the bundle's content id.""" name = EXPORT - version = 1 + version = 2 after = (CENSUS, SCRIPTS) - PARAMS = {"blobMin": 4096, "classes": None, "png": {"level": 6}} - LAYOUT_ROOT = "_bundles/{subject}" + PARAMS = {"blobMin": 4096, "png": {"level": 6}} def __init__(self, reader: Impl | None = None): self.reader = reader self._docs = _Docs() + self._atoms_memo: dict[tuple, dict] = {} + self._loaded: _Bundle | None = None def __getstate__(self): return {"reader": self.reader} @@ -426,113 +416,104 @@ def normalize(self, params: dict | None) -> dict: blob = p.get("blobMin", 4096) if not isinstance(blob, int) or isinstance(blob, bool) or blob < 1: raise ValueError(f"stage {self.name}: blobMin {blob!r}: expected a positive integer") - classes = p.get("classes") - if classes is not None: - if isinstance(classes, str) or not all(isinstance(c, str) and c for c in classes): - raise ValueError(f"stage {self.name}: classes {classes!r}: expected a list of class names") - classes = sorted(set(classes)) - return {"blobMin": blob, "classes": classes, "png": _png_params(p.get("png"))} + return {"blobMin": blob, "png": _png_params(p.get("png"))} def subjects(self, env) -> list[str]: - selected = set(env.fact("selected")) - out = set(selected) - if any("Sprite" in _census_classes(env, s) for s in sorted(selected)): - out.update(s for s in env.fact("bundles") if s not in out and "SpriteAtlas" in _census_classes(env, s)) - return sorted(out) + return units.table(env).subjects() + + def unit(self, subject: str, env) -> units.Unit: + return units.table(env).unit(subject) def inputs(self, subject: str, env) -> list[Input]: - inp = env.fact("bundles")[subject] + u = self.unit(subject, env) + inp = env.fact("bundles")[u.bundle] if inp.role != "bundle": raise ValueError(f"{self.name} {subject}: input role {inp.role!r}, expected 'bundle'") return [inp] - def census(self, subject: str, env) -> dict: - inp = env.input_of(contract.task_id(CENSUS, subject), "census") - return self._docs.get(env.depot, inp.sha256) - def uses(self, subject: str, env) -> dict: - """The atoms and converters of the classes the bundle's census lists; with the task's `classes` (env.params - while the task is described), of those of them only; converter cri.usm when a MonoBehaviour of the bundle is - a movie asset (its script context names the class).""" - classes = list(self.census(subject, env)["classes"]) - only = (getattr(env, "params", None) or {}).get("classes") - if only is not None: - keep = set(only) - classes = [c for c in classes if c in keep] - atoms = export_atoms(classes, self.reader_id()) - if "MonoBehaviour" in classes and USM_ASSET_SCRIPT in self.context(subject, env).get("scripts", {}).values(): - atoms = dict(sorted({**atoms, USM_CONVERTER: converter_id(USM_CONVERTER)}.items())) + """The atoms and converters of the classes of the unit's objects; converter cri.usm when one of them is a + movie asset (its script context names the class).""" + u = self.unit(subject, env) + usm = ( + "MonoBehaviour" in u.classes and USM_ASSET_SCRIPT in self.context(subject, env).get("scripts", {}).values() + ) + k = (u.classes, usm) + atoms = self._atoms_memo.get(k) + if atoms is None: + atoms = export_atoms(u.classes, self.reader_id()) + if usm: + atoms = dict(sorted({**atoms, USM_CONVERTER: converter_id(USM_CONVERTER)}.items())) + self._atoms_memo[k] = atoms return atoms def context(self, subject: str, env) -> dict: - doc = self.census(subject, env) - out = {} - refs = census_mod.script_refs(doc) - if refs: + u = self.unit(subject, env) + out: dict = {"unit": u.root, "bundle": u.bundle} + if u.scripts: table = self._docs.get(env.depot, env.input_of(SCRIPTS_TASK, "scripts").sha256)["scripts"] - out["scripts"] = link.script_context(table, refs) - types = _script_types(doc) - if types: - out["typetrees"] = types + scripts = link.script_context(table, u.scripts) + if scripts: + out["scripts"] = scripts + trees = {} + for version, h in u.trees: + nodes = scripttypes.script_types(version).get(h) + if nodes is not None: + trees[h] = nodes + if trees: + out["typetrees"] = dict(sorted(trees.items())) return out def depends(self, subject: str, env) -> list[str]: - return [contract.task_id(CENSUS, subject), SCRIPTS_TASK] + return [contract.task_id(CENSUS, self.unit(subject, env).bundle), SCRIPTS_TASK] def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: - """From the bundle's size, its census' object count and the size of its resource files (least squares over - a whole catalog: CPU seconds; peak resident bytes of a worker process, which grow with the objects).""" + """CPU: the unit's share of loading its bundle (from the bundle's size, among the requested units of the + bundle) and its own (its objects and their bytes); peak: the worker's for the whole bundle (from its size + and object count), which the tasks of its batch hold together.""" + u = self.unit(subject, env) + t = units.table(env) size = sum(i.size for i in inputs) - try: - doc = self.census(subject, env) - n = int(doc["objects"]) - res = sum(int(r.get("size") or 0) for r in doc.get("resources") or ()) - except Exception: - n = res = 0 - return Cost(0.12 + size * 3.3e-7 + n * 8.8e-5 + res * 6.6e-8, (80 << 20) + 80 * size + 17500 * n) + n = t.objects.get(u.bundle, 0) + load = 0.12 + size * 3.3e-7 + cpu = load / max(1, t.per_bundle[u.bundle]) + len(u.members) * 8.8e-5 + u.size * 6.6e-8 + return Cost(cpu, (80 << 20) + 80 * size + 17500 * n) + + def batch(self, task) -> str: + return task.input("bundle").sha256 + + def end_batch(self, batch: str) -> None: + if self._loaded is not None and self._loaded.sha == batch: + self._loaded = None def run(self, task, depot) -> Output: reader = self.reader if self.reader is not None else resolve("reader") _check_atoms(task, reader) - view = reader.fn(depot.input_bytes(task.input("bundle")), builtins.serialized_name(task.subject)) + inp = task.input("bundle") + if self._loaded is None or self._loaded.sha != inp.sha256: + self._loaded = None + name = builtins.serialized_name(task.context["bundle"]) + self._loaded = _Bundle(inp.sha256, reader.fn(depot.input_bytes(inp), name)) fns = {n: resolve(n).fn for n in task.atoms if n in ATOM_NAMES and n != "reader"} - arts, items = BundleRun(view, task, depot, fns).run() + arts, items = UnitRun(self._loaded, task, depot, fns).run() return Output(arts, items) -def _script_types(doc: dict) -> dict: - """{type hash: nodes} of the embedded type trees (nnnotes.unity.scripttypes, of each file's Unity version) of the - type hashes of the MonoBehaviours in the census' files that store no type trees ({} when it has none).""" - out = {} - for f in doc["files"]: - if f["typetree"]: - continue - types = scripttypes.script_types(f["unityVersion"] or "") - for o in f["objects"]: - h = o["typeHash"] if o["class"] == "MonoBehaviour" else None - if h in types: - out[h] = types[h] - return dict(sorted(out.items())) - - -def _census_classes(env, subject: str) -> dict: - """{class: objects} of a bundle's census ({} when its census has no result).""" - try: - return env.artifact(contract.task_id(CENSUS, subject), "census")["semantics"]["facts"]["classes"] - except (Pending, KeyError): - return {} - - def _store_input(role: str, record: dict) -> Input: c = record["content"] return Input(role, c["sha256"], c["size"], None, ({"kind": "depot"},)) +def _artifact(env, tid: str, aid: str) -> dict | None: + return next((a for a in env.result(tid)["artifacts"] if a["id"] == aid), None) + + class SpriteCropStage(Stage): - """sprite.crop: the images of the sprites whose SpriteAtlas is in another bundle. One task per atlas, subject - ":"; inputs: "atlas" (the atlas' JSON), "sprite:" (each sprite's meta), "texture:" (the image of each atlas texture the sprites use), all artifacts - of unity.export results. Waits for unity.export tasks that have not run. Artifacts "#image".""" + """sprite.crop: the images of the sprites whose SpriteAtlas is in another bundle. One task per atlas, subject the + unity.export subject of the atlas' unit; inputs: "atlas" (the atlas' JSON), "sprite:" (each sprite's + JSON), "texture:" (the image of each atlas texture the sprites use), all artifacts of unity.export + results. The sprites are those of the unity.export tasks of Sprite units whose JSON names an atlas of another + bundle; waits for unity.export tasks that have not run. Artifacts "#image".""" name = CROP version = 1 @@ -561,51 +542,40 @@ def normalize(self, params: dict | None) -> dict: raise ValueError(f"stage {self.name}: unknown parameters {', '.join(unknown)}") return {"png": _png_params(p.get("png"))} - @staticmethod - def _may_hold(env, tid: str) -> bool: - subject = contract.parse_task_id(tid, EXPORT)[1] - try: - classes = env.artifact(contract.task_id(CENSUS, subject), "census")["semantics"]["facts"]["classes"] - except Exception: - return True - return "Sprite" in classes or "SpriteAtlas" in classes - def index(self, env) -> dict: - """{subjects: {subject: (atlas object id, [(sprite object id, export task id, meta record)])}, atlases, - textures, missing: atlas object ids that sprites name but no result holds}.""" + """{subjects: {atlas subject: (atlas object id, [(sprite object id, export task id, its JSON record)])}, + missing: atlas object ids that sprites name but no unity.export result holds}.""" pending = env.pending(EXPORT) if pending: raise Pending(pending[0]) - tids = env.tasks(EXPORT) - sig = tuple(env.key(t) for t in tids) + table = units.table(env) + tids = [] + for t in env.tasks(EXPORT): + u = table.units.get(contract.parse_task_id(t, EXPORT)[1]) + if u is not None and u.cls == "Sprite": + tids.append(t) + sig = (id(table), tuple(env.key(t) for t in tids)) if self._memo is not None and self._memo[0] == sig: return self._memo[1] - atlases, textures, requests = {}, {}, defaultdict(list) + requests = defaultdict(list) for tid in tids: - if not self._may_hold(env, tid): - continue for a in env.result(tid)["artifacts"]: - owner, role = contract.parse_artifact_id(a["id"]) - sem, obj = a["semantics"], a["provenance"].get("object") or {} - facts = sem.get("facts") or {} - if sem["kind"] == "sprite.meta" and facts.get("crop") == CROP: - requests[facts["atlas"]].append((owner, tid, a)) - elif role == "json" and obj.get("class") == "SpriteAtlas": - atlases[owner] = (tid, a) - elif role == "image" and sem["kind"] == "texture.image": - textures[owner] = (tid, a) - subjects = {} - for atlas, reqs in requests.items(): - if atlas in atlases: - tid = atlases[atlas][0] - subject = f"{contract.parse_task_id(tid, EXPORT)[1]}:{contract.parse_object_id(atlas)[1]}" - subjects[subject] = (atlas, sorted(reqs, key=lambda r: r[0])) - idx = { - "subjects": subjects, - "atlases": atlases, - "textures": textures, - "missing": sorted(set(requests) - set(atlases)), - } + facts = a["semantics"].get("facts") or {} + if a["semantics"]["kind"] == "sprite.meta" and facts.get("crop") == CROP: + requests[facts["atlas"]].append((contract.parse_artifact_id(a["id"])[0], tid, a)) + subjects, missing = {}, [] + for atlas, reqs in sorted(requests.items()): + subject = table.subject_of(atlas) + if subject is None or not env.known(contract.task_id(EXPORT, subject)): + missing.append(atlas) + continue + try: + env.key(contract.task_id(EXPORT, subject)) + except Pending: + missing.append(atlas) + continue + subjects[subject] = (atlas, sorted(reqs, key=lambda r: r[0])) + idx = {"subjects": subjects, "missing": missing} self._memo = (sig, idx) return idx @@ -616,31 +586,53 @@ def _subject(self, subject: str, env): idx = self.index(env) if subject not in idx["subjects"]: raise ValueError(f"{self.name}: no sprites wait for atlas {subject}") - return idx, *idx["subjects"][subject] - - def inputs(self, subject: str, env) -> list[Input]: - idx, atlas, reqs = self._subject(subject, env) - rec = idx["atlases"][atlas][1] - ins = [_store_input("atlas", rec)] + return idx["subjects"][subject] + + def _textures(self, subject: str, env) -> tuple[dict, list]: + """(the atlas' JSON record, [(texture object id, the export task holding it)]) of the textures the + sprites use.""" + atlas, reqs = self._subject(subject, env) + rec = _artifact(env, contract.task_id(EXPORT, subject), contract.artifact_id(atlas, "json")) + if rec is None: + raise ValueError(f"{self.name} {subject}: the atlas {atlas} has no JSON") entries = textures.atlas_entries(self._docs.get(env.depot, rec["content"]["sha256"])) needed = set() - for owner, _tid, meta in reqs: - ins.append(_store_input("sprite:" + owner, meta)) + for _owner, _tid, meta in reqs: doc = self._docs.get(env.depot, meta["content"]["sha256"]) entry = textures.atlas_entry(entries, doc["sprite"].get("m_RenderDataKey")) if entry is not None: needed.update( t for t in (ref_object_id(entry.get("texture")), ref_object_id(entry.get("alphaTexture"))) if t ) - ins += [_store_input("texture:" + t, idx["textures"][t][1]) for t in sorted(needed) if t in idx["textures"]] + table = units.table(env) + out = [] + for t in sorted(needed): + s = table.subject_of(t) + if s is not None and env.known(contract.task_id(EXPORT, s)): + out.append((t, contract.task_id(EXPORT, s))) + return rec, out + + def inputs(self, subject: str, env) -> list[Input]: + _atlas, reqs = self._subject(subject, env) + rec, texs = self._textures(subject, env) + ins = [_store_input("atlas", rec)] + ins += [_store_input("sprite:" + owner, meta) for owner, _tid, meta in reqs] + for t, tid in texs: + try: + img = _artifact(env, tid, contract.artifact_id(t, "image")) + except Pending: + continue + if img is not None and img["semantics"]["kind"] == "texture.image": + ins.append(_store_input("texture:" + t, img)) return ins def uses(self, subject: str, env) -> dict: return dict(self.ATOMS) def depends(self, subject: str, env) -> list[str]: - idx, atlas, reqs = self._subject(subject, env) - return sorted({idx["atlases"][atlas][0], *(tid for _o, tid, _m in reqs)}) + _atlas, reqs = self._subject(subject, env) + _rec, texs = self._textures(subject, env) + return sorted({contract.task_id(EXPORT, subject), *(tid for _o, tid, _m in reqs), *(tid for _t, tid in texs)}) def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: tex = sum(i.size for i in inputs if i.role.startswith("texture:")) diff --git a/src/nnnotes/pipeline/units.py b/src/nnnotes/pipeline/units.py new file mode 100644 index 0000000..bfce293 --- /dev/null +++ b/src/nnnotes/pipeline/units.py @@ -0,0 +1,198 @@ +"""The units of unity.export, from the censuses: what one export task converts, and which of them a request needs. + +A unit is a hierarchy (a root GameObject with the GameObjects, Transforms and components its prefab holds: the +objects whose census record names it as their `unit`) or an object outside any hierarchy. Its subject is the +path-id stable address of its root object (link.path_address: ":", with the file after the bundle in +a bundle of several serialized files), which stays when the bundle's content changes; its object ids do not. + +The request (`table`): every unit holding an object of the selected bundles (fact "selected"), and every unit +reachable from those through what their objects reference (census `refs`) across the censuses of the bundles of +the fact "bundles"; with the fact "classes" (a list of class names; absent or None: every class), only the units +whose root object is of those classes. A bundle whose census failed has no units; a reference into a file no +census holds is left out. +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass + +from . import contract, link +from .census import CensusStage +from .stages import Pending + +CENSUS = CensusStage.name + + +@dataclass(frozen=True) +class Unit: + """One unit: its subject, stable bundle name, root object id and class, member object ids (the root first), + ((class, objects)) of its members by class, the MonoScripts they name (m_Script and an AnimationClip's binding + scripts), the (Unity version, type hash) of its MonoBehaviours in files without type trees, and the byte size of + its members.""" + + subject: str + bundle: str + root: str + cls: str + members: tuple + counts: tuple + scripts: tuple + trees: tuple + size: int + + @property + def classes(self) -> tuple: + return tuple(c for c, _ in self.counts) + + +class Units: + """The units of a request: `units` {subject: Unit} of the requested units; `of` {object id: subject} of the + objects of every unit of the censuses read; `files` {serialized file: stable bundle name}; `objects` {stable + bundle name: its objects}; `per_bundle` the number of requested units per bundle; `classes` the objects of the + requested units per class.""" + + def __init__(self, units: dict, of: dict, files: dict, objects: dict): + self.units, self.of, self.files, self.objects = units, of, files, objects + self.per_bundle = Counter(u.bundle for u in units.values()) + self.classes: Counter = Counter() + for u in units.values(): + for c, n in u.counts: + self.classes[c] += n + + def subjects(self) -> list[str]: + return sorted(self.units) + + def unit(self, subject: str) -> Unit: + try: + return self.units[subject] + except KeyError: + raise ValueError(f"unity.export: no requested unit {subject}") from None + + def subject_of(self, oid: str) -> str | None: + """The subject of the unit holding object `oid` (None when no census read holds it).""" + return self.of.get(oid) + + +def _ref(r: dict | None) -> str | None: + """The object id of a census reference record (None for null or an unknown file index).""" + if not r or "file" not in r: + return None + return contract.object_id(r["file"], r["pathId"]) + + +def _census_units(stable: str, doc: dict, need_refs: bool): + """(units, {object id: subject}, {file: stable}, {subject: referenced object ids}) of one census.""" + tags = link.file_tags([f["name"] for f in doc["files"]]) + groups: dict[str, list] = {} # root object id -> [(file record, object record)], the root first + for f in doc["files"]: + for o in f["objects"]: + oid = contract.object_id(f["name"], o["pathId"]) + root = o.get("unit", oid) + entry = (f, o) + if root == oid: + groups.setdefault(root, []).insert(0, entry) + else: + groups.setdefault(root, []).append(entry) + units, of, refs = {}, {}, {} + for root, members in groups.items(): + rf, ro = members[0] + if contract.object_id(rf["name"], ro["pathId"]) != root: + continue # members of a root the census does not list + subject = link.path_address(stable, tags, rf["name"], ro["pathId"]) + ids, classes, scripts, trees, size, out = [], Counter(), set(), set(), 0, set() + for f, o in members: + oid = contract.object_id(f["name"], o["pathId"]) + ids.append(oid) + of[oid] = subject + classes[o["class"]] += 1 + size += int(o.get("byteSize") or 0) + for s in [o.get("script"), *(o.get("bindingScripts") or ())]: + r = _ref(s) + if r is not None: + scripts.add(r) + if o["class"] == "MonoBehaviour" and not f["typetree"] and o.get("typeHash"): + trees.add((f["unityVersion"] or "", o["typeHash"])) + if need_refs: + out.update(o.get("refs") or ()) + units[subject] = Unit( + subject, + stable, + root, + ro["class"], + tuple(ids), + tuple(sorted(classes.items())), + tuple(sorted(scripts)), + tuple(sorted(trees)), + size, + ) + if need_refs: + refs[subject] = out.difference(ids) + return units, of, {f["name"]: stable for f in doc["files"]}, refs + + +def build(censuses, bundles, selected, classes=None) -> Units: + """The Units of a request over `censuses` [(stable bundle name, census document)] of the bundles `bundles` + (module documentation).""" + selected = set(selected) + every = selected >= set(bundles) # no reference can add a unit: none is read + units: dict[str, Unit] = {} + of: dict[str, str] = {} + files: dict[str, str] = {} + refs: dict[str, set] = {} + objects: dict[str, int] = {} + for stable, doc in censuses: + objects[stable] = int(doc["objects"]) + u, o, f, r = _census_units(stable, doc, need_refs=not every) + units.update(u) + of.update(o) + files.update(f) + refs.update(r) + if every: + want = set(units) + else: + want = {s for s, u in units.items() if u.bundle in selected} + todo = list(want) + while todo: + for oid in refs.get(todo.pop(), ()): + s = of.get(oid) + if s is not None and s not in want: + want.add(s) + todo.append(s) + if classes is not None: + keep = set(classes) + want = {s for s in want if units[s].cls in keep} + return Units({s: units[s] for s in sorted(want)}, of, files, objects) + + +_MEMO: list = [] # [(signature, Units)] of the last table + + +def table(env) -> Units: + """The Units of the request of a planning environment: the censuses of the fact "bundles" (Pending while one + has not run; a failed one is skipped), the facts "selected" and "classes". Kept for the planning state it was + made in.""" + bundles = sorted(env.fact("bundles")) + keys = [] + for b in bundles: + tid = contract.task_id(CENSUS, b) + try: + keys.append((b, env.key(tid))) + except Pending as p: + if not p.failed: + raise + classes = env.facts.get("classes") + sig = (id(env.depot), tuple(keys), tuple(env.fact("selected")), None if classes is None else tuple(classes)) + if _MEMO and _MEMO[0][0] == sig: + return _MEMO[0][1] + + def censuses(): + for b, key in keys: + tid = contract.task_id(CENSUS, b) + aid = contract.artifact_id(tid, "census") + rec = next(a for a in env.depot.committed(key, tid)["artifacts"] if a["id"] == aid) + yield b, contract.loads(env.depot.read(rec["content"]["sha256"])) + + t = build(censuses(), bundles, env.fact("selected"), classes) + _MEMO[:] = [(sig, t)] + return t diff --git a/src/nnnotes/pipeline/voices.py b/src/nnnotes/pipeline/voices.py index 52eefa9..147b060 100644 --- a/src/nnnotes/pipeline/voices.py +++ b/src/nnnotes/pipeline/voices.py @@ -62,7 +62,7 @@ from ..content import advcommand, storyvoices from ..formats import acb from ..source import languages -from . import contract, views +from . import contract, units, views from .contract import Cost, Input from .stages import Output, Pending, Stage @@ -820,16 +820,15 @@ def _sheet_name(name: str) -> str | None: return name if "/" not in name else None -_address_cache: dict = {} # content id of an address table -> (AddressIndex, files), the latest one +_address_cache: dict = {} # content id of an address table -> its AddressIndex, the latest one -def _address_parts(env, tid: str) -> tuple: - """(AddressIndex, {serialized file: stable bundle name}) of the result of the link.addresses task `tid`.""" +def _address_index(env, tid: str): + """The AddressIndex of the result of the link.addresses task `tid`.""" sha = env.artifact(tid, "addresses")["content"]["sha256"] hit = _address_cache.get(sha) if hit is None: - doc = contract.loads(env.depot.read(sha)) - hit = (views.AddressIndex.from_addresses(doc), dict(doc.get("files") or {})) + hit = views.AddressIndex.from_addresses(contract.loads(env.depot.read(sha))) _address_cache.clear() _address_cache[sha] = hit return hit @@ -930,7 +929,7 @@ def _plan(self, subject: str, env) -> tuple: ) if self._memo is not None and self._memo[0] == sig: return self._memo[1] - index, files = _address_parts(env, link) + index = _address_index(env, link) tables = { i.role[len(views.MASTER_ROLE) :]: views._cached_rows(i.sha256, Path(i.locators[0]["path"]).read_bytes) for i in masters @@ -945,15 +944,18 @@ def _plan(self, subject: str, env) -> tuple: } sheets = {s: v for s, v in found.items() if s in wanted} used = {v["acb"] for v in sheets.values()} - episodes, contents, read = self._episodes(env, rec, tables, files, set(exports)) + episodes, contents, read = self._episodes(env, rec, tables, set(exports)) plan = (masters, rec.subset(), sheets, [acbs[a] for a in sorted(acbs) if a in used], episodes, contents, read) self._memo = (sig, plan) return plan - def _episodes(self, env, index, tables, files: dict, exports: set) -> tuple[dict, list[Input], list[str]]: + def _episodes(self, env, index, tables, exports: set) -> tuple[dict, list[Input], list[str]]: """The story parts the unity.export results hold: the object a part's key names (the first of its class, - STORY_OBJECTS) in the bundle of its serialized file (the address table's `files`), and the artifact with - its content. ({asset: {part: {object, artifact, sha256}}}, the content inputs, the tasks read.)""" + STORY_OBJECTS) in the task of its unit (pipeline.units), and the artifact with its content. ({asset: + {part: {object, artifact, sha256}}}, the content inputs, the tasks read.)""" + if not exports: + return {}, [], [] + table = units.table(env) arts: dict[str, dict] = {} out: dict = {} contents: dict[str, Input] = {} @@ -965,8 +967,8 @@ def _episodes(self, env, index, tables, files: dict, exports: set) -> tuple[dict if not pick: continue oid = pick[0].id - stable = files.get(contract.parse_object_id(oid)[0]) - tid = contract.task_id(EXPORT_STAGE, stable) if stable is not None else None + subject = table.subject_of(oid) + tid = contract.task_id(EXPORT_STAGE, subject) if subject is not None else None if tid is None or tid not in exports: continue if tid not in arts: diff --git a/src/nnnotes/schemas/artifacts.schema.json b/src/nnnotes/schemas/artifacts.schema.json deleted file mode 100644 index c8ce2d3..0000000 --- a/src/nnnotes/schemas/artifacts.schema.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:nnnotes:schema:artifacts", - "title": "nnnotes.artifacts/1", - "description": "The artifact table (stage link.artifacts): the artifact ids of every object with artifacts, the artifact each contained object is part of, and per artifact its content id, size and extension. With a layout manifest it gives every object's files.", - "type": "object", - "required": [ - "schema", - "objects", - "contained", - "artifacts" - ], - "properties": { - "schema": { - "const": "nnnotes.artifacts/1" - }, - "objects": { - "type": "object", - "additionalProperties": { - "type": "array", - "items": { - "$ref": "urn:nnnotes:schema:common#/$defs/artifactId" - }, - "minItems": 1 - } - }, - "contained": { - "type": "object", - "additionalProperties": { - "$ref": "urn:nnnotes:schema:common#/$defs/artifactId" - } - }, - "artifacts": { - "type": "object", - "additionalProperties": { - "type": "object", - "required": [ - "sha256", - "size", - "ext" - ], - "properties": { - "sha256": { - "$ref": "urn:nnnotes:schema:common#/$defs/sha256" - }, - "size": { - "type": "integer", - "minimum": 0 - }, - "ext": { - "type": "string" - } - } - } - } - } -} diff --git a/src/nnnotes/unity/convert/__init__.py b/src/nnnotes/unity/convert/__init__.py index ce85c0c..e0cb8f7 100644 --- a/src/nnnotes/unity/convert/__init__.py +++ b/src/nnnotes/unity/convert/__init__.py @@ -2,8 +2,8 @@ A converter reads one bundle alone through its `BundleObjects` (objects.py) and writes its artifacts through the `emit` function the caller gives it; the caller (the unity.export stage) keeps the items, the depot and the -provenance. Each converter carries a version (CONVERTERS); a task's key includes the converters of the classes its -bundle holds, with the atoms they use (CONVERTER_ATOMS). +provenance. Each converter carries a version (CONVERTERS); a task's key includes the converters of the classes of the +objects it converts, with the atoms they use (CONVERTER_ATOMS). Texture2D tex.png `image`: PNG of the first mip level (HDR ASTC: the .astc file) Sprite sprite.png `image`: PNG on the sprite's rect; `json`: typetree, geometry, render data @@ -21,9 +21,7 @@ also `usm`: its USM (kind cri.usm) TextAsset data `data`: the stored bytes Font font `font`: the font file - MonoScript scripts.json "#scripts.json": every MonoScript of the bundle - AssetBundle bundle.json "#bundle.json": serialized files, externals, AssetBundle objects - any other class generic.json `json`: the object's typetree + any other class generic.json `json`: the object's typetree (a MonoScript, an AssetBundle, ...) Every object a converter takes has a `json` artifact (or is contained in one): the converter's document, or the typetree JSON (a Texture2D, Mesh, TextAsset or Font without the bytes its other artifact holds). @@ -77,8 +75,6 @@ "mono.json": 2, "data": 2, "font": 2, - "scripts.json": 1, - "bundle.json": 1, "shader.json": 1, "generic.json": 2, "cri.usm": 1, @@ -96,8 +92,6 @@ "MonoBehaviour": ("mono.json",), "TextAsset": ("data",), "Font": ("font",), - "MonoScript": ("scripts.json",), - "AssetBundle": ("bundle.json",), "Shader": ("shader.json",), } ALWAYS = ("generic.json",) @@ -134,7 +128,7 @@ def converter_id(name: str) -> str: def converters_for(classes) -> list[str]: - """The converters a bundle holding objects of `classes` may use.""" + """The converters objects of `classes` may use.""" out: set[str] = set(ALWAYS) for c in classes: out.update(CLASS_CONVERTERS.get(c, ())) @@ -142,7 +136,7 @@ def converters_for(classes) -> list[str]: def export_atoms(classes, reader_id: str | None = None) -> dict: - """The atoms and converters a bundle holding `classes` uses (the `atoms` of its unity.export task).""" + """The atoms and converters objects of `classes` use (the `atoms` of a unity.export task).""" convs = converters_for(classes) names = set(BASE_ATOMS) for c in convs: diff --git a/src/nnnotes/unity/convert/generic.py b/src/nnnotes/unity/convert/generic.py index c69e4b2..dfa9715 100644 --- a/src/nnnotes/unity/convert/generic.py +++ b/src/nnnotes/unity/convert/generic.py @@ -1,14 +1,10 @@ -"""Objects without a converter of their own (generic.json: the typetree JSON), and the bundle's AssetBundle objects -(bundle.json) and MonoScripts (scripts.json), which go into one document each.""" +"""Objects without a converter of their own (generic.json: the typetree JSON).""" from __future__ import annotations from .objects import BundleObjects, encode_object -__all__ = ["LOOSE", "generic", "loose_doc"] - -# converter of the objects that go into one document -> (class, kind of the document) -LOOSE = {"bundle.json": ("AssetBundle", "unity.bundle"), "scripts.json": ("MonoScript", "unity.scripts")} +__all__ = ["generic"] def generic(ctx: BundleObjects, info) -> str | None: @@ -18,15 +14,3 @@ def generic(ctx: BundleObjects, info) -> str | None: doc = ctx.tree(ctx.readers[key], ctx.tt(key, whole=False)) ctx.emit(key, "json", encode_object(doc), "json", "unity.object", "json", {}) return ctx.partial.get(key) - - -def loose_doc(ctx: BundleObjects, conv: str, entries: dict) -> dict: - """The document of converter `conv` (LOOSE) for `entries` ({object id: typetree JSON}): bundle.json also lists - the bundle's serialized files with their Unity versions and externals.""" - if conv == "bundle.json": - files = [ - {"name": f, "unityVersion": ctx.versions.get(f), "externals": list(ctx.view.externals(f))} - for f in ctx.files - ] - return {"files": files, "assetBundles": entries} - return {"scripts": entries} diff --git a/src/nnnotes/unity/convert/objects.py b/src/nnnotes/unity/convert/objects.py index d6383cf..ec167a7 100644 --- a/src/nnnotes/unity/convert/objects.py +++ b/src/nnnotes/unity/convert/objects.py @@ -34,6 +34,7 @@ "NO_SCRIPT", "PREFIXES", "TYPETREE", + "BundleIndex", "BundleObjects", "Capture", "ReadError", @@ -188,11 +189,37 @@ class Capture: facts: dict = field(default_factory=dict) +class BundleIndex: + """What the converters read of a bundle that depends on its bytes alone (`view`, a reader atom's BundleView): + its objects and their readers, files, externals, Unity versions and container paths, with memos of names and + type tree nodes. The BundleObjects of the tasks of one bundle share it.""" + + def __init__(self, view): + self.view = view + self.infos = view.objects() + self.info = {(i.file, i.path_id): i for i in self.infos} + self.readers = {k: view.reader(i) for k, i in self.info.items()} + self.files = view.files() + self.ext_files = {f: [_file_name(p) for p in view.externals(f)] for f in self.files} + self.versions = {f: view.unity_version(f) for f in self.files} + self.stripped = frozenset(f for f in self.files if not view.stores_typetrees(f)) + self.containers: dict[tuple, str] = {} + for path, file, pid in view.container(): + k = (file, int(pid)) + if k not in self.containers or path < self.containers[k]: + self.containers[k] = path + self.names: dict[tuple, str | None] = {} + self.special_memo: dict[int, tuple] = {} + self.children_memo: dict[int, tuple] = {} + self.heads: dict[int, tuple] = {} + + class BundleObjects: - """The objects of one bundle (`view`, a reader atom's BundleView). `fns`: the atom functions by name; `scripts`: - the script table entries the bundle's objects use ({MonoScript object id: "assembly|namespace|class"}); - `typetrees`: the type trees of script classes for the files that store none ({type hash: nodes}); `blob_min`: - byte arrays of this size or more are blobs; `png_level`: the zlib level of PNG files.""" + """The objects of one bundle (`view`, a reader atom's BundleView; `index`: its BundleIndex when one is shared). + `fns`: the atom functions by name; `scripts`: the script table entries the bundle's objects use ({MonoScript + object id: "assembly|namespace|class"}); `typetrees`: the type trees of script classes for the files that store + none ({type hash: nodes}); `blob_min`: byte arrays of this size or more are blobs; `png_level`: the zlib level of + PNG files.""" def __init__( self, @@ -204,30 +231,22 @@ def __init__( png_level: int, emit: Emit, typetrees: dict | None = None, + index: BundleIndex | None = None, ): + idx = index if index is not None else BundleIndex(view) + if idx.view is not view: + raise ValueError("the index is of another view") self.view, self.fns, self.emit = view, fns, emit self.scripts, self.blob_min, self.png_level = dict(scripts), int(blob_min), int(png_level) self.typetrees = dict(typetrees or {}) - self.infos = view.objects() - self.info = {(i.file, i.path_id): i for i in self.infos} - self.readers = {k: view.reader(i) for k, i in self.info.items()} - self.files = view.files() - self.ext_files = {f: [_file_name(p) for p in view.externals(f)] for f in self.files} - self.versions = {f: view.unity_version(f) for f in self.files} - self.stripped = frozenset(f for f in self.files if not view.stores_typetrees(f)) + self.infos, self.info, self.readers = idx.infos, idx.info, idx.readers + self.files, self.ext_files, self.versions, self.stripped = idx.files, idx.ext_files, idx.versions, idx.stripped + self.containers, self.names = idx.containers, idx.names + self._special_memo, self._children_memo, self._heads = idx.special_memo, idx.children_memo, idx.heads self.partial: dict[tuple, str] = {} # objects read in part -> why self._nodes: dict[tuple, object] = {} # objects read with a type tree of `typetrees` -> its node self._script_nodes: dict[str, object] = {} # type hash -> node - self.containers: dict[tuple, str] = {} - for path, file, pid in view.container(): - k = (file, int(pid)) - if k not in self.containers or path < self.containers[k]: - self.containers[k] = path - self.names: dict[tuple, str | None] = {} self._mono_scripts: dict[tuple, str | None] = {} - self._special_memo: dict[int, tuple] = {} - self._children_memo: dict[int, tuple] = {} - self._heads: dict[int, tuple] = {} self._captures: dict[str, Capture] = {} self._local: Callable[[tuple], dict | None] | None = None self.seen: set[str] = set() diff --git a/src/nnnotes/unity/convert/textures.py b/src/nnnotes/unity/convert/textures.py index ed05194..79763ba 100644 --- a/src/nnnotes/unity/convert/textures.py +++ b/src/nnnotes/unity/convert/textures.py @@ -1,7 +1,8 @@ """Texture2D (tex.png) and Sprite (sprite.png). -A texture's image is decoded once per bundle: `texture` (or `decode_into`, when the task leaves textures out) puts it -in a holder that the sprites cut from it read. Where a sprite's image comes from is its plan (`sprite_plan`). +A texture's image is decoded by `decode_into` into a holder, which `texture` and the sprites cut from the texture read, +so the caller decides how long a decoded image is kept. Where a sprite's image comes from is its plan +(`sprite_plan`). A sprite's `json` holds its typetree JSON (`sprite`), its decoded geometry (`mesh`: positions in the sprite's units, triangles per submesh, texture coordinates when stored; a texture coordinate of a position p is @@ -48,18 +49,14 @@ def _canon(v) -> str: return json.dumps(v, sort_keys=True, ensure_ascii=True) -def texture(ctx: BundleObjects, info, holder: dict) -> None: +def texture(ctx: BundleObjects, info, decoded: dict) -> None: """`image`: PNG of the first mip level (HDR ASTC: the .astc file); `json`: the typetree JSON without the pixels - (`image data`). The decoded image (or the error) goes into `holder` for the texture's sprites.""" + (`image data`). `decoded`: the texture's decoded image or the error, as decode_into gives them.""" from ..atoms.texture import ASTC_BLOCKS, format_name, is_hdr_astc key = (info.file, info.path_id) - try: - tt = ctx.tt(key) - inp = ctx.view.texture_input(info) - except Exception as e: - holder["error"] = e - raise + tt = ctx.tt(key) + inp = ctx.view.texture_input(info) fmt, w, h = int(inp.format), int(inp.width), int(inp.height) facts = { "width": w, @@ -69,19 +66,13 @@ def texture(ctx: BundleObjects, info, holder: dict) -> None: "exportedMips": [0], } if w and h and is_hdr_astc(fmt): - holder["error"] = Unsupported( - "unsupported.texture.format", f"{format_name(fmt)} (HDR ASTC; the texture is exported as .astc)" - ) data = ctx.fns["astc.container"](inp.data, w, h, ASTC_BLOCKS[fmt]) ctx.emit(key, "image", data, "astc", "texture.image", "astc", facts) _texture_json(ctx, key, tt) return - try: - img = ctx.fns["texture.decode"](inp) - except Exception as e: - holder["error"] = e - raise - holder["image"] = img + if "error" in decoded: + raise decoded["error"] + img = decoded["image"] facts["mode"] = img.mode data = ctx.fns["png.encode"](img, ctx.png_level) ctx.emit(key, "image", data, "png", "texture.image", "png", facts) @@ -94,7 +85,7 @@ def _texture_json(ctx: BundleObjects, key: tuple, tt: dict) -> None: def decode_into(ctx: BundleObjects, info, holder: dict) -> None: - """The texture's image for its sprites only (a task whose classes leave Texture2D out).""" + """The texture's decoded image into `holder` ("image"), or the error ("error"; HDR ASTC is not decoded).""" from ..atoms.texture import format_name, is_hdr_astc try: diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index 85388f5..9a1eca6 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -16,7 +16,7 @@ from nnnotes import cli from nnnotes.cli import assets, common from nnnotes.cli import catalog as catalog_cli -from nnnotes.pipeline import contract +from nnnotes.pipeline import contract, units from nnnotes.pipeline.census import CensusStage from nnnotes.pipeline.contract import Cost from nnnotes.pipeline.depot import Depot @@ -34,12 +34,14 @@ # ---------------------------------------------------------------- synthetic data -def obj(pid, cls, name=None, script=None): +def obj(pid, cls, name=None, script=None, unit=None): o = {"pathId": pid, "classId": 0, "class": cls, "byteSize": 10, "typeHash": None} if name is not None: o["name"] = name if script is not None: o["script"] = script + if unit is not None: + o["unit"] = unit return o @@ -82,7 +84,8 @@ def cdoc(file: str, objs: list, container=(), scripts=(), bundle_name=None) -> d def censuses(broken: bool = True) -> dict: """{bundle: census document}: two cards (a texture with a sub-sprite, a script-driven MonoBehaviour), the - shared bundle holding the MonoScript, a background bundle with an object that fails (`broken`).""" + shared bundle holding the MonoScript, a background bundle with a prefab (a GameObject and its Transform) and an + object that fails (`broken`).""" script = {"file": CABS["shared"], "pathId": 42} out = {} for n in ("card_1", "card_2"): @@ -113,7 +116,8 @@ def censuses(broken: bool = True) -> dict: ) out["bg_x"] = cdoc( CABS["bg_x"], - [obj(1, "AssetBundle", "ab"), obj(3, "Mesh", "floor")] + ([obj(4, "Broken", "bad")] if broken else []), + [obj(1, "AssetBundle", "ab"), obj(2, "Transform", "", unit=f"{CABS['bg_x']}:3"), obj(3, "GameObject", "x")] + + ([obj(4, "Broken", "bad")] if broken else []), [("assets/game/bg/x.prefab", 3)], ) return out @@ -189,26 +193,34 @@ def run(self, task, depot): class FakeExport(Stage): - """unity.export: one JSON artifact per object of the census (MonoBehaviours name their script class from the - link.scripts table), the AssetBundle object contained in the bundle's first artifact, class `Broken` failed.""" + """unity.export: one task per unit (pipeline.units), one JSON artifact for its root object (a MonoBehaviour + names its script class from the link.scripts table) that contains its other members, class `Broken` failed, + class `Crash` a ConfigError.""" name = "unity.export" version = 1 after = ("unity.census", "link.scripts") - PARAMS = {"png": {"encoder": "pillow", "level": 6}, "classes": None} + PARAMS = {"png": {"encoder": "pillow", "level": 6}} ATOMS = {"png.encode": "fake/1"} def subjects(self, env): - return list(env.fact("selected")) + return units.table(env).subjects() - def _census(self, subject): - return contract.task_id("unity.census", subject) + def _census(self, subject, env): + return contract.task_id("unity.census", units.table(env).unit(subject).bundle) def depends(self, subject, env): - return [self._census(subject), "link.scripts:all"] + return [self._census(subject, env), "link.scripts:all"] def inputs(self, subject, env): - return [env.input_of(self._census(subject), "census"), env.input_of("link.scripts:all", "scripts")] + return [env.input_of(self._census(subject, env), "census"), env.input_of("link.scripts:all", "scripts")] + + def context(self, subject, env): + unit = units.table(env).unit(subject) + return {"unit": unit.root, "members": list(unit.members)} + + def batch(self, task): + return task.input("census").sha256 def estimate(self, subject, env, inputs): return Cost(0.01, 1 << 20) @@ -216,55 +228,31 @@ def estimate(self, subject, env, inputs): def run(self, task, depot): doc = contract.loads(depot.input_bytes(task.input("census"))) scripts = contract.loads(depot.input_bytes(task.input("scripts")))["scripts"] - if any(o["class"] == "Crash" for f in doc["files"] for o in f["objects"]): - raise ConfigError("setting paths.tool is not set") container = {(e["file"], e["pathId"]): e["path"] for b in doc["assetBundles"] for e in b["container"]} - only = task.params["classes"] - arts, items, first = [], [], None - for f in doc["files"]: - for o in f["objects"]: - oid = contract.object_id(f["name"], o["pathId"]) - if only is not None and o["class"] not in only: - continue - if o["class"] == "AssetBundle": - continue - if o["class"] == "Broken": - items.append( - contract.item(oid, "failed", why=contract.reason("fake.broken", "broken"), cls=o["class"]) - ) - continue - body = {"object": oid, "class": o["class"], "level": task.params["png"]["level"]} - if o.get("script"): - body["script"] = scripts.get(contract.object_id(o["script"]["file"], o["script"]["pathId"])) - facts = { - "file": f["name"], - "pathId": o["pathId"], - "classId": 0, - "class": o["class"], - "name": o.get("name", ""), - } - if (f["name"], o["pathId"]) in container: - facts["container"] = container[(f["name"], o["pathId"])] - aid = contract.artifact_id(oid, "json") - arts.append( - contract.artifact( - aid, - depot.add(contract.encode(body), "json"), - contract.provenance(task, obj=facts), - {"kind": "fake.json"}, - ) - ) - items.append(contract.item(oid, "exported", artifacts=[aid], cls=o["class"])) - first = first or aid - for f in doc["files"]: - for o in f["objects"]: - if o["class"] == "AssetBundle" and first and (only is None or "AssetBundle" in only): - items.append( - contract.item( - contract.object_id(f["name"], o["pathId"]), "contained", within=first, cls="AssetBundle" - ) - ) - return Output(arts, items) + file, pid = contract.parse_object_id(task.context["unit"]) + (f,) = [f for f in doc["files"] if f["name"] == file] + (o,) = [o for o in f["objects"] if o["pathId"] == pid] + oid = task.context["unit"] + if o["class"] == "Crash": + raise ConfigError("setting paths.tool is not set") + if o["class"] == "Broken": + return Output( + [], [contract.item(oid, "failed", why=contract.reason("fake.broken", "broken"), cls="Broken")] + ) + body = {"object": oid, "class": o["class"], "level": task.params["png"]["level"]} + if o.get("script"): + body["script"] = scripts.get(contract.object_id(o["script"]["file"], o["script"]["pathId"])) + facts = {"file": f["name"], "pathId": o["pathId"], "classId": 0, "class": o["class"], "name": o.get("name", "")} + if (f["name"], o["pathId"]) in container: + facts["container"] = container[(f["name"], o["pathId"])] + aid = contract.artifact_id(oid, "json") + art = contract.artifact( + aid, depot.add(contract.encode(body), "json"), contract.provenance(task, obj=facts), {"kind": "fake.json"} + ) + items = [contract.item(oid, "exported", artifacts=[aid], cls=o["class"])] + cls = {contract.object_id(g["name"], m["pathId"]): m["class"] for g in doc["files"] for m in g["objects"]} + items += [contract.item(m, "contained", within=aid, cls=cls[m]) for m in task.context["members"][1:]] + return Output([art], items) REAL = {s.name: s for s in assets.STAGE_SOURCES} @@ -273,8 +261,7 @@ def run(self, task, depot): assets.StageSource("unity.census", "tests.cli.test_assets:FakeCensus"), REAL["link.scripts"], REAL["link.addresses"], - assets.StageSource("unity.export", "tests.cli.test_assets:FakeExport", output=True, root="_bundles/{subject}"), - REAL["link.artifacts"], + assets.StageSource("unity.export", "tests.cli.test_assets:FakeExport", output=True), ) @@ -370,7 +357,7 @@ def test_the_built_in_files_of_the_apk_are_exported(tmp_path, capsys, fake, monk base = [*flags(d), "--apk", apk, "depot", "plan", "--depot", tmp_path / "s", "--json"] code, out, err = nn(capsys, *base, "--select", "group:bg") ids = {n["id"] for n in json.loads(out)["nodes"]} - assert "unity.census:apk/data.unity3d" in ids and "unity.export:apk/data.unity3d" not in ids + assert "unity.census:apk/data.unity3d" in ids and not any(i.startswith("unity.export:apk/") for i in ids) code, out, err = nn( capsys, *flags(d), @@ -409,7 +396,7 @@ def test_export_writes_layouts_reports_and_hits_the_second_time(tmp_path, capsys code, out, err = export(capsys, tmp_path, d, "depot", "out") assert code == 1, err # the broken object is a failed item summary = json.loads(out) - assert summary["items"] == {"exported": 8, "contained": 4, "generic": 0, "unsupported": 0, "failed": 1} + assert summary["items"] == {"exported": 12, "contained": 1, "generic": 0, "unsupported": 0, "failed": 1} files = tree(tmp_path / "out") assert "Assets/Game/Card/1/full.png.json" in files # the catalog's case of the container path assert "Assets/Game/Card/1/full[card1].png.json" in files # the sub-sprite @@ -422,15 +409,16 @@ def test_export_writes_layouts_reports_and_hits_the_second_time(tmp_path, capsys assert [(f["code"], f["object"]) for f in fails] == [("fake.broken", f"{CABS['bg_x']}:4")] cov = contract.loads((reports / "coverage.json").read_bytes())["classes"] assert cov["Texture2D"]["total"] == 2 and cov["Texture2D"]["exported"] == 2 - assert cov["AssetBundle"] == { - "total": 4, + assert cov["Transform"] == { + "total": 1, "missing": 0, "exported": 0, - "contained": 4, + "contained": 1, "generic": 0, "unsupported": 0, "failed": 0, } + assert cov["AssetBundle"]["total"] == cov["AssetBundle"]["exported"] == 4 run = contract.loads((reports / "run.json").read_bytes()) assert Depot(tmp_path / "depot").run(run["id"]) == run assert run["context"]["catalog"]["sha"] == contract.sha256(d["catalog"].read_bytes()) @@ -518,7 +506,7 @@ def test_plan_emit_tasks_and_run_stage_give_the_exported_depot(tmp_path, capsys, code, out, err = nn(capsys, *flags(d), "depot", "run", tasks, "--depot", tmp_path / "q") assert code in (0, 1), err rounds += 1 - assert rounds == 4 # index + census; scripts + addresses; export; artifacts + assert rounds == 3 # index + census; scripts + addresses; export skip = ["inputs", "runs", "costs", "catalogs", "downloads"] assert tree(tmp_path / "orch", skip) == tree(tmp_path / "q", skip) @@ -857,7 +845,8 @@ def test_derived_documents_read_the_paths_of_the_objects_they_name(tmp_path, cap card = f"{CABS['card_1']}:5" assert doc["CAB-none:9"] == {} and doc[f"{CABS['bg_x']}:4"] == {} # unknown; failed assert doc[card] == {f"{card}#json": "Assets/Game/Card/1/full.png.json"} - assert doc[f"{CABS['card_1']}:1"] == doc[card] # contained in it + prefab = f"{CABS['bg_x']}:3" + assert doc[f"{CABS['bg_x']}:2"] == doc[prefab] == {f"{prefab}#json": "Assets/Game/Bg/x.prefab.json"} # contained assert (tmp_path / "o" / "Assets" / "Game" / "Card" / "1" / "full.png.json").is_file() code, out, _ = nn(capsys, *flags(d), "depot", "plan", "-o", tmp_path / "o", "--depot", tmp_path / "s", "--check") assert code == 0, out @@ -928,7 +917,6 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps contract.VIEW, "nnnotes.scripts/1", "nnnotes.addresses/1", - "nnnotes.artifacts/1", }, checked code, out, _ = nn( capsys, @@ -1028,7 +1016,7 @@ def test_export_records_the_calibration_and_plan_uses_it(tmp_path, capsys, fake) d = setup_data(tmp_path) export(capsys, tmp_path, d, "s", "o") cal = tmp_path / "s" / "costs" / assets.CALIBRATION - assert contract.loads(cal.read_bytes()) == {"stages": {}} # four bundles: too few + assert contract.loads(cal.read_bytes())["stages"]["unity.export"]["samples"] == 13 # one per unit run assets.write_calibration(tmp_path / "s", {"unity.export": {"cpu": 2.0, "peak": 3.0, "samples": 50}}) code, out, err = nn( capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "s", "--json", "--png-level", "1", "--explain" @@ -1208,7 +1196,7 @@ def test_names_see_the_tasks_that_ran(tmp_path, capsys, monkeypatch): code, out, err = export(capsys, tmp_path, d, "s", "o") assert code == 0, err placed = tree(tmp_path / "o") - assert "Named/4/a.flac" in placed and "Named/0/a.flac" not in placed # four bundles were exported + assert "Named/12/a.flac" in placed and "Named/0/a.flac" not in placed # the twelve units were exported code, out, _ = nn(capsys, *flags(d), "depot", "plan", "-o", tmp_path / "o", "--depot", tmp_path / "s", "--check") assert code == 0, out # plan places them the same way diff --git a/tests/golden/unity.export.json b/tests/golden/unity.export.json index 434f76a..de4fcb2 100644 --- a/tests/golden/unity.export.json +++ b/tests/golden/unity.export.json @@ -1,7 +1,6 @@ { "atoms": { "astc.container": "nnnotes/1", - "bundle.json": "bundle.json/1", "clip.json": "clip.json/2", "controller.json": "controller.json/2", "cri.usm": "cri.usm/1", @@ -16,7 +15,6 @@ "png.encode": "pillow-12.3.0/1", "prefab.json": "prefab.json/2", "reader": "fakeunity/1", - "scripts.json": "scripts.json/1", "shader.json": "shader.json/1", "sniff": "nnnotes/1", "sprite.crop": "pillow-12.3.0+numpy-2.4.6/1", @@ -25,69 +23,57 @@ "texture.decode": "unitypy-1.25.3+texture2ddecoder-1.0.6+astc-encoder-py-0.1.12+pillow-12.3.0/1" }, "fixtures": { - "all": { - "key": "bfa791aaeaa8744197413dca21b95285355376a687dcb4a5b511410b4761bd28", - "result": "d9f811e5ca222529d22d1c5e2ec9cd89328d0229eac92b7e01d0cb59dbc56077" - }, - "bundle.json": { - "key": "019bd61d3b8989c2f59289354d72d3857ed37cab1d90316359e6702ebab8a515", - "result": "f1516133579ff990af232a7b6d8b6303104c59997d20905f16aa280403d3a09e" - }, "clip.json": { - "key": "a9f5fb2a87fbcb197a5bc37f1c520b7b5a4a6fd5b5ad38b1198e9f8601dc223b", - "result": "6448026702f2239970f8041906335b3a2302a4463536d2f0394daa36906d9c79" + "key": "c2211c68d3286636469c74023637e15b3c11dade0f858ab2380498aa319171eb", + "result": "1f6ad672b184cd8b4cf9eeeb9ee0b8512ba532eaf708cb36b86c538b19910b75" }, "controller.json": { - "key": "472e6426b3af184e4fb17ccc0a147bd7b6fc9e5fc49f32d0ab2c2d79238417e1", - "result": "c43f0d2de335f10063d20f2d208b54cf7da37863860441a22781bce35f0cb1cc" + "key": "eaae782443b5874108271db45e6a0da404f91239f700dd827802081ba64dd184", + "result": "e63d57f287d5d388e1d6d2269bc03ffbf03165b11cb03a19d2ad003fb00cacf8" }, "cri.usm": { - "key": "72cba01cb4fe8fa95d052a2fd17380eaacdd722dc0e5fc802bce8a301560e7a0", - "result": "9fecf00c65642c4ebb635839f79d9001cf4308b367711c2acaf82fa53326cff5" + "key": "c69fabc7e24dd038b2375b59a0421cd514a8ea5e5dc3a21c181dda809902dc3f", + "result": "d829f22ea9a736a478f2963470f2bdca4d6a3200e40a510606956fe771d63e00" }, "data": { - "key": "1b44691876cbf5e28032ac2858d9616a8445e2357c543291e2806506c8c0a0b4", - "result": "cdaf30aeb760eef66ec41e029940a800af4d72b0ca0c4545c9c911b6c94fd331" + "key": "eec1daf7f06a873c434b64b5ab2d0629b61a71c331f16ed5434706b60ee3bee3", + "result": "493c173754a5a0088bc972fc8c373415d02b345060ab83df9ce41f8b0d0317f8" }, "font": { - "key": "7d5549079bee9a0778d9d698545f21bfd0fea5466e0efa160661cfb527a83ea9", - "result": "d58abbc2a1b9990fadca993ac0502dc2d586b79e4947bfd1b6fb0a9f7bbc41b0" + "key": "81155f5c29ca33162b4b0ee97d7559fb0460ad02c46a187097bb6e4893792b7c", + "result": "d0da497a3f6207d7e92e31ac4313f289eb483b5fcf0496476a3fb50e235b830a" }, "generic.json": { - "key": "22deb0da7906ad444ca282fb2a8f993106268f748f2fab35357c049e5b020410", - "result": "a41f325a63c894bfe54aba013c1ee7aa6d674fddd49715c5e3898b7f40e6559c" + "key": "76af5e1c7fd2b389a80c4725aa2b561ffb0ae552b0d65ce2eb4ad56710ae3201", + "result": "6d6d058a293360ee2065fbb205f35834198e7285a743f2e5b1dc847daa4cdad3" }, "material.json": { - "key": "b8b07211d9dbc578b3541a0e59653b01506a211426cc70b6e0aa2e82a948167e", - "result": "09085e48baebbda4a78a076e5dc66eeb19ae1f53710ec9f9b3553bb86dfdefde" + "key": "c391d41f8afdc880c74e24f143560ff80c122c97f82241cd2b4f0cdf8f5fad1f", + "result": "0990d8b5ccdc04c6335b2779354b3d64727ebccc701490ffe6a46346719e90e2" }, "mesh.glb": { - "key": "1c9b9a873c782e16650ef40cc0a41c17d37da55e05da6d44f47c488cc12d5027", - "result": "58304a1fd371109dbc38f99cd578ad8238453e769ae8c9d8562e1e187e2df1bc" + "key": "1e5a4fee78d3fd98334ce50a6b0b1cc56f5733252f2cb00ff974d515a362b911", + "result": "11949923ee05d67811b42f530d84313e9197afd74e203da44291761b8aa87994" }, "mono.json": { - "key": "f2109d87be61d8cf5b0b527326d2cdc474785104091c55e770a18b31cd967b23", - "result": "02bc065f80513aad7e9afc21704d0f9591e0f110c9ff6fd818f357fe139686f8" + "key": "0aeed465e4e3f8a310b4a2bd3cc06fa899fcdd5fb0ae04d3a5c2a5ccd85d09ea", + "result": "9711bc6d921e7c49ef0b86b487808c12b95e106bb791655a880fef9257ec6da3" }, "prefab.json": { - "key": "583e77337b68385457bb1f7ec789c943d69729706adc18b47d59354bd04fcf94", - "result": "3751c24516ea904f342e272234e57357b4c5d03ef6e90637c5c8bffbbca78d2a" - }, - "scripts.json": { - "key": "6e6a37fb9135463601b78a8a5f699996221242585bcc31c63863158ca0a3f9dd", - "result": "9e9b03c721f1f163ed6e81c202b767a7af84be95e662d4d3b5a9b8d46bfe3392" + "key": "0c801dd911545b249a8b629b3dfa05c7ea6c466482d7f93252f126717419dc94", + "result": "96507449031bd9d046088f01b64602d7191aa87d7cf5af29ecde677c051d20fe" }, "shader.json": { - "key": "f3c2c279ab2ed59a70932b19111c4f248593ec3ca9e0677b9402c38cb8505d03", - "result": "dac4f3d4f225dd3659ac78ab0b344daaa64e1a0a6f7dfe759f3ab49ffe3b3c43" + "key": "105e05e73efa131a339c80a936f0e8dc44a2cebe3c25d332f500b51871dfe83e", + "result": "0f670ee8b5f4ceb584329ecd3a4695d70dd5bf0b9690b4df63b87963fb83913b" }, "sprite.png": { - "key": "d35fe9220f41f10d16b8506f48f31774e9303a8ce6e7b2deba3eb2b26edc56d9", - "result": "ebe1ea68a2fb3289366624724ec4a9335ff6e42156ce4003c7669106e3981482" + "key": "ab28c0f55b23cb58aaf5b94f4357231ab1f4be8ce16c6a02e69bfc1917a3509c", + "result": "82cf3a23fe661f24f5e9fc3189e2b70db39c287e39392397556e2b081beb4c2c" }, "tex.png": { - "key": "1408f5b984cbcc1c74d23e3c6ef518288f2d112b6b2e4007aecb7f811ddfc809", - "result": "fec1a912d37045add008ea5c12941ea9c6e9eeb0cc0adb6b4d5be81c89ac26eb" + "key": "f5b96a51a6de3073767d863f7fbd7f835df241386588ae3fddbfa7eae4d02c5a", + "result": "ea26cd3f966ff9ebb9c7491f72e720aa311d5552f5ef0b136d9712958dac6ac7" } }, "libraries": { @@ -99,5 +85,5 @@ }, "provider": "tests.pipeline.test_objexport:export_golden", "stage": "unity.export", - "version": 1 + "version": 2 } diff --git a/tests/helpers/units.py b/tests/helpers/units.py new file mode 100644 index 0000000..bf36993 --- /dev/null +++ b/tests/helpers/units.py @@ -0,0 +1,76 @@ +"""The unity.census and unity.export results the stages that read unity.export results plan from: a bundle's census +(what pipeline.units reads for the unit table) and the result of one unit's task. Synthetic: one serialized file per +bundle, every object its own unit.""" + +from __future__ import annotations + +from collections import Counter + +from nnnotes.pipeline import census, contract, link +from nnnotes.pipeline.contract import Input, Task + +CENSUS = "unity.census" +EXPORT = "unity.export" + + +def census_doc(file: str, objects) -> dict: + """The census of a bundle of one serialized file `file` holding `objects` [(pathId, class)].""" + objs = [ + {"pathId": pid, "classId": 0, "class": cls, "byteSize": 10, "typeHash": None} for pid, cls in sorted(objects) + ] + return { + "schema": contract.CENSUS, + "files": [ + { + "name": file, + "unityVersion": "6000.0.0f1", + "platform": 13, + "typetree": True, + "externals": [], + "objects": objs, + } + ], + "resources": [], + "assetBundles": [], + "scripts": [], + "classes": dict(sorted(Counter(o["class"] for o in objs).items())), + "objects": len(objs), + "errors": [], + } + + +def add_census(env, bundle: str, file: str, objects) -> None: + """Commits the census of `bundle` (census_doc) and adds the bundle to the facts "bundles" and "selected".""" + depot = env.depot + data = f"bundle {bundle}".encode() + inp = Input("bundle", depot.put(data), len(data), bundle, ({"kind": "depot"},)) + t = Task(CENSUS, 3, bundle, {}, {}, (inp,)) + doc = census_doc(file, objects) + rec = contract.artifact( + contract.artifact_id(t.id, "census"), + depot.add(contract.encode(doc), "json"), + contract.provenance(t), + {"kind": CENSUS, "format": "json", "facts": census.facts(doc)}, + ) + res = contract.result(t, [rec], []) + depot.commit(res) + env.done(t.id, res["key"]) + env.facts["bundles"] = {**env.facts.get("bundles", {}), bundle: inp} + env.facts["selected"] = sorted({*env.facts.get("selected", ()), bundle}) + + +def add_unit(env, bundle: str, file: str, pid: int, artifacts, cls: str = "MonoBehaviour") -> str: + """Commits the unity.export result of the unit of object `file:pid` of `bundle` (a bundle of add_census): + artifacts [(role, bytes, extension, semantics)] of that object, one exported item; -> the task id.""" + depot = env.depot + oid = contract.object_id(file, pid) + subject = link.path_address(bundle, link.file_tags([file]), file, pid) + t = Task(EXPORT, 2, subject, {}, {"unit": oid, "bundle": bundle}, ()) + recs = [ + contract.artifact(contract.artifact_id(oid, role), depot.add(data, ext), contract.provenance(t), semantics) + for role, data, ext, semantics in artifacts + ] + res = contract.result(t, recs, [contract.item(oid, "exported", artifacts=[r["id"] for r in recs], cls=cls)]) + depot.commit(res) + env.done(t.id, res["key"]) + return t.id diff --git a/tests/pipeline/test_cristages.py b/tests/pipeline/test_cristages.py index 97b24f5..ca861b6 100644 --- a/tests/pipeline/test_cristages.py +++ b/tests/pipeline/test_cristages.py @@ -21,6 +21,7 @@ from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry from tests.formats.test_usm import ADX, HCA, KEY, MPEG1, VP9, audio_header, build, chunk, rand, sbt, video_header +from tests.helpers import units as unit_results from tests.unity.test_speed_exact import FakeTools ROOT = Path(__file__).resolve().parents[2] @@ -123,31 +124,29 @@ def planning(depot, tmp, files: dict, keys=None, boot=True, bundles=()) -> Env: return Env(depot, facts) -def export_result(depot, env, subject: str, acbs: dict, usms=None) -> str: - """A committed unity.export result holding `cri.acb` artifacts {cue sheet: ACB bytes} and `cri.usm` artifacts - {movie asset name: USM bytes}.""" - t = Task(EXPORT, 1, subject, {}, {}, (Input("bundle", depot.put(subject.encode()), len(subject)),)) - recs = [ - contract.artifact( - f"CAB-{subject}:{i + 1}#acb", - depot.add(data, "acb"), - contract.provenance(t), +def export_result(env, bundle: str, acbs: dict, usms=None) -> dict: + """Committed unity.export results of `bundle`: a MonoBehaviour unit per `cri.acb` artifact {cue sheet: ACB bytes} + and per `cri.usm` artifact {movie asset name: USM bytes}; -> {sheet or movie name: task id}.""" + held = [ + ( + i + 1, + sheet, + data, + "acb", {"kind": "cri.acb", "format": "acb", "facts": {"cueSheet": sheet, "layout": "embedded"}}, ) for i, (sheet, data) in enumerate(sorted(acbs.items())) ] - recs += [ - contract.artifact( - f"CAB-{subject}:{100 + i}#usm", - depot.add(data, "usm"), - contract.provenance(t), - {"kind": "cri.usm", "format": "usm", "facts": {"movie": name}}, - ) + held += [ + (100 + i, name, data, "usm", {"kind": "cri.usm", "format": "usm", "facts": {"movie": name}}) for i, (name, data) in enumerate(sorted((usms or {}).items())) ] - depot.commit(contract.result(t, recs, [])) - env.done(t.id, t.key) - return t.id + file = f"CAB-{bundle}" + unit_results.add_census(env, bundle, file, [(pid, "MonoBehaviour") for pid, *_ in held]) + return { + name: unit_results.add_unit(env, bundle, file, pid, [(role, data, role, semantics)]) + for pid, name, data, role, semantics in held + } EXPORT = "unity.export" @@ -219,12 +218,13 @@ def test_audio_subjects_by_content(tmp_path, tools): {"cri_assets_cri/sound/bgm_x": ACB, "cri_assets_cri/video/m": usm_file()[0]}, {"Cri/Sound/bgm_x": ["cri_assets_cri/sound/bgm_x"], "Cri/Video/m": ["cri_assets_cri/video/m"]}, ) - tid = export_result(depot, env, "cri_bundle", {"bgm_x": ACB, "se_live": other}) + tids = export_result(env, "cri_bundle", {"bgm_x": ACB, "se_live": other}) + tid = tids["bgm_x"] stage = AudioStage() sha, other_sha = contract.sha256(ACB), contract.sha256(other) assert stage.subjects(env) == sorted([sha, other_sha]) # the raw ACB and its bundle copy: one assert stage.names(env) == {sha: ["Cri/Sound/bgm_x"], other_sha: ["se_live"]} - assert stage.depends(sha, env) == [tid] and stage.depends(other_sha, env) == [tid] + assert stage.depends(sha, env) == [tid] and stage.depends(other_sha, env) == [tids["se_live"]] t = describe(stage, sha, None, env) assert t.id == f"{AUDIO}:{sha}" and [i.role for i in t.inputs] == ["acb", "boot"] assert t.input("acb").locators[0]["kind"] == "file" and t.atoms == FAKE_ATOMS @@ -559,12 +559,13 @@ def test_movie_held_in_a_bundle(tmp_path, tools): }, bundles=("embcri_video_common",), ) - tid = export_result(depot, env, "embcri_video_common", {}, {"warmup": held, "loose": raw}) + tids = export_result(env, "embcri_video_common", {}, {"warmup": held, "loose": raw}) + tid = tids["warmup"] stage = MovieStage() sha, raw_sha = contract.sha256(held), contract.sha256(raw) assert stage.subjects(env) == sorted([sha, raw_sha]) assert stage.names(env) == {sha: ["EmbCri/Video/common/warmup"], raw_sha: ["Cri/Video/m", "loose"]} - assert stage.depends(sha, env) == [tid] and stage.depends(raw_sha, env) == [tid] + assert stage.depends(sha, env) == [tid] and stage.depends(raw_sha, env) == [tids["loose"]] with fake_tools(key=KEY): t = describe(stage, sha, None, env) assert t.input("usm").locators == ({"kind": "depot"},) and sorted(i.role for i in t.inputs) == ["boot", "usm"] diff --git a/tests/pipeline/test_link.py b/tests/pipeline/test_link.py index 9f24dc9..eee3d1a 100644 --- a/tests/pipeline/test_link.py +++ b/tests/pipeline/test_link.py @@ -296,81 +296,6 @@ def test_link_stages(tmp_path): assert execute(at, depot, stages).status == "hit" -def test_artifacts_table_and_stage(tmp_path): - depot = Depot(tmp_path / "depot") - arts, items = [], [] - for pid, status in ((1, "exported"), (2, "contained"), (3, "failed")): - oid = contract.object_id(CAB_A, pid) - if status == "exported": - c = depot.add(b"{}\n", "json") - aid = contract.artifact_id(oid, "json") - arts.append({"id": aid, "content": c}) - items.append(contract.item(oid, "exported", artifacts=[aid])) - elif status == "contained": - items.append(contract.item(oid, "contained", within=contract.artifact_id(CAB_A + ":1", "json"))) - else: - items.append(contract.item(oid, "failed", why=contract.reason("x", "y"))) - table = link.artifacts([("unity.export:card_a", {"artifacts": arts, "items": items})]) - assert table["objects"] == {f"{CAB_A}:1": [f"{CAB_A}:1#json"]} - assert table["contained"] == {f"{CAB_A}:2": f"{CAB_A}:1#json"} - assert table["artifacts"][f"{CAB_A}:1#json"]["ext"] == "json" - stage = link.ArtifactsStage() - env = Env(depot, {}) - export = contract.Task("unity.export", 1, "card_a") - depot.commit( - contract.result( - export, - [contract.artifact(a["id"], a["content"], contract.provenance(export), {"kind": "x"}) for a in arts], - items, - ) - ) - env.waiting(export.id) - with pytest.raises(Pending): - describe(stage, "all", None, env) - env.done(export.id, export.key) - env.waiting("sprite.crop:atlas", failed=True) # failed tasks are skipped - t = describe(stage, "all", None, env) - assert t.context == {"results": {export.id: export.key}} and t.inputs == () - execute(t, depot, registry(stage)) - (rec,) = depot.result(t.key)["artifacts"] - assert rec["id"] == "link.artifacts:all#artifacts" - assert contract.loads(depot.read(rec["content"]["sha256"]))["objects"] == table["objects"] - - -def test_the_artifact_table_is_written_entry_by_entry_with_the_same_bytes(): - def res(n, oid, aid, *, within=None): - c = {"sha256": f"{n:064x}", "size": n, "ext": "png", "mediaType": "image/png"} - items = ( - [contract.item(oid, "contained", within=within)] - if within - else [contract.item(oid, "exported", artifacts=[aid, aid + "~"])] - ) - return ( - f"unity.export:{n}", - { - "artifacts": [] if within else [{"id": aid, "content": c}, {"id": aid + "~", "content": c}], - "items": items, - }, - ) - - results = [ - res(3, f"{CAB_A}:3", f"{CAB_A}:3#立ち絵"), - res(1, f"{CAB_A}:1", f"{CAB_A}:1#png"), - res(2, f"{CAB_A}:2", None, within=f"{CAB_A}:1#png"), - res(4, f"{CAB_A}:é", None, within=f"{CAB_A}:1#png"), - ] - only_exported = [results[0], results[1]] - only_contained = [ - ("unity.export:x", {"artifacts": [], "items": [contract.item(f"{CAB_A}:9", "contained", within="a#b")]}) - ] - for rs in (results, only_exported, only_contained, []): - want = contract.encode(link.artifacts(rs)) - for chunk in (1, 64, 1 << 20): - parts = list(link.encode_artifacts(*link.artifact_table(iter(rs)), chunk=chunk)) - assert b"".join(parts) == want - assert len(list(link.encode_artifacts(*link.artifact_table(rs), chunk=1))) > 1 or not rs - - def test_store_add_chunks_is_add_of_the_joined_bytes(tmp_path): s = Depot(tmp_path / "s") c = s.add_chunks(iter([b"ab", b"", "é".encode()]), ".JSON") diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index f2a1879..8ec6e27 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -1,9 +1,8 @@ -"""unity.export and sprite.crop on synthetic bundles (fakeunity environments behind a stand-in reader atom): every -object gets exactly one item, the converters' artifacts, references, byte arrays and float forms, the generic -fallback with its reason codes, class selection, determinism, the cross-bundle sprite crop, and the golden records' -providers.""" +"""unity.export and sprite.crop on synthetic bundles (fakeunity environments behind a stand-in reader atom, with +their real censuses): one task per unit, every object exactly one item, the converters' artifacts, references, byte +arrays and float forms, the generic fallback with its reason codes, unit selection, batches, determinism, the +cross-bundle sprite crop, and the golden records' providers.""" -import json import math import re import subprocess @@ -15,10 +14,10 @@ from PIL import Image from nnnotes.pipeline import census as census_mod -from nnnotes.pipeline import contract, objexport +from nnnotes.pipeline import contract, objexport, units from nnnotes.pipeline.contract import IncompatibleTask, Input, Task from nnnotes.pipeline.depot import Depot -from nnnotes.pipeline.objexport import CROP, EXPORT, ExportStage, SpriteCropStage +from nnnotes.pipeline.objexport import CROP, ExportStage, SpriteCropStage from nnnotes.pipeline.stages import Env, describe, execute, registry from nnnotes.unity.atoms import Impl, astc, estimate, gltf from nnnotes.unity.atoms.reader import UnityBundle @@ -624,26 +623,70 @@ def classes_of(name): return sorted({r.type.name for f in BUNDLES[name]().assets for r in f.objects.values()}) -def task_for(depot, name, classes=None, params=None, scripts=None, typetrees=None): - data = f"fake:{name}".encode() - sha = depot.put(data) - stage = ExportStage(FAKE_READER) - p = stage.normalize(dict(params or {}, classes=classes)) - return Task( - EXPORT, - 1, - name, - p, - export_atoms(classes_of(name), FAKE_READER.id), - (Input("bundle", sha, len(data), f"{name}.bundle", ({"kind": "depot"},)),), - {"scripts": dict(SCRIPTS if scripts is None else scripts), **({"typetrees": typetrees} if typetrees else {})}, +def planning_env(depot, names=("all", "atlas"), selected=None, scripts=SCRIPTS, classes=None): + """A planning environment over fake bundles: their inputs, their censuses (census.census_env, the shipped type + trees read for files without type trees), the script table; `selected` (default: every bundle of `names`) and + `classes` as the export command gives them.""" + names = tuple(sorted(set(names) | set(selected or ()))) + bundles = {} + for n in names: + data = f"fake:{n}".encode() + bundles[n] = Input("bundle", depot.put(data), len(data), f"{n}.bundle", ({"kind": "depot"},)) + facts = {"bundles": bundles, "selected": sorted(selected if selected is not None else names), "classes": classes} + env = Env(depot, facts) + for n in names: + t = Task(census_mod.CensusStage.name, census_mod.CensusStage.version, n, {}, {}, (bundles[n],)) + doc = census_mod.census_env(BUNDLES[n](), shipped=True) + rec = contract.artifact( + contract.artifact_id(t.id, "census"), + depot.add(contract.encode(doc), "json"), + contract.provenance(t), + {"kind": "unity.census", "format": "json", "facts": census_mod.facts(doc)}, + ) + depot.commit(contract.result(t, [rec], [])) + env.done(t.id, t.key) + doc = {"schema": "nnnotes.scripts/1", "scripts": scripts} + data = contract.encode(doc) # its input: another table is another task + table = Task("link.scripts", 1, "all", {}, {}, (Input("census", depot.put(data), len(data), None, ()),)) + committed(depot, env, table, "scripts", doc, "link.scripts") + return env + + +def committed(depot, env, task, role, doc, kind): + rec = contract.artifact( + contract.artifact_id(task.id, role), + depot.add(contract.encode(doc), "json"), + contract.provenance(task), + {"kind": kind, "format": "json", "facts": {}}, ) + depot.commit(contract.result(task, [rec], [])) + env.done(task.id, task.key) -def run(depot, task): - stage = ExportStage(FAKE_READER) - execute(task, depot, registry(stage), force=True) - return depot.result(task.key) +def unit_task(depot, env, subject, params=None): + return describe(ExportStage(FAKE_READER), subject, params, env) + + +def export(depot, env, params=None, stage=None) -> dict: + """Run every unity.export task of a planning environment (in one stage object, as a worker runs a batch): the + results merged {"items", "artifacts"} with {"tasks": {task id: Task}, "status": {task id: result status}}.""" + stage = stage or ExportStage(FAKE_READER) + out: dict = {"items": [], "artifacts": [], "tasks": {}, "status": {}} + for s in stage.subjects(env): + t = describe(stage, s, params, env) + execute(t, depot, registry(stage), force=True) + doc = depot.result(t.key) + env.done(t.id, t.key) + out["items"] += doc["items"] + out["artifacts"] += doc["artifacts"] + out["tasks"][t.id] = t + out["status"][t.id] = doc["status"] + return out + + +def run_all(depot, names=("all",), params=None, classes=None, scripts=SCRIPTS) -> dict: + """export() of the units of the bundles `names` (only those of `classes` when given).""" + return export(depot, planning_env(depot, names, scripts=scripts, classes=classes), params) def by_object(doc): @@ -669,19 +712,18 @@ def oid(pid, cab=CAB): @pytest.fixture def exported(tmp_path): depot = Depot(tmp_path / "depot") - task = task_for(depot, "all") - return depot, task, run(depot, task) + return depot, run_all(depot) # ---------------------------------------------------------------- items def test_every_object_has_exactly_one_item(exported): - depot, task, doc = exported + depot, doc = exported objects = [contract.object_id(f.name, pid) for f in bundle_all().assets for pid in f.objects] items = by_object(doc) assert sorted(items) == sorted(objects) and len(doc["items"]) == len(objects) status = {o: (i["status"], (i.get("reason") or {}).get("code")) for o, i in items.items()} assert status == { - oid(1): ("contained", None), + oid(1): ("exported", None), oid(10): ("exported", None), oid(11): ("exported", None), oid(12): ("exported", None), @@ -705,7 +747,7 @@ def test_every_object_has_exactly_one_item(exported): oid(70): ("exported", None), oid(71): ("generic", "script.unresolved"), oid(72): ("generic", "script.missing"), - oid(80): ("contained", None), + oid(80): ("exported", None), oid(90): ("exported", None), oid(91): ("generic", "generic.clip.legacy"), oid(92): ("exported", None), @@ -719,18 +761,21 @@ def test_every_object_has_exactly_one_item(exported): oid(74): ("exported", None), oid(51): ("exported", None), } - assert items[oid(61)]["in"] == f"{oid(60)}#json" and items[oid(80)]["in"] == "unity.export:all#scripts.json" - assert items[oid(1)]["in"] == "unity.export:all#bundle.json" + assert items[oid(61)]["in"] == f"{oid(60)}#json" assert items[oid(94)]["artifacts"] == [f"{oid(94)}#json"] - assert doc["status"] == "ok" and all(i["class"] for i in doc["items"]) + assert set(doc["status"].values()) == {"ok"} and all(i["class"] for i in doc["items"]) for a in doc["artifacts"]: - owner, _role = contract.parse_artifact_id(a["id"]) - assert ("object" in a["provenance"]) == (not owner.startswith("unity.export:")) - assert depot.has(a["content"]["sha256"], a["content"]["size"]) + assert "object" in a["provenance"] and depot.has(a["content"]["sha256"], a["content"]["size"]) + # one task per unit: a hierarchy (its root's subject) or an object outside any + assert sorted(doc["tasks"]) == sorted( + f"unity.export:all:{contract.parse_object_id(o)[1]}" + for o in objects + if o not in {oid(p) for p in (61, 62, 63, 64, 65)} + ) def test_provenance_facts_and_atoms(exported): - depot, task, doc = exported + depot, doc = exported a = art(doc, f"{oid(10)}#image") assert a["provenance"]["object"] == { "file": CAB, @@ -749,7 +794,9 @@ def test_provenance_facts_and_atoms(exported): "astc.container", "tex.png", } + task = doc["tasks"]["unity.export:all:10"] assert a["provenance"]["inputs"] == [{"role": "bundle", "sha256": task.inputs[0].sha256}] + assert a["provenance"]["stage"]["version"] == ExportStage.version == 2 g = art(doc, f"{oid(95)}#json") assert set(g["provenance"]["atoms"]) == {"reader", "sniff", "generic.json"} assert task.atoms["tex.png"] == "tex.png/2" and task.atoms["reader"] == "fakeunity/1" @@ -757,7 +804,7 @@ def test_provenance_facts_and_atoms(exported): # ---------------------------------------------------------------- converters def test_textures(exported): - depot, _task, doc = exported + depot, doc = exported png = load(depot, doc, f"{oid(10)}#image") img = Image.open(__import__("io").BytesIO(png)) assert img.size == (W, H) and img.mode == "RGBA" @@ -784,7 +831,7 @@ def test_textures(exported): def test_sprites_same_bundle(exported): - depot, _task, doc = exported + depot, doc = exported from nnnotes.unity.atoms import png, sprite from nnnotes.unity.atoms.mesh import arrays from nnnotes.unity.atoms.reader import mesh_input_from_typetree @@ -833,7 +880,7 @@ def test_sprites_same_bundle(exported): def test_sprite_with_its_atlas_elsewhere_gets_a_meta_only(exported): - depot, _task, doc = exported + depot, doc = exported assert by_object(doc)[oid(13)]["artifacts"] == [f"{oid(13)}#json"] a = art(doc, f"{oid(13)}#json") assert a["semantics"]["facts"] == {"crop": CROP, "atlas": f"{ATLAS_CAB}:99"} @@ -843,7 +890,7 @@ def test_sprite_with_its_atlas_elsewhere_gets_a_meta_only(exported): def test_mesh(exported): - depot, _task, doc = exported + depot, doc = exported d, body = gltf.read(load(depot, doc, f"{oid(30)}#mesh")) (prim,) = d["meshes"][0]["primitives"] assert set(prim["attributes"]) == {"POSITION", "NORMAL", "TEXCOORD_0"} and prim["mode"] == 4 @@ -858,7 +905,7 @@ def test_mesh(exported): def test_text_and_fonts(exported): - depot, _task, doc = exported + depot, doc = exported a = art(doc, f"{oid(40)}#data") assert load(depot, doc, a["id"]) == b"hello\n" and a["content"]["ext"] == "txt" assert load(depot, doc, f"{oid(41)}#data") == b"\xff\xfe\x00bin" @@ -868,7 +915,7 @@ def test_text_and_fonts(exported): def test_every_converted_object_has_its_json(exported): - depot, _task, doc = exported + depot, doc = exported tex = obj(depot, doc, f"{oid(10)}#json") assert "image data" not in tex and tex["m_Width"] == W and tex["m_TextureFormat"] == 4 assert art(doc, f"{oid(10)}#json")["semantics"] == {"kind": "unity.object", "format": "json", "facts": {}} @@ -886,12 +933,11 @@ def test_every_converted_object_has_its_json(exported): assert all( i["status"] != "exported" or any(a.endswith("#json") for a in i["artifacts"]) or i["object"] == oid(13) for i in doc["items"] - if i["class"] not in ("AssetBundle", "MonoScript") ) def test_items_name_the_objects_they_reference(exported): - _depot, _task, doc = exported + _depot, doc = exported items = by_object(doc) assert items[oid(50)]["refs"] == [oid(10), f"{ATLAS_CAB}:5"] # the material's shader and texture # a prefab: the objects its components reference outside it, other bundles' included; its own objects are named @@ -907,7 +953,7 @@ def test_items_name_the_objects_they_reference(exported): def test_prefab(exported): - depot, _task, doc = exported + depot, doc = exported p = obj(depot, doc, f"{oid(60)}#json") assert p["name"] == "root" and [n["path"] for n in p["nodes"]] == ["root", "root/child"] root, child = p["nodes"] @@ -930,7 +976,7 @@ def test_prefab(exported): def test_scriptable_objects_blobs_and_strings(exported): - depot, _task, doc = exported + depot, doc = exported m = obj(depot, doc, f"{oid(70)}#json") assert m["$script"] == "A|Game|Local" and m["m_Name"] == "moc" blob = m["_bytes"]["$blob"] @@ -949,7 +995,7 @@ def test_scriptable_objects_blobs_and_strings(exported): def test_cue_sheets_held_in_the_bundle(exported): - depot, _task, doc = exported + depot, doc = exported from nnnotes.formats import cri a = art(doc, f"{oid(73)}#acb") @@ -973,15 +1019,21 @@ def test_cue_sheets_held_in_the_bundle(exported): "$blob": {"sha256": contract.sha256(EMBEDDED_ACB), "size": len(EMBEDDED_ACB), "format": "acb"} } assert by_object(doc)[oid(74)]["artifacts"] == [f"{oid(74)}#acb", f"{oid(74)}#json"] # no blob: artifact - small = run(depot, task_for(depot, "all", classes=["MonoBehaviour"], params={"blobMin": 1 << 20})) + small = run_all(depot, classes=["MonoBehaviour"], params={"blobMin": 1 << 20}) assert load(depot, small, f"{oid(74)}#acb") == EMBEDDED_ACB # whatever its size def movie_task(depot, converter=True): - """The task of the movie bundle: converter cri.usm in its atoms, as uses() gives it for that bundle.""" - t = task_for(depot, "movie", scripts=USM_SCRIPTS) - atoms = {**t.atoms, "cri.usm": "cri.usm/1"} if converter else t.atoms - return Task(t.stage, t.version, t.subject, t.params, dict(sorted(atoms.items())), t.inputs, t.context) + """The task of the movie asset's unit: converter cri.usm in its atoms, as uses() gives it for that unit.""" + env = planning_env(depot, ("movie",), scripts=USM_SCRIPTS) + t = unit_task(depot, env, "movie:2") + atoms = t.atoms if converter else {k: v for k, v in t.atoms.items() if k != "cri.usm"} + return Task(t.stage, t.version, t.subject, t.params, atoms, t.inputs, t.context) + + +def run(depot, task): + execute(task, depot, registry(ExportStage(FAKE_READER)), force=True) + return depot.result(task.key) def test_movies_held_in_the_bundle(tmp_path): @@ -1000,22 +1052,23 @@ def test_movies_held_in_the_bundle(tmp_path): "$blob": {"sha256": contract.sha256(WARMUP), "size": len(WARMUP), "format": "usm"} } assert by_object(doc)[m]["status"] == "exported" and by_object(doc)[m]["artifacts"] == [f"{m}#json", f"{m}#usm"] - assert load(depot, doc, f"{a}#acb") == EMBEDDED_ACB old = run(depot, movie_task(depot, converter=False)) assert by_object(old)[m]["status"] == "generic" and by_object(old)[m]["reason"]["code"] == "generic.error" - assert f"{m}#usm" not in {x["id"] for x in old["artifacts"]} and load(depot, old, f"{a}#acb") == EMBEDDED_ACB + assert f"{m}#usm" not in {x["id"] for x in old["artifacts"]} + sheet = run_all(depot, ("movie",), scripts=USM_SCRIPTS) + assert load(depot, sheet, f"{a}#acb") == EMBEDDED_ACB def test_the_movie_converter_is_keyed_by_the_script_context(tmp_path): depot = Depot(tmp_path / "s") env = planning_env(depot, ("all", "movie"), scripts=USM_SCRIPTS) stage = ExportStage(FAKE_READER) - assert describe(stage, "movie", None, env).atoms["cri.usm"] == "cri.usm/1" - assert "cri.usm" not in describe(stage, "all", None, env).atoms - assert "cri.usm" not in describe(stage, "movie", {"classes": ["AssetBundle"]}, env).atoms + assert describe(stage, "movie:2", None, env).atoms["cri.usm"] == "cri.usm/1" + assert "cri.usm" not in describe(stage, "movie:3", None, env).atoms # a cue sheet asset + assert "cri.usm" not in describe(stage, "all:70", None, env).atoms assert "cri.usm" in stage.ATOMS plain = planning_env(Depot(tmp_path / "p"), ("movie",)) - assert "cri.usm" not in describe(stage, "movie", None, plain).atoms + assert "cri.usm" not in describe(stage, "movie:2", None, plain).atoms def test_acb_layouts(tmp_path): @@ -1064,14 +1117,14 @@ def test_a_split_sheet_with_a_missing_chunk_gives_the_typetree(tmp_path, monkeyp dict(real(tt), chunks=[pptr(44), pptr(99, 1)]) if real(tt) and real(tt)["layout"] == "split" else real(tt) ), ) - doc = run(depot, task_for(depot, "all", classes=["MonoBehaviour"])) + doc = run_all(depot, classes=["MonoBehaviour"]) item = by_object(doc)[oid(73)] assert item["status"] == "generic" and item["reason"]["code"] == "generic.error" assert "is not a TextAsset of the bundle" in item["reason"]["message"] def test_shader(exported): - depot, _task, doc = exported + depot, doc = exported from nnnotes.formats import shader env = bundle_all() @@ -1099,7 +1152,7 @@ def test_shader(exported): def test_clip_bindings_name_scripts_of_other_bundles(exported): - depot, _task, doc = exported + depot, doc = exported clip = obj(depot, doc, f"{oid(96)}#json") (b,) = clip["bindings"] assert b == { @@ -1114,11 +1167,11 @@ def test_clip_bindings_name_scripts_of_other_bundles(exported): "script": {"$ref": {"file": "CAB-scripts", "pathId": 12}}, "scriptClass": "Assembly-CSharp|Game|Fader", } - assert census_mod.script_refs(census_doc(open_fake(b"fake:all"))) == sorted(SCRIPTS) + ["CAB-scripts:9"] + assert census_mod.script_refs(census_mod.census_env(bundle_all())) == sorted(SCRIPTS) + ["CAB-scripts:9"] def test_material_clip_controller(exported): - depot, _task, doc = exported + depot, doc = exported mat = obj(depot, doc, f"{oid(50)}#json") assert mat["shader"] == {"$ref": {"file": ATLAS_CAB, "pathId": 5}} assert mat["textures"]["_MainTex"]["texture"]["$ref"]["pathId"] == 10 @@ -1160,19 +1213,11 @@ def test_material_clip_controller(exported): assert obj(depot, doc, f"{oid(95)}#json")["m_Speed"] == INF -def test_loose_documents(exported): - depot, task, doc = exported - s = obj(depot, doc, "unity.export:all#scripts.json") - assert list(s["scripts"]) == [oid(80)] and s["scripts"][oid(80)]["m_ClassName"] == "Local" - b = obj(depot, doc, "unity.export:all#bundle.json") - assert b["files"] == [ - { - "name": CAB, - "unityVersion": "6000.3.12f1", - "externals": ["archive:/CAB-atlas/CAB-atlas", "archive:/CAB-scripts/CAB-scripts"], - } - ] - ab = b["assetBundles"][oid(1)] +def test_scripts_and_asset_bundles_have_their_typetree(exported): + depot, doc = exported + s = obj(depot, doc, f"{oid(80)}#json") + assert s["m_ClassName"] == "Local" and by_object(doc)[oid(80)]["artifacts"] == [f"{oid(80)}#json"] + ab = obj(depot, doc, f"{oid(1)}#json") assert ab["m_Container"][0] == [ "assets/x/tex.png", { @@ -1185,33 +1230,35 @@ def test_loose_documents(exported): def test_two_runs_are_identical(tmp_path): a, b = Depot(tmp_path / "a"), Depot(tmp_path / "b") - ta, tb = task_for(a, "all"), task_for(b, "all") - assert ta.key == tb.key - da, db = run(a, ta), run(b, tb) - assert contract.encode(da) == contract.encode(db) + da, db = run_all(a), run_all(b) + assert {k: x.key for k, x in da["tasks"].items()} == {k: x.key for k, x in db["tasks"].items()} + assert contract.encode(da["items"]) == contract.encode(db["items"]) + assert contract.encode(da["artifacts"]) == contract.encode(db["artifacts"]) for x in da["artifacts"]: assert a.read(x["content"]["sha256"]) == b.read(x["content"]["sha256"]) -def test_key_follows_the_census_classes_and_params(tmp_path): +def test_a_units_key_holds_its_classes_and_the_params(tmp_path): depot = Depot(tmp_path / "s") - t = task_for(depot, "atlas") + env = planning_env(depot, ("atlas",)) + t = unit_task(depot, env, "atlas:97") # a sprite assert "mesh.glb" not in t.atoms and "mesh.arrays" in t.atoms # sprites use mesh.arrays - assert {"tex.png", "sprite.png", "generic.json", "bundle.json"} <= set(t.atoms) - assert task_for(depot, "atlas", params={"png": {"level": 3}}).key != t.key + assert set(t.atoms) == set(export_atoms(["Sprite"], FAKE_READER.id)) + assert t.context == {"unit": f"{ATLAS_CAB}:97", "bundle": "atlas"} + assert unit_task(depot, env, "atlas:97", {"png": {"level": 3}}).key != t.key assert export_atoms(["Texture2D"]) == export_atoms(["Texture2D", "Texture2D"]) assert set(export_atoms(["Mesh"])) == {"reader", "sniff", "mesh.arrays", "gltf.write", "mesh.glb", "generic.json"} with pytest.raises(ValueError, match="png level"): ExportStage().normalize({"png": {"level": 11}}) with pytest.raises(ValueError, match="unknown parameters"): - ExportStage().normalize({"mesh": "obj"}) - assert ExportStage().normalize(None) == {"blobMin": 4096, "classes": None, "png": {"level": 6}} + ExportStage().normalize({"classes": ["Mesh"]}) + assert ExportStage().normalize(None) == {"blobMin": 4096, "png": {"level": 6}} def test_png_level_changes_bytes_not_pixels(tmp_path): depot = Depot(tmp_path / "s") - d6 = run(depot, task_for(depot, "all", classes=["Texture2D"])) - d0 = run(depot, task_for(depot, "all", classes=["Texture2D"], params={"png": {"level": 0}})) + d6 = run_all(depot, classes=["Texture2D"]) + d0 = run_all(depot, classes=["Texture2D"], params={"png": {"level": 0}}) p6, p0 = load(depot, d6, f"{oid(10)}#image"), load(depot, d0, f"{oid(10)}#image") assert p6 != p0 io = __import__("io") @@ -1220,27 +1267,31 @@ def test_png_level_changes_bytes_not_pixels(tmp_path): def test_blob_threshold(tmp_path): depot = Depot(tmp_path / "s") - doc = run(depot, task_for(depot, "all", classes=["MonoBehaviour"], params={"blobMin": 1 << 20})) + doc = run_all(depot, classes=["MonoBehaviour"], params={"blobMin": 1 << 20}) m = obj(depot, doc, f"{oid(70)}#json") assert m["_bytes"] == {"$hex": MOC.hex()} -def test_class_selection(tmp_path): +def test_classes_select_units_by_their_root(tmp_path): depot = Depot(tmp_path / "s") - doc = run(depot, task_for(depot, "all", classes=["Sprite"])) + doc = run_all(depot, classes=["Sprite"]) items = by_object(doc) - assert set(items) == {oid(11), oid(12), oid(13)} # textures decoded, not exported + assert set(items) == {oid(11), oid(12), oid(13)} # their textures decoded, not exported assert all(i["status"] == "exported" for i in items.values()) - doc = run(depot, task_for(depot, "all", classes=["GameObject"])) + doc = run_all(depot, classes=["GameObject"]) assert set(by_object(doc)) == {oid(p) for p in (60, 61, 62, 63, 64, 65)} + assert list(doc["tasks"]) == ["unity.export:all:60"] + keys = {k: x.key for k, x in run_all(depot)["tasks"].items()} + assert keys["unity.export:all:60"] == doc["tasks"]["unity.export:all:60"].key # a selection is no part of keys def test_scripts_come_from_the_context_only(tmp_path): depot = Depot(tmp_path / "s") - doc = run(depot, task_for(depot, "all", classes=["MonoBehaviour"], scripts={})) + doc = run_all(depot, classes=["MonoBehaviour", "GameObject"], scripts={}) items = by_object(doc) assert items[oid(70)]["reason"]["code"] == "script.unresolved" - assert items[oid(62)]["reason"]["code"] == "script.unresolved" # no hierarchy: one by one + issues = art(doc, f"{oid(60)}#json")["semantics"]["issues"] # the components of a hierarchy + assert issues[0]["code"] == "script.unresolved" and issues[0]["message"].startswith("2 MonoBehaviours") def test_a_converter_error_gives_the_typetree(tmp_path, monkeypatch): @@ -1250,7 +1301,7 @@ def broken(tt, ref): raise KeyError("m_SavedProperties") monkeypatch.setattr(material, "material_doc", broken) - doc = run(depot, task_for(depot, "all", classes=["Material"])) + doc = run_all(depot, classes=["Material"]) item = by_object(doc)[oid(50)] assert item["status"] == "generic" and item["reason"] == { "code": "generic.error", @@ -1266,7 +1317,7 @@ def broken(*args): raise RuntimeError("component failed") monkeypatch.setattr(prefab, "_component", broken) - doc = run(depot, task_for(depot, "all", classes=["GameObject"])) + doc = run_all(depot, classes=["GameObject"]) items = by_object(doc) assert {i["status"] for i in items.values()} == {"generic"} assert all(i["reason"]["code"] == "generic.error" for i in items.values()) @@ -1283,14 +1334,16 @@ def typetree(self, o): return real(self, o) monkeypatch.setattr(UnityBundle, "typetree", typetree) - doc = run(depot, task_for(depot, "all", classes=["Animator"])) + doc = run_all(depot, classes=["Animator"]) (item,) = doc["items"] - assert item["status"] == "failed" and item["reason"]["code"] == "failed.read" and doc["status"] == "partial" + assert item["status"] == "failed" and item["reason"]["code"] == "failed.read" + assert doc["status"] == {"unity.export:all:95": "partial"} def test_task_with_other_atoms_is_refused(tmp_path): depot = Depot(tmp_path / "s") - t = task_for(depot, "all") + env = planning_env(depot, ("all",)) + t = unit_task(depot, env, "all:10") bad = Task(t.stage, t.version, t.subject, t.params, {**t.atoms, "tex.png": "tex.png/0"}, t.inputs, t.context) with pytest.raises(IncompatibleTask, match="tex.png"): execute(bad, depot, registry(ExportStage(FAKE_READER))) @@ -1299,119 +1352,90 @@ def test_task_with_other_atoms_is_refused(tmp_path): t.version, t.subject, t.params, - {k: v for k, v in t.atoms.items() if k != "mesh.glb"}, + {k: v for k, v in t.atoms.items() if k != "tex.png"}, t.inputs, t.context, ) - with pytest.raises(IncompatibleTask, match="mesh.glb"): + with pytest.raises(IncompatibleTask, match="tex.png"): execute(missing, depot, registry(ExportStage(FAKE_READER))) + other = Task(t.stage, t.version, t.subject, t.params, t.atoms, t.inputs, {**t.context, "unit": f"{CAB}:999"}) + with pytest.raises(IncompatibleTask, match="not in the bundle"): + execute(other, depot, registry(ExportStage(FAKE_READER))) # ---------------------------------------------------------------- describing tasks -def census_doc(view: UnityBundle) -> dict: - """The part of a census the export stage reads (classes, objects, m_Script references).""" - files, classes = [], {} - for f in view.files(): - objs = [] - for o in view.objects(): - if o.file != f: - continue - classes[o.class_name] = classes.get(o.class_name, 0) + 1 - rec = {"pathId": o.path_id, "classId": o.class_id, "class": o.class_name, "typeHash": o.type_hash} - tt = view.reader(o).read_typetree(check_read=False) # a header read, as the census' - ext = view.externals(f) - - def ref(s): - file = f if not s["m_FileID"] else ext[s["m_FileID"] - 1].rsplit("/", 1)[-1] - return {"file": file, "pathId": s["m_PathID"]} - - s = tt.get("m_Script") - if isinstance(s, dict) and s.get("m_PathID"): - rec["script"] = ref(s) - bindings = [ - b["script"] - for b in (tt.get("m_ClipBindingConstant") or {}).get("genericBindings", ()) - if b["script"].get("m_PathID") - ] - if bindings: - rec["bindingScripts"] = [ref(b) for b in bindings] - objs.append(rec) - files.append( - {"name": f, "unityVersion": view.unity_version(f), "typetree": view.stores_typetrees(f), "objects": objs} - ) - return { - "schema": contract.CENSUS, - "files": files, - "resources": [], - "assetBundles": [], - "scripts": [], - "classes": dict(sorted(classes.items())), - "objects": sum(classes.values()), - "errors": [], - } - - -def committed(depot, env, task, role, doc, kind): - rec = contract.artifact( - contract.artifact_id(task.id, role), - depot.add(contract.encode(doc), "json"), - contract.provenance(task), - {"kind": kind, "format": "json", "facts": {}}, - ) - depot.commit(contract.result(task, [rec], [])) - env.done(task.id, task.key) - - -def planning_env(depot, names=("all", "atlas"), selected=None, scripts=SCRIPTS): - names = tuple(sorted(set(names) | set(selected or ()))) - bundles = {} - for n in names: - data = f"fake:{n}".encode() - bundles[n] = Input("bundle", depot.put(data), len(data), f"{n}.bundle", ({"kind": "depot"},)) - env = Env(depot, {"bundles": bundles, "selected": sorted(selected if selected is not None else names)}) - for n in names: - t = Task(census_mod.CensusStage.name, 1, n, {}, {}, (bundles[n],)) - doc = census_doc(open_fake(f"fake:{n}".encode())) - rec = contract.artifact( - contract.artifact_id(t.id, "census"), - depot.add(contract.encode(doc), "json"), - contract.provenance(t), - {"kind": "unity.census", "format": "json", "facts": census_mod.facts(doc)}, - ) - depot.commit(contract.result(t, [rec], [])) - env.done(t.id, t.key) - table = Task("link.scripts", 1, "all") - committed(depot, env, table, "scripts", {"schema": "nnnotes.scripts/1", "scripts": scripts}, "link.scripts") - return env - - -def test_describe_reads_the_census_and_the_script_table(tmp_path): +def test_describe_reads_the_units_and_the_script_table(tmp_path): depot = Depot(tmp_path / "s") env = planning_env(depot) stage = ExportStage(FAKE_READER) - assert stage.subjects(env) == ["all", "atlas"] - # every closure bundle holding a SpriteAtlas joins a selection that holds sprites (a sprite reaches its - # texture through its atlas' bundle); a selection without sprites stays as it is - assert stage.subjects(planning_env(Depot(tmp_path / "t"), selected=["all"])) == ["all", "atlas"] - assert stage.subjects(planning_env(Depot(tmp_path / "u"), selected=["plain"])) == ["plain"] - t = describe(stage, "all", None, env) - assert t.atoms == export_atoms(classes_of("all"), FAKE_READER.id) - assert t.context == {"scripts": SCRIPTS} and t.key == task_for(depot, "all").key - assert stage.depends("all", env) == ["unity.census:all", "link.scripts:all"] - a = describe(stage, "atlas", None, env) - assert a.context == {} and "mesh.glb" not in a.atoms - assert t.cost.peak_bytes >= 72 << 20 + subjects = stage.subjects(env) + assert "all:60" in subjects and "all:61" not in subjects and "atlas:1" in subjects + # a selection's units and the units their objects reach in other bundles (a sprite's atlas, its texture and + # the sprites it packs); not the rest of those bundles + assert [ + s for s in stage.subjects(planning_env(Depot(tmp_path / "t"), selected=["all"])) if s.startswith("atlas") + ] == [ + "atlas:97", + "atlas:98", + "atlas:99", + ] + assert stage.subjects(planning_env(Depot(tmp_path / "u"), selected=["plain"])) == ["plain:1", "plain:2"] + t = describe(stage, "all:10", None, env) + assert t.atoms == export_atoms(["Texture2D"], FAKE_READER.id) and t.subject == "all:10" + assert t.context == {"unit": oid(10), "bundle": "all"} + assert stage.depends("all:10", env) == ["unity.census:all", "link.scripts:all"] + m = describe(stage, "all:70", None, env) + assert m.context["scripts"] == {oid(80): "A|Game|Local"} + p = describe(stage, "all:60", None, env) # a hierarchy: its objects' classes and scripts + assert set(p.atoms) == set(export_atoms(["GameObject", "MonoBehaviour", "Transform"], FAKE_READER.id)) + assert p.context["scripts"] == { + "CAB-scripts:7": SCRIPTS["CAB-scripts:7"], + "CAB-scripts:8": SCRIPTS["CAB-scripts:8"], + } + clip = describe(stage, "all:96", None, env) # its bindings' scripts + assert clip.context["scripts"] == {"CAB-scripts:12": "Assembly-CSharp|Game|Fader"} + assert t.cost.peak_bytes >= 72 << 20 and stage.batch(t) == t.inputs[0].sha256 == stage.batch(p) fresh = Env(depot, env.facts) from nnnotes.pipeline.stages import Pending fresh.waiting("unity.census:all") with pytest.raises(Pending): - describe(stage, "all", None, fresh) + stage.subjects(fresh) + + +def test_the_tasks_of_a_bundle_load_it_once_and_their_order_changes_nothing(tmp_path): + opened = [] + + def counting(data, name=None): + opened.append(bytes(data)) + return open_fake(data, name) + reader = Impl("fakeunity/1", counting, lambda facts: estimate(0.01, 1 << 20)) + depot = Depot(tmp_path / "s") + env = planning_env(depot, ("all",)) + stage = ExportStage(reader) + subjects = stage.subjects(env) + tasks = [describe(stage, s, None, env) for s in subjects] + results = {} + for t in tasks: + results[t.id] = stage.run(t, depot) + assert opened == [b"fake:all"] + stage.end_batch(stage.batch(tasks[0])) + assert stage._loaded is None + backwards = ExportStage(reader) + for t in reversed(tasks): + out = backwards.run(t, depot) + assert contract.encode(out.items) == contract.encode(results[t.id].items), t.id + assert contract.encode(out.artifacts) == contract.encode(results[t.id].artifacts), t.id + assert len(opened) == 2 + + +def test_files_without_type_trees_read_their_scripts_with_the_shipped_ones(tmp_path, monkeypatch): + from nnnotes.unity import scripttypes -def test_files_without_type_trees_read_their_scripts_with_the_shipped_ones(tmp_path): + monkeypatch.setattr(scripttypes, "script_types", lambda version: PLAYER_TYPETREES) depot = Depot(tmp_path / "s") - doc = run(depot, task_for(depot, "player", scripts=PLAYER_SCRIPTS, typetrees=PLAYER_TYPETREES)) + doc = run_all(depot, ("player",), scripts=PLAYER_SCRIPTS) items = by_object(doc) def p(pid): @@ -1469,39 +1493,35 @@ def test_the_context_holds_the_shipped_type_trees_of_its_scripts(tmp_path, monke trees = {**PLAYER_TYPETREES, shipped["typeHash"]: shipped["nodes"]} monkeypatch.setattr(scripttypes, "script_types", lambda version: trees) env = planning_env(Depot(tmp_path / "s"), names=("player",), scripts=PLAYER_SCRIPTS) - t = describe(ExportStage(FAKE_READER), "player", None, env) - assert t.context["typetrees"] == PLAYER_TYPETREES # the trees of its MonoBehaviours' type hashes - assert ( - "typetrees" not in describe(ExportStage(FAKE_READER), "all", None, planning_env(Depot(tmp_path / "t"))).context - ) - - -def test_class_selection_keeps_only_its_converters_in_the_key(tmp_path): - depot = Depot(tmp_path / "s") - env = planning_env(depot) stage = ExportStage(FAKE_READER) - full = describe(stage, "all", None, env) - assert full.key == describe(stage, "all", {"classes": None}, env).key == task_for(depot, "all").key - meshes = describe(stage, "all", {"classes": ["Mesh"]}, env) - assert meshes.atoms == export_atoms(["Mesh"], FAKE_READER.id) and meshes.params["classes"] == ["Mesh"] - assert "tex.png" not in meshes.atoms and meshes.key != full.key - assert describe(stage, "all", {"classes": ["Mesh", "Nothing"]}, env).atoms == meshes.atoms - assert describe(stage, "all", {"classes": ["Nothing"]}, env).atoms == export_atoms([], FAKE_READER.id) - assert env.params == {} # set while describing only - for classes in (["Mesh"], ["Sprite"], ["GameObject"], ["Material", "AnimationClip"]): - t = describe(stage, "all", {"classes": classes}, env) - execute(t, depot, registry(stage)) - doc = depot.result(t.key) - assert doc["items"] and all(i["class"] in {*classes, "Transform", "MonoBehaviour"} for i in doc["items"]) + # the trees of its MonoBehaviours' type hashes; a hierarchy whose component has none: no trees + assert describe(stage, "player:2", None, env).context["typetrees"] == {PLAYER_HASH: SETTINGS_ROWS} + assert describe(stage, "player:4", None, env).context["typetrees"] == {OTHER_HASH: SETTINGS_ROWS[:-1]} + assert "typetrees" not in describe(stage, "player:20", None, env).context + assert "typetrees" not in describe(stage, "all:70", None, planning_env(Depot(tmp_path / "t"))).context + + +def test_the_unit_table(tmp_path): + env = planning_env(Depot(tmp_path / "s"), ("all", "atlas", "plain"), selected=["all"], classes=["Sprite"]) + t = units.table(env) + assert t.subjects() == ["all:11", "all:12", "all:13", "atlas:97"] + u = t.unit("all:13") + assert (u.bundle, u.root, u.cls, u.members) == ("all", oid(13), "Sprite", (oid(13),)) + assert t.subject_of(oid(62)) == "all:60" and t.subject_of(f"{ATLAS_CAB}:98") == "atlas:98" + assert t.subject_of("CAB-plain:2") == "plain:2" and t.subject_of("CAB-x:1") is None + assert t.files == {CAB: "all", ATLAS_CAB: "atlas", "CAB-plain": "plain"} + assert t.per_bundle == {"all": 3, "atlas": 1} and t.classes == {"Sprite": 4} + assert units.table(env) is t # kept for the same planning state + h = units.table(planning_env(Depot(tmp_path / "t"), ("all",))).unit("all:60") + assert h.members == tuple(oid(p) for p in (60, 61, 62, 63, 64, 65)) + assert h.counts == (("GameObject", 2), ("MonoBehaviour", 2), ("Transform", 2)) + with pytest.raises(ValueError, match="no requested unit"): + t.unit("all:61") def crop_setup(depot): env = planning_env(depot) - stage = ExportStage(FAKE_READER) - for n in ("all", "atlas"): - t = describe(stage, n, None, env) - execute(t, depot, registry(stage)) - env.done(t.id, t.key) + export(depot, env) return env @@ -1512,7 +1532,11 @@ def test_sprite_crop_equals_the_same_sprite_cropped_in_its_atlas_bundle(tmp_path assert stage.subjects(env) == ["atlas:99"] t = describe(stage, "atlas:99", None, env) assert [i.role for i in t.inputs] == ["atlas", f"sprite:{oid(13)}", f"texture:{ATLAS_CAB}:98"] - assert stage.depends("atlas:99", env) == ["unity.export:all", "unity.export:atlas"] + assert stage.depends("atlas:99", env) == [ + "unity.export:all:13", + "unity.export:atlas:98", + "unity.export:atlas:99", + ] assert set(t.atoms) == {"png.encode", "sprite.crop"} execute(t, depot, registry(stage)) doc = depot.result(t.key) @@ -1520,7 +1544,7 @@ def test_sprite_crop_equals_the_same_sprite_cropped_in_its_atlas_bundle(tmp_path assert item == {"object": oid(13), "status": "exported", "artifacts": [f"{oid(13)}#image"], "class": "Sprite"} a = doc["artifacts"][0] assert a["provenance"]["object"]["name"] == "remote" and a["provenance"]["object"]["file"] == CAB - local = depot.result(env.key("unity.export:atlas")) + local = depot.result(env.key("unity.export:atlas:97")) same = art(local, f"{ATLAS_CAB}:97#image") assert a["content"] == same["content"] # the same bytes as a local crop assert Image.open(__import__("io").BytesIO(depot.read(a["content"]["sha256"]))).size == (8, 8) @@ -1529,7 +1553,7 @@ def test_sprite_crop_equals_the_same_sprite_cropped_in_its_atlas_bundle(tmp_path def test_sprite_crop_waits_for_exports(tmp_path): depot = Depot(tmp_path / "s") env = planning_env(depot) - env.waiting("unity.export:all") + env.waiting("unity.export:all:13") from nnnotes.pipeline.stages import Pending with pytest.raises(Pending): @@ -1570,34 +1594,32 @@ def test_every_reason_code_and_converter_is_documented(): # ---------------------------------------------------------------- golden records -CONVERTER_CLASSES = { - "tex.png": ["Texture2D"], - "sprite.png": ["Sprite"], - "mesh.glb": ["Mesh"], - "prefab.json": ["GameObject"], - "material.json": ["Material"], - "clip.json": ["AnimationClip"], - "controller.json": ["AnimatorController", "AnimatorOverrideController"], - "mono.json": ["MonoBehaviour"], - "data": ["TextAsset"], - "font": ["Font"], - "shader.json": ["Shader"], - "scripts.json": ["MonoScript"], - "bundle.json": ["AssetBundle"], - "generic.json": ["Animator", "AudioClip", "SpriteAtlas"], +# converter -> the unit of the synthetic bundles its golden fixture converts +CONVERTER_UNITS = { + "tex.png": "all:10", + "sprite.png": "all:12", + "mesh.glb": "all:30", + "prefab.json": "all:60", + "material.json": "all:50", + "clip.json": "all:96", + "controller.json": "all:92", + "mono.json": "all:73", + "data": "all:40", + "font": "all:42", + "shader.json": "all:51", + "generic.json": "all:95", } LIBRARIES = ["Pillow", "UnityPy", "astc-encoder-py", "numpy", "texture2ddecoder"] def export_golden(): - """Provider of tests/golden/unity.export.json: one fixture per converter (the synthetic bundle with the - classes the converter takes), and the whole bundle.""" + """Provider of tests/golden/unity.export.json: one fixture per converter (a unit of the synthetic bundles that + the converter converts).""" - def fixture(classes): - return lambda depot: task_for(depot, "all", classes=classes) + def fixture(subject): + return lambda depot: unit_task(depot, planning_env(depot, ("all",)), subject) - fixtures = {name: fixture(classes) for name, classes in CONVERTER_CLASSES.items()} - fixtures["all"] = fixture(None) + fixtures = {name: fixture(subject) for name, subject in CONVERTER_UNITS.items()} fixtures["cri.usm"] = movie_task return {"stage": ExportStage(FAKE_READER), "fixtures": fixtures, "libraries": LIBRARIES} @@ -1614,8 +1636,7 @@ def fixture(depot): def test_golden_providers_cover_every_converter(): p = export_golden() - assert set(p["fixtures"]) == set(CONVERTERS) | {"all"} - assert json.dumps(CONVERTER_CLASSES) # every converter named once + assert set(p["fixtures"]) == set(CONVERTERS) def test_a_converter_change_without_a_version_bump_fails_its_golden(tmp_path, monkeypatch): @@ -1639,5 +1660,4 @@ def changed(ctx, info): actual = golden(p["stage"], p["fixtures"], Depot(tmp_path / "s"), p["libraries"]) problems = golden_problems(expected, actual, p["stage"]) assert "unity.export: fixture font: output changed without a version bump" in problems - assert "unity.export: fixture all: output changed without a version bump" in problems assert not any("fixture tex.png" in x for x in problems) diff --git a/tests/pipeline/test_voices_story.py b/tests/pipeline/test_voices_story.py index 8fd0dd3..c6da211 100644 --- a/tests/pipeline/test_voices_story.py +++ b/tests/pipeline/test_voices_story.py @@ -10,11 +10,11 @@ from nnnotes.content import advcommand, storyvoices from nnnotes.pipeline import contract, voices -from nnnotes.pipeline.contract import Task from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Pending, describe, execute, registry from nnnotes.pipeline.views import AddressIndex, Obj, ViewError from nnnotes.source import languages +from tests.helpers import units as unit_results from tests.pipeline.test_views import addresses_doc from tests.pipeline.test_voices import KEYS, SHEETS, TABLES, Paths, audio, audio_task, planning, row, text @@ -483,28 +483,19 @@ def test_story_rules_are_checked(): # ---------------------------------------------------------------- the stage def export_results(depot, env) -> dict: - """unity.export results holding the exported episode parts, one bundle per episode: {asset: {part: oid}}.""" + """unity.export results holding the exported episode parts, one bundle per episode and a unit per part: {asset: + {part: oid}}.""" oids = {} for n, (asset, parts) in enumerate(sorted(EPISODES.items())): - t = Task("unity.export", 1, f"adv_{asset}", {}, {}, ()) - arts, items = [], [] - for m, part in enumerate(sorted(parts), 1): - oid = contract.object_id(f"CAB-{n:04x}", 10 * n + m) - role = voices.STORY_OBJECTS[part][1] - data = part_bytes(asset, part) - arts.append( - contract.artifact( - contract.artifact_id(oid, role), - depot.add(data, "json" if part == "script" else "txt"), - contract.provenance(t), - {"kind": "unity.object", "format": "json"}, - ) - ) - items.append(contract.item(oid, "exported", artifacts=[arts[-1]["id"]])) - oids.setdefault(asset, {})[part] = oid - res = contract.result(t, arts, items) - depot.commit(res) - env.done(t.id, res["key"]) + file, bundle = f"CAB-{n:04x}", f"adv_{asset}" + pids = {part: 10 * n + m for m, part in enumerate(sorted(parts), 1)} + unit_results.add_census(env, bundle, file, [(pid, voices.STORY_OBJECTS[p][0]) for p, pid in pids.items()]) + for part, pid in pids.items(): + cls, role = voices.STORY_OBJECTS[part] + ext = "json" if part == "script" else "txt" + semantics = {"kind": "unity.object", "format": "json"} + unit_results.add_unit(env, bundle, file, pid, [(role, part_bytes(asset, part), ext, semantics)], cls) + oids.setdefault(asset, {})[part] = contract.object_id(file, pid) return oids @@ -556,7 +547,8 @@ def test_stage_reads_the_exported_episodes(tmp_path): shas = {p["sha256"] for parts in eps.values() for p in parts.values()} assert sorted(i.role for i in task.inputs if i.role.startswith("adv:")) == sorted(f"adv:{s}" for s in shas) deps = stage.depends("main", env) - assert sorted(d for d in deps if d.startswith("unity.export:")) == sorted(f"unity.export:adv_{a}" for a in EPISODES) + parts = [f"unity.export:adv_{a}:{p['object'].split(':')[1]}" for a, ps in eps.items() for p in ps.values()] + assert sorted(d for d in deps if d.startswith("unity.export:")) == sorted(parts) # the unit of each part ex = execute(task, depot, registry(stage)) assert (ex.status, ex.result_status) == ("ran", "ok") res = depot.result(task.key) @@ -576,7 +568,7 @@ def test_stage_reads_the_exported_episodes(tmp_path): def test_stage_waits_for_the_exports(tmp_path): depot = Depot(tmp_path / "s") env = story_env(depot, tmp_path) - env.waiting("unity.export:adv_adv_script_x_01") + env.waiting("unity.export:adv_adv_script_x_01:11") # the unit of its script with pytest.raises(Pending, match="unity.export"): describe(voices.VoiceStage(), "main", None, env) From 5197557a1559c8b91365fdeb41a05c4ec9644bf1 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 04:33:51 +0800 Subject: [PATCH 13/91] feat(assemble): add product stages assembled from the exported objects A product stage (nnnotes.pipeline.assemble.ProductStage) makes one task per subject from root objects: the objects of the catalog keys it gives for the subject, through the address table, and the object ids it names. Its task reaches the units holding its roots and every unit an item of a reached unit references, through the unity.export results; a sprite whose atlas is in another bundle takes its image from sprite.crop. - The task's input is the object table of what it reaches (nnnotes.objects/1): each object's class, name, item status and files, the object holding a contained one, and the referenced objects no census holds. Its key therefore follows exactly the files it can read. - The result holds the product's own files as `#`, its object table as `#objects` and one item for the subject. - The unit request takes the fact `roots` ({keys, objects}): `assemble.request` gives it from the products asked for, so a run exports the units their roots reach and nothing else of their bundles. - layout.product_entries and layout.product_manifest place products for a player: every file at `assets/.` and a manifest (nnnotes.product/1) per product listing its files and its objects' files. - view.voices reads the address table through the same cached reader. Co-Authored-By: Claude Opus 5.5 --- docs/contracts.md | 8 + docs/products.md | 83 ++++++ docs/schemas.md | 2 + docs/stages.md | 8 +- mkdocs.yml | 1 + src/nnnotes/pipeline/assemble.py | 304 +++++++++++++++++++++ src/nnnotes/pipeline/contract.py | 2 + src/nnnotes/pipeline/layout.py | 57 ++++ src/nnnotes/pipeline/units.py | 64 ++++- src/nnnotes/pipeline/voices.py | 16 +- src/nnnotes/schemas/objects.schema.json | 89 ++++++ src/nnnotes/schemas/product.schema.json | 86 ++++++ tests/pipeline/test_assemble.py | 343 ++++++++++++++++++++++++ 13 files changed, 1031 insertions(+), 32 deletions(-) create mode 100644 docs/products.md create mode 100644 src/nnnotes/pipeline/assemble.py create mode 100644 src/nnnotes/schemas/objects.schema.json create mode 100644 src/nnnotes/schemas/product.schema.json create mode 100644 tests/pipeline/test_assemble.py diff --git a/docs/contracts.md b/docs/contracts.md index 369222e..dcd8fe2 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -202,6 +202,13 @@ bytes nor keys; the file names of `original` and `cas` both follow from the reco `original` manifest alone. Documents derived for `original` from records and paths (the views with the files of their objects) are the exception: `original` has the derived documents, `cas` the artifacts they are made from. +## Product manifest: nnnotes.product/1 + +A product layout writes, for each product of `pipeline.assemble`, a manifest +([product.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/product.schema.json)): the product's +files and the files of the objects it uses, each a content-addressed `assets/.`. The product's object +table is `nnnotes.objects/1`. See [products](products.md). + ## Reports - Coverage, `nnnotes.coverage/1` ([coverage.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/coverage.schema.json)): per class the census @@ -224,6 +231,7 @@ Documents written by particular stages (artifacts of no object: their ids are `< | `nnnotes.scripts/1` | `link.scripts` | [scripts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/scripts.schema.json) | | `nnnotes.addresses/1` | `link.addresses` | [addresses.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/addresses.schema.json) | | `nnnotes.view/1` | `view.` | [view.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/view.schema.json) | +| `nnnotes.objects/1` | `pipeline.assemble` (a product's object table) | [objects.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/objects.schema.json) | The catalog versions of a depot, `nnnotes.catalogs/1` ([catalogs.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalogs.schema.json), `/catalogs/index.json`), and the difference between two of them, `nnnotes.catalog-diff/1` diff --git a/docs/products.md b/docs/products.md new file mode 100644 index 0000000..8606ccd --- /dev/null +++ b/docs/products.md @@ -0,0 +1,83 @@ +# Products + +A product is something a player or a site reads as a whole: a Live2D model, a story episode, a live. Products are +assembled from the results of `unity.export` ([stages](stages.md)), never from a bundle: by the time a product is +made, every object it uses has been converted once, and the product combines those conversions with what it adds +itself. + +## Product stages + +A product stage (`nnnotes.pipeline.assemble.ProductStage`) has one task per subject. Its task starts from root +objects: + +- the objects of the catalog keys the stage gives for the subject (`keys`), through the address table of + `link.addresses`; +- the object ids the stage names besides (`root_objects`). + +Both are read from the planning facts alone, so a command can ask for what its products need before anything runs: +`assemble.request(stages, products, env)` gives the fact `roots` (`{"keys", "objects"}`), and the request of +`unity.export` then holds the units of those objects and every unit reachable from them through the census `refs` +([units](stages.md#units)), and nothing else of their bundles. + +What a task reaches: the units holding its roots, then every unit that an item of a reached unit references (the +item's `refs`), through the `unity.export` results. A sprite whose atlas is in another bundle has its `image` from the +`sprite.crop` result of that atlas. A reached unit that the request did not export is an error of the request; an +object that a reached object references and no census holds is listed as `missing`. + +| Part of the task | | +|---|---| +| inputs | `objects`: the object table of what it reaches (`nnnotes.objects/1`); the stage's own inputs (master tables, data its parameters name) | +| depends | `link.addresses` (when it has keys), the `unity.export` and `sprite.crop` tasks whose results it read | +| artifacts | `#`: the product's own files; `#objects`: the object table of the objects its files use (the whole table unless the stage narrows it) | +| item | one for the subject: `exported`, or `unsupported` with a reason code | + +The key holds the object table, so a product is assembled again exactly when a file of an object it reaches +changes, or what it reaches changes. + +## Object table: nnnotes.objects/1 + +([objects.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/objects.schema.json)) + +```json +{"schema": "nnnotes.objects/1", + "roots": ["CAB-…:-4153…"], + "objects": { + "CAB-…:-4153…": {"status": "exported", "class": "GameObject", "name": "model", + "files": {"json": {"sha256": "…", "size": 5120, "ext": "json"}}}, + "CAB-…:12": {"status": "contained", "class": "Transform", "in": "CAB-…:-4153…"}, + "CAB-…:40": {"status": "exported", "class": "Texture2D", "name": "texture_00", + "files": {"image": {"sha256": "…", "size": 812345, "ext": "png"}, + "json": {"sha256": "…", "size": 640, "ext": "json"}}}}, + "missing": []} +``` + +Each object has the status of its item (the most severe when two stages made items for it), its class and name +where known, its files by role (the artifacts of its items), the object whose file holds it (`in`, a contained +object), or the reason code of a generic, unsupported or failed object. + +## Product layout + +A product layout places products for a player: + +- every file of a product and every file of the objects of its object table at `assets/.` (a file + several products or objects share is written once); +- each product's manifest at `/.json` (the stage's `manifest(subject)`, by default + `/.json` with the subject escaped as one path segment). + +## Product manifest: nnnotes.product/1 + +([product.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/product.schema.json)) + +```json +{"schema": "nnnotes.product/1", "product": "live2d.model", "subject": "…", + "files": {"model.json": {"asset": "assets/.json", "size": 2048}}, + "objects": { + "CAB-…:-4153…": {"class": "GameObject", "name": "model", + "files": {"json": {"asset": "assets/.json", "size": 5120}}}, + "CAB-…:12": {"class": "Transform", "in": "CAB-…:-4153…"}}} +``` + +`files` are the product's own files by path; `objects` are the objects of its object table that have files, or are +contained in an object that has one. An asset path is relative to the directory above the manifest's directory, as +in the chart manifests of ournotes-player. A reference in an object's JSON (`{"$ref": {"file", "pathId"}}`) names +the object `:` of `objects`. diff --git a/docs/schemas.md b/docs/schemas.md index 0fa3405..fd0a6b7 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -26,7 +26,9 @@ schemas.load("plan") # the schema as a dict | `failures` | `nnnotes.failures/1` | the failed tasks and items of a run | | `layout` | `nnnotes.layout/1` | the files a layout owns in its output directory | | `music-data` | `nnnotes.music-data/2` | every song and chart of one master data version ([music data](music-data.md)) | +| `objects` | `nnnotes.objects/1` | the objects a product reaches: their class, item status and files ([products](products.md)) | | `plan` | `nnnotes.plan/1` | what a run would do and why (`depot plan`) | +| `product` | `nnnotes.product/1` | a product's manifest: its files and the files of the objects it uses, as content-addressed assets ([products](products.md)) | | `result` | `nnnotes.result/1` | the result of a task, a function of its key alone | | `run` | `nnnotes.run/1` | a run manifest: context, selection, tasks with keys and statuses, layouts | | `scripts` | `nnnotes.scripts/1` | every MonoScript of the censuses, by object id | diff --git a/docs/stages.md b/docs/stages.md index 7d46629..bc7eb16 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -109,10 +109,10 @@ hierarchy. Its subject is the stable address of its root object, `#", the +table of the objects they use as "#objects", and one item for the subject. A product layout +(layout.product_entries) places the files and the objects' files and writes the product's manifest. +""" + +from __future__ import annotations + +from collections import OrderedDict +from dataclasses import dataclass, field + +from ..unity.atoms import Unsupported +from . import contract, units +from .contract import Cost, Input +from .layout import PRODUCT_TABLE, escape_segment +from .orchestrate import SEVERITY +from .stages import Output, Stage + +EXPORT, CROP = "unity.export", "sprite.crop" +ADDRESSES = units.ADDRESSES +TABLE_KIND = "product.objects" + + +def request(stages: dict, products: dict, env) -> dict: + """The fact "roots" (pipeline.units) of a request for `products` {stage name: subjects}: the catalog keys and the + root objects of every subject, sorted.""" + keys, objects = set(), set() + for name, subjects in sorted(products.items()): + stage = stages[name] + for s in subjects: + keys.update(stage.keys(s, env)) + objects.update(stage.root_objects(s, env)) + return {"keys": sorted(keys), "objects": sorted(objects)} + + +def _entry(item: dict, arts: dict) -> dict: + """The object table entry of a unity.export or sprite.crop item.""" + e: dict = {"status": item["status"]} + cls, name, files = item.get("class"), None, {} + for aid in item.get("artifacts", ()): + a = arts[aid] + c = a["content"] + files[contract.parse_artifact_id(aid)[1]] = {"sha256": c["sha256"], "size": c["size"], "ext": c["ext"]} + obj = (a.get("provenance") or {}).get("object") or {} + cls, name = cls or obj.get("class"), name or obj.get("name") + if cls: + e["class"] = cls + if name: + e["name"] = name + if files: + e["files"] = files + if "in" in item: + e["in"] = contract.parse_artifact_id(item["in"])[0] + if "reason" in item: + e["reason"] = item["reason"]["code"] + return e + + +def _merge(e: dict, other: dict) -> None: + """Add the files of another item of the same object; the status becomes the more severe.""" + e.setdefault("files", {}).update(other.get("files", {})) + if SEVERITY.index(other["status"]) < SEVERITY.index(e["status"]): + e["status"] = other["status"] + if "reason" in other: + e["reason"] = other["reason"] + + +@dataclass +class Reach: + """What a product reaches: its object table (nnnotes.objects/1), the tasks whose results it read, the bytes of + its objects' files.""" + + table: dict + read: list = field(default_factory=list) + size: int = 0 + + +class _Results: + """The items and artifact records of the results read, by result key (a result never changes).""" + + def __init__(self, size: int = 4096): + self.size, self._docs = size, OrderedDict() + + def get(self, env, tid: str) -> tuple[list, dict]: + key = env.key(tid) + doc = self._docs.get(key) + if doc is None: + res = env.result(tid) + doc = self._docs[key] = (res["items"], {a["id"]: a for a in res["artifacts"]}) + while len(self._docs) > self.size: + self._docs.popitem(last=False) + else: + self._docs.move_to_end(key) + return doc + + +def reach(env, roots, results: _Results | None = None) -> Reach: + """The objects reachable from the object ids `roots` (module documentation).""" + results = results or _Results() + table = units.table(env) + todo, seen, missing = [], set(), set() + for o in roots: + s = table.subject_of(o) + if s is None: + missing.add(o) + else: + todo.append(s) + objects: dict[str, dict] = {} + read = [] + crops: dict[str, list] = {} # atlas object id -> sprite object ids + while todo: + s = todo.pop() + if s in seen: + continue + seen.add(s) + table.unit(s) # ValueError: a unit the request does not export + tid = contract.task_id(EXPORT, s) + items, arts = results.get(env, tid) + read.append(tid) + for it in items: + oid = it["object"] + objects[oid] = _entry(it, arts) + for aid in it.get("artifacts", ()): + facts = arts[aid]["semantics"].get("facts") or {} + if facts.get("crop") == CROP: + crops.setdefault(facts["atlas"], []).append(oid) + for r in it.get("refs", ()): + t = table.subject_of(r) + if t is None: + missing.add(r) + elif t not in seen: + todo.append(t) + for atlas, sprites in sorted(crops.items()): + s = table.subject_of(atlas) + tid = contract.task_id(CROP, s) if s is not None else None + if tid is None or not env.known(tid): # the atlas is not exported: its sprites have no image + missing.add(atlas) + continue + items, arts = results.get(env, tid) + read.append(tid) + made = {it["object"]: it for it in items} + for oid in sprites: + if oid in made: + _merge(objects[oid], _entry(made[oid], arts)) + doc = { + "schema": contract.OBJECTS, + "roots": sorted(set(roots)), + "objects": dict(sorted(objects.items())), + "missing": sorted(missing), + } + size = sum(f["size"] for e in objects.values() for f in e.get("files", {}).values()) + return Reach(doc, sorted(read), size) + + +class Objects: + """The object table of a product task and the bytes of its objects' files (from the depot).""" + + def __init__(self, doc: dict, depot): + self.doc, self.depot = doc, depot + self.roots: list[str] = doc["roots"] + self._objects: dict = doc["objects"] + self._json: dict = {} + + def __contains__(self, oid: str) -> bool: + return oid in self._objects + + def ids(self, cls: str | None = None) -> list[str]: + """The object ids of the table (of class `cls`), sorted.""" + return [o for o, e in self._objects.items() if cls is None or e.get("class") == cls] + + def info(self, oid: str) -> dict: + try: + return self._objects[oid] + except KeyError: + raise KeyError(f"object {oid} is not in the product's object table") from None + + def bytes(self, oid: str, role: str) -> bytes: + files = self.info(oid).get("files") or {} + if role not in files: + raise KeyError(f"object {oid} has no {role!r} file") + return self.depot.read(files[role]["sha256"]) + + def json(self, oid: str, role: str = "json"): + """The decoded JSON of a file of an object (kept for the task).""" + k = (oid, role) + if k not in self._json: + self._json[k] = contract.loads(self.bytes(oid, role)) + return self._json[k] + + +@dataclass +class Assembly: + """What `assemble` makes: the product's files {path: (bytes, extension, kind)} and the object ids its files use + (None: every object of the table).""" + + files: dict + objects: set | None = None + + +class ProductStage(Stage): + """Base of the product stages (module documentation). Subclasses set `name`, `version` and `KIND`, and define + `keys` and/or `root_objects` and `assemble`; `subjects` defaults to the subjects of the fact "products".""" + + after = (ADDRESSES, EXPORT, CROP) + KIND = "product" + + def __init__(self): + self._memo: dict = {} + self._results = _Results() + + def __getstate__(self): + return {k: v for k, v in self.__dict__.items() if k not in ("_memo", "_results")} + + def __setstate__(self, state): + self.__dict__.update(state) + self._memo, self._results = {}, _Results() + + # ---------------------------------------------------------------- what a subclass defines + def subjects(self, env) -> list[str]: + return sorted((env.facts.get("products") or {}).get(self.name, ())) + + def keys(self, subject: str, env) -> list[str]: + """The catalog keys whose objects are roots of `subject` (from the facts alone).""" + return [] + + def root_objects(self, subject: str, env) -> list[str]: + """Object ids that are roots of `subject` besides those of its keys (from the facts alone).""" + return [] + + def more_inputs(self, subject: str, env) -> list[Input]: + """Inputs besides the object table.""" + return [] + + def assemble(self, task, objects: Objects) -> Assembly: + raise NotImplementedError + + def manifest(self, subject: str) -> str: + """The path of the subject's manifest in a product layout: one directory, then a file.""" + return f"{self.name}/{escape_segment(subject)}.json" + + # ---------------------------------------------------------------- planning + def reach(self, subject: str, env) -> Reach: + """What `subject` reaches (kept while the results it read are the same).""" + roots = units.root_objects(env, self.keys(subject, env), self.root_objects(subject, env)) + sig = (tuple(roots), id(units.table(env))) + memo = self._memo.get(subject) + if memo is not None and memo[0] == sig and all(env.key(t) == k for t, k in memo[1]): + return memo[2] + r = reach(env, roots, self._results) + self._memo[subject] = (sig, [(t, env.key(t)) for t in r.read], r) + return r + + def inputs(self, subject: str, env) -> list[Input]: + data = contract.encode(self.reach(subject, env).table) + table = Input("objects", env.depot.put(data), len(data), None, ({"kind": "depot"},)) + return [table, *self.more_inputs(subject, env)] + + def depends(self, subject: str, env) -> list[str]: + own = [contract.task_id(ADDRESSES, s) for s in sorted(env.fact("catalogs"))] if self.keys(subject, env) else [] + return own + self.reach(subject, env).read + + def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: + """From the bytes of the objects' files: a product reads each of them about once.""" + size = self.reach(subject, env).size + sum(i.size for i in inputs) + return Cost(0.02 + size / 100e6, (64 << 20) + 2 * size) + + # ---------------------------------------------------------------- running + def run(self, task, depot) -> Output: + data = depot.input_bytes(task.input("objects")) + doc = contract.loads(data) + try: + made = self.assemble(task, Objects(doc, depot)) + except Unsupported as e: + why = contract.reason(e.code, e.message) + return Output([], [contract.item(task.id, "unsupported", why=why, cls=self.KIND)]) + prov = contract.provenance(task) + arts = [] + for path, (body, ext, kind) in sorted(made.files.items()): + if path == PRODUCT_TABLE: + raise ValueError(f"{task.id}: a product file cannot be named {PRODUCT_TABLE!r}") + aid = contract.artifact_id(task.id, path) + arts.append(contract.artifact(aid, depot.add(body, ext), prov, {"kind": kind})) + if made.objects is not None: + unknown = sorted(set(made.objects) - set(doc["objects"])) + if unknown: + raise ValueError(f"{task.id}: objects not in the table: {', '.join(unknown[:5])}") + doc = dict(doc, objects={o: e for o, e in doc["objects"].items() if o in made.objects}) + data = contract.encode(doc) + aid = contract.artifact_id(task.id, PRODUCT_TABLE) + arts.append(contract.artifact(aid, depot.add(data, "json"), prov, {"kind": TABLE_KIND})) + item = contract.item(task.id, "exported", artifacts=[a["id"] for a in arts], cls=self.KIND) + return Output(arts, [item]) diff --git a/src/nnnotes/pipeline/contract.py b/src/nnnotes/pipeline/contract.py index 97ff7f6..bf25f02 100644 --- a/src/nnnotes/pipeline/contract.py +++ b/src/nnnotes/pipeline/contract.py @@ -55,6 +55,8 @@ CATALOG_INDEX = "nnnotes.catalog-index/1" CENSUS = "nnnotes.census/2" VIEW = "nnnotes.view/1" +OBJECTS = "nnnotes.objects/1" +PRODUCT = "nnnotes.product/1" # item statuses of a result: every object a task covers has exactly one ITEM_STATUSES = ("exported", "contained", "generic", "unsupported", "failed") diff --git a/src/nnnotes/pipeline/layout.py b/src/nnnotes/pipeline/layout.py index 3fbb13d..b20eb4d 100644 --- a/src/nnnotes/pipeline/layout.py +++ b/src/nnnotes/pipeline/layout.py @@ -27,6 +27,9 @@ - paths that coincide ignoring case (or a file named like another path's directory) keep the first by path and id; each other gets `~` before its extension and is reported; artifacts with the same path and the same content share the file +`product` (product_entries): the products of pipeline.assemble. Every file of a product and of the objects of its +object table is at `assets/.`; each product has a manifest (nnnotes.product/1, product_manifest) at +`/.json` (ProductStage.manifest), whose asset paths are relative to the directory above its own. So every path ends with its artifact's extension, and `cas` follows from the `original` manifest alone; `original` follows from `cas` plus the results (the records). Documents a stage derives for `original` from its records and the paths (the views with the files of their objects) are the exception: `original` has the derived documents, @@ -63,6 +66,7 @@ MANIFEST = "manifest.json" JOURNAL = ".nnnotes-layout-journal.json" LAYOUTS = ("original", "cas") +PRODUCT_TABLE = "objects" # the role of a product's object table (nnnotes.objects/1) among its artifacts PLACEMENTS = ("clone", "hard", "copy") # how a file is made from its byte source LINKS = ("auto", *PLACEMENTS) # what a run may ask for: auto probes FICLONE = 0x40049409 # Linux ioctl: make a file share another's blocks @@ -353,6 +357,59 @@ def _resolve(placed: list) -> tuple[list[dict], dict]: } +def _asset(c: dict) -> dict: + return {"asset": f"assets/{c['sha256']}.{c['ext']}", "size": c["size"]} + + +def product_manifest(task: str, result: dict, table: dict) -> dict: + """The manifest (nnnotes.product/1) of a product: its files {path: {asset, size}} and the objects of its object + table `table` that have files ({class, name, files: {role: {asset, size}}}) or are contained in one ({class, + in}).""" + stage, subject = contract.parse_task_id(task) + files = {} + for a in result["artifacts"]: + role = contract.parse_artifact_id(a["id"])[1] + if role != PRODUCT_TABLE: + files[role] = _asset(a["content"]) + objects = {} + for oid, o in table["objects"].items(): + e = {k: o[k] for k in ("class", "name") if k in o} + if o.get("files"): + e["files"] = {r: _asset(f) for r, f in sorted(o["files"].items())} + elif "in" in o: + e["in"] = o["in"] + else: + continue + objects[oid] = e + return {"schema": contract.PRODUCT, "product": stage, "subject": subject, "files": files, "objects": objects} + + +def product_entries(products) -> list[dict]: + """The entries of the `product` layout: `products` [(manifest path, the manifest's record {id, sha256, size}, + result, object table)].""" + out: dict[tuple, dict] = {} + for path, manifest, result, table in products: + segs = path.split("/") + if len(segs) != 2 or not all(segs) or not segs[1].endswith(".json"): + raise ValueError(f"product manifest path {path!r}: expected /.json") + out[(path, manifest["id"])] = { + "path": path, + "sha256": manifest["sha256"], + "size": manifest["size"], + "id": manifest["id"], + } + for a in result["artifacts"]: + if contract.parse_artifact_id(a["id"])[1] != PRODUCT_TABLE: + e = _entry(_asset(a["content"])["asset"], a) + out[(e["path"], e["id"])] = e + for oid, o in table["objects"].items(): + for role, f in (o.get("files") or {}).items(): + aid = contract.artifact_id(oid, role) + p = f"assets/{f['sha256']}.{f['ext']}" + out[(p, aid)] = {"path": p, "sha256": f["sha256"], "size": f["size"], "id": aid} + return [out[k] for k in sorted(out)] + + def entries(name: str, sources, **kw) -> tuple[list[dict], dict]: """The entries and report of layout `name`.""" if name == "cas": diff --git a/src/nnnotes/pipeline/units.py b/src/nnnotes/pipeline/units.py index bfce293..a82dff1 100644 --- a/src/nnnotes/pipeline/units.py +++ b/src/nnnotes/pipeline/units.py @@ -5,11 +5,15 @@ path-id stable address of its root object (link.path_address: ":", with the file after the bundle in a bundle of several serialized files), which stays when the bundle's content changes; its object ids do not. -The request (`table`): every unit holding an object of the selected bundles (fact "selected"), and every unit -reachable from those through what their objects reference (census `refs`) across the censuses of the bundles of -the fact "bundles"; with the fact "classes" (a list of class names; absent or None: every class), only the units -whose root object is of those classes. A bundle whose census failed has no units; a reference into a file no -census holds is left out. +The request (`table`): every unit holding an object of the selected bundles (fact "selected") or a root object of the +fact "roots", and every unit reachable from those through what their objects reference (census `refs`) across the +censuses of the bundles of the fact "bundles"; with the fact "classes" (a list of class names; absent or None: +every class), only the units whose root object is of those classes. A bundle whose census failed has no units; a +reference into a file no census holds is left out. + +The fact "roots" (absent: none) is {"keys": catalog keys, "objects": object ids}: the objects of each key (all those +the address table of link.addresses lists for it; a key without objects adds none) and the objects named. The +products of a request start from them (pipeline.assemble). """ from __future__ import annotations @@ -17,11 +21,12 @@ from collections import Counter from dataclasses import dataclass -from . import contract, link +from . import contract, link, views from .census import CensusStage from .stages import Pending CENSUS = CensusStage.name +ADDRESSES = views.ADDRESSES_STAGE @dataclass(frozen=True) @@ -131,9 +136,9 @@ def _census_units(stable: str, doc: dict, need_refs: bool): return units, of, {f["name"]: stable for f in doc["files"]}, refs -def build(censuses, bundles, selected, classes=None) -> Units: - """The Units of a request over `censuses` [(stable bundle name, census document)] of the bundles `bundles` - (module documentation).""" +def build(censuses, bundles, selected, classes=None, roots=()) -> Units: + """The Units of a request over `censuses` [(stable bundle name, census document)] of the bundles `bundles`, from + the units of the bundles `selected` and of the objects `roots` (module documentation).""" selected = set(selected) every = selected >= set(bundles) # no reference can add a unit: none is read units: dict[str, Unit] = {} @@ -152,6 +157,7 @@ def build(censuses, bundles, selected, classes=None) -> Units: want = set(units) else: want = {s for s, u in units.items() if u.bundle in selected} + want.update(of[o] for o in roots if o in of) todo = list(want) while todo: for oid in refs.get(todo.pop(), ()): @@ -166,12 +172,36 @@ def build(censuses, bundles, selected, classes=None) -> Units: _MEMO: list = [] # [(signature, Units)] of the last table +_ADDRESSES: dict = {} # {sha256: AddressIndex} of the last address table read + + +def address_index(env, subject: str) -> views.AddressIndex: + """The AddressIndex of the result of link.addresses for the catalog `subject` (Pending while it has not run).""" + sha = env.artifact(contract.task_id(ADDRESSES, subject), "addresses")["content"]["sha256"] + index = _ADDRESSES.get(sha) + if index is None: + index = views.AddressIndex.from_addresses(contract.loads(env.depot.read(sha))) + _ADDRESSES.clear() + _ADDRESSES[sha] = index + return index + + +def root_objects(env, keys=(), objects=()) -> list[str]: + """The object ids of the catalog `keys` (through the address tables of the fact "catalogs") and the `objects`, + sorted.""" + out = set(objects) + if keys: + for subject in sorted(env.fact("catalogs")): + index = address_index(env, subject) + for k in keys: + out.update(o.id for o in index.objects(k) or ()) + return sorted(out) def table(env) -> Units: """The Units of the request of a planning environment: the censuses of the fact "bundles" (Pending while one - has not run; a failed one is skipped), the facts "selected" and "classes". Kept for the planning state it was - made in.""" + has not run; a failed one is skipped), the facts "selected", "roots" (Pending while link.addresses has not run, + when it names keys) and "classes". Kept for the planning state it was made in.""" bundles = sorted(env.fact("bundles")) keys = [] for b in bundles: @@ -182,7 +212,15 @@ def table(env) -> Units: if not p.failed: raise classes = env.facts.get("classes") - sig = (id(env.depot), tuple(keys), tuple(env.fact("selected")), None if classes is None else tuple(classes)) + request = env.facts.get("roots") or {} + roots = root_objects(env, request.get("keys", ()), request.get("objects", ())) + sig = ( + id(env.depot), + tuple(keys), + tuple(env.fact("selected")), + None if classes is None else tuple(classes), + tuple(roots), + ) if _MEMO and _MEMO[0][0] == sig: return _MEMO[0][1] @@ -193,6 +231,6 @@ def censuses(): rec = next(a for a in env.depot.committed(key, tid)["artifacts"] if a["id"] == aid) yield b, contract.loads(env.depot.read(rec["content"]["sha256"])) - t = build(censuses(), bundles, env.fact("selected"), classes) + t = build(censuses(), bundles, env.fact("selected"), classes, roots) _MEMO[:] = [(sig, t)] return t diff --git a/src/nnnotes/pipeline/voices.py b/src/nnnotes/pipeline/voices.py index 147b060..d6c519c 100644 --- a/src/nnnotes/pipeline/voices.py +++ b/src/nnnotes/pipeline/voices.py @@ -820,20 +820,6 @@ def _sheet_name(name: str) -> str | None: return name if "/" not in name else None -_address_cache: dict = {} # content id of an address table -> its AddressIndex, the latest one - - -def _address_index(env, tid: str): - """The AddressIndex of the result of the link.addresses task `tid`.""" - sha = env.artifact(tid, "addresses")["content"]["sha256"] - hit = _address_cache.get(sha) - if hit is None: - hit = views.AddressIndex.from_addresses(contract.loads(env.depot.read(sha))) - _address_cache.clear() - _address_cache[sha] = hit - return hit - - class VoiceStage(Stage): """view.voices (module documentation). Artifact "#view" (nnnotes.voices/1); facts rows, entries (rows per status), gaps, unreferenced (streams of the decoded sheets no row reaches). No items.""" @@ -929,7 +915,7 @@ def _plan(self, subject: str, env) -> tuple: ) if self._memo is not None and self._memo[0] == sig: return self._memo[1] - index = _address_index(env, link) + index = units.address_index(env, subject) tables = { i.role[len(views.MASTER_ROLE) :]: views._cached_rows(i.sha256, Path(i.locators[0]["path"]).read_bytes) for i in masters diff --git a/src/nnnotes/schemas/objects.schema.json b/src/nnnotes/schemas/objects.schema.json new file mode 100644 index 0000000..b9d006e --- /dev/null +++ b/src/nnnotes/schemas/objects.schema.json @@ -0,0 +1,89 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:nnnotes:schema:objects", + "title": "nnnotes.objects/1", + "description": "The object table of a product (pipeline.assemble): the objects reachable from its roots through the unity.export results, with their item status and files.", + "type": "object", + "required": [ + "schema", + "roots", + "objects", + "missing" + ], + "properties": { + "schema": { + "const": "nnnotes.objects/1" + }, + "roots": { + "type": "array", + "items": { + "type": "string" + } + }, + "objects": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "status" + ], + "properties": { + "status": { + "enum": [ + "exported", + "contained", + "generic", + "unsupported", + "failed" + ] + }, + "class": { + "type": "string" + }, + "name": { + "type": "string" + }, + "files": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "sha256", + "size", + "ext" + ], + "properties": { + "sha256": { + "$ref": "urn:nnnotes:schema:common#/$defs/sha256" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "ext": { + "type": "string", + "pattern": "^[0-9a-z]+$" + } + } + } + }, + "in": { + "type": "string", + "description": "The object whose file holds a contained object." + }, + "reason": { + "type": "string", + "description": "The reason code of a generic, unsupported or failed object." + } + } + } + }, + "missing": { + "type": "array", + "description": "Objects a reached object references that no census holds.", + "items": { + "type": "string" + } + } + } +} diff --git a/src/nnnotes/schemas/product.schema.json b/src/nnnotes/schemas/product.schema.json new file mode 100644 index 0000000..bb01211 --- /dev/null +++ b/src/nnnotes/schemas/product.schema.json @@ -0,0 +1,86 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:nnnotes:schema:product", + "title": "nnnotes.product/1", + "description": "The manifest of a product in a product layout: its files and the files of the objects it uses, each a content-addressed asset whose path is relative to the directory above the manifest's.", + "type": "object", + "required": [ + "schema", + "product", + "subject", + "files", + "objects" + ], + "properties": { + "schema": { + "const": "nnnotes.product/1" + }, + "product": { + "$ref": "urn:nnnotes:schema:common#/$defs/stageName" + }, + "subject": { + "type": "string" + }, + "files": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/asset" + } + }, + "objects": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "class": { + "type": "string" + }, + "name": { + "type": "string" + }, + "files": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/asset" + } + }, + "in": { + "type": "string" + } + }, + "oneOf": [ + { + "required": [ + "files" + ] + }, + { + "required": [ + "in" + ] + } + ] + } + } + }, + "$defs": { + "asset": { + "type": "object", + "required": [ + "asset", + "size" + ], + "additionalProperties": false, + "properties": { + "asset": { + "type": "string", + "pattern": "^assets/[0-9a-f]{64}\\.[0-9a-z]+$" + }, + "size": { + "type": "integer", + "minimum": 0 + } + } + } + } +} diff --git a/tests/pipeline/test_assemble.py b/tests/pipeline/test_assemble.py new file mode 100644 index 0000000..547d509 --- /dev/null +++ b/tests/pipeline/test_assemble.py @@ -0,0 +1,343 @@ +"""Product stages (pipeline.assemble) over the fake bundles of test_objexport: a toy product rooted at a prefab and +at an atlas sprite, its object table, its result and the product layout.""" + +from __future__ import annotations + +import json + +import pytest + +from nnnotes.pipeline import assemble, contract, layout, link +from nnnotes.pipeline.contract import Task +from nnnotes.pipeline.depot import Depot +from nnnotes.pipeline.objexport import ExportStage, SpriteCropStage +from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry +from tests.pipeline.test_objexport import ATLAS_CAB, FAKE_READER, oid, planning_env + + +def addresses(depot, env, keys: dict) -> None: + """A committed link.addresses result whose table maps `keys` {catalog key: [object ids]}.""" + t = Task("link.addresses", 1, "main", {}, {}, ()) + doc = { + "schema": link.ADDRESSES, + "keys": {k: [{"objects": ids}] for k, ids in sorted(keys.items())}, + "objects": {}, + "files": {}, + "bundles": {}, + } + rec = contract.artifact( + contract.artifact_id(t.id, "addresses"), + depot.add(contract.encode(doc), "json"), + contract.provenance(t), + {"kind": "link.addresses", "format": "json"}, + ) + depot.commit(contract.result(t, [rec], [])) + env.done(t.id, t.key) + + +class Toy(assemble.ProductStage): + """A product whose roots are the objects of the keys of its subjects: `json` with the objects it uses and a + `data` file of theirs.""" + + name = "toy.thing" + version = 1 + KIND = "Toy" + KEYS: dict = {} + + def __init__(self, keys=None): + super().__init__() + self.KEYS = dict(Toy.KEYS if keys is None else keys) + + def keys(self, subject, env): + return [self.KEYS.get(subject, subject)] + + def assemble(self, task, objects): + files = { + "json": ( + contract.encode({"subject": task.subject, "objects": objects.ids()}), + "json", + "toy.json", + ) + } + for i, o in enumerate(sorted(objects.ids("TextAsset"))): + files[f"data/{i}"] = (objects.bytes(o, "data"), "txt", "toy.data") + return assemble.Assembly(files) + + +class Rooted(Toy): + """A Toy with root objects besides those of its keys.""" + + def __init__(self, keys, objects): + super().__init__(keys) + self.objects = list(objects) + + def root_objects(self, subject, env): + return self.objects + + +def env_for(depot, keys=None, names=("all", "atlas"), roots_objects=(), selected=None): + """A planning environment over the fake bundles with a committed address table and the facts of a request.""" + env = planning_env(depot, names, selected=selected) + env.facts["catalogs"] = {"main": []} + addresses(depot, env, keys if keys is not None else {}) + subjects = sorted(keys if keys is not None else {}) + env.facts["products"] = {"toy.thing": subjects} + env.facts["roots"] = {"keys": subjects, "objects": list(roots_objects)} + return env + + +def exported(depot, env) -> dict: + """Every unity.export task of the request, then every sprite.crop task: {stage: {subject: result}}.""" + out = {"unity.export": {}, "sprite.crop": {}} + for stage in (ExportStage(FAKE_READER), SpriteCropStage()): + for s in stage.subjects(env): + t = describe(stage, s, None, env) + execute(t, depot, registry(stage), force=True) + env.done(t.id, t.key) + out[stage.name][s] = depot.result(t.key) + return out + + +def product(depot, env, subject="card", keys=None, stage=None): + stage = stage or Toy(keys) + t = describe(stage, subject, None, env) + execute(t, depot, registry(stage), force=True) + return t, depot.result(t.key) + + +def table_of(depot, doc) -> dict: + rec = next(a for a in doc["artifacts"] if a["id"].endswith("#objects")) + return contract.loads(depot.read(rec["content"]["sha256"])) + + +# ---------------------------------------------------------------- the request and the table +def test_a_products_key_is_asked_for_before_anything_runs(tmp_path): + """request(): the roots fact of the products, from the keys of their subjects alone.""" + depot = Depot(tmp_path / "s") + stage = Toy({"m/a": "key/a", "m/b": "key/b"}) + stages = {"toy.thing": stage} + env = Env(depot, {"keys": {}}) + assert assemble.request(stages, {"toy.thing": ["m/b", "m/a"]}, env) == {"keys": ["key/a", "key/b"], "objects": []} + assert stage.subjects(Env(depot, {"products": {"toy.thing": ["m/b"]}})) == ["m/b"] + + +def test_the_object_table_is_what_the_root_reaches(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + table = table_of(depot, doc) + assert table["roots"] == [oid(60)] + assert table["missing"] and all(m.startswith("CAB-scripts:") for m in table["missing"]) # no census of it + # the hierarchy: the root exported, its members contained in its prefab + assert table["objects"][oid(60)] == { + "status": "exported", + "class": "GameObject", + "name": "root", + "files": {"json": oid_file(table, oid(60), "json")}, + } + assert table["objects"][oid(61)]["status"] == "contained" and table["objects"][oid(61)]["in"] == oid(60) + assert table["objects"][oid(65)]["class"] == "MonoBehaviour" + # what the prefab references across bundles: the texture and the material + assert {"json", "image"} <= set(table["objects"][oid(10)]["files"]) + assert oid(70) not in table["objects"] # not referenced + item = doc["items"][0] + assert (item["object"], item["status"], item["class"]) == (t.id, "exported", "Toy") + assert item["artifacts"] == sorted(f"{t.id}#{p}" for p in ("json", "objects")) # no TextAsset is reached + + +def oid_file(table, o, role): + return table["objects"][o]["files"][role] + + +def test_a_reached_object_that_no_census_holds_is_missing(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}, roots_objects=["CAB-none:1"]) + exported(depot, env) + _, doc = product(depot, env, "m/a", {"m/a": "key/a"}, stage=Rooted({"m/a": "key/a"}, ["CAB-none:1"])) + table = table_of(depot, doc) + assert "CAB-none:1" in table["missing"] and "CAB-none:1" in table["roots"] + + +def test_a_root_of_a_unit_the_request_leaves_out_is_an_error(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}, selected=["atlas"]) + env.facts["roots"] = {"keys": [], "objects": []} # the request does not start from the product's key + with pytest.raises(ValueError, match="no requested unit"): + describe(Toy({"m/a": "key/a"}), "m/a", None, env) + + +def test_the_request_roots_select_the_units_and_nothing_else(tmp_path): + """A request for one prefab exports its units, not every unit of its bundle.""" + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}, names=("all",), selected=[]) + subjects = ExportStage(FAKE_READER).subjects(env) + assert subjects == ["all:10", "all:60"] # the prefab and the texture it references + + +def test_sprite_images_come_from_the_crop_of_their_atlas(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(13)]}, names=("all", "atlas"), roots_objects=[f"{ATLAS_CAB}:99"]) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + table = table_of(depot, doc) + sprite = table["objects"][oid(13)] + assert set(sprite["files"]) == {"json", "image"} # the image from sprite.crop + assert table["objects"][f"{ATLAS_CAB}:99"]["class"] == "SpriteAtlas" + assert table["objects"][f"{ATLAS_CAB}:97"]["status"] == "exported" # the atlas references its own sprite + crop = [c for c in [*doc["artifacts"]] if c["semantics"]["kind"] == "sprite.meta"] + assert crop == [] # the sprite's meta belongs to its export result, not to the product + assert f"{t.id}#objects" in {a["id"] for a in doc["artifacts"]} + + +def test_depends_names_the_units_it_reads(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + deps = Toy({"m/a": "key/a"}).depends("m/a", env) + assert deps[0] == "link.addresses:main" + assert deps[1:] == ["unity.export:all:10", "unity.export:all:60"] # what it reads, not every unit + + +def test_inputs_and_the_key_follow_the_objects(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + stage = Toy({"m/a": "key/a"}) + t = describe(stage, "m/a", None, env) + assert [(i.role, i.size) for i in t.inputs] == [("objects", len(contract.encode(stage.reach("m/a", env).table)))] + assert t.context == {} and t.atoms == {} and t.params == {} + # another object of the same unit is not in the table: the key does not follow it + t2 = describe(Toy({"m/a": "key/a"}), "m/a", None, env) + assert t2.key == t.key and t2.cost.cpu_seconds > 0 + + +def test_a_product_waits_for_its_units(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + with pytest.raises(Pending, match="unity.export"): + describe(Toy({"m/a": "key/a"}), "m/a", None, env) + + +# ---------------------------------------------------------------- the result and the layout +def test_a_product_result_is_the_files_and_the_table(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + body = contract.loads( + depot.read(next(a for a in doc["artifacts"] if a["id"] == f"{t.id}#json")["content"]["sha256"]) + ) + assert body["subject"] == "m/a" and body["objects"] + assert doc["status"] == "ok" + again = execute(t, depot, registry(Toy({"m/a": "key/a"})), force=True) + assert again.status == "ran" and contract.encode(depot.result(t.key)) == contract.encode(doc) + + +def test_an_unsupported_product_is_an_item(tmp_path): + class Picky(Toy): + def assemble(self, task, objects): + raise assemble.Unsupported("unsupported.toy", "nothing to make") + + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}, stage=Picky({"m/a": "key/a"})) + assert doc["artifacts"] == [] + assert doc["items"] == [ + { + "object": t.id, + "status": "unsupported", + "class": "Toy", + "reason": {"code": "unsupported.toy", "message": "nothing to make"}, + } + ] + assert doc["status"] == "ok" # an unsupported product is a reported outcome, not a failure + + +def test_the_table_can_be_pruned_to_the_objects_the_product_uses(tmp_path): + class Pruned(Toy): + def assemble(self, task, objects): + made = super().assemble(task, objects) + return assemble.Assembly(made.files, {oid(60)}) + + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + _, doc = product(depot, env, "m/a", {"m/a": "key/a"}, stage=Pruned({"m/a": "key/a"})) + assert list(table_of(depot, doc)["objects"]) == [oid(60)] + + +def test_a_product_names_an_object_the_table_lacks(tmp_path): + class Wrong(Toy): + def assemble(self, task, objects): + made = super().assemble(task, objects) + return assemble.Assembly(made.files, {"CAB-none:1"}) + + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t = describe(Wrong({"m/a": "key/a"}), "m/a", None, env) + with pytest.raises(ValueError, match="not in the table"): + execute(t, depot, registry(Wrong({"m/a": "key/a"})), force=True) + + +# ---------------------------------------------------------------- the layout +def manifest_of(depot, t, doc): + """(entries, the manifest) of a product layout for one result.""" + table = table_of(depot, doc) + stage = Toy({"m/a": "key/a"}) + path = stage.manifest("m/a") + body = contract.encode(layout.product_manifest(t.id, doc, table)) + sha = depot.put(body) + rec = {"id": f"{t.id}#manifest", "sha256": sha, "size": len(body)} + return layout.product_entries([(path, rec, doc, table)]), contract.loads(body) + + +def test_the_product_layout_places_the_files_and_the_objects(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + entries, manifest = manifest_of(depot, t, doc) + place = {e["path"]: e["id"] for e in entries} + assert place["toy.thing/m%2Fa.json"] == f"{t.id}#manifest" + json_rec = next(a for a in doc["artifacts"] if a["id"] == f"{t.id}#json") + assert place[f"assets/{json_rec['content']['sha256']}.json"] == f"{t.id}#json" + assert place[f"assets/{oid_file(table_of(depot, doc), oid(10), 'image')['sha256']}.png"] == f"{oid(10)}#image" + assert manifest["files"]["json"] == { + "asset": f"assets/{json_rec['content']['sha256']}.json", + "size": json_rec["content"]["size"], + } + assert manifest["objects"][oid(10)]["class"] == "Texture2D" + assert manifest["objects"][oid(61)]["in"] == oid(60) + assert "assets" in manifest["objects"][oid(10)]["files"]["image"]["asset"] + # the manifest path is one directory and one file + assert len(entries) == len({(e["path"], e["id"]) for e in entries}) + + +def test_the_product_layout_refuses_a_nested_manifest_path(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + table = table_of(depot, doc) + body = contract.encode(layout.product_manifest(t.id, doc, table)) + rec = {"id": f"{t.id}#manifest", "sha256": depot.put(body), "size": len(body)} + with pytest.raises(ValueError, match="manifest path"): + layout.product_entries([("a/b/c.json", rec, doc, table)]) + + +def test_the_product_layout_writes_and_hits(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + entries, _ = manifest_of(depot, t, doc) + w = layout.materialize(tmp_path / "out", "product", {}, entries, layout.from_depot(depot)) + assert w["write"] == len(entries) and w["remove"] == 0 + assert (tmp_path / "out" / "toy.thing" / "m%2Fa.json").is_file() + assert layout.diff(tmp_path / "out", entries) == {"write": 0, "remove": 0, "keep": len(entries)} + assert json.loads((tmp_path / "out" / "toy.thing" / "m%2Fa.json").read_text(encoding="utf-8"))["schema"] == ( + contract.PRODUCT + ) From e5a0103c5dda01a7f3ddab65c71abe75ca4f519b Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 10:29:34 +0800 Subject: [PATCH 14/91] feat(link)!: find objects by their Resources.Load path The census lists the container of a ResourceManager (`resourcePaths`: the paths Resources.Load finds objects at, in stored order); unity.census is at version 4. The address table (nnnotes.addresses/2, link.addresses version 2) adds `resources`, each path with the object ids it names, and gives those objects stable addresses as container paths of the built-in file holding the ResourceManager. AddressIndex.resource looks a path up ignoring case. A request's roots and a product stage take resource paths besides catalog keys and object ids (`resources`), so a product can start from the objects a runtime loads by name, such as the materials of a model's masks. BREAKING CHANGE: address tables are nnnotes.addresses/2 with a required `resources` table, and census documents carry `resourcePaths`. Co-Authored-By: Claude Opus 5.5 --- docs/contracts.md | 2 +- docs/products.md | 8 +++-- docs/schemas.md | 2 +- docs/stages.md | 16 +++++---- src/nnnotes/pipeline/assemble.py | 31 +++++++++++------ src/nnnotes/pipeline/census.py | 18 ++++++++-- src/nnnotes/pipeline/link.py | 25 +++++++++----- src/nnnotes/pipeline/units.py | 21 +++++++----- src/nnnotes/pipeline/views.py | 35 ++++++++++++++----- src/nnnotes/schemas/addresses.schema.json | 18 ++++++++-- src/nnnotes/schemas/census.schema.json | 32 +++++++++++++++++- tests/cli/test_assets.py | 7 ++-- tests/helpers/fakeunity.py | 1 + tests/helpers/units.py | 1 + tests/pipeline/test_assemble.py | 41 ++++++++++++++++++++--- tests/pipeline/test_census.py | 22 +++++++++++- tests/pipeline/test_link.py | 41 +++++++++++++++++++++-- tests/pipeline/test_modelstages.py | 1 + tests/pipeline/test_views.py | 14 +++++--- 19 files changed, 266 insertions(+), 70 deletions(-) diff --git a/docs/contracts.md b/docs/contracts.md index dcd8fe2..7c63b3c 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -229,7 +229,7 @@ Documents written by particular stages (artifacts of no object: their ids are `< | `nnnotes.catalog-index/1` | `catalog.index` | [catalog-index.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalog-index.schema.json) | | `nnnotes.census/2` | `unity.census` | [census.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/census.schema.json) | | `nnnotes.scripts/1` | `link.scripts` | [scripts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/scripts.schema.json) | -| `nnnotes.addresses/1` | `link.addresses` | [addresses.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/addresses.schema.json) | +| `nnnotes.addresses/2` | `link.addresses` | [addresses.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/addresses.schema.json) | | `nnnotes.view/1` | `view.` | [view.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/view.schema.json) | | `nnnotes.objects/1` | `pipeline.assemble` (a product's object table) | [objects.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/objects.schema.json) | diff --git a/docs/products.md b/docs/products.md index 8606ccd..002ce50 100644 --- a/docs/products.md +++ b/docs/products.md @@ -12,10 +12,12 @@ objects: - the objects of the catalog keys the stage gives for the subject (`keys`), through the address table of `link.addresses`; +- the objects of the Resources.Load paths it gives (`resources`): paths of the ResourceManager in the APK's built-in + files, such as the materials a runtime loads by name, through the same table and compared ignoring case; - the object ids the stage names besides (`root_objects`). -Both are read from the planning facts alone, so a command can ask for what its products need before anything runs: -`assemble.request(stages, products, env)` gives the fact `roots` (`{"keys", "objects"}`), and the request of +All are read from the planning facts alone, so a command can ask for what its products need before anything runs: +`assemble.request(stages, products, env)` gives the fact `roots` (`{"keys", "resources", "objects"}`), and the request of `unity.export` then holds the units of those objects and every unit reachable from them through the census `refs` ([units](stages.md#units)), and nothing else of their bundles. @@ -27,7 +29,7 @@ object that a reached object references and no census holds is listed as `missin | Part of the task | | |---|---| | inputs | `objects`: the object table of what it reaches (`nnnotes.objects/1`); the stage's own inputs (master tables, data its parameters name) | -| depends | `link.addresses` (when it has keys), the `unity.export` and `sprite.crop` tasks whose results it read | +| depends | `link.addresses` (when it has keys or resource paths), the `unity.export` and `sprite.crop` tasks whose results it read | | artifacts | `#`: the product's own files; `#objects`: the object table of the objects its files use (the whole table unless the stage narrows it) | | item | one for the subject: `exported`, or `unsupported` with a reason code | diff --git a/docs/schemas.md b/docs/schemas.md index fd0a6b7..6ffa241 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -14,7 +14,7 @@ schemas.load("plan") # the schema as a dict | Schema | Format | Document | |---|---|---| -| `addresses` | `nnnotes.addresses/1` | the address table of a catalog version and the censuses of its bundles | +| `addresses` | `nnnotes.addresses/2` | the address table of a catalog version and the censuses of its bundles | | `artifact` | `nnnotes.artifact/1` | the content, provenance and semantics of one output file | | `catalog-diff` | `nnnotes.catalog-diff/1` | the difference between two catalog versions (`catalog diff`) | | `catalog-index` | `nnnotes.catalog-index/1` | the index of a catalog pair: remote and APK catalogs, every location in catalog order | diff --git a/docs/stages.md b/docs/stages.md index bc7eb16..9e8e106 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -12,14 +12,15 @@ The asset export is built from three layers: ## Stages -Every stage is at version 1, except `unity.census` (version 3: it lists what every object references and the -hierarchy holding it), `unity.export` (version 2: one task per unit), `cri.movie` (version 2: every stream kind, +Every stage is at version 1, except `unity.census` (version 4: it lists what every object references, the +hierarchy holding it and the Resources.Load paths of a ResourceManager), `link.addresses` (version 2: the +Resources.Load paths), `unity.export` (version 2: one task per unit), `cri.movie` (version 2: every stream kind, codec and channel of a USM) and `catalog.index` (version 2: gzip input and Japanese remote asset placeholders). | Stage | Subject | Inputs | Outputs | |---|---|---|---| | `catalog.index` | a catalog (remote + APK) | the catalog files | every location (with the bundle hash, crc and size), bundles and raw remote files | -| `unity.census` | a bundle, or a built-in file of the APK | the bundle | serialized files, externals, objects (file, path id, class, size, type hash, name, script, the scripts an AnimationClip's bindings name, the objects it references, the hierarchy holding it), container, scene hashes, dependencies, script fields | +| `unity.census` | a bundle, or a built-in file of the APK | the bundle | serialized files, externals, objects (file, path id, class, size, type hash, name, script, the scripts an AnimationClip's bindings name, the objects it references, the hierarchy holding it), container, scene hashes, dependencies, script fields, the ResourceManager's paths (Resources.Load) | | `link.scripts` | global | the censuses holding MonoScripts | serialized file and path id -> assembly, namespace, class | | `unity.export` | a unit of a bundle or of a built-in file of the APK (below), by the stable address of its root object | the bundle; context `unit` (the root's object id), `bundle`, the script entries its objects use and, for files without type trees, the type trees nnnotes ships for their type hashes | its objects' artifacts and one item per object | | `sprite.crop` | a SpriteAtlas | the atlas' JSON, the images of its textures, the metas of the sprites of other bundles it packs | the sprites' images | @@ -27,7 +28,7 @@ codec and channel of a USM) and `catalog.index` (version 2: gzip input and Japan | `cri.movie` | USM content | `usm` (a raw file of the catalog, or the `usm` artifact of a movie held in a bundle), `boot` | every stream as stored (video, alpha, audio, subtitles), `movie.mkv` (or `movie.webm`) | | `live2d.model` | a Live2D model key (`Character/Live2D///model/`) whose bundle is selected | `bundle:`: the bundles of the key's dependency closure in the order the extractor loads them (the order is part of the key); context `internalId` (the key's location); atom `live2d.extract_model` (the extractor of `nnnotes export live2d`, run unchanged) | `live2d.model:#`: every file `nnnotes export live2d` writes for the model, `` relative to the model's directory; placed under `/` | | `spine.skeleton` | a SkeletonDataAsset of a selected bundle, by its stable address `:` (in a bundle of several serialized files the file's tag follows the bundle name, as in `link.addresses`) | `bundle:`: the bundles holding the serialized files the asset's file reaches through its externals; context `object` (the asset's object id); atom `spot.write_skeleton` (the Spine writer of `nnnotes export spot`) | `spine.skeleton:#`: the skeleton (`.json` or `.skel`), its atlases (`.atlas`) and their page images, the bytes `nnnotes export spot` writes into `spine/`; one item for the asset | -| `link.addresses` | global | catalog index, censuses (the built-in files' too) | key -> objects, serialized file -> bundle or built-in file, bundle -> keys | +| `link.addresses` | global | catalog index, censuses (the built-in files' too) | key -> objects, Resources.Load path -> objects, serialized file -> bundle or built-in file, bundle -> keys | | `view.` | a catalog (`main`) | `master:
` per master table it reads, `rules` (the view's rule and the resolvers it uses), `addresses` (the part of the address table it read: the keys looked up and the prefixes listed); no parameters, atoms or context | `view.:#view` (facts: rows, entries, gaps, unreferenced) | | `view.voices` | a catalog (`main`) | `master:
` per master table its rules read, `rules`, `addresses`, `sheets` (each cue sheet the `cri.audio` results are known by that a voice names or the rules' prefixes list: its task and ACB content id), `acb:` (the ACB of each decoded one), `episodes` (per story episode and part, the `unity.export` artifact holding the object its key names), `adv:` (those contents); context `audio` (the keys of those `cri.audio` results); runs after `unity.export` and `cri.audio` | `view.voices:#view` (facts: rows, entries, gaps, unreferenced streams) | @@ -110,9 +111,10 @@ several serialized files the file's tag follows the bundle name, as in `link.add bundle's content changes, so a unit keeps its task across catalog versions. A run exports the units holding an object of the selected bundles, the units of the objects the fact "roots" names -(a product's roots: [products](products.md)) and every unit reachable from those through what their objects reference -(the census `refs`, across the censuses of the run); with `--only-class` only the units whose root object is of those -classes. The selection is not part of any key: a unit's task is the same whichever request reaches it. +(catalog keys, Resources.Load paths and object ids: a product's roots, [products](products.md)) and every unit reachable +from those through what their objects reference (the census `refs`, across the censuses of the run); with `--only-class` +only the units whose root object is of those classes. The selection is not part of any key: a unit's task is the same +whichever request reaches it. A task's key includes the atoms `reader` and `sniff`, the converter `generic.json`, the converters (with their atoms) of the classes of the unit's objects, its bundle, its root and the script entries and shipped type trees its diff --git a/src/nnnotes/pipeline/assemble.py b/src/nnnotes/pipeline/assemble.py index 1c2e9a1..4fc3020 100644 --- a/src/nnnotes/pipeline/assemble.py +++ b/src/nnnotes/pipeline/assemble.py @@ -1,8 +1,9 @@ """Product stages: products assembled from the unity.export results of the objects they reach, never from a bundle. A product (a Live2D model, a story, a live, ...) is a stage whose task per subject starts from root objects: the -objects of the catalog keys `keys(subject)` gives and the objects `root_objects(subject)` names (both read from the -planning facts alone, so a command can ask for the units its products need before anything runs: `request`). The +objects of the catalog keys `keys(subject)` gives, of the Resources.Load paths `resources(subject)` gives and the +objects `root_objects(subject)` names (all read from the planning facts alone, so a command can ask for the units its +products need before anything runs: `request`). The objects it reaches are those of the units holding its roots and every unit an item of a reached unit references (contract.item `refs`), through the unity.export results; a sprite whose atlas is in another bundle has its image from sprite.crop. A reached unit that the request did not export is an error of the request; a referenced object @@ -34,15 +35,16 @@ def request(stages: dict, products: dict, env) -> dict: - """The fact "roots" (pipeline.units) of a request for `products` {stage name: subjects}: the catalog keys and the - root objects of every subject, sorted.""" - keys, objects = set(), set() + """The fact "roots" (pipeline.units) of a request for `products` {stage name: subjects}: the catalog keys, the + resource paths and the root objects of every subject, sorted.""" + keys, resources, objects = set(), set(), set() for name, subjects in sorted(products.items()): stage = stages[name] for s in subjects: keys.update(stage.keys(s, env)) + resources.update(stage.resources(s, env)) objects.update(stage.root_objects(s, env)) - return {"keys": sorted(keys), "objects": sorted(objects)} + return {"keys": sorted(keys), "resources": sorted(resources), "objects": sorted(objects)} def _entry(item: dict, arts: dict) -> dict: @@ -211,7 +213,8 @@ class Assembly: class ProductStage(Stage): """Base of the product stages (module documentation). Subclasses set `name`, `version` and `KIND`, and define - `keys` and/or `root_objects` and `assemble`; `subjects` defaults to the subjects of the fact "products".""" + `keys`, `resources` and/or `root_objects` and `assemble`; `subjects` defaults to the subjects of the fact + "products".""" after = (ADDRESSES, EXPORT, CROP) KIND = "product" @@ -235,8 +238,13 @@ def keys(self, subject: str, env) -> list[str]: """The catalog keys whose objects are roots of `subject` (from the facts alone).""" return [] + def resources(self, subject: str, env) -> list[str]: + """The Resources.Load paths whose objects are roots of `subject` (from the facts alone).""" + return [] + def root_objects(self, subject: str, env) -> list[str]: - """Object ids that are roots of `subject` besides those of its keys (from the facts alone).""" + """Object ids that are roots of `subject` besides those of its keys and resource paths (from the facts + alone).""" return [] def more_inputs(self, subject: str, env) -> list[Input]: @@ -253,7 +261,9 @@ def manifest(self, subject: str) -> str: # ---------------------------------------------------------------- planning def reach(self, subject: str, env) -> Reach: """What `subject` reaches (kept while the results it read are the same).""" - roots = units.root_objects(env, self.keys(subject, env), self.root_objects(subject, env)) + roots = units.root_objects( + env, self.keys(subject, env), self.root_objects(subject, env), self.resources(subject, env) + ) sig = (tuple(roots), id(units.table(env))) memo = self._memo.get(subject) if memo is not None and memo[0] == sig and all(env.key(t) == k for t, k in memo[1]): @@ -268,7 +278,8 @@ def inputs(self, subject: str, env) -> list[Input]: return [table, *self.more_inputs(subject, env)] def depends(self, subject: str, env) -> list[str]: - own = [contract.task_id(ADDRESSES, s) for s in sorted(env.fact("catalogs"))] if self.keys(subject, env) else [] + named = self.keys(subject, env) or self.resources(subject, env) + own = [contract.task_id(ADDRESSES, s) for s in sorted(env.fact("catalogs"))] if named else [] return own + self.reach(subject, env).read def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: diff --git a/src/nnnotes/pipeline/census.py b/src/nnnotes/pipeline/census.py index c7b43c3..a660cdc 100644 --- a/src/nnnotes/pipeline/census.py +++ b/src/nnnotes/pipeline/census.py @@ -17,6 +17,9 @@ assetBundles the AssetBundle objects: {file, pathId, name, assetBundleName, dependencies, isStreamedSceneAssetBundle, sceneHashes [[scene, hash]], container [{path, file, pathId, preloadIndex, preloadSize}] (stored order)} + resourcePaths the container of the ResourceManager objects (the paths Resources.Load finds objects at, in the + APK's global managers): [{path, file, pathId}] in stored order; a path names one object per + class it was saved as scripts the MonoScript objects: {file, pathId, assembly, namespace, class, name} classes objects per class; `objects` the total errors [{object, message}] of the objects whose typetree could not be read (their record has no name, @@ -140,6 +143,14 @@ def _asset_bundle(d: dict, refs: _Refs, oid: tuple[str, int]) -> dict: } +def _resource_paths(d: dict, refs: _Refs) -> list: + out = [] + for path, p in _pairs(d.get("m_Container")): + target = refs.ref(p) or {} + out.append({"path": path, "file": target.get("file"), "pathId": target.get("pathId")}) + return out + + def _binding_scripts(d: dict, refs: _Refs) -> list: """The MonoScripts an AnimationClip's generic bindings name, {file, pathId} (or {fileId, pathId}), sorted.""" out = {} @@ -170,7 +181,7 @@ def census_env(env, shipped: bool = False) -> dict: by_file: dict[str, list] = {} for i in ctx.infos: by_file.setdefault(i.file, []).append(i) - files, bundles, scripts, errors, classes = [], [], [], [], {} + files, bundles, paths, scripts, errors, classes = [], [], [], [], [], {} for name in sorted(by_file): sf = view.serialized_file(name) refs = _Refs(name, ctx.ext_files[name]) @@ -204,6 +215,8 @@ def census_env(env, shipped: bool = False) -> dict: rec["refs"] = sorted(contract.object_id(*k) for k in found) if i.class_name == "AssetBundle": bundles.append(_asset_bundle(d, refs, key)) + elif i.class_name == "ResourceManager": + paths.extend(_resource_paths(d, refs)) elif i.class_name == "MonoScript": scripts.append( { @@ -241,6 +254,7 @@ def census_env(env, shipped: bool = False) -> dict: "files": files, "resources": resources, "assetBundles": sorted(bundles, key=lambda b: (b["file"], b["pathId"])), + "resourcePaths": paths, "scripts": sorted(scripts, key=lambda s: (s["file"], s["pathId"])), "classes": dict(sorted(classes.items())), "objects": sum(classes.values()), @@ -332,7 +346,7 @@ class CensusStage(Stage): MonoBehaviours with.""" name = "unity.census" - version = 3 + version = 4 ATOMS = {"reader": impl_id(("UnityPy",), 1)} def subjects(self, env) -> list[str]: diff --git a/src/nnnotes/pipeline/link.py b/src/nnnotes/pipeline/link.py index bf41aa9..968454c 100644 --- a/src/nnnotes/pipeline/link.py +++ b/src/nnnotes/pipeline/link.py @@ -5,7 +5,7 @@ stays empty). A MonoBehaviour's class is the entry of its m_Script; `script_context` picks the entries a task consumes. -link.addresses (nnnotes.addresses/1), one task per catalog pair: the catalog index and the censuses of its bundles +link.addresses (nnnotes.addresses/2), one task per catalog pair: the catalog index and the censuses of its bundles (by stable bundle name) and of the APK's built-in files (subjects `apk/...`, pipeline.builtins) -> bundles {stable name: {name, files, assetBundleName, keys}}: the file name in this catalog version, its serialized files, the AssetBundle name the census found and the keys located in it; a built-in file @@ -14,7 +14,10 @@ keys {primary key: [{location, type, internalId, bundle | raw, objects}]}: an asset location's bundle is the dependency whose container holds its internal id (compared ignoring case); `objects` the object ids the container path names (null when a dependency bundle has no census) - objects {stable address: {object, class, name, byteSize, typeHash}} of the objects a container path names + resources {resource path: [object ids]}: the paths of the ResourceManager container of the built-in files (what + Resources.Load finds, as stored), the objects each names in stored order + objects {stable address: {object, class, name, byteSize, typeHash}} of the objects a container path or a + resource path names problems files named by two bundles, bundles whose census AssetBundle name is not the catalog's, keys whose internal id no container holds @@ -24,8 +27,9 @@ "/[]" for another object under it with a name no other object under it has, else ":". In a bundle of several serialized files (their path ids may coincide) the path id form names the file after the bundle: the part of the file name after its CAB- prefix ("scene_x.sharedAssets:12"), or -"|" ("apk/data.unity3d|level0:3"). -Objects that no container path names are not listed; `address_of` gives any object's address. +"|" ("apk/data.unity3d|level0:3"). A resource path counts as a container path of the built-in file holding the +ResourceManager ("apk/data.unity3d/live2d/cubism/materials/mask"). +Objects that no container or resource path names are not listed; `address_of` gives any object's address. """ @@ -40,7 +44,7 @@ from .stages import Output, Pending, Stage SCRIPTS = "nnnotes.scripts/1" -ADDRESSES = "nnnotes.addresses/1" +ADDRESSES = "nnnotes.addresses/2" _CAB = re.compile(r"^CAB-[0-9a-f]+") @@ -125,7 +129,7 @@ def addresses(index: dict, censuses) -> dict: the index does not list are ignored.""" files_of = catalogdb.file_refs(index) listed = {b["stable"]: b for b in index["bundles"]} - bundles, owners, info, groups, tags_of = {}, {}, {}, {}, {} + bundles, owners, info, groups, tags_of, resources = {}, {}, {}, {}, {}, {} problems = {"files": [], "bundleNames": [], "keys": []} pairs = sorted(censuses.items()) if isinstance(censuses, dict) else censuses for stable, doc in pairs: @@ -141,8 +145,11 @@ def addresses(index: dict, censuses) -> dict: if b.get("bundleName") and found and b["bundleName"] + ".bundle" not in found: problems["bundleNames"].append({"bundle": stable, "catalog": b["bundleName"] + ".bundle", "census": found}) g: dict[str, list[dict]] = {} - for e in census.container(doc): + for e in census.container(doc) + doc["resourcePaths"]: g.setdefault(e["path"].casefold(), []).append(e) + for e in doc["resourcePaths"]: + if e["file"] is not None: + resources.setdefault(e["path"], []).append(contract.object_id(e["file"], e["pathId"])) wanted = {(e["file"], e["pathId"]) for es in g.values() for e in es if e["file"] is not None} for file, o in census.objects(doc): if (file, o["pathId"]) in wanted: @@ -208,6 +215,7 @@ def addresses(index: dict, censuses) -> dict: "bundles": dict(sorted(bundles.items())), "files": dict(sorted(cab.items())), "keys": dict(sorted(keys.items())), + "resources": dict(sorted(resources.items())), "objects": dict(sorted(objects.items())), "problems": problems, } @@ -292,7 +300,7 @@ class AddressesStage(Stage): skips failed ones. Artifact "link.addresses:#addresses".""" name = "link.addresses" - version = 1 + version = 2 after = ("catalog.index", "unity.census") def subjects(self, env) -> list[str]: @@ -323,6 +331,7 @@ def run(self, task, depot) -> Output: facts = { "bundles": len(doc["bundles"]), "keys": len(doc["keys"]), + "resources": len(doc["resources"]), "objects": len(doc["objects"]), "problems": {k: len(v) for k, v in doc["problems"].items()}, } diff --git a/src/nnnotes/pipeline/units.py b/src/nnnotes/pipeline/units.py index a82dff1..788caf7 100644 --- a/src/nnnotes/pipeline/units.py +++ b/src/nnnotes/pipeline/units.py @@ -11,9 +11,10 @@ every class), only the units whose root object is of those classes. A bundle whose census failed has no units; a reference into a file no census holds is left out. -The fact "roots" (absent: none) is {"keys": catalog keys, "objects": object ids}: the objects of each key (all those -the address table of link.addresses lists for it; a key without objects adds none) and the objects named. The -products of a request start from them (pipeline.assemble). +The fact "roots" (absent: none) is {"keys": catalog keys, "resources": resource paths, "objects": object ids}, each +list optional: the objects of each key and of each resource path (all those the address tables of link.addresses +list for it: a resource path is a Resources.Load path of the APK's built-in files, compared ignoring case; one +without objects adds none) and the objects named. The products of a request start from them (pipeline.assemble). """ from __future__ import annotations @@ -186,22 +187,24 @@ def address_index(env, subject: str) -> views.AddressIndex: return index -def root_objects(env, keys=(), objects=()) -> list[str]: - """The object ids of the catalog `keys` (through the address tables of the fact "catalogs") and the `objects`, - sorted.""" +def root_objects(env, keys=(), objects=(), resources=()) -> list[str]: + """The object ids of the catalog `keys` and the resource paths `resources` (through the address tables of the + fact "catalogs") and the `objects`, sorted.""" out = set(objects) - if keys: + if keys or resources: for subject in sorted(env.fact("catalogs")): index = address_index(env, subject) for k in keys: out.update(o.id for o in index.objects(k) or ()) + for p in resources: + out.update(o.id for o in index.resource(p) or ()) return sorted(out) def table(env) -> Units: """The Units of the request of a planning environment: the censuses of the fact "bundles" (Pending while one has not run; a failed one is skipped), the facts "selected", "roots" (Pending while link.addresses has not run, - when it names keys) and "classes". Kept for the planning state it was made in.""" + when it names keys or resource paths) and "classes". Kept for the planning state it was made in.""" bundles = sorted(env.fact("bundles")) keys = [] for b in bundles: @@ -213,7 +216,7 @@ def table(env) -> Units: raise classes = env.facts.get("classes") request = env.facts.get("roots") or {} - roots = root_objects(env, request.get("keys", ()), request.get("objects", ())) + roots = root_objects(env, request.get("keys", ()), request.get("objects", ()), request.get("resources", ())) sig = ( id(env.depot), tuple(keys), diff --git a/src/nnnotes/pipeline/views.py b/src/nnnotes/pipeline/views.py index 18c7012..a8c3683 100644 --- a/src/nnnotes/pipeline/views.py +++ b/src/nnnotes/pipeline/views.py @@ -121,11 +121,17 @@ def objects(self, key: str) -> list[Obj] | None: class AddressIndex: - """Addresses from {catalog key: objects in stored order} (a key mapped to None or [] has no census object).""" + """Addresses from {catalog key: objects in stored order} (a key mapped to None or [] has no census object), and + the objects of the resource paths of the APK's built-in files ({path: objects in stored order}, `resource`).""" - def __init__(self, entries: Mapping[str, Iterable[Obj] | None]): + def __init__( + self, entries: Mapping[str, Iterable[Obj] | None], resources: Mapping[str, Iterable[Obj]] | None = None + ): self._objs = {k: list(v or ()) for k, v in entries.items()} self._keys = sorted(self._objs) + self._resources: dict[str, list[Obj]] = {} + for path, objs in (resources or {}).items(): + self._resources.setdefault(path.casefold(), []).extend(objs) def keys(self, prefix: str = "") -> list[str]: i = bisect.bisect_left(self._keys, prefix) @@ -140,6 +146,12 @@ def objects(self, key: str) -> list[Obj] | None: objs = self._objs.get(key) return None if objs is None else list(objs) + def resource(self, path: str) -> list[Obj] | None: + """The objects a Resources.Load `path` finds (compared ignoring case, as Unity does), in stored order; None + for a path no ResourceManager lists.""" + objs = self._resources.get(path.casefold()) + return None if objs is None else list(objs) + @classmethod def build(cls, locations: Mapping[str, str], containers: Mapping[str, Iterable[Obj]]) -> AddressIndex: """The index of a catalog and its censuses: `locations` {key: internal id} (the asset path a key loads), @@ -152,22 +164,27 @@ def build(cls, locations: Mapping[str, str], containers: Mapping[str, Iterable[O @classmethod def from_addresses(cls, doc: dict) -> AddressIndex: - """The index of an address table (nnnotes.addresses/1, the link.addresses stage): a key's objects are those - of all its locations (in the table's order, the order of the bundle's container), with class, name and - stable address from the table's `objects`; a location whose bundle has no census yet contributes none.""" + """The index of an address table (nnnotes.addresses/2, the link.addresses stage): a key's objects are those + of all its locations (in the table's order, the order of the bundle's container), a resource path's those + its table lists, with class, name and stable address from the table's `objects`; a location whose bundle has + no census yet contributes none.""" where: dict[str, str] = {} for addr, info in doc["objects"].items(): where.setdefault(info["object"], addr) + + def obj(oid: str) -> Obj: + addr = where.get(oid) + info = doc["objects"][addr] if addr is not None else {} + return Obj(oid, info.get("class") or "", info.get("name") or "", addr) + entries = {} for key, locs in doc["keys"].items(): objs = {} for loc in locs: for oid in loc.get("objects") or (): - addr = where.get(oid) - info = doc["objects"][addr] if addr is not None else {} - objs[oid] = Obj(oid, info.get("class") or "", info.get("name") or "", addr) + objs[oid] = obj(oid) entries[key] = list(objs.values()) - return cls(entries) + return cls(entries, {path: [obj(o) for o in oids] for path, oids in doc["resources"].items()}) def to_json(self) -> dict: return {"keys": {k: [o.to_json() for o in self._objs[k]] for k in self._keys}} diff --git a/src/nnnotes/schemas/addresses.schema.json b/src/nnnotes/schemas/addresses.schema.json index 320e4e3..d503b25 100644 --- a/src/nnnotes/schemas/addresses.schema.json +++ b/src/nnnotes/schemas/addresses.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "urn:nnnotes:schema:addresses", - "title": "nnnotes.addresses/1", - "description": "The address table of a catalog version and the censuses of its bundles (stage link.addresses): per stable bundle name its file name, serialized files, AssetBundle name and keys; serialized file -> bundle; per catalog key its locations with their bundle (or raw files) and the objects the container path names; per stable address the object it names; the problems found.", + "title": "nnnotes.addresses/2", + "description": "The address table of a catalog version and the censuses of its bundles (stage link.addresses): per stable bundle name its file name, serialized files, AssetBundle name and keys; serialized file -> bundle; per catalog key its locations with their bundle (or raw files) and the objects the container path names; per resource path (Resources.Load, the APK's built-in files) the objects it names; per stable address the object it names; the problems found.", "type": "object", "required": [ "schema", @@ -10,12 +10,13 @@ "bundles", "files", "keys", + "resources", "objects", "problems" ], "properties": { "schema": { - "const": "nnnotes.addresses/1" + "const": "nnnotes.addresses/2" }, "catalog": { "type": "object", @@ -148,6 +149,17 @@ } } }, + "resources": { + "description": "Per path of a ResourceManager container (the paths Resources.Load finds objects at, as stored) the object ids it names, in stored order.", + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, "objects": { "type": "object", "additionalProperties": { diff --git a/src/nnnotes/schemas/census.schema.json b/src/nnnotes/schemas/census.schema.json index a1c0354..4cecd0e 100644 --- a/src/nnnotes/schemas/census.schema.json +++ b/src/nnnotes/schemas/census.schema.json @@ -2,13 +2,14 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "urn:nnnotes:schema:census", "title": "nnnotes.census/2", - "description": "The census of one bundle (stage unity.census), read without decoding any object: its serialized files by name (Unity version, platform, type trees, externals, objects by path id with class, size, type hash, m_Name, m_Script, the objects they reference and the hierarchy holding them), the other files of the bundle, its AssetBundle objects (container in stored order, dependencies, scene hashes), its MonoScripts, objects per class and the objects whose typetree could not be read. A function of the bundle's bytes.", + "description": "The census of one bundle (stage unity.census), read without decoding any object: its serialized files by name (Unity version, platform, type trees, externals, objects by path id with class, size, type hash, m_Name, m_Script, the objects they reference and the hierarchy holding them), the other files of the bundle, its AssetBundle objects (container in stored order, dependencies, scene hashes), the container of its ResourceManager (the paths Resources.Load finds objects at), its MonoScripts, objects per class and the objects whose typetree could not be read. A function of the bundle's bytes.", "type": "object", "required": [ "schema", "files", "resources", "assetBundles", + "resourcePaths", "scripts", "classes", "objects", @@ -329,6 +330,35 @@ } } }, + "resourcePaths": { + "description": "The container of the ResourceManager objects (in the APK's global managers): the paths Resources.Load finds objects at, in stored order; a path names one object per class it was saved as.", + "type": "array", + "items": { + "type": "object", + "required": [ + "path", + "file", + "pathId" + ], + "properties": { + "path": { + "type": "string" + }, + "file": { + "type": [ + "string", + "null" + ] + }, + "pathId": { + "type": [ + "integer", + "null" + ] + } + } + } + }, "scripts": { "type": "array", "items": { diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index 9a1eca6..e52104d 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -16,7 +16,7 @@ from nnnotes import cli from nnnotes.cli import assets, common from nnnotes.cli import catalog as catalog_cli -from nnnotes.pipeline import contract, units +from nnnotes.pipeline import contract, link, units from nnnotes.pipeline.census import CensusStage from nnnotes.pipeline.contract import Cost from nnnotes.pipeline.depot import Depot @@ -75,6 +75,7 @@ def cdoc(file: str, objs: list, container=(), scripts=(), bundle_name=None) -> d ], "resources": [], "assetBundles": [ab], + "resourcePaths": [], "scripts": list(scripts), "classes": classes, "objects": len(objs), @@ -915,8 +916,8 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps contract.CATALOG_INDEX, contract.CENSUS, contract.VIEW, - "nnnotes.scripts/1", - "nnnotes.addresses/1", + link.SCRIPTS, + link.ADDRESSES, }, checked code, out, _ = nn( capsys, diff --git a/tests/helpers/fakeunity.py b/tests/helpers/fakeunity.py index 49089c1..ead4cb3 100644 --- a/tests/helpers/fakeunity.py +++ b/tests/helpers/fakeunity.py @@ -65,6 +65,7 @@ "PlayerSettings": 129, "Font": 128, "AssetBundle": 142, + "ResourceManager": 147, "AnimatorController": 91, "AnimatorOverrideController": 221, "RectTransform": 224, diff --git a/tests/helpers/units.py b/tests/helpers/units.py index bf36993..87c86b1 100644 --- a/tests/helpers/units.py +++ b/tests/helpers/units.py @@ -32,6 +32,7 @@ def census_doc(file: str, objects) -> dict: ], "resources": [], "assetBundles": [], + "resourcePaths": [], "scripts": [], "classes": dict(sorted(Counter(o["class"] for o in objs).items())), "objects": len(objs), diff --git a/tests/pipeline/test_assemble.py b/tests/pipeline/test_assemble.py index 547d509..b14dc93 100644 --- a/tests/pipeline/test_assemble.py +++ b/tests/pipeline/test_assemble.py @@ -15,12 +15,14 @@ from tests.pipeline.test_objexport import ATLAS_CAB, FAKE_READER, oid, planning_env -def addresses(depot, env, keys: dict) -> None: - """A committed link.addresses result whose table maps `keys` {catalog key: [object ids]}.""" +def addresses(depot, env, keys: dict, resources=None) -> None: + """A committed link.addresses result whose table maps `keys` {catalog key: [object ids]} and `resources` + {resource path: [object ids]}.""" t = Task("link.addresses", 1, "main", {}, {}, ()) doc = { "schema": link.ADDRESSES, "keys": {k: [{"objects": ids}] for k, ids in sorted(keys.items())}, + "resources": dict(sorted((resources or {}).items())), "objects": {}, "files": {}, "bundles": {}, @@ -75,11 +77,25 @@ def root_objects(self, subject, env): return self.objects -def env_for(depot, keys=None, names=("all", "atlas"), roots_objects=(), selected=None): +class Loaded(Toy): + """A Toy rooted at the objects of Resources.Load paths alone.""" + + def __init__(self, paths): + super().__init__() + self.paths = list(paths) + + def keys(self, subject, env): + return [] + + def resources(self, subject, env): + return self.paths + + +def env_for(depot, keys=None, names=("all", "atlas"), roots_objects=(), selected=None, resources=None): """A planning environment over the fake bundles with a committed address table and the facts of a request.""" env = planning_env(depot, names, selected=selected) env.facts["catalogs"] = {"main": []} - addresses(depot, env, keys if keys is not None else {}) + addresses(depot, env, keys if keys is not None else {}, resources) subjects = sorted(keys if keys is not None else {}) env.facts["products"] = {"toy.thing": subjects} env.facts["roots"] = {"keys": subjects, "objects": list(roots_objects)} @@ -117,7 +133,8 @@ def test_a_products_key_is_asked_for_before_anything_runs(tmp_path): stage = Toy({"m/a": "key/a", "m/b": "key/b"}) stages = {"toy.thing": stage} env = Env(depot, {"keys": {}}) - assert assemble.request(stages, {"toy.thing": ["m/b", "m/a"]}, env) == {"keys": ["key/a", "key/b"], "objects": []} + roots = assemble.request(stages, {"toy.thing": ["m/b", "m/a"]}, env) + assert roots == {"keys": ["key/a", "key/b"], "resources": [], "objects": []} assert stage.subjects(Env(depot, {"products": {"toy.thing": ["m/b"]}})) == ["m/b"] @@ -175,6 +192,20 @@ def test_the_request_roots_select_the_units_and_nothing_else(tmp_path): assert subjects == ["all:10", "all:60"] # the prefab and the texture it references +def test_a_resources_load_path_is_a_root(tmp_path): + """A product's Resources.Load paths name its roots through the address table, ignoring case.""" + depot = Depot(tmp_path / "s") + env = env_for(depot, {}, names=("all",), selected=[], resources={"shaders/thing": [oid(60)]}) + stage = Loaded(["Shaders/Thing"]) + env.facts["roots"] = assemble.request({"toy.thing": stage}, {"toy.thing": ["m/a"]}, env) + assert env.facts["roots"] == {"keys": [], "resources": ["Shaders/Thing"], "objects": []} + assert ExportStage(FAKE_READER).subjects(env) == ["all:10", "all:60"] + exported(depot, env) + assert stage.depends("m/a", env) == ["link.addresses:main", "unity.export:all:10", "unity.export:all:60"] + _, doc = product(depot, env, "m/a", stage=stage) + assert table_of(depot, doc)["roots"] == [oid(60)] + + def test_sprite_images_come_from_the_crop_of_their_atlas(tmp_path): depot = Depot(tmp_path / "s") env = env_for(depot, {"key/a": [oid(13)]}, names=("all", "atlas"), roots_objects=[f"{ATLAS_CAB}:99"]) diff --git a/tests/pipeline/test_census.py b/tests/pipeline/test_census.py index 8a6b73a..705e86a 100644 --- a/tests/pipeline/test_census.py +++ b/tests/pipeline/test_census.py @@ -251,6 +251,26 @@ def test_monoscripts_and_unknown_file_index(): assert refs.ref(None) is None and refs.ref(pptr(5)) == {"file": "CAB-1", "pathId": 5} +def test_the_resource_manager_lists_the_resources_load_paths(): + env = FakeEnvironment() + f = env.file("globalgamemanagers.assets", externals=["resources.assets"]) + container = [ + ("live2d/cubism/materials/mask", pptr(4, 1)), + ("live2d/cubism/materials/mask", pptr(5, 1)), + ("missing", pptr(0)), + ] + f.add(3, "ResourceManager", {"m_Container": container, "m_DependentAssets": []}) + doc = census.census_env(env) + assert doc["resourcePaths"] == [ + {"path": "live2d/cubism/materials/mask", "file": "resources.assets", "pathId": 4}, + {"path": "live2d/cubism/materials/mask", "file": "resources.assets", "pathId": 5}, + {"path": "missing", "file": None, "pathId": None}, + ] + (rm,) = doc["files"][0]["objects"] + assert rm["refs"] == ["resources.assets:4", "resources.assets:5"] + assert doc["assetBundles"] == [] + + def player_file() -> FakeEnvironment: """A player's own file without type trees: a MonoBehaviour that references a built-in texture.""" env = FakeEnvironment() @@ -293,7 +313,7 @@ def load(path, name=None): env = Env(depot, {"bundles": {"x": inp}}) assert stage.subjects(env) == ["x"] task = describe(stage, "x", None, env) - assert task.id == "unity.census:x" and task.atoms == census.CensusStage.ATOMS and task.version == 3 + assert task.id == "unity.census:x" and task.atoms == census.CensusStage.ATOMS and task.version == 4 assert task.atoms["reader"].startswith("unitypy-") assert task.cost.peak_bytes > size assert execute(task, depot, registry(stage)).status == "ran" diff --git a/tests/pipeline/test_link.py b/tests/pipeline/test_link.py index eee3d1a..67b9333 100644 --- a/tests/pipeline/test_link.py +++ b/tests/pipeline/test_link.py @@ -1,6 +1,6 @@ import pytest -from nnnotes.pipeline import catalogdb, census, contract, link +from nnnotes.pipeline import catalogdb, census, contract, link, views from nnnotes.pipeline.catalogdb import CatalogDB, IndexStage, index from nnnotes.pipeline.contract import Input from nnnotes.pipeline.depot import Depot @@ -20,8 +20,9 @@ def obj(pid, cls, name=None, size=10, script=None): return o -def cdoc(files: dict, container=(), name=None, scripts=()): - """A census document: files {file: [objects]}, container [(path, file, pathId)] on the first file's AssetBundle.""" +def cdoc(files: dict, container=(), name=None, scripts=(), resource_paths=()): + """A census document: files {file: [objects]}, container [(path, file, pathId)] on the first file's AssetBundle, + resource_paths [(path, file, pathId)] of a ResourceManager.""" fs = [ { "name": f, @@ -55,6 +56,7 @@ def cdoc(files: dict, container=(), name=None, scripts=()): "files": fs, "resources": [], "assetBundles": [ab], + "resourcePaths": [{"path": p, "file": f, "pathId": i} for p, f, i in resource_paths], "scripts": list(scripts), "classes": classes, "objects": sum(classes.values()), @@ -133,6 +135,39 @@ def censuses(): return {"card_a": card, "shared": shared, "scene_x": scene, "scripts": scripts} +def test_resource_paths_name_objects_of_the_builtin_files(): + """A built-in file's ResourceManager paths: the resources table, and addresses like container paths.""" + idx = index(synth.CatalogWriter().build(catalog_entries())) + data = cdoc( + { + "globalgamemanagers.assets": [obj(1, "ResourceManager")], + "resources.assets": [obj(4, "Material", "Mask"), obj(5, "Texture2D", "Mask"), obj(6, "Shader", "S")], + }, + resource_paths=[ + ("live2d/cubism/materials/mask", "resources.assets", 4), + ("live2d/cubism/materials/mask", "resources.assets", 5), + ("shaders/s", "resources.assets", 6), + ("gone", None, None), + ], + ) + doc = link.addresses(idx, {**censuses(), "apk/data.unity3d": data}) + assert doc["resources"] == { + "live2d/cubism/materials/mask": ["resources.assets:4", "resources.assets:5"], + "shaders/s": ["resources.assets:6"], + } + objs = doc["objects"] + assert objs["apk/data.unity3d/live2d/cubism/materials/mask"]["object"] == "resources.assets:4" + assert objs["apk/data.unity3d/live2d/cubism/materials/mask[Mask]"]["object"] == "resources.assets:5" + assert objs["apk/data.unity3d/shaders/s"]["class"] == "Shader" + assert doc["bundles"]["apk/data.unity3d"]["keys"] == [] and doc["problems"]["keys"] == ["Card/Gone"] + ix = views.AddressIndex.from_addresses(doc) + mask = ix.resource("Live2D/Cubism/Materials/Mask") + assert [(o.id, o.cls) for o in mask] == [("resources.assets:4", "Material"), ("resources.assets:5", "Texture2D")] + assert mask[0].stable == "apk/data.unity3d/live2d/cubism/materials/mask" + assert ix.resource("live2d/cubism/materials/maskculling") is None + assert link.addresses(idx, censuses())["resources"] == {} + + def test_scripts_table_and_context(): docs = censuses() table = link.scripts(docs.values()) diff --git a/tests/pipeline/test_modelstages.py b/tests/pipeline/test_modelstages.py index 49afcd8..bb59984 100644 --- a/tests/pipeline/test_modelstages.py +++ b/tests/pipeline/test_modelstages.py @@ -299,6 +299,7 @@ def cdoc(files, scripts=()): "files": fs, "resources": [], "assetBundles": [], + "resourcePaths": [], "scripts": list(scripts), "classes": classes, "objects": sum(classes.values()), diff --git a/tests/pipeline/test_views.py b/tests/pipeline/test_views.py index dd9dda9..29c3865 100644 --- a/tests/pipeline/test_views.py +++ b/tests/pipeline/test_views.py @@ -10,7 +10,7 @@ import pytest -from nnnotes.pipeline import contract, views +from nnnotes.pipeline import contract, link, views from nnnotes.pipeline.contract import Input from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.orchestrate import Orchestrator @@ -116,6 +116,7 @@ def test_stored_order_decides_not_ids(): } ] }, + "resources": {}, "objects": {f"b/s.png[{x.name}]": {"object": x.id, "class": x.cls, "name": x.name} for x in sheet}, } assert [x.id for x in AddressIndex.from_addresses(doc).objects("S")] == [x.id for x in sheet] @@ -162,7 +163,7 @@ def test_address_index_build_and_prefixes(): def test_address_index_of_an_address_table(): doc = { - "schema": "nnnotes.addresses/1", + "schema": link.ADDRESSES, "keys": { "MemberCard/1/member_full": [ { @@ -191,7 +192,9 @@ def test_address_index_of_an_address_table(): ], "Cri/Sound/bgm": [{"location": 4, "type": "System.Object", "raw": "bgm.acb", "internalId": "y"}], }, + "resources": {"live2d/mask": ["resources.assets:3"]}, "objects": { + "apk/data.unity3d/live2d/mask": {"object": "resources.assets:3", "class": "Material", "name": "Mask"}, "membercard_1/assets/membercard/1/member_full.png": { "object": "CAB-a:1", "class": "Texture2D", @@ -212,6 +215,8 @@ def test_address_index_of_an_address_table(): assert ix.objects("MemberCard/2/member_full") == [] and ix.objects("Cri/Sound/bgm") == [] r = views.resolve(ix, "MemberCard/1/member_full[member_full]", "Sprite") assert (r["object"], r["stable"]) == ("CAB-a:2", "membercard_1/assets/membercard/1/member_full.png[member_full]") + assert ix.resource("Live2D/Mask") == [Obj("resources.assets:3", "Material", "Mask", "apk/data.unity3d/live2d/mask")] + assert ix.resource("Live2D/Other") is None and ix.objects("live2d/mask") is None # paths are no catalog keys # ---------------------------------------------------------------- rules @@ -712,7 +717,7 @@ def test_master_tables_from_a_directory(tmp_path): # ---------------------------------------------------------------- the view stages def addresses_doc(entries: dict) -> dict: - """A synthetic address table (nnnotes.addresses/1) of {catalog key: objects}: one location per key in bundle + """A synthetic address table (nnnotes.addresses/2) of {catalog key: objects}: one location per key in bundle `b`, every object listed under a stable address of its own.""" keys, objects = {}, {} for n, (key, objs) in enumerate(sorted(entries.items())): @@ -734,11 +739,12 @@ def addresses_doc(entries: dict) -> dict: "typeHash": None, } return { - "schema": "nnnotes.addresses/1", + "schema": link.ADDRESSES, "catalog": {"remote": None, "apk": None}, "bundles": {}, "files": {}, "keys": keys, + "resources": {}, "objects": objects, "problems": {"files": [], "bundleNames": [], "keys": []}, } From 786b7c1979455f1f96f22c3ca39f697cdb4e86c5 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 10:31:05 +0800 Subject: [PATCH 15/91] feat(assemble): let a product keep some files of an object An Assembly names, besides the objects its files use, the roles it keeps of an object of which it uses some files only (`roles`), such as a shader's programs for one graphics API. The product's object table, and so its manifest and layout, hold only those files. Naming an object the table lacks or a role the object has no file for is an error of the stage. Co-Authored-By: Claude Opus 5.5 --- docs/products.md | 2 +- src/nnnotes/pipeline/assemble.py | 35 +++++++++++++++++++++++++------- tests/pipeline/test_assemble.py | 35 ++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 8 deletions(-) diff --git a/docs/products.md b/docs/products.md index 002ce50..d9c5158 100644 --- a/docs/products.md +++ b/docs/products.md @@ -30,7 +30,7 @@ object that a reached object references and no census holds is listed as `missin |---|---| | inputs | `objects`: the object table of what it reaches (`nnnotes.objects/1`); the stage's own inputs (master tables, data its parameters name) | | depends | `link.addresses` (when it has keys or resource paths), the `unity.export` and `sprite.crop` tasks whose results it read | -| artifacts | `#`: the product's own files; `#objects`: the object table of the objects its files use (the whole table unless the stage narrows it) | +| artifacts | `#`: the product's own files; `#objects`: the object table of the objects its files use (the whole table unless the stage narrows it to some objects, or to some files of an object: a shader's programs for one graphics API) | | item | one for the subject: `exported`, or `unsupported` with a reason code | The key holds the object table, so a product is assembled again exactly when a file of an object it reaches diff --git a/src/nnnotes/pipeline/assemble.py b/src/nnnotes/pipeline/assemble.py index 4fc3020..a1ab4e7 100644 --- a/src/nnnotes/pipeline/assemble.py +++ b/src/nnnotes/pipeline/assemble.py @@ -204,11 +204,35 @@ def json(self, oid: str, role: str = "json"): @dataclass class Assembly: - """What `assemble` makes: the product's files {path: (bytes, extension, kind)} and the object ids its files use - (None: every object of the table).""" + """What `assemble` makes: the product's files {path: (bytes, extension, kind)} and what of the object table they + use: `objects` the object ids (None: every object of the table), `roles` {object id: the roles of its files the + product keeps} for an object of which it uses some files only (a shader's programs for one graphics API). The + product's object table, and so its manifest, holds only those.""" files: dict objects: set | None = None + roles: dict = field(default_factory=dict) + + +def _prune(tid: str, objects: dict, made: Assembly) -> dict: + """The entries of an object table an Assembly keeps.""" + keep = set(objects) if made.objects is None else set(made.objects) + unknown = sorted((keep | set(made.roles)) - set(objects)) + if unknown: + raise ValueError(f"{tid}: objects not in the table: {', '.join(unknown[:5])}") + out = {} + for o, e in objects.items(): + if o not in keep: + continue + roles = made.roles.get(o) + if roles is not None: + files = e.get("files") or {} + lacking = sorted(set(roles) - set(files)) + if lacking: + raise ValueError(f"{tid}: object {o} has no {', '.join(lacking)} file") + e = dict(e, files={r: f for r, f in files.items() if r in roles}) + out[o] = e + return out class ProductStage(Stage): @@ -303,11 +327,8 @@ def run(self, task, depot) -> Output: raise ValueError(f"{task.id}: a product file cannot be named {PRODUCT_TABLE!r}") aid = contract.artifact_id(task.id, path) arts.append(contract.artifact(aid, depot.add(body, ext), prov, {"kind": kind})) - if made.objects is not None: - unknown = sorted(set(made.objects) - set(doc["objects"])) - if unknown: - raise ValueError(f"{task.id}: objects not in the table: {', '.join(unknown[:5])}") - doc = dict(doc, objects={o: e for o, e in doc["objects"].items() if o in made.objects}) + if made.objects is not None or made.roles: + doc = dict(doc, objects=_prune(task.id, doc["objects"], made)) data = contract.encode(doc) aid = contract.artifact_id(task.id, PRODUCT_TABLE) arts.append(contract.artifact(aid, depot.add(data, "json"), prov, {"kind": TABLE_KIND})) diff --git a/tests/pipeline/test_assemble.py b/tests/pipeline/test_assemble.py index b14dc93..4cd733d 100644 --- a/tests/pipeline/test_assemble.py +++ b/tests/pipeline/test_assemble.py @@ -313,6 +313,41 @@ def assemble(self, task, objects): execute(t, depot, registry(Wrong({"m/a": "key/a"})), force=True) +class Roles(Toy): + """A Toy that keeps the `json` file alone of the texture its prefab references.""" + + ROLES: dict = {oid(10): {"json"}} + + def assemble(self, task, objects): + made = super().assemble(task, objects) + return assemble.Assembly(made.files, roles=self.ROLES) + + +def test_a_product_keeps_some_files_of_an_object(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}, stage=Roles({"m/a": "key/a"})) + table = table_of(depot, doc) + assert set(table["objects"][oid(10)]["files"]) == {"json"} # no image + assert oid(60) in table["objects"] and oid(61) in table["objects"] # the others whole + entries, manifest = manifest_of(depot, t, doc) + assert set(manifest["objects"][oid(10)]["files"]) == {"json"} + assert f"{oid(10)}#image" not in {e["id"] for e in entries} + + +def test_a_product_cannot_keep_a_file_an_object_lacks(tmp_path): + class Wrong(Roles): + ROLES = {oid(10): {"json", "mesh"}} + + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t = describe(Wrong({"m/a": "key/a"}), "m/a", None, env) + with pytest.raises(ValueError, match="has no mesh file"): + execute(t, depot, registry(Wrong({"m/a": "key/a"})), force=True) + + # ---------------------------------------------------------------- the layout def manifest_of(depot, t, doc): """(entries, the manifest) of a product layout for one result.""" From ca0956445d8f626cbfb6d4c33a1fb53e61d1fb32 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 10:40:45 +0800 Subject: [PATCH 16/91] feat(unity): name clip bindings in the hierarchy that plays the clip nnnotes.unity.bindings.resolve gives, for the bindings of an AnimationClip document, the path and the attribute each names in a prefab.json hierarchy played by the Animator on a given GameObject: paths relative to that GameObject by their crc32, attributes among the field paths of the bound component (a MonoBehaviour's by its script), the properties of a renderer's materials for a material binding, and the engine property names. A binding that no name, or several, fit stays unresolved; several fitting paths are listed. The binding name tables, field path listing and the material attribute bit layout live in this module, and the typetree exporter uses them from there. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 2 +- src/nnnotes/content/storyhost.py | 3 +- src/nnnotes/unity/bindings.py | 251 +++++++++++++++++++++++++++++++ src/nnnotes/unity/export.py | 117 +------------- tests/unity/test_bindings.py | 149 ++++++++++++++++++ 5 files changed, 408 insertions(+), 114 deletions(-) create mode 100644 src/nnnotes/unity/bindings.py create mode 100644 tests/unity/test_bindings.py diff --git a/docs/architecture.md b/docs/architecture.md index c55d9c0..21d43e3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -12,7 +12,7 @@ subpackage of `nnnotes` and imports only the layers below it. | Products | `nnnotes.site`, `nnnotes.musicdata` | the ournotes-player site (charts, models, stories, indexes); the music data file and its deck statistics | | Pipeline | `nnnotes.pipeline` | stages, task descriptions and results, the content-addressed depot, plans, layouts, the catalog versions, the views of the master data | | Content | `nnnotes.content` | the game's subjects as exports: ADV episodes and stories, lives and scores, spots, rooms, the UI library | -| Unity objects | `nnnotes.unity` | bundle reading, object ids, the atoms (the pure functions objects are converted with: texture decoding, sprite images, mesh arrays, glTF, PNG, file sniffing), the converters (one per Unity class, `nnnotes.unity.convert`), the type trees shipped for the player's script classes (`nnnotes.unity.scripttypes`), the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | +| Unity objects | `nnnotes.unity` | bundle reading, object ids, the atoms (the pure functions objects are converted with: texture decoding, sprite images, mesh arrays, glTF, PNG, file sniffing), the converters (one per Unity class, `nnnotes.unity.convert`), clip bindings named in the hierarchy that plays a clip (`nnnotes.unity.bindings`), the type trees shipped for the player's script classes (`nnnotes.unity.scripttypes`), the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | | File formats | `nnnotes.formats` | formats read from bytes: CRI ACB/AWB and HCA, USM movies, CRI Lips data, shaders, motions | | Sources | `nnnotes.source` | the configuration, the APK, Addressables catalogs and bundles, the game API, master data, languages, `Session` | | Core | `nnnotes.core` | the one JSON writer, canonical JSON and its hashes, media types, the process caches and the downloads cache | diff --git a/src/nnnotes/content/storyhost.py b/src/nnnotes/content/storyhost.py index 9c51813..462af37 100644 --- a/src/nnnotes/content/storyhost.py +++ b/src/nnnotes/content/storyhost.py @@ -39,8 +39,9 @@ from ..formats import shader as shader_mod from ..source import languages, master from ..unity import fontsettings, textstyle, tmpfont +from ..unity.bindings import RECT_PROPS from ..unity.convert.animation import streamed_frames -from ..unity.export import RECT_PROPS, Exporter, TexelPacker +from ..unity.export import Exporter, TexelPacker from ..unity.reader import SceneGraph, deref, mesh_arrays, referent, script_class from . import advui, room, spot, storyfonts diff --git a/src/nnnotes/unity/bindings.py b/src/nnnotes/unity/bindings.py new file mode 100644 index 0000000..7c8cc5d --- /dev/null +++ b/src/nnnotes/unity/bindings.py @@ -0,0 +1,251 @@ +"""Clip bindings named in the hierarchy that plays the clip. + +A binding of an AnimationClip (clip.json `bindings`, convert.animation.clip_doc) names what it animates by crc32 +values: `pathCrc`, the path of the animated object relative to the GameObject whose Animator plays the clip, and +`attributeCrc`, the animated property. Only that hierarchy names them. `resolve` takes the bindings, the hierarchy +(a prefab.json document: its nodes with their paths and components) and the path of the Animator's GameObject in it, +and gives per binding the path and the attribute it names, or None where no name, or several, fit. + +Path: the crc32 of the path, relative to the Animator's GameObject, of a transform below it ("" for itself). A path +no transform below it has stays unresolved: the Animator leaves such a binding unbound (an object parented there at +run time aside). + +Attribute, from the bound component (the components of the binding's class on the GameObjects at its path; for a +MonoBehaviour those of its script): +- a Transform binding's attribute the clip names already (convert.animation.TRANSFORM_ATTR); +- a renderer material binding (customType 22) names a property of the renderer's materials (`material_attribute`); +- any other binding names a field path of the bound component (`flat_names` of its fields), else one of the engine + properties (ENGINE_PROPS, RECT_PROPS), a material property (`material_prop_names`) or a field path of another + component of its class in the hierarchy. +""" + +from __future__ import annotations + +import zlib +from collections.abc import Callable, Iterable + +from UnityPy.enums import ClassIDType + +__all__ = [ + "ENGINE_PROPS", + "MATERIAL_PROPS", + "RECT_PROPS", + "RENDERER_MATERIAL", + "crc", + "flat_names", + "material_attribute", + "material_prop_names", + "resolve", +] + +RENDERER_MATERIAL = 22 # GenericBinding.customType of a renderer's material property +# Animatable property names a binding can name that are no field path of a component's fields: common component +# properties and RectTransform driven fields. +ENGINE_PROPS = ( + "m_IsActive", + "m_Enabled", + "m_Color.r", + "m_Color.g", + "m_Color.b", + "m_Color.a", + "m_Size.x", + "m_Size.y", + "m_FlipX", + "m_FlipY", + "m_SortingOrder", + "m_Sprite", + "m_Alpha", + "m_SizeDelta.x", + "m_SizeDelta.y", + "m_AnchoredPosition.x", + "m_AnchoredPosition.y", + "m_LocalPosition.x", + "m_LocalPosition.y", + "m_LocalPosition.z", + "m_LocalScale.x", + "m_LocalScale.y", + "m_LocalScale.z", +) +RECT_PROPS = ( + "m_AnchoredPosition.x", + "m_AnchoredPosition.y", + "m_SizeDelta.x", + "m_SizeDelta.y", + "m_AnchorMin.x", + "m_AnchorMin.y", + "m_AnchorMax.x", + "m_AnchorMax.y", + "m_Pivot.x", + "m_Pivot.y", +) +# material properties a binding can name as "material.[.]" (renderer material property blocks) +MATERIAL_PROPS = ( + "_Color", + "_TintColor", + "_MainTex_ST", + "_BaseColor", + "_EmissionColor", + "_Alpha", + "_Cutoff", + "_Intensity", + "_Offset", + "_Scroll", + "_ScrollX", + "_ScrollY", + "_Power", + "_Value", + "_Glow", + "_GlowColor", + "_AddColor", + "_MultiplyColor", + "_Progress", + "_Fade", + "_Brightness", + "_Dissolve", + "_MainTex", + "_Speed", + "_Rotation", +) +_CHANNELS = ("r", "g", "b", "a", "x", "y", "z", "w") + + +def crc(s: str) -> int: + """The crc32 a binding names a path or an attribute by.""" + return zlib.crc32(s.encode("utf-8")) + + +def flat_names(tt, prefix: str = "") -> list[str]: + """Every field path of a typetree or of an object's fields ("m_Color", "m_Color.r", "m_Enabled", ...); lists are + leaves.""" + out = [] + if isinstance(tt, dict): + for k, v in tt.items(): + p = f"{prefix}.{k}" if prefix else k + out.append(p) + out.extend(flat_names(v, p)) + return out + + +def material_prop_names(extra: Iterable[str] = ()) -> list[str]: + """The attributes of material properties (MATERIAL_PROPS and `extra`): "material." and every channel.""" + out = [] + for p in sorted(set(MATERIAL_PROPS) | set(extra)): + out.append(f"material.{p}") + out.extend(f"material.{p}.{c}" for c in _CHANNELS) + return out + + +def material_attribute(attribute: int, kinds: dict[str, str]) -> str | None: + """The attribute a renderer material binding (customType 22) names, "material.[.]", among + the properties of the renderer's materials `kinds` {name: "float" | "vector"} (a colour is a vector); None when + no property, or several, fit. The attribute is (crc32(property) & 0x0FFFFFFF) | channel << 28 with bit 30 set + for a colour channel (r, g, b, a) and clear for a vector channel (x, y, z, w) or a float property.""" + hits = [k for k in kinds if crc(k) & 0x0FFFFFFF == attribute & 0x0FFFFFFF] + if len(hits) != 1: + return None + name, channel, colour = hits[0], (attribute >> 28) & 3, (attribute >> 30) & 1 + if kinds[name] == "float" and not colour and channel == 0: + return f"material.{name}" + return f"material.{name}.{('rgba' if colour else 'xyzw')[channel]}" + + +def _class_name(type_id: int) -> str | None: + try: + return ClassIDType(type_id).name + except ValueError: + return None + + +class _Hierarchy: + """A prefab document's nodes by path and the crc32 tables of paths relative to a GameObject.""" + + def __init__(self, prefab: dict): + self.nodes: dict[str, list[dict]] = {} + for n in prefab["nodes"]: + self.nodes.setdefault(n["path"], []).append(n) + self._tables: dict[str, dict[int, set[str]]] = {} + self._names: dict[object, set[str]] = {} + + def relative(self, root: str) -> dict[int, set[str]]: + t = self._tables.get(root) + if t is None: + if root not in self.nodes: + raise KeyError(f"the Animator's GameObject {root!r} is not in the hierarchy") + t = {} + for path in self.nodes: + if path == root or path.startswith(root + "/"): + rel = path[len(root) + 1 :] + t.setdefault(crc(rel), set()).add(rel) + self._tables[root] = t + return t + + def class_names(self, match: Callable[[dict], bool], key) -> set[str]: + """The field paths of every component of the hierarchy `match` accepts (cached by `key`).""" + names = self._names.get(key) + if names is None: + names = self._names[key] = { + x + for ns in self.nodes.values() + for n in ns + for c in n["components"] + if match(c) + for x in flat_names(c.get("fields")) + } + return names + + +def resolve( + bindings: list[dict], prefab: dict, root: str, material: Callable[[object], dict | None] = lambda ref: None +) -> list[dict]: + """[{path, attribute}] of clip bindings (module documentation) in the hierarchy of `prefab` (a prefab.json + document) played by the Animator on the GameObject at `root` (a node path); `pathCandidates` (sorted) where + several paths fit. `material(ref)`: the material.json document of a reference a renderer's m_Materials holds (None + when unknown).""" + h = _Hierarchy(prefab) + table = h.relative(root) + out = [] + for b in bindings: + cands = table.get(b["pathCrc"], set()) + path = next(iter(cands)) if len(cands) == 1 else None + r: dict = {"path": path, "attribute": b.get("attribute")} + if len(cands) > 1: + r["pathCandidates"] = sorted(cands) + if r["attribute"] is None: + r["attribute"] = _attribute(h, b, root, path, material) + out.append(r) + return out + + +def _attribute(h: _Hierarchy, b: dict, root: str, path: str | None, material) -> str | None: + tid = b["typeID"] + if tid == 114: + script = b.get("scriptClass") + + def match(c: dict) -> bool: + return c["type"] == "MonoBehaviour" and script is not None and c.get("script") == script + + key: object = ("script", script) + else: + cls = b.get("class") or _class_name(tid) + + def match(c: dict) -> bool: + return c["type"] == cls + + key = cls + nodes = h.nodes.get(root if path == "" else f"{root}/{path}", []) if path is not None else [] + bound = [c for n in nodes for c in n["components"] if match(c)] + if b.get("customType") == RENDERER_MATERIAL: + kinds: dict[str, str] = {} + for c in bound: + for ref in (c.get("fields") or {}).get("m_Materials") or (): + m = material(ref) if ref is not None else None + if m is not None: + kinds.update({k: "float" for k in m["floats"]}) + kinds.update({k: "vector" for k in m["colors"]}) + return material_attribute(b["attributeCrc"], kinds) + a = b["attributeCrc"] + hit = {x for c in bound for x in flat_names(c.get("fields")) if crc(x) == a} + if len(hit) != 1: + names = set(ENGINE_PROPS) | set(RECT_PROPS) | set(material_prop_names()) | h.class_names(match, key) + hit = {x for x in names if crc(x) == a} + return next(iter(hit)) if len(hit) == 1 else None diff --git a/src/nnnotes/unity/export.py b/src/nnnotes/unity/export.py index 4b0eb68..2f8b1cc 100644 --- a/src/nnnotes/unity/export.py +++ b/src/nnnotes/unity/export.py @@ -26,7 +26,6 @@ import json import math import re -import zlib from pathlib import Path from typing import Literal @@ -37,6 +36,7 @@ from ..core import cache from ..source.catalog import Catalog +from .bindings import ENGINE_PROPS, RECT_PROPS, crc, flat_names, material_attribute, material_prop_names from .convert.animation import CLASS_ID, TRANSFORM_ATTR, controller_doc, streamed_frames from .convert.material import material_doc from .reader import ( @@ -116,100 +116,6 @@ def _rid(o, name: str) -> str: return f"{name}#{hashlib.sha1(f'{o.assets_file.name}:{o.path_id}'.encode()).hexdigest()[:8]}" -def _crc(s: str) -> int: - return zlib.crc32(s.encode("utf-8")) - - -def flat_names(tt, prefix: str = "") -> list[str]: - """Every field path of a typetree ("m_Color", "m_Color.r", "m_Enabled", ...); lists are leaves.""" - out = [] - if isinstance(tt, dict): - for k, v in tt.items(): - p = f"{prefix}.{k}" if prefix else k - out.append(p) - out.extend(flat_names(v, p)) - return out - - -# --- clip bindings ------------------------------------------------------------ -# Animatable property names a binding can name (attribute = crc32 of the name) that are not field paths of a -# component the exporter has seen: common component properties, RectTransform driven fields, and material -# properties ("material.[.]", renderer material property blocks). -ENGINE_PROPS = ( - "m_IsActive", - "m_Enabled", - "m_Color.r", - "m_Color.g", - "m_Color.b", - "m_Color.a", - "m_Size.x", - "m_Size.y", - "m_FlipX", - "m_FlipY", - "m_SortingOrder", - "m_Sprite", - "m_Alpha", - "m_SizeDelta.x", - "m_SizeDelta.y", - "m_AnchoredPosition.x", - "m_AnchoredPosition.y", - "m_LocalPosition.x", - "m_LocalPosition.y", - "m_LocalPosition.z", - "m_LocalScale.x", - "m_LocalScale.y", - "m_LocalScale.z", -) -RECT_PROPS = ( - "m_AnchoredPosition.x", - "m_AnchoredPosition.y", - "m_SizeDelta.x", - "m_SizeDelta.y", - "m_AnchorMin.x", - "m_AnchorMin.y", - "m_AnchorMax.x", - "m_AnchorMax.y", - "m_Pivot.x", - "m_Pivot.y", -) -MATERIAL_PROPS = ( - "_Color", - "_TintColor", - "_MainTex_ST", - "_BaseColor", - "_EmissionColor", - "_Alpha", - "_Cutoff", - "_Intensity", - "_Offset", - "_Scroll", - "_ScrollX", - "_ScrollY", - "_Power", - "_Value", - "_Glow", - "_GlowColor", - "_AddColor", - "_MultiplyColor", - "_Progress", - "_Fade", - "_Brightness", - "_Dissolve", - "_MainTex", - "_Speed", - "_Rotation", -) - - -def material_prop_names(extra) -> list[str]: - out = [] - for p in sorted(set(MATERIAL_PROPS) | set(extra)): - out.append(f"material.{p}") - for c in ("r", "g", "b", "a", "x", "y", "z", "w"): - out.append(f"material.{p}.{c}") - return out - - def _events(tt: dict) -> list[dict]: return [ {x: e[x] for x in ("time", "functionName", "data", "floatParameter", "intParameter", "messageOptions")} @@ -822,7 +728,7 @@ def _skin(m, tt: dict, vertex_count: int) -> dict: def _crc_of(self, name: str) -> int: c = self._crcs.get(name) if c is None: - c = self._crcs[name] = _crc(name) + c = self._crcs[name] = crc(name) return c def _names(self, o, tt: dict | None = None) -> list[str]: @@ -932,15 +838,8 @@ def _bound_names(self, r: dict) -> set[str]: return names def _material_attribute(self, r: dict) -> str | None: - """Attribute of a renderer material property binding (customType 22, RendererMaterial): - attribute = (crc32(property) & 0x0FFFFFFF) | channel << 28, bit 30 set for colour channels (r, g, b, a), - clear for vector channels (x, y, z, w) or a float property. The property is one of the materials of the - renderer the binding names; returns "material.[.]" (Unity's property path). - The bit layout is native engine code (not in the C# reference); the layout above is the one the data - shows: the four _TintColor bindings of the live intro's lane_mesh carry 4..7 << 28 over - crc32("_TintColor") & 0x0FFFFFFF, with 4..6 constant (0.538, 0.510, 2.040: the effect's _TintColor rgb) - and 7 animated 0 -> 1 -> 0 (alpha).""" - a = r["attributeCrc"] + """Attribute of a renderer material property binding (customType 22): a property of the materials of the + renderer the binding names (bindings.material_attribute).""" kinds: dict[str, str] = {} for g, go_pid in self._binding_gos(r): for c in g.go[go_pid]["m_Component"]: @@ -954,13 +853,7 @@ def _material_attribute(self, r: dict) -> str | None: sp = mo.read_typetree()["m_SavedProperties"] kinds.update({k: "float" for k, _ in sp.get("m_Floats", [])}) kinds.update({k: "vector" for k, _ in sp.get("m_Colors", [])}) - hits = [k for k in kinds if self._crc_of(k) & 0x0FFFFFFF == a & 0x0FFFFFFF] - if len(hits) != 1: - return None - name, channel, colour = hits[0], (a >> 28) & 3, (a >> 30) & 1 - if kinds[name] == "float" and not colour and channel == 0: - return f"material.{name}" - return f"material.{name}.{('rgba' if colour else 'xyzw')[channel]}" + return material_attribute(r["attributeCrc"], kinds) def _class_names(self, r: dict) -> set[str]: """Names any binding of the class can target: engine properties, material properties of the diff --git a/tests/unity/test_bindings.py b/tests/unity/test_bindings.py new file mode 100644 index 0000000..c10f216 --- /dev/null +++ b/tests/unity/test_bindings.py @@ -0,0 +1,149 @@ +"""Clip bindings named in a synthetic prefab hierarchy (nnnotes.unity.bindings).""" + +import pytest + +from nnnotes.unity import bindings +from nnnotes.unity.bindings import crc + +PARAM = "Live2D.Cubism|Live2D.Cubism.Core|CubismParameter" +PART = "Live2D.Cubism|Live2D.Cubism.Core|CubismPart" +MAT = {"$ref": {"file": "CAB-m", "pathId": 7, "class": "Material"}} + + +def mono(script, **fields): + return {"type": "MonoBehaviour", "pathId": 1, "script": script, "fields": fields} + + +def node(path, *components): + return {"path": path, "name": path.rsplit("/", 1)[-1], "components": list(components)} + + +def prefab(): + return { + "name": "model", + "nodes": [ + node("model", mono("Live2D.Cubism|Live2D.Cubism.Rendering|CubismRenderController", Opacity=1.0)), + node("model/Parameters"), + node("model/Parameters/ParamAngleX", mono(PARAM, Value=0.0, MinimumValue=-30.0)), + node("model/Parameters/ParamAngleY", mono(PARAM, Value=0.0, MinimumValue=-30.0)), + node("model/Parts"), + node("model/Parts/PartArm", mono(PART, Opacity=1.0)), + node("model/Drawables"), + node( + "model/Drawables/ArtMesh0", + {"type": "MeshRenderer", "pathId": 2, "fields": {"m_Enabled": 1, "m_Materials": [MAT, None]}}, + ), + node("model/Twin"), + node("model/Twin"), + node("other"), + ], + } + + +def binding(path, attr, type_id=114, script=None, **kw): + b = {"pathCrc": crc(path), "attributeCrc": crc(attr) if isinstance(attr, str) else attr} + b.update(typeID=type_id, customType=0, pptr=False, int=False, serializeReference=False) + if script is not None: + b.update(script={"$ref": {"file": "CAB-s", "pathId": 3}}, scriptClass=script) + b.update(kw) + return b + + +def test_paths_are_relative_to_the_animators_game_object(): + out = bindings.resolve( + [ + binding("Parameters/ParamAngleX", "Value", script=PARAM), + binding("Parts/PartArm", "Opacity", script=PART), + binding("", "Opacity", script="Live2D.Cubism|Live2D.Cubism.Rendering|CubismRenderController"), + binding("Parameters/ParamGone", "Value", script=PARAM), + ], + prefab(), + "model", + ) + assert out == [ + {"path": "Parameters/ParamAngleX", "attribute": "Value"}, + {"path": "Parts/PartArm", "attribute": "Opacity"}, + {"path": "", "attribute": "Opacity"}, + {"path": None, "attribute": "Value"}, # not below the Animator: a field of the class in the hierarchy + ] + below = bindings.resolve([binding("ParamAngleY", "Value", script=PARAM)], prefab(), "model/Parameters") + assert below == [{"path": "ParamAngleY", "attribute": "Value"}] + + +def test_a_path_two_transforms_share_names_them_both(): + (r,) = bindings.resolve([binding("Twin", "m_IsActive", type_id=1)], prefab(), "model") + assert r == {"path": "Twin", "attribute": "m_IsActive"} # an engine property + + +def test_the_bound_component_is_the_bindings_script(): + other = "Game|Game|Other" + (r,) = bindings.resolve([binding("Parameters/ParamAngleX", "MinimumValue", script=other)], prefab(), "model") + assert r == {"path": "Parameters/ParamAngleX", "attribute": None} # no Other there, nor anywhere + + +def test_a_transform_bindings_attribute_is_the_clips(): + b = binding("Parts", 2, type_id=4, attribute="m_LocalRotation", curves=4) + assert bindings.resolve([b], prefab(), "model") == [{"path": "Parts", "attribute": "m_LocalRotation"}] + + +def test_a_renderer_material_binding_names_a_property_of_its_materials(): + tint = crc("_TintColor") & 0x0FFFFFFF + docs = {7: {"floats": {"_Cutoff": 0.5}, "colors": {"_TintColor": {"r": 1, "g": 1, "b": 1, "a": 1}}}} + + def material(ref): + return docs.get(ref["$ref"]["pathId"]) + + out = bindings.resolve( + [ + binding( + "Drawables/ArtMesh0", tint | 3 << 28 | 1 << 30, type_id=23, customType=22, **{"class": "MeshRenderer"} + ), + binding("Drawables/ArtMesh0", tint | 1 << 28, type_id=23, customType=22), + binding("Drawables/ArtMesh0", crc("_Cutoff") & 0x0FFFFFFF, type_id=23, customType=22), + binding("Drawables/ArtMesh0", crc("_Other") & 0x0FFFFFFF, type_id=23, customType=22), + ], + prefab(), + "model", + material, + ) + assert [r["attribute"] for r in out] == ["material._TintColor.a", "material._TintColor.y", "material._Cutoff", None] + assert bindings.resolve([binding("Drawables/ArtMesh0", tint, type_id=23, customType=22)], prefab(), "model") == [ + {"path": "Drawables/ArtMesh0", "attribute": None} # its materials unknown + ] + + +def test_a_field_of_the_bound_renderer(): + (r,) = bindings.resolve([binding("Drawables/ArtMesh0", "m_Enabled", type_id=23)], prefab(), "model") + assert r == {"path": "Drawables/ArtMesh0", "attribute": "m_Enabled"} + + +def test_an_ambiguous_path_lists_its_candidates(monkeypatch): + monkeypatch.setattr(bindings, "crc", lambda s: 1 if s in ("Parts", "Drawables") else 0x7FFFFFFF) + (r,) = bindings.resolve([{"pathCrc": 1, "attributeCrc": 5, "typeID": 1}], prefab(), "model") + assert r == {"path": None, "attribute": None, "pathCandidates": ["Drawables", "Parts"]} + + +def test_the_animators_game_object_is_in_the_hierarchy(): + with pytest.raises(KeyError, match="not in the hierarchy"): + bindings.resolve([], prefab(), "nope") + + +def test_material_attribute_layout(): + kinds = {"_Color": "vector", "_Alpha": "float"} + c = crc("_Color") & 0x0FFFFFFF + assert bindings.material_attribute(c | 0 << 28 | 1 << 30, kinds) == "material._Color.r" + assert bindings.material_attribute(c | 2 << 28, kinds) == "material._Color.z" + assert bindings.material_attribute(crc("_Alpha") & 0x0FFFFFFF, kinds) == "material._Alpha" + assert bindings.material_attribute(crc("_Alpha") & 0x0FFFFFFF | 1 << 30, kinds) == "material._Alpha.r" + assert bindings.material_prop_names(["_Extra"])[:3] == [ + "material._AddColor", + "material._AddColor.r", + "material._AddColor.g", + ] + assert "material._Extra.w" in bindings.material_prop_names(["_Extra"]) + assert bindings.flat_names({"m_Color": {"r": 1, "g": 0}, "list": [1]}) == [ + "m_Color", + "m_Color.r", + "m_Color.g", + "list", + ] From 33426711f3fac5ea7bcae602e423b4c47b85d783 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 11:07:54 +0800 Subject: [PATCH 17/91] feat(live2d)!: assemble Live2D models from the exported objects Two product stages make a Live2D model from the unity.export results of the objects its prefab reaches, never from a bundle: - live2d.model (version 2) is the model the player reads: model.json (format 3) names the prefab, the moc, the canvas, the atlas pages, the shaders, the mask materials loaded by Resources.Load path, MotionSync and the bindings of every clip named in the model's hierarchy. Its object table drops the MonoScripts and keeps, of a shader, the parsed form, the variant index and the GLSL ES 3.00 programs the drawn materials use. - live2d.cubism is the model as Cubism SDK files: moc3, atlas pages, physics3, exp3, motion3 and model3. `export assets --layout product` places the products of the selection: each product stage makes the products of the keys the selected bundles hold, and without an object layout the run exports only what the products reach. `export live2d MODEL -o OUT` runs the same pipeline for one model and writes its Cubism SDK files, or with `--player` the player's model in the product layout. The Cubism format conversions live in nnnotes.formats.cubism, the choice of the player's shader program in formats.shader.player_program, and the model ids in content.live2d; the site's model export uses them from there. BREAKING CHANGE: `export live2d` writes the Cubism SDK files only (no prefab JSON, no motions/_fades.json) and runs through the depot; live2d.model tasks take the object table of the model instead of its bundles. Co-Authored-By: Claude Opus 5.5 --- docs/assets.md | 10 +- docs/cli.md | 12 +- docs/commands.md | 21 +- docs/export.md | 6 +- docs/products.md | 46 +++- docs/stages.md | 4 +- src/nnnotes/cli/assets.py | 108 +++++++- src/nnnotes/cli/export.py | 37 ++- src/nnnotes/content/live2d.py | 368 ++++++++++++++++++++++++++++ src/nnnotes/formats/cubism.py | 106 ++++++++ src/nnnotes/formats/motion.py | 37 --- src/nnnotes/formats/shader.py | 28 ++- src/nnnotes/pipeline/assemble.py | 4 + src/nnnotes/pipeline/layout.py | 23 +- src/nnnotes/pipeline/live2d.py | 87 +++++++ src/nnnotes/pipeline/modelstages.py | 241 +----------------- src/nnnotes/site/sitefiles.py | 4 +- src/nnnotes/site/storysite.py | 4 +- src/nnnotes/site/webmodel.py | 98 ++------ src/nnnotes/unity/bindings.py | 4 +- src/nnnotes/unity/live2d.py | 181 +------------- tests/cli/test_assets.py | 98 +++++++- tests/content/test_live2d.py | 161 ++++++++++++ tests/helpers/live2dmodel.py | 243 ++++++++++++++++++ tests/pipeline/test_live2d.py | 84 +++++++ tests/pipeline/test_modelstages.py | 245 +----------------- tests/site/test_webmodel.py | 28 --- 27 files changed, 1439 insertions(+), 849 deletions(-) create mode 100644 src/nnnotes/content/live2d.py create mode 100644 src/nnnotes/formats/cubism.py create mode 100644 src/nnnotes/pipeline/live2d.py create mode 100644 tests/content/test_live2d.py create mode 100644 tests/helpers/live2dmodel.py create mode 100644 tests/pipeline/test_live2d.py diff --git a/docs/assets.md b/docs/assets.md index 96e04b3..e3fb2ff 100644 --- a/docs/assets.md +++ b/docs/assets.md @@ -18,7 +18,7 @@ nnnotes depot plan --check && echo "up to date" ## Commands ``` -nnnotes export assets -o OUT [--layout original[,cas]] [--select group:G|key:PREFIX|bundle:GLOB ...] [--views all|none|a,b] +nnnotes export assets -o OUT [--layout original[,cas][,product]] [--select group:G|key:PREFIX|bundle:GLOB ...] [--views all|none|a,b] [--depot DIR] [--workers N] [--memory GiB] [--fetch-workers N] [--stored-catalog L|SHA] [--png-level N] [--flac-level N] [--only-class C,...] [--link auto|clone|hard|copy] [--strict] [--dry-run] [--explain] @@ -37,7 +37,7 @@ nnnotes depot verify [--quick] [--workers N] | `-o OUT` | output directory | | `--select SEL` | what to export (repeatable, the union; default everything): `group:G` the bundles of an Addressables group (the stable bundle name up to `_assets_`, else up to its first `_`), `key:PREFIX` the bundles and raw files the catalog keys under `PREFIX` load, `bundle:GLOB` the bundles and raw files whose stable name matches | | `--views` | the semantic views to build ([views.md](views.md)): `all`, `none` or names, comma-separated; default `all` when master data is set, else `none` | -| `--layout` | `original`, `cas` or both (below) | +| `--layout` | `original`, `cas`, `product`, comma-separated (below) | | `--depot DIR` | the depot (`[paths] depot`) | | `--workers N` | worker processes (default: the CPUs the process may run on, its affinity mask as `taskset` or a container's cpuset sets it; `0`: every task in this process) | | `--memory GiB` | memory budget (default: 80 % of the physical memory where it can be read, else none) | @@ -183,6 +183,12 @@ replaces or removes it where the system needs that (Windows), and the depot obje from the `original` manifest alone, `original` from `cas` and the results. The views are the exception: `original` has them with the paths of their objects, `cas` the view documents they are made from. +`product` places the products of the selection ([products](products.md)): every product stage makes its products +of the catalog keys the selected bundles hold (the Live2D stages: each model key), and the layout writes each +product's manifest (`models/.json`, `cubism/.json`) with its files and the files of its objects at +`assets/.`. The product stages run only with this layout. Without `original` or `cas`, the run exports +only the objects the products reach, not every object of the selection. + ## Reports `OUT/_reports/`: diff --git a/docs/cli.md b/docs/cli.md index ec38edb..158e10c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -178,7 +178,7 @@ every object of the selected bundles -> common formats (the depot and its layout | `--workers` `N` | worker processes (default: the CPUs this process may use; 0: this process) | | `--memory` `GiB` | memory budget of the running tasks and workers (default: 80 % of the physical memory) | | `--fetch-workers` `N` | parallel downloads (default 8) | -| `--layout` `LAYOUT` | original, cas or original,cas (default original) | +| `--layout` `LAYOUT` | original, cas, product, comma-separated (default original; product: the products of the selection) | | `--explain` | print every setting of the run first | | `--link` `LINK` | how the layout files are made from the depot: auto (default; [export] link): a clone where the file system makes them, else a hard link (read-only), else a copy; or that method only (one of `auto`, `clone`, `hard`, `copy`) | | `--strict` | exit 1 when a view has gaps | @@ -228,12 +228,18 @@ At most one of `--band`, `--leader-card`. ## nnnotes export live2d -a Live2D model -> runtime model directory +a Live2D model -> its Cubism SDK files, or the player's model | Argument | Description | | --- | --- | | `key` | model key Character/Live2D///model/, or its model id | | `-o`, `--out` `OUT` | output directory (required) | +| `--player` | write the player's model (models/.json and assets/) instead of the Cubism SDK files | +| `--stored-catalog` `LABEL\|SHA` | a catalog version of the depot (`catalog versions`; default: the current catalog) | +| `--workers` `N` | worker processes (default: the usable CPUs) | +| `--link` `LINK` | as export assets --link (one of `auto`, `clone`, `hard`, `copy`) | +| `--explain` | print every setting of the run first | +| `--dry-run` | print the plan, run nothing | ## nnnotes export spot @@ -321,7 +327,7 @@ what `export assets` would run and why, without running it (--json: nnnotes.plan | `--workers` `N` | worker processes (default: the CPUs this process may use; 0: this process) | | `--memory` `GiB` | memory budget of the running tasks and workers (default: 80 % of the physical memory) | | `--fetch-workers` `N` | parallel downloads (default 8) | -| `--layout` `LAYOUT` | original, cas or original,cas (default original) | +| `--layout` `LAYOUT` | original, cas, product, comma-separated (default original; product: the products of the selection) | | `--explain` | print every setting of the run first | | `--since` `RUN` | explain against this run (default: the latest of the selection) | | `--check` | exit 1 when anything would run or change | diff --git a/docs/commands.md b/docs/commands.md index fe4738b..db5be94 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -287,24 +287,29 @@ way the TMP shader applies them. The document-level `textStyle.roles` gives, per ## export live2d ``` -nnnotes export live2d MODEL -o OUT +nnnotes export live2d MODEL -o OUT [--player] ``` -A Live2D (Cubism) model prefab as a Cubism runtime directory. `MODEL` is the model's key +A Live2D (Cubism) model as the Cubism SDK files other tools read. `MODEL` is the model's key `Character/Live2D///model/` or its id `` (`nnnotes catalog keys --prefix Character/Live2D/` lists the keys): ``` OUT/.moc3 the model's moc3 -OUT/textures/*.png atlas pages -OUT/.prefab.json the whole prefab: every GameObject and component, motions and expressions inlined -OUT/.exp3.json expressions +OUT/textures/.png the atlas pages the drawables draw with, by name OUT/.physics3.json physics (not written for a model without physics) -OUT/motions/.motion3.json motions (and motions/_fades.json) -OUT/.model3.json file references, EyeBlink / LipSync groups +OUT/.exp3.json expressions +OUT/motions/.motion3.json motions +OUT/.model3.json file references, the model's fade times per motion, EyeBlink / LipSync groups ``` -Needs `[paths] apk` (component classes are resolved through the APK). Prints a JSON summary. +With `--player`, the model the player reads instead: the product layout of `live2d.model`, `OUT/models/.json` +and the files it names under `OUT/assets/` ([products](products.md#live2d-models)). + +The command runs the depot's pipeline for the model alone (the census of the bundles of its key, the objects its +prefab reaches, the product) like `export assets`, and writes the same reports in `OUT/_reports/`; it takes +`--stored-catalog`, `--workers`, `--link`, `--explain` and `--dry-run` as `export assets` does. Needs `[paths] apk` +(the component classes and the mask materials are in the APK). ## export spot diff --git a/docs/export.md b/docs/export.md index 6cce4fc..dbf7d9c 100644 --- a/docs/export.md +++ b/docs/export.md @@ -51,9 +51,9 @@ nnnotes catalog keys --prefix Character/Live2D/ --limit 0 # every model key nnnotes export live2d adv_live2d_rana_003_casual_spring_01 -o out/live2d/rana ``` -`MODEL` is the model's key or its id, the last part of the key. The output is a Cubism runtime directory: moc3, atlas -pages, motions, expressions, physics, `model3.json` and the whole prefab as JSON. Details: -[export live2d](commands.md#export-live2d). +`MODEL` is the model's key or its id, the last part of the key. The output is the model's Cubism SDK files: moc3, +atlas pages, motions, expressions, physics and `model3.json`; `--player` writes the model the player reads instead. +Details: [export live2d](commands.md#export-live2d). ## A spot or a room diff --git a/docs/products.md b/docs/products.md index d9c5158..1aa989a 100644 --- a/docs/products.md +++ b/docs/products.md @@ -59,13 +59,18 @@ object), or the reason code of a generic, unsupported or failed object. ## Product layout -A product layout places products for a player: +A product layout places products for a player (`export assets --layout product`, `export live2d --player`): - every file of a product and every file of the objects of its object table at `assets/.` (a file several products or objects share is written once); - each product's manifest at `/.json` (the stage's `manifest(subject)`, by default `/.json` with the subject escaped as one path segment). +A command asks for products in one of two ways: `export assets --layout product` makes, with each product stage, the +products of the catalog keys the selected bundles hold (the stage's `select`); `export live2d` asks for one model. +`export live2d` without `--player` places the files of its one product at their own paths instead (the `files` +placement: the Cubism SDK files as a directory). + ## Product manifest: nnnotes.product/1 ([product.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/product.schema.json)) @@ -83,3 +88,42 @@ A product layout places products for a player: contained in an object that has one. An asset path is relative to the directory above the manifest's directory, as in the chart manifests of ournotes-player. A reference in an object's JSON (`{"$ref": {"file", "pathId"}}`) names the object `:` of `objects`. + +## Live2D models + +Two product stages (`nnnotes.pipeline.live2d`) assemble a Live2D model from the objects its prefab reaches. Their +subjects are model keys, `Character/Live2D///model/`; a model's id is its ``. + +`live2d.model` is the model the player reads. Its roots are the model's GameObject (the object of its key) and the +materials the Cubism mask pass draws with, which the APK's built-in files hold at the `Resources.Load` paths +`Live2D/Cubism/Materials/Mask` and `Live2D/Cubism/Materials/MaskCulling`. Its manifest is `models/.json`, and its +one file `model.json` (format 3) names objects of its object table: + +| Field | | +|---|---| +| `name`, `key` | the model's id and key | +| `prefab` | the object id of the model's GameObject; its `json` is the hierarchy (prefab.json) | +| `moc` | the object id of its CubismMoc (the root's `CubismModel._moc`); the moc3 is its file `blob:_bytes` | +| `canvas` | the moc3's canvas: `pixelsPerUnit`, `originX`, `originY`, `width`, `height`, `mocVersion` | +| `textures` | the object ids of the atlas pages the drawables draw with (`CubismRenderer._mainTexture`), by name | +| `shaders` | `{shader name: object id}` of the shaders of the object table | +| `resources` | `{"cubismMask", "cubismMaskCulling"}`: the object ids of the two mask materials | +| `motionSync` | the root has the MotionSync controller and its CRI audio input | +| `bindings` | `{clip object id: [{path, attribute}]}`: the bindings of every AnimationClip of the table named in the model's hierarchy, played from its root (`nnnotes.unity.bindings`; `null` where no name, or several, fit) | + +Its object table holds every object the model reaches but the MonoScripts. Of a shader it keeps the parsed form +(`json`), the variant index (`index`) and the GLSL ES 3.00 programs (subshader 0, pass 0) the player draws the +drawables' materials with: for each material's keywords, alone and with `_ADDITIONAL_LIGHTS_VERTEX` (the keyword the +story renderer adds to a character draw), and the mask materials' when a drawable is masked (`CUBISM_MASK_ON`). + +`live2d.cubism` is the model as the Cubism SDK files other tools read (`export live2d`); its manifest is +`cubism/.json` and it has no objects: + +| File | From | +|---|---| +| `.moc3` | the CubismMoc's bytes | +| `textures/.png` | the atlas pages, by name (a name two pages share is numbered: `-2.png`) | +| `.physics3.json` | the root's `CubismPhysicsController._rig` (when it has sub-rigs) | +| `.exp3.json` | each CubismExpressionData of the table | +| `motions/.motion3.json` | each CubismFadeMotionData, with the sample rate of the clip of its name | +| `.model3.json` | the files, the model's fade times of each motion, and the EyeBlink and LipSync groups (the GameObjects holding a CubismEyeBlinkParameter or a CubismMouthParameter) | diff --git a/docs/stages.md b/docs/stages.md index 9e8e106..4dc4fdc 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -14,7 +14,7 @@ The asset export is built from three layers: Every stage is at version 1, except `unity.census` (version 4: it lists what every object references, the hierarchy holding it and the Resources.Load paths of a ResourceManager), `link.addresses` (version 2: the -Resources.Load paths), `unity.export` (version 2: one task per unit), `cri.movie` (version 2: every stream kind, +Resources.Load paths), `live2d.model` (version 2: assembled from the exported objects), `unity.export` (version 2: one task per unit), `cri.movie` (version 2: every stream kind, codec and channel of a USM) and `catalog.index` (version 2: gzip input and Japanese remote asset placeholders). | Stage | Subject | Inputs | Outputs | @@ -26,7 +26,7 @@ codec and channel of a USM) and `catalog.index` (version 2: gzip input and Japan | `sprite.crop` | a SpriteAtlas | the atlas' JSON, the images of its textures, the metas of the sprites of other bundles it packs | the sprites' images | | `cri.audio` | ACB content | `acb` (a raw file of the catalog, or the `acb` artifact of a cue sheet held in a bundle), `awb` (an AWB the catalog pairs with it), `boot` (the game's boot data: the key's source) | FLAC per stream, `cues.json`, `streams.json` | | `cri.movie` | USM content | `usm` (a raw file of the catalog, or the `usm` artifact of a movie held in a bundle), `boot` | every stream as stored (video, alpha, audio, subtitles), `movie.mkv` (or `movie.webm`) | -| `live2d.model` | a Live2D model key (`Character/Live2D///model/`) whose bundle is selected | `bundle:`: the bundles of the key's dependency closure in the order the extractor loads them (the order is part of the key); context `internalId` (the key's location); atom `live2d.extract_model` (the extractor of `nnnotes export live2d`, run unchanged) | `live2d.model:#`: every file `nnnotes export live2d` writes for the model, `` relative to the model's directory; placed under `/` | +| `live2d.model`, `live2d.cubism` | a Live2D model key (`Character/Live2D///model/`) a command asks for | `objects`: the object table of what the model reaches ([products](products.md#live2d-models)); depends on `link.addresses` and the `unity.export` tasks it reads | `model.json` (the player's model) and the object table; the Cubism SDK files | | `spine.skeleton` | a SkeletonDataAsset of a selected bundle, by its stable address `:` (in a bundle of several serialized files the file's tag follows the bundle name, as in `link.addresses`) | `bundle:`: the bundles holding the serialized files the asset's file reaches through its externals; context `object` (the asset's object id); atom `spot.write_skeleton` (the Spine writer of `nnnotes export spot`) | `spine.skeleton:#`: the skeleton (`.json` or `.skel`), its atlases (`.atlas`) and their page images, the bytes `nnnotes export spot` writes into `spine/`; one item for the asset | | `link.addresses` | global | catalog index, censuses (the built-in files' too) | key -> objects, Resources.Load path -> objects, serialized file -> bundle or built-in file, bundle -> keys | | `view.` | a catalog (`main`) | `master:
` per master table it reads, `rules` (the view's rule and the resolvers it uses), `addresses` (the part of the address table it read: the keys looked up and the prefixes listed); no parameters, atoms or context | `view.:#view` (facts: rows, entries, gaps, unreferenced) | diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index 81d2ad4..c570b66 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -1,7 +1,7 @@ """The asset commands: the default orchestration of the stages (`export assets`), its plan and single tasks (`depot plan`, `depot run`). - nnnotes export assets -o OUT [--layout original[,cas]] [--select group:G|key:PREFIX|bundle:GLOB ...] + nnnotes export assets -o OUT [--layout original[,cas][,product]] [--select group:G|key:PREFIX|bundle:GLOB ...] [--views all|none|a,b] [--workers N] [--memory GiB] [--fetch-workers N] [--stored-catalog L|SHA] [--png-level N] [--flac-level N] [--only-class C,...] [--link auto|clone|hard|copy] [--strict] [--dry-run] [--explain] @@ -21,12 +21,16 @@ bundles {stable bundle name: Input "bundle"} of the selected bundles and their dependencies (a bundle whose bytes are not known yet: a Pending "fetch:"), and of the APK's built-in files (`apk/data.unity3d`, `apk/Resources/unity default resources`: pipeline.builtins) - selected the selected bundles' stable names, sorted (the subjects of the export stages) + selected the selected bundles' stable names, sorted (the subjects of the export stages; none when only products + are asked for: the run exports what they reach) raw {stable raw file name: Input "raw"} of the selected raw files (only with a stage that reads them; export fetches them while the stages before run, and reading the fact waits for that) boot the Input of the game's boot data (with a CRI stage: cristages.boot_input; None without an APK) master the decoded master data directory, or None views the names of the selected views, sorted + products {product stage: subjects} (with a product stage in the pipeline: the `product` layout makes each + stage's products of the keys the selected bundles hold, held_keys; `export live2d` asks for one) + roots the root objects of those products (pipeline.assemble.request) Selection (--select, repeatable; the union; none: everything): `group:G` the bundles of an Addressables group (the stable name up to `_assets_`, else up to its first `_`), `key:PREFIX` the bundles and raw files the catalog keys @@ -69,7 +73,8 @@ from .common import nonnegative, print_json, usage GIB = 1 << 30 -LAYOUT_NAMES = ("original", "cas") +LAYOUT_NAMES = ("original", "cas", "product") +OBJECT_LAYOUTS = ("original", "cas") # the layouts of the objects: they ask for every object of the selection FETCH_WORKERS = 8 RECYCLE_TASKS = 500 RECYCLE_RSS = 3 * GIB # a worker whose resident memory after a task is above this is replaced ... @@ -88,12 +93,14 @@ class StageSource: """Where a stage (or, for a name ending with '.', a family of stages) comes from. `output`: its artifacts are placed in the layouts; `root`: the layout root of its artifacts without a container path (format fields - {stage} and {subject}; a stage's LAYOUT_ROOT attribute wins).""" + {stage} and {subject}; a stage's LAYOUT_ROOT attribute wins); `product`: a product stage (pipeline.assemble), + in the pipeline only when products are asked for, its results placed by the product layouts only.""" name: str target: str output: bool = False root: str | None = None + product: bool = False def matches(self, stage: str) -> bool: return stage.startswith(self.name) if self.name.endswith(".") else stage == self.name @@ -108,8 +115,9 @@ def matches(self, stage: str) -> bool: StageSource("sprite.crop", "nnnotes.pipeline.objexport:SpriteCropStage", output=True), StageSource("cri.audio", "nnnotes.pipeline.cristages:AudioStage", output=True), StageSource("cri.movie", "nnnotes.pipeline.cristages:MovieStage", output=True), - StageSource("live2d.model", "nnnotes.pipeline.modelstages:Live2DStage", output=True), StageSource("spine.skeleton", "nnnotes.pipeline.modelstages:SpineStage", output=True), + StageSource("live2d.model", "nnnotes.pipeline.live2d:ModelStage", output=True, product=True), + StageSource("live2d.cubism", "nnnotes.pipeline.live2d:CubismStage", output=True, product=True), StageSource("view.", "nnnotes.pipeline.views:stages", output=True), StageSource("view.voices", "nnnotes.pipeline.voices:VoiceStage", output=True), ) @@ -232,6 +240,23 @@ def select(index: dict, selection: list[str], builtin=()) -> Selection: return Selection(sorted(pick(chosen, "bundle") + own), sorted(pick(deps, "bundle") + builtin), pick(chosen, "raw")) +def held_keys(index: dict, bundles) -> list[str]: + """The catalog keys of the asset locations the bundles `bundles` hold (a location's first dependency is the + bundle holding it), sorted.""" + from ..pipeline import catalogdb + + files, want = catalogdb.file_refs(index), {("bundle", b) for b in bundles} + return sorted( + { + loc["primaryKey"] + for loc in index["locations"] + if loc["kind"] not in catalogdb.FILE_KINDS + and loc["dependencies"] + and files.get(loc["dependencies"][0]) in want + } + ) + + # ---------------------------------------------------------------- settings def default_memory() -> int | None: """80 % of the physical memory (Linux), else None (no budget).""" @@ -459,11 +484,13 @@ class Workspace: """What export and plan share: the depot, the catalog version, its index, the selection, the pipeline and its parameters, the facts.""" - def __init__(self, args, cfg, log=_log): + def __init__(self, args, cfg, log=_log, layouts=()): from ..pipeline.catalogdb import CatalogDB from ..pipeline.depot import open_depot self.args, self.cfg, self.log = args, cfg, log + self.layouts = list(layouts) + self.products: dict | None = None # {product stage: subjects} a command asks for (request) self.root = Session(cfg).depot_root() self.apk = cfg.path("paths", "apk") self.fetcher = CatalogFetcher(self.root, cfg) @@ -613,9 +640,19 @@ def views(self) -> list[str]: raise self.cfg.missing(*self.cfg.master(self.cfg.get("catalog", "region"))) return want + def request(self, products: dict, selection: list[str]) -> None: + """Ask for the products `products` {product stage: subjects} alone, with the selection `selection` (what + their census needs) instead of the command's: only what the products reach is exported.""" + self.products, self.selection = {k: sorted(v) for k, v in products.items()}, list(selection) + + def _wanted(self, name: str) -> bool: + if not self.installed.sources[name].product: + return True + return name in self.products if self.products is not None else "product" in self.layouts + def pipeline(self) -> list[str]: views = set(self.views()) - return [n for n in self.installed.stages if not n.startswith("view.") or n in views] + return [n for n in self.installed.stages if (not n.startswith("view.") or n in views) and self._wanted(n)] def params(self, pipeline: list[str]) -> dict: """{stage: parameters} from the flags; a flag no stage of the pipeline takes is a usage error.""" @@ -695,6 +732,19 @@ def facts(self, pipeline: list[str], fetch: bool, background: bool = False) -> d "classes": self.classes(), "raw": Inputs({}), } + products = [n for n in pipeline if self.installed.sources[n].product] + if products: + from ..pipeline import assemble + + if self.products is not None: + chosen = {n: self.products.get(n, []) for n in products} + else: + held = held_keys(index, sel.selected) + chosen = {n: self.installed.stages[n].select(held) for n in products} + facts["products"] = chosen + facts["roots"] = assemble.request(self.installed.stages, chosen, Env(self.depot, facts)) + if self.products is not None or not set(self.layouts) & set(OBJECT_LAYOUTS): + facts["selected"] = [] # products alone: what they reach if any(n in pipeline for n in RAW_STAGES): from ..pipeline.cristages import boot_input @@ -906,9 +956,13 @@ def layout_entries(self, name: str, placed: LayoutInput, *, store_derived: bool from ..pipeline import layout stages = self.installed.stages + if name not in OBJECT_LAYOUTS: + return self._product_entries(name, placed, store_derived) sources, named = [], [] for tid, doc in placed.results: stage_name, subject = contract.parse_task_id(tid) + if self.installed.sources[stage_name].product: + continue stage = stages[stage_name] names = self.names(stage_name).get(subject) if name == "original" else None if names: @@ -937,6 +991,25 @@ def layout_entries(self, name: str, placed: LayoutInput, *, store_derived: bool taken.add(e["path"].casefold()) return sorted(entries + extra, key=lambda e: (e["path"], e["id"])), report + def _product_entries(self, name: str, placed: LayoutInput, store: bool) -> tuple[list[dict], dict]: + """The entries of a product layout (`product`: manifests and files; `files`: the products' own files at + their paths) for the product results given to `placed`, and its report.""" + from ..pipeline import layout + + results = [(t, d) for t, d in placed.results if self.installed.sources[contract.parse_task_id(t)[0]].product] + if name == "files": + return layout.product_files(d for _, d in results), {"collisions": [], "products": len(results)} + products = [] + for tid, doc in results: + stage_name, subject = contract.parse_task_id(tid) + rec = next(a for a in doc["artifacts"] if a["id"] == contract.artifact_id(tid, layout.PRODUCT_TABLE)) + table = contract.loads(self.depot.read(rec["content"]["sha256"])) + body = contract.encode(layout.product_manifest(tid, doc, table)) + sha = self.depot.put(body) if store else contract.sha256(body) + manifest = {"id": contract.artifact_id(tid, "manifest"), "sha256": sha, "size": len(body)} + products.append((self.installed.stages[stage_name].manifest(subject), manifest, doc, table)) + return layout.product_entries(products), {"collisions": [], "products": len(products)} + def done(self, tasks: dict) -> None: """The tasks whose results the layouts place, {task id: key} (after a run: every task with a result).""" self._done, self._names = dict(tasks), {} @@ -1161,15 +1234,20 @@ def _write_report(d: Path, name: str, doc) -> Path: def cmd_export(args, cfg): - from ..pipeline import builtins, layout - from ..pipeline.orchestrate import Orchestrator, failure - try: names = parse_layouts(args.layout) except ValueError as e: usage(args, str(e)) use(cfg) # the tools the atoms name are part of the keys - ws = Workspace(args, cfg) + export(args, cfg, Workspace(args, cfg, layouts=names)) + + +def export(args, cfg, ws: Workspace) -> None: + """Run the request of `ws` and place its layouts (ws.layouts) in args.out (cmd_export; export live2d).""" + from ..pipeline import builtins, layout + from ..pipeline.orchestrate import Orchestrator, failure + + names = ws.layouts pipeline = ws.pipeline() params = ws.params(pipeline) orch = Orchestrator( @@ -1334,7 +1412,7 @@ def cmd_plan(args, cfg): except ValueError as e: usage(args, str(e)) use(cfg) # the tools the atoms name are part of the keys - ws = Workspace(args, cfg) + ws = Workspace(args, cfg, layouts=layouts) pipeline = ws.pipeline() params = ws.params(pipeline) if args.explain: @@ -1552,5 +1630,9 @@ def run_args(c) -> None: help="memory budget of the running tasks and workers (default: 80 %% of the physical memory)", ) c.add_argument("--fetch-workers", type=int, metavar="N", help=f"parallel downloads (default {FETCH_WORKERS})") - c.add_argument("--layout", default="original", help="original, cas or original,cas (default original)") + c.add_argument( + "--layout", + default="original", + help="original, cas, product, comma-separated (default original; product: the products of the selection)", + ) c.add_argument("--explain", action="store_true", help="print every setting of the run first") diff --git a/src/nnnotes/cli/export.py b/src/nnnotes/cli/export.py index a59b440..bc35a77 100644 --- a/src/nnnotes/cli/export.py +++ b/src/nnnotes/cli/export.py @@ -9,6 +9,7 @@ from pathlib import Path from ..core.jsonio import write_json +from ..source.config import use from ..source.session import Session from . import assets from .common import ( @@ -24,6 +25,7 @@ known_row, live_option_arg, live_options, + nonnegative, out_arg, report, usage, @@ -122,16 +124,20 @@ def cmd_live(args, cfg): def cmd_live2d(args, cfg): - from ..site import webmodel - from ..unity import live2d - - cfg.require_path("paths", "apk") # the Cubism component classes are read from the APK - cat = Session(cfg).catalog() + from ..content import live2d + from ..pipeline.live2d import CUBISM, MODEL + + cfg.require_path("paths", "apk") # the Cubism component classes and the mask materials are in the APK + use(cfg) # the tools the atoms name are part of the keys + ws = assets.Workspace(args, cfg, layouts=["product" if args.player else "files"]) + ws.run_index() + keys = (loc["primaryKey"] for loc in ws.index()["locations"]) try: # a model id or a model key - (key,) = webmodel.catalog_models(cat, [args.key]).values() + (key,) = live2d.select_models([args.key], live2d.model_keys(keys)).values() except ValueError as e: usage(args, str(e)) - report(args, live2d.extract_model(cat, key, Path(args.out))) + ws.request({MODEL if args.player else CUBISM: [key]}, [f"key:{key}"]) + assets.export(args, cfg, ws) def cmd_spot(args, cfg): @@ -293,9 +299,24 @@ def register(sub) -> None: live_option_arg(c) out_arg(c, "output directory") - c = command(es, "live2d", cmd_live2d, "a Live2D model -> runtime model directory") + c = command(es, "live2d", cmd_live2d, "a Live2D model -> its Cubism SDK files, or the player's model") c.add_argument("key", help="model key Character/Live2D///model/, or its model id ") out_arg(c, "output directory") + c.add_argument( + "--player", + action="store_true", + help="write the player's model (models/.json and assets/) instead of the Cubism SDK files", + ) + c.add_argument( + "--stored-catalog", + metavar="LABEL|SHA", + help="a catalog version of the depot (`catalog versions`; default: the current catalog)", + ) + c.add_argument("--workers", type=nonnegative, metavar="N", help="worker processes (default: the usable CPUs)") + c.add_argument("--link", choices=("auto", "clone", "hard", "copy"), help="as export assets --link") + c.add_argument("--explain", action="store_true", help="print every setting of the run first") + c.add_argument("--dry-run", action="store_true", help="print the plan, run nothing") + c.set_defaults(views="none", strict=False) c = command(es, "spot", cmd_spot, "a spot -> spot.json + Spine + room.glb + shaders") c.add_argument("spot_id", type=int, help="MasterHomeSpot id") diff --git a/src/nnnotes/content/live2d.py b/src/nnnotes/content/live2d.py new file mode 100644 index 0000000..5897b3e --- /dev/null +++ b/src/nnnotes/content/live2d.py @@ -0,0 +1,368 @@ +"""Live2D models assembled from the objects their prefab reaches: the player's model document and the Cubism SDK +interchange files. + +Both read a product's object table (`Objects`, pipeline.assemble), never a bundle. The model is the GameObject among +the table's roots (the object its catalog key names, Character/Live2D///model/); its components +are read from its prefab document (prefab.json), a MonoBehaviour's class being the last part of its script entry. + +`model` -> model.json (MODEL_FORMAT): + + name, key the model's id (model_id: the key's last part) and its key + prefab the object id of the model's GameObject + moc the object id of its CubismMoc (the root's CubismModel._moc); the moc3 is its file `blob:_bytes` + canvas the moc3's canvas (formats.cubism.moc3_canvas) + textures the object ids of the atlas pages the drawables draw with (CubismRenderer._mainTexture), by name + shaders {shader name: object id} of the shaders of the table (the drawables' materials', the mask + materials', ...) + resources {name: object id} of the materials the Cubism mask pass draws with (RESOURCES: Resources.Load + paths, each the Material among the roots named as the path's last part) + motionSync the root has the MotionSync controller and its CRI audio input (MOTION_SYNC_COMPONENTS); a model + without them takes its mouth in a story from the voice's lip sync analysis + bindings {clip object id: [{path, attribute}]}: the bindings of every AnimationClip of the table named in the + model's hierarchy, played from its root (unity.bindings.resolve) + +Of the objects, the player reads all but the MonoScripts; of a shader, its parsed form (`json`), its variant index +(`index`) and the programs it draws the materials with (formats.shader.player_program): those of each drawable +material's keywords, alone and with STORY_KEYWORDS, and, when a drawable is masked (MASK_KEYWORD), the mask +materials'. A shader no drawn material uses keeps its parsed form and index. + +`cubism` -> the Cubism SDK files other tools read: .moc3, textures/.png (the atlas pages, by name), +.physics3.json (CubismPhysicsController._rig; when the rig has sub-rigs), .exp3.json (each +CubismExpressionData), motions/.motion3.json (each CubismFadeMotionData, with the sample rate of the clip of +its name) and .model3.json (the files, the model's fade times of each motion, and the EyeBlink and LipSync +groups: the names of the GameObjects holding a CubismEyeBlinkParameter or a CubismMouthParameter). +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator +from typing import Any, Protocol + +from ..core.jsonio import dumps +from ..formats import motion +from ..formats.cubism import expression_json, moc3_canvas, physics_json +from ..formats.shader import player_program +from ..unity import bindings +from ..unity.ids import object_id + +__all__ = [ + "MASK_KEYWORD", + "LIVE2D_PREFIX", + "MODEL_FORMAT", + "MODEL_ID", + "MODEL_KEY", + "MOTION_SYNC_COMPONENTS", + "RESOURCES", + "STORY_KEYWORDS", + "Objects", + "cubism", + "model", + "model_id", + "model_keys", + "model_match", + "select_models", +] + +MODEL_FORMAT = 3 # model.json "format" +LIVE2D_PREFIX = "Character/Live2D/" +MODEL_KEY = re.compile(r"Character/Live2D/(?P[^/]+)/(?P[^/]+)/model/(?P=name)") +MODEL_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*") +# Resources.Load paths of the materials the Cubism mask pass draws with +RESOURCES = { + "cubismMask": "Live2D/Cubism/Materials/Mask", + "cubismMaskCulling": "Live2D/Cubism/Materials/MaskCulling", +} +MASK_KEYWORD = "CUBISM_MASK_ON" # a drawable material's keyword: the drawable is masked +# keywords the story renderer adds to a character draw (_ADDITIONAL_LIGHTS_VERTEX: at quality 4); the mask pass draws +# with none +STORY_KEYWORDS = ("_ADDITIONAL_LIGHTS_VERTEX",) +# a model's MotionSync: both components on the prefab's root +MOTION_SYNC_COMPONENTS = ("CubismMotionSyncController", "Live2DMotionSyncCriAudioInput") +SHADER_ROLES = ("json", "index") # a shader's files the player reads besides its programs +_UNSAFE = re.compile(r"[^A-Za-z0-9_.-]+") + + +class Objects(Protocol): + """A product's object table and its objects' files (pipeline.assemble.Objects).""" + + roots: list[str] + + def __contains__(self, oid: str) -> bool: ... + + def ids(self, cls: str | None = None) -> list[str]: ... + + def info(self, oid: str) -> dict: ... + + def bytes(self, oid: str, role: str) -> bytes: ... + + def json(self, oid: str, role: str = "json") -> Any: ... + + +def model_match(key: str) -> re.Match[str]: + """The parts of a model key: group, name (ValueError for a key that is not a model's).""" + m = MODEL_KEY.fullmatch(key) + if m is None: + raise ValueError(f"{key}: not a Live2D model key (Character/Live2D///model/)") + return m + + +def model_id(key: str) -> str: + """The id of a model key: its (ValueError for a key that is not a model's, or a name that is not + URL-safe).""" + m = model_match(key) + if not MODEL_ID.fullmatch(m["name"]): + raise ValueError(f"{key}: model name {m['name']!r} is not URL-safe") + return m["name"] + + +def model_keys(keys) -> dict[str, str]: + """{id: key} of the model keys among `keys`, in id order; two keys with the same id raise.""" + out: dict[str, str] = {} + for k in keys: + if MODEL_KEY.fullmatch(k): + i = model_id(k) + if out.get(i, k) != k: + raise ValueError(f"model id {i}: keys {out[i]} and {k}") + out[i] = k + return dict(sorted(out.items())) + + +def select_models(specs, models: dict[str, str]) -> dict[str, str]: + """The models `specs` names (ids or keys; None: all) as {id: key} in id order, from `models` ({id: key}).""" + if specs is None: + return dict(models) + by_key = {k: i for i, k in models.items()} + out, unknown = {}, [] + for s in specs: + i = s if s in models else by_key.get(s) + if i is None: + unknown.append(s) + else: + out[i] = models[i] + if unknown: + raise ValueError( + f"not a Live2D model of the catalog: {', '.join(unknown)} " + f"(give a model id or a key Character/Live2D///model/)" + ) + return dict(sorted(out.items())) + + +def ref_id(v) -> str | None: + """The object id of a reference value ({"$ref": {file, pathId}}); None for anything else.""" + r = v.get("$ref") if isinstance(v, dict) else None + return object_id(r["file"], r["pathId"]) if isinstance(r, dict) and "file" in r else None + + +def _class(c: dict) -> str: + """A component's class: a MonoBehaviour's is its script's.""" + if c["type"] == "MonoBehaviour" and isinstance(c.get("script"), str): + return c["script"].rsplit("|", 1)[-1] + return c["type"] + + +class _Model: + """A model's GameObject, prefab document and components.""" + + def __init__(self, objects: Objects, key: str): + self.objects, self.key, self.name = objects, key, model_id(key) + gos = [o for o in objects.roots if o in objects and objects.info(o).get("class") == "GameObject"] + if len(gos) != 1: + raise ValueError(f"{key}: {len(gos)} GameObjects among the roots, expected the model's") + self.id = gos[0] + self.prefab = objects.json(self.id) + self.root = self.prefab["nodes"][0] + + def components(self, cls: str, root_only: bool = False) -> Iterator[tuple[dict, dict]]: + """(node, component) of every component of class `cls` (of the root node only).""" + for n in [self.root] if root_only else self.prefab["nodes"]: + for c in n["components"]: + if _class(c) == cls: + yield n, c + + def one(self, cls: str) -> dict: + found = [c for _, c in self.components(cls, root_only=True)] + if len(found) != 1: + raise ValueError(f"{self.key}: {len(found)} {cls} on the model's root") + return found[0] + + def moc(self) -> str: + oid = ref_id(self.one("CubismModel")["fields"].get("_moc")) + if oid is None or oid not in self.objects: + raise ValueError(f"{self.key}: the CubismModel's moc is not in the object table") + return oid + + def moc_bytes(self) -> bytes: + return self.objects.bytes(self.moc(), "blob:_bytes") + + def textures(self) -> list[str]: + """The atlas pages the drawables draw with, by name and object id.""" + pages = {ref_id(c["fields"].get("_mainTexture")) for _, c in self.components("CubismRenderer")} + pages.discard(None) + return sorted((o for o in pages if o is not None), key=lambda o: (self.objects.info(o).get("name") or "", o)) + + def drawable_materials(self) -> list[str]: + """The first material of every drawable's MeshRenderer.""" + out = [] + for _, c in self.components("MeshRenderer"): + mats = c["fields"].get("m_Materials") or [] + oid = ref_id(mats[0]) if mats else None + if oid is not None: + out.append(oid) + return out + + def scripted(self, cls: str) -> list[str]: + """The MonoBehaviours of the table whose script class is `cls` (outside the prefab: data assets).""" + return [o for o in self.objects.ids("MonoBehaviour") if _script_class(self.objects, o) == cls] + + +def _script_class(objects: Objects, oid: str) -> str | None: + if "json" not in (objects.info(oid).get("files") or {}): + return None + entry = objects.json(oid).get("$script") + return entry.rsplit("|", 1)[-1] if isinstance(entry, str) else None + + +def _resources(m: _Model) -> dict[str, str]: + out = {} + for name, path in RESOURCES.items(): + leaf = path.rsplit("/", 1)[-1] + hits = [ + o + for o in m.objects.roots + if o in m.objects and m.objects.info(o).get("class") == "Material" and m.objects.info(o).get("name") == leaf + ] + if len(hits) != 1: + raise ValueError(f"{m.key}: {len(hits)} Materials {leaf!r} among the roots (Resources.Load {path})") + out[name] = hits[0] + return out + + +def model(objects: Objects, key: str) -> tuple[dict, set[str], dict[str, set[str]]]: + """(model.json, the object ids the player reads, {shader id: the roles of its files it reads}) of the model + `key` (module documentation).""" + m = _Model(objects, key) + moc = m.moc() + resources = _resources(m) + sets: dict[str, set[tuple]] = {} # shader id -> keyword sets + drawn = [objects.json(o) for o in m.drawable_materials()] + for mat in drawn: + sid = ref_id(mat["shader"]) + if sid is None: + raise ValueError(f"{key}: material {mat['material']!r} has no shader") + kws = tuple(mat["keywords"]) + sets.setdefault(sid, set()).update({kws, (*kws, *STORY_KEYWORDS)}) + masked = any(MASK_KEYWORD in mat["keywords"] for mat in drawn) + for oid in resources.values(): + mat = objects.json(oid) + sid = ref_id(mat["shader"]) + if sid is None: + raise ValueError(f"{key}: material {mat['material']!r} has no shader") + sets.setdefault(sid, set()) + if masked: + sets[sid].add(tuple(mat["keywords"])) + for sid in objects.ids("Shader"): + sets.setdefault(sid, set()) # a shader no drawn material uses: its parsed form and index only + shaders, roles = {}, {} + for sid, kw_sets in sorted(sets.items()): + index = objects.json(sid, "index") + shaders[index["name"]] = sid + keep = set(SHADER_ROLES) + for kws in sorted(kw_sets): + try: + keep.add(player_program(index["variants"], kws)["role"]) + except ValueError as e: + raise ValueError(f"{key}: shader {index['name']}: {e}") from None + roles[sid] = keep + classes = {_class(c) for c in m.root["components"]} + + def material(ref) -> dict | None: + oid = ref_id(ref) + return objects.json(oid) if oid is not None and oid in objects else None + + clips = {} + for cid in objects.ids("AnimationClip"): + if "json" in (objects.info(cid).get("files") or {}): + clips[cid] = bindings.resolve(objects.json(cid)["bindings"], m.prefab, m.root["path"], material) + doc = { + "format": MODEL_FORMAT, + "name": m.name, + "key": key, + "prefab": m.id, + "moc": moc, + "canvas": moc3_canvas(m.moc_bytes()), + "textures": m.textures(), + "shaders": dict(sorted(shaders.items())), + "resources": resources, + "motionSync": all(k in classes for k in MOTION_SYNC_COMPONENTS), + "bindings": clips, + } + keep = {o for o in objects.ids() if objects.info(o).get("class") != "MonoScript"} + return doc, keep, roles + + +def _json(doc, indent: int | None = None) -> bytes: + return dumps(doc, indent=indent, ensure_ascii=False).encode("utf-8") + + +def _page_files(m: _Model) -> dict[str, str]: + """{object id: path} of the atlas pages: textures/., a name two pages share numbered after it.""" + out, used = {}, set() + for oid in m.textures(): + info = m.objects.info(oid) + stem = _UNSAFE.sub("_", info.get("name") or "texture") + ext = (info.get("files") or {}).get("image", {}).get("ext", "png") + path, n = f"textures/{stem}.{ext}", 1 + while path.casefold() in used: + n += 1 + path = f"textures/{stem}-{n}.{ext}" + used.add(path.casefold()) + out[oid] = path + return out + + +def cubism(objects: Objects, key: str) -> dict[str, bytes]: + """{path: bytes} of the Cubism SDK files of the model `key` (module documentation).""" + m = _Model(objects, key) + name = m.name + files = {f"{name}.moc3": m.moc_bytes()} + pages = _page_files(m) + for oid, path in pages.items(): + files[path] = objects.bytes(oid, "image") + refs: dict = {"Moc": f"{name}.moc3", "Textures": list(pages.values())} + controllers = [c for _, c in m.components("CubismPhysicsController", root_only=True)] + rig = controllers[0]["fields"].get("_rig") if controllers else None + if rig and rig.get("SubRigs"): + refs["Physics"] = f"{name}.physics3.json" + files[refs["Physics"]] = _json(physics_json(rig)) + expressions = [] + for oid in m.scripted("CubismExpressionData"): + tt = objects.json(oid) + ename = tt["m_Name"].removesuffix(".exp3") + files[f"{ename}.exp3.json"] = _json(expression_json(tt)) + expressions.append({"Name": ename, "File": f"{ename}.exp3.json"}) + if expressions: + refs["Expressions"] = sorted(expressions, key=lambda e: e["Name"]) + rates = {} + for cid in objects.ids("AnimationClip"): + if "json" in (objects.info(cid).get("files") or {}): + clip = objects.json(cid) + rates[clip["name"]] = float(clip["sampleRate"]) + motions = {} + for oid in m.scripted("CubismFadeMotionData"): + tt = objects.json(oid) + mname = motion.motion_name(tt) + files[f"motions/{mname}.motion3.json"] = _json(motion.convert_fade(tt, rates.get(mname))) + entry = {"File": f"motions/{mname}.motion3.json"} + for k, field in (("FadeInTime", "ModelFadeInTime"), ("FadeOutTime", "ModelFadeOutTime")): + if tt[field] >= 0: + entry[k] = tt[field] + motions[mname] = [entry] + refs["Motions"] = dict(sorted(motions.items())) + groups = [] + for group, cls in (("EyeBlink", "CubismEyeBlinkParameter"), ("LipSync", "CubismMouthParameter")): + ids = sorted(n["name"] for n, _ in m.components(cls)) + if ids: + groups.append({"Target": "Parameter", "Name": group, "Ids": ids}) + files[f"{name}.model3.json"] = _json({"Version": 3, "FileReferences": refs, "Groups": groups}, indent=1) + return files diff --git a/src/nnnotes/formats/cubism.py b/src/nnnotes/formats/cubism.py new file mode 100644 index 0000000..effbcd7 --- /dev/null +++ b/src/nnnotes/formats/cubism.py @@ -0,0 +1,106 @@ +"""Cubism-for-Unity data -> the Cubism SDK interchange files: the canvas of a moc3, physics3.json, exp3.json. + +Cubism-for-Unity keeps a model's physics rig (CubismPhysicsController._rig) and its expressions +(CubismExpressionData) in its own serialized layout; these functions write them in the layout of the Cubism SDK's +JSON files (motion3.json: formats.motion). +""" + +from __future__ import annotations + +import struct + +__all__ = ["EXP_BLEND", "PHYS_TYPE", "expression_json", "moc3_canvas", "physics_json"] + +# Live2D.Cubism.Core.CubismParameterBlendMode -> exp3.json "Blend" +EXP_BLEND = {0: "Overwrite", 1: "Add", 2: "Multiply"} +# Live2D.Cubism.Framework.Physics.CubismPhysicsSourceComponent -> physics3 "Type" +PHYS_TYPE = {0: "X", 1: "Y", 2: "Angle"} + + +def moc3_canvas(moc: bytes) -> dict: + """CanvasInfo from a moc3 (section table at 0x40; entry 1 = canvas info).""" + if moc[:4] != b"MOC3": + raise ValueError("not a moc3") + canvas_off = struct.unpack_from(" dict: + """physics3.json of a CubismPhysicsController's rig (the inverse of Cubism-for-Unity's import, ToRig).""" + settings, dictionary = [], [] + n_in = n_out = n_vtx = 0 + for i, sr in enumerate(rig["SubRigs"]): + sid = f"PhysicsSetting{i + 1}" + dictionary.append({"Id": sid, "Name": sr["Name"]}) + inputs = [ + { + "Source": {"Target": "Parameter", "Id": x["SourceId"]}, + "Weight": x["Weight"], + "Type": PHYS_TYPE[x["SourceComponent"]], + "Reflect": bool(x["IsInverted"]), + } + for x in sr["Input"] + ] + outputs = [ + { + "Destination": {"Target": "Parameter", "Id": x["DestinationId"]}, + "VertexIndex": x["ParticleIndex"], + "Scale": x["AngleScale"], + "Weight": x["Weight"], + "Type": PHYS_TYPE[x["SourceComponent"]], + "Reflect": bool(x["IsInverted"]), + } + for x in sr["Output"] + ] + verts = [ + { + "Position": _vec2(p["InitialPosition"]), + "Mobility": p["Mobility"], + "Delay": p["Delay"], + "Acceleration": p["Acceleration"], + "Radius": p["Radius"], + } + for p in sr["Particles"] + ] + nz = sr["Normalization"] + settings.append( + { + "Id": sid, + "Input": inputs, + "Output": outputs, + "Vertices": verts, + "Normalization": { + "Position": {k: nz["Position"][k] for k in ("Minimum", "Default", "Maximum")}, + "Angle": {k: nz["Angle"][k] for k in ("Minimum", "Default", "Maximum")}, + }, + } + ) + n_in += len(inputs) + n_out += len(outputs) + n_vtx += len(verts) + meta = { + "PhysicsSettingCount": len(settings), + "TotalInputCount": n_in, + "TotalOutputCount": n_out, + "VertexCount": n_vtx, + "EffectiveForces": {"Gravity": _vec2(rig["Gravity"]), "Wind": _vec2(rig["Wind"])}, + "PhysicsDictionary": dictionary, + } + if rig["Fps"] > 0: # 0 = unset in physics3 terms + meta["Fps"] = rig["Fps"] + return {"Version": 3, "Meta": meta, "PhysicsSettings": settings} + + +def expression_json(tt: dict) -> dict: + """exp3.json of a CubismExpressionData's fields.""" + return { + "Type": tt["Type"], + "FadeInTime": tt["FadeInTime"], + "FadeOutTime": tt["FadeOutTime"], + "Parameters": [{"Id": p["Id"], "Value": p["Value"], "Blend": EXP_BLEND[p["Blend"]]} for p in tt["Parameters"]], + } diff --git a/src/nnnotes/formats/motion.py b/src/nnnotes/formats/motion.py index 82f140e..b33888e 100644 --- a/src/nnnotes/formats/motion.py +++ b/src/nnnotes/formats/motion.py @@ -17,8 +17,6 @@ import math from pathlib import Path -from ..core.jsonio import write_json - SEG_LINEAR, SEG_BEZIER, SEG_STEPPED = 0, 1, 2 WEIGHT_IN, WEIGHT_OUT = 1, 2 # UnityEngine.WeightedMode flags @@ -56,14 +54,6 @@ def _segments(kf: list[dict]): return segs, nseg, npts, restricted -def _clip_rates(env) -> dict[str, float]: - return { - o.read_typetree()["m_Name"]: float(o.read_typetree()["m_SampleRate"]) - for o in env.objects - if o.type.name == "AnimationClip" - } - - def convert_fade(fade: dict, fps: float | None) -> dict: curves = [] total_seg = total_pts = 0 @@ -110,30 +100,3 @@ def convert_fade(fade: dict, fps: float | None) -> dict: def motion_name(fade: dict) -> str: return Path(str(fade["MotionName"])).name.removesuffix(".motion3.json") - - -def extract_motions(env, out_dir: Path) -> dict[str, str]: - """Every CubismFadeMotionData in env -> /.motion3.json. - - Returns {motionName: fileName}. Also writes `_fades.json` with the - model-level fade overrides (ModelFadeIn/OutTime, -1 = unset) the model3 - motion entries need. - """ - out_dir = Path(out_dir) - out_dir.mkdir(parents=True, exist_ok=True) - rates = _clip_rates(env) - manifest: dict[str, str] = {} - model_fades: dict[str, dict] = {} - for o in env.objects: - if o.type.name != "MonoBehaviour": - continue - if o.read().m_Script.read().m_ClassName != "CubismFadeMotionData": - continue - tt = o.read_typetree() - name = motion_name(tt) - fn = f"{name}.motion3.json" - write_json(out_dir / fn, convert_fade(tt, rates.get(name)), indent=None) - manifest[name] = fn - model_fades[name] = {"FadeInTime": tt["ModelFadeInTime"], "FadeOutTime": tt["ModelFadeOutTime"]} - write_json(out_dir / "_fades.json", model_fades, indent=None, ensure_ascii=True) - return manifest diff --git a/src/nnnotes/formats/shader.py b/src/nnnotes/formats/shader.py index aac32f8..a957473 100644 --- a/src/nnnotes/formats/shader.py +++ b/src/nnnotes/formats/shader.py @@ -29,10 +29,6 @@ import struct from pathlib import Path -import UnityPy -from UnityPy.enums import ShaderGpuProgramType -from UnityPy.export.ShaderConverter import CompressionHelper - from ..core import cache from ..core.jsonio import dumps, write_json @@ -59,6 +55,8 @@ } TEXT_TYPES = {"GLES", "GLES3", "GLES31", "GLES31AEP", "GLCore32", "GLCore41", "GLCore43", "MetalVS", "MetalFS"} STAGES = ("progVertex", "progFragment", "progGeometry", "progHull", "progDomain") +# the programs the player draws with: GLSL ES 3.00 (WebGL2) +PLAYER_PLATFORM, PLAYER_TYPE = "gles3", "GLES3" def _safe(name: str) -> str: @@ -71,6 +69,8 @@ def _entry(arr, i): def _type_name(v: int) -> str: + from UnityPy.enums import ShaderGpuProgramType + return ShaderGpuProgramType(v).name.replace("kShaderGpuProgram", "") @@ -97,6 +97,8 @@ def _code_of(entry: bytes) -> bytes: def platform_blobs(shader) -> dict[int, list[bytes]]: + from UnityPy.export.ShaderConverter import CompressionHelper + blob = bytes(shader.compressedBlob or b"") out = {} n = min(len(shader.platforms), len(shader.compressedLengths), len(shader.decompressedLengths)) @@ -181,6 +183,22 @@ def _render(o, tt: dict, name: str) -> tuple[str, list[dict], list[bytes]]: return text, recs, codes +def player_program(variants: list[dict], keywords) -> dict: + """The variant record of a shader (platform, subShader, pass, type, keywords, ...) the player draws a material + with `keywords` by: the PLAYER_TYPE program of subshader 0, pass 0 whose keywords are exactly the material's + among those the programs there use. ValueError when not exactly one fits.""" + vs = [ + v + for v in variants + if v["platform"] == PLAYER_PLATFORM and v["type"] == PLAYER_TYPE and v["subShader"] == 0 and v["pass"] == 0 + ] + want = set(keywords) & {k for v in vs for k in v["keywords"]} + hits = [v for v in vs if set(v["keywords"]) == want] + if len(hits) != 1: + raise ValueError(f"{len(hits)} {PLAYER_TYPE} programs with the keywords {sorted(want)}") + return hits[0] + + def _dump_key(o) -> str: st = o.serialized_type type_hash = bytes(st.old_type_hash) if st is not None and st.old_type_hash else b"" @@ -239,6 +257,8 @@ def write_index(index: list, out_dir: Path) -> dict: def dump(bundles: list, out_dir: Path) -> dict: + import UnityPy + index: list = [] for bpath in bundles: env = UnityPy.load(str(bpath)) diff --git a/src/nnnotes/pipeline/assemble.py b/src/nnnotes/pipeline/assemble.py index a1ab4e7..f2fa27e 100644 --- a/src/nnnotes/pipeline/assemble.py +++ b/src/nnnotes/pipeline/assemble.py @@ -258,6 +258,10 @@ def __setstate__(self, state): def subjects(self, env) -> list[str]: return sorted((env.facts.get("products") or {}).get(self.name, ())) + def select(self, keys: list[str]) -> list[str]: + """The subjects among the catalog keys `keys` (those a selection's bundles hold), sorted.""" + return [] + def keys(self, subject: str, env) -> list[str]: """The catalog keys whose objects are roots of `subject` (from the facts alone).""" return [] diff --git a/src/nnnotes/pipeline/layout.py b/src/nnnotes/pipeline/layout.py index b20eb4d..705df28 100644 --- a/src/nnnotes/pipeline/layout.py +++ b/src/nnnotes/pipeline/layout.py @@ -30,6 +30,8 @@ `product` (product_entries): the products of pipeline.assemble. Every file of a product and of the objects of its object table is at `assets/.`; each product has a manifest (nnnotes.product/1, product_manifest) at `/.json` (ProductStage.manifest), whose asset paths are relative to the directory above its own. +`files` (product_files): the files of one product at their own paths (a model's Cubism SDK files as a directory); a +path two products give is an error. So every path ends with its artifact's extension, and `cas` follows from the `original` manifest alone; `original` follows from `cas` plus the results (the records). Documents a stage derives for `original` from its records and the paths (the views with the files of their objects) are the exception: `original` has the derived documents, @@ -65,7 +67,7 @@ MANIFEST = "manifest.json" JOURNAL = ".nnnotes-layout-journal.json" -LAYOUTS = ("original", "cas") +LAYOUTS = ("original", "cas", "product", "files") PRODUCT_TABLE = "objects" # the role of a product's object table (nnnotes.objects/1) among its artifacts PLACEMENTS = ("clone", "hard", "copy") # how a file is made from its byte source LINKS = ("auto", *PLACEMENTS) # what a run may ask for: auto probes @@ -410,13 +412,28 @@ def product_entries(products) -> list[dict]: return [out[k] for k in sorted(out)] +def product_files(results) -> list[dict]: + """The entries of the `files` layout: the files of the product results `results` at their own paths.""" + out: dict[str, dict] = {} + for result in results: + for a in result["artifacts"]: + path = contract.parse_artifact_id(a["id"])[1] + if path == PRODUCT_TABLE: + continue + e = _entry(escape_path(path.split("/")), a) + seen = out.setdefault(e["path"].casefold(), e) + if seen["id"] != e["id"]: + raise LayoutError(f"{e['path']}: a file of {seen['id']} and of {e['id']}") + return sorted(out.values(), key=lambda e: (e["path"], e["id"])) + + def entries(name: str, sources, **kw) -> tuple[list[dict], dict]: - """The entries and report of layout `name`.""" + """The entries and report of layout `original` or `cas`.""" if name == "cas": return cas_entries(sources, kw.get("extra", ())), {"collisions": [], "shared": []} if name == "original": return original_entries(sources, **kw) - raise ValueError(f"unknown layout {name!r} (one of {', '.join(LAYOUTS)})") + raise ValueError(f"layout {name!r}: entries() places original or cas") def paths(entries_: list[dict]) -> dict[str, str]: diff --git a/src/nnnotes/pipeline/live2d.py b/src/nnnotes/pipeline/live2d.py new file mode 100644 index 0000000..fc4c198 --- /dev/null +++ b/src/nnnotes/pipeline/live2d.py @@ -0,0 +1,87 @@ +"""The Live2D products: a model for the player (live2d.model) and its Cubism SDK files (live2d.cubism). + +Both are product stages (pipeline.assemble) whose subjects are model keys (Character/Live2D///model/ +, the fact "products") and whose roots are the objects of the key; the player's model also starts from the +materials the Cubism mask pass loads by Resources.Load path (content.live2d.RESOURCES), which the APK's built-in +files hold. They assemble with content.live2d: + +live2d.model "#model.json" (kind live2d.model: content.live2d.model) and the object table of what the + player reads; its manifest is models/.json +live2d.cubism "#" for every Cubism SDK file (content.live2d.cubism; kinds by file: CUBISM_KINDS) and + no objects; its manifest is cubism/.json, and its files are the model's directory +""" + +from __future__ import annotations + +from ..content import live2d +from . import contract +from .assemble import Assembly, ProductStage + +__all__ = ["CUBISM", "CUBISM_KINDS", "MODEL", "CubismStage", "ModelStage", "cubism_kind"] + +MODEL, CUBISM = "live2d.model", "live2d.cubism" +# Cubism SDK files -> artifact kinds (by suffix, the first that matches) +CUBISM_KINDS = ( + (".model3.json", "live2d.model3"), + (".physics3.json", "live2d.physics3"), + (".exp3.json", "live2d.expression"), + (".motion3.json", "live2d.motion"), + (".moc3", "live2d.moc3"), + ("textures/", "live2d.texture"), +) + + +def cubism_kind(path: str) -> str: + for suffix, kind in CUBISM_KINDS: + if path.endswith(suffix) or path.startswith(suffix): + return kind + raise ValueError(f"{path}: not a Cubism SDK file") + + +class ModelStage(ProductStage): + """live2d.model: a model key -> model.json and the object table the player reads (module documentation).""" + + name = MODEL + version = 2 + KIND = "Live2DModel" + + def select(self, keys: list[str]) -> list[str]: + return sorted(live2d.model_keys(keys).values()) + + def keys(self, subject: str, env) -> list[str]: + live2d.model_id(subject) + return [subject] + + def resources(self, subject: str, env) -> list[str]: + return sorted(live2d.RESOURCES.values()) + + def manifest(self, subject: str) -> str: + return f"models/{live2d.model_id(subject)}.json" + + def assemble(self, task, objects) -> Assembly: + doc, keep, roles = live2d.model(objects, task.subject) + return Assembly({"model.json": (contract.encode(doc), "json", "live2d.model")}, keep, roles) + + +class CubismStage(ProductStage): + """live2d.cubism: a model key -> its Cubism SDK files (module documentation).""" + + name = CUBISM + version = 1 + KIND = "CubismModel" + + def select(self, keys: list[str]) -> list[str]: + return sorted(live2d.model_keys(keys).values()) + + def keys(self, subject: str, env) -> list[str]: + live2d.model_id(subject) + return [subject] + + def manifest(self, subject: str) -> str: + return f"cubism/{live2d.model_id(subject)}.json" + + def assemble(self, task, objects) -> Assembly: + files = {} + for path, data in sorted(live2d.cubism(objects, task.subject).items()): + files[path] = (data, path.rsplit(".", 1)[-1], cubism_kind(path)) + return Assembly(files, set()) diff --git a/src/nnnotes/pipeline/modelstages.py b/src/nnnotes/pipeline/modelstages.py index 290feda..ca4c19a 100644 --- a/src/nnnotes/pipeline/modelstages.py +++ b/src/nnnotes/pipeline/modelstages.py @@ -1,17 +1,5 @@ -"""The model stages: Live2D models and Spine skeletons, written by the extractors of `nnnotes export live2d` and -`nnnotes export spot` (unchanged) from the bundles of their tasks. - -live2d.model, one task per Live2D model key (Character/Live2D///model/, the keys `nnnotes export -live2d` takes) whose own bundle (the first dependency of the key's location) is selected: - inputs "bundle:": the bundles of the key's dependency closure in the order the catalog gives them - (Catalog.resolve; n is the position: the extractor loads them in this order, so the order is part of - the key) - context {"internalId": the internal id of the key's location} (the container path the model is read from) - artifacts "live2d.model:#": every file live2d.extract_model writes, relative to the model's - directory (.moc3, .prefab.json, textures/*.png, .model3.json, *.exp3.json, - .physics3.json, motions/*.motion3.json, motions/_fades.json), with its bytes -The extractor runs through TaskCatalog, which answers what it asks of a catalog from the task. It reads nothing of -the APK but the APK's bundles of the closure, which are inputs like the others. +"""The Spine skeleton stage, written by the extractor of `nnnotes export spot` (unchanged) from the bundles of its +tasks. spine.skeleton, one task per SkeletonDataAsset (a MonoBehaviour whose script the script table names SkeletonDataAsset) of a selected bundle; its subject is the asset's stable address by path id, @@ -24,46 +12,28 @@ the atlas pages (PNG), as spot.write_skeleton writes them for `nnnotes export spot` item the asset: exported (its artifacts), or failed with the writer's error -Both tasks name their extractor as their one atom ("live2d.extract_model", "spot.write_skeleton"), whose id carries -the versions of the libraries the files depend on; this module imports UnityPy only when a task runs. +The task names its extractor as its one atom ("spot.write_skeleton"), whose id carries the versions of the libraries +the files depend on; this module imports UnityPy only when a task runs. """ from __future__ import annotations import re -import tempfile -from pathlib import Path -from ..source.catalog import Bundle from ..unity.atoms import impl_id -from . import catalogdb, contract, link +from . import contract, link from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage -LIVE2D = "live2d.model" SPINE = "spine.skeleton" CENSUS = "unity.census" SCRIPTS = "link.scripts" SCRIPTS_TASK = contract.task_id(SCRIPTS, "all") -# the model keys of `nnnotes export live2d` (webmodel.MODEL_KEY; webmodel imports UnityPy) -MODEL_KEY = re.compile(r"Character/Live2D/(?P[^/]+)/(?P[^/]+)/model/(?P=name)") SKELETON_CLASS = "SkeletonDataAsset" -# the libraries the extractors' output depends on (export._LIBS, the salt of export's PNG cache) +# the libraries the extractor's output depends on (export._LIBS, the salt of export's PNG cache) LIBRARIES = ("UnityPy", "Pillow", "texture2ddecoder", "astc-encoder-py", "etcpak", "numpy") -_ASSET_PREFIXES = ("Assets/", "Packages/") _ADDRESS = re.compile(r"0x[0-9a-fA-F]{6,}") _EXT = re.compile(r"[0-9a-z]+") -# live2d.extract_model files -> artifact kinds (by suffix, the first that matches) -LIVE2D_KINDS = ( - (".model3.json", "live2d.model3"), - (".prefab.json", "live2d.prefab"), - (".physics3.json", "live2d.physics3"), - (".exp3.json", "live2d.expression"), - (".motion3.json", "live2d.motion"), - ("motions/_fades.json", "live2d.fades"), - (".moc3", "live2d.moc3"), - ("textures/", "live2d.texture"), -) def _message(e: BaseException) -> str: @@ -94,203 +64,6 @@ def _fact_input(bundles, stable: str, what: str) -> Input: return inp -def _ordered(task, prefix: str = "bundle:") -> list[Input]: - """The inputs "bundle:" of a task by n.""" - ins = [i for i in task.inputs if i.role.startswith(prefix)] - return sorted(ins, key=lambda i: int(i.role[len(prefix) :])) - - -# ---------------------------------------------------------------- the catalog of a task -class Closures: - """Key lookup and dependency closures of a catalog index, as Catalog.entry and Catalog.resolve give them on the - catalogs the index was made of: a key's location is its first one inside a bundle (internal id under Assets/ or - Packages/), else its first one (remote catalog before APK catalog, each by offset); its closure is every bundle - location its dependencies reach, depth first, in the order Catalog.resolve lists them.""" - - def __init__(self, index: dict): - self.index = index - self.locations = catalogdb.by_id(index) - self.files = catalogdb.file_refs(index) - self.by_key: dict[str, list[dict]] = {} - for loc in index["locations"]: - self.by_key.setdefault(loc["primaryKey"], []).append(loc) - - def keys(self) -> list[str]: - return sorted(self.by_key) - - def entry(self, key: str) -> dict: - locs = self.by_key.get(key) - if not locs: - raise KeyError(key) - return next((loc for loc in locs if loc["internalId"].startswith(_ASSET_PREFIXES)), locs[0]) - - def own_bundle(self, key: str) -> str | None: - """The stable name of the bundle a key's location names first (the bundle holding it), else None.""" - deps = self.entry(key)["dependencies"] - ref = self.files.get(deps[0]) if deps else None - return ref[1] if ref is not None and ref[0] == "bundle" else None - - def closure(self, key: str) -> list[str]: - """The location ids of the bundles of a key's closure, in Catalog.resolve order.""" - seen: set[str] = set() - out: dict[str, None] = {} - stack = [self.entry(key)["id"]] - while stack: - lid = stack.pop() - if lid in seen: - continue - seen.add(lid) - loc = self.locations.get(lid) - if loc is None: - continue - if loc["kind"] == "bundle": - out.setdefault(lid, None) - for d in loc["dependencies"]: - if d not in seen: - stack.append(d) - return list(out) - - def closure_bundles(self, key: str) -> list[str]: - """The stable names of the bundles of a key's closure, in Catalog.resolve order (a file two locations name - is listed twice, as Catalog.fetch_key gives it twice).""" - return [self.files[lid][1] for lid in self.closure(key)] - - -# the value of TaskCatalog.apk: not None, the APK's bundles of the closure are inputs of the task -APK_FROM_INPUTS = "" - - -class TaskCatalog: - """What the model extractors ask of a Catalog (live2d.extract_model through export.Exporter: fetch_key, entry, - apk), answered from a task: its key, the internal id of the key's location and the bundles of the key's closure - in load order, each bundle read through the depot (Depot.open_input: its locators, checked against its content - id). Another key is not in it (KeyError). `apk` is not None: the extractors need the APK's bundles, and those of - the closure are inputs of the task like the others.""" - - def __init__(self, key: str, internal_id: str, bundles: list[Input], depot): - self.key, self.internal_id, self.bundles, self.depot = key, internal_id, list(bundles), depot - self.apk = APK_FROM_INPUTS - - @classmethod - def from_task(cls, task, depot) -> TaskCatalog: - """The catalog of a live2d.model task: its subject, context internalId and inputs "bundle:".""" - return cls(task.subject, task.context["internalId"], _ordered(task), depot) - - def _check(self, key: str) -> None: - if key != self.key: - raise KeyError(key) - - def has(self, key: str) -> bool: - return key == self.key - - def keys(self, prefix: str = "") -> list[str]: - return [self.key] if self.key.startswith(prefix) else [] - - def entry(self, key: str) -> dict: - self._check(key) - return {"primary_key": key, "internal_id": self.internal_id} - - def entries_for(self, key: str) -> list[dict]: - return [self.entry(key)] if key == self.key else [] - - def resolve(self, key: str) -> list[Bundle]: - """The closure as Bundles in load order: offset = the position, name = the input's file name (its content - id when it has none); internal_id is empty and remote True (a task has a file's content, not its place in - a catalog; every bundle is read from the inputs).""" - self._check(key) - return [Bundle(i, "", inp.name or inp.sha256, True) for i, inp in enumerate(self.bundles)] - - def fetch_key(self, key: str) -> list[Path]: - """The files of the closure's bundles, in load order.""" - self._check(key) - return [self.depot.open_input(inp) for inp in self.bundles] - - -# ---------------------------------------------------------------- live2d.model -def live2d_kind(path: str) -> str: - """The artifact kind of a file live2d.extract_model writes (LIVE2D_KINDS; else "live2d.file").""" - for suffix, kind in LIVE2D_KINDS: - if path.endswith(suffix) or (suffix.endswith("/") and path.startswith(suffix)): - return kind - return "live2d.file" - - -def _tree(root: Path) -> list[tuple[str, Path]]: - """(relative POSIX path, file) of every file under `root`, sorted by path.""" - return sorted((p.relative_to(root).as_posix(), p) for p in root.rglob("*") if p.is_file()) - - -class Live2DStage(Stage): - """live2d.model: a Live2D model key -> the files of `nnnotes export live2d` (module documentation). Subjects and - inputs come from the facts "index" (the catalog index), "selected" and "bundles" ({stable name: Input "bundle"}).""" - - name = LIVE2D - version = 1 - ATOMS = {"live2d.extract_model": impl_id(LIBRARIES, 1)} - LAYOUT_ROOT = "{subject}" - - def __init__(self): - self._closures: Closures | None = None - - def __getstate__(self): - return {} - - def __setstate__(self, state): - self.__init__() - - def closures(self, env) -> Closures: - index = env.fact("index") - if self._closures is None or self._closures.index is not index: - self._closures = Closures(index) - return self._closures - - def subjects(self, env) -> list[str]: - c, selected = self.closures(env), set(env.fact("selected")) - return [k for k in c.keys() if MODEL_KEY.fullmatch(k) and c.own_bundle(k) in selected] - - def inputs(self, subject: str, env) -> list[Input]: - bundles, what = env.fact("bundles"), f"{self.name} {subject}" - names = self.closures(env).closure_bundles(subject) - width = max(3, len(str(len(names) - 1))) - out = [] - for i, stable in enumerate(names): - inp = _fact_input(bundles, stable, what) - out.append(Input(f"bundle:{i:0{width}d}", inp.sha256, inp.size, inp.name, inp.locators)) - return out - - def context(self, subject: str, env) -> dict: - return {"internalId": self.closures(env).entry(subject)["internalId"]} - - def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: - """From the closure's bytes (least squares over the models of a catalog: CPU seconds; peak resident bytes - of a worker process).""" - size = sum(i.size for i in inputs) - return Cost(0.9 + size * 2.2e-7, (420 << 20) + 10 * size) - - def run(self, task, depot) -> Output: - _check_atoms(task, self.ATOMS) - from ..unity import live2d - - cat = TaskCatalog.from_task(task, depot) - arts = [] - with tempfile.TemporaryDirectory(prefix="nnnotes-live2d-") as d: - # TaskCatalog answers the catalog calls of the model extractor - summary = live2d.extract_model(cat, task.subject, Path(d)) # pyright: ignore[reportArgumentType] - for rel, p in _tree(Path(d)): - kind = live2d_kind(rel) - facts = {k: summary[k] for k in sorted(summary)} if kind == "live2d.model3" else {} - ext = _ext(rel) - arts.append( - contract.artifact( - contract.artifact_id(task.id, rel), - depot.add(p.read_bytes(), ext), - contract.provenance(task), - {"kind": kind, "format": ext, "facts": facts}, - ) - ) - return Output(arts, []) - - # ---------------------------------------------------------------- spine.skeleton def skeleton_index(env, selected) -> dict: """What spine.skeleton plans from, read from the census results and the script table: {"subjects": {subject: @@ -407,7 +180,7 @@ def depends(self, subject: str, env) -> list[str]: return sorted({*(contract.task_id(CENSUS, s) for s in self.closure(subject, env)), SCRIPTS_TASK}) def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: - """From the closure's bytes (least squares over the skeletons of a catalog, as Live2DStage.estimate).""" + """From the closure's bytes (least squares over the skeletons of a catalog).""" size = sum(i.size for i in inputs) return Cost(0.27 + size * 1.9e-7, (440 << 20) + 20 * size) diff --git a/src/nnnotes/site/sitefiles.py b/src/nnnotes/site/sitefiles.py index aded184..d8398ed 100644 --- a/src/nnnotes/site/sitefiles.py +++ b/src/nnnotes/site/sitefiles.py @@ -26,6 +26,8 @@ from pathlib import Path from ..core import cache, jsonio +from ..formats.shader import PLAYER_PLATFORM as SHADER_PLATFORM +from ..formats.shader import PLAYER_TYPE as SHADER_TYPE from ..source.config import Config, ConfigError from . import compress from .compress import DEFAULT_ENCODING @@ -73,8 +75,6 @@ ".skel", ".bin", } # .glb / .atlas / .skel: an Overlay story's home spot; .bin: a story's CRI Lips data (crilips.py) -SHADER_PLATFORM = "gles3" -SHADER_TYPE = "GLES3" CHARTS_INDEX = "charts.json" # documentation fields of the extractors the player does not read (live-audio.json `spec`, the voice rule notes, # scene.json `slice` / `derived` notes): left out of the site's copies diff --git a/src/nnnotes/site/storysite.py b/src/nnnotes/site/storysite.py index e166319..a3fb2f2 100644 --- a/src/nnnotes/site/storysite.py +++ b/src/nnnotes/site/storysite.py @@ -49,7 +49,7 @@ import uuid from pathlib import Path -from ..content import adv, advui, story, storyfonts, storyhost +from ..content import adv, advui, live2d, story, storyfonts, storyhost from ..core import jsonio from ..source import languages, master from ..source.config import Config, ConfigError, usable_cpus, use @@ -802,7 +802,7 @@ def build( if keys: model_build = webmodel.build( site, - webmodel.model_keys(keys), + live2d.model_keys(keys), cfg, player_dir, force, diff --git a/src/nnnotes/site/webmodel.py b/src/nnnotes/site/webmodel.py index ee2a6f8..8cb37a8 100644 --- a/src/nnnotes/site/webmodel.py +++ b/src/nnnotes/site/webmodel.py @@ -47,13 +47,23 @@ import json import multiprocessing as mp import os -import re import shutil import tempfile import time import traceback from pathlib import Path +from ..content.live2d import ( + LIVE2D_PREFIX, + MASK_KEYWORD, + MOTION_SYNC_COMPONENTS, + RESOURCES, + STORY_KEYWORDS, + model_id, + model_keys, + model_match, + select_models, +) from ..core import jsonio from ..formats import shader as shader_mod from ..source import languages, master @@ -63,8 +73,6 @@ from .sitefiles import ( MANIFEST_FORMAT, MODELS_DIR, - SHADER_PLATFORM, - SHADER_TYPE, Store, _dump, _lock_fetches, @@ -78,80 +86,19 @@ ) from .siteindex import write_index -__all__ = ["catalog_models", "model_id", "model_keys"] +__all__ = ["catalog_models"] -LIVE2D_PREFIX = "Character/Live2D/" -MODEL_KEY = re.compile(r"Character/Live2D/(?P[^/]+)/(?P[^/]+)/model/(?P=name)") -MODEL_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*") MODEL_INDEX = "model.json" MODEL_FORMAT = 2 # model.json "format" SHADER_DIR = "shaders" -MASK_KEYWORD = "CUBISM_MASK_ON" # a drawable material's keyword: the drawable is masked -# keywords the story renderer adds to a character draw (_ADDITIONAL_LIGHTS_VERTEX: at quality 4); the mask pass draws -# with none -STORY_KEYWORDS = ("_ADDITIONAL_LIGHTS_VERTEX",) -# Resources.Load paths of the materials the Cubism mask pass draws with -RESOURCES = { - "cubismMask": "Live2D/Cubism/Materials/Mask", - "cubismMaskCulling": "Live2D/Cubism/Materials/MaskCulling", -} -# a model's MotionSync path (the player's CubismMotionSyncController.fromPrefab): both components on the prefab's root -MOTION_SYNC_COMPONENTS = ("CubismMotionSyncController", "Live2DMotionSyncCriAudioInput") COSTUME_TABLE = "MasterCharacterCostume" # _live2dPath (the key after LIVE2D_PREFIX) -> _characterID NAME_FIELDS = ("character", "names", "label") # listing fields from the master data (model_names) # ---------------------------------------------------------------- models and ids -def _model_key(key: str) -> re.Match[str]: - m = MODEL_KEY.fullmatch(key) - if not m: - raise ValueError(f"{key}: not a Live2D model key (Character/Live2D///model/)") - return m - - -def model_id(key: str) -> str: - """The id of a model key: its .""" - m = _model_key(key) - if not MODEL_ID.fullmatch(m["name"]): - raise ValueError(f"{key}: model name {m['name']!r} is not URL-safe") - return m["name"] - - -def model_keys(keys) -> dict[str, str]: - """{id: key} of the model keys among `keys`, in id order; two keys with the same id raise.""" - out: dict[str, str] = {} - for k in keys: - if MODEL_KEY.fullmatch(k): - i = model_id(k) - if out.get(i, k) != k: - raise ValueError(f"model id {i}: keys {out[i]} and {k}") - out[i] = k - return dict(sorted(out.items())) - - -def select(specs, models: dict[str, str]) -> dict[str, str]: - """The models `specs` names (ids or keys; None: all) as {id: key} in id order, from `models` ({id: key}).""" - if specs is None: - return dict(models) - by_key = {k: i for i, k in models.items()} - out, unknown = {}, [] - for s in specs: - i = s if s in models else by_key.get(s) - if i is None: - unknown.append(s) - else: - out[i] = models[i] - if unknown: - raise ValueError( - f"not a Live2D model of the catalog: {', '.join(unknown)} " - f"(give a model id or a key Character/Live2D///model/)" - ) - return dict(sorted(out.items())) - - def catalog_models(cat, specs=None) -> dict[str, str]: - """`select` over the model keys of the catalog `cat`.""" - return select(specs, model_keys(cat.keys(LIVE2D_PREFIX))) + """content.live2d.select_models over the model keys of the catalog `cat`.""" + return select_models(specs, model_keys(cat.keys(LIVE2D_PREFIX))) # ---------------------------------------------------------------- names from the master data @@ -275,16 +222,11 @@ def shader_files(index: list, materials: list[dict], extra: tuple[str, ...] = () for name, kws in sorted(sets): if name not in recs: raise RuntimeError(f"shader {name}: not in the shader index") - variants = [ - v - for v in recs[name]["variants"] - if v["platform"] == SHADER_PLATFORM and v["type"] == SHADER_TYPE and v["subShader"] == 0 and v["pass"] == 0 - ] - want = set(kws) & {k for v in variants for k in v["keywords"]} - hits = [v for v in variants if set(v["keywords"]) == want] - if len(hits) != 1: - raise RuntimeError(f"shader {name}: {len(hits)} GLES3 programs with the keywords {sorted(want)}") - files |= {recs[name]["parsed"], hits[0]["file"]} + try: + program = shader_mod.player_program(recs[name]["variants"], kws) + except ValueError as e: + raise RuntimeError(f"shader {name}: {e}") from None + files |= {recs[name]["parsed"], program["file"]} return sorted(files) @@ -354,7 +296,7 @@ def model_facts(key: str, summary: dict, names: dict | None = None) -> dict: """What a model listing needs, from the key and the export summary, plus the NAME_FIELDS of `names` (an entry of model_names).""" facts = { - "group": _model_key(key)["group"], + "group": model_match(key)["group"], "canvas": summary["canvas"], "textures": len(summary["textures"]), "nodes": summary["nodes"], diff --git a/src/nnnotes/unity/bindings.py b/src/nnnotes/unity/bindings.py index 7c8cc5d..b4ddb51 100644 --- a/src/nnnotes/unity/bindings.py +++ b/src/nnnotes/unity/bindings.py @@ -24,8 +24,6 @@ import zlib from collections.abc import Callable, Iterable -from UnityPy.enums import ClassIDType - __all__ = [ "ENGINE_PROPS", "MATERIAL_PROPS", @@ -150,6 +148,8 @@ def material_attribute(attribute: int, kinds: dict[str, str]) -> str | None: def _class_name(type_id: int) -> str | None: + from UnityPy.enums import ClassIDType + try: return ClassIDType(type_id).name except ValueError: diff --git a/src/nnnotes/unity/live2d.py b/src/nnnotes/unity/live2d.py index 1cecd2d..e426c65 100644 --- a/src/nnnotes/unity/live2d.py +++ b/src/nnnotes/unity/live2d.py @@ -16,130 +16,26 @@ expression list and every AnimationClip (Mecanim clip data + resolved bindings) inlined; plus `canvas` (moc3) -`extract_model` adds the Cubism SDK interchange files for other tools: - - .exp3.json CubismExpressionData - .physics3.json CubismPhysicsController._rig (inverse of ToRig), when the - model has physics - motions/.motion3.json CubismFadeMotionData (see motion.py) - .model3.json file refs + EyeBlink/LipSync groups from the - model's own CubismEyeBlinkParameter/CubismMouthParameter - -model3.json lists the atlas pages sorted by name (the Cubism Editor's -texture_NN naming). The texture each drawable is actually drawn with is its -CubismRenderer._mainTexture in prefab.json, keyed by CubismDrawable._unmanagedIndex -(the Cubism Core drawable index). +The texture each drawable is drawn with is its CubismRenderer._mainTexture in prefab.json, keyed by +CubismDrawable._unmanagedIndex (the Cubism Core drawable index). """ from __future__ import annotations -import json -import struct from dataclasses import dataclass from pathlib import Path from ..core.jsonio import write_json -from ..formats import motion as motion_mod +from ..formats.cubism import moc3_canvas from ..source.catalog import Catalog from ..source.config import apk_missing from .export import Exporter from .reader import SceneGraph, script_class -# Live2D.Cubism.Core.CubismParameterBlendMode -> exp3.json "Blend" -EXP_BLEND = {0: "Overwrite", 1: "Add", 2: "Multiply"} -# Live2D.Cubism.Framework.Physics.CubismPhysicsSourceComponent -> physics3 "Type" -PHYS_TYPE = {0: "X", 1: "Y", 2: "Angle"} # the moc is written as .moc3; prefab.json names it only STUB_ASSETS = ("CubismMoc",) -def moc3_canvas(moc: bytes) -> dict: - """CanvasInfo from a moc3 (section table at 0x40; entry 1 = canvas info).""" - if moc[:4] != b"MOC3": - raise ValueError("not a moc3") - canvas_off = struct.unpack_from(" dict: - settings, dictionary = [], [] - n_in = n_out = n_vtx = 0 - for i, sr in enumerate(rig["SubRigs"]): - sid = f"PhysicsSetting{i + 1}" - dictionary.append({"Id": sid, "Name": sr["Name"]}) - inputs = [ - { - "Source": {"Target": "Parameter", "Id": x["SourceId"]}, - "Weight": x["Weight"], - "Type": PHYS_TYPE[x["SourceComponent"]], - "Reflect": bool(x["IsInverted"]), - } - for x in sr["Input"] - ] - outputs = [ - { - "Destination": {"Target": "Parameter", "Id": x["DestinationId"]}, - "VertexIndex": x["ParticleIndex"], - "Scale": x["AngleScale"], - "Weight": x["Weight"], - "Type": PHYS_TYPE[x["SourceComponent"]], - "Reflect": bool(x["IsInverted"]), - } - for x in sr["Output"] - ] - verts = [ - { - "Position": _vec2(p["InitialPosition"]), - "Mobility": p["Mobility"], - "Delay": p["Delay"], - "Acceleration": p["Acceleration"], - "Radius": p["Radius"], - } - for p in sr["Particles"] - ] - nz = sr["Normalization"] - settings.append( - { - "Id": sid, - "Input": inputs, - "Output": outputs, - "Vertices": verts, - "Normalization": { - "Position": {k: nz["Position"][k] for k in ("Minimum", "Default", "Maximum")}, - "Angle": {k: nz["Angle"][k] for k in ("Minimum", "Default", "Maximum")}, - }, - } - ) - n_in += len(inputs) - n_out += len(outputs) - n_vtx += len(verts) - meta = { - "PhysicsSettingCount": len(settings), - "TotalInputCount": n_in, - "TotalOutputCount": n_out, - "VertexCount": n_vtx, - "EffectiveForces": {"Gravity": _vec2(rig["Gravity"]), "Wind": _vec2(rig["Wind"])}, - "PhysicsDictionary": dictionary, - } - if rig["Fps"] > 0: # 0 = unset in physics3 terms - meta["Fps"] = rig["Fps"] - return {"Version": 3, "Meta": meta, "PhysicsSettings": settings} - - -def expression_json(tt: dict) -> dict: - return { - "Type": tt["Type"], - "FadeInTime": tt["FadeInTime"], - "FadeOutTime": tt["FadeOutTime"], - "Parameters": [{"Id": p["Id"], "Value": p["Value"], "Blend": EXP_BLEND[p["Blend"]]} for p in tt["Parameters"]], - } - - def _classes(env) -> dict[str | None, list]: by_cls: dict[str | None, list] = {} for o in env.objects: @@ -201,74 +97,3 @@ def extract_runtime(cat: Catalog, key: str, out_dir: Path) -> dict: """Write the runtime files of the model `key` into `out_dir`; returns the summary: name, moc3, prefab, moc3Bytes, textures, nodes.""" return export_runtime(cat, key, out_dir).summary - - -def extract_model(cat: Catalog, key: str, out_dir: Path) -> dict: - out_dir = Path(out_dir) - rt = export_runtime(cat, key, out_dir) - res, env, graph, by_cls = rt.summary, rt.env, rt.graph, rt.classes - name = res["name"] - (out_dir / "motions").mkdir(parents=True, exist_ok=True) - - def owner_names(cls): - return sorted(graph.go[o.read_typetree()["m_GameObject"]["m_PathID"]]["m_Name"] for o in by_cls.get(cls, [])) - - # expressions - exp_refs = [] - for o in by_cls.get("CubismExpressionData", []): - tt = o.read_typetree() - ename = tt["m_Name"].removesuffix(".exp3") - fn = f"{ename}.exp3.json" - write_json(out_dir / fn, expression_json(tt), indent=None) - exp_refs.append({"Name": ename, "File": fn}) - exp_refs.sort(key=lambda e: e["Name"]) - - # physics (none without a CubismPhysicsController) - rig = ( - _only(by_cls, "CubismPhysicsController", name)["_rig"] - if by_cls.get("CubismPhysicsController") - else {"SubRigs": []} - ) - phys_file = None - if rig["SubRigs"]: - phys_file = f"{name}.physics3.json" - write_json(out_dir / phys_file, physics_json(rig, []), indent=None) - - # motions - motions = motion_mod.extract_motions(env, out_dir / "motions") - model_fades = json.loads((out_dir / "motions" / "_fades.json").read_text(encoding="utf-8")) - - # model3.json - fr = {"Moc": res["moc3"], "Textures": res["textures"]} - if phys_file: - fr["Physics"] = phys_file - if exp_refs: - fr["Expressions"] = exp_refs - - def motion_entry(m, f): - e = {"File": f"motions/{f}"} - for k in ("FadeInTime", "FadeOutTime"): - if model_fades[m][k] >= 0: - e[k] = model_fades[m][k] - return [e] - - fr["Motions"] = {m: motion_entry(m, f) for m, f in sorted(motions.items())} - groups = [] - eye_ids = owner_names("CubismEyeBlinkParameter") - mouth_ids = owner_names("CubismMouthParameter") - if eye_ids: - groups.append({"Target": "Parameter", "Name": "EyeBlink", "Ids": eye_ids}) - if mouth_ids: - groups.append({"Target": "Parameter", "Name": "LipSync", "Ids": mouth_ids}) - model3 = {"Version": 3, "FileReferences": fr, "Groups": groups} - write_json(out_dir / f"{name}.model3.json", model3) - - return { - **res, - "model3": f"{name}.model3.json", - "expressions": len(exp_refs), - "motions": len(motions), - "physicsSettings": len(rig["SubRigs"]), - "eyeBlink": eye_ids, - "lipSync": mouth_ids, - } diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index e52104d..1b4f66b 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -16,7 +16,7 @@ from nnnotes import cli from nnnotes.cli import assets, common from nnnotes.cli import catalog as catalog_cli -from nnnotes.pipeline import contract, link, units +from nnnotes.pipeline import assemble, catalogdb, contract, link, units from nnnotes.pipeline.census import CensusStage from nnnotes.pipeline.contract import Cost from nnnotes.pipeline.depot import Depot @@ -1428,3 +1428,99 @@ def test_worker_counts_follow_the_cpus_this_process_may_use(monkeypatch): assert ws.workers() == 12 and cri.default_workers() == 3 ws.args.workers = 0 assert ws.workers() == 0 + + +# ---------------------------------------------------------------- products +class FakeProduct(assemble.ProductStage): + """A product per Bg/ key of the selection: index.json naming its roots and objects.""" + + name = "fake.product" + version = 1 + KIND = "FakeProduct" + + def select(self, keys): + return [k for k in keys if k.startswith("Bg/")] + + def keys(self, subject, env): + return [subject] + + def assemble(self, task, objects): + doc = {"roots": objects.roots, "objects": objects.ids()} + return assemble.Assembly({"index.json": (contract.encode(doc), "json", "fake.index")}) + + +class FakeCubism(assemble.ProductStage): + """live2d.cubism standing in for the real stage: a model3.json and a texture.""" + + name = "live2d.cubism" + version = 1 + KIND = "CubismModel" + + def keys(self, subject, env): + return [subject] + + def assemble(self, task, objects): + files = {"m.model3.json": (b'{"Version": 3}', "json", "live2d.model3"), "textures/t.png": (b"png", "png", "x")} + return assemble.Assembly(files, set()) + + +PRODUCT_SOURCES = FAKE_SOURCES + ( + assets.StageSource("fake.product", "tests.cli.test_assets:FakeProduct", output=True, product=True), + assets.StageSource("live2d.cubism", "tests.cli.test_assets:FakeCubism", output=True, product=True), +) +MODEL_KEY = "Character/Live2D/g/m/model/m" + + +def test_held_keys_are_those_of_the_bundles_holding_them(): + idx = catalogdb.index(catalog_bytes([(MODEL_KEY, "Assets/Game/Bg/x.prefab", [3])])) + assert assets.held_keys(idx, ["bg_x"]) == ["Bg/x", MODEL_KEY] + assert assets.held_keys(idx, [contract.stable_bundle_name(NAMES["card_1"])]) == ["Card/1/full"] + assert assets.held_keys(idx, []) == [] + + +def test_the_product_layout_places_the_products_of_the_selection(tmp_path, capsys, monkeypatch): + monkeypatch.setattr(assets, "STAGE_SOURCES", PRODUCT_SOURCES) + d = setup_data(tmp_path, censuses(broken=False)) + code, out, err = export(capsys, tmp_path, d, "s", "out", "--layout", "product") + assert code == 0, err + run = contract.loads((tmp_path / "out" / assets.REPORTS / "run.json").read_bytes()) + ids = sorted(t["id"] for t in run["tasks"]) + bg = f"{contract.stable_bundle_name(NAMES['bg_x'])}:3" + assert [i for i in ids if i.startswith(("unity.export:", "fake.product:", "live2d."))] == [ + "fake.product:Bg/x", + f"unity.export:{bg}", # what the product reaches, not every unit of the selection + ] + manifest = contract.loads((tmp_path / "out" / "fake.product" / "Bg%2Fx.json").read_bytes()) + assert manifest["product"] == "fake.product" and manifest["subject"] == "Bg/x" + index = (tmp_path / "out" / manifest["files"]["index.json"]["asset"]).read_bytes() + assert contract.loads(index)["roots"] == [f"{CABS['bg_x']}:3"] + go = manifest["objects"][f"{CABS['bg_x']}:3"] + assert go["class"] == "GameObject" and (tmp_path / "out" / go["files"]["json"]["asset"]).is_file() + assert manifest["objects"][f"{CABS['bg_x']}:2"] == {"class": "Transform", "in": f"{CABS['bg_x']}:3"} + assert sorted(p.name for p in (tmp_path / "out").iterdir()) == [ + assets.REPORTS, + "assets", + "fake.product", + "manifest.json", + ] # no object layout + code, out, err = export(capsys, tmp_path, d, "s", "both", "--layout", "original,product") + assert code == 0, err + assert (tmp_path / "both" / "original" / "Assets" / "Game" / "Card" / "1" / "full.png.json").is_file() + assert (tmp_path / "both" / "product" / "fake.product" / "Bg%2Fx.json").is_file() + + +def test_export_live2d_writes_the_cubism_files_of_one_model(tmp_path, capsys, monkeypatch): + monkeypatch.setattr(assets, "STAGE_SOURCES", PRODUCT_SOURCES) + monkeypatch.setattr("nnnotes.pipeline.catalogdb.apk_version", lambda apk: None) + d = setup_data(tmp_path, censuses(broken=False), extra=[(MODEL_KEY, "Assets/Game/Bg/x.prefab", [3])]) + apk = builtin_apk(tmp_path / "base.apk", {"assets/aa/catalog.bin": synth.CatalogWriter().build([])}) + argv = [*flags(d), "--apk", apk, "export", "live2d", "m", "-o", tmp_path / "m", "--depot", tmp_path / "s"] + code, out, err = nn(capsys, *argv, "--workers", "0") + assert code == 0, err + assert tree(tmp_path / "m", skip=(assets.REPORTS, "manifest.json")) == { + "m.model3.json": hashlib.sha256(b'{"Version": 3}').hexdigest(), + "textures/t.png": hashlib.sha256(b"png").hexdigest(), + } + run = contract.loads((tmp_path / "m" / assets.REPORTS / "run.json").read_bytes()) + ids = {t["id"] for t in run["tasks"]} + assert f"live2d.cubism:{MODEL_KEY}" in ids and "fake.product:Bg/x" not in ids diff --git a/tests/content/test_live2d.py b/tests/content/test_live2d.py new file mode 100644 index 0000000..0a82828 --- /dev/null +++ b/tests/content/test_live2d.py @@ -0,0 +1,161 @@ +"""Live2D models assembled from a synthetic object table (content.live2d).""" + +import json + +import pytest + +from nnnotes.content import live2d +from nnnotes.pipeline.assemble import Objects +from nnnotes.pipeline.depot import Depot +from tests.helpers import live2dmodel as lm + +KEY_A = "Character/Live2D/001_adv/adv_model_a/model/adv_model_a" +KEY_B = "Character/Live2D/sub_x/adv_model_b/model/adv_model_b" + + +def objects_of(tmp_path, objs=None, roots=(lm.ROOT, lm.MASK_MAT, lm.CULL_MAT)) -> Objects: + depot = Depot(tmp_path / "d") + return Objects(lm.table(depot, objs, roots), depot) + + +def test_model_document(tmp_path): + doc, keep, roles = live2d.model(objects_of(tmp_path), lm.KEY) + assert doc == { + "format": live2d.MODEL_FORMAT, + "name": "m001", + "key": lm.KEY, + "prefab": lm.ROOT, + "moc": lm.MOC, + "canvas": { + "pixelsPerUnit": 100.0, + "originX": 0.5, + "originY": 0.5, + "width": 20.0, + "height": 30.0, + "mocVersion": 5, + }, + "textures": [lm.TEX0, lm.TEX1], # by name + "shaders": {"Live2D Cubism/Mask": lm.MASK_SHADER, "Live2D Cubism/Unlit": lm.SHADER}, + "resources": {"cubismMask": lm.MASK_MAT, "cubismMaskCulling": lm.CULL_MAT}, + "motionSync": True, + "bindings": {lm.CLIP: [{"path": "Parameters/ParamEyeLOpen", "attribute": "Value"}]}, + } + assert lm.SCRIPT not in keep and keep == set(lm.objects()) - {lm.SCRIPT} + + +def test_the_player_reads_the_programs_of_the_drawn_keywords(tmp_path): + _, _, roles = live2d.model(objects_of(tmp_path), lm.KEY) + programs = {v["role"] for v in lm.SHADER_VARIANTS if v["platform"] == "gles3"} # each set alone and with ALV + assert roles[lm.SHADER] == {"json", "index"} | programs # no typetree, no other platform + assert roles[lm.MASK_SHADER] == {"json", "index", "program:gles3/s0p0_progVertex_0"} # a drawable is masked + + +def test_unmasked_models_read_no_mask_program(tmp_path): + objs = lm.objects() + cls, name, files = objs[lm.MAT1] + objs[lm.MAT1] = (cls, name, {"json": {**files["json"], "keywords": []}}) + _, _, roles = live2d.model(objects_of(tmp_path, objs), lm.KEY) + assert roles[lm.MASK_SHADER] == {"json", "index"} + assert roles[lm.SHADER] == {"json", "index", "program:gles3/s0p0_progVertex_0", "program:gles3/s0p0_progVertex_2"} + + +def test_a_model_without_motion_sync(tmp_path): + objs = lm.objects() + prefab = objs[lm.ROOT][2]["json"] + prefab["nodes"][0]["components"] = [ + c for c in prefab["nodes"][0]["components"] if c.get("script") is None or "MotionSync" not in c["script"] + ] + doc, _, _ = live2d.model(objects_of(tmp_path, objs), lm.KEY) + assert doc["motionSync"] is False + + +def test_a_keyword_set_without_a_program_is_an_error(tmp_path): + objs = lm.objects() + shader = objs[lm.SHADER][2] + shader["index"] = {**shader["index"], "variants": [v for v in lm.SHADER_VARIANTS if v["keywords"] != [lm.ALV]]} + with pytest.raises(ValueError, match="Live2D Cubism/Unlit: 0 GLES3 programs"): + live2d.model(objects_of(tmp_path, objs), lm.KEY) + + +def test_the_roots_name_the_model_and_the_mask_materials(tmp_path): + with pytest.raises(ValueError, match="0 GameObjects among the roots"): + live2d.model(objects_of(tmp_path, roots=(lm.MASK_MAT, lm.CULL_MAT)), lm.KEY) + with pytest.raises(ValueError, match="'MaskCulling' among the roots"): + live2d.model(objects_of(tmp_path, roots=(lm.ROOT, lm.MASK_MAT)), lm.KEY) + with pytest.raises(ValueError, match="not a Live2D model key"): + live2d.model(objects_of(tmp_path), "Character/Live2D/c001/m001/model/other") + + +def test_cubism_files(tmp_path): + files = live2d.cubism(objects_of(tmp_path), lm.KEY) + assert sorted(files) == [ + "m001.moc3", + "m001.model3.json", + "m001.physics3.json", + "motions/idle.motion3.json", + "smile.exp3.json", + "textures/texture_00.png", + "textures/texture_01.png", + ] + assert files["m001.moc3"] == lm.moc3() and files["textures/texture_00.png"] == b"png0" + model3 = json.loads(files["m001.model3.json"]) + assert model3 == { + "Version": 3, + "FileReferences": { + "Moc": "m001.moc3", + "Textures": ["textures/texture_00.png", "textures/texture_01.png"], + "Physics": "m001.physics3.json", + "Expressions": [{"Name": "smile", "File": "smile.exp3.json"}], + "Motions": {"idle": [{"File": "motions/idle.motion3.json", "FadeInTime": 0.5}]}, + }, + "Groups": [ + {"Target": "Parameter", "Name": "EyeBlink", "Ids": ["ParamEyeLOpen"]}, + {"Target": "Parameter", "Name": "LipSync", "Ids": ["ParamMouthOpenY"]}, + ], + } + motion = json.loads(files["motions/idle.motion3.json"]) + assert motion["Meta"]["Fps"] == 30.0 and motion["Curves"][0]["Id"] == "ParamEyeLOpen" + assert json.loads(files["smile.exp3.json"])["Parameters"] == [ + {"Id": "ParamEyeLOpen", "Value": 0.0, "Blend": "Multiply"} + ] + physics = json.loads(files["m001.physics3.json"]) + assert physics["Meta"]["PhysicsSettingCount"] == 1 and physics["PhysicsSettings"][0]["Output"][0]["Type"] == "Angle" + + +def test_pages_that_share_a_name_are_numbered(tmp_path): + objs = lm.objects() + cls, _, files = objs[lm.TEX1] + objs[lm.TEX1] = (cls, "texture_00", files) + files_ = live2d.cubism(objects_of(tmp_path, objs), lm.KEY) + assert [p for p in sorted(files_) if p.startswith("textures/")] == [ + "textures/texture_00-2.png", + "textures/texture_00.png", + ] + + +def test_model_id_is_the_model_name(): + assert live2d.model_id(KEY_A) == "adv_model_a" + for bad in ( + "Character/Live2D/001_adv/a/model/b", + "Character/Live2D/a/model/a", + "Other/001/a/model/a", + "Character/Live2D/001_adv/a/model/a/extra", + ): + with pytest.raises(ValueError, match="not a Live2D model key"): + live2d.model_id(bad) + with pytest.raises(ValueError, match="not URL-safe"): + live2d.model_id("Character/Live2D/g/a b/model/a b") + + +def test_model_keys_and_select(): + keys = [KEY_B, "Character/Live2D/001_adv/adv_model_a/motion/x", KEY_A, "Live/Score/1"] + models = live2d.model_keys(keys) + assert models == {"adv_model_a": KEY_A, "adv_model_b": KEY_B} + assert list(models) == ["adv_model_a", "adv_model_b"] + assert live2d.select_models(None, models) == models + assert live2d.select_models([KEY_B, "adv_model_a"], models) == models + assert live2d.select_models(["adv_model_b"], models) == {"adv_model_b": KEY_B} + with pytest.raises(ValueError, match="nope"): + live2d.select_models(["adv_model_a", "nope"], models) + with pytest.raises(ValueError, match="model id adv_model_a"): + live2d.model_keys([KEY_A, "Character/Live2D/002_adv/adv_model_a/model/adv_model_a"]) diff --git a/tests/helpers/live2dmodel.py b/tests/helpers/live2dmodel.py new file mode 100644 index 0000000..ee2a305 --- /dev/null +++ b/tests/helpers/live2dmodel.py @@ -0,0 +1,243 @@ +"""A synthetic Live2D model as a product's object table (nnnotes.objects/1) whose files are in a depot: a prefab with +a moc, two drawables (one masked), a physics rig, MotionSync, eye blink and lip sync parameters, a clip, its fade +motion data, an expression, the mask materials of the built-in files and a MonoScript.""" + +from __future__ import annotations + +import struct + +from nnnotes.pipeline import contract + +KEY = "Character/Live2D/c001/m001/model/m001" +CAB, RES = "CAB-m", "resources.assets" +CUBISM = "Live2D.Cubism|Live2D.Cubism.Core|" +FRAMEWORK = "Live2D.Cubism|Live2D.Cubism.Framework|" +ALV, MASK = "_ADDITIONAL_LIGHTS_VERTEX", "CUBISM_MASK_ON" +ROOT, MOC, TEX0, TEX1, MAT0, MAT1, SHADER = (f"{CAB}:{i}" for i in (1, 10, 20, 21, 30, 31, 40)) +CLIP, FADE, EXPR, SCRIPT = f"{CAB}:50", f"{CAB}:60", f"{CAB}:70", "CAB-s:1" +MASK_MAT, CULL_MAT, MASK_SHADER = f"{RES}:4", f"{RES}:5", f"{RES}:6" +CANVAS = (100.0, 0.5, 0.5, 20.0, 30.0) + + +def ref(oid: str, cls: str | None = None) -> dict: + file, _, pid = oid.rpartition(":") + r = {"file": file, "pathId": int(pid)} + if cls: + r["class"] = cls + return {"$ref": r} + + +def moc3() -> bytes: + """A moc3 header whose section table's entry 1 points at the canvas info.""" + data = bytearray(0x80) + data[:4] = b"MOC3" + data[4] = 5 + struct.pack_into(" dict: + return {"type": "MonoBehaviour", "pathId": 0, "script": script, "fields": fields} + + +def node(path: str, *components) -> dict: + return {"path": path, "name": path.rsplit("/", 1)[-1], "components": list(components)} + + +def prefab() -> dict: + rig = { + "Gravity": {"x": 0.0, "y": -1.0}, + "Wind": {"x": 0.0, "y": 0.0}, + "Fps": 0.0, + "SubRigs": [ + { + "Name": "Hair", + "Input": [{"SourceId": "ParamAngleX", "Weight": 60.0, "SourceComponent": 0, "IsInverted": 0}], + "Output": [ + { + "DestinationId": "ParamHair", + "ParticleIndex": 1, + "AngleScale": 1.5, + "Weight": 100.0, + "SourceComponent": 2, + "IsInverted": 1, + } + ], + "Particles": [ + { + "InitialPosition": {"x": 0.0, "y": 0.0}, + "Mobility": 1.0, + "Delay": 1.0, + "Acceleration": 1.0, + "Radius": 0.0, + } + ], + "Normalization": { + "Position": {"Minimum": -10.0, "Default": 0.0, "Maximum": 10.0}, + "Angle": {"Minimum": -10.0, "Default": 0.0, "Maximum": 10.0}, + }, + } + ], + } + return { + "name": "m001", + "nodes": [ + node( + "m001", + mono(CUBISM + "CubismModel", _moc=ref(MOC, "MonoBehaviour")), + mono(FRAMEWORK + "CubismPhysicsController", _rig=rig), + mono("Live2D.Cubism|Live2D.Cubism.Framework.MotionSync|CubismMotionSyncController"), + mono("Assembly-CSharp||Live2DMotionSyncCriAudioInput"), + {"type": "Animator", "pathId": 2, "fields": {}}, + ), + node("m001/Parameters"), + node( + "m001/Parameters/ParamEyeLOpen", + mono(CUBISM + "CubismParameter", Value=1.0), + mono(FRAMEWORK + "CubismEyeBlinkParameter"), + ), + node( + "m001/Parameters/ParamMouthOpenY", + mono(CUBISM + "CubismParameter", Value=0.0), + mono(FRAMEWORK + "CubismMouthParameter"), + ), + node("m001/Drawables"), + node( + "m001/Drawables/ArtMesh0", + mono("Live2D.Cubism|Live2D.Cubism.Rendering|CubismRenderer", _mainTexture=ref(TEX1, "Texture2D")), + {"type": "MeshRenderer", "pathId": 3, "fields": {"m_Materials": [ref(MAT0, "Material")]}}, + ), + node( + "m001/Drawables/ArtMesh1", + mono("Live2D.Cubism|Live2D.Cubism.Rendering|CubismRenderer", _mainTexture=ref(TEX0, "Texture2D")), + {"type": "MeshRenderer", "pathId": 4, "fields": {"m_Materials": [ref(MAT1, "Material")]}}, + ), + ], + } + + +def variant(n: int, keywords, platform="gles3", type_="GLES3") -> dict: + return { + "file": f"s/{platform}/s0p0_progVertex_{n}.glsl", + "platform": platform, + "subShader": 0, + "pass": 0, + "stage": "progVertex", + "type": type_, + "keywords": list(keywords), + "role": f"program:{platform}/s0p0_progVertex_{n}", + } + + +SHADER_VARIANTS = [ + variant(0, []), + variant(1, [MASK]), + variant(2, [ALV]), + variant(3, [MASK, ALV]), + variant(4, [], "vulkan", "SPIRV"), +] +MASK_VARIANTS = [variant(0, []), variant(1, ["CUBISM_INVERT"])] + + +def fade() -> dict: + keys = [ + {"time": 0.0, "value": 1.0, "inSlope": 0.0, "outSlope": 0.0, "weightedMode": 0, "inWeight": 0, "outWeight": 0}, + {"time": 1.0, "value": 0.0, "inSlope": 0.0, "outSlope": 0.0, "weightedMode": 0, "inWeight": 0, "outWeight": 0}, + ] + return { + "$script": FRAMEWORK + "CubismFadeMotionData", + "m_Name": "idle", + "MotionName": "motions/idle.motion3.json", + "ModelFadeInTime": 0.5, + "ModelFadeOutTime": -1.0, + "FadeInTime": 1.0, + "FadeOutTime": 1.0, + "ParameterIds": ["ParamEyeLOpen"], + "ParameterCurves": [{"m_Curve": keys}], + "ParameterFadeInTimes": [-1.0], + "ParameterFadeOutTimes": [-1.0], + "MotionLength": 1.0, + } + + +def objects() -> dict: + """{object id: (class, name, {role: bytes | JSON document})} of the model.""" + clip_binding = { + "pathCrc": 0, + "attributeCrc": 0, + "typeID": 114, + "customType": 0, + "pptr": False, + "int": False, + "serializeReference": False, + "script": ref("CAB-s:2"), + "scriptClass": CUBISM + "CubismParameter", + } + from nnnotes.unity.bindings import crc + + clip_binding.update(pathCrc=crc("Parameters/ParamEyeLOpen"), attributeCrc=crc("Value")) + material = {"material": "Unlit", "shader": ref(SHADER, "Shader"), "keywords": [], "floats": {}, "colors": {}} + return { + ROOT: ("GameObject", "m001", {"json": prefab()}), + MOC: ("MonoBehaviour", "m001", {"json": {"$script": CUBISM + "CubismMoc"}, "blob:_bytes": moc3()}), + TEX0: ("Texture2D", "texture_00", {"json": {"m_Name": "texture_00"}, "image": b"png0"}), + TEX1: ("Texture2D", "texture_01", {"json": {"m_Name": "texture_01"}, "image": b"png1"}), + MAT0: ("Material", "Unlit", {"json": material}), + MAT1: ("Material", "UnlitMasked", {"json": {**material, "material": "UnlitMasked", "keywords": [MASK]}}), + SHADER: ( + "Shader", + "Live2D Cubism/Unlit", + { + "json": {"name": "Live2D Cubism/Unlit"}, + "index": {"name": "Live2D Cubism/Unlit", "variants": SHADER_VARIANTS}, + "typetree": {"m_Name": ""}, + **{v["role"]: f"program {v['role']}".encode() for v in SHADER_VARIANTS}, + }, + ), + CLIP: ("AnimationClip", "idle", {"json": {"name": "idle", "sampleRate": 30.0, "bindings": [clip_binding]}}), + FADE: ("MonoBehaviour", "idle", {"json": fade()}), + EXPR: ( + "MonoBehaviour", + "smile.exp3", + { + "json": { + "$script": FRAMEWORK + "CubismExpressionData", + "m_Name": "smile.exp3", + "Type": "Live2D Expression", + "FadeInTime": 0.5, + "FadeOutTime": 0.5, + "Parameters": [{"Id": "ParamEyeLOpen", "Value": 0.0, "Blend": 2}], + } + }, + ), + SCRIPT: ("MonoScript", "CubismModel", {"json": {"m_ClassName": "CubismModel"}}), + MASK_MAT: ("Material", "Mask", {"json": {**material, "material": "Mask", "shader": ref(MASK_SHADER)}}), + CULL_MAT: ( + "Material", + "MaskCulling", + {"json": {**material, "material": "MaskCulling", "shader": ref(MASK_SHADER)}}, + ), + MASK_SHADER: ( + "Shader", + "Live2D Cubism/Mask", + { + "json": {"name": "Live2D Cubism/Mask"}, + "index": {"name": "Live2D Cubism/Mask", "variants": MASK_VARIANTS}, + **{v["role"]: b"mask program" for v in MASK_VARIANTS}, + }, + ), + } + + +def table(depot, objs: dict | None = None, roots=(ROOT, MASK_MAT, CULL_MAT)) -> dict: + """The object table of `objs` (objects()), every file put in `depot`.""" + out = {} + for oid, (cls, name, files) in sorted((objs if objs is not None else objects()).items()): + recs = {} + for role, body in sorted(files.items()): + data = body if isinstance(body, bytes) else contract.encode(body) + ext = ("png" if role == "image" else "bin") if isinstance(body, bytes) else "json" + recs[role] = {"sha256": depot.put(data), "size": len(data), "ext": ext} + out[oid] = {"status": "exported", "class": cls, "name": name, "files": recs} + return {"schema": contract.OBJECTS, "roots": sorted(roots), "objects": out, "missing": []} diff --git a/tests/pipeline/test_live2d.py b/tests/pipeline/test_live2d.py new file mode 100644 index 0000000..d67f7ee --- /dev/null +++ b/tests/pipeline/test_live2d.py @@ -0,0 +1,84 @@ +"""The Live2D product stages (pipeline.live2d) run on a synthetic object table.""" + +import json +import subprocess +import sys + +import pytest + +from nnnotes.pipeline import contract, layout +from nnnotes.pipeline.contract import Input, Task +from nnnotes.pipeline.depot import Depot +from nnnotes.pipeline.live2d import CubismStage, ModelStage, cubism_kind +from nnnotes.pipeline.stages import execute, registry +from tests.helpers import live2dmodel as lm + + +def run(tmp_path, stage): + depot = Depot(tmp_path / "d") + data = contract.encode(lm.table(depot)) + inp = Input("objects", depot.put(data), len(data), None, ({"kind": "depot"},)) + t = Task(stage.name, stage.version, lm.KEY, {}, {}, (inp,)) + execute(t, depot, registry(stage), force=True) + doc = depot.result(t.key) + arts = {contract.parse_artifact_id(a["id"])[1]: a for a in doc["artifacts"]} + return depot, t, doc, arts + + +def test_the_stages_start_from_the_key_and_the_mask_materials(): + model, cubism = ModelStage(), CubismStage() + assert model.keys(lm.KEY, None) == [lm.KEY] and cubism.keys(lm.KEY, None) == [lm.KEY] + assert model.resources(lm.KEY, None) == ["Live2D/Cubism/Materials/Mask", "Live2D/Cubism/Materials/MaskCulling"] + assert cubism.resources(lm.KEY, None) == [] + assert model.manifest(lm.KEY) == "models/m001.json" and cubism.manifest(lm.KEY) == "cubism/m001.json" + with pytest.raises(ValueError, match="not a Live2D model key"): + model.keys("Character/Live2D/c001/m001/other", None) + + +def test_the_model_product(tmp_path): + depot, t, doc, arts = run(tmp_path, ModelStage()) + assert set(arts) == {"model.json", "objects"} and arts["model.json"]["semantics"]["kind"] == "live2d.model" + model = contract.loads(depot.read(arts["model.json"]["content"]["sha256"])) + assert model["prefab"] == lm.ROOT and model["format"] == 3 + table = contract.loads(depot.read(arts["objects"]["content"]["sha256"])) + assert lm.SCRIPT not in table["objects"] + shader = table["objects"][lm.SHADER]["files"] + assert "typetree" not in shader and "program:vulkan/s0p0_progVertex_4" not in shader + assert doc["items"][0]["class"] == "Live2DModel" and doc["items"][0]["status"] == "exported" + manifest = layout.product_manifest(t.id, doc, table) + assert manifest["product"] == "live2d.model" and set(manifest["files"]) == {"model.json"} + assert lm.TEX0 in manifest["objects"] and "image" in manifest["objects"][lm.TEX0]["files"] + + +def test_the_cubism_product(tmp_path): + depot, t, doc, arts = run(tmp_path, CubismStage()) + assert {p: a["semantics"]["kind"] for p, a in arts.items()} == { + "m001.moc3": "live2d.moc3", + "m001.model3.json": "live2d.model3", + "m001.physics3.json": "live2d.physics3", + "motions/idle.motion3.json": "live2d.motion", + "smile.exp3.json": "live2d.expression", + "textures/texture_00.png": "live2d.texture", + "textures/texture_01.png": "live2d.texture", + "objects": "product.objects", + } + table = contract.loads(depot.read(arts["objects"]["content"]["sha256"])) + assert table["objects"] == {} + model3 = json.loads(depot.read(arts["m001.model3.json"]["content"]["sha256"])) + assert model3["FileReferences"]["Moc"] == "m001.moc3" + assert doc["items"][0]["class"] == "CubismModel" + + +def test_cubism_kinds(): + assert cubism_kind("x.model3.json") == "live2d.model3" and cubism_kind("textures/a.png") == "live2d.texture" + with pytest.raises(ValueError, match="not a Cubism SDK file"): + cubism_kind("other.txt") + + +def test_planning_the_stages_does_not_import_unitypy(): + code = ( + "import sys, nnnotes.pipeline.live2d as m; m.ModelStage(); m.CubismStage(); " + "print(sorted(x for x in ('UnityPy', 'numpy', 'PIL') if x in sys.modules))" + ) + r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) + assert r.returncode == 0 and r.stdout.strip() == "[]", r.stderr[-2000:] diff --git a/tests/pipeline/test_modelstages.py b/tests/pipeline/test_modelstages.py index bb59984..5edc140 100644 --- a/tests/pipeline/test_modelstages.py +++ b/tests/pipeline/test_modelstages.py @@ -1,263 +1,28 @@ -"""The model stages: the closures of Live2D model keys (as Catalog.resolve gives them), the task catalog the -Live2D extractor runs on, the keys of both stages, and their runs with synthetic extractors, censuses and objects.""" +"""The Spine skeleton stage: its subjects, closures, keys and runs with synthetic censuses and objects.""" import subprocess import sys -import zipfile from pathlib import Path import pytest from nnnotes.pipeline import census, contract, link, modelstages -from nnnotes.pipeline.catalogdb import index from nnnotes.pipeline.census import CensusStage -from nnnotes.pipeline.contract import IncompatibleTask, Input, Task +from nnnotes.pipeline.contract import Input, Task from nnnotes.pipeline.depot import Depot -from nnnotes.pipeline.modelstages import Closures, Live2DStage, SpineStage, TaskCatalog +from nnnotes.pipeline.modelstages import SpineStage from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry -from nnnotes.source.catalog import Catalog -from tests.helpers import synth - -H = [f"{i:x}" * 32 for i in range(1, 10)] -M1 = "Character/Live2D/g1/m1/model/m1" -M2 = "Character/Live2D/g2/m2/model/m2" -PREFAB = "Assets/AddressableResources/Character/Live2D/g1/m1/model/m1.prefab" - - -def remote_entries(order=(2, 3, 5), m1_prefab=PREFAB): - return [ - (M1, "Character/Live2D/g1/m1/model/m1", [3]), # an alias location first - (M1, m1_prefab, list(order), {"type": "UnityEngine.GameObject"}), - (f"model_m1_{H[0]}.bundle", synth.remote(f"model_m1_{H[0]}.bundle"), [4]), - (f"shared_{H[1]}.bundle", synth.remote(f"shared_{H[1]}.bundle"), []), - (f"tex_{H[2]}.bundle", synth.remote(f"tex_{H[2]}.bundle"), [2]), # a cycle back to the model bundle - ("monoscripts.bundle", synth.local("monoscripts.bundle"), []), - (M2, "Assets/AddressableResources/Character/Live2D/g2/m2/model/m2.prefab", [7, 5, 3]), - (f"model_m2_{H[3]}.bundle", synth.remote(f"model_m2_{H[3]}.bundle"), []), - ("Character/Live2D/g2/m2/motion/idle", "Assets/Game/idle.anim", [7]), - ("Character/Live2D/g3/m3/model/m3", "Assets/Game/m3.prefab", []), # no bundle - ] - - -APK = [("Shared/S", "Assets/Game/Shared/S.prefab", [1]), ("monoscripts.bundle", synth.local("monoscripts.bundle"), [])] - - -def build(entries): - return synth.CatalogWriter().build(entries) - - -def make_apk(path, entries=APK): - with zipfile.ZipFile(path, "w") as z: - z.writestr("assets/aa/catalog.bin", build(entries)) - return path - - -def test_closures_are_those_of_catalog_resolve(tmp_path): - remote = build(remote_entries()) - cat = Catalog(remote, tmp_path / "downloads", apk=make_apk(tmp_path / "base.apk")) - doc = index(remote, build(APK)) - c = Closures(doc) - names = {b["stable"]: b["name"] for b in doc["bundles"]} - assert c.keys() == cat.keys() - for key in cat.keys(): - assert c.entry(key)["internalId"] == cat.entry(key)["internal_id"] - assert [names[s] for s in c.closure_bundles(key)] == [b.name for b in cat.resolve(key)], key - assert c.entry(M1)["internalId"] == PREFAB # the location inside a bundle - assert c.closure_bundles(M1) == ["monoscripts", "shared", "model_m1", "tex"] - assert c.own_bundle(M1) == "model_m1" and c.own_bundle("Character/Live2D/g3/m3/model/m3") is None - with pytest.raises(KeyError): - c.entry("Character/Live2D/none") - - -# ---------------------------------------------------------------- live2d.model -def contents(tag=b""): - return { - "model_m1": b"UnityFS m1" + tag, - "shared": b"UnityFS shared", - "tex": b"UnityFS tex", - "monoscripts": b"UnityFS scripts", - "model_m2": b"UnityFS m2", - } - - -def bundles_fact(doc, data, locators=True): - out = {} - for b in doc["bundles"]: - body = data[b["stable"]] - locs = ({"kind": "download", "path": f"bundles/{b['name']}"},) if locators else () - out[b["stable"]] = Input("bundle", contract.sha256(body), len(body), b["name"], locs) - return out - - -def live2d_env(depot, entries=None, data=None, selected=None, locators=True): - doc = index(build(entries or remote_entries()), build(APK)) - data = data or contents() - bundles = bundles_fact(doc, data, locators) - return Env(depot, {"index": doc, "bundles": bundles, "selected": sorted(selected or bundles)}) - - -def test_live2d_subjects_inputs_and_context(tmp_path): - stage = Live2DStage() - env = live2d_env(Depot(tmp_path / "s")) - assert stage.subjects(env) == [M1, M2] # model keys with their bundle - assert stage.subjects(live2d_env(Depot(tmp_path / "s"), selected=["model_m2", "shared"])) == [M2] - t = describe(stage, M1, None, env) - assert t.id == "live2d.model:" + M1 and t.params == {} and t.atoms == Live2DStage.ATOMS - assert [(i.role, i.name) for i in t.inputs] == [ - ("bundle:000", "monoscripts.bundle"), - ("bundle:001", f"shared_{H[1]}.bundle"), - ("bundle:002", f"model_m1_{H[0]}.bundle"), - ("bundle:003", f"tex_{H[2]}.bundle"), - ] - assert t.inputs[2].sha256 == contract.sha256(contents()["model_m1"]) - assert t.context == {"internalId": PREFAB} - assert t.inputs[0].locators == ({"kind": "download", "path": "bundles/monoscripts.bundle"},) - assert set(Live2DStage.ATOMS) == {"live2d.extract_model"} - assert all(lib.lower() in Live2DStage.ATOMS["live2d.extract_model"] for lib in modelstages.LIBRARIES) - - -def test_live2d_keys_follow_the_closure_contents_and_order_only(tmp_path): - stage, depot = Live2DStage(), Depot(tmp_path / "s") - key = describe(stage, M1, None, live2d_env(depot)).key - # other file names, no locators, another catalog with more keys: the same task - renamed = [(e[0].replace(H[1], H[8]), e[1].replace(H[1], H[8]), *e[2:]) for e in remote_entries()] - renamed.append(("Other/key", "Assets/Game/other.asset", [3])) - assert describe(stage, M1, None, live2d_env(depot, renamed, locators=False)).key == key - # a bundle of the closure changed, the closure's order changed, the key's location changed: another task - assert describe(stage, M1, None, live2d_env(depot, data=contents(b"!"))).key != key - assert describe(stage, M1, None, live2d_env(depot, remote_entries(order=(3, 2, 5)))).key != key - assert describe(stage, M1, None, live2d_env(depot, remote_entries(m1_prefab=PREFAB + "x"))).key != key - # a bundle outside the closure changed: the same task - other = dict(contents(), model_m2=b"UnityFS m2 changed") - assert describe(stage, M1, None, live2d_env(depot, data=other)).key == key - bumped = type("Bumped", (Live2DStage,), {"version": 2})() - assert describe(bumped, M1, None, live2d_env(depot)).key != key - - -def test_live2d_waits_for_unfetched_bundles_and_names_unreadable_ones(tmp_path): - stage, depot = Live2DStage(), Depot(tmp_path / "s") - env = live2d_env(depot) - env.facts["bundles"] = dict(env.facts["bundles"], tex=Pending("fetch:tex")) - with pytest.raises(Pending, match="fetch:tex"): - describe(stage, M1, None, env) - env.facts["bundles"]["tex"] = None - with pytest.raises(Pending, match="fetch:tex"): - describe(stage, M1, None, env) - del env.facts["bundles"]["tex"] - with pytest.raises(ValueError, match="bundle tex is not readable"): - describe(stage, M1, None, env) - - -def stored_task(depot, stage=None): - """The live2d.model task of M1 with every bundle in the depot (depot locators only).""" - for body in contents().values(): - depot.put(body) - return describe(stage or Live2DStage(), M1, None, live2d_env(depot, locators=False)) - - -def test_task_catalog_answers_from_the_task(tmp_path): - depot = Depot(tmp_path / "s") - t = stored_task(depot) - cat = TaskCatalog.from_task(t, depot) - assert cat.apk is not None and cat.has(M1) and not cat.has(M2) - assert cat.keys("Character/") == [M1] and cat.keys("Other/") == [] - assert cat.entry(M1)["internal_id"] == PREFAB and cat.entries_for(M2) == [] - order = ["monoscripts", "shared", "model_m1", "tex"] - assert [p.read_bytes() for p in cat.fetch_key(M1)] == [contents()[s] for s in order] - assert [b.name for b in cat.resolve(M1)] == [i.name for i in t.inputs] - assert [b.offset for b in cat.resolve(M1)] == [0, 1, 2, 3] - for f in (cat.entry, cat.fetch_key, cat.resolve): - with pytest.raises(KeyError): - f(M2) - # inputs are ordered by position, not by role text - many = [Input(f"bundle:{i}", contract.sha256(b"x%d" % i), 2) for i in (10, 2, 1)] - assert [ - i.role - for i in TaskCatalog( - "k", "Assets/k", modelstages._ordered(Task(Live2DStage.name, 1, "k", inputs=tuple(many))), depot - ).bundles - ] == ["bundle:1", "bundle:2", "bundle:10"] - - -def fake_extract(calls): - def extract_model(cat, key, out_dir): - assert cat.apk is not None - data = b"".join(p.read_bytes() for p in cat.fetch_key(key)) - name = key.rsplit("/", 1)[-1] - calls.append((key, cat.entry(key)["internal_id"])) - (out_dir / "textures").mkdir() - (out_dir / "motions").mkdir() - (out_dir / f"{name}.moc3").write_bytes(b"MOC3" + data) - (out_dir / "textures" / "texture_00-1a2b3c4d.png").write_bytes(b"\x89PNG page") - (out_dir / f"{name}.prefab.json").write_text('{"nodes": []}', encoding="utf-8") - (out_dir / "motions" / "idle.motion3.json").write_text("{}", encoding="utf-8") - (out_dir / "motions" / "_fades.json").write_text("{}", encoding="utf-8") - (out_dir / "smile.exp3.json").write_text("{}", encoding="utf-8") - (out_dir / f"{name}.model3.json").write_text('{"Version": 3}\n', encoding="utf-8") - return {"name": name, "textures": ["textures/texture_00-1a2b3c4d.png"], "motions": 1, "expressions": 1} - - return extract_model - - -def test_live2d_run_stores_every_file_of_the_extractor(tmp_path, monkeypatch): - from nnnotes.unity import live2d - - calls = [] - monkeypatch.setattr(live2d, "extract_model", fake_extract(calls)) - depot = Depot(tmp_path / "s") - stage = Live2DStage() - t = stored_task(depot, stage) - ex = execute(t, depot, registry(stage)) - assert ex.status == "ran" and calls == [(M1, PREFAB)] - doc = depot.result(t.key) - tid = "live2d.model:" + M1 - got = {contract.parse_artifact_id(a["id"])[1]: a for a in doc["artifacts"]} - assert sorted(got) == [ - "m1.moc3", - "m1.model3.json", - "m1.prefab.json", - "motions/_fades.json", - "motions/idle.motion3.json", - "smile.exp3.json", - "textures/texture_00-1a2b3c4d.png", - ] - assert all(a["id"].startswith(tid + "#") for a in doc["artifacts"]) and doc["items"] == [] - kinds = {r: (a["semantics"]["kind"], a["content"]["ext"]) for r, a in got.items()} - assert kinds == { - "m1.model3.json": ("live2d.model3", "json"), - "m1.moc3": ("live2d.moc3", "moc3"), - "m1.prefab.json": ("live2d.prefab", "json"), - "motions/_fades.json": ("live2d.fades", "json"), - "motions/idle.motion3.json": ("live2d.motion", "json"), - "smile.exp3.json": ("live2d.expression", "json"), - "textures/texture_00-1a2b3c4d.png": ("live2d.texture", "png"), - } - order = ["monoscripts", "shared", "model_m1", "tex"] - assert depot.read(got["m1.moc3"]["content"]["sha256"]) == b"MOC3" + b"".join(contents()[s] for s in order) - assert got["m1.model3.json"]["semantics"]["facts"] == { - "expressions": 1, - "motions": 1, - "name": "m1", - "textures": ["textures/texture_00-1a2b3c4d.png"], - } - assert execute(t, depot, registry(stage)).status == "hit" - assert execute(t, depot, registry(stage), force=True).status == "ran" # the same result again - other = Task(t.stage, t.version, t.subject, t.params, {"live2d.extract_model": "unitypy-0/1"}, t.inputs, t.context) - with pytest.raises(IncompatibleTask): - execute(other, depot, registry(stage)) -def test_the_model_key_and_libraries_are_those_of_the_extractors(): - from nnnotes.site import webmodel +def test_the_libraries_are_those_of_the_extractor(): from nnnotes.unity import export - assert modelstages.MODEL_KEY.pattern == webmodel.MODEL_KEY.pattern assert modelstages.LIBRARIES == export._LIBS def test_importing_the_stages_does_not_import_unitypy(): code = ( - "import sys, nnnotes.pipeline.modelstages as m; m.Live2DStage(); m.SpineStage(); " + "import sys, nnnotes.pipeline.modelstages as m; m.SpineStage(); " "assert 'UnityPy' not in sys.modules, 'UnityPy imported'" ) r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) diff --git a/tests/site/test_webmodel.py b/tests/site/test_webmodel.py index 814231f..93c3ff3 100644 --- a/tests/site/test_webmodel.py +++ b/tests/site/test_webmodel.py @@ -18,34 +18,6 @@ KEY_B = "Character/Live2D/sub_x/adv_model_b/model/adv_model_b" -def test_model_id_is_the_model_name(): - assert webmodel.model_id(KEY_A) == "adv_model_a" - for bad in ( - "Character/Live2D/001_adv/a/model/b", - "Character/Live2D/a/model/a", - "Other/001/a/model/a", - "Character/Live2D/001_adv/a/model/a/extra", - ): - with pytest.raises(ValueError, match="not a Live2D model key"): - webmodel.model_id(bad) - with pytest.raises(ValueError, match="not URL-safe"): - webmodel.model_id("Character/Live2D/g/a b/model/a b") - - -def test_model_keys_and_select(): - keys = [KEY_B, "Character/Live2D/001_adv/adv_model_a/motion/x", KEY_A, "Live/Score/1"] - models = webmodel.model_keys(keys) - assert models == {"adv_model_a": KEY_A, "adv_model_b": KEY_B} - assert list(models) == ["adv_model_a", "adv_model_b"] - assert webmodel.select(None, models) == models - assert webmodel.select([KEY_B, "adv_model_a"], models) == models - assert webmodel.select(["adv_model_b"], models) == {"adv_model_b": KEY_B} - with pytest.raises(ValueError, match="nope"): - webmodel.select(["adv_model_a", "nope"], models) - with pytest.raises(ValueError, match="model id adv_model_a"): - webmodel.model_keys([KEY_A, "Character/Live2D/002_adv/adv_model_a/model/adv_model_a"]) - - def test_shader_names(): doc = { "nodes": [ From 93cfb757a309948cf823cc44318561efc1c7f790 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 11:33:38 +0800 Subject: [PATCH 18/91] feat(layout)!: list only the objects with files in a product manifest A product manifest named every object of its object table, including the Transforms and components contained in a prefab's file. Those entries carried no files, and the references a product reads from other objects' files name objects with files, so the manifest now lists only those. An object contained in a prefab is named by the prefab's JSON, whose nodes and components carry their path ids. For a Live2D model this drops the hierarchy's objects (several thousand entries) from models/.json, so a player fetches and parses only the entries it resolves. Co-Authored-By: Claude Opus 5.5 --- docs/products.md | 8 ++++---- src/nnnotes/pipeline/layout.py | 12 ++++-------- src/nnnotes/schemas/product.schema.json | 19 ++++--------------- tests/cli/test_assets.py | 2 +- tests/pipeline/test_assemble.py | 2 +- 5 files changed, 14 insertions(+), 29 deletions(-) diff --git a/docs/products.md b/docs/products.md index 1aa989a..c3be8a6 100644 --- a/docs/products.md +++ b/docs/products.md @@ -80,12 +80,12 @@ placement: the Cubism SDK files as a directory). "files": {"model.json": {"asset": "assets/.json", "size": 2048}}, "objects": { "CAB-…:-4153…": {"class": "GameObject", "name": "model", - "files": {"json": {"asset": "assets/.json", "size": 5120}}}, - "CAB-…:12": {"class": "Transform", "in": "CAB-…:-4153…"}}} + "files": {"json": {"asset": "assets/.json", "size": 5120}}}}} ``` -`files` are the product's own files by path; `objects` are the objects of its object table that have files, or are -contained in an object that has one. An asset path is relative to the directory above the manifest's directory, as +`files` are the product's own files by path; `objects` are the objects of its object table that have files. An object +contained in another object's file (a Transform or a component in a prefab) is not listed: the prefab's JSON holds it +(its nodes and components carry their `pathId`). An asset path is relative to the directory above the manifest's directory, as in the chart manifests of ournotes-player. A reference in an object's JSON (`{"$ref": {"file", "pathId"}}`) names the object `:` of `objects`. diff --git a/src/nnnotes/pipeline/layout.py b/src/nnnotes/pipeline/layout.py index 705df28..1b961aa 100644 --- a/src/nnnotes/pipeline/layout.py +++ b/src/nnnotes/pipeline/layout.py @@ -365,8 +365,8 @@ def _asset(c: dict) -> dict: def product_manifest(task: str, result: dict, table: dict) -> dict: """The manifest (nnnotes.product/1) of a product: its files {path: {asset, size}} and the objects of its object - table `table` that have files ({class, name, files: {role: {asset, size}}}) or are contained in one ({class, - in}).""" + table `table` that have files ({class, name, files: {role: {asset, size}}}). An object contained in another's + file (a Transform or a component in a prefab) is named by that file and not listed.""" stage, subject = contract.parse_task_id(task) files = {} for a in result["artifacts"]: @@ -375,14 +375,10 @@ def product_manifest(task: str, result: dict, table: dict) -> dict: files[role] = _asset(a["content"]) objects = {} for oid, o in table["objects"].items(): - e = {k: o[k] for k in ("class", "name") if k in o} if o.get("files"): + e = {k: o[k] for k in ("class", "name") if k in o} e["files"] = {r: _asset(f) for r, f in sorted(o["files"].items())} - elif "in" in o: - e["in"] = o["in"] - else: - continue - objects[oid] = e + objects[oid] = e return {"schema": contract.PRODUCT, "product": stage, "subject": subject, "files": files, "objects": objects} diff --git a/src/nnnotes/schemas/product.schema.json b/src/nnnotes/schemas/product.schema.json index bb01211..b9377c0 100644 --- a/src/nnnotes/schemas/product.schema.json +++ b/src/nnnotes/schemas/product.schema.json @@ -43,23 +43,12 @@ "additionalProperties": { "$ref": "#/$defs/asset" } - }, - "in": { - "type": "string" } }, - "oneOf": [ - { - "required": [ - "files" - ] - }, - { - "required": [ - "in" - ] - } - ] + "required": [ + "files" + ], + "additionalProperties": false } } }, diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index 1b4f66b..185a9b6 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -1496,7 +1496,7 @@ def test_the_product_layout_places_the_products_of_the_selection(tmp_path, capsy assert contract.loads(index)["roots"] == [f"{CABS['bg_x']}:3"] go = manifest["objects"][f"{CABS['bg_x']}:3"] assert go["class"] == "GameObject" and (tmp_path / "out" / go["files"]["json"]["asset"]).is_file() - assert manifest["objects"][f"{CABS['bg_x']}:2"] == {"class": "Transform", "in": f"{CABS['bg_x']}:3"} + assert f"{CABS['bg_x']}:2" not in manifest["objects"] # a Transform of the prefab assert sorted(p.name for p in (tmp_path / "out").iterdir()) == [ assets.REPORTS, "assets", diff --git a/tests/pipeline/test_assemble.py b/tests/pipeline/test_assemble.py index 4cd733d..49ff357 100644 --- a/tests/pipeline/test_assemble.py +++ b/tests/pipeline/test_assemble.py @@ -376,7 +376,7 @@ def test_the_product_layout_places_the_files_and_the_objects(tmp_path): "size": json_rec["content"]["size"], } assert manifest["objects"][oid(10)]["class"] == "Texture2D" - assert manifest["objects"][oid(61)]["in"] == oid(60) + assert oid(61) not in manifest["objects"] # contained in the prefab of oid(60), named by its file assert "assets" in manifest["objects"][oid(10)]["files"]["image"]["asset"] # the manifest path is one directory and one file assert len(entries) == len({(e["path"], e["id"]) for e in entries}) From 7d6cc5842e777f99abfb8cc02768f4516ac1e1ec Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 11:49:19 +0800 Subject: [PATCH 19/91] feat(convert)!: write path ids as decimal strings in object JSON A path id is a 64-bit integer. Written as a JSON number, a reader that parses numbers as doubles (JavaScript's JSON.parse) rounds most of them, so the object a reference names could not be looked up by its id there. Object JSON now writes every path id as its decimal string: the pathId of a {"$ref"}, and the pathId and transformPathId of a prefab's nodes and components. The object id ":" is the reference's file and pathId as written. The converters whose documents hold references move to a new version (sprite.png/3, prefab.json/3, material.json/2, clip.json/3, controller.json/3, mono.json/3, font/3, shader.json/2, generic.json/3), so their results are made again. Co-Authored-By: Claude Opus 5.5 --- docs/stages.md | 18 +++---- src/nnnotes/unity/convert/__init__.py | 20 ++++---- src/nnnotes/unity/convert/objects.py | 4 +- src/nnnotes/unity/convert/prefab.py | 6 +-- tests/golden/sprite.crop.json | 4 +- tests/golden/unity.export.json | 70 +++++++++++++-------------- tests/helpers/live2dmodel.py | 10 ++-- tests/pipeline/test_objexport.py | 30 ++++++------ tests/unity/test_bindings.py | 10 ++-- 9 files changed, 87 insertions(+), 85 deletions(-) diff --git a/docs/stages.md b/docs/stages.md index 4dc4fdc..693b619 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -49,17 +49,17 @@ with its reason code; any other error of a converter gives it with `generic.erro | Class | Converter | Artifacts | Atoms | Notes | |---|---|---|---|---| | Texture2D | `tex.png/2` | `image`: PNG of the first mip level (HDR ASTC: `.astc`), `json`: the typetree without `image data` (settings, colour space, mip count, stream data) | `texture.decode`, `png.encode`, `astc.container` | further mip levels as facts; a texture without pixels: `empty.texture` | -| Sprite | `sprite.png/2` | `image`: PNG on the sprite's rect (packing undone, tight mask applied), `json`: the typetree, geometry, render data and image (see below) | `texture.decode`, `sprite.crop`, `png.encode`, `mesh.arrays` | the tasks of a bundle share the textures it decoded last; a sprite whose SpriteAtlas is in another bundle gets its `json` here (with its geometry and object facts) and its `image` from `sprite.crop` | +| Sprite | `sprite.png/3` | `image`: PNG on the sprite's rect (packing undone, tight mask applied), `json`: the typetree, geometry, render data and image (see below) | `texture.decode`, `sprite.crop`, `png.encode`, `mesh.arrays` | the tasks of a bundle share the textures it decoded last; a sprite whose SpriteAtlas is in another bundle gets its `json` here (with its geometry and object facts) and its `image` from `sprite.crop` | | Mesh | `mesh.glb/2` | `mesh`: binary glTF in the mesh's local space, `json`: the typetree without the vertex, index and compressed mesh data | `mesh.arrays`, `gltf.write` | see `gltf.write` below; limits of the skin and blend shapes as issues of the artifact | -| GameObject (a root) | `prefab.json/2` | `json`: the hierarchy's nodes (path, path ids, transform) with their components' fields | | the objects below it are `contained`; a root of a scene's file names its scene | -| Material | `material.json/1` | `json`: shader, keywords, textures, floats, colours | | | -| Shader | `shader.json/1` | `json`: the parsed form (the bytes of `nnnotes export shader`'s `.json`), `program:/sp

__`: each compiled sub-program as stored (`.glsl`, `.metal`, `.vkprog`), `index`: the variants (platform, subshader, pass, stage, GPU program type, keywords, program role), `typetree`: the typetree JSON | | a shader without a parsed form: its typetree with `generic.error` | -| AnimationClip | `clip.json/2` | `json`: the Mecanim clip data and its bindings as stored (see below) | | a legacy or compressed clip: `generic.clip.legacy` | -| AnimatorController | `controller.json/2` | `json`: name, clips (`$ref`), layers, state machines, parameters, default values | | an AnimatorOverrideController: `generic.controller.override` | -| MonoBehaviour (not in a hierarchy) | `mono.json/2`, `cri.usm/1` | `json`: `$script` (assembly, namespace, class) and the typetree; a cue sheet asset also `acb`, a movie asset holding its USM also `usm` (see below) | `sniff` | the class from the task's script table | +| GameObject (a root) | `prefab.json/3` | `json`: the hierarchy's nodes (path, path ids, transform) with their components' fields | | the objects below it are `contained`; a root of a scene's file names its scene | +| Material | `material.json/2` | `json`: shader, keywords, textures, floats, colours | | | +| Shader | `shader.json/2` | `json`: the parsed form (the bytes of `nnnotes export shader`'s `.json`), `program:/sp

__`: each compiled sub-program as stored (`.glsl`, `.metal`, `.vkprog`), `index`: the variants (platform, subshader, pass, stage, GPU program type, keywords, program role), `typetree`: the typetree JSON | | a shader without a parsed form: its typetree with `generic.error` | +| AnimationClip | `clip.json/3` | `json`: the Mecanim clip data and its bindings as stored (see below) | | a legacy or compressed clip: `generic.clip.legacy` | +| AnimatorController | `controller.json/3` | `json`: name, clips (`$ref`), layers, state machines, parameters, default values | | an AnimatorOverrideController: `generic.controller.override` | +| MonoBehaviour (not in a hierarchy) | `mono.json/3`, `cri.usm/1` | `json`: `$script` (assembly, namespace, class) and the typetree; a cue sheet asset also `acb`, a movie asset holding its USM also `usm` (see below) | `sniff` | the class from the task's script table | | TextAsset | `data/2` | `data`: the stored bytes, `json`: the typetree without `m_Script` | `sniff` | extension from the container path, else from the content | -| Font | `font/2` | `font`: the font file, `json`: the typetree without `m_FontData` | `sniff` | | -| any other class (a MonoScript, an AssetBundle, ...) | `generic.json/2` | `json`: the typetree | `sniff` | an object that can be read only in part (see below): that part, `unsupported.typetree` | +| Font | `font/3` | `font`: the font file, `json`: the typetree without `m_FontData` | `sniff` | | +| any other class (a MonoScript, an AssetBundle, ...) | `generic.json/3` | `json`: the typetree | `sniff` | an object that can be read only in part (see below): that part, `unsupported.typetree` | Every object a converter takes has a `json` artifact or is contained in another object's. Each item lists in `refs` the objects its documents reference (by object id, other bundles' objects included, the objects of its own diff --git a/src/nnnotes/unity/convert/__init__.py b/src/nnnotes/unity/convert/__init__.py index e0cb8f7..0a53d68 100644 --- a/src/nnnotes/unity/convert/__init__.py +++ b/src/nnnotes/unity/convert/__init__.py @@ -27,6 +27,8 @@ typetree JSON (a Texture2D, Mesh, TextAsset or Font without the bytes its other artifact holds). Object JSON (canon/1 in the object's own field order): +- a path id is a 64-bit integer, written as its decimal string (a JSON number of that size does not survive a + reader that parses numbers as doubles), so the object a reference names is ":" as it is written; - a reference is {"$ref": {"file", "pathId"}}, with "class" and "name" when the target is in the bundle; a null reference is null. Inside a prefab, a GameObject, Transform or component of the same prefab is named by its path: {"gameObject": path}, {"transform": path}, {"component": class, "gameObject": path}; @@ -66,17 +68,17 @@ # converter -> version CONVERTERS = { "tex.png": 2, - "sprite.png": 2, + "sprite.png": 3, "mesh.glb": 2, - "prefab.json": 2, - "material.json": 1, - "clip.json": 2, - "controller.json": 2, - "mono.json": 2, + "prefab.json": 3, + "material.json": 2, + "clip.json": 3, + "controller.json": 3, + "mono.json": 3, "data": 2, - "font": 2, - "shader.json": 1, - "generic.json": 2, + "font": 3, + "shader.json": 2, + "generic.json": 3, "cri.usm": 1, } # class -> the converters its objects may use (every class may fall back to generic.json) diff --git a/src/nnnotes/unity/convert/objects.py b/src/nnnotes/unity/convert/objects.py index ec167a7..b93a937 100644 --- a/src/nnnotes/unity/convert/objects.py +++ b/src/nnnotes/unity/convert/objects.py @@ -454,7 +454,7 @@ def dref(self, key: tuple) -> dict: """{"$ref"} of an object, with its class and name when it is in the bundle (recorded in `seen`).""" file, pid = key self.seen.add(object_id(file, pid)) - d = {"file": label(file), "pathId": pid} + d = {"file": label(file), "pathId": str(pid)} r = self.readers.get(key) if r is not None: d["class"] = r.type.name @@ -472,7 +472,7 @@ def ref(self, owner, pptr: dict): fid = int(pptr.get("m_FileID", 0) or 0) file = self.file_of(owner.assets_file.name, fid) if file is None: - return {"$ref": {"fileId": fid, "pathId": pid}} + return {"$ref": {"fileId": fid, "pathId": str(pid)}} key = (file, pid) if self._local is not None: local = self._local(key) diff --git a/src/nnnotes/unity/convert/prefab.py b/src/nnnotes/unity/convert/prefab.py index 1610952..c424706 100644 --- a/src/nnnotes/unity/convert/prefab.py +++ b/src/nnnotes/unity/convert/prefab.py @@ -133,7 +133,7 @@ def _subtree(ctx: BundleObjects, h: Hierarchies, f: str, g, root_tf: int): def _component(ctx: BundleObjects, h: Hierarchies, o, root: tuple, unresolved: list) -> dict: key = ctx.key(o) tt = ctx.tt(key, whole=False) - out = {"type": o.type.name, "pathId": key[1]} + out = {"type": o.type.name, "pathId": str(key[1])} if o.type.name == "MonoBehaviour": entry = ctx.mono_script(key, tt) out["script"] = entry if entry and entry != NO_SCRIPT else None @@ -164,8 +164,8 @@ def prefab(ctx: BundleObjects, info, h: Hierarchies, root: Root) -> None: node = { "path": g.path(tf), "name": go["m_Name"], - "pathId": go_pid, - "transformPathId": tf, + "pathId": str(go_pid), + "transformPathId": str(tf), "active": bool(go["m_IsActive"]), "layer": go["m_Layer"], "tag": go.get("m_Tag"), diff --git a/tests/golden/sprite.crop.json b/tests/golden/sprite.crop.json index 6cc0b0b..d15f112 100644 --- a/tests/golden/sprite.crop.json +++ b/tests/golden/sprite.crop.json @@ -5,8 +5,8 @@ }, "fixtures": { "atlas": { - "key": "b1f819e7c9328fb5accdbd6366070fb94a45f592ae5cc1d1a531d4326c90d873", - "result": "6f2bc5adaf8e95fe9a17028d47fdac301436432523758fbfedc8a0034338945c" + "key": "8bff1abc736f8e76c8ff92e0f87f28317e25bcd2a59dfbe4b8753144b862566b", + "result": "746170244ebd43a73004fc8c369d817d50216f51062ab73ebb9b57fbb7f6ad2f" } }, "libraries": { diff --git a/tests/golden/unity.export.json b/tests/golden/unity.export.json index de4fcb2..15f9837 100644 --- a/tests/golden/unity.export.json +++ b/tests/golden/unity.export.json @@ -1,79 +1,79 @@ { "atoms": { "astc.container": "nnnotes/1", - "clip.json": "clip.json/2", - "controller.json": "controller.json/2", + "clip.json": "clip.json/3", + "controller.json": "controller.json/3", "cri.usm": "cri.usm/1", "data": "data/2", - "font": "font/2", - "generic.json": "generic.json/2", + "font": "font/3", + "generic.json": "generic.json/3", "gltf.write": "numpy-2.4.6/1", - "material.json": "material.json/1", + "material.json": "material.json/2", "mesh.arrays": "numpy-2.4.6/1", "mesh.glb": "mesh.glb/2", - "mono.json": "mono.json/2", + "mono.json": "mono.json/3", "png.encode": "pillow-12.3.0/1", - "prefab.json": "prefab.json/2", + "prefab.json": "prefab.json/3", "reader": "fakeunity/1", - "shader.json": "shader.json/1", + "shader.json": "shader.json/2", "sniff": "nnnotes/1", "sprite.crop": "pillow-12.3.0+numpy-2.4.6/1", - "sprite.png": "sprite.png/2", + "sprite.png": "sprite.png/3", "tex.png": "tex.png/2", "texture.decode": "unitypy-1.25.3+texture2ddecoder-1.0.6+astc-encoder-py-0.1.12+pillow-12.3.0/1" }, "fixtures": { "clip.json": { - "key": "c2211c68d3286636469c74023637e15b3c11dade0f858ab2380498aa319171eb", - "result": "1f6ad672b184cd8b4cf9eeeb9ee0b8512ba532eaf708cb36b86c538b19910b75" + "key": "5174601cce42b801f88496a18f30601c9b96a98415e5ea634b60ef5e6e65cc6e", + "result": "2f98d3aa5e65721ad5549b8d39a32d632a4056b62306ecae289a259f8dd73b96" }, "controller.json": { - "key": "eaae782443b5874108271db45e6a0da404f91239f700dd827802081ba64dd184", - "result": "e63d57f287d5d388e1d6d2269bc03ffbf03165b11cb03a19d2ad003fb00cacf8" + "key": "47cb3924c6de41c3b40606809353c7f7b6e615a9fe4924772ceae7f6920274ab", + "result": "139a51c4e08d27211fdeb01d27cf9d81d0cd4256e7ee5d382b85ed088f3dea80" }, "cri.usm": { - "key": "c69fabc7e24dd038b2375b59a0421cd514a8ea5e5dc3a21c181dda809902dc3f", - "result": "d829f22ea9a736a478f2963470f2bdca4d6a3200e40a510606956fe771d63e00" + "key": "8a8b0f2d4ef9b6e9bd697efe2b27a8e6a9ae891dfba203c33420fc6eefa241d3", + "result": "2031ae1e748492317d8565e02f1a7c745333ff9e026f9334711b0a4f48d64758" }, "data": { - "key": "eec1daf7f06a873c434b64b5ab2d0629b61a71c331f16ed5434706b60ee3bee3", - "result": "493c173754a5a0088bc972fc8c373415d02b345060ab83df9ce41f8b0d0317f8" + "key": "313e7f1b4c9cad52dd5e62069ebf8372536735b7ab6548b41861cb923d7e63d2", + "result": "980c3dc7410587621768f517fd8b4fa34756dc10bdb3fbb8c7230e510c0a9f59" }, "font": { - "key": "81155f5c29ca33162b4b0ee97d7559fb0460ad02c46a187097bb6e4893792b7c", - "result": "d0da497a3f6207d7e92e31ac4313f289eb483b5fcf0496476a3fb50e235b830a" + "key": "f75bcfd326253ea01d22297c76e9fd1b209bed5c7470acc7769cd025a5a1c3d6", + "result": "b1d520e50ffbc83211fd51c8f7a7e84657ad2e932f4077525f36e9862d8603fb" }, "generic.json": { - "key": "76af5e1c7fd2b389a80c4725aa2b561ffb0ae552b0d65ce2eb4ad56710ae3201", - "result": "6d6d058a293360ee2065fbb205f35834198e7285a743f2e5b1dc847daa4cdad3" + "key": "dea8a6e77d1b1848d43b0ba28d16ca55812c455a0b254bb7a6703ea948e18b99", + "result": "097447e4235618804dcee54a5c5e9d27ec77a0012389b1c726fe463dca9694e3" }, "material.json": { - "key": "c391d41f8afdc880c74e24f143560ff80c122c97f82241cd2b4f0cdf8f5fad1f", - "result": "0990d8b5ccdc04c6335b2779354b3d64727ebccc701490ffe6a46346719e90e2" + "key": "7d4b88d018c77afbc9a9c745667d489830c2bbf97a20efd8ca0660d0d35112cf", + "result": "f58509825571f63771d27fa675bb98b849a6e3d17afd9047eaf8e5c1ce86034e" }, "mesh.glb": { - "key": "1e5a4fee78d3fd98334ce50a6b0b1cc56f5733252f2cb00ff974d515a362b911", - "result": "11949923ee05d67811b42f530d84313e9197afd74e203da44291761b8aa87994" + "key": "608ccea80ac1758cf022d3a21b9bca4bf61cfc4a716bbb05e58a5db26d531b0e", + "result": "04060bd45010e278b6659d5619ae5f9e4207211411f68573c95202119e8b4b44" }, "mono.json": { - "key": "0aeed465e4e3f8a310b4a2bd3cc06fa899fcdd5fb0ae04d3a5c2a5ccd85d09ea", - "result": "9711bc6d921e7c49ef0b86b487808c12b95e106bb791655a880fef9257ec6da3" + "key": "a005d9bbfb99bf1d67498f26ec4fa2d1f59cab6f30361d93d41ae815f4e25311", + "result": "0337425cb6d6fb774c3c297756fc97cb6c416471833a5eff3eaa6cc0680874fb" }, "prefab.json": { - "key": "0c801dd911545b249a8b629b3dfa05c7ea6c466482d7f93252f126717419dc94", - "result": "96507449031bd9d046088f01b64602d7191aa87d7cf5af29ecde677c051d20fe" + "key": "b5ae5dcac2a7c2bd8c49995a55397ae4539307c36425286b10c75cbaae4d2451", + "result": "c568f4b1773c3437518596374da5aaae7d0d9e5d54d436794410f528277a2c7a" }, "shader.json": { - "key": "105e05e73efa131a339c80a936f0e8dc44a2cebe3c25d332f500b51871dfe83e", - "result": "0f670ee8b5f4ceb584329ecd3a4695d70dd5bf0b9690b4df63b87963fb83913b" + "key": "7b4f858df794ef86cc62c1a5201a630ae91ab86ff0e6dbf946d836cf29a3b309", + "result": "aab00b71441e5978a98431d780fa1c4facb394389ca66901414541a238583197" }, "sprite.png": { - "key": "ab28c0f55b23cb58aaf5b94f4357231ab1f4be8ce16c6a02e69bfc1917a3509c", - "result": "82cf3a23fe661f24f5e9fc3189e2b70db39c287e39392397556e2b081beb4c2c" + "key": "f8fcd13c515ca0d0acd08cd983b23e917fbec42e7de0c351980b69cab4f4af09", + "result": "d053cdaf856f61c785987e324a847e04c4cdc0e9556059eba66dc737f7809531" }, "tex.png": { - "key": "f5b96a51a6de3073767d863f7fbd7f835df241386588ae3fddbfa7eae4d02c5a", - "result": "ea26cd3f966ff9ebb9c7491f72e720aa311d5552f5ef0b136d9712958dac6ac7" + "key": "5659b2e28ffb247c956a6273a41b1f6c0196e64737060a7228fde0ff1d946c3e", + "result": "84ac758d275634016b105b57698e5cac22ac57379ea8783486b833f1650c27f5" } }, "libraries": { diff --git a/tests/helpers/live2dmodel.py b/tests/helpers/live2dmodel.py index ee2a305..604d649 100644 --- a/tests/helpers/live2dmodel.py +++ b/tests/helpers/live2dmodel.py @@ -21,7 +21,7 @@ def ref(oid: str, cls: str | None = None) -> dict: file, _, pid = oid.rpartition(":") - r = {"file": file, "pathId": int(pid)} + r = {"file": file, "pathId": pid} if cls: r["class"] = cls return {"$ref": r} @@ -38,7 +38,7 @@ def moc3() -> bytes: def mono(script: str, **fields) -> dict: - return {"type": "MonoBehaviour", "pathId": 0, "script": script, "fields": fields} + return {"type": "MonoBehaviour", "pathId": "0", "script": script, "fields": fields} def node(path: str, *components) -> dict: @@ -89,7 +89,7 @@ def prefab() -> dict: mono(FRAMEWORK + "CubismPhysicsController", _rig=rig), mono("Live2D.Cubism|Live2D.Cubism.Framework.MotionSync|CubismMotionSyncController"), mono("Assembly-CSharp||Live2DMotionSyncCriAudioInput"), - {"type": "Animator", "pathId": 2, "fields": {}}, + {"type": "Animator", "pathId": "2", "fields": {}}, ), node("m001/Parameters"), node( @@ -106,12 +106,12 @@ def prefab() -> dict: node( "m001/Drawables/ArtMesh0", mono("Live2D.Cubism|Live2D.Cubism.Rendering|CubismRenderer", _mainTexture=ref(TEX1, "Texture2D")), - {"type": "MeshRenderer", "pathId": 3, "fields": {"m_Materials": [ref(MAT0, "Material")]}}, + {"type": "MeshRenderer", "pathId": "3", "fields": {"m_Materials": [ref(MAT0, "Material")]}}, ), node( "m001/Drawables/ArtMesh1", mono("Live2D.Cubism|Live2D.Cubism.Rendering|CubismRenderer", _mainTexture=ref(TEX0, "Texture2D")), - {"type": "MeshRenderer", "pathId": 4, "fields": {"m_Materials": [ref(MAT1, "Material")]}}, + {"type": "MeshRenderer", "pathId": "4", "fields": {"m_Materials": [ref(MAT1, "Material")]}}, ), ], } diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index 8ec6e27..292f288 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -855,7 +855,7 @@ def test_sprites_same_bundle(exported): assert load(depot, doc, f"{oid(pid)}#image") == png.encode(want) meta = obj(depot, doc, f"{oid(pid)}#json") rd = meta["renderData"] - assert rd["texture"] == {"$ref": {"file": CAB, "pathId": 10, "class": "Texture2D", "name": "tex"}} + assert rd["texture"] == {"$ref": {"file": CAB, "pathId": "10", "class": "Texture2D", "name": "tex"}} assert rd["textureRect"] == src["textureRect"] and rd["uvTransform"] == src["uvTransform"] assert rd["settingsRaw"] == src["settingsRaw"] and rd["alphaTexture"] is None assert meta["sprite"]["m_Name"] == tt["m_Name"] @@ -873,7 +873,7 @@ def test_sprites_same_bundle(exported): meta = obj(depot, doc, f"{oid(11)}#json") assert meta["renderData"]["source"] == "renderData" and meta["renderData"]["atlas"] is None meta = obj(depot, doc, f"{oid(12)}#json") - assert meta["renderData"]["source"] == "atlas" and meta["renderData"]["atlas"]["$ref"]["pathId"] == 20 + assert meta["renderData"]["source"] == "atlas" and meta["renderData"]["atlas"]["$ref"]["pathId"] == "20" assert meta["sprite"]["m_RD"]["m_VertexData"]["m_DataSize"]["$hex"] # small: inline refs = art(doc, f"{oid(12)}#image")["semantics"]["refs"] assert refs == [{"rel": "texture", "object": oid(10)}, {"rel": "atlas", "object": oid(20)}] @@ -885,7 +885,7 @@ def test_sprite_with_its_atlas_elsewhere_gets_a_meta_only(exported): a = art(doc, f"{oid(13)}#json") assert a["semantics"]["facts"] == {"crop": CROP, "atlas": f"{ATLAS_CAB}:99"} meta = obj(depot, doc, a["id"]) - assert meta["image"] == {"source": "atlas", "atlas": {"$ref": {"file": ATLAS_CAB, "pathId": 99}}, "stage": CROP} + assert meta["image"] == {"source": "atlas", "atlas": {"$ref": {"file": ATLAS_CAB, "pathId": "99"}}, "stage": CROP} assert meta["object"]["name"] == "remote" and len(meta["mesh"]["positions"]) == 4 @@ -957,13 +957,13 @@ def test_prefab(exported): p = obj(depot, doc, f"{oid(60)}#json") assert p["name"] == "root" and [n["path"] for n in p["nodes"]] == ["root", "root/child"] root, child = p["nodes"] - assert root["pathId"] == 60 and root["transformPathId"] == 61 and root["layer"] == 5 + assert root["pathId"] == "60" and root["transformPathId"] == "61" and root["layer"] == 5 (b,) = root["components"] assert b["type"] == "MonoBehaviour" and b["script"] == "Assembly-CSharp|Game|Behaviour" fields = b["fields"] assert fields["target"] == {"gameObject": "root/child"} - assert fields["tex"] == {"$ref": {"file": CAB, "pathId": 10, "class": "Texture2D", "name": "tex"}} - assert fields["m_Script"] == {"$ref": {"file": "CAB-scripts", "pathId": 7}} + assert fields["tex"] == {"$ref": {"file": CAB, "pathId": "10", "class": "Texture2D", "name": "tex"}} + assert fields["m_Script"] == {"$ref": {"file": "CAB-scripts", "pathId": "7"}} assert math.isnan(fields["value"]) and fields["small"] == {"$hex": "616263"} assert b'{"$float": "nan"}' in load(depot, doc, f"{oid(60)}#json").replace(b"\n", b"").replace( b" ", b"" @@ -971,7 +971,7 @@ def test_prefab(exported): (c,) = child["components"] assert c["fields"]["up"] == {"transform": "root"} assert c["fields"]["comp"] == {"component": "MonoBehaviour", "gameObject": "root", "class": "Behaviour"} - assert c["fields"]["far"] == {"$ref": {"file": "CAB-scripts", "pathId": 3}} + assert c["fields"]["far"] == {"$ref": {"file": "CAB-scripts", "pathId": "3"}} assert c["script"] == "Assembly-CSharp||Holder" @@ -991,7 +991,7 @@ def test_scriptable_objects_blobs_and_strings(exported): assert m["text"] == {"$hex": b"caf".hex() + "c3"} assert by_object(doc)[oid(70)]["artifacts"] == [f"{oid(70)}#blob:_bytes", f"{oid(70)}#json"] lost = obj(depot, doc, f"{oid(71)}#json") - assert "$script" not in lost and lost["m_Script"] == {"$ref": {"file": "CAB-scripts", "pathId": 9}} + assert "$script" not in lost and lost["m_Script"] == {"$ref": {"file": "CAB-scripts", "pathId": "9"}} def test_cue_sheets_held_in_the_bundle(exported): @@ -1164,7 +1164,7 @@ def test_clip_bindings_name_scripts_of_other_bundles(exported): "pptr": False, "int": False, "serializeReference": False, - "script": {"$ref": {"file": "CAB-scripts", "pathId": 12}}, + "script": {"$ref": {"file": "CAB-scripts", "pathId": "12"}}, "scriptClass": "Assembly-CSharp|Game|Fader", } assert census_mod.script_refs(census_mod.census_env(bundle_all())) == sorted(SCRIPTS) + ["CAB-scripts:9"] @@ -1173,8 +1173,8 @@ def test_clip_bindings_name_scripts_of_other_bundles(exported): def test_material_clip_controller(exported): depot, doc = exported mat = obj(depot, doc, f"{oid(50)}#json") - assert mat["shader"] == {"$ref": {"file": ATLAS_CAB, "pathId": 5}} - assert mat["textures"]["_MainTex"]["texture"]["$ref"]["pathId"] == 10 + assert mat["shader"] == {"$ref": {"file": ATLAS_CAB, "pathId": "5"}} + assert mat["textures"]["_MainTex"]["texture"]["$ref"]["pathId"] == "10" assert mat["colors"]["_Color"]["g"] == INF and b"1e999" in load(depot, doc, f"{oid(50)}#json") clip = obj(depot, doc, f"{oid(90)}#json") assert list(clip) == [ @@ -1209,7 +1209,7 @@ def test_material_clip_controller(exported): ] ctl = obj(depot, doc, f"{oid(92)}#json") assert list(ctl) == ["name", "clips", "layers", "stateMachines", "parameters", "defaultValues"] - assert ctl["clips"] == [{"$ref": {"file": CAB, "pathId": 90, "class": "AnimationClip", "name": "clip"}}] + assert ctl["clips"] == [{"$ref": {"file": CAB, "pathId": "90", "class": "AnimationClip", "name": "clip"}}] assert obj(depot, doc, f"{oid(95)}#json")["m_Speed"] == INF @@ -1223,7 +1223,7 @@ def test_scripts_and_asset_bundles_have_their_typetree(exported): { "preloadIndex": 0, "preloadSize": 1, - "asset": {"$ref": {"file": CAB, "pathId": 10, "class": "Texture2D", "name": "tex"}}, + "asset": {"$ref": {"file": CAB, "pathId": "10", "class": "Texture2D", "name": "tex"}}, }, ] @@ -1442,7 +1442,7 @@ def p(pid): return oid(pid, PLAYER) def script(pid, cls): - return {"$ref": {"file": PLAYER, "pathId": pid, "class": "MonoScript", "name": cls}} + return {"$ref": {"file": PLAYER, "pathId": str(pid), "class": "MonoScript", "name": cls}} assert items[p(2)]["status"] == "exported" assert obj(depot, doc, f"{p(2)}#json") == { @@ -1451,7 +1451,7 @@ def script(pid, cls): "m_Enabled": 1, "m_Script": script(50, "SoundVolumeSettings"), "m_Name": "volume", - "target": {"$ref": {"file": "unity default resources", "pathId": 10}}, + "target": {"$ref": {"file": "unity default resources", "pathId": "10"}}, "volume": 0.5, } assert items[p(2)]["refs"] == [p(50), "unity default resources:10"] diff --git a/tests/unity/test_bindings.py b/tests/unity/test_bindings.py index c10f216..651e02e 100644 --- a/tests/unity/test_bindings.py +++ b/tests/unity/test_bindings.py @@ -7,11 +7,11 @@ PARAM = "Live2D.Cubism|Live2D.Cubism.Core|CubismParameter" PART = "Live2D.Cubism|Live2D.Cubism.Core|CubismPart" -MAT = {"$ref": {"file": "CAB-m", "pathId": 7, "class": "Material"}} +MAT = {"$ref": {"file": "CAB-m", "pathId": "7", "class": "Material"}} def mono(script, **fields): - return {"type": "MonoBehaviour", "pathId": 1, "script": script, "fields": fields} + return {"type": "MonoBehaviour", "pathId": "1", "script": script, "fields": fields} def node(path, *components): @@ -31,7 +31,7 @@ def prefab(): node("model/Drawables"), node( "model/Drawables/ArtMesh0", - {"type": "MeshRenderer", "pathId": 2, "fields": {"m_Enabled": 1, "m_Materials": [MAT, None]}}, + {"type": "MeshRenderer", "pathId": "2", "fields": {"m_Enabled": 1, "m_Materials": [MAT, None]}}, ), node("model/Twin"), node("model/Twin"), @@ -44,7 +44,7 @@ def binding(path, attr, type_id=114, script=None, **kw): b = {"pathCrc": crc(path), "attributeCrc": crc(attr) if isinstance(attr, str) else attr} b.update(typeID=type_id, customType=0, pptr=False, int=False, serializeReference=False) if script is not None: - b.update(script={"$ref": {"file": "CAB-s", "pathId": 3}}, scriptClass=script) + b.update(script={"$ref": {"file": "CAB-s", "pathId": "3"}}, scriptClass=script) b.update(kw) return b @@ -88,7 +88,7 @@ def test_a_transform_bindings_attribute_is_the_clips(): def test_a_renderer_material_binding_names_a_property_of_its_materials(): tint = crc("_TintColor") & 0x0FFFFFFF - docs = {7: {"floats": {"_Cutoff": 0.5}, "colors": {"_TintColor": {"r": 1, "g": 1, "b": 1, "a": 1}}}} + docs = {"7": {"floats": {"_Cutoff": 0.5}, "colors": {"_TintColor": {"r": 1, "g": 1, "b": 1, "a": 1}}}} def material(ref): return docs.get(ref["$ref"]["pathId"]) From 68d5d3b289a2a7bb7f9af767ca1ba2fb542a8a50 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 14:15:49 +0800 Subject: [PATCH 20/91] feat(spot)!: assemble spots and rooms from the exported objects Two product stages make a spot from the unity.export results of the objects its prefabs reach, never from a bundle: - room.gltf (subject a background key Spot//Background/, manifest rooms/.json) writes room.glb and room.json: every mesh the background prefab draws, baked into the prefab's space from the Mesh objects' local glTF, with materials translated from each Material's document and its Shader's parsed form, and textures from the Texture2D images and settings. What draws nothing is listed in room.json: nullMaterialSubmeshes, nullMeshFilters and emptyMeshes (a mesh without vertices or triangles). - spot.files (subject a situation key Spot///, manifest spots/.json) writes spot.json and the Spine files under spine/. Its input `master` is the spot's record of MasterHomeSpot, MasterText and MasterStoryHomeSpotTapTalkEpisode, made at planning. A product stage that reads master tables names them in MASTER; the planning fact "master" is set when one is in the pipeline, and `export assets --layout product` gives such a stage no subjects without master data. content.prefabs reads a prefab document as a hierarchy (parents, world matrices, activity, in-prefab references) for the Live2D, room and spot assemblers. The story host keeps its bundle reader of a spot in content.hostspot. `export spot SPOT_ID -o OUT` and `export room KEY -o OUT` run the pipeline for the spot's two products, or the one room, and write their files; they take the options of `export live2d`. BREAKING CHANGE: `export room` writes OUT/room.glb and OUT/room.json and draws only the meshes of the background prefab's hierarchy; `export spot` writes no shaders/ and spot.json has no `resources`; room.json lists empty meshes as emptyMeshes; the spine.skeleton stage is removed. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 7 +- docs/cli.md | 18 +- docs/commands.md | 40 +-- docs/export.md | 6 +- docs/products.md | 69 +++- docs/stages.md | 3 +- src/nnnotes/cli/assets.py | 17 +- src/nnnotes/cli/export.py | 83 +++-- src/nnnotes/content/hostspot.py | 372 ++++++++++++++++++++ src/nnnotes/content/live2d.py | 50 +-- src/nnnotes/content/prefabs.py | 148 ++++++++ src/nnnotes/content/room.py | 491 +++++++++++++------------- src/nnnotes/content/spot.py | 353 +++++++++---------- src/nnnotes/content/storyhost.py | 51 +-- src/nnnotes/formats/shader.py | 4 +- src/nnnotes/pipeline/assemble.py | 3 +- src/nnnotes/pipeline/modelstages.py | 227 ------------ src/nnnotes/pipeline/spot.py | 106 ++++++ src/nnnotes/pipeline/views.py | 14 + src/nnnotes/unity/reader.py | 20 +- src/nnnotes/unity/transform.py | 27 ++ tests/cli/test_assets.py | 45 +++ tests/cli/test_cli.py | 3 +- tests/content/test_hostspot.py | 509 +++++++++++++++++++++++++++ tests/content/test_room.py | 525 +++++++--------------------- tests/content/test_spot.py | 278 ++++++--------- tests/content/test_story_host.py | 12 +- tests/helpers/spotobjects.py | 357 +++++++++++++++++++ tests/pipeline/test_modelstages.py | 295 ---------------- tests/pipeline/test_spot.py | 103 ++++++ tests/unity/test_speed_exact.py | 2 +- 31 files changed, 2541 insertions(+), 1697 deletions(-) create mode 100644 src/nnnotes/content/hostspot.py create mode 100644 src/nnnotes/content/prefabs.py delete mode 100644 src/nnnotes/pipeline/modelstages.py create mode 100644 src/nnnotes/pipeline/spot.py create mode 100644 src/nnnotes/unity/transform.py create mode 100644 tests/content/test_hostspot.py create mode 100644 tests/helpers/spotobjects.py delete mode 100644 tests/pipeline/test_modelstages.py create mode 100644 tests/pipeline/test_spot.py diff --git a/docs/architecture.md b/docs/architecture.md index 21d43e3..42acf20 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -36,9 +36,10 @@ decrypted bundles in the depot's `downloads/` directory. ## Two ways to export -- **One subject.** `export episode`, `export story`, `export live`, `export live2d`, `export spot`, `export room`, - `export shader`, `export audio`, `export ui`, `site build` and `music-data` read what one subject needs through the - Session and write their output directory directly ([Export stories and lives](export.md)). +- **One subject.** `export episode`, `export story`, `export live`, `export shader`, `export audio`, `export ui`, + `site build` and `music-data` read what one subject needs through the Session and write their output directory + directly ([Export stories and lives](export.md)). `export live2d`, `export spot` and `export room` ask the pipeline + below for the products of one subject ([products](products.md)). - **Every asset.** `export assets` splits the work into stages ([stages](stages.md)). A task is one stage applied to one subject (a bundle, a content id, a view); its key covers exactly what its output depends on: the stage and converter versions, the input contents and the options ([contracts](contracts.md)). Results and their files are diff --git a/docs/cli.md b/docs/cli.md index 158e10c..3baa75e 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -243,21 +243,31 @@ a Live2D model -> its Cubism SDK files, or the player's model ## nnnotes export spot -a spot -> spot.json + Spine + room.glb + shaders +a spot -> spot.json + Spine files + room.glb + room.json | Argument | Description | | --- | --- | | `spot_id` | MasterHomeSpot id | | `-o`, `--out` `OUT` | output directory (required) | +| `--stored-catalog` `LABEL\|SHA` | a catalog version of the depot (`catalog versions`; default: the current catalog) | +| `--workers` `N` | worker processes (default: the usable CPUs) | +| `--link` `LINK` | as export assets --link (one of `auto`, `clone`, `hard`, `copy`) | +| `--explain` | print every setting of the run first | +| `--dry-run` | print the plan, run nothing | ## nnnotes export room -a background prefab -> glb +a spot background -> room.glb + room.json | Argument | Description | | --- | --- | -| `key` | addressable key of the prefab | -| `-o`, `--out` `OUT` | output .glb file (required) | +| `key` | background key Spot//Background/ | +| `-o`, `--out` `OUT` | output directory (required) | +| `--stored-catalog` `LABEL\|SHA` | a catalog version of the depot (`catalog versions`; default: the current catalog) | +| `--workers` `N` | worker processes (default: the usable CPUs) | +| `--link` `LINK` | as export assets --link (one of `auto`, `clone`, `hard`, `copy`) | +| `--explain` | print every setting of the run first | +| `--dry-run` | print the plan, run nothing | ## nnnotes export shader diff --git a/docs/commands.md b/docs/commands.md index db5be94..95f4d0e 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -320,42 +320,26 @@ nnnotes export spot SPOT_ID -o OUT ``` OUT/spot.json master row, names, tap-talk episodes, situation settings, tap targets, Spine characters OUT/spine/ Spine skeletons (.json or .skel), atlases, atlas page PNGs -OUT/room.glb the spot's background as binary glTF (as `export room`), with room.json -OUT/shaders/ shaders of the spot's bundles (as `export shader`) +OUT/room.glb the spot's background as binary glTF (as `export room`) +OUT/room.json its summary ``` -Needs `[paths] apk` and master data (`MasterHomeSpot`, `MasterText`, `MasterStoryHomeSpotTapTalkEpisode`). `room.glb` and -`room.json` are the `export room` export of the spot's background. - -A reference the situation prefab leaves empty is written as null: a Spine character with no `_animation` has null -`skeletonData`, `animation` and `world`, and a tap target with no `_focus` has a null `focusWorld`. +The spot whose `MasterHomeSpot` row has the id `SPOT_ID`: the products `spot.files` of its situation +(`_situationAssetPath`) and `room.gltf` of its background (`_backgroundAssetPath`), their files at their own paths +([products](products.md#spots)). The command runs the depot's pipeline for the two products alone, as `export live2d` +does, writes the same reports in `OUT/_reports/` and takes the same options. Needs `[paths] apk` (the Spot and Spine +component classes are in the APK) and master data (`MasterHomeSpot`, `MasterText`, +`MasterStoryHomeSpotTapTalkEpisode`). ## export room ``` -nnnotes export room KEY -o OUT.glb +nnnotes export room KEY -o OUT ``` -A spot background prefab as binary glTF: every mesh baked into prefab space, converted to glTF's right-handed space, -textures embedded, materials translated from the shader's render state. Objects inactive in the prefab are kept with -`extras.unityActive = false`. A summary (`meshCount`, materials, textures, samplers) is written next to it as -`OUT.json`. - -Materials and textures are told apart by serialized file and path id (the prefab's bundle and its dependencies can -reuse a path id). Each texture is one glTF texture with its own sampler, and textures whose PNG bytes are equal share -one glTF image. The summary's `textures` has one entry per glTF texture, in glTF order (the index a material's -`baseColorTexture` names): the Unity texture's name. - -The render state is that of the shader's first pass for the material: blend factors (colour and alpha), blend -operations, colour mask, culling, depth write, depth test and alpha to mask. A state the shader takes from a -property (`Blend [_SrcBlend] [_DstBlend]`, `Cull [_Cull]`, `ZWrite [_ZWrite]`, ...) is the material's value of that -property, else the shader's default for it. Each glTF material keeps the resolved state in -`extras.unityRenderState` (and `OUT.json` in `materials[].renderState`); a shader or a state glTF cannot express -stops the command. A submesh whose material slot is empty is not written (there is no material to draw it with): the -summary lists each as `nullMaterialSubmeshes` (`mesh`, object `path` in the prefab, `submesh` index), and a mesh -left with no submesh is not written. A MeshFilter with no mesh, or a mesh with no triangles (no index data), draws -nothing and is not written; the summary lists the object path in `nullMeshFilters`, or the mesh and object path in -`meshesWithoutTriangles`. +A spot background as binary glTF, `OUT/room.glb`, with its summary `OUT/room.json`: the product `room.gltf` of the +background key `KEY` (`Spot//Background/`; [products](products.md#spots)). Runs and reports as +`export spot`. ## export shader diff --git a/docs/export.md b/docs/export.md index dbf7d9c..5ddeded 100644 --- a/docs/export.md +++ b/docs/export.md @@ -59,11 +59,11 @@ Details: [export live2d](commands.md#export-live2d). ```sh nnnotes export spot SPOT_ID -o out/spots/SPOT_ID -nnnotes export room KEY -o out/rooms/room.glb # a background prefab key +nnnotes export room KEY -o out/rooms/PLACE # a background key ``` -`export spot` writes the spot's master row with its names and tap-talk episodes, its Spine characters, its -background as binary glTF and the shaders of its bundles. `export room` writes one background prefab as binary glTF +`export spot` writes the spot's master row with its names and tap-talk episodes, its tap targets, its Spine +characters and their files, and its background as binary glTF. `export room` writes one background as binary glTF with a summary next to it. Details: [export spot](commands.md#export-spot), [export room](commands.md#export-room). ## Shaders, audio and the UI library diff --git a/docs/products.md b/docs/products.md index c3be8a6..366e81d 100644 --- a/docs/products.md +++ b/docs/products.md @@ -67,9 +67,10 @@ A product layout places products for a player (`export assets --layout product`, `/.json` with the subject escaped as one path segment). A command asks for products in one of two ways: `export assets --layout product` makes, with each product stage, the -products of the catalog keys the selected bundles hold (the stage's `select`); `export live2d` asks for one model. -`export live2d` without `--player` places the files of its one product at their own paths instead (the `files` -placement: the Cubism SDK files as a directory). +products of the catalog keys the selected bundles hold (the stage's `select`; a stage that reads master data has none +without it); `export live2d`, `export spot` and `export room` ask for the products of one subject. `export live2d` +without `--player`, `export spot` and `export room` place the files of their products at their own paths instead (the +`files` placement: the Cubism SDK files as a directory, a spot's files). ## Product manifest: nnnotes.product/1 @@ -127,3 +128,65 @@ story renderer adds to a character draw), and the mask materials' when a drawabl | `.exp3.json` | each CubismExpressionData of the table | | `motions/.motion3.json` | each CubismFadeMotionData, with the sample rate of the clip of its name | | `.model3.json` | the files, the model's fade times of each motion, and the EyeBlink and LipSync groups (the GameObjects holding a CubismEyeBlinkParameter or a CubismMouthParameter) | + +## Spots + +A spot of the home is a background prefab (a room) and a situation prefab placed on it. Two product stages +(`nnnotes.pipeline.spot`) assemble them; the roots of each are the objects of its subject's catalog key, and neither +keeps objects: its files are all it holds. + +`room.gltf` is a background as binary glTF. Its subject is a background key, `Spot//Background/`, and +its manifest `rooms/.json`. Its files: + +| File | | +|---|---| +| `room.glb` | every drawn mesh of the prefab (a MeshFilter with a mesh beside an enabled MeshRenderer), in hierarchy order | +| `room.json` | the summary: `backgroundKey`, `glb`, `meshCount`, `inactiveMeshes`, `materials`, `textures`, `samplers`, and what draws nothing (below) | + +Each drawn mesh is one glTF node without a transform: its vertices are baked into the prefab's space through the +world matrix of its node (the root's own transform included) and written in glTF's right-handed space (z negated, +v = 1 - v). Each non-empty submesh is one primitive with the renderer's material of the same index. A node inactive in +the prefab is kept with `extras.unityActive = false`. The situation's `backgroundPosition`, `backgroundRotation` and +`backgroundScale` are not baked in: they go on top, as the game applies them to the prefab root. + +Materials are translated from the shader's own render state, the first pass's: blend factors (colour and alpha), blend +operations, colour mask, culling, depth write, depth test and alpha to mask. A state the shader takes from a property +(`Blend [_SrcBlend] [_DstBlend]`, `Cull [_Cull]`, `ZWrite [_ZWrite]`, ...) is the material's value of that property, +else the shader's default for it. Each glTF material keeps the resolved state in `extras.unityRenderState` (and +`room.json` in `materials[].renderState`); a shader or a state glTF cannot express is an error of the task. Each +material and each texture is one object of the table, so materials and textures are told apart by object id. Each +texture a material's main slot names is one glTF texture with its own sampler (from the texture's filter, wrap and +mip count), and textures whose PNG bytes are equal share one glTF image. The `textures` of `room.json` has one entry +per glTF texture, in glTF order: the Unity texture's name. + +What draws nothing is not written and `room.json` lists it: a submesh whose material slot is empty +(`nullMaterialSubmeshes`: `mesh`, object `path` in the prefab, `submesh` index), a MeshFilter without a mesh +(`nullMeshFilters`: the object path) and a mesh without vertices or triangles (`emptyMeshes`: `mesh` and `path`). + +`spot.files` is a situation: its subject is a situation key, `Spot///`, and its manifest +`spots/.json`. Its input `master` is the spot's record of the master tables `MasterHomeSpot`, +`MasterText` and `MasterStoryHomeSpotTapTalkEpisode` (the row whose `_situationAssetPath` is the key, its name, its +tap talks), made at planning from the master data. Its files are `spot.json` and the Spine files under `spine/`: + +| Field of spot.json | | +|---|---| +| `spotId`, `master` | the spot's id and its `MasterHomeSpot` row | +| `name` | its name per language (`japanese`, `english`, `traditionalChinese`, `simplifiedChinese`, `korean`) | +| `tapTalk` | `{character id: [ADV ids]}` of its tap talks | +| `situationSettings` | the fields of the prefab's SpotSituationSettings (camera and interaction parameters) | +| `controller` | the fields of its SpotCharacterController but the character lists | +| `characters` | each SpotCharacter (a tap target) by character id: its object `path`, its fields, its `world` matrix and the world matrix of its `_focus` (`focusWorld`) | +| `spineCharacters` | each SpotSpineCharacter in hierarchy order: `path`, the name of the skeleton its SkeletonAnimation draws (`skeletonData`), that animation's settings (`animation`) and its `world` matrix | +| `skeletons` | each SkeletonDataAsset of the table, by name: `name`, `skeleton` and `atlases` (its files), `scale`, `defaultMix`, `mixes` | +| `matrixConvention` | `Unity world space (left-handed), 4x4 column-major` | + +World matrices are in the space of the situation prefab, its root's own transform included. Fields that name objects +are written as the prefab document writes them. A reference the prefab leaves empty is written as null: a Spine +character with no `_animation` has null `skeletonData`, `animation` and `world`, and a tap target with no `_focus` a +null `focusWorld`. + +| Spine file | From | +|---|---| +| `spine/.json` or `.skel` | the bytes of the SkeletonDataAsset's `skeletonJSON` (JSON when they start with `{`) | +| `spine/.atlas` | the bytes of each SpineAtlasAsset's `atlasFile`, as stored (an atlas two skeletons share is written once) | +| `spine/` | for each page of the atlas, the PNG of the `_MainTex` of the atlas asset's material of the same index | diff --git a/docs/stages.md b/docs/stages.md index 693b619..6e26c6f 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -27,7 +27,8 @@ codec and channel of a USM) and `catalog.index` (version 2: gzip input and Japan | `cri.audio` | ACB content | `acb` (a raw file of the catalog, or the `acb` artifact of a cue sheet held in a bundle), `awb` (an AWB the catalog pairs with it), `boot` (the game's boot data: the key's source) | FLAC per stream, `cues.json`, `streams.json` | | `cri.movie` | USM content | `usm` (a raw file of the catalog, or the `usm` artifact of a movie held in a bundle), `boot` | every stream as stored (video, alpha, audio, subtitles), `movie.mkv` (or `movie.webm`) | | `live2d.model`, `live2d.cubism` | a Live2D model key (`Character/Live2D///model/`) a command asks for | `objects`: the object table of what the model reaches ([products](products.md#live2d-models)); depends on `link.addresses` and the `unity.export` tasks it reads | `model.json` (the player's model) and the object table; the Cubism SDK files | -| `spine.skeleton` | a SkeletonDataAsset of a selected bundle, by its stable address `:` (in a bundle of several serialized files the file's tag follows the bundle name, as in `link.addresses`) | `bundle:`: the bundles holding the serialized files the asset's file reaches through its externals; context `object` (the asset's object id); atom `spot.write_skeleton` (the Spine writer of `nnnotes export spot`) | `spine.skeleton:#`: the skeleton (`.json` or `.skel`), its atlases (`.atlas`) and their page images, the bytes `nnnotes export spot` writes into `spine/`; one item for the asset | +| `room.gltf` | a spot background key (`Spot//Background/`) | `objects`: the object table of what the background reaches ([products](products.md#spots)) | `room.glb` and `room.json` | +| `spot.files` | a spot situation key (`Spot///`) | `objects`; `master`: the spot's master record | `spot.json` and the Spine files | | `link.addresses` | global | catalog index, censuses (the built-in files' too) | key -> objects, Resources.Load path -> objects, serialized file -> bundle or built-in file, bundle -> keys | | `view.` | a catalog (`main`) | `master:

` per master table it reads, `rules` (the view's rule and the resolvers it uses), `addresses` (the part of the address table it read: the keys looked up and the prefixes listed); no parameters, atoms or context | `view.:#view` (facts: rows, entries, gaps, unreferenced) | | `view.voices` | a catalog (`main`) | `master:
` per master table its rules read, `rules`, `addresses`, `sheets` (each cue sheet the `cri.audio` results are known by that a voice names or the rules' prefixes list: its task and ACB content id), `acb:` (the ACB of each decoded one), `episodes` (per story episode and part, the `unity.export` artifact holding the object its key names), `adv:` (those contents); context `audio` (the keys of those `cri.audio` results); runs after `unity.export` and `cri.audio` | `view.voices:#view` (facts: rows, entries, gaps, unreferenced streams) | diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index c570b66..ae71088 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -115,9 +115,10 @@ def matches(self, stage: str) -> bool: StageSource("sprite.crop", "nnnotes.pipeline.objexport:SpriteCropStage", output=True), StageSource("cri.audio", "nnnotes.pipeline.cristages:AudioStage", output=True), StageSource("cri.movie", "nnnotes.pipeline.cristages:MovieStage", output=True), - StageSource("spine.skeleton", "nnnotes.pipeline.modelstages:SpineStage", output=True), StageSource("live2d.model", "nnnotes.pipeline.live2d:ModelStage", output=True, product=True), StageSource("live2d.cubism", "nnnotes.pipeline.live2d:CubismStage", output=True, product=True), + StageSource("room.gltf", "nnnotes.pipeline.spot:RoomStage", output=True, product=True), + StageSource("spot.files", "nnnotes.pipeline.spot:SpotStage", output=True, product=True), StageSource("view.", "nnnotes.pipeline.views:stages", output=True), StageSource("view.voices", "nnnotes.pipeline.voices:VoiceStage", output=True), ) @@ -722,25 +723,31 @@ def facts(self, pipeline: list[str], fetch: bool, background: bool = False) -> d inputs = self._inputs("bundle", [bundles[s] for s in sel.census if s not in builtin], fetch) inputs = inputs.plus(builtin) sel.selected = [s for s in sel.selected if s in inputs] # files that cannot be read: problems() + products = [n for n in pipeline if self.installed.sources[n].product] + readers = [n for n in products if self.installed.stages[n].MASTER] # products that read master tables facts = { "catalogs": self.catalogs_fact(), "index": index, "selected": sel.selected, - "master": self.master() if any(n.startswith("view.") for n in pipeline) else None, + "master": self.master() if readers or any(n.startswith("view.") for n in pipeline) else None, "views": sorted(n[5:] for n in pipeline if n.startswith("view.")), "bundles": inputs, "classes": self.classes(), "raw": Inputs({}), } - products = [n for n in pipeline if self.installed.sources[n].product] if products: from ..pipeline import assemble if self.products is not None: chosen = {n: self.products.get(n, []) for n in products} - else: + if facts["master"] is None and any(chosen[n] for n in readers): + raise self.cfg.missing(*self.cfg.master(self.cfg.get("catalog", "region"))) + else: # without master data, the products that read it have no subjects (as the views) held = held_keys(index, sel.selected) - chosen = {n: self.installed.stages[n].select(held) for n in products} + chosen = { + n: [] if n in readers and facts["master"] is None else self.installed.stages[n].select(held) + for n in products + } facts["products"] = chosen facts["roots"] = assemble.request(self.installed.stages, chosen, Env(self.depot, facts)) if self.products is not None or not set(self.layouts) & set(OBJECT_LAYOUTS): diff --git a/src/nnnotes/cli/export.py b/src/nnnotes/cli/export.py index bc35a77..d66b5e1 100644 --- a/src/nnnotes/cli/export.py +++ b/src/nnnotes/cli/export.py @@ -141,43 +141,35 @@ def cmd_live2d(args, cfg): def cmd_spot(args, cfg): - from ..content import room, spot - from ..formats import shader + from ..pipeline.spot import ROOM, SPOT + from ..source.master import table cfg.require_path("paths", "apk") # the Spot / Spine component classes are read from the APK + use(cfg) # the tools the atoms name are part of the keys md = Session(cfg).master_dir() known_row(args, md, "MasterHomeSpot", args.spot_id, "spot") - cat = Session(cfg).catalog() - out = Path(args.out) - doc = spot.extract(cat, md, args.spot_id, out) - bg = doc["master"]["_backgroundAssetPath"] - rm = room.extract_room(cat, bg, out / "room.glb") - bundles = cat.fetch_key(bg) + cat.fetch_key(doc["master"]["_situationAssetPath"]) - s = shader.dump(bundles, out / "shaders") - r = { - "out": str(out), - "spot": {"tapTargets": len(doc["characters"]), "skeletons": len(doc["skeletons"])}, - "room": {"meshes": rm["meshCount"], "inactiveMeshes": rm["inactiveMeshes"]}, - "shaders": {"shaders": s["count"], "variants": s["variants"]}, - } - report( - args, - r, - lambda r: [ - f"{r['out']}/spot.json ({r['spot']['tapTargets']} tap targets, {r['spot']['skeletons']} skeletons)", - f"{r['out']}/room.glb ({r['room']['meshes']} meshes, {r['room']['inactiveMeshes']} inactive)", - f"{r['out']}/shaders/ ({r['shaders']['shaders']} shaders, {r['shaders']['variants']} variants)", - ], - ) + (row,) = [r for r in table(md, "MasterHomeSpot") if r.get("_id") == args.spot_id] + situation, background = row["_situationAssetPath"], row["_backgroundAssetPath"] + ws = assets.Workspace(args, cfg, layouts=["files"]) + ws.request({SPOT: [situation], ROOM: [background]}, [f"key:{situation}", f"key:{background}"]) + assets.export(args, cfg, ws) def cmd_room(args, cfg): from ..content import room + from ..pipeline.spot import ROOM - cat = Session(cfg).catalog() - known_keys(args, cat, [args.key]) - r = room.extract_room(cat, args.key, Path(args.out)) - report(args, r, lambda r: [f"{r['glb']} ({r['meshCount']} meshes, {r['inactiveMeshes']} inactive)"]) + try: + room.room_name(args.key) + except ValueError as e: + usage(args, str(e)) + use(cfg) + ws = assets.Workspace(args, cfg, layouts=["files"]) + ws.run_index() + if not any(loc["primaryKey"] == args.key for loc in ws.index()["locations"]): + usage(args, f"{args.key}: not a key of the catalog") + ws.request({ROOM: [args.key]}, [f"key:{args.key}"]) + assets.export(args, cfg, ws) def cmd_shader(args, cfg): @@ -240,6 +232,20 @@ def cmd_ui(args, cfg): sys.exit(1) +def product_args(c) -> None: + """The options of a command that exports products through the depot (as export assets).""" + c.add_argument( + "--stored-catalog", + metavar="LABEL|SHA", + help="a catalog version of the depot (`catalog versions`; default: the current catalog)", + ) + c.add_argument("--workers", type=nonnegative, metavar="N", help="worker processes (default: the usable CPUs)") + c.add_argument("--link", choices=("auto", "clone", "hard", "copy"), help="as export assets --link") + c.add_argument("--explain", action="store_true", help="print every setting of the run first") + c.add_argument("--dry-run", action="store_true", help="print the plan, run nothing") + c.set_defaults(views="none", strict=False) + + def register(sub) -> None: es = group(sub, "export", "game data -> common formats: the depot's asset export and single subjects") @@ -307,24 +313,17 @@ def register(sub) -> None: action="store_true", help="write the player's model (models/.json and assets/) instead of the Cubism SDK files", ) - c.add_argument( - "--stored-catalog", - metavar="LABEL|SHA", - help="a catalog version of the depot (`catalog versions`; default: the current catalog)", - ) - c.add_argument("--workers", type=nonnegative, metavar="N", help="worker processes (default: the usable CPUs)") - c.add_argument("--link", choices=("auto", "clone", "hard", "copy"), help="as export assets --link") - c.add_argument("--explain", action="store_true", help="print every setting of the run first") - c.add_argument("--dry-run", action="store_true", help="print the plan, run nothing") - c.set_defaults(views="none", strict=False) + product_args(c) - c = command(es, "spot", cmd_spot, "a spot -> spot.json + Spine + room.glb + shaders") + c = command(es, "spot", cmd_spot, "a spot -> spot.json + Spine files + room.glb + room.json") c.add_argument("spot_id", type=int, help="MasterHomeSpot id") out_arg(c, "output directory") + product_args(c) - c = command(es, "room", cmd_room, "a background prefab -> glb") - c.add_argument("key", help="addressable key of the prefab") - out_arg(c, "output .glb file") + c = command(es, "room", cmd_room, "a spot background -> room.glb + room.json") + c.add_argument("key", help="background key Spot//Background/") + out_arg(c, "output directory") + product_args(c) c = command(es, "shader", cmd_shader, "the shaders of a key's bundle closure or of APK bundles") g = c.add_mutually_exclusive_group(required=True) diff --git a/src/nnnotes/content/hostspot.py b/src/nnnotes/content/hostspot.py new file mode 100644 index 0000000..0f74f88 --- /dev/null +++ b/src/nnnotes/content/hostspot.py @@ -0,0 +1,372 @@ +"""The spot files a story's home host writes from the bundles of a spot (content.storyhost): the background as +binary glTF and the situation's spot.json with its Spine files, read with UnityPy from the bundle closures of the +spot's catalog keys. + +`extract_room` writes .glb and .json as content.room makes room.glb and room.json from a product's objects +(the same material translation, sampler and glTF writer): every MeshFilter of the background's bundle closure, in the +bundles' object order; the empty meshes listed apart (`skippedEmptyMeshes`: no vertex data, +`meshesWithoutTriangles`: vertices only). `extract_spot` writes spot.json and spine/ as content.spot makes them, +the skeletons in the bundles' object order, and `resources`: the bundle names of the background and situation keys. +""" + +from __future__ import annotations + +import io +import json +from pathlib import Path + +import numpy as np +import UnityPy + +from ..core.jsonio import write_json +from ..formats.shader import parsed_summary +from ..source.catalog import Catalog +from ..source.config import apk_missing +from ..unity.convert.material import material_doc +from ..unity.reader import SceneGraph, deref, mesh_arrays, referent, script_class, strip_pptrs, texture_image +from . import room +from .prefabs import mat4 +from .spot import LANGS, atlas_pages + +__all__ = [ + "drawn_meshes", + "extract_room", + "extract_spot", + "main_textures", + "mesh_components", + "object_key", + "submesh_primitives", + "text_bytes", +] + + +def object_key(obj) -> tuple[str, int]: + """(serialized file name, path id) of an object: a path id is unique only within its file.""" + return obj.assets_file.name, obj.path_id + + +def main_textures(owner, mat_tt: dict, shader_name: str) -> list: + """[(texture object, texture env)] of a material's main texture slot (room.main_tex_slot) when it holds a texture; + `owner` is the material's object, whose file the env's PPtr is read from.""" + slot = room.main_tex_slot(shader_name) + return [ + (deref(owner, env["m_Texture"]), env) + for key, env in mat_tt["m_SavedProperties"]["m_TexEnvs"] + if key == slot and env["m_Texture"]["m_PathID"] + ] + + +# ---- extraction --------------------------------------------------------- +def submesh_primitives(tris: list, mats: list, material_for, on_null) -> list: + """[(glTF triangle indices, material index)] of a mesh's submeshes: `tris` (per submesh, Unity winding), + `mats` the renderer's material PPtrs of the same index. An empty submesh gives nothing; a submesh whose material + slot is empty (path id 0) gives nothing and is reported to `on_null(submesh index)`.""" + prims = [] + for i, t in enumerate(tris): + if not len(t): + continue + if not mats[i].path_id: # empty slot: no material to draw the submesh with + on_null(i) + continue + prims.append((t[:, [0, 2, 1]].reshape(-1).astype(np.uint32), material_for(mats[i]))) + return prims + + +def mesh_components(env) -> tuple[dict, dict]: + """({GameObject key: MeshFilter}, {GameObject key: MeshRenderer}) of an environment. The key is (file name, + GameObject path id), as object_key: a component is in its GameObject's file.""" + mf_by_go, mr_by_go = {}, {} + for o in env.objects: + if o.type.name == "MeshFilter": + mf_by_go[(o.assets_file.name, o.read_typetree()["m_GameObject"]["m_PathID"])] = o + elif o.type.name == "MeshRenderer": + mr_by_go[(o.assets_file.name, o.read_typetree()["m_GameObject"]["m_PathID"])] = o + return mf_by_go, mr_by_go + + +def drawn_meshes(mf_by_go: dict, mr_by_go: dict, graph, on_null_mesh): + """(transform path id, Mesh, material PPtrs) of each GameObject with a MeshFilter and an enabled MeshRenderer + (mesh_components maps). A MeshFilter whose mesh reference is empty (path id 0) draws nothing: its object path + goes to `on_null_mesh(path)`.""" + for key, mf in mf_by_go.items(): + tf_pid = graph.tf_of_go.get(key[1]) + mr = mr_by_go.get(key) + if tf_pid is None or mr is None: + continue + if not mr.read_typetree().get("m_Enabled", 1): + continue + mesh_ref = mf.read().m_Mesh + if not mesh_ref.path_id: + on_null_mesh(graph.path(tf_pid)) + continue + yield tf_pid, mesh_ref.read(), mr.read().m_Materials + + +def extract_room(cat: Catalog, bg_key: str, out_path: Path) -> dict: + env = UnityPy.load(*[str(p) for p in cat.fetch_key(bg_key)]) + graph = SceneGraph(env) + mf_by_go, mr_by_go = mesh_components(env) + + glb = room.Glb() + tex_cache: dict[tuple[str, int], int] = {} # texture object_key -> glTF texture + mat_cache: dict[tuple[str, int], int] = {} # material object_key -> glTF material + + def material_for(pptr) -> int: + mat = pptr.deref() + mkey = object_key(mat) + if mkey in mat_cache: + return mat_cache[mkey] + mat_tt = mat.read_typetree() + shader_tt = referent(mat, mat_tt["m_Shader"]).read_typetree() + gm = room.translate_material(material_doc(mat_tt, lambda p: p), parsed_summary(shader_tt)) + for tex_obj, env_ in main_textures(mat, mat_tt, shader_tt["m_ParsedForm"]["m_Name"]): + tid = object_key(tex_obj) + if tid not in tex_cache: + tex_tt = tex_obj.read_typetree() + buf = io.BytesIO() + texture_image(tex_obj.read()).save(buf, format="PNG") + tex_cache[tid] = glb.texture(buf.getvalue(), tex_tt["m_Name"], room.sampler(tex_tt)) + info, transform = room.texture_info(tex_cache[tid], env_["m_Scale"], env_["m_Offset"]) + if transform: + glb.ext_used.add("KHR_texture_transform") + gm["pbrMetallicRoughness"]["baseColorTexture"] = info + mat_cache[mkey] = glb.material(gm) + return mat_cache[mkey] + + n_mesh = 0 + n_inactive = 0 + skipped = [] + null_slots = [] # submeshes whose material slot is empty + null_meshes = [] # objects whose MeshFilter has no mesh + no_triangles = [] # meshes with vertices but no triangles + for tf_pid, mesh, mats in drawn_meshes(mf_by_go, mr_by_go, graph, null_meshes.append): + arrays = mesh_arrays(mesh) + if arrays is None: + skipped.append(mesh.m_Name) # empty mesh (no vertex data) + continue + v, vn, uv0, tris = arrays + if not any(len(t) for t in tris): # vertices only (e.g. a placeholder mesh without index data) + no_triangles.append({"mesh": mesh.m_Name, "path": graph.path(tf_pid)}) + continue + if len(mats) < len(tris): + raise NotImplementedError(f"{mesh.m_Name}: {len(tris)} submeshes, {len(mats)} materials") + m = graph.world(tf_pid) + p = (np.hstack([v, np.ones((len(v), 1))]) @ m.T)[:, :3] + p[:, 2] *= -1.0 # Unity LH -> glTF RH + if len(vn): + nw = vn @ np.linalg.inv(m[:3, :3]).T + nw /= np.maximum(np.linalg.norm(nw, axis=1, keepdims=True), 1e-12) + nw[:, 2] *= -1.0 + else: + nw = np.empty((0, 3)) + uv = uv0.copy() + uv[:, 1] = 1.0 - uv[:, 1] # Unity UV origin bottom-left -> glTF top-left + prims = submesh_primitives( + tris, + mats, + material_for, + lambda i: null_slots.append({"mesh": mesh.m_Name, "path": graph.path(tf_pid), "submesh": i}), + ) + if not prims: + continue + is_active = graph.active(tf_pid) + glb.add_mesh(mesh.m_Name, p, nw, uv, prims, active=is_active) + n_mesh += 1 + n_inactive += 0 if is_active else 1 + + out_path = Path(out_path) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(glb.bytes({"backgroundKey": bg_key})) + doc = { + "backgroundKey": bg_key, + "glb": out_path.name, + "meshCount": n_mesh, + "inactiveMeshes": n_inactive, + "skippedEmptyMeshes": skipped, + "nullMaterialSubmeshes": null_slots, + "nullMeshFilters": null_meshes, + "meshesWithoutTriangles": no_triangles, + **glb.summary(), + } + write_json(out_path.with_suffix(".json"), doc) + return doc + + +def _master(md: Path, name: str) -> list[dict]: + return json.loads((md / f"{name}.json").read_text(encoding="utf-8"))["_allData"] + + +def text_bytes(ta) -> bytes: + s = ta.m_Script + return s.encode("utf-8", "surrogateescape") if isinstance(s, str) else bytes(s) + + +def _target(ref): + """The object a PPtr refers to, None for an empty reference (path id 0).""" + return ref.read() if ref.path_id else None + + +def write_skeleton(o, write, written_atlases: dict[int, str] | None = None) -> dict: + """The Spine files of the SkeletonDataAsset object `o` (module docstring), each given to `write(file name, + bytes)` in this order: the skeleton, then for each of its atlases not in `written_atlases` ({SpineAtlasAsset + path id: atlas file name}, updated here) the atlas text as stored and one PNG per page. Returns the skeleton's + record of spot.json: name, skeleton file, atlas files, scale, default mix, mixes.""" + written = {} if written_atlases is None else written_atlases + sda = o.read_typetree() + sk = o.read().skeletonJSON.read() + raw = text_bytes(sk) + is_json = raw.lstrip()[:1] == b"{" + ext = ".json" if is_json else ".skel" + sk_file = sk.m_Name if sk.m_Name.endswith(ext) else f"{sk.m_Name}{ext}" # as the atlas below + write(sk_file, raw) + atlases = [] + for ap in o.read().atlasAssets: + atlas_asset = ap.read() + if ap.path_id not in written: + af = atlas_asset.atlasFile.read() + araw = text_bytes(af) + atext = araw.decode("utf-8") + a_file = af.m_Name if af.m_Name.endswith(".atlas") else f"{af.m_Name}.atlas" + write(a_file, araw) # the atlas text exactly as the game stores it + pages = atlas_pages(atext) + mats = list(atlas_asset.materials) + if len(mats) != len(pages): + raise RuntimeError(f"{a_file}: {len(pages)} pages vs {len(mats)} materials") + for page, mp in zip(pages, mats): + mat = mp.read() + tex = next(e.m_Texture.read() for k, e in mat.m_SavedProperties.m_TexEnvs if k == "_MainTex") + buf = io.BytesIO() + texture_image(tex).save(buf, format="PNG") + write(page, buf.getvalue()) + written[ap.path_id] = a_file + atlases.append(written[ap.path_id]) + return { + "name": sda["m_Name"], + "skeleton": sk_file, + "atlases": atlases, + "scale": sda["scale"], + "defaultMix": sda["defaultMix"], + "mixes": [ + {"from": f, "to": t, "duration": d} + for f, t, d in zip(sda.get("fromAnimation", []), sda.get("toAnimation", []), sda.get("duration", [])) + ], + } + + +def spine_characters(objs: list, graph, skeletons: dict, world_of) -> list[dict]: + """SpotSpineCharacter components: object path, the SkeletonAnimation they drive (its skeleton data name, + initial animation settings and world matrix). `skeletons`: SkeletonDataAsset path id -> skeleton record. One + whose `_animation` is empty has those three null.""" + out = [] + for o in objs: + ssc = o.read_typetree() + path = graph.path(graph.tf_of_go[ssc["m_GameObject"]["m_PathID"]]) + anim_o = _target(o.read()._animation) + if anim_o is None: + out.append({"path": path, "skeletonData": None, "animation": None, "world": None}) + continue + at = anim_o.object_reader.read_typetree() + out.append( + { + "path": path, + "skeletonData": skeletons[anim_o.skeletonDataAsset.path_id]["name"], + "animation": { + k: at[k] + for k in ( + "_animationName", + "loop", + "timeScale", + "initialSkinName", + "initialFlipX", + "initialFlipY", + "pmaVertexColors", + "tintBlack", + "zSpacing", + ) + }, + "world": world_of(at["m_GameObject"]), + } + ) + return out + + +def tap_targets(objs: list, graph, world_of) -> list[dict]: + """SpotCharacter components (tap targets) by character id: object path, their fields, world matrix and the world + matrix of their `_focus` transform (null when the reference is empty).""" + out = [] + for o in objs: + sc = o.read_typetree() + focus_o = _target(o.read()._focus) + out.append( + { + "path": graph.path(graph.tf_of_go[sc["m_GameObject"]["m_PathID"]]), + **strip_pptrs({k: v for k, v in sc.items() if k != "_focus"}), + "world": world_of(sc["m_GameObject"]), + "focusWorld": None + if focus_o is None + else world_of(focus_o.object_reader.read_typetree()["m_GameObject"]), + } + ) + out.sort(key=lambda c: c["_characterId"]) + return out + + +def extract_spot(cat: Catalog, md: Path, spot_id: int, out_dir: Path) -> dict: + if cat.apk is None: + raise apk_missing("resolving the Spot / Spine component classes") + spots = {r["_id"]: r for r in _master(md, "MasterHomeSpot")} + sp = spots[spot_id] + txt = {r["_id"]: r for r in _master(md, "MasterText")} + taps: dict[int, list[int]] = {} + for r in _master(md, "MasterStoryHomeSpotTapTalkEpisode"): + if r.get("_spotId") == spot_id: + taps.setdefault(r["_characterId"], []).append(r["_advId"]) + + out_dir = Path(out_dir) + spine_dir = out_dir / "spine" + spine_dir.mkdir(parents=True, exist_ok=True) + + env = UnityPy.load(*[str(p) for p in cat.fetch_key(sp["_situationAssetPath"])]) + graph = SceneGraph(env) + by_cls: dict[str | None, list] = {} + for o in env.objects: + if o.type.name == "MonoBehaviour": + by_cls.setdefault(script_class(o), []).append(o) + + def world_of(go_pptr) -> list[float]: + return mat4(graph.world_of_go(go_pptr["m_PathID"])) + + settings = strip_pptrs(by_cls["SpotSituationSettings"][0].read_typetree()) + controller_o = by_cls["SpotCharacterController"][0] + controller = strip_pptrs(controller_o.read_typetree()) + + # Spine skeleton data -> files + def write(name: str, data: bytes) -> None: + (spine_dir / name).write_bytes(data) + + skeleton_files: dict[int, dict] = {} + written_atlases: dict[int, str] = {} + for o in by_cls.get("SkeletonDataAsset", []): + skeleton_files[o.path_id] = write_skeleton(o, write, written_atlases) + + spine_chars = spine_characters(by_cls.get("SpotSpineCharacter", []), graph, skeleton_files, world_of) + characters = tap_targets(by_cls.get("SpotCharacter", []), graph, world_of) + + doc = { + "spotId": spot_id, + "master": sp, + "name": {lang[1:]: txt.get(sp.get("_nameTextId"), {}).get(lang, "") for lang in LANGS}, + "tapTalk": {str(k): v for k, v in sorted(taps.items())}, + "situationSettings": settings, + "controller": {k: v for k, v in controller.items() if k not in ("_spotSpineCharacters", "_characters")}, + "characters": characters, + "spineCharacters": spine_chars, + "skeletons": list(skeleton_files.values()), + "resources": { + "background": [b.name for b in cat.resolve(sp["_backgroundAssetPath"])], + "situation": [b.name for b in cat.resolve(sp["_situationAssetPath"])], + }, + "matrixConvention": "Unity world space (left-handed), 4x4 column-major", + } + write_json(out_dir / "spot.json", doc) + return doc diff --git a/src/nnnotes/content/live2d.py b/src/nnnotes/content/live2d.py index 5897b3e..471b03c 100644 --- a/src/nnnotes/content/live2d.py +++ b/src/nnnotes/content/live2d.py @@ -37,14 +37,13 @@ import re from collections.abc import Iterator -from typing import Any, Protocol from ..core.jsonio import dumps from ..formats import motion from ..formats.cubism import expression_json, moc3_canvas, physics_json from ..formats.shader import player_program from ..unity import bindings -from ..unity.ids import object_id +from .prefabs import Objects, component_class, ref_id, root_game_object, script_class __all__ = [ "MASK_KEYWORD", @@ -83,22 +82,6 @@ _UNSAFE = re.compile(r"[^A-Za-z0-9_.-]+") -class Objects(Protocol): - """A product's object table and its objects' files (pipeline.assemble.Objects).""" - - roots: list[str] - - def __contains__(self, oid: str) -> bool: ... - - def ids(self, cls: str | None = None) -> list[str]: ... - - def info(self, oid: str) -> dict: ... - - def bytes(self, oid: str, role: str) -> bytes: ... - - def json(self, oid: str, role: str = "json") -> Any: ... - - def model_match(key: str) -> re.Match[str]: """The parts of a model key: group, name (ValueError for a key that is not a model's).""" m = MODEL_KEY.fullmatch(key) @@ -148,28 +131,12 @@ def select_models(specs, models: dict[str, str]) -> dict[str, str]: return dict(sorted(out.items())) -def ref_id(v) -> str | None: - """The object id of a reference value ({"$ref": {file, pathId}}); None for anything else.""" - r = v.get("$ref") if isinstance(v, dict) else None - return object_id(r["file"], r["pathId"]) if isinstance(r, dict) and "file" in r else None - - -def _class(c: dict) -> str: - """A component's class: a MonoBehaviour's is its script's.""" - if c["type"] == "MonoBehaviour" and isinstance(c.get("script"), str): - return c["script"].rsplit("|", 1)[-1] - return c["type"] - - class _Model: """A model's GameObject, prefab document and components.""" def __init__(self, objects: Objects, key: str): self.objects, self.key, self.name = objects, key, model_id(key) - gos = [o for o in objects.roots if o in objects and objects.info(o).get("class") == "GameObject"] - if len(gos) != 1: - raise ValueError(f"{key}: {len(gos)} GameObjects among the roots, expected the model's") - self.id = gos[0] + self.id = root_game_object(objects, key) self.prefab = objects.json(self.id) self.root = self.prefab["nodes"][0] @@ -177,7 +144,7 @@ def components(self, cls: str, root_only: bool = False) -> Iterator[tuple[dict, """(node, component) of every component of class `cls` (of the root node only).""" for n in [self.root] if root_only else self.prefab["nodes"]: for c in n["components"]: - if _class(c) == cls: + if component_class(c) == cls: yield n, c def one(self, cls: str) -> dict: @@ -213,14 +180,7 @@ def drawable_materials(self) -> list[str]: def scripted(self, cls: str) -> list[str]: """The MonoBehaviours of the table whose script class is `cls` (outside the prefab: data assets).""" - return [o for o in self.objects.ids("MonoBehaviour") if _script_class(self.objects, o) == cls] - - -def _script_class(objects: Objects, oid: str) -> str | None: - if "json" not in (objects.info(oid).get("files") or {}): - return None - entry = objects.json(oid).get("$script") - return entry.rsplit("|", 1)[-1] if isinstance(entry, str) else None + return [o for o in self.objects.ids("MonoBehaviour") if script_class(self.objects, o) == cls] def _resources(m: _Model) -> dict[str, str]: @@ -274,7 +234,7 @@ def model(objects: Objects, key: str) -> tuple[dict, set[str], dict[str, set[str except ValueError as e: raise ValueError(f"{key}: shader {index['name']}: {e}") from None roles[sid] = keep - classes = {_class(c) for c in m.root["components"]} + classes = {component_class(c) for c in m.root["components"]} def material(ref) -> dict | None: oid = ref_id(ref) diff --git a/src/nnnotes/content/prefabs.py b/src/nnnotes/content/prefabs.py new file mode 100644 index 0000000..3bd73da --- /dev/null +++ b/src/nnnotes/content/prefabs.py @@ -0,0 +1,148 @@ +"""Prefab documents of a product's object table read as the game's hierarchy. + +A root GameObject's `json` is its prefab document (prefab.json): `{name, nodes}`, the nodes in hierarchy order (depth +first), each with its path, local transform, `active` (GameObject.m_IsActive) and components `{type, pathId, script, +fields}`. A component's fields name objects of the table by reference (`{"$ref": {file, pathId}}`) and objects of the +same hierarchy by path: `{"gameObject": path}`, `{"transform": path}`, or `{"component": type, "gameObject": path, +"class": script class}` for a component. `Prefab` gives each node's parent, its world matrix (Unity's left-handed +space, the root's own transform included) and whether it is active in the hierarchy, and resolves those references. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import TYPE_CHECKING, Any, Protocol + +from ..unity.ids import object_id + +if TYPE_CHECKING: + import numpy as np + +__all__ = ["Objects", "Prefab", "component_class", "mat4", "ref_id", "root_game_object", "script_class"] + + +class Objects(Protocol): + """A product's object table and its objects' files (pipeline.assemble.Objects).""" + + roots: list[str] + + def __contains__(self, oid: str) -> bool: ... + + def ids(self, cls: str | None = None) -> list[str]: ... + + def info(self, oid: str) -> dict: ... + + def bytes(self, oid: str, role: str) -> bytes: ... + + def json(self, oid: str, role: str = "json") -> Any: ... + + +def ref_id(v) -> str | None: + """The object id of a reference value ({"$ref": {file, pathId}}); None for anything else.""" + r = v.get("$ref") if isinstance(v, dict) else None + return object_id(r["file"], r["pathId"]) if isinstance(r, dict) and "file" in r else None + + +def component_class(c: dict) -> str: + """A component's class: a MonoBehaviour's is its script's (the last part of its script entry).""" + if c["type"] == "MonoBehaviour" and isinstance(c.get("script"), str): + return c["script"].rsplit("|", 1)[-1] + return c["type"] + + +def script_class(objects: Objects, oid: str) -> str | None: + """The script class of a MonoBehaviour outside a hierarchy (its `json`'s `$script`), None without one.""" + if "json" not in (objects.info(oid).get("files") or {}): + return None + entry = objects.json(oid).get("$script") + return entry.rsplit("|", 1)[-1] if isinstance(entry, str) else None + + +def root_game_object(objects: Objects, what: str) -> str: + """The one GameObject among the table's roots (ValueError naming `what` otherwise).""" + gos = [o for o in objects.roots if o in objects and objects.info(o).get("class") == "GameObject"] + if len(gos) != 1: + raise ValueError(f"{what}: {len(gos)} GameObjects among the roots, expected one") + return gos[0] + + +def mat4(m: np.ndarray) -> list[float]: + """A 4x4 matrix as 16 floats, column-major.""" + return [float(x) for x in m.T.reshape(-1)] + + +class Prefab: + """The nodes of a prefab document with their parents, world matrices and activity (module documentation).""" + + def __init__(self, doc: dict): + self.name: str = doc["name"] + self.nodes: list[dict] = doc["nodes"] + self.parent: list[int | None] = [] + stack: list[int] = [] + for n in self.nodes: + while stack and not n["path"].startswith(self.nodes[stack[-1]]["path"] + "/"): + stack.pop() + self.parent.append(stack[-1] if stack else None) + stack.append(len(self.parent) - 1) + self._by_path: dict[str, list[int]] = {} + for i, n in enumerate(self.nodes): + self._by_path.setdefault(n["path"], []).append(i) + self._world: dict[int, np.ndarray] = {} # numpy is imported when a world matrix is asked for + + def at(self, path: str) -> int: + """The node at `path` (ValueError when no node, or several, has it).""" + hits = self._by_path.get(path, []) + if len(hits) != 1: + raise ValueError(f"{self.name}: {len(hits)} nodes at {path!r}") + return hits[0] + + def world(self, i: int) -> np.ndarray: + """The world matrix of node `i`: its local transform under its parents'.""" + from ..unity.transform import trs + + if i not in self._world: + n = self.nodes[i] + m = trs(n["localPosition"], n["localRotation"], n["localScale"]) + p = self.parent[i] + self._world[i] = m if p is None else self.world(p) @ m + return self._world[i] + + def active(self, i: int | None) -> bool: + """Node `i` and every parent of it are active.""" + while i is not None: + if not self.nodes[i].get("active", True): + return False + i = self.parent[i] + return True + + def components(self, cls: str) -> Iterator[tuple[int, dict]]: + """(node, component) of every component of class `cls`, in hierarchy order.""" + for i, n in enumerate(self.nodes): + for c in n["components"]: + if component_class(c) == cls: + yield i, c + + def node_of(self, v) -> int | None: + """The node a reference to an object of the hierarchy names (its GameObject, Transform or a component of + it); None for null.""" + if v is None: + return None + if not isinstance(v, dict) or not ({"gameObject", "transform"} & v.keys()): + raise ValueError(f"{self.name}: not a reference to an object of the hierarchy: {v!r}") + return self.at(v["gameObject"] if "gameObject" in v else v["transform"]) + + def component_of(self, v) -> tuple[int, dict] | None: + """(node, component) a reference to a component of the hierarchy names; None for null.""" + i = self.node_of(v) + if i is None: + return None + if "component" not in v: + raise ValueError(f"{self.name}: not a reference to a component: {v!r}") + hits = [ + c + for c in self.nodes[i]["components"] + if c["type"] == v["component"] and ("class" not in v or component_class(c) == v["class"]) + ] + if len(hits) != 1: + raise ValueError(f"{self.name}: {len(hits)} components {v!r}") + return i, hits[0] diff --git a/src/nnnotes/content/room.py b/src/nnnotes/content/room.py index e8ba1ac..f3ce16e 100644 --- a/src/nnnotes/content/room.py +++ b/src/nnnotes/content/room.py @@ -1,49 +1,45 @@ -"""Spot background prefab -> binary glTF (.glb). - -A spot background is a few hundred textured cards (MeshFilter + MeshRenderer -under a Transform hierarchy). Each mesh is baked into prefab space by walking -the Transform TRS chain, converted from Unity's left-handed space to glTF's -right-handed space (negate Z, reverse winding), and written with its texture(s) -embedded. Geometry is read from the mesh's vertex/index buffers; each submesh -becomes one primitive with the renderer's material of the same index. - -Objects are told apart by (serialized file name, path id) (object_key): a path -id is unique only within its file, and a bundle's closure loads several files. -Each Unity texture is one glTF texture with its own sampler; textures whose PNG -bytes are equal share one glTF image. - -Materials are translated from the Unity material's shader and that shader's own -pass render state (culling, blend factors and operations, depth write and test, -colour mask, alpha to mask, queue tags) -- not from names. A render state value -that names a property (URP/Lit's `Blend [_SrcBlend] [_DstBlend]`, `Cull [_Cull]`, -`ZWrite [_ZWrite]`) takes the material's value of that property, else the -shader's default for it, else the value the shader stores (render_state). Each -glTF material records the resolved state in `extras.unityRenderState`. -Unsupported shaders and states raise instead of being approximated. A submesh -whose material slot is empty (null) is not written: nothing says what it would -be drawn with; room.json lists it (`nullMaterialSubmeshes`). A MeshFilter with -no mesh, or a mesh with no triangles, draws nothing; room.json lists the object -(`nullMeshFilters`, `meshesWithoutTriangles`). - -The situation's backgroundPosition/Rotation/Scale (spot.json) is not baked in: it -goes on top of this, as the game applies it to the prefab root. Objects that -are inactive in the prefab are kept, flagged `extras.unityActive = false`. +"""A spot background (a room) assembled from the objects its prefab reaches: binary glTF (.glb) and its summary. + +A spot background is a few hundred textured cards: MeshFilter and MeshRenderer components under a Transform +hierarchy. `room` reads them from the prefab document of the background's GameObject (content.prefabs) and the +objects its components name: each Mesh's `mesh` (binary glTF in the mesh's local space, unity.atoms.gltf), each +Material's `json` and its Shader's parsed form, each Texture2D's `json` and `image`. It never reads a bundle. + +Each drawn mesh (a MeshFilter with a mesh beside an enabled MeshRenderer) becomes one node without a transform: its +vertices are baked into the prefab's space through the world matrix of its node (the root's own transform included), +in glTF's right-handed space (z negated), with Unity's texture coordinates flipped (v = 1 - v), as the mesh's glTF +holds them. Each non-empty submesh becomes one primitive with the renderer's material of the same index. A node +inactive in the prefab is kept, flagged `extras.unityActive = false`. + +Materials are translated from the Unity material's shader and that shader's own pass render state (culling, blend +factors and operations, depth write and test, colour mask, alpha to mask, queue tags), not from names. A render +state value that names a property (URP/Lit's `Blend [_SrcBlend] [_DstBlend]`, `Cull [_Cull]`, `ZWrite [_ZWrite]`) +takes the material's value of that property, else the shader's default for it, else the value the shader stores +(render_state). Each glTF material records the resolved state in `extras.unityRenderState`. Unsupported shaders and +states raise instead of being approximated. Each Texture2D a material's main slot names is one glTF texture with its +own sampler; textures whose PNG bytes are equal share one glTF image. + +What draws nothing is not written and room.json lists it: a submesh whose material slot is empty +(`nullMaterialSubmeshes`), a MeshFilter without a mesh (`nullMeshFilters`), a mesh without vertices or triangles +(`emptyMeshes`). + +The situation's backgroundPosition, backgroundRotation and backgroundScale (spot.json) are not baked in: they go on +top of this, as the game applies them to the prefab root. """ from __future__ import annotations import hashlib -import io +import re import struct -from pathlib import Path -import numpy as np -import UnityPy +from ..core.jsonio import dumps +from .prefabs import Objects, Prefab, component_class, ref_id, root_game_object -from ..core.jsonio import dumps, write_json -from ..source.catalog import Catalog -from ..unity.reader import SceneGraph, deref, mesh_arrays, referent, texture_image +__all__ = ["BACKGROUND_KEY", "room", "room_name", "render_state", "translate_material"] +# a spot background's catalog key: Spot//Background/ +BACKGROUND_KEY = r"Spot/(?P[^/]+)/Background/(?P=place)" # UnityEngine.Rendering.BlendMode BLEND_ONE, BLEND_ZERO, BLEND_SRC_ALPHA, BLEND_ONE_MINUS_SRC_ALPHA = 1, 0, 5, 10 BLEND_OP_ADD = 0 # UnityEngine.Rendering.BlendOp.Add @@ -55,7 +51,7 @@ # UnityEngine.TextureWrapMode WRAP = {0: REPEAT, 1: CLAMP, 2: MIRROR, 3: MIRROR} UNLIT_SHADERS = ("Unlit/", "Universal Render Pipeline/Unlit") -# render state of a pass (SerializedShaderState) read by room: record key -> path in m_State (render target 0) +# render state of a pass (SerializedShaderState) read by room: record key -> path in the pass state (render target 0) STATE_FIELDS = { "srcBlend": ("rtBlend0", "srcBlend"), "dstBlend": ("rtBlend0", "destBlend"), @@ -70,19 +66,28 @@ "alphaToMask": ("alphaToMask",), } NO_PROPERTY = "" # SerializedShaderFloatValue.name of a fixed value +TRIANGLES = 4 # glTF primitive mode + + +def room_name(key: str) -> str: + """The of a background key (ValueError for another key).""" + m = re.fullmatch(BACKGROUND_KEY, key) + if m is None: + raise ValueError(f"{key}: not a spot background key (Spot//Background/)") + return m["place"] # ---- material translation ------------------------------------------------- -def shader_defaults(shader_tt: dict) -> dict: - """{property name: default value} of a shader's properties (m_ParsedForm.m_PropInfo, first component).""" - props = shader_tt["m_ParsedForm"].get("m_PropInfo", {}).get("m_Props", []) - return {p["m_Name"]: p["m_DefValue[0]"] for p in props if "m_DefValue[0]" in p} +# A material is its `json` (material.json: material, shader, keywords, floats, colors, ints, textures), a shader its +# parsed form (name, properties, subShaders[].tags, subShaders[].passes[].tags and state). +def shader_defaults(shader: dict) -> dict: + """{property name: default value} of a shader's properties (the first component).""" + return {p["m_Name"]: p["m_DefValue[0]"] for p in shader.get("properties") or () if "m_DefValue[0]" in p} -def material_values(mat_tt: dict) -> dict: +def material_values(material: dict) -> dict: """{property name: value} of a material's int and float properties.""" - sp = mat_tt["m_SavedProperties"] - return {**dict(sp.get("m_Ints", [])), **dict(sp.get("m_Floats", []))} + return {**(material.get("ints") or {}), **(material.get("floats") or {})} def state_value(x, values: dict, defaults: dict): @@ -98,10 +103,19 @@ def state_value(x, values: dict, defaults: dict): return int(v) if isinstance(v, float) and v.is_integer() else v -def render_state(mat_tt: dict, shader_tt: dict) -> dict: +def _pass_state(shader: dict) -> tuple[dict, dict]: + """(render state of the first pass, merged subshader and pass tags).""" + ss = shader["subShaders"][0] + ps = ss["passes"][0] + tags = dict((ss.get("tags") or {}).get("tags", [])) + tags.update(dict((ps.get("tags") or {}).get("tags", []))) + return ps["state"], tags + + +def render_state(material: dict, shader: dict) -> dict: """The render state (STATE_FIELDS) of the shader's first pass for this material.""" - state, _ = _pass_state(shader_tt) - values, defaults = material_values(mat_tt), shader_defaults(shader_tt) + state, _ = _pass_state(shader) + values, defaults = material_values(material), shader_defaults(shader) out = {} for key, path in STATE_FIELDS.items(): x = state @@ -111,29 +125,20 @@ def render_state(mat_tt: dict, shader_tt: dict) -> dict: return out -def _pass_state(shader_tt: dict) -> tuple[dict, dict]: - """(render state of the first pass, merged subshader+pass tags).""" - ss = shader_tt["m_ParsedForm"]["m_SubShaders"][0] - ps = ss["m_Passes"][0] - tags = dict(ss.get("m_Tags", {}).get("tags", [])) - tags.update(dict(ps.get("m_Tags", {}).get("tags", []))) - return ps["m_State"], tags - - -def translate_material(mat_tt: dict, shader_tt: dict) -> dict: - """Unity material + shader -> glTF material fields (texture filled by caller).""" - shader_name = shader_tt["m_ParsedForm"]["m_Name"] - _, tags = _pass_state(shader_tt) - rs = render_state(mat_tt, shader_tt) +def translate_material(material: dict, shader: dict) -> dict: + """A Unity material and its shader -> glTF material fields (the texture is the caller's).""" + shader_name = shader["name"] + _, tags = _pass_state(shader) + rs = render_state(material, shader) src, dst, cull = rs["srcBlend"], rs["dstBlend"], rs["cull"] - floats = dict(mat_tt["m_SavedProperties"]["m_Floats"]) - colors = dict(mat_tt["m_SavedProperties"]["m_Colors"]) + floats = material.get("floats") or {} + colors = material.get("colors") or {} unlit = shader_name.startswith(UNLIT_SHADERS) if not unlit and shader_name != "Universal Render Pipeline/Lit": raise NotImplementedError(f"shader not translated: {shader_name}") - out: dict = {"name": mat_tt["m_Name"], "extras": {"unityShader": shader_name, "unityRenderState": rs}} + out: dict = {"name": material["material"], "extras": {"unityShader": shader_name, "unityRenderState": rs}} if (src, dst) == (BLEND_ONE, BLEND_ZERO): cutout = tags.get("RenderType") == "TransparentCutout" or tags.get("QUEUE") == "AlphaTest" if cutout: @@ -164,29 +169,27 @@ def translate_material(mat_tt: dict, shader_tt: dict) -> dict: return out -def _main_tex_slot(shader_name: str) -> str: +def main_tex_slot(shader_name: str) -> str: + """The texture property a shader samples as its main texture.""" return "_BaseMap" if shader_name.startswith("Universal Render Pipeline/") else "_MainTex" -def object_key(obj) -> tuple[str, int]: - """(serialized file name, path id) of an object: a path id is unique only within its file.""" - return obj.assets_file.name, obj.path_id - +def texture_info(index: int, scale: dict, offset: dict) -> tuple[dict, bool]: + """(glTF texture info of texture `index`, whether it uses KHR_texture_transform) for a texture slot's scale and + offset.""" + info: dict = {"index": index} + sx, sy, ox, oy = scale["x"], scale["y"], offset["x"], offset["y"] + if (sx, sy, ox, oy) == (1.0, 1.0, 0.0, 0.0): + return info, False + # Unity samples (u*sx+ox, v*sy+oy) with a bottom-left origin; in glTF's top-left texture space that is this + info["extensions"] = {"KHR_texture_transform": {"scale": [sx, sy], "offset": [ox, 1.0 - sy - oy]}} + return info, True -def main_textures(owner, mat_tt: dict, shader_name: str) -> list: - """[(texture object, texture env)] of a material's main texture slot (_main_tex_slot) when it holds a texture; - `owner` is the material's object, whose file the env's PPtr is read from.""" - slot = _main_tex_slot(shader_name) - return [ - (deref(owner, env["m_Texture"]), env) - for key, env in mat_tt["m_SavedProperties"]["m_TexEnvs"] - if key == slot and env["m_Texture"]["m_PathID"] - ] - -def _sampler(tex_tt: dict) -> dict: - s = tex_tt.get("m_TextureSettings", {}) - mips = tex_tt.get("m_MipCount", 1) > 1 +def sampler(texture: dict) -> dict: + """The glTF sampler of a Texture2D's settings and mip count (its typetree fields).""" + s = texture.get("m_TextureSettings", {}) + mips = texture.get("m_MipCount", 1) > 1 fm = s.get("m_FilterMode", 1) mag = NEAREST if fm == 0 else LINEAR if fm == 0: @@ -202,7 +205,9 @@ def _sampler(tex_tt: dict) -> dict: # ---- glTF builder ------------------------------------------------------- -class _Glb: +class Glb: + """A binary glTF of baked meshes: one node per mesh, one primitive per submesh.""" + def __init__(self): self.bin = bytearray() self.bufferViews, self.accessors, self.meshes, self.nodes = [], [], [], [] @@ -220,7 +225,9 @@ def _view(self, data: bytes, target=None) -> int: self.bufferViews.append(bv) return len(self.bufferViews) - 1 - def accessor_f32(self, arr: np.ndarray, kind: str, minmax=False) -> int: + def accessor_f32(self, arr, kind: str, minmax=False) -> int: + import numpy as np + a = np.ascontiguousarray(arr, dtype=" int: self.accessors.append(acc) return len(self.accessors) - 1 - def accessor_idx(self, idx: np.ndarray) -> int: + def accessor_idx(self, idx) -> int: + import numpy as np + a = np.ascontiguousarray(idx, dtype=" None: attrs["NORMAL"] = self.accessor_f32(nrm, "VEC3") primitives = [] for idx, material in prims: - p = {"attributes": attrs, "indices": self.accessor_idx(idx), "mode": 4} + p = {"attributes": attrs, "indices": self.accessor_idx(idx), "mode": TRIANGLES} if material is not None: p["material"] = material primitives.append(p) self.meshes.append({"name": name, "primitives": primitives}) self.nodes.append({"name": name, "mesh": len(self.meshes) - 1, "extras": {"unityActive": active}}) - def write(self, path: Path, extras: dict): + def bytes(self, extras: dict) -> bytes: """glTF requires the arrays it has to be non-empty and a buffer to hold bytes: an empty array is left out, and a room without geometry is a scene without nodes and a file without the BIN chunk.""" scene = {"nodes": list(range(len(self.nodes)))} if self.nodes else {} @@ -300,166 +309,174 @@ def write(self, path: Path, extras: dict): gltf["extensionsUsed"] = sorted(self.ext_used) js = dumps(gltf, separators=(",", ":")).encode("utf-8") js += b" " * (-len(js) % 4) - self.bin.extend(b"\0" * (-len(self.bin) % 4)) - size = 12 + 8 + len(js) + (8 + len(self.bin) if self.bin else 0) - with open(path, "wb") as f: - f.write(struct.pack(" list: - """[(glTF triangle indices, material index)] of a mesh's submeshes: `tris` (per submesh, Unity winding), - `mats` the renderer's material PPtrs of the same index. An empty submesh gives nothing; a submesh whose material - slot is empty (path id 0) gives nothing and is reported to `on_null(submesh index)`.""" - prims = [] - for i, t in enumerate(tris): - if not len(t): - continue - if not mats[i].path_id: # empty slot: no material to draw the submesh with - on_null(i) - continue - prims.append((t[:, [0, 2, 1]].reshape(-1).astype(np.uint32), material_for(mats[i]))) - return prims - - -def mesh_components(env) -> tuple[dict, dict]: - """({GameObject key: MeshFilter}, {GameObject key: MeshRenderer}) of an environment. The key is (file name, - GameObject path id), as object_key: a component is in its GameObject's file.""" - mf_by_go, mr_by_go = {}, {} - for o in env.objects: - if o.type.name == "MeshFilter": - mf_by_go[(o.assets_file.name, o.read_typetree()["m_GameObject"]["m_PathID"])] = o - elif o.type.name == "MeshRenderer": - mr_by_go[(o.assets_file.name, o.read_typetree()["m_GameObject"]["m_PathID"])] = o - return mf_by_go, mr_by_go - - -def drawn_meshes(mf_by_go: dict, mr_by_go: dict, graph, on_null_mesh): - """(transform path id, Mesh, material PPtrs) of each GameObject with a MeshFilter and an enabled MeshRenderer - (mesh_components maps). A MeshFilter whose mesh reference is empty (path id 0) draws nothing: its object path - goes to `on_null_mesh(path)`.""" - for key, mf in mf_by_go.items(): - tf_pid = graph.tf_of_go.get(key[1]) - mr = mr_by_go.get(key) - if tf_pid is None or mr is None: + body = bytes(self.bin) + b"\0" * (-len(self.bin) % 4) + size = 12 + 8 + len(js) + (8 + len(body) if body else 0) + out = struct.pack(" dict: + """The materials, textures and samplers as room.json lists them.""" + return { + "materials": [ + { + "name": m["name"], + "shader": m["extras"]["unityShader"], + "alphaMode": m.get("alphaMode", "OPAQUE"), + "alphaCutoff": m.get("alphaCutoff"), + "doubleSided": m.get("doubleSided", False), + "unlit": "KHR_materials_unlit" in m.get("extensions", {}), + "renderState": m["extras"]["unityRenderState"], + } + for m in self.materials + ], + "textures": [t["name"] for t in self.textures], + "samplers": self.samplers, + } + + +# ---- assembly ------------------------------------------------------------- +def _floats(doc: dict, body: bytes, index: int): + """The values of a glTF accessor as float32, a normalized integer accessor dequantized.""" + import numpy as np + + from ..unity.atoms.gltf import accessor_array + + a = accessor_array(doc, body, index) + if doc["accessors"][index].get("normalized"): + return (a.astype(np.float64) / np.iinfo(a.dtype).max).astype(np.float32) + return a.astype(np.float32) + + +class _Mesh: + """A Mesh object's local-space glTF: positions and normals back in Unity's space (z as the game stores it), + texture coordinates as the glTF holds them (v = 1 - v), and per submesh its triangle indices (glTF winding) or + None for an empty submesh; no submeshes for a mesh without an index buffer (one points primitive).""" + + def __init__(self, objects: Objects, oid: str): + from ..unity.atoms.gltf import accessor_array, read + + doc, body = read(objects.bytes(oid, "mesh")) + (mesh,) = doc["meshes"] + prims = mesh["primitives"] + attrs = prims[0]["attributes"] + self.name = mesh.get("name") or objects.info(oid).get("name") or oid + self.pos = _floats(doc, body, attrs["POSITION"])[:, :3].copy() + self.pos[:, 2] *= -1.0 + self.nrm = _floats(doc, body, attrs["NORMAL"])[:, :3].copy() if "NORMAL" in attrs else None + if self.nrm is not None: + self.nrm[:, 2] *= -1.0 + self.uv = _floats(doc, body, attrs["TEXCOORD_0"])[:, :2] if "TEXCOORD_0" in attrs else None + unity = (mesh.get("extras") or {}).get("unity") or {} + empty = set(unity.get("emptySubmeshes") or ()) + count = len(unity.get("submeshes") or ()) + self.submeshes: list = [] + if any("indices" not in p for p in prims): + return + it = iter(prims) + for i in range(count): + if i in empty: + self.submeshes.append(None) + continue + p = next(it) + if p.get("mode", TRIANGLES) != TRIANGLES or "indices" not in p: + raise NotImplementedError(f"mesh {self.name!r}: submesh {i} is not a triangle list") + self.submeshes.append(accessor_array(doc, body, p["indices"]).reshape(-1)) + if next(it, None) is not None: + raise ValueError(f"mesh {self.name!r}: more primitives than non-empty submeshes") + + +def room(objects: Objects, key: str) -> tuple[bytes, dict]: + """(room.glb, room.json) of the background `key` (module documentation).""" + import numpy as np + + prefab = Prefab(objects.json(root_game_object(objects, key))) + glb = Glb() + textures: dict[str, int] = {} # Texture2D object id -> glTF texture + materials: dict[str, int] = {} # Material object id -> glTF material + + def material_for(oid: str) -> int: + if oid in materials: + return materials[oid] + mat = objects.json(oid) + sid = ref_id(mat["shader"]) + if sid is None or sid not in objects: + raise ValueError(f"{key}: material {mat['material']!r} has no shader in the object table") + shader = objects.json(sid) + gm = translate_material(mat, shader) + slot = (mat.get("textures") or {}).get(main_tex_slot(shader["name"])) + tid = ref_id(slot["texture"]) if slot else None + if slot and tid is not None: + if tid not in textures: + tex = objects.json(tid) + textures[tid] = glb.texture(objects.bytes(tid, "image"), tex["m_Name"], sampler(tex)) + info, transform = texture_info(textures[tid], slot["scale"], slot["offset"]) + if transform: + glb.ext_used.add("KHR_texture_transform") + gm["pbrMetallicRoughness"]["baseColorTexture"] = info + materials[oid] = glb.material(gm) + return materials[oid] + + counts = {"meshes": 0, "inactive": 0} + null_slots, null_meshes, empty = [], [], [] + for i, node in enumerate(prefab.nodes): + comps = {component_class(c): c["fields"] for c in node["components"]} + mf, mr = comps.get("MeshFilter"), comps.get("MeshRenderer") + if mf is None or mr is None or not mr.get("m_Enabled", 1): continue - if not mr.read_typetree().get("m_Enabled", 1): + path = node["path"] + mesh_id = ref_id(mf.get("m_Mesh")) + if mesh_id is None: + null_meshes.append(path) continue - mesh_ref = mf.read().m_Mesh - if not mesh_ref.path_id: - on_null_mesh(graph.path(tf_pid)) + info = objects.info(mesh_id) + if "mesh" not in (info.get("files") or {}): + if info.get("reason") != "empty.mesh": + raise ValueError(f"{key}: {path}: mesh {info.get('name')!r} was not converted ({info.get('reason')})") + empty.append({"mesh": info.get("name"), "path": path}) continue - yield tf_pid, mesh_ref.read(), mr.read().m_Materials - - -def extract_room(cat: Catalog, bg_key: str, out_path: Path) -> dict: - env = UnityPy.load(*[str(p) for p in cat.fetch_key(bg_key)]) - graph = SceneGraph(env) - mf_by_go, mr_by_go = mesh_components(env) - - glb = _Glb() - tex_cache: dict[tuple[str, int], int] = {} # texture object_key -> glTF texture - mat_cache: dict[tuple[str, int], int] = {} # material object_key -> glTF material - - def material_for(pptr) -> int: - mat = pptr.deref() - mkey = object_key(mat) - if mkey in mat_cache: - return mat_cache[mkey] - mat_tt = mat.read_typetree() - shader_tt = referent(mat, mat_tt["m_Shader"]).read_typetree() - gm = translate_material(mat_tt, shader_tt) - for tex_obj, env_ in main_textures(mat, mat_tt, shader_tt["m_ParsedForm"]["m_Name"]): - tid = object_key(tex_obj) - if tid not in tex_cache: - tex_tt = tex_obj.read_typetree() - buf = io.BytesIO() - texture_image(tex_obj.read()).save(buf, format="PNG") - tex_cache[tid] = glb.texture(buf.getvalue(), tex_tt["m_Name"], _sampler(tex_tt)) - info: dict = {"index": tex_cache[tid]} - sx, sy = env_["m_Scale"]["x"], env_["m_Scale"]["y"] - ox, oy = env_["m_Offset"]["x"], env_["m_Offset"]["y"] - if (sx, sy, ox, oy) != (1.0, 1.0, 0.0, 0.0): - # Unity samples (u*sx+ox, v*sy+oy) with a bottom-left origin; - # in glTF's top-left texture space that is this transform. - info["extensions"] = {"KHR_texture_transform": {"scale": [sx, sy], "offset": [ox, 1.0 - sy - oy]}} - glb.ext_used.add("KHR_texture_transform") - gm["pbrMetallicRoughness"]["baseColorTexture"] = info - mat_cache[mkey] = glb.material(gm) - return mat_cache[mkey] - - n_mesh = 0 - n_inactive = 0 - skipped = [] - null_slots = [] # submeshes whose material slot is empty - null_meshes = [] # objects whose MeshFilter has no mesh - no_triangles = [] # meshes with vertices but no triangles - for tf_pid, mesh, mats in drawn_meshes(mf_by_go, mr_by_go, graph, null_meshes.append): - arrays = mesh_arrays(mesh) - if arrays is None: - skipped.append(mesh.m_Name) # empty mesh (no vertex data) + mesh = _Mesh(objects, mesh_id) + if not any(tri is not None and len(tri) for tri in mesh.submeshes): + empty.append({"mesh": mesh.name, "path": path}) continue - v, vn, uv0, tris = arrays - if not any(len(t) for t in tris): # vertices only (e.g. a placeholder mesh without index data) - no_triangles.append({"mesh": mesh.m_Name, "path": graph.path(tf_pid)}) + mats = mr.get("m_Materials") or [] + if len(mats) < len(mesh.submeshes): + raise NotImplementedError(f"{mesh.name}: {len(mesh.submeshes)} submeshes, {len(mats)} materials") + prims = [] + for s, tri in enumerate(mesh.submeshes): + if tri is None or not len(tri): + continue + mid = ref_id(mats[s]) + if mid is None: # empty slot: no material to draw the submesh with + null_slots.append({"mesh": mesh.name, "path": path, "submesh": s}) + continue + prims.append((tri.astype(np.uint32), material_for(mid))) + if not prims: continue - if len(mats) < len(tris): - raise NotImplementedError(f"{mesh.m_Name}: {len(tris)} submeshes, {len(mats)} materials") - m = graph.world(tf_pid) - p = (np.hstack([v, np.ones((len(v), 1))]) @ m.T)[:, :3] + m = prefab.world(i) + p = (np.hstack([mesh.pos, np.ones((len(mesh.pos), 1))]) @ m.T)[:, :3] p[:, 2] *= -1.0 # Unity LH -> glTF RH - if len(vn): - nw = vn @ np.linalg.inv(m[:3, :3]).T + if mesh.nrm is not None: + nw = mesh.nrm @ np.linalg.inv(m[:3, :3]).T nw /= np.maximum(np.linalg.norm(nw, axis=1, keepdims=True), 1e-12) nw[:, 2] *= -1.0 else: nw = np.empty((0, 3)) - uv = uv0.copy() - uv[:, 1] = 1.0 - uv[:, 1] # Unity UV origin bottom-left -> glTF top-left - prims = submesh_primitives( - tris, - mats, - material_for, - lambda i: null_slots.append({"mesh": mesh.m_Name, "path": graph.path(tf_pid), "submesh": i}), - ) - if not prims: - continue - is_active = graph.active(tf_pid) - glb.add_mesh(mesh.m_Name, p, nw, uv, prims, active=is_active) - n_mesh += 1 - n_inactive += 0 if is_active else 1 - - out_path = Path(out_path) - out_path.parent.mkdir(parents=True, exist_ok=True) - glb.write(out_path, {"backgroundKey": bg_key}) + uv = mesh.uv if mesh.uv is not None else np.zeros((len(mesh.pos), 2), np.float32) + active = prefab.active(i) + glb.add_mesh(mesh.name, p, nw, uv, prims, active=active) + counts["meshes"] += 1 + counts["inactive"] += 0 if active else 1 + doc = { - "backgroundKey": bg_key, - "glb": out_path.name, - "meshCount": n_mesh, - "inactiveMeshes": n_inactive, - "skippedEmptyMeshes": skipped, + "backgroundKey": key, + "glb": "room.glb", + "meshCount": counts["meshes"], + "inactiveMeshes": counts["inactive"], "nullMaterialSubmeshes": null_slots, "nullMeshFilters": null_meshes, - "meshesWithoutTriangles": no_triangles, - "materials": [ - { - "name": m["name"], - "shader": m["extras"]["unityShader"], - "alphaMode": m.get("alphaMode", "OPAQUE"), - "alphaCutoff": m.get("alphaCutoff"), - "doubleSided": m.get("doubleSided", False), - "unlit": "KHR_materials_unlit" in m.get("extensions", {}), - "renderState": m["extras"]["unityRenderState"], - } - for m in glb.materials - ], - "textures": [t["name"] for t in glb.textures], - "samplers": glb.samplers, + "emptyMeshes": empty, + **glb.summary(), } - write_json(out_path.with_suffix(".json"), doc) - return doc + return glb.bytes({"backgroundKey": key}), doc diff --git a/src/nnnotes/content/spot.py b/src/nnnotes/content/spot.py index 6c05f16..4a34a6a 100644 --- a/src/nnnotes/content/spot.py +++ b/src/nnnotes/content/spot.py @@ -1,46 +1,79 @@ -"""Spot (explorable 2.5D scene) extractor -> spot.json + Spine assets. - -A spot = background prefab (cardboard room, see room.py) + situation prefab: -SpotSituationSettings (camera/interaction parameters), SpotCharacterController, -SpotCharacter x N (tap targets with focus transforms) and SpotSpineCharacter x M, -each driving a Spine.Unity SkeletonAnimation. Components are identified by -their MonoScript class (Catalog must be opened with `apk=`). - -A reference the prefab leaves empty is written as null, not filled in: a -SpotSpineCharacter with no `_animation` has null `skeletonData`, `animation` -and `world`; a SpotCharacter with no `_focus` has a null `focusWorld`. - -Spine files are written from the SkeletonDataAsset reference chain (write_skeleton): - SkeletonDataAsset.skeletonJSON -> skeleton (.json or binary .skel) - SkeletonDataAsset.atlasAssets[i].atlasFile -> .atlas - SpineAtlasAsset.materials[j]._MainTex -> PNG named after atlas page j +"""A spot (an explorable 2.5D scene of the home) assembled from the objects its situation prefab reaches: spot.json and +the Spine files of its characters. + +A spot is a background prefab (a room, content.room) and a situation prefab: SpotSituationSettings (camera and +interaction parameters), SpotCharacterController, SpotCharacter x N (tap targets with focus transforms) and +SpotSpineCharacter x M, each driving a Spine.Unity SkeletonAnimation. `spot` reads the situation prefab's document +(content.prefabs) and the objects its components reach, never a bundle; a MonoBehaviour's class is its script's. The +spot's row of MasterHomeSpot, its name and its tap talks come from the master data (`master_record`). + +A reference the prefab leaves empty is written as null: a SpotSpineCharacter with no `_animation` has null +`skeletonData`, `animation` and `world`; a SpotCharacter with no `_focus` has a null `focusWorld`. Fields that name +objects are written as the prefab document writes them (references). World matrices are 4x4 column-major in Unity's +world space (left-handed) of the situation prefab, its root's own transform included. + +Spine files (spine/) of each SkeletonDataAsset of the table (spine_files): + SkeletonDataAsset.skeletonJSON -> the TextAsset's bytes: the skeleton (.json or binary .skel) + SkeletonDataAsset.atlasAssets[i].atlasFile -> .atlas, as stored + SpineAtlasAsset.materials[j]._MainTex -> the Texture2D's PNG, named after atlas page j """ from __future__ import annotations -import io -import json -from pathlib import Path +import re -import UnityPy +from .prefabs import Objects, Prefab, mat4, ref_id, root_game_object, script_class -from ..core.jsonio import write_json -from ..source.catalog import Catalog -from ..source.config import apk_missing -from ..unity.reader import SceneGraph, script_class, strip_pptrs, texture_image +__all__ = ["LANGS", "SITUATION_KEY", "atlas_pages", "master_record", "situation_name", "spine_files", "spot"] +# a spot situation's catalog key: Spot/// +SITUATION_KEY = r"Spot/(?P[^/]+)/(?P[^/]+)/(?P=situation)" LANGS = ("_japanese", "_english", "_traditionalChinese", "_simplifiedChinese", "_korean") +MASTER_TABLES = ("MasterHomeSpot", "MasterText", "MasterStoryHomeSpotTapTalkEpisode") +HEADER = ("m_GameObject", "m_Script", "m_Enabled", "m_Name") # a component's Unity header fields +ANIMATION_FIELDS = ( + "_animationName", + "loop", + "timeScale", + "initialSkinName", + "initialFlipX", + "initialFlipY", + "pmaVertexColors", + "tintBlack", + "zSpacing", +) +MATRIX_CONVENTION = "Unity world space (left-handed), 4x4 column-major" + + +def situation_name(key: str) -> str: + """The of a situation key (ValueError for another key, a background's included).""" + m = re.fullmatch(SITUATION_KEY, key) + if m is None or m["situation"] == "Background": + raise ValueError(f"{key}: not a spot situation key (Spot///)") + return m["situation"] + + +def master_record(tables: dict[str, list[dict]], key: str) -> dict: + """{row, name, tapTalk} of the spot whose situation is `key`, from the master tables MASTER_TABLES ({table: + rows}): its MasterHomeSpot row, its name per language and {character id: adv ids} of its tap talks (ValueError + when no row, or several, has the key).""" + rows = [r for r in tables["MasterHomeSpot"] if r.get("_situationAssetPath") == key] + if len(rows) != 1: + raise ValueError(f"{key}: {len(rows)} MasterHomeSpot rows have it as their situation") + row = rows[0] + text = next((t for t in tables["MasterText"] if t.get("_id") == row.get("_nameTextId")), {}) + taps: dict[int, list[int]] = {} + for r in tables["MasterStoryHomeSpotTapTalkEpisode"]: + if r.get("_spotId") == row["_id"]: + taps.setdefault(r["_characterId"], []).append(r["_advId"]) + return { + "row": row, + "name": {lang[1:]: text.get(lang, "") for lang in LANGS}, + "tapTalk": {str(k): v for k, v in sorted(taps.items())}, + } -def _master(md: Path, name: str) -> list[dict]: - return json.loads((md / f"{name}.json").read_text(encoding="utf-8"))["_allData"] - - -def _mat4(m) -> list[float]: - return [float(x) for x in m.T.reshape(-1)] # column-major - - -def _atlas_pages(text: str) -> list[str]: +def atlas_pages(text: str) -> list[str]: """Page file names of a Spine atlas (a page line is followed by 'size:').""" lines = [ln.strip() for ln in text.splitlines()] return [ @@ -50,177 +83,129 @@ def _atlas_pages(text: str) -> list[str]: ] -def _text_bytes(ta) -> bytes: - s = ta.m_Script - return s.encode("utf-8", "surrogateescape") if isinstance(s, str) else bytes(s) +def _object(objects: Objects, v, what: str) -> str: + oid = ref_id(v) + if oid is None or oid not in objects: + raise ValueError(f"{what} is not in the object table") + return oid -def _target(ref): - """The object a PPtr refers to, None for an empty reference (path id 0).""" - return ref.read() if ref.path_id else None +def _text_asset(objects: Objects, v, what: str) -> tuple[str, bytes]: + """(name, bytes) of the TextAsset a reference names.""" + oid = _object(objects, v, what) + return objects.json(oid)["m_Name"], objects.bytes(oid, "data") -def write_skeleton(o, write, written_atlases: dict[int, str] | None = None) -> dict: - """The Spine files of the SkeletonDataAsset object `o` (module docstring), each given to `write(file name, - bytes)` in this order: the skeleton, then for each of its atlases not in `written_atlases` ({SpineAtlasAsset - path id: atlas file name}, updated here) the atlas text as stored and one PNG per page. Returns the skeleton's - record of spot.json: name, skeleton file, atlas files, scale, default mix, mixes.""" - written = {} if written_atlases is None else written_atlases - sda = o.read_typetree() - sk = o.read().skeletonJSON.read() - raw = _text_bytes(sk) - is_json = raw.lstrip()[:1] == b"{" - ext = ".json" if is_json else ".skel" - sk_file = sk.m_Name if sk.m_Name.endswith(ext) else f"{sk.m_Name}{ext}" # as the atlas below - write(sk_file, raw) +def spine_files(objects: Objects, sda: str, written: dict[str, str]) -> tuple[dict, dict[str, bytes]]: + """(the skeleton's record of spot.json: name, skeleton, atlases, scale, defaultMix, mixes; {file name: bytes}) + of the SkeletonDataAsset `sda`: the skeleton, then for each of its atlases not in `written` ({SpineAtlasAsset + object id: atlas file name}, updated here) the atlas text as stored and one PNG per page.""" + d = objects.json(sda) + name = d["m_Name"] + files: dict[str, bytes] = {} + sk_name, raw = _text_asset(objects, d.get("skeletonJSON"), f"{name}: the skeleton") + ext = ".json" if raw.lstrip()[:1] == b"{" else ".skel" + sk_file = sk_name if sk_name.endswith(ext) else f"{sk_name}{ext}" + files[sk_file] = raw atlases = [] - for ap in o.read().atlasAssets: - atlas_asset = ap.read() - if ap.path_id not in written: - af = atlas_asset.atlasFile.read() - araw = _text_bytes(af) - atext = araw.decode("utf-8") - a_file = af.m_Name if af.m_Name.endswith(".atlas") else f"{af.m_Name}.atlas" - write(a_file, araw) # the atlas text exactly as the game stores it - pages = _atlas_pages(atext) - mats = list(atlas_asset.materials) + for ap in d.get("atlasAssets") or (): + aid = _object(objects, ap, f"{name}: an atlas asset") + if aid not in written: + asset = objects.json(aid) + a_name, a_raw = _text_asset(objects, asset.get("atlasFile"), f"{name}: an atlas file") + a_file = a_name if a_name.endswith(".atlas") else f"{a_name}.atlas" + files[a_file] = a_raw # the atlas text exactly as the game stores it + pages = atlas_pages(a_raw.decode("utf-8")) + mats = asset.get("materials") or [] if len(mats) != len(pages): - raise RuntimeError(f"{a_file}: {len(pages)} pages vs {len(mats)} materials") + raise ValueError(f"{a_file}: {len(pages)} pages vs {len(mats)} materials") for page, mp in zip(pages, mats): - mat = mp.read() - tex = next(e.m_Texture.read() for k, e in mat.m_SavedProperties.m_TexEnvs if k == "_MainTex") - buf = io.BytesIO() - texture_image(tex).save(buf, format="PNG") - write(page, buf.getvalue()) - written[ap.path_id] = a_file - atlases.append(written[ap.path_id]) - return { - "name": sda["m_Name"], + mat = objects.json(_object(objects, mp, f"{a_file}: the material of page {page}")) + slot = (mat.get("textures") or {}).get("_MainTex") + files[page] = objects.bytes(_object(objects, slot and slot["texture"], f"{page}: the texture"), "image") + written[aid] = a_file + atlases.append(written[aid]) + record = { + "name": name, "skeleton": sk_file, "atlases": atlases, - "scale": sda["scale"], - "defaultMix": sda["defaultMix"], + "scale": d["scale"], + "defaultMix": d["defaultMix"], "mixes": [ - {"from": f, "to": t, "duration": d} - for f, t, d in zip(sda.get("fromAnimation", []), sda.get("toAnimation", []), sda.get("duration", [])) + {"from": f, "to": t, "duration": x} + for f, t, x in zip(d.get("fromAnimation", []), d.get("toAnimation", []), d.get("duration", [])) ], } - - -def spine_characters(objs: list, graph, skeletons: dict, world_of) -> list[dict]: - """SpotSpineCharacter components: object path, the SkeletonAnimation they drive (its skeleton data name, - initial animation settings and world matrix). `skeletons`: SkeletonDataAsset path id -> skeleton record. One - whose `_animation` is empty has those three null.""" - out = [] - for o in objs: - ssc = o.read_typetree() - path = graph.path(graph.tf_of_go[ssc["m_GameObject"]["m_PathID"]]) - anim_o = _target(o.read()._animation) - if anim_o is None: - out.append({"path": path, "skeletonData": None, "animation": None, "world": None}) + return record, files + + +def _fields(f: dict, *drop: str) -> dict: + return {k: v for k, v in f.items() if k not in HEADER and k not in drop} + + +def _first(prefab: Prefab, cls: str, key: str) -> dict: + found = [c for _, c in prefab.components(cls)] + if not found: + raise ValueError(f"{key}: no {cls} in the situation prefab") + return found[0]["fields"] + + +def spot(objects: Objects, key: str, master: dict) -> tuple[dict, dict[str, bytes]]: + """(spot.json, {spine/: bytes}) of the spot whose situation is `key` and whose master_record is `master` + (module documentation).""" + prefab = Prefab(objects.json(root_game_object(objects, key))) + files: dict[str, bytes] = {} + skeletons: dict[str, dict] = {} # SkeletonDataAsset object id -> its record + written: dict[str, str] = {} + sdas = [o for o in objects.ids("MonoBehaviour") if script_class(objects, o) == "SkeletonDataAsset"] + for sda in sorted(sdas, key=lambda o: (objects.json(o)["m_Name"], o)): + record, made = spine_files(objects, sda, written) + skeletons[sda] = record + for f, data in made.items(): + if f"spine/{f}" in files: + raise ValueError(f"{key}: two Spine files are named {f!r}") + files[f"spine/{f}"] = data + + spine_characters = [] + for i, c in prefab.components("SpotSpineCharacter"): + path = prefab.nodes[i]["path"] + anim = prefab.component_of(c["fields"].get("_animation")) + if anim is None: + spine_characters.append({"path": path, "skeletonData": None, "animation": None, "world": None}) continue - at = anim_o.object_reader.read_typetree() - out.append( + j, a = anim + sda = _object(objects, a["fields"].get("skeletonDataAsset"), f"{path}: the skeleton data") + spine_characters.append( { "path": path, - "skeletonData": skeletons[anim_o.skeletonDataAsset.path_id]["name"], - "animation": { - k: at[k] - for k in ( - "_animationName", - "loop", - "timeScale", - "initialSkinName", - "initialFlipX", - "initialFlipY", - "pmaVertexColors", - "tintBlack", - "zSpacing", - ) - }, - "world": world_of(at["m_GameObject"]), + "skeletonData": skeletons[sda]["name"], + "animation": {k: a["fields"][k] for k in ANIMATION_FIELDS}, + "world": mat4(prefab.world(j)), } ) - return out - - -def tap_targets(objs: list, graph, world_of) -> list[dict]: - """SpotCharacter components (tap targets) by character id: object path, their fields, world matrix and the world - matrix of their `_focus` transform (null when the reference is empty).""" - out = [] - for o in objs: - sc = o.read_typetree() - focus_o = _target(o.read()._focus) - out.append( + characters = [] + for i, c in prefab.components("SpotCharacter"): + focus = prefab.node_of(c["fields"].get("_focus")) + characters.append( { - "path": graph.path(graph.tf_of_go[sc["m_GameObject"]["m_PathID"]]), - **strip_pptrs({k: v for k, v in sc.items() if k != "_focus"}), - "world": world_of(sc["m_GameObject"]), - "focusWorld": None - if focus_o is None - else world_of(focus_o.object_reader.read_typetree()["m_GameObject"]), + "path": prefab.nodes[i]["path"], + **_fields(c["fields"], "_focus"), + "world": mat4(prefab.world(i)), + "focusWorld": None if focus is None else mat4(prefab.world(focus)), } ) - out.sort(key=lambda c: c["_characterId"]) - return out - - -def extract(cat: Catalog, md: Path, spot_id: int, out_dir: Path) -> dict: - if cat.apk is None: - raise apk_missing("resolving the Spot / Spine component classes") - spots = {r["_id"]: r for r in _master(md, "MasterHomeSpot")} - sp = spots[spot_id] - txt = {r["_id"]: r for r in _master(md, "MasterText")} - taps: dict[int, list[int]] = {} - for r in _master(md, "MasterStoryHomeSpotTapTalkEpisode"): - if r.get("_spotId") == spot_id: - taps.setdefault(r["_characterId"], []).append(r["_advId"]) - - out_dir = Path(out_dir) - spine_dir = out_dir / "spine" - spine_dir.mkdir(parents=True, exist_ok=True) - - env = UnityPy.load(*[str(p) for p in cat.fetch_key(sp["_situationAssetPath"])]) - graph = SceneGraph(env) - by_cls: dict[str | None, list] = {} - for o in env.objects: - if o.type.name == "MonoBehaviour": - by_cls.setdefault(script_class(o), []).append(o) - - def world_of(go_pptr) -> list[float]: - return _mat4(graph.world_of_go(go_pptr["m_PathID"])) - - settings = strip_pptrs(by_cls["SpotSituationSettings"][0].read_typetree()) - controller_o = by_cls["SpotCharacterController"][0] - controller = strip_pptrs(controller_o.read_typetree()) - - # Spine skeleton data -> files - def write(name: str, data: bytes) -> None: - (spine_dir / name).write_bytes(data) - - skeleton_files: dict[int, dict] = {} - written_atlases: dict[int, str] = {} - for o in by_cls.get("SkeletonDataAsset", []): - skeleton_files[o.path_id] = write_skeleton(o, write, written_atlases) - - spine_chars = spine_characters(by_cls.get("SpotSpineCharacter", []), graph, skeleton_files, world_of) - characters = tap_targets(by_cls.get("SpotCharacter", []), graph, world_of) - + characters.sort(key=lambda x: x["_characterId"]) + row = master["row"] doc = { - "spotId": spot_id, - "master": sp, - "name": {lang[1:]: txt.get(sp.get("_nameTextId"), {}).get(lang, "") for lang in LANGS}, - "tapTalk": {str(k): v for k, v in sorted(taps.items())}, - "situationSettings": settings, - "controller": {k: v for k, v in controller.items() if k not in ("_spotSpineCharacters", "_characters")}, + "spotId": row["_id"], + "master": row, + "name": master["name"], + "tapTalk": master["tapTalk"], + "situationSettings": _fields(_first(prefab, "SpotSituationSettings", key)), + "controller": _fields(_first(prefab, "SpotCharacterController", key), "_spotSpineCharacters", "_characters"), "characters": characters, - "spineCharacters": spine_chars, - "skeletons": list(skeleton_files.values()), - "resources": { - "background": [b.name for b in cat.resolve(sp["_backgroundAssetPath"])], - "situation": [b.name for b in cat.resolve(sp["_situationAssetPath"])], - }, - "matrixConvention": "Unity world space (left-handed), 4x4 column-major", + "spineCharacters": spine_characters, + "skeletons": list(skeletons.values()), + "matrixConvention": MATRIX_CONVENTION, } - write_json(out_dir / "spot.json", doc) - return doc + return doc, files diff --git a/src/nnnotes/content/storyhost.py b/src/nnnotes/content/storyhost.py index 462af37..8183627 100644 --- a/src/nnnotes/content/storyhost.py +++ b/src/nnnotes/content/storyhost.py @@ -9,8 +9,8 @@ host/host.json kind, the capture camera of a slot (CameraTargetRenderer prefab), the SimpleAdv root and its SimpleAdvLayoutRoot fields, the sequences the host screen has played when the talk starts, and per kind: - home spot.json + Spine files (spot.extract) and room.glb + room.json - (room.extract_room) under host/spot/; per placed glb node its object path, its + home spot.json + Spine files (hostspot.extract_spot) and room.glb + room.json + (hostspot.extract_room) under host/spot/; per placed glb node its object path, its state after SpotBackground.Prepare and the SpotSceneRoot.SetObject "floor" render queue; the room materials as saved; the "Spot" scene's object root, lights, volumes and RenderSettings; the lights of the placed prefabs; the background @@ -43,7 +43,8 @@ from ..unity.convert.animation import streamed_frames from ..unity.export import Exporter, TexelPacker from ..unity.reader import SceneGraph, deref, mesh_arrays, referent, script_class -from . import advui, room, spot, storyfonts +from . import advui, hostspot, storyfonts +from .spot import atlas_pages FORMAT = "ournotes.story-host/1" OVERLAY = 1 # MasterAdv._playbackMode of the episodes SimpleAdvPlayer plays @@ -600,14 +601,14 @@ def _go_ref(owner, pptr: dict) -> int | None: def background(cat, key: str, situation_name: str, ex: Exporter) -> dict: """The placed background prefab `key` as SpotSceneRoot.SetObject leaves it, over the nodes and materials of - room.extract_room (the same bundle load, glb_order): SpotBackground.Prepare(situation_name) on the prefab + hostspot.extract_room (the same bundle load, glb_order): SpotBackground.Prepare(situation_name) on the prefab root, the "floor" render queue (FindFirstChildWithName(background, "floor", "shadow"), its GetComponentInChildren().material.renderQueue = 2000, before the root is activated), then SetActiveFast(background, true). Objects outside the placed prefab (other prefabs of the bundle closure) are not placed: inactive.""" env = UnityPy.load(*[str(p) for p in cat.fetch_key(key)]) graph = SceneGraph(env) - mf_by_go, mr_by_go = room.mesh_components(env) + mf_by_go, mr_by_go = hostspot.mesh_components(env) rendered = {go for _, go in mr_by_go} # SceneGraph: a GameObject path id is in one file only root_go = container_go(env, cat.entry(key)["internal_id"]).path_id root = graph.tf_of_go[root_go] @@ -648,7 +649,7 @@ def background(cat, key: str, situation_name: str, ex: Exporter) -> dict: ) ) shown = {**overrides, root_go: True} - nodes, mats = glb_order(room.drawn_meshes(mf_by_go, mr_by_go, graph, lambda p: None)) + nodes, mats = glb_order(hostspot.drawn_meshes(mf_by_go, mr_by_go, graph, lambda p: None)) rows = [] for p in mats: mat = p.deref() @@ -712,7 +713,7 @@ def background(cat, key: str, situation_name: str, ex: Exporter) -> dict: def spine_records(env, ex: Exporter) -> tuple[dict, list, dict]: - """The Spine data of a situation prefab's bundle closure (loaded as spot.extract loads it): the material of + """The Spine data of a situation prefab's bundle closure (loaded as hostspot.extract_spot loads it): the material of every atlas page (Exporter.material records by name), per SpotSpineCharacter in spot.json's order its SkeletonAnimation's MeshRenderer sorting order / layer and {atlas page: material name} (SpineAtlasAsset materials in page order), and the shaders the materials name. -> (materials, characters, {name: shader}).""" @@ -734,7 +735,7 @@ def pages(sda) -> dict: if a is None: continue att = a.read_typetree() - names = spot._atlas_pages(spot._text_bytes(referent(a, att["atlasFile"]).read()).decode("utf-8")) + names = atlas_pages(hostspot.text_bytes(referent(a, att["atlasFile"]).read()).decode("utf-8")) mats = [deref(a, m) for m in att["materials"]] if len(names) != len(mats): raise RuntimeError(f"atlas {att.get('m_Name')}: {len(names)} pages vs {len(mats)} materials") @@ -860,17 +861,17 @@ def scene_root(ex: Exporter, key: str | None) -> dict: def home_host(cat, master_dir, player, host: dict, hdir: Path, keys: dict) -> dict: - """host.json `home`: spot.extract and room.extract_room into host/spot/, the placed background (background), - the Spine records, the Spot scene root, the blur, the ambient sound and the shaders (host/shaders/, GLES3 - programs only) with each material's GLES3 keyword set.""" + """host.json `home`: hostspot.extract_spot and hostspot.extract_room into host/spot/, the placed background + (background), the Spine records, the Spot scene root, the blur, the ambient sound and the shaders (host/shaders/, + GLES3 programs only) with each material's GLES3 keyword set.""" sdir = hdir / "spot" - doc = spot.extract(cat, Path(master_dir), host["spotId"], sdir) + doc = hostspot.extract_spot(cat, Path(master_dir), host["spotId"], sdir) row = doc["master"] # A tap talk's character need not be among the spot's SpotCharacters (spot.json `characters`): the game then has # no tap target that starts the talk (HomeSpotAfterTalkRequestFactory.TrySelectAdv matches the tapped # SpotCharacter's _characterId), and the player plays it without the camera focus bg_key, sit_key = row["_backgroundAssetPath"], row["_situationAssetPath"] - room_doc = room.extract_room(cat, bg_key, sdir / "room.glb") + room_doc = hostspot.extract_room(cat, bg_key, sdir / "room.glb") ex = Exporter(cat, hdir, player=player, textures="deferred", inline_meshes=False) sit_env = UnityPy.load(*[str(p) for p in cat.fetch_key(sit_key)]) sit_go = container_go(sit_env, cat.entry(sit_key)["internal_id"]) @@ -1082,15 +1083,15 @@ def component_in_children(graph, tf: int, has, active) -> int | None: def glb_order(drawn, arrays_of=mesh_arrays) -> tuple[list, list]: - """The nodes and materials room.extract_room writes, in its order, from its room.drawn_meshes items + """The nodes and materials hostspot.extract_room writes, in its order, from its hostspot.drawn_meshes items [(transform id, Mesh, material PPtrs)]: a mesh without vertex data or triangles, or whose submeshes all lack - a material, is no node; a material is numbered at its first use (room.submesh_primitives order, by - room.object_key). -> ([(transform id, mesh name)], [material PPtr]).""" + a material, is no node; a material is numbered at its first use (hostspot.submesh_primitives order, by + hostspot.object_key). -> ([(transform id, mesh name)], [material PPtr]).""" index: dict[tuple[str, int], int] = {} mats, nodes = [], [] def material_for(p) -> int: - key = room.object_key(p.deref()) + key = hostspot.object_key(p.deref()) if key not in index: index[key] = len(mats) mats.append(p) @@ -1105,29 +1106,29 @@ def material_for(p) -> int: continue if len(pptrs) < len(tris): raise NotImplementedError(f"{mesh.m_Name}: {len(tris)} submeshes, {len(pptrs)} materials") - if room.submesh_primitives(tris, pptrs, material_for, lambda i: None): + if hostspot.submesh_primitives(tris, pptrs, material_for, lambda i: None): nodes.append((tf, mesh.m_Name)) return nodes, mats def glb_textures(materials: list) -> dict[tuple[str, int], int]: - """{texture room.object_key: glb texture index} of room.extract_room for its materials [(material object, - typetree, shader name)] in glb order: the main texture slot of each (room.main_textures), numbered at first + """{texture hostspot.object_key: glb texture index} of hostspot.extract_room for its materials [(material object, + typetree, shader name)] in glb order: the main texture slot of each (hostspot.main_textures), numbered at first use.""" index: dict[tuple[str, int], int] = {} for owner, tt, shader_name in materials: - for tex, _ in room.main_textures(owner, tt, shader_name): - index.setdefault(room.object_key(tex), len(index)) + for tex, _ in hostspot.main_textures(owner, tt, shader_name): + index.setdefault(hostspot.object_key(tex), len(index)) return index def _texture_key(owner, ref: dict) -> tuple[str, int] | None: - """room.object_key of the texture a material's PPtr `ref` names (read from `owner`'s typetree); None for an + """hostspot.object_key of the texture a material's PPtr `ref` names (read from `owner`'s typetree); None for an empty slot or a texture outside the loaded files (so not in the glb).""" if not ref["m_PathID"]: return None try: - return room.object_key(deref(owner, ref)) + return hostspot.object_key(deref(owner, ref)) except FileNotFoundError: return None @@ -1169,7 +1170,7 @@ def glb_json(path) -> dict: def check_room(key: str, glb: dict, room_doc: dict, bg: dict) -> None: - """room.extract_room's room.glb (its JSON chunk `glb`) and room.json (`room_doc`) have the nodes, materials and + """hostspot.extract_room's room.glb (its JSON chunk `glb`) and room.json (`room_doc`) have the nodes, materials and glTF textures of the background record `bg` (background), in its order.""" if ( [n["name"] for n in glb["nodes"]] != bg["check"]["nodes"] diff --git a/src/nnnotes/formats/shader.py b/src/nnnotes/formats/shader.py index a957473..08a5a72 100644 --- a/src/nnnotes/formats/shader.py +++ b/src/nnnotes/formats/shader.py @@ -132,7 +132,7 @@ def variants(tt: dict, blobs: dict[int, list[bytes]]): n += 1 -def _parsed_summary(tt: dict) -> dict: +def parsed_summary(tt: dict) -> dict: pf = tt["m_ParsedForm"] return { "name": pf.get("m_Name"), @@ -160,7 +160,7 @@ def _parsed_summary(tt: dict) -> dict: def _render(o, tt: dict, name: str) -> tuple[str, list[dict], list[bytes]]: """(parsed summary JSON text, variant records, variant program bytes) of one Shader object.""" safe = _safe(name) - text = dumps(_parsed_summary(tt), indent=1, ensure_ascii=False, default=str) + text = dumps(parsed_summary(tt), indent=1, ensure_ascii=False, default=str) recs, codes = [], [] for plat, si, pi, stage, n, t, kws, code in variants(tt, platform_blobs(o.read())): tag = PLATFORM.get(plat, str(plat)) diff --git a/src/nnnotes/pipeline/assemble.py b/src/nnnotes/pipeline/assemble.py index f2fa27e..9ba12f6 100644 --- a/src/nnnotes/pipeline/assemble.py +++ b/src/nnnotes/pipeline/assemble.py @@ -238,10 +238,11 @@ def _prune(tid: str, objects: dict, made: Assembly) -> dict: class ProductStage(Stage): """Base of the product stages (module documentation). Subclasses set `name`, `version` and `KIND`, and define `keys`, `resources` and/or `root_objects` and `assemble`; `subjects` defaults to the subjects of the fact - "products".""" + "products". A stage that reads master tables names them in `MASTER` (its inputs come from the fact "master").""" after = (ADDRESSES, EXPORT, CROP) KIND = "product" + MASTER: tuple[str, ...] = () def __init__(self): self._memo: dict = {} diff --git a/src/nnnotes/pipeline/modelstages.py b/src/nnnotes/pipeline/modelstages.py deleted file mode 100644 index ca4c19a..0000000 --- a/src/nnnotes/pipeline/modelstages.py +++ /dev/null @@ -1,227 +0,0 @@ -"""The Spine skeleton stage, written by the extractor of `nnnotes export spot` (unchanged) from the bundles of its -tasks. - -spine.skeleton, one task per SkeletonDataAsset (a MonoBehaviour whose script the script table names -SkeletonDataAsset) of a selected bundle; its subject is the asset's stable address by path id, -":" (link.file_tags): - inputs "bundle:": the bundles holding the serialized files the asset's file reaches through - externals, transitively (every object a reference chain from the asset can reach); externals no - census holds (the built-in files) are left out - context {"object": the asset's object id} - artifacts "spine.skeleton:#": the skeleton (.json or .skel), each of its atlases (.atlas) and - the atlas pages (PNG), as spot.write_skeleton writes them for `nnnotes export spot` - item the asset: exported (its artifacts), or failed with the writer's error - -The task names its extractor as its one atom ("spot.write_skeleton"), whose id carries the versions of the libraries -the files depend on; this module imports UnityPy only when a task runs. -""" - -from __future__ import annotations - -import re - -from ..unity.atoms import impl_id -from . import contract, link -from .contract import Cost, IncompatibleTask, Input -from .stages import Output, Pending, Stage - -SPINE = "spine.skeleton" -CENSUS = "unity.census" -SCRIPTS = "link.scripts" -SCRIPTS_TASK = contract.task_id(SCRIPTS, "all") -SKELETON_CLASS = "SkeletonDataAsset" -# the libraries the extractor's output depends on (export._LIBS, the salt of export's PNG cache) -LIBRARIES = ("UnityPy", "Pillow", "texture2ddecoder", "astc-encoder-py", "etcpak", "numpy") -_ADDRESS = re.compile(r"0x[0-9a-fA-F]{6,}") -_EXT = re.compile(r"[0-9a-z]+") - - -def _message(e: BaseException) -> str: - return _ADDRESS.sub("0x?", f"{type(e).__name__}: {e}")[:500] - - -def _ext(name: str, default: str = "bin") -> str: - ext = name.rsplit("/", 1)[-1].rpartition(".")[2].lower() if "." in name.rsplit("/", 1)[-1] else "" - return ext if _EXT.fullmatch(ext) else default - - -def _check_atoms(task, atoms: dict) -> None: - if task.atoms != atoms: - raise IncompatibleTask(f"task {task.id}: this installation runs {atoms}, the task names {task.atoms}") - - -def _fact_input(bundles, stable: str, what: str) -> Input: - """The Input of a bundle from the fact "bundles" (Pending while it is not fetched).""" - if stable not in bundles: - raise ValueError(f"{what}: bundle {stable} is not readable") - inp = bundles[stable] - if inp is None: - raise Pending(f"fetch:{stable}") - if isinstance(inp, Exception): - raise inp - if inp.role != "bundle": - raise ValueError(f"{what}: input role {inp.role!r}, expected 'bundle'") - return inp - - -# ---------------------------------------------------------------- spine.skeleton -def skeleton_index(env, selected) -> dict: - """What spine.skeleton plans from, read from the census results and the script table: {"subjects": {subject: - (stable bundle name, file, pathId)} of the SkeletonDataAssets of the `selected` bundles, "files": {serialized - file (case-folded): stable bundle name} (a file two bundles hold: the first by name), "externals": {serialized - file (case-folded): [its external files]}}. Raises Pending while a census task has not run or the script table - is missing; failed censuses are left out.""" - pending = env.pending(CENSUS) - if pending: - raise Pending(pending[0]) - table = contract.loads(env.depot.read(env.input_of(SCRIPTS_TASK, "scripts").sha256))["scripts"] - scripts = {oid for oid, entry in table.items() if entry.rsplit("|", 1)[-1] == SKELETON_CLASS} - files: dict[str, str] = {} - externals: dict[str, list[str]] = {} - subjects: dict[str, tuple] = {} - for tid in env.tasks(CENSUS): - stable = contract.parse_task_id(tid, CENSUS)[1] - doc = contract.loads(env.depot.read(env.artifact(tid, "census")["content"]["sha256"])) - for f in doc["files"]: - files.setdefault(f["name"].casefold(), stable) - externals.setdefault(f["name"].casefold(), [x["file"] for x in f["externals"]]) - if not scripts or stable not in selected or "MonoBehaviour" not in doc["classes"]: - continue - tags = link.file_tags(f["name"] for f in doc["files"]) - for f in doc["files"]: - for o in f["objects"]: - s = o.get("script") - if ( - o["class"] == "MonoBehaviour" - and s - and s.get("file") is not None - and contract.object_id(s["file"], s["pathId"]) in scripts - ): - subject = contract.stable_address(stable + tags[f["name"]], path_id=o["pathId"]) - subjects[subject] = (stable, f["name"], int(o["pathId"])) - return {"subjects": subjects, "files": files, "externals": externals} - - -def reach(index: dict, file: str) -> list[str]: - """The stable names of the bundles holding `file` and every serialized file it reaches through externals, - sorted (skeleton_index document; files no census holds are left out).""" - start = file.casefold() - seen, stack, out = {start}, [start], set() - while stack: - f = stack.pop() - b = index["files"].get(f) - if b is None: - continue - out.add(b) - for x in index["externals"].get(f, ()): - xf = x.casefold() - if xf not in seen: - seen.add(xf) - stack.append(xf) - return sorted(out) - - -class SpineStage(Stage): - """spine.skeleton: a SkeletonDataAsset -> its Spine files as `nnnotes export spot` writes them (module - documentation). Subjects from the census results, the script table (link.scripts:all) and the fact "selected"; - inputs from the fact "bundles". Waits for census tasks that have not run.""" - - name = SPINE - version = 1 - after = (CENSUS, SCRIPTS) - ATOMS = {"spot.write_skeleton": impl_id(LIBRARIES, 2)} - - def __init__(self): - self._memo = None - - def __getstate__(self): - return {} - - def __setstate__(self, state): - self.__init__() - - def index(self, env) -> dict: - """skeleton_index, kept while the census results, the script table and the selection stay the same.""" - pending = env.pending(CENSUS) - if pending: - raise Pending(pending[0]) - selected = tuple(sorted(env.fact("selected"))) - sig = (selected, tuple(env.key(t) for t in env.tasks(CENSUS)), env.key(SCRIPTS_TASK)) - if self._memo is None or self._memo[0] != sig: - self._memo = (sig, skeleton_index(env, set(selected))) - return self._memo[1] - - def _subject(self, subject: str, env) -> tuple[dict, tuple]: - idx = self.index(env) - if subject not in idx["subjects"]: - raise ValueError(f"{self.name}: no SkeletonDataAsset {subject} in the selected bundles") - return idx, idx["subjects"][subject] - - def subjects(self, env) -> list[str]: - return sorted(self.index(env)["subjects"]) - - def closure(self, subject: str, env) -> list[str]: - idx, (_stable, file, _pid) = self._subject(subject, env) - return reach(idx, file) - - def inputs(self, subject: str, env) -> list[Input]: - bundles, what = env.fact("bundles"), f"{self.name} {subject}" - out = [] - for stable in self.closure(subject, env): - inp = _fact_input(bundles, stable, what) - out.append(Input(f"bundle:{stable}", inp.sha256, inp.size, inp.name, inp.locators)) - return out - - def context(self, subject: str, env) -> dict: - _idx, (_stable, file, pid) = self._subject(subject, env) - return {"object": contract.object_id(file, pid)} - - def depends(self, subject: str, env) -> list[str]: - return sorted({*(contract.task_id(CENSUS, s) for s in self.closure(subject, env)), SCRIPTS_TASK}) - - def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: - """From the closure's bytes (least squares over the skeletons of a catalog).""" - size = sum(i.size for i in inputs) - return Cost(0.27 + size * 1.9e-7, (440 << 20) + 20 * size) - - def run(self, task, depot) -> Output: - _check_atoms(task, self.ATOMS) - from ..content import spot - from ..unity import reader as unity - - oid = task.context["object"] - paths = [depot.open_input(i) for i in task.inputs if i.role.startswith("bundle:")] - env = unity.load_closure(paths) - file, pid = contract.parse_object_id(oid) - written: dict[str, bytes] = {} - try: - sf = env.get_cab(file) - o = getattr(sf, "objects", {}).get(pid) - if o is None: - raise KeyError(f"{oid} is not in the task's bundles") - record = spot.write_skeleton(o, written.__setitem__) - except (MemoryError, OSError): - raise - except Exception as e: - return Output( - [], - [contract.item(oid, "failed", why=contract.reason("failed.export", _message(e)), cls="MonoBehaviour")], - ) - arts = [] - for name in sorted(written): - data = written[name] - if name == record["skeleton"]: - ext, kind, facts = _ext(name), "spine.skeleton", record - elif name in record["atlases"]: - ext, kind, facts = _ext(name, "atlas"), "spine.atlas", {} - else: - ext, kind, facts = "png", "spine.page", {} - arts.append( - contract.artifact( - contract.artifact_id(task.id, name), - depot.add(data, ext), - contract.provenance(task), - {"kind": kind, "format": ext, "facts": facts}, - ) - ) - return Output(arts, [contract.item(oid, "exported", artifacts=[a["id"] for a in arts], cls="MonoBehaviour")]) diff --git a/src/nnnotes/pipeline/spot.py b/src/nnnotes/pipeline/spot.py new file mode 100644 index 0000000..28e1bc1 --- /dev/null +++ b/src/nnnotes/pipeline/spot.py @@ -0,0 +1,106 @@ +"""The spot products: a spot background as binary glTF (room.gltf) and a spot situation's spot.json with its Spine +files (spot.files). + +Both are product stages (pipeline.assemble) whose roots are the objects of their subject's catalog key; they +assemble with content.room and content.spot and keep no objects (their files are all they hold): + +room.gltf subject a background key (Spot//Background/): "#room.glb" (kind spot.room) and + "#room.json" (kind spot.room-summary); its manifest is rooms/.json +spot.files subject a situation key (Spot///): "#spot.json" (kind spot.situation) + and "#spine/" for each Spine file (kind spine.skeleton, spine.atlas or spine.texture); its + manifest is spots/.json. Its input "master" is the spot's master record + (content.spot.master_record) from the master tables of the fact "master", made at planning +""" + +from __future__ import annotations + +import re + +from ..content import room, spot +from . import contract +from .assemble import Assembly, ProductStage +from .contract import Input +from .views import master_rows + +__all__ = ["ROOM", "SPOT", "RoomStage", "SpotStage", "spine_files"] + +ROOM, SPOT = "room.gltf", "spot.files" +MASTER_INPUT = "master" + + +def spine_files(doc: dict, spine: dict[str, bytes]) -> dict: + """{path: (bytes, extension, kind)} of the Spine files of spot.json `doc`: the skeletons (spine.skeleton), the + atlases (spine.atlas) and the PNG of each atlas page (spine.texture).""" + skeletons = {f"spine/{s['skeleton']}" for s in doc["skeletons"]} + atlases = {f"spine/{a}" for s in doc["skeletons"] for a in s["atlases"]} + out = {} + for path, data in spine.items(): + if path in skeletons: + out[path] = (data, path.rsplit(".", 1)[-1], "spine.skeleton") + elif path in atlases: + out[path] = (data, "atlas", "spine.atlas") + else: + out[path] = (data, "png", "spine.texture") + return out + + +class RoomStage(ProductStage): + """room.gltf: a background key -> room.glb and room.json (module documentation).""" + + name = ROOM + version = 1 + KIND = "SpotRoom" + + def select(self, keys: list[str]) -> list[str]: + return sorted(k for k in keys if re.fullmatch(room.BACKGROUND_KEY, k)) + + def keys(self, subject: str, env) -> list[str]: + room.room_name(subject) + return [subject] + + def manifest(self, subject: str) -> str: + return f"rooms/{room.room_name(subject)}.json" + + def assemble(self, task, objects) -> Assembly: + glb, doc = room.room(objects, task.subject) + files = { + "room.glb": (glb, "glb", "spot.room"), + "room.json": (contract.encode(doc), "json", "spot.room-summary"), + } + return Assembly(files, set()) + + +class SpotStage(ProductStage): + """spot.files: a situation key -> spot.json and its Spine files (module documentation).""" + + name = SPOT + version = 1 + KIND = "SpotSituation" + MASTER = spot.MASTER_TABLES + + def select(self, keys: list[str]) -> list[str]: + out = [] + for k in keys: + try: + spot.situation_name(k) + except ValueError: + continue + out.append(k) + return sorted(out) + + def keys(self, subject: str, env) -> list[str]: + spot.situation_name(subject) + return [subject] + + def more_inputs(self, subject: str, env) -> list[Input]: + data = contract.encode(spot.master_record(master_rows(env, self.MASTER), subject)) + return [Input(MASTER_INPUT, env.depot.put(data), len(data), None, ({"kind": "depot"},))] + + def manifest(self, subject: str) -> str: + return f"spots/{spot.situation_name(subject)}.json" + + def assemble(self, task, objects) -> Assembly: + master = contract.loads(objects.depot.input_bytes(task.input(MASTER_INPUT))) + doc, spine = spot.spot(objects, task.subject, master) + files = {"spot.json": (contract.encode(doc), "json", "spot.situation"), **spine_files(doc, spine)} + return Assembly(files, set()) diff --git a/src/nnnotes/pipeline/views.py b/src/nnnotes/pipeline/views.py index a8c3683..9ba60a7 100644 --- a/src/nnnotes/pipeline/views.py +++ b/src/nnnotes/pipeline/views.py @@ -874,6 +874,20 @@ def _cached_rows(sha: str, read) -> list[dict]: return rows +def master_rows(env, tables: Iterable[str]) -> dict[str, list[dict]]: + """{table: rows} of the master tables `tables` in the directory of the fact "master" (ViewError for a table the + master data lacks).""" + mdir = Path(env.fact("master")) + out = {} + for t in tables: + p = mdir / f"{t}.json" + if not p.is_file(): + raise ViewError(f"master table {t} is not in the master data") + sha, _ = env.depot.identify(p) + out[t] = _cached_rows(sha, p.read_bytes) + return out + + def _address_index(env, tid: str) -> AddressIndex: """The address index of the result of the link.addresses task `tid` (Pending until it has run).""" sha = env.artifact(tid, "addresses")["content"]["sha256"] diff --git a/src/nnnotes/unity/reader.py b/src/nnnotes/unity/reader.py index fd600e4..eef4285 100644 --- a/src/nnnotes/unity/reader.py +++ b/src/nnnotes/unity/reader.py @@ -19,6 +19,7 @@ from UnityPy.helpers.TypeTreeNode import TypeTreeNode from ..core import cache +from .transform import trs # ---- typetree blobs --------------------------------------------------------- # A serialized file lists the typetree of each of its classes as a blob, and UnityPy parses every blob in Python @@ -249,25 +250,6 @@ def texture_image(tex): # ---- scene graph ---------------------------------------------------------- -def quat_matrix(q: dict) -> np.ndarray: - x, y, z, w = q["x"], q["y"], q["z"], q["w"] - return np.array( - [ - [1 - 2 * (y * y + z * z), 2 * (x * y - z * w), 2 * (x * z + y * w), 0], - [2 * (x * y + z * w), 1 - 2 * (x * x + z * z), 2 * (y * z - x * w), 0], - [2 * (x * z - y * w), 2 * (y * z + x * w), 1 - 2 * (x * x + y * y), 0], - [0, 0, 0, 1], - ], - dtype=np.float64, - ) - - -def trs(pos: dict, rot: dict, scale: dict) -> np.ndarray: - t = np.eye(4) - t[:3, 3] = [pos["x"], pos["y"], pos["z"]] - return t @ quat_matrix(rot) @ np.diag([scale["x"], scale["y"], scale["z"], 1.0]) - - class SceneGraph: """GameObject/Transform hierarchy of an environment, with world matrices.""" diff --git a/src/nnnotes/unity/transform.py b/src/nnnotes/unity/transform.py new file mode 100644 index 0000000..2b69bfb --- /dev/null +++ b/src/nnnotes/unity/transform.py @@ -0,0 +1,27 @@ +"""Unity transforms as 4x4 matrices: a quaternion's rotation and a local position, rotation and scale (TRS), in +Unity's left-handed space.""" + +from __future__ import annotations + +import numpy as np + +__all__ = ["quat_matrix", "trs"] + + +def quat_matrix(q: dict) -> np.ndarray: + x, y, z, w = q["x"], q["y"], q["z"], q["w"] + return np.array( + [ + [1 - 2 * (y * y + z * z), 2 * (x * y - z * w), 2 * (x * z + y * w), 0], + [2 * (x * y + z * w), 1 - 2 * (x * x + z * z), 2 * (y * z - x * w), 0], + [2 * (x * z - y * w), 2 * (y * z + x * w), 1 - 2 * (x * x + y * y), 0], + [0, 0, 0, 1], + ], + dtype=np.float64, + ) + + +def trs(pos: dict, rot: dict, scale: dict) -> np.ndarray: + t = np.eye(4) + t[:3, 3] = [pos["x"], pos["y"], pos["z"]] + return t @ quat_matrix(rot) @ np.diag([scale["x"], scale["y"], scale["z"], 1.0]) diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index 185a9b6..f91890d 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -1464,6 +1464,31 @@ def assemble(self, task, objects): return assemble.Assembly(files, set()) +class FakeReader(assemble.ProductStage): + """A product per Bg/ key of the selection that reads the master table MasterX: its rows as rows.json.""" + + name = "fake.reader" + version = 1 + KIND = "FakeReader" + MASTER = ("MasterX",) + + def select(self, keys): + return [k for k in keys if k.startswith("Bg/")] + + def keys(self, subject, env): + return [subject] + + def more_inputs(self, subject, env): + from nnnotes.pipeline.views import master_rows + + data = contract.encode(master_rows(env, self.MASTER)) + return [contract.Input("master", env.depot.put(data), len(data), None, ({"kind": "depot"},))] + + def assemble(self, task, objects): + rows = objects.depot.input_bytes(task.input("master")) + return assemble.Assembly({"rows.json": (rows, "json", "fake.rows")}, set()) + + PRODUCT_SOURCES = FAKE_SOURCES + ( assets.StageSource("fake.product", "tests.cli.test_assets:FakeProduct", output=True, product=True), assets.StageSource("live2d.cubism", "tests.cli.test_assets:FakeCubism", output=True, product=True), @@ -1509,6 +1534,26 @@ def test_the_product_layout_places_the_products_of_the_selection(tmp_path, capsy assert (tmp_path / "both" / "product" / "fake.product" / "Bg%2Fx.json").is_file() +def test_products_that_read_master_data_have_subjects_only_with_it(tmp_path, capsys, monkeypatch): + sources = PRODUCT_SOURCES + ( + assets.StageSource("fake.reader", "tests.cli.test_assets:FakeReader", output=True, product=True), + ) + monkeypatch.setattr(assets, "STAGE_SOURCES", sources) + d = setup_data(tmp_path, censuses(broken=False)) + code, out, err = export(capsys, tmp_path, d, "s", "out", "--layout", "product") + assert code == 0, err + run = contract.loads((tmp_path / "out" / assets.REPORTS / "run.json").read_bytes()) + assert [t["id"] for t in run["tasks"] if t["id"].startswith("fake.")] == ["fake.product:Bg/x"] + master = tmp_path / "master" + master.mkdir() + (master / "MasterX.json").write_text('{"_allData": [{"_id": 1}]}', encoding="utf-8") + code, out, err = export(capsys, tmp_path, d, "s", "out2", "--layout", "product", "--master", master) + assert code == 0, err + manifest = contract.loads((tmp_path / "out2" / "fake.reader" / "Bg%2Fx.json").read_bytes()) + rows = (tmp_path / "out2" / manifest["files"]["rows.json"]["asset"]).read_bytes() + assert contract.loads(rows) == {"MasterX": [{"_id": 1}]} + + def test_export_live2d_writes_the_cubism_files_of_one_model(tmp_path, capsys, monkeypatch): monkeypatch.setattr(assets, "STAGE_SOURCES", PRODUCT_SOURCES) monkeypatch.setattr("nnnotes.pipeline.catalogdb.apk_version", lambda apk: None) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 9ec5b87..2fa65f5 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -368,7 +368,8 @@ def data(catalog_file, tmp_path, cdn, monkeypatch): "argv, message", [ (["bundle", "pull", "Char/A", "Nope/1", "Nope/2", "-o", "b"], "not a key of the catalog: Nope/1, Nope/2"), - (["export", "room", "Nope/1", "-o", "r.glb"], "not a key of the catalog: Nope/1"), + (["export", "room", "Nope/1", "-o", "r"], "Nope/1: not a spot background key"), + (["export", "room", "Spot/a/Background/a", "-o", "r"], "Spot/a/Background/a: not a key of the catalog"), (["export", "shader", "--key", "Nope/1", "-o", "s"], "not a key of the catalog: Nope/1"), (["export", "shader", "--apk-bundle", "zz", "-o", "s"], "--apk-bundle 'zz': 0 APK bundles match"), (["export", "audio", "Nope", "-o", "a"], "cue sheet Nope: no key Cri/Sound/Nope in the catalog"), diff --git a/tests/content/test_hostspot.py b/tests/content/test_hostspot.py new file mode 100644 index 0000000..cabd194 --- /dev/null +++ b/tests/content/test_hostspot.py @@ -0,0 +1,509 @@ +"""hostspot (the spot files of a story's home host, read from bundles): empty material slots and mesh filters, what +draws nothing, materials and textures of a bundle whose serialized files reuse path ids, Spine characters and tap +targets with and without the references the prefab leaves empty, and the Spine files of a skeleton (synthetic +objects).""" + +import copy +import io +import json +import struct +from types import SimpleNamespace + +import numpy as np +import pytest +from PIL import Image +from UnityPy.classes import PPtr as UnityPPtr + +from nnnotes.content import hostspot, room +from nnnotes.unity.reader import DEFAULT_RESOURCES +from tests.helpers.fakeunity import F32, POSITION, UV0, mesh_tt, submesh, unitypy_mesh +from tests.helpers.spotobjects import fv, material, shader + + +class PPtr: + def __init__(self, path_id): + self.path_id = path_id + + +def test_empty_material_slots_are_skipped_and_reported(): + tris = [np.array([[0, 1, 2]]), np.zeros((0, 3), int), np.array([[2, 3, 0], [0, 1, 3]])] + mats = [PPtr(0), PPtr(11), PPtr(12)] + nulls, used = [], [] + + def material_for(p): + used.append(p.path_id) + return p.path_id * 10 + + prims = hostspot.submesh_primitives(tris, mats, material_for, nulls.append) + assert nulls == [0] and used == [12] # the empty submesh 1 gives nothing, silently + assert len(prims) == 1 and prims[0][1] == 120 + assert prims[0][0].tolist() == [2, 0, 3, 0, 3, 1] # winding reversed for glTF + assert hostspot.submesh_primitives(tris[:1], mats[:1], material_for, nulls.append) == [] and nulls == [0, 0] + + +class Obj: + """A component: its typetree and the object UnityPy reads for it.""" + + def __init__(self, tt, **fields): + self.tt, self.fields = tt, fields + + def read_typetree(self): + return self.tt + + def read(self): + return type("Read", (), self.fields)() + + +class Ref(PPtr): + def __init__(self, path_id, target=None): + super().__init__(path_id) + self.target = target + + def read(self): + assert self.path_id, "an empty reference is not read" + return self.target + + +class Graph: + tf_of_go = {1: 11, 2: 12, 3: 13, 4: 14} + + def path(self, tf_pid): + return f"Room/obj{tf_pid}" + + +def test_a_mesh_filter_without_a_mesh_is_skipped_and_reported(): + mesh = object() + mfs = { + ("f", 1): Obj({}, m_Mesh=Ref(21, mesh)), + ("f", 2): Obj({}, m_Mesh=Ref(0)), + ("f", 3): Obj({}, m_Mesh=Ref(0)), + ("f", 4): Obj({}, m_Mesh=Ref(24, mesh)), + } + mrs = { + ("f", 1): Obj({"m_Enabled": 1}, m_Materials=["m1"]), + ("f", 2): Obj({"m_Enabled": 1}, m_Materials=[]), + ("f", 3): Obj({"m_Enabled": 0}, m_Materials=[]), # 3: disabled renderer, 4: no renderer + ("g", 4): Obj({"m_Enabled": 1}, m_Materials=["m4"]), + } # another file's GameObject 4 + nulls = [] + drawn = list(hostspot.drawn_meshes(mfs, mrs, Graph(), nulls.append)) + assert drawn == [(11, mesh, ["m1"])] + assert nulls == ["Room/obj12"] # the disabled one draws nothing either way + + +def test_extract_room_lists_what_draws_nothing(tmp_path, monkeypatch): + """A mesh without triangles, a MeshFilter without a mesh and a submesh without a material are not written but + listed in hostspot.json.""" + + class Mesh: + def __init__(self, name): + self.m_Name = name + + class Cat: + def fetch_key(self, key): + return [] + + class SceneGraphStub(Graph): + def __init__(self, env): + pass + + def world(self, tf_pid): + return np.eye(4) + + def active(self, tf_pid): + return True + + tri = np.array([[0, 1, 2]]) + arrays = { + "quad": (np.zeros((3, 3)), np.empty((0, 3)), np.zeros((3, 2)), [tri]), + "placeholder": (np.zeros((4, 3)), np.empty((0, 3)), np.zeros((4, 2)), [np.zeros((0, 3), int)]), + "bare": (np.zeros((4, 3)), np.empty((0, 3)), np.zeros((4, 2)), []), + } + + def drawn(mf_by_go, mr_by_go, graph, on_null_mesh): + on_null_mesh("Room/obj14") + yield 11, Mesh("quad"), [PPtr(0)] + yield 12, Mesh("placeholder"), [] + yield 13, Mesh("bare"), [] + + monkeypatch.setattr(hostspot.UnityPy, "load", lambda *a: type("Env", (), {"objects": []})()) + monkeypatch.setattr(hostspot, "SceneGraph", SceneGraphStub) + monkeypatch.setattr(hostspot, "drawn_meshes", drawn) + monkeypatch.setattr(hostspot, "mesh_arrays", lambda mesh: arrays[mesh.m_Name]) + doc = hostspot.extract_room(Cat(), "Spot/a/Background/b", tmp_path / "hostspot.glb") + assert doc["meshCount"] == 0 and doc["materials"] == [] + assert doc["nullMeshFilters"] == ["Room/obj14"] + assert doc["meshesWithoutTriangles"] == [ + {"mesh": "placeholder", "path": "Room/obj12"}, + {"mesh": "bare", "path": "Room/obj13"}, + ] + assert doc["nullMaterialSubmeshes"] == [{"mesh": "quad", "path": "Room/obj11", "submesh": 0}] + assert (tmp_path / "hostspot.glb").is_file() and (tmp_path / "hostspot.json").is_file() + + +# ---------------------------------------------------------------- a bundle whose serialized files reuse path ids +V0, V1 = {"x": 0.0, "y": 0.0, "z": 0.0}, {"x": 1.0, "y": 1.0, "z": 1.0} +Q0 = {"x": 0.0, "y": 0.0, "z": 0.0, "w": 1.0} +ROOM_KEY = "Spot/a/Background/b" +PREFAB = "Assets/Room.prefab" +# the quads under the prefab root: (object name, material path id, its file id in CAB-a: 1 is CAB-b) +ROOM_OBJECTS = [("wall", 5, 0), ("wall_b", 5, 1), ("floor", 6, 0), ("wall_c", 5, 0)] + + +def ref(pid, file_id=0): + return {"m_FileID": file_id, "m_PathID": pid} + + +class File: + """A serialized file as UnityPy's PPtr resolves it: name, objects by path id, externals (paths) and the bundle + holding it (`parent`); a file outside the loaded bundles is not found.""" + + def __init__(self, name, bundle, externals=()): + self.name, self.parent, self.objects = name, bundle, {} + self.externals = [SimpleNamespace(path=p) for p in externals] + self.environment = SimpleNamespace(find_file=lambda name: None) + bundle.files[name] = self + + def add(self, kind, pid, tt, obj=None): + o = SimpleNamespace( + type=SimpleNamespace(name=kind), + path_id=pid, + assets_file=self, + read_typetree=lambda: copy.deepcopy(tt), + read=lambda: obj, + ) + self.objects[pid] = o + return o + + def pptr(self, pid, file_id=0): + """A PPtr as UnityPy parses it from an object of this file.""" + return UnityPPtr(m_FileID=file_id, m_PathID=pid, assetsfile=self) + + +def tex_env(pid, file_id=0): + return {"m_Texture": ref(pid, file_id), "m_Scale": {"x": 1.0, "y": 1.0}, "m_Offset": {"x": 0.0, "y": 0.0}} + + +def textured(name, **slots): + """A material typetree of shader 3 with the texture envs `slots` {slot: tex_env}.""" + tt = material(name=name) + tt.update(m_Shader=ref(3), m_ValidKeywords=[], m_CustomRenderQueue=-1) + tt["m_SavedProperties"]["m_TexEnvs"] = [[k, v] for k, v in slots.items()] + return tt + + +def quad(name): + return unitypy_mesh( + mesh_tt(name, 3, {POSITION: (F32, 3, np.eye(3)), UV0: (F32, 2, np.zeros((3, 2)))}, [0, 1, 2], [submesh(0, 3)]) + ) + + +def two_file_room(): + """The environment of a bundle of two serialized files that reuse path ids. CAB-a holds the background prefab + (root `room`, GameObject 100, over ROOM_OBJECTS), shader 3, material 5 `a_wall` (main texture its texture 7, + `_Mask` in unity default resources), material 6 `a_floor` (main texture CAB-b's texture 7, `_Detail` its own + texture 7) and texture 7 `wall_a`. CAB-b holds shader 3, material 5 `b_wall` (main texture its texture 7) and + texture 7 `wall_b`: the pixels of `wall_a` with another filter.""" + bundle = SimpleNamespace(files={}) + a = File("CAB-a", bundle, ["archive:/CAB-b/CAB-b", DEFAULT_RESOURCES]) + b = File("CAB-b", bundle) + pixels = Image.new("RGBA", (2, 2), (200, 120, 40, 255)) + for f in (a, b): + f.add("Shader", 3, shader(name="Unlit/Transparent", srcBlend=fv(5.0), destBlend=fv(10.0))) + a.add("Material", 5, textured("a_wall", _MainTex=tex_env(7), _Mask=tex_env(9, 2))) + a.add("Material", 6, textured("a_floor", _MainTex=tex_env(7, 1), _Detail=tex_env(7))) + b.add("Material", 5, textured("b_wall", _MainTex=tex_env(7))) + for f, name, filter_mode in ((a, "wall_a", 1), (b, "wall_b", 0)): + f.add( + "Texture2D", + 7, + { + "m_Name": name, + "m_MipCount": 1, + "m_TextureSettings": {"m_FilterMode": filter_mode, "m_WrapU": 1, "m_WrapV": 1}, + }, + pixels.copy(), + ) + a.add("AssetBundle", 1, {"m_Container": [[PREFAB.lower(), {"asset": ref(100)}]]}) + a.add("GameObject", 100, {"m_Name": "room", "m_IsActive": 1, "m_Layer": 0}) + a.add( + "Transform", + 200, + { + "m_GameObject": ref(100), + "m_Father": ref(0), + "m_LocalPosition": V0, + "m_LocalRotation": Q0, + "m_LocalScale": V1, + "m_Children": [ref(201 + i) for i in range(len(ROOM_OBJECTS))], + }, + ) + for i, (name, mat, file_id) in enumerate(ROOM_OBJECTS): + go, mesh = 101 + i, quad(f"{name}_mesh") + a.add("GameObject", go, {"m_Name": name, "m_IsActive": 1, "m_Layer": 0}) + a.add( + "Transform", + 201 + i, + { + "m_GameObject": ref(go), + "m_Father": ref(200), + "m_Children": [], + "m_LocalPosition": V0, + "m_LocalRotation": Q0, + "m_LocalScale": V1, + }, + ) + a.add( + "MeshFilter", + 301 + i, + {"m_GameObject": ref(go)}, + SimpleNamespace(m_Mesh=SimpleNamespace(path_id=21 + i, read=lambda mesh=mesh: mesh)), + ) + a.add( + "MeshRenderer", + 401 + i, + {"m_GameObject": ref(go), "m_Enabled": 1}, + SimpleNamespace(m_Materials=[a.pptr(mat, file_id)]), + ) + return SimpleNamespace(objects=[o for f in (a, b) for o in f.objects.values()]) + + +class RoomCatalog: + def fetch_key(self, key): + return [] + + def entry(self, key): + return {"internal_id": PREFAB} + + +def load_room(monkeypatch, env): + """UnityPy.load gives `env`; a texture's image is the object it reads as.""" + monkeypatch.setattr(hostspot.UnityPy, "load", lambda *a: env) + monkeypatch.setattr(hostspot, "texture_image", lambda image: image) + + +def glb_json(path): + data = path.read_bytes() + (n,) = struct.unpack_from(" bytes: + buf = io.BytesIO() + image.save(buf, format="PNG") + return buf.getvalue() + + +def test_write_skeleton_writes_the_skeleton_atlas_and_pages_in_order(): + a, b = images() + shared = SpineRef(40, atlas_asset("chr", [Tex(a), Tex(b)])) + got = [] + written = {} + rec = hostspot.write_skeleton(skeleton("chr", b"\x00binary", [shared]), lambda n, d: got.append((n, d)), written) + assert [n for n, _ in got] == ["chr.skel", "chr.atlas", "p1.png", "p2.png"] + assert got[0][1] == b"\x00binary" and got[1][1] == ATLAS_TEXT.encode() + assert got[2][1] == png_of(a) and got[3][1] == png_of(b) + assert rec == { + "name": "chr_SkeletonData", + "skeleton": "chr.skel", + "atlases": ["chr.atlas"], + "scale": 0.01, + "defaultMix": 0.2, + "mixes": [{"from": "a", "to": "b", "duration": 0.5}], + } + assert written == {40: "chr.atlas"} + got.clear() # a second skeleton on the same atlas: written once + rec2 = hostspot.write_skeleton( + skeleton("chr2", ' {"skeleton": 1}', [shared]), lambda n, d: got.append((n, d)), written + ) + assert got == [("chr2.json", b' {"skeleton": 1}')] and rec2["atlases"] == ["chr.atlas"] + + +def test_write_skeleton_keeps_a_name_that_already_has_the_extension(): + a, b = images() + got = [] + shared = SpineRef(40, atlas_asset("chr.atlas", [Tex(a), Tex(b)])) + rec = hostspot.write_skeleton(skeleton("chr.skel", b"\x00binary", [shared]), lambda n, d: got.append((n, d))) + assert [n for n, _ in got] == ["chr.skel", "chr.atlas", "p1.png", "p2.png"] and rec["skeleton"] == "chr.skel" + + +def test_write_skeleton_checks_pages_against_materials(): + a, _ = images() + with pytest.raises(RuntimeError, match="2 pages vs 1 materials"): + hostspot.write_skeleton( + skeleton("x", b"s", [SpineRef(41, atlas_asset("x.atlas", [Tex(a)]))]), lambda n, d: None + ) diff --git a/tests/content/test_room.py b/tests/content/test_room.py index b01a938..f57a36f 100644 --- a/tests/content/test_room.py +++ b/tests/content/test_room.py @@ -1,79 +1,26 @@ -"""room: material translation from the pass render state (property-driven values), empty material slots, and the -glTF textures, images and materials of a bundle whose serialized files reuse path ids (synthetic shader / material -typetrees and objects).""" +"""room: material translation from the pass render state (property-driven values) on the documents of a Material and +its Shader, the glTF writer, and room.glb / room.json of a synthetic background's object table.""" -import copy import json import struct -from types import SimpleNamespace import numpy as np import pytest -from PIL import Image -from UnityPy.classes import PPtr as UnityPPtr from nnnotes.content import room -from nnnotes.unity.reader import DEFAULT_RESOURCES -from tests.helpers.fakeunity import F32, POSITION, UV0, mesh_tt, submesh, unitypy_mesh +from nnnotes.pipeline.assemble import Objects +from nnnotes.pipeline.depot import Depot +from nnnotes.unity.atoms import gltf +from tests.helpers import spotobjects as so +from tests.helpers.spotobjects import fv, material, shader -FIXED = "" +def translate(mat_tt, shader_tt): + return room.translate_material(so.material_json(mat_tt), so.shader_json(shader_tt)) -def fv(val, name=FIXED): - return {"val": val, "name": name} - -def shader(name="Universal Render Pipeline/Lit", props=None, tags=(), **state): - """A shader typetree: one subshader, one pass with render target 0 and the other states given (defaults: - opaque, back-face culling, depth write on, LEqual, all colour channels).""" - rt = { - "srcBlend": fv(1.0), - "destBlend": fv(0.0), - "srcBlendAlpha": fv(1.0), - "destBlendAlpha": fv(0.0), - "blendOp": fv(0.0), - "blendOpAlpha": fv(0.0), - "colMask": fv(15.0), - } - st = {"culling": fv(2.0), "zWrite": fv(1.0), "zTest": fv(4.0), "alphaToMask": fv(0.0)} - for k, v in state.items(): - (rt if k in rt else st)[k] = v - return { - "m_ParsedForm": { - "m_Name": name, - "m_PropInfo": { - "m_Props": [ - { - "m_Name": n, - "m_Type": 2, - "m_DefValue[0]": d, - "m_DefValue[1]": 0.0, - "m_DefValue[2]": 0.0, - "m_DefValue[3]": 0.0, - } - for n, d in (props or {}).items() - ] - }, - "m_SubShaders": [ - { - "m_Tags": {"tags": list(tags)}, - "m_Passes": [{"m_State": {"rtBlend0": rt, **st}, "m_Tags": {"tags": []}}], - } - ], - } - } - - -def material(floats=None, ints=None, name="m"): - return { - "m_Name": name, - "m_SavedProperties": { - "m_TexEnvs": [], - "m_Ints": list((ints or {}).items()), - "m_Floats": list((floats or {}).items()), - "m_Colors": [], - }, - } +def render_state(mat_tt, shader_tt): + return room.render_state(so.material_json(mat_tt), so.shader_json(shader_tt)) LIT_BLEND = dict( @@ -84,26 +31,26 @@ def material(floats=None, ints=None, name="m"): def test_property_driven_blend_takes_the_material_value(): sh = shader(props=LIT_PROPS, **LIT_BLEND) - opaque = room.translate_material(material({"_SrcBlend": 1.0, "_DstBlend": 0.0, "_Cull": 2.0}), sh) + opaque = translate(material({"_SrcBlend": 1.0, "_DstBlend": 0.0, "_Cull": 2.0}), sh) assert "alphaMode" not in opaque and "doubleSided" not in opaque assert opaque["extras"]["unityRenderState"]["srcBlend"] == 1 and opaque["extras"]["unityRenderState"]["zWrite"] == 1 - blended = room.translate_material(material({"_SrcBlend": 5.0, "_DstBlend": 10.0, "_Cull": 0.0, "_ZWrite": 0.0}), sh) + blended = translate(material({"_SrcBlend": 5.0, "_DstBlend": 10.0, "_Cull": 0.0, "_ZWrite": 0.0}), sh) assert blended["alphaMode"] == "BLEND" and blended["doubleSided"] is True assert blended["extras"]["unityRenderState"]["zWrite"] == 0 with pytest.raises(NotImplementedError, match="cull mode 1"): - room.translate_material(material({"_SrcBlend": 1.0, "_DstBlend": 0.0, "_Cull": 1.0}), sh) + translate(material({"_SrcBlend": 1.0, "_DstBlend": 0.0, "_Cull": 1.0}), sh) with pytest.raises(NotImplementedError, match="blend 1/1"): - room.translate_material(material({"_SrcBlend": 1.0, "_DstBlend": 1.0}), sh) + translate(material({"_SrcBlend": 1.0, "_DstBlend": 1.0}), sh) def test_missing_material_value_falls_back_to_the_shader_default_then_the_stored_value(): sh = shader(props=LIT_PROPS, **LIT_BLEND) - rs = room.render_state(material({}), sh) # the material has none of the properties + rs = render_state(material({}), sh) # the material has none of the properties assert (rs["srcBlend"], rs["dstBlend"], rs["cull"], rs["zWrite"]) == (1, 0, 2, 1) - rs = room.render_state(material({}), shader(**LIT_BLEND)) # no shader default either: the stored value + rs = render_state(material({}), shader(**LIT_BLEND)) # no shader default either: the stored value assert (rs["srcBlend"], rs["dstBlend"], rs["cull"], rs["zWrite"]) == (0, 0, 0, 0) # a fixed value ignores a material property of the same name - rs = room.render_state(material({"culling": 0.0, "_Cull": 0.0}), shader(culling=fv(2.0))) + rs = render_state(material({"culling": 0.0, "_Cull": 0.0}), shader(culling=fv(2.0))) assert rs["cull"] == 2 @@ -125,23 +72,23 @@ def test_missing_material_value_falls_back_to_the_shader_default_then_the_stored ) def test_every_state_field_resolves_through_its_property(key, field): sh = shader(props={"_P": 3.0}, **{field: fv(7.0, "_P")}) - assert room.render_state(material({"_P": 6.0}), sh)[key] == 6 # the material's float - assert room.render_state(material(ints={"_P": 5}), sh)[key] == 5 # or int - assert room.render_state(material(), sh)[key] == 3 # the shader's default - assert room.render_state(material(), shader(**{field: fv(7.0, "_P")}))[key] == 7 # the stored value - assert room.render_state(material({"_P": 6.0}), shader(**{field: fv(7.0)}))[key] == 7 # fixed: stored value + assert render_state(material({"_P": 6.0}), sh)[key] == 6 # the material's float + assert render_state(material(ints={"_P": 5}), sh)[key] == 5 # or int + assert render_state(material(), sh)[key] == 3 # the shader's default + assert render_state(material(), shader(**{field: fv(7.0, "_P")}))[key] == 7 # the stored value + assert render_state(material({"_P": 6.0}), shader(**{field: fv(7.0)}))[key] == 7 # fixed: stored value def test_blend_op_other_than_add_is_not_approximated(): sh = shader(name="Unlit/Transparent", srcBlend=fv(5.0), destBlend=fv(10.0), blendOp=fv(2.0, "_BlendOp")) with pytest.raises(NotImplementedError, match="blend op 2"): - room.translate_material(material(), sh) - assert room.translate_material(material({"_BlendOp": 0.0}), sh)["alphaMode"] == "BLEND" + translate(material(), sh) + assert translate(material({"_BlendOp": 0.0}), sh)["alphaMode"] == "BLEND" -def test_fixed_state_translation_is_unchanged(): +def test_fixed_state_translation(): sh = shader(name="Unlit/Transparent Cutout", tags=[("RenderType", "TransparentCutout")]) - cutout = room.translate_material(material({"_Cutoff": 0.5}), sh) + cutout = translate(material({"_Cutoff": 0.5}), sh) assert cutout["alphaMode"] == "MASK" and cutout["alphaCutoff"] == 0.5 assert cutout["extensions"] == {"KHR_materials_unlit": {}} rs = cutout["extras"]["unityRenderState"] @@ -159,280 +106,24 @@ def test_fixed_state_translation_is_unchanged(): "alphaToMask": 0, } with pytest.raises(NotImplementedError, match="shader not translated"): - room.translate_material(material(), shader(name="Custom/Other")) - - -class PPtr: - def __init__(self, path_id): - self.path_id = path_id - - -def test_empty_material_slots_are_skipped_and_reported(): - tris = [np.array([[0, 1, 2]]), np.zeros((0, 3), int), np.array([[2, 3, 0], [0, 1, 3]])] - mats = [PPtr(0), PPtr(11), PPtr(12)] - nulls, used = [], [] - - def material_for(p): - used.append(p.path_id) - return p.path_id * 10 - - prims = room.submesh_primitives(tris, mats, material_for, nulls.append) - assert nulls == [0] and used == [12] # the empty submesh 1 gives nothing, silently - assert len(prims) == 1 and prims[0][1] == 120 - assert prims[0][0].tolist() == [2, 0, 3, 0, 3, 1] # winding reversed for glTF - assert room.submesh_primitives(tris[:1], mats[:1], material_for, nulls.append) == [] and nulls == [0, 0] - - -class Obj: - """A component: its typetree and the object UnityPy reads for it.""" - - def __init__(self, tt, **fields): - self.tt, self.fields = tt, fields + translate(material(), shader(name="Custom/Other")) - def read_typetree(self): - return self.tt - def read(self): - return type("Read", (), self.fields)() - - -class Ref(PPtr): - def __init__(self, path_id, target=None): - super().__init__(path_id) - self.target = target - - def read(self): - assert self.path_id, "an empty reference is not read" - return self.target - - -class Graph: - tf_of_go = {1: 11, 2: 12, 3: 13, 4: 14} - - def path(self, tf_pid): - return f"Room/obj{tf_pid}" - - -def test_a_mesh_filter_without_a_mesh_is_skipped_and_reported(): - mesh = object() - mfs = { - ("f", 1): Obj({}, m_Mesh=Ref(21, mesh)), - ("f", 2): Obj({}, m_Mesh=Ref(0)), - ("f", 3): Obj({}, m_Mesh=Ref(0)), - ("f", 4): Obj({}, m_Mesh=Ref(24, mesh)), - } - mrs = { - ("f", 1): Obj({"m_Enabled": 1}, m_Materials=["m1"]), - ("f", 2): Obj({"m_Enabled": 1}, m_Materials=[]), - ("f", 3): Obj({"m_Enabled": 0}, m_Materials=[]), # 3: disabled renderer, 4: no renderer - ("g", 4): Obj({"m_Enabled": 1}, m_Materials=["m4"]), - } # another file's GameObject 4 - nulls = [] - drawn = list(room.drawn_meshes(mfs, mrs, Graph(), nulls.append)) - assert drawn == [(11, mesh, ["m1"])] - assert nulls == ["Room/obj12"] # the disabled one draws nothing either way - - -def test_extract_room_lists_what_draws_nothing(tmp_path, monkeypatch): - """A mesh without triangles, a MeshFilter without a mesh and a submesh without a material are not written but - listed in room.json.""" - - class Mesh: - def __init__(self, name): - self.m_Name = name - - class Cat: - def fetch_key(self, key): - return [] - - class SceneGraphStub(Graph): - def __init__(self, env): - pass - - def world(self, tf_pid): - return np.eye(4) - - def active(self, tf_pid): - return True - - tri = np.array([[0, 1, 2]]) - arrays = { - "quad": (np.zeros((3, 3)), np.empty((0, 3)), np.zeros((3, 2)), [tri]), - "placeholder": (np.zeros((4, 3)), np.empty((0, 3)), np.zeros((4, 2)), [np.zeros((0, 3), int)]), - "bare": (np.zeros((4, 3)), np.empty((0, 3)), np.zeros((4, 2)), []), - } - - def drawn(mf_by_go, mr_by_go, graph, on_null_mesh): - on_null_mesh("Room/obj14") - yield 11, Mesh("quad"), [PPtr(0)] - yield 12, Mesh("placeholder"), [] - yield 13, Mesh("bare"), [] - - monkeypatch.setattr(room.UnityPy, "load", lambda *a: type("Env", (), {"objects": []})()) - monkeypatch.setattr(room, "SceneGraph", SceneGraphStub) - monkeypatch.setattr(room, "drawn_meshes", drawn) - monkeypatch.setattr(room, "mesh_arrays", lambda mesh: arrays[mesh.m_Name]) - doc = room.extract_room(Cat(), "Spot/a/Background/b", tmp_path / "room.glb") - assert doc["meshCount"] == 0 and doc["materials"] == [] - assert doc["nullMeshFilters"] == ["Room/obj14"] - assert doc["meshesWithoutTriangles"] == [ - {"mesh": "placeholder", "path": "Room/obj12"}, - {"mesh": "bare", "path": "Room/obj13"}, - ] - assert doc["nullMaterialSubmeshes"] == [{"mesh": "quad", "path": "Room/obj11", "submesh": 0}] - assert (tmp_path / "room.glb").is_file() and (tmp_path / "room.json").is_file() - - -# ---------------------------------------------------------------- a bundle whose serialized files reuse path ids -V0, V1 = {"x": 0.0, "y": 0.0, "z": 0.0}, {"x": 1.0, "y": 1.0, "z": 1.0} -Q0 = {"x": 0.0, "y": 0.0, "z": 0.0, "w": 1.0} -ROOM_KEY = "Spot/a/Background/b" -PREFAB = "Assets/Room.prefab" -# the quads under the prefab root: (object name, material path id, its file id in CAB-a: 1 is CAB-b) -ROOM_OBJECTS = [("wall", 5, 0), ("wall_b", 5, 1), ("floor", 6, 0), ("wall_c", 5, 0)] - - -def ref(pid, file_id=0): - return {"m_FileID": file_id, "m_PathID": pid} - - -class File: - """A serialized file as UnityPy's PPtr resolves it: name, objects by path id, externals (paths) and the bundle - holding it (`parent`); a file outside the loaded bundles is not found.""" - - def __init__(self, name, bundle, externals=()): - self.name, self.parent, self.objects = name, bundle, {} - self.externals = [SimpleNamespace(path=p) for p in externals] - self.environment = SimpleNamespace(find_file=lambda name: None) - bundle.files[name] = self - - def add(self, kind, pid, tt, obj=None): - o = SimpleNamespace( - type=SimpleNamespace(name=kind), - path_id=pid, - assets_file=self, - read_typetree=lambda: copy.deepcopy(tt), - read=lambda: obj, - ) - self.objects[pid] = o - return o - - def pptr(self, pid, file_id=0): - """A PPtr as UnityPy parses it from an object of this file.""" - return UnityPPtr(m_FileID=file_id, m_PathID=pid, assetsfile=self) - - -def tex_env(pid, file_id=0): - return {"m_Texture": ref(pid, file_id), "m_Scale": {"x": 1.0, "y": 1.0}, "m_Offset": {"x": 0.0, "y": 0.0}} - - -def textured(name, **slots): - """A material typetree of shader 3 with the texture envs `slots` {slot: tex_env}.""" - tt = material(name=name) - tt.update(m_Shader=ref(3), m_ValidKeywords=[], m_CustomRenderQueue=-1) - tt["m_SavedProperties"]["m_TexEnvs"] = [[k, v] for k, v in slots.items()] - return tt - - -def quad(name): - return unitypy_mesh( - mesh_tt(name, 3, {POSITION: (F32, 3, np.eye(3)), UV0: (F32, 2, np.zeros((3, 2)))}, [0, 1, 2], [submesh(0, 3)]) - ) - - -def two_file_room(): - """The environment of a bundle of two serialized files that reuse path ids. CAB-a holds the background prefab - (root `room`, GameObject 100, over ROOM_OBJECTS), shader 3, material 5 `a_wall` (main texture its texture 7, - `_Mask` in unity default resources), material 6 `a_floor` (main texture CAB-b's texture 7, `_Detail` its own - texture 7) and texture 7 `wall_a`. CAB-b holds shader 3, material 5 `b_wall` (main texture its texture 7) and - texture 7 `wall_b`: the pixels of `wall_a` with another filter.""" - bundle = SimpleNamespace(files={}) - a = File("CAB-a", bundle, ["archive:/CAB-b/CAB-b", DEFAULT_RESOURCES]) - b = File("CAB-b", bundle) - pixels = Image.new("RGBA", (2, 2), (200, 120, 40, 255)) - for f in (a, b): - f.add("Shader", 3, shader(name="Unlit/Transparent", srcBlend=fv(5.0), destBlend=fv(10.0))) - a.add("Material", 5, textured("a_wall", _MainTex=tex_env(7), _Mask=tex_env(9, 2))) - a.add("Material", 6, textured("a_floor", _MainTex=tex_env(7, 1), _Detail=tex_env(7))) - b.add("Material", 5, textured("b_wall", _MainTex=tex_env(7))) - for f, name, filter_mode in ((a, "wall_a", 1), (b, "wall_b", 0)): - f.add( - "Texture2D", - 7, - { - "m_Name": name, - "m_MipCount": 1, - "m_TextureSettings": {"m_FilterMode": filter_mode, "m_WrapU": 1, "m_WrapV": 1}, - }, - pixels.copy(), - ) - a.add("AssetBundle", 1, {"m_Container": [[PREFAB.lower(), {"asset": ref(100)}]]}) - a.add("GameObject", 100, {"m_Name": "room", "m_IsActive": 1, "m_Layer": 0}) - a.add( - "Transform", - 200, - { - "m_GameObject": ref(100), - "m_Father": ref(0), - "m_LocalPosition": V0, - "m_LocalRotation": Q0, - "m_LocalScale": V1, - "m_Children": [ref(201 + i) for i in range(len(ROOM_OBJECTS))], - }, - ) - for i, (name, mat, file_id) in enumerate(ROOM_OBJECTS): - go, mesh = 101 + i, quad(f"{name}_mesh") - a.add("GameObject", go, {"m_Name": name, "m_IsActive": 1, "m_Layer": 0}) - a.add( - "Transform", - 201 + i, - { - "m_GameObject": ref(go), - "m_Father": ref(200), - "m_Children": [], - "m_LocalPosition": V0, - "m_LocalRotation": Q0, - "m_LocalScale": V1, - }, - ) - a.add( - "MeshFilter", - 301 + i, - {"m_GameObject": ref(go)}, - SimpleNamespace(m_Mesh=SimpleNamespace(path_id=21 + i, read=lambda mesh=mesh: mesh)), - ) - a.add( - "MeshRenderer", - 401 + i, - {"m_GameObject": ref(go), "m_Enabled": 1}, - SimpleNamespace(m_Materials=[a.pptr(mat, file_id)]), - ) - return SimpleNamespace(objects=[o for f in (a, b) for o in f.objects.values()]) - - -class RoomCatalog: - def fetch_key(self, key): - return [] - - def entry(self, key): - return {"internal_id": PREFAB} - - -def load_room(monkeypatch, env): - """UnityPy.load gives `env`; a texture's image is the object it reads as.""" - monkeypatch.setattr(room.UnityPy, "load", lambda *a: env) - monkeypatch.setattr(room, "texture_image", lambda image: image) - - -def glb_json(path): - data = path.read_bytes() - (n,) = struct.unpack_from(" Objects: + depot = Depot(tmp_path / "d") + return Objects(so.table(depot, objs or so.room_objects(), [so.ROOM_ROOT]), depot) + + +def test_room_bakes_the_drawn_meshes_and_lists_what_draws_nothing(tmp_path): + glb, doc = room.room(objects_of(tmp_path), so.BACKGROUND) + gdoc, body = gltf.read(glb) + assert [n["name"] for n in gdoc["nodes"]] == ["quad", "quad2", "quad"] # wall, floor, wall2: hierarchy order + assert [n["extras"]["unityActive"] for n in gdoc["nodes"]] == [True, False, True] + assert doc == { + "backgroundKey": so.BACKGROUND, + "glb": "room.glb", + "meshCount": 3, + "inactiveMeshes": 1, + "nullMaterialSubmeshes": [{"mesh": "quad2", "path": "room/floor", "submesh": 1}], + "nullMeshFilters": ["room/nomesh"], + "emptyMeshes": [{"mesh": "empty_mesh", "path": "room/empty"}, {"mesh": "points", "path": "room/points"}], + **{k: doc[k] for k in ("materials", "textures", "samplers")}, + } + assert gdoc["asset"]["extras"] == {"backgroundKey": so.BACKGROUND} + prim = gdoc["meshes"][0]["primitives"][0] + pos = gltf.accessor_array(gdoc, body, prim["attributes"]["POSITION"]) + # the mesh's vertices under the root at (1, 0, 2), in glTF's space (z negated) + expected = np.array(so.VERTICES) + [1.0, 0.0, 2.0] + expected[:, 2] *= -1.0 + assert np.allclose(pos, expected) + assert np.allclose(gltf.accessor_array(gdoc, body, prim["attributes"]["NORMAL"]), [[0.0, 0.0, 1.0]] * 4) + uv = gltf.accessor_array(gdoc, body, prim["attributes"]["TEXCOORD_0"]) + assert np.allclose(uv, [(u, 1.0 - v) for u, v in so.UVS]) # v flipped once + assert gltf.accessor_array(gdoc, body, prim["indices"]).reshape(-1).tolist() == [0, 2, 1, 0, 3, 2] + assert len(gdoc["meshes"][1]["primitives"]) == 1 # the floor's second submesh has no material + + +def test_room_materials_and_textures_are_shared_by_object(tmp_path): + glb, doc = room.room(objects_of(tmp_path), so.BACKGROUND) + gdoc, _ = gltf.read(glb) + assert [m["name"] for m in gdoc["materials"]] == ["a_wall", "a_floor"] + assert [p["material"] for m in gdoc["meshes"] for p in m["primitives"]] == [0, 1, 0] + wall, floor = gdoc["materials"] + assert wall["alphaMode"] == "BLEND" and wall["extensions"] == {"KHR_materials_unlit": {}} + assert wall["pbrMetallicRoughness"]["baseColorTexture"] == {"index": 0} + assert floor["pbrMetallicRoughness"] == { + "baseColorFactor": [1.0, 0.5, 0.25, 1.0], + "metallicFactor": 0.0, + "roughnessFactor": 0.75, + "baseColorTexture": { + "index": 1, + "extensions": {"KHR_texture_transform": {"scale": [2.0, 2.0], "offset": [0.0, -1.5]}}, + }, + } + assert gdoc["extensionsUsed"] == ["KHR_materials_unlit", "KHR_texture_transform"] + assert doc["textures"] == ["wall_a", "floor_b"] and len(gdoc["images"]) == 2 + assert doc["samplers"] == [ + {"magFilter": room.LINEAR, "minFilter": room.LINEAR, "wrapS": room.CLAMP, "wrapT": room.CLAMP}, + {"magFilter": room.NEAREST, "minFilter": room.NEAREST, "wrapS": room.REPEAT, "wrapT": room.REPEAT}, ] - assert [im["name"] for im in gltf["images"]] == ["wall_a"] - assert [s["magFilter"] for s in gltf["samplers"]] == [room.LINEAR, room.NEAREST] - assert doc["textures"] == ["wall_a", "wall_b"] and doc["samplers"] == gltf["samplers"] + assert [m["name"] for m in doc["materials"]] == ["a_wall", "a_floor"] + assert doc["materials"][1]["alphaMode"] == "OPAQUE" and doc["materials"][0]["unlit"] is True + + +def test_a_mesh_the_conversion_failed_on_is_an_error(tmp_path): + objs = so.room_objects() + objs[so.EMPTY] = ("Mesh", "broken", {"json": {}}, {"status": "generic", "reason": "generic.error"}) + with pytest.raises(ValueError, match="room/empty: mesh 'broken' was not converted"): + room.room(objects_of(tmp_path, objs), so.BACKGROUND) + + +def test_a_material_without_its_shader_is_an_error(tmp_path): + objs = so.room_objects() + del objs[so.LIT] + with pytest.raises(ValueError, match="material 'a_floor' has no shader in the object table"): + room.room(objects_of(tmp_path, objs), so.BACKGROUND) + + +def test_room_names(): + assert room.room_name(so.BACKGROUND) == "home_a" + with pytest.raises(ValueError, match="not a spot background key"): + room.room_name(so.SITUATION) diff --git a/tests/content/test_spot.py b/tests/content/test_spot.py index 3048981..19d928c 100644 --- a/tests/content/test_spot.py +++ b/tests/content/test_spot.py @@ -1,188 +1,134 @@ -"""spot: Spine characters and tap targets, with and without the references the prefab leaves empty -(synthetic components), and the Spine files of a skeleton (synthetic objects).""" - -import io +"""spot: spot.json and the Spine files of a synthetic situation's object table, and the spot's master record.""" import pytest -from PIL import Image from nnnotes.content import spot +from nnnotes.pipeline.assemble import Objects +from nnnotes.pipeline.depot import Depot +from tests.helpers import spotobjects as so -class Ref: - def __init__(self, path_id, target=None): - self.path_id, self.target = path_id, target - - def read(self): - assert self.path_id, "an empty reference is not read" - return self.target - - -class Reader: - def __init__(self, tt): - self.tt = tt - - def read_typetree(self): - return self.tt - - -class Read: - def __init__(self, tt=None, **fields): - self.object_reader = Reader(tt) - self.__dict__.update(fields) - - -class Obj: - def __init__(self, tt, **fields): - self.tt, self.fields = tt, fields - - def read_typetree(self): - return self.tt +def objects_of(tmp_path, objs=None) -> Objects: + depot = Depot(tmp_path / "d") + return Objects(so.table(depot, objs or so.situation_objects(), [so.SIT_ROOT]), depot) - def read(self): - return Read(**self.fields) +def translation(x, y, z): + return [1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, x, y, z, 1.0] # column-major -class Graph: - tf_of_go = {1: 11, 2: 12, 3: 13, 4: 14} - def path(self, tf_pid): - return f"Situation/obj{tf_pid}" - - -def go(pid): - return {"m_FileID": 0, "m_PathID": pid} - - -def world_of(go_ref): - return [float(go_ref["m_PathID"])] - - -ANIM = { - "m_GameObject": go(3), - "_animationName": "idle", - "loop": 1, - "timeScale": 1.0, - "initialSkinName": "default", - "initialFlipX": 0, - "initialFlipY": 0, - "pmaVertexColors": 1, - "tintBlack": 0, - "zSpacing": 0.0, -} - - -def test_spine_characters_keep_one_without_animation_as_null(): - anim = Read(ANIM, skeletonDataAsset=Ref(50)) - objs = [Obj({"m_GameObject": go(1)}, _animation=Ref(30, anim)), Obj({"m_GameObject": go(2)}, _animation=Ref(0))] - got = spot.spine_characters(objs, Graph(), {50: {"name": "chr_sd"}}, world_of) - assert got[0] == { - "path": "Situation/obj11", - "skeletonData": "chr_sd", - "animation": {k: v for k, v in ANIM.items() if k != "m_GameObject"}, - "world": [3.0], +def test_master_record(): + rec = spot.master_record(so.MASTER, so.SITUATION) + assert rec == { + "row": so.MASTER["MasterHomeSpot"][0], + "name": { + "japanese": "カフェ", + "english": "Cafe", + "traditionalChinese": "", + "simplifiedChinese": "", + "korean": "", + }, + "tapTalk": {"5": [51], "7": [71, 72]}, } - assert got[1] == {"path": "Situation/obj12", "skeletonData": None, "animation": None, "world": None} - - -def test_tap_targets_without_focus_have_a_null_focus_world(): - focus = Read({"m_GameObject": go(3)}) - objs = [ - Obj({"m_GameObject": go(2), "m_Enabled": 1, "_characterId": 7, "_focus": go(0)}, _focus=Ref(0)), - Obj({"m_GameObject": go(1), "m_Enabled": 1, "_characterId": 5, "_focus": go(33)}, _focus=Ref(33, focus)), - ] - got = spot.tap_targets(objs, Graph(), world_of) - assert [c["_characterId"] for c in got] == [5, 7] - assert got[0] == {"path": "Situation/obj11", "_characterId": 5, "world": [1.0], "focusWorld": [3.0]} - assert got[1] == {"path": "Situation/obj12", "_characterId": 7, "world": [2.0], "focusWorld": None} - - -# ---------------------------------------------------------------- Spine files (synthetic objects) -class Text: - def __init__(self, name, script): - self.m_Name, self.m_Script = name, script - - -class Tex: - m_TextureFormat = 4 # RGBA32 - - def __init__(self, image): - self.m_Name, self.image = "page", image - - -class TexEnv: - def __init__(self, tex): - self.m_Texture = Ref(1, tex) - - -class Material: - def __init__(self, tex): - self.m_SavedProperties = Read(m_TexEnvs=[("_BumpMap", TexEnv(None)), ("_MainTex", TexEnv(tex))]) - - -ATLAS_TEXT = "\np1.png\nsize: 2,2\nformat: RGBA8888\nr\n bounds: 0,0,1,1\n\np2.png\nsize: 1,1\nr2\n bounds: 0,0,1,1\n" - - -def atlas_asset(name, pages): - return Read( - atlasFile=Ref(20, Text(name, ATLAS_TEXT)), materials=[Ref(30 + i, Material(t)) for i, t in enumerate(pages)] - ) - - -def skeleton(name, script, atlases): - tt = { - "m_Name": name + "_SkeletonData", - "scale": 0.01, - "defaultMix": 0.2, - "fromAnimation": ["a"], - "toAnimation": ["b"], - "duration": [0.5], + with pytest.raises(ValueError, match="0 MasterHomeSpot rows"): + spot.master_record(so.MASTER, "Spot/c/c_01/c_01") + + +def test_spot_document(tmp_path): + doc, files = spot.spot(objects_of(tmp_path), so.SITUATION, spot.master_record(so.MASTER, so.SITUATION)) + assert doc == { + "spotId": 3, + "master": so.MASTER["MasterHomeSpot"][0], + "name": doc["name"], + "tapTalk": {"5": [51], "7": [71, 72]}, + "situationSettings": {"backgroundPosition": so.V0, "cameraFov": 30.0}, + "controller": {"_speed": 2.0}, + "characters": [ # by character id; the root at (0, 0, 1) + { + "path": "sit/chr5", + "_characterId": 5, + "world": translation(2.0, 0.0, 1.0), + "focusWorld": translation(2.0, 1.0, 1.0), + }, + { + "path": "sit/chr7", + "_characterId": 7, + "_radius": 0.5, + "world": translation(0.0, 0.0, 1.0), + "focusWorld": None, + }, + ], + "spineCharacters": [ # hierarchy order + {"path": "sit/chr7", "skeletonData": None, "animation": None, "world": None}, + { + "path": "sit/chr5", + "skeletonData": "chr_SkeletonData", + "animation": so.ANIMATION, + "world": translation(2.0, 0.0, 1.0), + }, + ], + "skeletons": [ # by name; the shared atlas once + { + "name": "chr2_SkeletonData", + "skeleton": "chr2.json", + "atlases": ["chr.atlas"], + "scale": 0.01, + "defaultMix": 0.2, + "mixes": [{"from": "a", "to": "b", "duration": 0.5}], + }, + { + "name": "chr_SkeletonData", + "skeleton": "chr.skel", + "atlases": ["chr.atlas"], + "scale": 0.01, + "defaultMix": 0.2, + "mixes": [{"from": "a", "to": "b", "duration": 0.5}], + }, + ], + "matrixConvention": spot.MATRIX_CONVENTION, + } + objs = so.situation_objects() + assert files == { + "spine/chr2.json": b' {"skeleton": 1}', + "spine/chr.atlas": so.ATLAS_TEXT.encode(), + "spine/p1.png": objs[so.PAGE1][2]["image"], + "spine/p2.png": objs[so.PAGE2][2]["image"], + "spine/chr.skel": b"\x00binary", } - return Obj(tt, skeletonJSON=Ref(9, Text(name, script)), atlasAssets=atlases) -def images(): - return Image.new("RGBA", (2, 2), (1, 2, 3, 4)), Image.new("RGBA", (1, 1), (9, 8, 7, 6)) +def test_a_skeleton_name_with_its_extension_is_kept(tmp_path): + objs = so.situation_objects() + cls, name, files, extra = objs[so.SKEL] + objs[so.SKEL] = (cls, name, {**files, "json": {"m_Name": "chr.skel"}}, extra) + _, files = spot.spot(objects_of(tmp_path, objs), so.SITUATION, spot.master_record(so.MASTER, so.SITUATION)) + assert "spine/chr.skel" in files and "spine/chr.skel.skel" not in files -def png(image) -> bytes: - buf = io.BytesIO() - image.save(buf, format="PNG") - return buf.getvalue() +def test_atlas_pages_match_its_materials(tmp_path): + objs = so.situation_objects() + cls, name, files, extra = objs[so.ATLAS] + objs[so.ATLAS] = (cls, name, {"json": {**files["json"], "materials": files["json"]["materials"][:1]}}, extra) + with pytest.raises(ValueError, match="chr.atlas: 2 pages vs 1 materials"): + spot.spot(objects_of(tmp_path, objs), so.SITUATION, spot.master_record(so.MASTER, so.SITUATION)) -def test_write_skeleton_writes_the_skeleton_atlas_and_pages_in_order(): - a, b = images() - shared = Ref(40, atlas_asset("chr", [Tex(a), Tex(b)])) - got = [] - written = {} - rec = spot.write_skeleton(skeleton("chr", b"\x00binary", [shared]), lambda n, d: got.append((n, d)), written) - assert [n for n, _ in got] == ["chr.skel", "chr.atlas", "p1.png", "p2.png"] - assert got[0][1] == b"\x00binary" and got[1][1] == ATLAS_TEXT.encode() - assert got[2][1] == png(a) and got[3][1] == png(b) - assert rec == { - "name": "chr_SkeletonData", - "skeleton": "chr.skel", - "atlases": ["chr.atlas"], - "scale": 0.01, - "defaultMix": 0.2, - "mixes": [{"from": "a", "to": "b", "duration": 0.5}], - } - assert written == {40: "chr.atlas"} - got.clear() # a second skeleton on the same atlas: written once - rec2 = spot.write_skeleton(skeleton("chr2", ' {"skeleton": 1}', [shared]), lambda n, d: got.append((n, d)), written) - assert got == [("chr2.json", b' {"skeleton": 1}')] and rec2["atlases"] == ["chr.atlas"] +def test_a_situation_without_its_settings_is_an_error(tmp_path): + objs = so.situation_objects() + cls, name, files, extra = objs[so.SIT_ROOT] + doc = files["json"] + root = {**doc["nodes"][0], "components": doc["nodes"][0]["components"][1:]} + objs[so.SIT_ROOT] = (cls, name, {"json": {**doc, "nodes": [root, *doc["nodes"][1:]]}}, extra) + with pytest.raises(ValueError, match="no SpotSituationSettings in the situation prefab"): + spot.spot(objects_of(tmp_path, objs), so.SITUATION, spot.master_record(so.MASTER, so.SITUATION)) -def test_write_skeleton_keeps_a_name_that_already_has_the_extension(): - a, b = images() - got = [] - shared = Ref(40, atlas_asset("chr.atlas", [Tex(a), Tex(b)])) - rec = spot.write_skeleton(skeleton("chr.skel", b"\x00binary", [shared]), lambda n, d: got.append((n, d))) - assert [n for n, _ in got] == ["chr.skel", "chr.atlas", "p1.png", "p2.png"] and rec["skeleton"] == "chr.skel" +def test_situation_names(): + assert spot.situation_name(so.SITUATION) == "home_a_01" + for key in (so.BACKGROUND, "Spot/a/b/c"): + with pytest.raises(ValueError, match="not a spot situation key"): + spot.situation_name(key) -def test_write_skeleton_checks_pages_against_materials(): - a, _ = images() - with pytest.raises(RuntimeError, match="2 pages vs 1 materials"): - spot.write_skeleton(skeleton("x", b"s", [Ref(41, atlas_asset("x.atlas", [Tex(a)]))]), lambda n, d: None) +def test_atlas_pages(): + assert spot.atlas_pages(so.ATLAS_TEXT) == ["p1.png", "p2.png"] diff --git a/tests/content/test_story_host.py b/tests/content/test_story_host.py index f1b01d2..57e9462 100644 --- a/tests/content/test_story_host.py +++ b/tests/content/test_story_host.py @@ -9,9 +9,9 @@ import numpy as np import pytest -from nnnotes.content import advui, room, storyhost +from nnnotes.content import advui, hostspot, storyhost from nnnotes.unity.reader import SceneGraph -from tests.content.test_room import ROOM_KEY, File, RoomCatalog, load_room, two_file_room +from tests.content.test_hostspot import ROOM_KEY, File, RoomCatalog, load_room, two_file_room V0 = {"x": 0.0, "y": 0.0, "z": 0.0} V1 = {"x": 1.0, "y": 1.0, "z": 1.0} @@ -153,7 +153,7 @@ def test_glb_json_reads_the_json_chunk(tmp_path): storyhost.glb_json(tmp_path / "x.glb") -def test_glb_order_follows_room_extract_room(): +def test_glb_order_follows_hostspot_extract_room(): tri = [[0, 1, 2]] drawn = [ (1, Mesh("a", [tri, tri]), [pp(20), pp(10)]), @@ -165,7 +165,7 @@ def test_glb_order_follows_room_extract_room(): ] # path id 10 of another file: another material nodes, mats = storyhost.glb_order(drawn, arrays) assert nodes == [(1, "a"), (5, "b"), (6, "c")] - assert [room.object_key(m.deref()) for m in mats] == [("CAB-a", 20), ("CAB-a", 10), ("CAB-b", 10)] + assert [hostspot.object_key(m.deref()) for m in mats] == [("CAB-a", 20), ("CAB-a", 10), ("CAB-b", 10)] with pytest.raises(NotImplementedError, match="2 submeshes, 1 materials"): storyhost.glb_order([(1, Mesh("c", [tri, tri]), [pp(1)])], arrays) @@ -226,11 +226,11 @@ def test_glb_textures_and_room_materials_keep_the_glb_order(): assert storyhost.room_material(oa, empty, "U", index)["texEnvs"]["_MainTex"]["texture"] is None -def test_background_numbers_the_room_as_room_extract_room(tmp_path, monkeypatch): +def test_background_numbers_the_room_as_hostspot_extract_room(tmp_path, monkeypatch): """Materials and textures of two serialized files that reuse path ids: the background record and room.glb agree (check_room), and a texture slot names the glTF texture, not the image.""" load_room(monkeypatch, two_file_room()) - doc = room.extract_room(RoomCatalog(), ROOM_KEY, tmp_path / "room.glb") + doc = hostspot.extract_room(RoomCatalog(), ROOM_KEY, tmp_path / "room.glb") gltf = storyhost.glb_json(tmp_path / "room.glb") bg = storyhost.background(RoomCatalog(), ROOM_KEY, "situation(Clone)", None) storyhost.check_room(ROOM_KEY, gltf, doc, bg) diff --git a/tests/helpers/spotobjects.py b/tests/helpers/spotobjects.py new file mode 100644 index 0000000..29ff81d --- /dev/null +++ b/tests/helpers/spotobjects.py @@ -0,0 +1,357 @@ +"""A synthetic spot as products' object tables (nnnotes.objects/1) whose files are in a depot: a background prefab +(content.room) with drawn, inactive, hidden, mesh-less and empty meshes, two materials of two shaders and their +textures; a situation prefab (content.spot) with tap targets, Spine characters, two skeletons sharing one atlas; and +the master tables of the spot. Shaders and materials are made as typetrees and turned into their object documents +(formats.shader.parsed_summary, unity.convert.material.material_doc).""" + +from __future__ import annotations + +import io + +import numpy as np +from PIL import Image + +from nnnotes.formats.shader import parsed_summary +from nnnotes.pipeline import contract +from nnnotes.unity.atoms import gltf +from nnnotes.unity.atoms.mesh import arrays +from nnnotes.unity.atoms.reader import mesh_input_from_typetree +from nnnotes.unity.convert.material import material_doc +from tests.helpers.fakeunity import F32, NORMAL, POSITION, UV0, VERSION, mesh_tt, submesh + +FIXED = "" +BACKGROUND = "Spot/home_a/Background/home_a" +SITUATION = "Spot/home_a/home_a_01/home_a_01" +CAB, SIT = "CAB-r", "CAB-s" +ROOM_ROOT, QUAD, QUAD2, POINTS, EMPTY = (f"{CAB}:{i}" for i in (1, 10, 11, 12, 13)) +MAT_WALL, MAT_FLOOR, UNLIT, LIT, TEX_WALL, TEX_FLOOR = (f"{CAB}:{i}" for i in (20, 21, 30, 31, 40, 41)) +SIT_ROOT, SDA, SDA2, ATLAS, SKEL, SKEL2, ATLAS_TEXT_ASSET = (f"{SIT}:{i}" for i in (1, 50, 51, 52, 60, 61, 62)) +PAGE_MAT1, PAGE_MAT2, PAGE1, PAGE2 = (f"{SIT}:{i}" for i in (70, 71, 80, 81)) +SPINE = "spine-unity|Spine.Unity|" +GAME = "Assembly-CSharp||" +ATLAS_TEXT = "\np1.png\nsize: 2,2\nformat: RGBA8888\nr\n bounds: 0,0,1,1\n\np2.png\nsize: 1,1\nr2\n bounds: 0,0,1,1\n" +V0, V1, Q0 = {"x": 0.0, "y": 0.0, "z": 0.0}, {"x": 1.0, "y": 1.0, "z": 1.0}, {"x": 0.0, "y": 0.0, "z": 0.0, "w": 1.0} +ROOM_OFFSET = {"x": 1.0, "y": 0.0, "z": 2.0} # the background root's own position +VERTICES = [(0.0, 0.0, 0.5), (1.0, 0.0, 0.5), (1.0, 1.0, 0.5), (0.0, 1.0, 0.5)] +UVS = [(0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 1.0)] + + +def ref(oid: str | None, cls: str | None = None): + if oid is None: + return None + file, _, pid = oid.rpartition(":") + r = {"file": file, "pathId": pid} + if cls: + r["class"] = cls + return {"$ref": r} + + +# ---------------------------------------------------------------- shader and material typetrees +def fv(val, name=FIXED): + return {"val": val, "name": name} + + +def shader(name="Universal Render Pipeline/Lit", props=None, tags=(), **state): + """A shader typetree: one subshader, one pass with render target 0 and the other states given (defaults: + opaque, back-face culling, depth write on, LEqual, all colour channels).""" + rt = { + "srcBlend": fv(1.0), + "destBlend": fv(0.0), + "srcBlendAlpha": fv(1.0), + "destBlendAlpha": fv(0.0), + "blendOp": fv(0.0), + "blendOpAlpha": fv(0.0), + "colMask": fv(15.0), + } + st = {"culling": fv(2.0), "zWrite": fv(1.0), "zTest": fv(4.0), "alphaToMask": fv(0.0)} + for k, v in state.items(): + (rt if k in rt else st)[k] = v + return { + "m_ParsedForm": { + "m_Name": name, + "m_PropInfo": { + "m_Props": [ + { + "m_Name": n, + "m_Type": 2, + "m_DefValue[0]": d, + "m_DefValue[1]": 0.0, + "m_DefValue[2]": 0.0, + "m_DefValue[3]": 0.0, + } + for n, d in (props or {}).items() + ] + }, + "m_SubShaders": [ + { + "m_Tags": {"tags": list(tags)}, + "m_Passes": [{"m_State": {"rtBlend0": rt, **st}, "m_Tags": {"tags": []}}], + } + ], + } + } + + +def material(floats=None, ints=None, name="m", textures=None, colors=None, shader_ref=None): + """A material typetree; `textures` {slot: (texture reference, scale (x, y), offset (x, y))}.""" + return { + "m_Name": name, + "m_Shader": shader_ref, + "m_SavedProperties": { + "m_TexEnvs": [ + [k, {"m_Texture": t, "m_Scale": {"x": s[0], "y": s[1]}, "m_Offset": {"x": o[0], "y": o[1]}}] + for k, (t, s, o) in (textures or {}).items() + ], + "m_Ints": list((ints or {}).items()), + "m_Floats": list((floats or {}).items()), + "m_Colors": list((colors or {}).items()), + }, + } + + +def material_json(tt: dict) -> dict: + """The `json` of a Material of typetree `tt` (its references already {"$ref"}).""" + return material_doc(tt, lambda r: r) + + +def shader_json(tt: dict) -> dict: + return parsed_summary(tt) + + +# ---------------------------------------------------------------- object files +def png(color, size=(2, 2)) -> bytes: + buf = io.BytesIO() + Image.new("RGBA", size, color).save(buf, format="PNG") + return buf.getvalue() + + +def mesh_glb(name: str, indices=(), submeshes=()) -> bytes: + """The `mesh` of a Mesh: four vertices (VERTICES, UVS, normals -z) with `indices` in `submeshes`.""" + ch = { + POSITION: (F32, 3, np.array(VERTICES)), + NORMAL: (F32, 3, np.tile([0.0, 0.0, -1.0], (4, 1))), + UV0: (F32, 2, np.array(UVS)), + } + return gltf.write([arrays(mesh_input_from_typetree(mesh_tt(name, 4, ch, list(indices), list(submeshes)), VERSION))]) + + +def texture(name: str, filter_mode: int, wrap: int) -> dict: + return { + "m_Name": name, + "m_MipCount": 1, + "m_TextureSettings": {"m_FilterMode": filter_mode, "m_WrapU": wrap, "m_WrapV": wrap}, + } + + +def node(path: str, *components, position=V0, active=True) -> dict: + return { + "path": path, + "name": path.rsplit("/", 1)[-1], + "active": active, + "localPosition": position, + "localRotation": Q0, + "localScale": V1, + "components": list(components), + } + + +def component(type_: str, **fields) -> dict: + return {"type": type_, "pathId": "0", "fields": fields} + + +def mono(script: str, **fields) -> dict: + return {"type": "MonoBehaviour", "pathId": "0", "script": script, "fields": fields} + + +def drawn(path: str, mesh: str | None, materials, enabled=1, **kw) -> dict: + return node( + path, + component("MeshFilter", m_Mesh=ref(mesh, "Mesh")), + component("MeshRenderer", m_Enabled=enabled, m_Materials=[ref(m, "Material") for m in materials]), + **kw, + ) + + +# ---------------------------------------------------------------- the background +def room_objects() -> dict: + """{object id: (class, name, files, entry extras)} of the background BACKGROUND: under the root `room` (at + ROOM_OFFSET) `wall` (QUAD, MAT_WALL), `floor` (inactive, QUAD2: two submeshes, the second without a material), + `wall2` (QUAD and MAT_WALL again), `hidden` (a disabled renderer), `nomesh` (no mesh), `empty` (a mesh the + conversion left empty) and `points` (a mesh without an index buffer).""" + prefab = { + "name": "room", + "nodes": [ + node("room", position=ROOM_OFFSET), + drawn("room/wall", QUAD, [MAT_WALL]), + drawn("room/floor", QUAD2, [MAT_FLOOR, None], active=False), + drawn("room/wall2", QUAD, [MAT_WALL]), + drawn("room/hidden", QUAD, [MAT_WALL], enabled=0), + drawn("room/nomesh", None, []), + drawn("room/empty", EMPTY, [MAT_WALL]), + drawn("room/points", POINTS, [MAT_WALL]), + ], + } + unlit = shader(name="Unlit/Transparent", srcBlend=fv(5.0), destBlend=fv(10.0)) + wall = material( + name="a_wall", + shader_ref=ref(UNLIT, "Shader"), + textures={"_MainTex": (ref(TEX_WALL, "Texture2D"), (1.0, 1.0), (0.0, 0.0)), "_Mask": (None, (1, 1), (0, 0))}, + ) + floor = material( + name="a_floor", + shader_ref=ref(LIT, "Shader"), + textures={"_BaseMap": (ref(TEX_FLOOR, "Texture2D"), (2.0, 2.0), (0.0, 0.5))}, + floats={"_Metallic": 0.0, "_Smoothness": 0.25}, + colors={"_BaseColor": {"r": 1.0, "g": 0.5, "b": 0.25, "a": 1.0}}, + ) + return { + ROOM_ROOT: ("GameObject", "room", {"json": prefab}, {}), + QUAD: ("Mesh", "quad", {"mesh": mesh_glb("quad", [0, 1, 2, 0, 2, 3], [submesh(0, 6)])}, {}), + QUAD2: ("Mesh", "quad2", {"mesh": mesh_glb("quad2", [0, 1, 2, 0, 2, 3], [submesh(0, 3), submesh(3, 3)])}, {}), + POINTS: ("Mesh", "points", {"mesh": mesh_glb("points")}, {}), + EMPTY: ( + "Mesh", + "empty_mesh", + {"json": {"m_Name": "empty_mesh"}}, + {"status": "generic", "reason": "empty.mesh"}, + ), + MAT_WALL: ("Material", "a_wall", {"json": material_json(wall)}, {}), + MAT_FLOOR: ("Material", "a_floor", {"json": material_json(floor)}, {}), + UNLIT: ("Shader", "Unlit/Transparent", {"json": shader_json(unlit)}, {}), + LIT: ("Shader", "Universal Render Pipeline/Lit", {"json": shader_json(shader())}, {}), + TEX_WALL: ("Texture2D", "wall_a", {"json": texture("wall_a", 1, 1), "image": png((200, 120, 40, 255))}, {}), + TEX_FLOOR: ("Texture2D", "floor_b", {"json": texture("floor_b", 0, 0), "image": png((9, 8, 7, 255))}, {}), + } + + +# ---------------------------------------------------------------- the situation +ANIMATION = { + "_animationName": "idle", + "loop": 1, + "timeScale": 1.0, + "initialSkinName": "default", + "initialFlipX": 0, + "initialFlipY": 0, + "pmaVertexColors": 1, + "tintBlack": 0, + "zSpacing": 0.0, +} +HEADER = {"m_Enabled": 1, "m_Script": ref("CAB-x:9", "MonoScript"), "m_Name": ""} +SITUATION_OFFSET = {"x": 0.0, "y": 0.0, "z": 1.0} + + +def skeleton_data(name: str, skeleton: str) -> dict: + return { + "$script": SPINE + "SkeletonDataAsset", + "m_Name": name, + "skeletonJSON": ref(skeleton, "TextAsset"), + "atlasAssets": [ref(ATLAS, "MonoBehaviour")], + "scale": 0.01, + "defaultMix": 0.2, + "fromAnimation": ["a"], + "toAnimation": ["b"], + "duration": [0.5], + } + + +def situation_objects() -> dict: + """{object id: (class, name, files, entry extras)} of the situation SITUATION: the root `sit` (at + SITUATION_OFFSET) with SpotSituationSettings and SpotCharacterController; `sit/chr7` (character 7, no focus, a + Spine character without animation) before `sit/chr5` (character 5 at (2, 0, 0), focus `sit/chr5/focus` at + (0, 1, 0), a Spine character driving `sit/chr5/spine`'s SkeletonAnimation of SDA); skeletons SDA (binary .skel) + and SDA2 (JSON) sharing the two-page atlas ATLAS.""" + prefab = { + "name": "sit", + "nodes": [ + node( + "sit", + mono(GAME + "SpotSituationSettings", **HEADER, backgroundPosition=V0, cameraFov=30.0), + mono( + GAME + "SpotCharacterController", + **HEADER, + _spotSpineCharacters=[{"component": "MonoBehaviour", "gameObject": "sit/chr5"}], + _characters=[], + _speed=2.0, + ), + position=SITUATION_OFFSET, + ), + node( + "sit/chr7", + mono(GAME + "SpotCharacter", **HEADER, _characterId=7, _focus=None, _radius=0.5), + mono(GAME + "SpotSpineCharacter", **HEADER, _animation=None), + ), + node( + "sit/chr5", + mono(GAME + "SpotCharacter", **HEADER, _characterId=5, _focus={"transform": "sit/chr5/focus"}), + mono( + GAME + "SpotSpineCharacter", + **HEADER, + _animation={ + "component": "MonoBehaviour", + "gameObject": "sit/chr5/spine", + "class": "SkeletonAnimation", + }, + ), + position={"x": 2.0, "y": 0.0, "z": 0.0}, + ), + node("sit/chr5/focus", position={"x": 0.0, "y": 1.0, "z": 0.0}), + node( + "sit/chr5/spine", + mono(SPINE + "SkeletonAnimation", **HEADER, skeletonDataAsset=ref(SDA, "MonoBehaviour"), **ANIMATION), + ), + ], + } + page = lambda tex: {"json": material_json(material(textures={"_MainTex": (ref(tex, "Texture2D"), (1, 1), (0, 0))}))} # noqa: E731 + return { + SIT_ROOT: ("GameObject", "sit", {"json": prefab}, {}), + SDA: ("MonoBehaviour", "chr_SkeletonData", {"json": skeleton_data("chr_SkeletonData", SKEL)}, {}), + SDA2: ("MonoBehaviour", "chr2_SkeletonData", {"json": skeleton_data("chr2_SkeletonData", SKEL2)}, {}), + ATLAS: ( + "MonoBehaviour", + "chr_Atlas", + { + "json": { + "$script": SPINE + "SpineAtlasAsset", + "m_Name": "chr_Atlas", + "atlasFile": ref(ATLAS_TEXT_ASSET, "TextAsset"), + "materials": [ref(PAGE_MAT1, "Material"), ref(PAGE_MAT2, "Material")], + } + }, + {}, + ), + SKEL: ("TextAsset", "chr", {"json": {"m_Name": "chr"}, "data": b"\x00binary"}, {}), + SKEL2: ("TextAsset", "chr2", {"json": {"m_Name": "chr2"}, "data": b' {"skeleton": 1}'}, {}), + ATLAS_TEXT_ASSET: ("TextAsset", "chr", {"json": {"m_Name": "chr"}, "data": ATLAS_TEXT.encode()}, {}), + PAGE_MAT1: ("Material", "p1", page(PAGE1), {}), + PAGE_MAT2: ("Material", "p2", page(PAGE2), {}), + PAGE1: ("Texture2D", "p1", {"json": texture("p1", 1, 1), "image": png((1, 2, 3, 4))}, {}), + PAGE2: ("Texture2D", "p2", {"json": texture("p2", 1, 1), "image": png((9, 8, 7, 6), (1, 1))}, {}), + } + + +MASTER = { + "MasterHomeSpot": [ + {"_id": 3, "_nameTextId": 900, "_backgroundAssetPath": BACKGROUND, "_situationAssetPath": SITUATION}, + {"_id": 4, "_nameTextId": 901, "_backgroundAssetPath": BACKGROUND, "_situationAssetPath": "Spot/b/b_01/b_01"}, + ], + "MasterText": [{"_id": 900, "_japanese": "カフェ", "_english": "Cafe"}], + "MasterStoryHomeSpotTapTalkEpisode": [ + {"_spotId": 3, "_characterId": 7, "_advId": 71}, + {"_spotId": 4, "_characterId": 7, "_advId": 99}, + {"_spotId": 3, "_characterId": 5, "_advId": 51}, + {"_spotId": 3, "_characterId": 7, "_advId": 72}, + ], +} + + +def table(depot, objs: dict, roots) -> dict: + """The object table of `objs` (room_objects(), situation_objects()), every file put in `depot`.""" + out = {} + for oid, (cls, name, files, extra) in sorted(objs.items()): + recs = {} + for role, body in sorted(files.items()): + data = body if isinstance(body, bytes) else contract.encode(body) + ext = {"image": "png", "mesh": "glb"}.get(role, "bin") if isinstance(body, bytes) else "json" + recs[role] = {"sha256": depot.put(data), "size": len(data), "ext": ext} + out[oid] = {"status": "exported", "class": cls, "name": name, "files": recs, **extra} + return {"schema": contract.OBJECTS, "roots": sorted(roots), "objects": out, "missing": []} diff --git a/tests/pipeline/test_modelstages.py b/tests/pipeline/test_modelstages.py deleted file mode 100644 index 5edc140..0000000 --- a/tests/pipeline/test_modelstages.py +++ /dev/null @@ -1,295 +0,0 @@ -"""The Spine skeleton stage: its subjects, closures, keys and runs with synthetic censuses and objects.""" - -import subprocess -import sys -from pathlib import Path - -import pytest - -from nnnotes.pipeline import census, contract, link, modelstages -from nnnotes.pipeline.census import CensusStage -from nnnotes.pipeline.contract import Input, Task -from nnnotes.pipeline.depot import Depot -from nnnotes.pipeline.modelstages import SpineStage -from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry - - -def test_the_libraries_are_those_of_the_extractor(): - from nnnotes.unity import export - - assert modelstages.LIBRARIES == export._LIBS - - -def test_importing_the_stages_does_not_import_unitypy(): - code = ( - "import sys, nnnotes.pipeline.modelstages as m; m.SpineStage(); " - "assert 'UnityPy' not in sys.modules, 'UnityPy imported'" - ) - r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) - assert r.returncode == 0, r.stderr[-2000:] - - -# ---------------------------------------------------------------- spine.skeleton -CAB = {k: f"CAB-{c * 32}" for k, c in zip(("a", "t", "s", "d", "e", "m", "f"), "1234567")} - - -def obj(pid, cls, name=None, script=None): - o = {"pathId": pid, "classId": 0, "class": cls, "byteSize": 10, "typeHash": None} - if name is not None: - o["name"] = name - if script is not None: - o["script"] = {"file": CAB["m"], "pathId": script} - return o - - -def cdoc(files, scripts=()): - """A census document: files [(name, externals, objects)].""" - fs = [ - { - "name": n, - "unityVersion": "6000.0.0f1", - "platform": 13, - "typetree": True, - "externals": [{"file": x, "path": f"archive:/{x}/{x}", "guid": None, "type": 0} for x in ext], - "objects": objs, - } - for n, ext, objs in files - ] - classes = {} - for f in fs: - for o in f["objects"]: - classes[o["class"]] = classes.get(o["class"], 0) + 1 - return { - "schema": contract.CENSUS, - "files": fs, - "resources": [], - "assetBundles": [], - "resourcePaths": [], - "scripts": list(scripts), - "classes": classes, - "objects": sum(classes.values()), - "errors": [], - } - - -def spine_censuses(): - sda, atlas = 1, 2 - return { - "spine_a": cdoc( - [ - ( - CAB["a"], - [CAB["t"], CAB["m"], "unity default resources"], - [ - obj(10, "MonoBehaviour", "chr_SkeletonData", sda), - obj(11, "MonoBehaviour", "chr_Atlas", atlas), - obj(12, "TextAsset", "chr"), - ], - ) - ] - ), - "tex_b": cdoc([(CAB["t"], [CAB["s"]], [obj(1, "Texture2D", "chr"), obj(2, "Material", "chr_Material")])]), - "shader_c": cdoc([(CAB["s"], [], [obj(1, "Shader", "Spine/Skeleton")])]), - "other_d": cdoc([(CAB["d"], [], [obj(5, "MonoBehaviour", "x_SkeletonData", sda)])]), - "scene_e": cdoc( - [ - (CAB["e"], [], [obj(7, "GameObject", "root")]), - (CAB["e"] + ".sharedAssets", [CAB["t"]], [obj(7, "MonoBehaviour", "y_SkeletonData", sda)]), - ] - ), - "scripts": cdoc( - [(CAB["m"], [], [obj(1, "MonoScript", "SkeletonDataAsset"), obj(2, "MonoScript", "SpineAtlasAsset")])], - scripts=[ - { - "file": CAB["m"], - "pathId": 1, - "assembly": "spine-unity.dll", - "namespace": "Spine.Unity", - "class": "SkeletonDataAsset", - "name": "SkeletonDataAsset", - }, - { - "file": CAB["m"], - "pathId": 2, - "assembly": "spine-unity.dll", - "namespace": "Spine.Unity", - "class": "SpineAtlasAsset", - "name": "SpineAtlasAsset", - }, - ], - ), - "unsel_f": cdoc([(CAB["f"], [], [obj(3, "MonoBehaviour", "z_SkeletonData", sda)])]), - } - - -def census_result(depot, env, subject, doc, body): - inp = Input("bundle", contract.sha256(body), len(body), subject + ".bundle", ()) - t = describe(CensusStage(), subject, None, Env(depot, {"bundles": {subject: inp}})) - rec = contract.artifact( - contract.artifact_id(t.id, "census"), - depot.add(contract.encode(doc), "json"), - contract.provenance(t), - {"kind": "unity.census", "format": "json", "facts": census.facts(doc)}, - ) - depot.commit(contract.result(t, [rec], [])) - env.done(t.id, t.key) - return inp - - -def spine_env(depot, docs=None, data=None, selected=("other_d", "scene_e", "spine_a")): - docs = docs or spine_censuses() - data = data or {} - env = Env(depot, {"selected": sorted(selected)}) - env.facts["bundles"] = { - s: census_result(depot, env, s, d, data.get(s, b"UnityFS " + s.encode())) for s, d in sorted(docs.items()) - } - st = describe(link.ScriptsStage(), "all", None, env) - execute(st, depot, registry(link.ScriptsStage())) - env.done(st.id, st.key) - return env - - -def test_spine_subjects_closures_and_context(tmp_path): - stage, depot = SpineStage(), Depot(tmp_path / "s") - env = spine_env(depot) - assert stage.subjects(env) == ["other_d:5", "scene_e.sharedAssets:7", "spine_a:10"] - t = describe(stage, "spine_a:10", None, env) - assert t.id == "spine.skeleton:spine_a:10" and t.atoms == SpineStage.ATOMS and t.params == {} - assert [i.role for i in t.inputs] == ["bundle:scripts", "bundle:shader_c", "bundle:spine_a", "bundle:tex_b"] - assert t.inputs[2].sha256 == contract.sha256(b"UnityFS spine_a") - assert t.context == {"object": CAB["a"] + ":10"} - assert stage.depends("spine_a:10", env) == [ - "link.scripts:all", - "unity.census:scripts", - "unity.census:shader_c", - "unity.census:spine_a", - "unity.census:tex_b", - ] - e = describe(stage, "scene_e.sharedAssets:7", None, env) - assert [i.role for i in e.inputs] == ["bundle:scene_e", "bundle:shader_c", "bundle:tex_b"] - assert e.context == {"object": CAB["e"] + ".sharedAssets:7"} - assert [i.role for i in describe(stage, "other_d:5", None, env).inputs] == ["bundle:other_d"] - with pytest.raises(ValueError, match="no SkeletonDataAsset"): - describe(stage, "unsel_f:3", None, env) - - -def test_spine_keys_follow_the_closure_contents_only(tmp_path): - stage = SpineStage() - key = describe(stage, "spine_a:10", None, spine_env(Depot(tmp_path / "a"))).key - env = spine_env(Depot(tmp_path / "b"), data={"other_d": b"UnityFS other, changed"}) - assert describe(stage, "spine_a:10", None, env).key == key - env = spine_env(Depot(tmp_path / "c"), data={"shader_c": b"UnityFS shader, changed"}) - assert describe(stage, "spine_a:10", None, env).key != key - - -def test_spine_waits_for_censuses_and_the_script_table(tmp_path): - stage, depot = SpineStage(), Depot(tmp_path / "s") - env = spine_env(depot) - env.waiting("unity.census:tex_b") - with pytest.raises(Pending, match="unity.census:tex_b"): - stage.subjects(env) - env.waiting("unity.census:tex_b", failed=True) # a failed census is left out of the closures - assert [i.role for i in describe(stage, "spine_a:10", None, env).inputs] == ["bundle:scripts", "bundle:spine_a"] - env.waiting("link.scripts:all") - with pytest.raises(Pending, match="link.scripts:all"): - stage.subjects(env) - - -class FakeRef: - def __init__(self, path_id, target): - self.path_id, self.target = path_id, target - - def read(self): - return self.target - - -class FakeObj: - def __init__(self, tt, **fields): - self.tt, self.fields = tt, fields - - def read_typetree(self): - return self.tt - - def read(self): - return type("Read", (), self.fields)() - - -class FakeFile: - def __init__(self, objects): - self.objects = objects - - -class FakeEnv: - def __init__(self, files): - self.files = files - - def get_cab(self, name): - return self.files.get(name.lower()) - - -def fake_skeleton(pages=1): - from PIL import Image - - text = lambda name, s: type("T", (), {"m_Name": name, "m_Script": s})() # noqa: E731 - tex = type("Tex", (), {"m_Name": "chr", "m_TextureFormat": 4, "image": Image.new("RGBA", (1, 1))})() - mat = type( - "Mat", - (), - { - "m_SavedProperties": type( - "P", (), {"m_TexEnvs": [("_MainTex", type("E", (), {"m_Texture": FakeRef(1, tex)})())]} - )() - }, - )() - atlas = type( - "A", - (), - {"atlasFile": FakeRef(2, text("chr", "\nchr.png\nsize: 1,1\n")), "materials": [FakeRef(3, mat)] * pages}, - )() - return FakeObj( - {"m_Name": "chr_SkeletonData", "scale": 1.0, "defaultMix": 0.0}, - skeletonJSON=FakeRef(4, text("chr", b"\x01skel")), - atlasAssets=[FakeRef(11, atlas)], - ) - - -def test_spine_run_stores_the_files_of_the_skeleton(tmp_path, monkeypatch): - from nnnotes.unity import reader as unity - - stage, depot = SpineStage(), Depot(tmp_path / "s") - env = spine_env(depot) - for s in ("scripts", "shader_c", "spine_a", "tex_b"): - depot.put(b"UnityFS " + s.encode()) - loaded = [] - - def load_closure(paths): - loaded.append([Path(p).read_bytes() for p in paths]) - return FakeEnv({CAB["a"].lower(): FakeFile({10: sda})}) - - monkeypatch.setattr(unity, "load_closure", load_closure) - sda = fake_skeleton() - t = describe(stage, "spine_a:10", None, env) - assert execute(t, depot, registry(stage)).status == "ran" - assert loaded == [[b"UnityFS scripts", b"UnityFS shader_c", b"UnityFS spine_a", b"UnityFS tex_b"]] - doc = depot.result(t.key) - got = {contract.parse_artifact_id(a["id"])[1]: a for a in doc["artifacts"]} - assert {r: (a["semantics"]["kind"], a["content"]["ext"]) for r, a in got.items()} == { - "chr.skel": ("spine.skeleton", "skel"), - "chr.atlas": ("spine.atlas", "atlas"), - "chr.png": ("spine.page", "png"), - } - assert depot.read(got["chr.skel"]["content"]["sha256"]) == b"\x01skel" - assert got["chr.skel"]["semantics"]["facts"]["atlases"] == ["chr.atlas"] - (item,) = doc["items"] - assert item == { - "object": CAB["a"] + ":10", - "status": "exported", - "class": "MonoBehaviour", - "artifacts": sorted(a["id"] for a in doc["artifacts"]), - } - sda = fake_skeleton(pages=2) # a writer error: a failed item, cached - t2 = Task(t.stage, t.version, t.subject, {}, t.atoms, t.inputs[:3], t.context) # another key - assert execute(t2, depot, registry(stage)).result_status == "partial" - doc2 = depot.result(t2.key) - assert doc2["artifacts"] == [] and doc2["items"][0]["reason"]["code"] == "failed.export" - assert "1 pages vs 2 materials" in doc2["items"][0]["reason"]["message"] diff --git a/tests/pipeline/test_spot.py b/tests/pipeline/test_spot.py new file mode 100644 index 0000000..b34bb30 --- /dev/null +++ b/tests/pipeline/test_spot.py @@ -0,0 +1,103 @@ +"""The spot product stages (pipeline.spot) run on synthetic object tables.""" + +import json +import subprocess +import sys + +import pytest + +from nnnotes.content import spot +from nnnotes.pipeline import contract, layout +from nnnotes.pipeline.contract import Input, Task +from nnnotes.pipeline.depot import Depot +from nnnotes.pipeline.spot import RoomStage, SpotStage +from nnnotes.pipeline.stages import Env, execute, registry +from nnnotes.pipeline.views import ViewError +from tests.helpers import spotobjects as so + + +def master_dir(tmp_path): + md = tmp_path / "master" + md.mkdir() + for name, rows in so.MASTER.items(): + (md / f"{name}.json").write_text(json.dumps({"_allData": rows}), encoding="utf-8") + return md + + +def run(depot, stage, subject, objs, roots, *more): + data = contract.encode(so.table(depot, objs, roots)) + inp = Input("objects", depot.put(data), len(data), None, ({"kind": "depot"},)) + t = Task(stage.name, stage.version, subject, {}, {}, (inp, *more)) + execute(t, depot, registry(stage), force=True) + doc = depot.result(t.key) + arts = {contract.parse_artifact_id(a["id"])[1]: a for a in doc["artifacts"]} + return t, doc, arts + + +def test_the_stages_start_from_their_key(): + rooms, spots = RoomStage(), SpotStage() + assert rooms.keys(so.BACKGROUND, None) == [so.BACKGROUND] and spots.keys(so.SITUATION, None) == [so.SITUATION] + assert rooms.manifest(so.BACKGROUND) == "rooms/home_a.json" + assert spots.manifest(so.SITUATION) == "spots/home_a_01.json" + keys = [so.BACKGROUND, so.SITUATION, "Spot/b/b_01/b_01", "Spot/a/b/c", "Character/x"] + assert rooms.select(keys) == [so.BACKGROUND] + assert spots.select(keys) == ["Spot/b/b_01/b_01", so.SITUATION] + assert rooms.MASTER == () and spots.MASTER == spot.MASTER_TABLES + with pytest.raises(ValueError, match="not a spot situation key"): + spots.keys(so.BACKGROUND, None) + + +def test_the_room_product(tmp_path): + depot = Depot(tmp_path / "d") + t, doc, arts = run(depot, RoomStage(), so.BACKGROUND, so.room_objects(), [so.ROOM_ROOT]) + assert {p: a["semantics"]["kind"] for p, a in arts.items()} == { + "room.glb": "spot.room", + "room.json": "spot.room-summary", + "objects": "product.objects", + } + assert arts["room.glb"]["content"]["ext"] == "glb" + summary = contract.loads(depot.read(arts["room.json"]["content"]["sha256"])) + assert summary["meshCount"] == 3 and summary["glb"] == "room.glb" + table = contract.loads(depot.read(arts["objects"]["content"]["sha256"])) + assert table["objects"] == {} + assert doc["items"][0]["class"] == "SpotRoom" and doc["items"][0]["status"] == "exported" + manifest = layout.product_manifest(t.id, doc, table) + assert manifest["product"] == "room.gltf" and set(manifest["files"]) == {"room.glb", "room.json"} + + +def test_the_spot_product_reads_its_master_record(tmp_path): + depot = Depot(tmp_path / "d") + stage = SpotStage() + env = Env(depot, {"master": master_dir(tmp_path)}) + (master,) = stage.more_inputs(so.SITUATION, env) + assert master.role == "master" + assert contract.loads(depot.input_bytes(master)) == spot.master_record(so.MASTER, so.SITUATION) + t, doc, arts = run(depot, stage, so.SITUATION, so.situation_objects(), [so.SIT_ROOT], master) + assert {p: (a["semantics"]["kind"], a["content"]["ext"]) for p, a in arts.items()} == { + "spot.json": ("spot.situation", "json"), + "spine/chr2.json": ("spine.skeleton", "json"), + "spine/chr.skel": ("spine.skeleton", "skel"), + "spine/chr.atlas": ("spine.atlas", "atlas"), + "spine/p1.png": ("spine.texture", "png"), + "spine/p2.png": ("spine.texture", "png"), + "objects": ("product.objects", "json"), + } + doc_json = contract.loads(depot.read(arts["spot.json"]["content"]["sha256"])) + assert doc_json["spotId"] == 3 and doc_json["tapTalk"] == {"5": [51], "7": [71, 72]} + assert doc["items"][0]["class"] == "SpotSituation" + + +def test_a_master_table_the_master_data_lacks_is_an_error(tmp_path): + md = master_dir(tmp_path) + (md / "MasterText.json").unlink() + with pytest.raises(ViewError, match="master table MasterText is not in the master data"): + SpotStage().more_inputs(so.SITUATION, Env(Depot(tmp_path / "d"), {"master": md})) + + +def test_planning_the_stages_does_not_import_unitypy(): + code = ( + "import sys, nnnotes.pipeline.spot as m; m.RoomStage(); m.SpotStage(); " + "print(sorted(x for x in ('UnityPy', 'numpy', 'PIL') if x in sys.modules))" + ) + r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) + assert r.returncode == 0 and r.stdout.strip() == "[]", r.stderr[-2000:] diff --git a/tests/unity/test_speed_exact.py b/tests/unity/test_speed_exact.py index 24fa7c5..45c9a4b 100644 --- a/tests/unity/test_speed_exact.py +++ b/tests/unity/test_speed_exact.py @@ -425,7 +425,7 @@ def ref_dump_objects(shaders, out_dir: Path, source: str, index: list) -> list: continue done.add(name) safe = shader._safe(name) - jsonio.write_json(out_dir / f"{safe}.json", shader._parsed_summary(tt), default=str) + jsonio.write_json(out_dir / f"{safe}.json", shader.parsed_summary(tt), default=str) rec = {"name": name, "source": source, "parsed": f"{safe}.json", "variants": []} for plat, si, pi, stage, n, t, kws, code in shader.variants(tt, shader.platform_blobs(o.read())): tag = shader.PLATFORM.get(plat, str(plat)) From 2e5872ee522e5be127a336dbc0fe9f56034a4b9d Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 14:52:32 +0800 Subject: [PATCH 21/91] feat(pipeline)!: name built-in objects by class as product roots The address table (nnnotes.addresses/3) gains a `builtins` table: the objects of the APK's built-in files by class ("GraphicsSettings") and MonoBehaviours by their script's class ("MonoBehaviour:"), without MonoScripts and with a hierarchy listed by its root only. AddressIndex.builtin(selector) reads it. A product stage can name these selectors as roots (ProductStage.builtins); the request fact "roots" carries them, so the units of settings assets that the game reads without a catalog key or a Resources.Load path are exported with the products that use them. BREAKING CHANGE: link.addresses is at version 3 and writes nnnotes.addresses/3; the roots fact has a "builtins" list. Co-Authored-By: Claude Opus 5.5 --- docs/contracts.md | 2 +- docs/products.md | 11 +- docs/schemas.md | 2 +- docs/stages.md | 8 +- src/nnnotes/pipeline/assemble.py | 28 +- src/nnnotes/pipeline/link.py | 41 +- src/nnnotes/pipeline/units.py | 28 +- src/nnnotes/pipeline/views.py | 36 +- src/nnnotes/schemas/addresses.schema.json | 18 +- tests/pipeline/test_assemble.py | 48 +- tests/pipeline/test_link.py | 41 + tests/pipeline/test_views.py | 7 +- uv.lock | 2135 +++++++++++++++++++++ 13 files changed, 2356 insertions(+), 49 deletions(-) create mode 100644 uv.lock diff --git a/docs/contracts.md b/docs/contracts.md index 7c63b3c..d23c9e9 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -229,7 +229,7 @@ Documents written by particular stages (artifacts of no object: their ids are `< | `nnnotes.catalog-index/1` | `catalog.index` | [catalog-index.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalog-index.schema.json) | | `nnnotes.census/2` | `unity.census` | [census.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/census.schema.json) | | `nnnotes.scripts/1` | `link.scripts` | [scripts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/scripts.schema.json) | -| `nnnotes.addresses/2` | `link.addresses` | [addresses.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/addresses.schema.json) | +| `nnnotes.addresses/3` | `link.addresses` | [addresses.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/addresses.schema.json) | | `nnnotes.view/1` | `view.` | [view.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/view.schema.json) | | `nnnotes.objects/1` | `pipeline.assemble` (a product's object table) | [objects.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/objects.schema.json) | diff --git a/docs/products.md b/docs/products.md index 366e81d..10b4573 100644 --- a/docs/products.md +++ b/docs/products.md @@ -14,12 +14,15 @@ objects: `link.addresses`; - the objects of the Resources.Load paths it gives (`resources`): paths of the ResourceManager in the APK's built-in files, such as the materials a runtime loads by name, through the same table and compared ignoring case; +- the objects of the built-in selectors it gives (`builtins`): objects of the APK's built-in files by class + (`GraphicsSettings`) or a MonoBehaviour by its script's class (`MonoBehaviour:`), such as settings assets + the game reads without a path, through the same table; - the object ids the stage names besides (`root_objects`). All are read from the planning facts alone, so a command can ask for what its products need before anything runs: -`assemble.request(stages, products, env)` gives the fact `roots` (`{"keys", "resources", "objects"}`), and the request of -`unity.export` then holds the units of those objects and every unit reachable from them through the census `refs` -([units](stages.md#units)), and nothing else of their bundles. +`assemble.request(stages, products, env)` gives the fact `roots` (`{"keys", "resources", "builtins", "objects"}`), and +the request of `unity.export` then holds the units of those objects and every unit reachable from them through the +census `refs` ([units](stages.md#units)), and nothing else of their bundles. What a task reaches: the units holding its roots, then every unit that an item of a reached unit references (the item's `refs`), through the `unity.export` results. A sprite whose atlas is in another bundle has its `image` from the @@ -29,7 +32,7 @@ object that a reached object references and no census holds is listed as `missin | Part of the task | | |---|---| | inputs | `objects`: the object table of what it reaches (`nnnotes.objects/1`); the stage's own inputs (master tables, data its parameters name) | -| depends | `link.addresses` (when it has keys or resource paths), the `unity.export` and `sprite.crop` tasks whose results it read | +| depends | `link.addresses` (when it has keys, resource paths or built-in selectors), the `unity.export` and `sprite.crop` tasks whose results it read | | artifacts | `#`: the product's own files; `#objects`: the object table of the objects its files use (the whole table unless the stage narrows it to some objects, or to some files of an object: a shader's programs for one graphics API) | | item | one for the subject: `exported`, or `unsupported` with a reason code | diff --git a/docs/schemas.md b/docs/schemas.md index 6ffa241..fa3c1f9 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -14,7 +14,7 @@ schemas.load("plan") # the schema as a dict | Schema | Format | Document | |---|---|---| -| `addresses` | `nnnotes.addresses/2` | the address table of a catalog version and the censuses of its bundles | +| `addresses` | `nnnotes.addresses/3` | the address table of a catalog version and the censuses of its bundles | | `artifact` | `nnnotes.artifact/1` | the content, provenance and semantics of one output file | | `catalog-diff` | `nnnotes.catalog-diff/1` | the difference between two catalog versions (`catalog diff`) | | `catalog-index` | `nnnotes.catalog-index/1` | the index of a catalog pair: remote and APK catalogs, every location in catalog order | diff --git a/docs/stages.md b/docs/stages.md index 6e26c6f..661a8d5 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -13,8 +13,8 @@ The asset export is built from three layers: ## Stages Every stage is at version 1, except `unity.census` (version 4: it lists what every object references, the -hierarchy holding it and the Resources.Load paths of a ResourceManager), `link.addresses` (version 2: the -Resources.Load paths), `live2d.model` (version 2: assembled from the exported objects), `unity.export` (version 2: one task per unit), `cri.movie` (version 2: every stream kind, +hierarchy holding it and the Resources.Load paths of a ResourceManager), `link.addresses` (version 3: the +Resources.Load paths and the built-in selectors), `live2d.model` (version 2: assembled from the exported objects), `unity.export` (version 2: one task per unit), `cri.movie` (version 2: every stream kind, codec and channel of a USM) and `catalog.index` (version 2: gzip input and Japanese remote asset placeholders). | Stage | Subject | Inputs | Outputs | @@ -29,7 +29,7 @@ codec and channel of a USM) and `catalog.index` (version 2: gzip input and Japan | `live2d.model`, `live2d.cubism` | a Live2D model key (`Character/Live2D///model/`) a command asks for | `objects`: the object table of what the model reaches ([products](products.md#live2d-models)); depends on `link.addresses` and the `unity.export` tasks it reads | `model.json` (the player's model) and the object table; the Cubism SDK files | | `room.gltf` | a spot background key (`Spot//Background/`) | `objects`: the object table of what the background reaches ([products](products.md#spots)) | `room.glb` and `room.json` | | `spot.files` | a spot situation key (`Spot///`) | `objects`; `master`: the spot's master record | `spot.json` and the Spine files | -| `link.addresses` | global | catalog index, censuses (the built-in files' too) | key -> objects, Resources.Load path -> objects, serialized file -> bundle or built-in file, bundle -> keys | +| `link.addresses` | global | catalog index, censuses (the built-in files' too) | key -> objects, Resources.Load path -> objects, built-in selector (class, MonoBehaviour script class) -> objects, serialized file -> bundle or built-in file, bundle -> keys | | `view.` | a catalog (`main`) | `master:
` per master table it reads, `rules` (the view's rule and the resolvers it uses), `addresses` (the part of the address table it read: the keys looked up and the prefixes listed); no parameters, atoms or context | `view.:#view` (facts: rows, entries, gaps, unreferenced) | | `view.voices` | a catalog (`main`) | `master:
` per master table its rules read, `rules`, `addresses`, `sheets` (each cue sheet the `cri.audio` results are known by that a voice names or the rules' prefixes list: its task and ACB content id), `acb:` (the ACB of each decoded one), `episodes` (per story episode and part, the `unity.export` artifact holding the object its key names), `adv:` (those contents); context `audio` (the keys of those `cri.audio` results); runs after `unity.export` and `cri.audio` | `view.voices:#view` (facts: rows, entries, gaps, unreferenced streams) | @@ -112,7 +112,7 @@ several serialized files the file's tag follows the bundle name, as in `link.add bundle's content changes, so a unit keeps its task across catalog versions. A run exports the units holding an object of the selected bundles, the units of the objects the fact "roots" names -(catalog keys, Resources.Load paths and object ids: a product's roots, [products](products.md)) and every unit reachable +(catalog keys, Resources.Load paths, built-in selectors and object ids: a product's roots, [products](products.md)) and every unit reachable from those through what their objects reference (the census `refs`, across the censuses of the run); with `--only-class` only the units whose root object is of those classes. The selection is not part of any key: a unit's task is the same whichever request reaches it. diff --git a/src/nnnotes/pipeline/assemble.py b/src/nnnotes/pipeline/assemble.py index 9ba12f6..94cdc44 100644 --- a/src/nnnotes/pipeline/assemble.py +++ b/src/nnnotes/pipeline/assemble.py @@ -1,7 +1,8 @@ """Product stages: products assembled from the unity.export results of the objects they reach, never from a bundle. A product (a Live2D model, a story, a live, ...) is a stage whose task per subject starts from root objects: the -objects of the catalog keys `keys(subject)` gives, of the Resources.Load paths `resources(subject)` gives and the +objects of the catalog keys `keys(subject)` gives, of the Resources.Load paths `resources(subject)` gives, of the +built-in selectors `builtins(subject)` gives (objects of the APK's built-in files by class or script class) and the objects `root_objects(subject)` names (all read from the planning facts alone, so a command can ask for the units its products need before anything runs: `request`). The objects it reaches are those of the units holding its roots and every unit an item of a reached unit references @@ -36,15 +37,21 @@ def request(stages: dict, products: dict, env) -> dict: """The fact "roots" (pipeline.units) of a request for `products` {stage name: subjects}: the catalog keys, the - resource paths and the root objects of every subject, sorted.""" - keys, resources, objects = set(), set(), set() + resource paths, the built-in selectors and the root objects of every subject, sorted.""" + keys, resources, builtins, objects = set(), set(), set(), set() for name, subjects in sorted(products.items()): stage = stages[name] for s in subjects: keys.update(stage.keys(s, env)) resources.update(stage.resources(s, env)) + builtins.update(stage.builtins(s, env)) objects.update(stage.root_objects(s, env)) - return {"keys": sorted(keys), "resources": sorted(resources), "objects": sorted(objects)} + return { + "keys": sorted(keys), + "resources": sorted(resources), + "builtins": sorted(builtins), + "objects": sorted(objects), + } def _entry(item: dict, arts: dict) -> dict: @@ -271,6 +278,11 @@ def resources(self, subject: str, env) -> list[str]: """The Resources.Load paths whose objects are roots of `subject` (from the facts alone).""" return [] + def builtins(self, subject: str, env) -> list[str]: + """The built-in selectors ("", "MonoBehaviour:\n' - files[f"{sitefiles.STORY_PAGE_DIR}/story-list.js"] = 'import { x } from "../../src/story/define.js";\n' - if story_bundle: - files[sitefiles.STORY_BUNDLES[0]] = "/* story */\n" - for rel, text in files.items(): - write(root / rel, text) - return root - - -def test_write_player_story_page(tmp_path): - site = tmp_path / "site" - r = sitefiles.write_player(site, fake_player(tmp_path / "p")) - assert r["storyPageFiles"] == 2 - assert (site / "story/story-list.js").read_text( - encoding="utf-8" - ) == 'import { x } from "./ournotes-player.story.element.min.js";\n' - assert (site / "story/ournotes-player.story.element.min.js").is_file() - site2 = tmp_path / "site2" - r = sitefiles.write_player(site2, fake_player(tmp_path / "q", story_bundle=False)) - assert "storyPageFiles" not in r and not (site2 / "story").exists() - with pytest.raises(ConfigError, match="story page"): - sitefiles.write_player(site2, fake_player(tmp_path / "q", story_bundle=False), stories=True) - - -# ---------------------------------------------------------------- the models of the stories -MODEL_A, MODEL_B = "Character/Live2D/g/a/model/a", "Character/Live2D/g/b/model/b" - - -def put_model(site, key, motion_sync=False): - """A model manifest as webmodel.ingest writes it (made-up model.json and moc3).""" - mid = webmodel.model_id(key) - store = sitefiles.Store(site) - doc = {"format": webmodel.MODEL_FORMAT, "key": key, "motionSync": motion_sync} - man = { - "format": sitefiles.MANIFEST_FORMAT, - "id": mid, - "key": key, - "model": {}, - "files": { - "model.json": store.put("model.json", json.dumps(doc).encode()), - f"{mid}.moc3": store.put(f"{mid}.moc3", b"MOC3-" + mid.encode()), - }, - } - write(site / "models" / f"{mid}.json", sitefiles._dump(man)) - return {"id": mid, "ok": True, "files": 2, "bytes": sum(e["size"] for e in man["files"].values())} - - -@pytest.fixture -def story_build(tmp_path, monkeypatch): - """storysite.build over synthetic master data (MasterAdv 5, 6, 7, 8) and episodes: episode_models gives the - models each episode loads, build_dirs writes the story directory with the models it takes from the site - (webmodel.SiteModels, as story.build does), webmodel's model task writes a model manifest. -> run(ids, **kw), - with the calls in run.episodes (pre-pass) and run.models (model builds).""" - uses = {5: [MODEL_A], 6: [MODEL_B, MODEL_A], 7: [], 8: [MODEL_A]} - md = tmp_path / "master" - write(md / "MasterAdv.json", {"_allData": [{"_id": i} for i in uses]}) - font = tmp_path / "f.otf" - font.write_bytes(b"font") - cfg = Config( - { - "catalog": {"region": "zz", "language": "en"}, - "servers": {"zz": {"cdn": "https://cdn.invalid"}}, - "paths": {"master": str(md), "fonts": {"ja": str(font), "en": str(font)}}, - }, - environ={}, - ) - site = tmp_path / "site" - episodes, models = [], [] - - def episode_models(cat, master_dir, adv_id): - episodes.append(adv_id) - return sorted(uses[adv_id]) - - def build_dirs(cat, master_dir, player, adv_id, work, job): - src = webmodel.SiteModels(job["site"]) - return story_dirs(work, adv_id, mode=0, models={k: src.ensure(k)["id"] for k in uses[adv_id]}) - - def model_task(mid, key, job, data=None): - models.append(mid) - return put_model(Path(job["site"]), key) - - monkeypatch.setattr(tmpfont, "require_extra", lambda what=None: None) - monkeypatch.setattr(storysite, "open_data", lambda cfg, region=None: (None, md, None)) - monkeypatch.setattr(storysite, "episode_models", episode_models) - monkeypatch.setattr(storysite, "build_dirs", build_dirs) - monkeypatch.setattr(storysite.storyhost, "build", lambda *a, **kw: None) - monkeypatch.setattr(storysite, "font_file", lambda job, lang: None) - monkeypatch.setattr(webmodel, "model_task", model_task) - monkeypatch.setattr(webmodel, "_open", lambda cfg, region=None: (None, None)) - player = fake_player(tmp_path / "p") - - def run(ids, **kw): - return storysite.build(site, ids, cfg, player, audio=False, workers=1, story_languages=["ja", "en"], **kw) - - run.site, run.episodes, run.models = site, episodes, models - return run - - -def manifest(site, adv_id): - return json.loads((site / "stories" / f"{adv_id}.json").read_text(encoding="utf-8")) - - -def test_story_build_builds_the_models_of_its_stories_first(story_build): - site = story_build.site - r = story_build([5, 6, 7]) - assert r["storiesFailed"] == [] and [s["id"] for s in r["storiesBuilt"]] == ["5", "6", "7"] - assert sorted(story_build.episodes) == [5, 6, 7] - assert story_build.models == ["a", "b"] # each model once, whatever the stories using it - assert [m["id"] for m in r["storyModels"]["modelsBuilt"]] == ["a", "b"] and r["models"] == 2 - index = json.loads((site / "models.json").read_text(encoding="utf-8")) - assert [m["id"] for m in index["models"]] == ["a", "b"] - assert manifest(site, 5)["models"] == {"a": "models/a.json"} - assert manifest(site, 6)["models"] == {"a": "models/a.json", "b": "models/b.json"} - assert manifest(site, 7)["models"] == {} and manifest(site, 7)["root"] == "../" - assert json.loads(sitefiles.entry_text(site, manifest(site, 6)["files"]["story.json"]))["models"] == { - MODEL_B: "b", - MODEL_A: "a", - } - stories = {e["id"]: e for e in json.loads((site / "stories.json").read_text(encoding="utf-8"))["stories"]} - size = { - m: sum(e["size"] for e in json.loads((site / f"models/{m}.json").read_text(encoding="utf-8"))["files"].values()) - for m in ("a", "b") - } - assert (stories["5"]["size"]["models"], stories["6"]["size"]["models"], stories["7"]["size"]["models"]) == ( - size["a"], - size["a"] + size["b"], - 0, - ) - for mf in (site / "models").glob("*.json"): # write_index kept the models' assets - for e in json.loads(mf.read_text(encoding="utf-8"))["files"].values(): - assert all((site / a).is_file() for a in sitefiles.entry_assets(e)) - - -def test_story_build_skips_existing_models_unless_forced(story_build): - story_build([5]) - assert story_build.models == ["a"] - # stories whose manifest exists: no pre-pass, no model build - r = story_build([5]) - assert r["storiesSkipped"] == ["5"] and r["storyModels"] is None and story_build.episodes == [5] - # a new story whose model exists: the model manifest is skipped - r = story_build([8]) - assert story_build.models == ["a"] and r["storyModels"]["modelsSkipped"] == ["a"] - assert manifest(story_build.site, 8)["models"] == {"a": "models/a.json"} - # force: the stories and their models built again, each model once - r = story_build([5, 8, 6], force=True) - assert story_build.models == ["a", "a", "b"] and r["storyModels"]["modelsSkipped"] == [] - assert sorted(s["id"] for s in r["storiesBuilt"]) == ["5", "6", "8"] - - -def test_story_build_rebuilds_outdated_models(story_build): - """A model manifest whose model.json is older than MODEL_FORMAT (no motionSync) is built again by the stories' - model build without force; a current one is skipped.""" - site = story_build.site - put_model(site, MODEL_B) # current - put_model(site, MODEL_A) - man = json.loads((site / "models/a.json").read_text(encoding="utf-8")) - man["files"]["model.json"] = sitefiles.Store(site).put( - "model.json", json.dumps({"format": 1, "key": MODEL_A}).encode() - ) - write(site / "models/a.json", sitefiles._dump(man)) - r = story_build([6]) - assert story_build.models == ["a"] - assert r["storyModels"]["modelsRebuilt"] == [{"id": "a", "reason": "outdated"}] - assert r["storyModels"]["modelsSkipped"] == ["b"] and [m["id"] for m in r["storyModels"]["modelsBuilt"]] == ["a"] - assert r["storiesFailed"] == [] and webmodel.outdated(site, "a") is None - assert manifest(site, 6)["models"] == {"a": "models/a.json", "b": "models/b.json"} - - -def test_a_story_without_its_model_fails(story_build, monkeypatch): - monkeypatch.setattr( - webmodel, - "model_task", - lambda mid, key, job, data=None: {"id": mid, "ok": False, "stage": "export", "error": "x"}, - ) - r = story_build([5, 7]) - assert r["storyModels"]["modelsFailed"] == [{"id": "a", "stage": "export", "error": "x"}] - assert [(f["id"], f["stage"]) for f in r["storiesFailed"]] == [("5", "export")] - assert "no model manifest models/a.json" in r["storiesFailed"][0]["error"] - assert [s["id"] for s in r["storiesBuilt"]] == ["7"] - - -@pytest.mark.parametrize( - "argv, message", - [ - (["--story", "1", "--player-only"], "cannot be combined"), - (["--font", "fr=x.otf", "--all-stories"], "expected ="), - (["--story-languages", "en,xx", "--all-stories"], "expected languages"), - (["--story", "1", "--all-stories"], "not allowed with argument"), - ], -) -def test_cli_web_story_usage(tmp_path, capsys, argv, message): - from nnnotes import cli - - with pytest.raises(SystemExit) as e: - cli.main( - ["--json", "site", "build", "-o", str(tmp_path / "s"), "--player", str(fake_player(tmp_path / "p")), *argv] - ) - assert e.value.code == 2 and message in capsys.readouterr().err diff --git a/tests/site/test_web.py b/tests/site/test_web.py deleted file mode 100644 index b11a168..0000000 --- a/tests/site/test_web.py +++ /dev/null @@ -1,232 +0,0 @@ -import gzip -import hashlib -import json - -import pytest - -from nnnotes.formats import compression as compress -from nnnotes.site import sitefiles, siteindex, web -from nnnotes.source.config import ConfigError - -DOC = sitefiles._minify({"rows": [{"id": i, "name": f"row {i}", "tags": ["a", "b"]} for i in range(200)]}).encode() - - -def test_text_asset_minifies_and_drops_doc_fields(): - doc = {"spec": "notes for readers", "music": {"soundId": 1}, "voice": {"rule": "r", "x": 1}, "v": 1e999} - out = sitefiles.text_asset("audio/live-audio.json", json.dumps(doc, indent=2).replace("Infinity", "1e999")) - assert out == b'{"music":{"soundId":1},"voice":{"x":1},"v":1e999}' - glsl = "#ifdef VERTEX\nvoid main() {}\n#endif\n" - assert sitefiles.text_asset("shaders/a.glsl", glsl) == glsl.encode("utf-8") - - -def test_split_json_rejoins(): - assert sitefiles.split_json(b'{"a":1,"b":2}') is None - big = sitefiles._minify({"a": "x" * (sitefiles.SPLIT_MIN_BYTES // 2), "b": ["y"] * 200000, "c": {"d": 1}}).encode() - parts = sitefiles.split_json(big) - assert [k for k, _ in parts] == ["a", "b", "c"] - assert sitefiles.join_parts(parts) == big - odd = sitefiles._minify({"a b": "x" * sitefiles.SPLIT_MIN_BYTES, "c": 1}).encode() - assert sitefiles.split_json(odd) is None - - -def test_store_is_content_addressed(tmp_path): - store = sitefiles.Store(tmp_path) - e = store.put("x/data.json", b"{}") - assert e == {"asset": f"assets/{hashlib.sha256(b'{}').hexdigest()}.json", "size": 2} - assert store.put("y/other.json", b"{}") == e - assert (store.written, store.reused) == (1, 1) - assert store.put("noext", b"z")["asset"].endswith(".bin") - - -def sha(data): - return hashlib.sha256(data).hexdigest() - - -def test_store_encodes_compressible_files_when_smaller(tmp_path): - store = sitefiles.Store(tmp_path) - e = store.put("x/doc.json", DOC) - assert e == {"asset": f"assets/{sha(DOC)}.json.gz", "size": len(DOC), "stored": len(compress.gzip_bytes(DOC))} - assert e["stored"] < e["size"] - assert (tmp_path / e["asset"]).read_bytes() == compress.gzip_bytes(DOC) - assert sitefiles.read_asset(tmp_path, e["asset"]) == DOC - shader = b"void main() { gl_FragColor = vec4(1.0); }\n" * 20 - assert store.put("s/a.glsl", shader)["asset"] == f"assets/{sha(shader)}.glsl.gz" - assert store.put("x/tiny.json", b"{}") == {"asset": f"assets/{sha(b'{}')}.json", "size": 2} # gzip not smaller - assert store.put("t/a.png", DOC) == {"asset": f"assets/{sha(DOC)}.png", "size": len(DOC)} # not compressible - with pytest.raises(ValueError, match="asset encoding"): - sitefiles.Store(tmp_path, "zstd") - - -def test_store_brotli(tmp_path): - pytest.importorskip("brotli") - e = sitefiles.site_store(tmp_path, "br").put("m/model.moc3", DOC) - assert e == {"asset": f"assets/{sha(DOC)}.moc3.br", "size": len(DOC), "stored": len(compress.brotli_bytes(DOC))} - assert sitefiles.read_asset(tmp_path, e["asset"]) == DOC - - -def test_store_none_keeps_todays_names_and_bytes(tmp_path): - store = sitefiles.site_store(tmp_path, "none") - assert store.put("x/doc.json", DOC) == {"asset": f"assets/{sha(DOC)}.json", "size": len(DOC)} - assert [p.name for p in (tmp_path / "assets").iterdir()] == [f"{sha(DOC)}.json"] - assert (tmp_path / "assets" / f"{sha(DOC)}.json").read_bytes() == DOC - - -def test_store_reuses_encoded_names(tmp_path): - store = sitefiles.Store(tmp_path) - e = store.put("a.json", DOC) - assert store.put("b.json", DOC) == e and (store.written, store.reused) == (1, 1) - again = sitefiles.Store(tmp_path) - assert again.put("c.json", DOC) == e and (again.written, again.reused) == (0, 1) - # a file of that name that holds the bytes is kept whatever encoder wrote it; its length is the entry's `stored` - other = gzip.compress(DOC, 1, mtime=0) - (tmp_path / e["asset"]).write_bytes(other) - kept = sitefiles.Store(tmp_path).put("d.json", DOC) - assert kept == {**e, "stored": len(other)} and (tmp_path / e["asset"]).read_bytes() == other - (tmp_path / e["asset"]).write_bytes(other[:-5]) # damaged: written again - fresh = sitefiles.Store(tmp_path) - assert fresh.put("e.json", DOC) == e and fresh.written == 1 - assert (tmp_path / e["asset"]).read_bytes() == compress.gzip_bytes(DOC) - - -def test_split_parts_carry_stored(tmp_path): - big = sitefiles._minify({"a": "x" * (sitefiles.SPLIT_MIN_BYTES // 2), "b": ["y"] * 200000, "c": {"d": 1}}).encode() - e = sitefiles.Store(tmp_path).put_file("scene.json", big) - assert e["size"] == len(big) and [p[0] for p in e["parts"]] == ["a", "b", "c"] - a, b, c = e["parts"] - small = b'{"d":1}' - assert len(a) == len(b) == 4 and a[1].endswith(".json.gz") and a[3] < a[2] - assert c == ["c", f"assets/{sha(small)}.json", len(small)] # a part gzip does not make smaller - assert sitefiles.entry_text(tmp_path, e) == big - raw = sitefiles.Store(tmp_path / "raw", "none").put_file("scene.json", big) - assert all(len(p) == 3 for p in raw["parts"]) and sitefiles.entry_text(tmp_path / "raw", raw) == big - - -def test_stored_text_memo_follows_the_encoding(tmp_path): - doc = json.dumps({"k": list(range(300))}) - a = web.stored_text(sitefiles.Store(tmp_path), "x/a.json", doc, memo="m") - b = web.stored_text(sitefiles.Store(tmp_path, "none"), "x/a.json", doc, memo="m") - assert a["asset"].endswith(".json.gz") and b == {"asset": a["asset"][:-3], "size": a["size"]} - - -def test_ingest_task_stores_in_the_jobs_encoding(tmp_path, monkeypatch): - seen = [] - monkeypatch.setattr(web, "ingest", lambda store, *a, **k: seen.append(store.encoding) or {"id": "1_easy"}) - cfg = {"site": str(tmp_path), "audioFormat": "aac", "audio": True, "language": "en"} - web.ingest_task(1, [("easy", str(tmp_path), {}, [])], str(tmp_path), {**cfg, "encoding": "none"}) - web.ingest_task(1, [("easy", str(tmp_path), {}, [])], str(tmp_path), cfg) - assert seen == ["none", "gzip"] - - -def manifest(site_dir, music_id, difficulty, store, files): - entries = {p: store.put(p, data) for p, data in files.items()} - doc = { - "musicId": music_id, - "difficulty": difficulty, - "audio": True, - "audioFormat": "aac", - "flows": ["direct"], - "files": entries, - "chart": {"title": f"t{music_id}", "level": 1}, - } - (site_dir / "charts" / f"{music_id}_{difficulty}.json").write_text(json.dumps(doc), encoding="utf-8") - - -def test_write_index_orders_charts_and_prunes_assets(tmp_path): - (tmp_path / "charts").mkdir() - store = sitefiles.Store(tmp_path) - manifest(tmp_path, 2, "easy", store, {"live.json": b"{}"}) - manifest(tmp_path, 1, "expert", store, {"live.json": b"{}", "score/a.json": b"[1]"}) - manifest(tmp_path, 1, "easy", store, {"live.json": b"{}"}) - stale = store.put("old.json", b"[0]") - r = siteindex.write_index(tmp_path) - assert (r["charts"], r["assets"], r["removedAssets"], r["assetBytes"], r["bytes"]) == (3, 2, 1, 5, 5) - assert not (tmp_path / stale["asset"]).exists() - index = json.loads((tmp_path / "charts.json").read_text(encoding="utf-8")) - assert index["format"] == sitefiles.SITE_FORMAT - assert [c["id"] for c in index["charts"]] == ["1_easy", "1_expert", "2_easy"] - assert index["charts"][1]["bytes"] == 5 and index["charts"][1]["title"] == "t1" - - -def encoded_site(site): - """Two charts sharing an encoded file, a raw file each, and a stale encoded asset.""" - (site / "charts").mkdir(parents=True) - store = sitefiles.Store(site) - manifest(site, 1, "easy", store, {"live.json": b"{}", "scene/a.json": DOC}) - manifest(site, 1, "hard", store, {"live.json": b"[1]", "scene/a.json": DOC}) - return store.put("old.json", DOC + b" ") - - -def test_write_index_prunes_encoded_names(tmp_path): - stale = encoded_site(tmp_path) - assert stale["asset"].endswith(".json.gz") - r = siteindex.write_index(tmp_path) - assert (r["charts"], r["assets"], r["removedAssets"]) == (2, 3, 1) - assert not (tmp_path / stale["asset"]).exists() - names = sorted(p.name for p in (tmp_path / "assets").iterdir()) - assert names == sorted([f"{sha(DOC)}.json.gz", f"{sha(b'{}')}.json", f"{sha(b'[1]')}.json"]) - assert (r["assetBytes"], r["bytes"]) == (len(compress.gzip_bytes(DOC)) + 2 + 3, len(DOC) + 2 + 3) - index = json.loads((tmp_path / "charts.json").read_text(encoding="utf-8")) - assert [c["bytes"] for c in index["charts"]] == [len(DOC) + 2, len(DOC) + 3] # decoded sizes - - -def fake_player(root): - for rel, text in { - sitefiles.READ_SET_SCRIPT: "// read set\n", - sitefiles.PLAYER_BUNDLES[0]: "/* bundle */\n", - f"{sitefiles.PLAYER_PAGE_DIR}/index.html": ( - '\n' - ), - f"{sitefiles.PLAYER_PAGE_DIR}/chart-list.js": 'import "../../src/element.js";\n', - }.items(): - p = root / rel - p.parent.mkdir(parents=True, exist_ok=True) - p.write_bytes(text.encode("utf-8")) - return root - - -def test_check_player(tmp_path): - with pytest.raises(ConfigError, match="NNNOTES_PATHS_PLAYER"): - sitefiles.check_player(None) - with pytest.raises(ConfigError, match="not a built ournotes-player"): - sitefiles.check_player(tmp_path) - assert sitefiles.check_player(fake_player(tmp_path / "p")) == (tmp_path / "p").resolve() - - -def test_write_player(tmp_path): - player = fake_player(tmp_path / "p") - out = tmp_path / "site" - r = sitefiles.write_player(out, player) - version = hashlib.sha256(b"/* bundle */\n").hexdigest()[:16] - assert r["playerVersion"] == version and r["pageFiles"] == 2 - assert (out / "chart-list.js").read_text(encoding="utf-8") == 'import "./ournotes-player.element.min.js";\n' - assert version in (out / "index.html").read_text(encoding="utf-8") - assert (out / "ournotes-player.element.min.js").is_file() - (player / sitefiles.PLAYER_PAGE_DIR / "extra.js").write_bytes(b'import "../../src/other.js";\n') - with pytest.raises(RuntimeError, match="beyond PAGE_IMPORTS"): - sitefiles.write_player(out, player) - - -def test_write_player_copies_the_complete_songs_module_tree(tmp_path): - player = fake_player(tmp_path / "p") - files = { - "index.html": b'', - "songs.js": b'import "./catalog.js"; import "./replay-panel.js";', - "catalog.js": b"export const catalog = {};", - "text.js": b"export const text = {};", - "replay-panel.js": b'import "./replay-preset.js";', - "replay-worker.js": b"// worker", - "replay-preset.js": b"// preset", - "nested/future-dependency.json": b"{}", - } - for name, data in files.items(): - path = player / sitefiles.SONGS_PAGE_DIR / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - site = tmp_path / "site" - result = sitefiles.write_player(site, player) - assert result["songsPageFiles"] == len(files) - assert { - path.relative_to(site / "songs").as_posix(): path.read_bytes() - for path in (site / "songs").rglob("*") - if path.is_file() - } == files diff --git a/tests/site/test_webmodel.py b/tests/site/test_webmodel.py deleted file mode 100644 index f2f2ca8..0000000 --- a/tests/site/test_webmodel.py +++ /dev/null @@ -1,705 +0,0 @@ -import ast -import hashlib -import inspect -import json -from types import SimpleNamespace - -import pytest - -from nnnotes import cli -from nnnotes.core import jsonio -from nnnotes.site import sitefiles, siteindex, webmodel -from nnnotes.source.config import Config, ConfigError -from nnnotes.unity import live2d - -KEY_A = "Character/Live2D/001_adv/adv_model_a/model/adv_model_a" -KEY_B = "Character/Live2D/sub_x/adv_model_b/model/adv_model_b" - - -def test_shader_names(): - doc = { - "nodes": [ - { - "materials": [ - {"material": "Lit", "shader": {"shader": "S/Lit"}}, - {"material": "Mask", "shader": {"shader": "S/Mask"}}, - ] - } - ], - "other": {"shader": "not a reference", "x": 1}, - } - assert webmodel.shader_names(doc) == {"S/Lit", "S/Mask"} - - -def drawable(tex, keywords, shader="S/Lit"): - return { - "path": "m/Drawables/d", - "components": [ - { - "type": "MeshRenderer", - "m_Materials": [{"material": "Lit", "shader": {"shader": shader}, "keywords": keywords}], - }, - {"type": "MonoBehaviour", "class": "CubismRenderer", "_mainTexture": {"texture": tex, "name": "page"}}, - ], - } - - -def variant(shader, n, keywords, platform="gles3", kind="GLES3", ext="glsl"): - return { - "file": f"{shader}/{platform}/s0p0_vertex_{n}.{ext}", - "platform": platform, - "subShader": 0, - "pass": 0, - "stage": "vertex", - "type": kind, - "keywords": keywords, - } - - -ALV = "_ADDITIONAL_LIGHTS_VERTEX" -SHADER_INDEX = [ - { - "name": "S/Lit", - "source": "x", - "parsed": "S_Lit.json", - "variants": [ - variant("S_Lit", 0, []), - variant("S_Lit", 1, ["CUBISM_MASK_ON"]), - variant("S_Lit", 2, ["_ADDITIONAL_LIGHTS"]), - variant("S_Lit", 3, [], kind="GLES31"), - variant("S_Lit", 4, [], platform="vulkan", kind="SPIRV", ext="vkprog"), - variant("S_Lit", 5, [ALV]), - variant("S_Lit", 6, ["CUBISM_MASK_ON", ALV]), - ], - }, - {"name": "S/Mask", "source": "y", "parsed": "S_Mask.json", "variants": [variant("S_Mask", 0, [])]}, -] -MASKS = { - "cubismMask": {"material": "Mask", "shader": {"shader": "S/Mask"}, "keywords": [], "floats": {"_Cull": 0.0}}, - "cubismMaskCulling": { - "material": "MaskCulling", - "shader": {"shader": "S/Mask"}, - "keywords": [], - "floats": {"_Cull": 1.0}, - }, -} - - -def fake_model(root, name="m", tex=b"\x89PNG texture a", moc=b"MOC3 bytes", masked=True, key=KEY_A, motion_sync=False): - """A model directory in the export layout (made-up contents) and its export summary. Two atlas pages, the - drawables use page 0; drawables masked or not; the root node with the MotionSync components or not.""" - kws = [[], ["CUBISM_MASK_ON"]] if masked else [[], []] - root_node = { - "path": name, - "components": [ - {"type": "MonoBehaviour", "class": c} for c in (webmodel.MOTION_SYNC_COMPONENTS if motion_sync else ()) - ], - } - prefab = { - "key": name, - "nodes": [root_node, *(drawable("textures/p0-00.png", k) for k in kws)], - "canvas": {"width": 1.0}, - } - doc = { - "format": webmodel.MODEL_FORMAT, - "name": name, - "key": key, - "moc3": f"{name}.moc3", - "prefab": f"{name}.prefab.json", - "textures": webmodel.drawable_textures(prefab), - "canvas": prefab["canvas"], - "shaders": "shaders/shaders.json", - "resources": MASKS, - "motionSync": webmodel.has_motion_sync(prefab), - } - files = { - f"{name}.moc3": moc, - f"{name}.prefab.json": json.dumps(prefab, indent=1).encode(), - "textures/p0-00.png": tex, - "textures/p1-11.png": b"unused page", - "shaders/shaders.json": json.dumps(SHADER_INDEX).encode(), - webmodel.MODEL_INDEX: json.dumps(doc).encode(), - "shaders/S_Lit.json": b'{"name": "S/Lit"}', - "shaders/S_Mask.json": b'{"name": "S/Mask"}', - } - for rec in SHADER_INDEX: - for v in rec["variants"]: - files[f"shaders/{v['file']}"] = f"#ifdef VERTEX\n// {v['file']}\n#endif\n".encode() - for rel, data in files.items(): - p = root / rel - p.parent.mkdir(parents=True, exist_ok=True) - p.write_bytes(data) - summary = { - "name": name, - "textures": doc["textures"], - "nodes": len(prefab["nodes"]), - "canvas": prefab["canvas"], - "files": webmodel.read_files(doc, prefab, SHADER_INDEX), - } - return root, summary - - -MASKED_FILES = [ - "m.moc3", - "m.prefab.json", - "model.json", - "shaders/S_Lit.json", - "shaders/S_Lit/gles3/s0p0_vertex_0.glsl", - "shaders/S_Lit/gles3/s0p0_vertex_1.glsl", - "shaders/S_Lit/gles3/s0p0_vertex_5.glsl", - "shaders/S_Lit/gles3/s0p0_vertex_6.glsl", - "shaders/S_Mask.json", - "shaders/S_Mask/gles3/s0p0_vertex_0.glsl", - "shaders/shaders.json", - "textures/p0-00.png", -] - - -def test_read_rule(tmp_path): - _, summary = fake_model(tmp_path / "a", masked=True) - assert summary["files"] == MASKED_FILES - _, summary = fake_model(tmp_path / "b", masked=False) - assert summary["files"] == [ - "m.moc3", - "m.prefab.json", - "model.json", - "shaders/S_Lit.json", - "shaders/S_Lit/gles3/s0p0_vertex_0.glsl", - "shaders/S_Lit/gles3/s0p0_vertex_5.glsl", - "shaders/shaders.json", - "textures/p0-00.png", - ] - # keywords the shader's programs do not use are ignored - lit = {"shader": {"shader": "S/Lit"}, "keywords": ["CUBISM_INVERT_ON"]} - assert webmodel.shader_files(SHADER_INDEX, [lit]) == ["S_Lit.json", "S_Lit/gles3/s0p0_vertex_0.glsl"] - with pytest.raises(RuntimeError, match="0 GLES3 programs"): - webmodel.shader_files( - SHADER_INDEX, [{"shader": {"shader": "S/Lit"}, "keywords": ["CUBISM_MASK_ON", "_ADDITIONAL_LIGHTS"]}] - ) - with pytest.raises(RuntimeError, match="not in the shader index"): - webmodel.shader_files(SHADER_INDEX, [{"shader": {"shader": "S/Other"}, "keywords": []}]) - - -def test_story_keywords_select_the_story_programs_too(): - lit = {"shader": {"shader": "S/Lit"}, "keywords": ["CUBISM_MASK_ON"]} - mask = MASKS["cubismMask"] - assert webmodel.STORY_KEYWORDS == (ALV,) - assert webmodel.shader_files(SHADER_INDEX, [lit], webmodel.STORY_KEYWORDS) == [ - "S_Lit.json", - "S_Lit/gles3/s0p0_vertex_1.glsl", - "S_Lit/gles3/s0p0_vertex_6.glsl", - ] - # a shader whose programs do not use the keyword: its one program - assert webmodel.shader_files(SHADER_INDEX, [mask], webmodel.STORY_KEYWORDS) == [ - "S_Mask.json", - "S_Mask/gles3/s0p0_vertex_0.glsl", - ] - # the keyword without a program for a material's set - index = [{**SHADER_INDEX[0], "variants": [v for v in SHADER_INDEX[0]["variants"] if v["keywords"] != [ALV]]}] - with pytest.raises(RuntimeError, match=r"0 GLES3 programs with the keywords \['_ADDITIONAL_LIGHTS_VERTEX'\]"): - webmodel.shader_files(index, [{"shader": {"shader": "S/Lit"}, "keywords": []}], webmodel.STORY_KEYWORDS) - - -def test_ingest_writes_a_manifest_of_the_read_files(tmp_path): - site = tmp_path / "site" - (site / sitefiles.MODELS_DIR).mkdir(parents=True) - mdir, summary = fake_model(tmp_path / "m") - r = webmodel.ingest(sitefiles.Store(site), site, "adv_model_a", KEY_A, mdir, summary) - man = json.loads((site / "models" / "adv_model_a.json").read_text(encoding="utf-8")) - assert r["ok"] and r["files"] == len(man["files"]) == 12 and sorted(man["files"]) == summary["files"] - assert man["format"] == sitefiles.MANIFEST_FORMAT == 3 and man["id"] == "adv_model_a" and man["key"] == KEY_A - assert man["model"] == {"group": "001_adv", "canvas": {"width": 1.0}, "textures": 1, "nodes": 3} - index = json.loads(sitefiles.entry_text(site, man["files"]["shaders/shaders.json"])) - assert [(r["name"], [v["file"] for v in r["variants"]]) for r in index] == [ - ( - "S/Lit", - [ - "S_Lit/gles3/s0p0_vertex_0.glsl", - "S_Lit/gles3/s0p0_vertex_1.glsl", - "S_Lit/gles3/s0p0_vertex_5.glsl", - "S_Lit/gles3/s0p0_vertex_6.glsl", - ], - ), - ("S/Mask", ["S_Mask/gles3/s0p0_vertex_0.glsl"]), - ] - prefab = sitefiles.entry_text(site, man["files"]["m.prefab.json"]) - assert json.loads(prefab)["canvas"] == {"width": 1.0} and b" " not in prefab - assert sitefiles.entry_text(site, man["files"]["m.moc3"]) == b"MOC3 bytes" - doc = json.loads(sitefiles.entry_text(site, man["files"]["model.json"])) - assert doc["format"] == webmodel.MODEL_FORMAT == 2 and doc["motionSync"] is False - - -def test_write_index_keeps_model_assets_and_writes_models_json(tmp_path): - site = tmp_path / "site" - (site / "charts").mkdir(parents=True) - (site / sitefiles.MODELS_DIR).mkdir() - store = sitefiles.Store(site) - shared = store.put("livescene/shaders/x.glsl", b"#ifdef VERTEX\n#endif\n") - chart = {"musicId": 1, "difficulty": "easy", "audio": False, "files": {"x.glsl": shared}, "chart": {}} - (site / "charts" / "1_easy.json").write_text(json.dumps(chart), encoding="utf-8") - webmodel.ingest(store, site, "adv_model_b", KEY_B, *fake_model(tmp_path / "b", "b", tex=b"tex b", masked=False)) - webmodel.ingest(store, site, "adv_model_a", KEY_A, *fake_model(tmp_path / "a", "a")) - stale = store.put("old.json", b"[0]") - r = siteindex.write_index(site) - assert (r["charts"], r["models"], r["removedAssets"]) == (1, 2, 1) - assert not (site / stale["asset"]).exists() - for mf in (site / "models").glob("*.json"): - for e in json.loads(mf.read_text(encoding="utf-8"))["files"].values(): - assert all((site / a).is_file() for a in sitefiles.entry_assets(e)) - index = json.loads((site / "models.json").read_text(encoding="utf-8")) - assert index["format"] == sitefiles.SITE_FORMAT - assert [m["id"] for m in index["models"]] == ["adv_model_a", "adv_model_b"] - a = index["models"][0] - assert a["manifest"] == "models/adv_model_a.json" and a["key"] == KEY_A and a["group"] == "001_adv" - assert a["files"] == 12 and a["bytes"] > 0 and a["textures"] == 1 - # the two models share their shader files: stored once - shared_glsl = { - json.loads((site / "models" / f"{i}.json").read_text(encoding="utf-8"))["files"][ - "shaders/S_Lit/gles3/s0p0_vertex_0.glsl" - ]["asset"] - for i in ("adv_model_a", "adv_model_b") - } - assert len(shared_glsl) == 1 - # removing a model's manifest frees its own assets on the next index - (site / "models" / "adv_model_b.json").unlink() - r = siteindex.write_index(site) - assert r["models"] == 1 and r["removedAssets"] >= 1 - - -def test_no_models_directory_no_models_json(tmp_path): - (tmp_path / "models.json").write_bytes(b"{}") - r = siteindex.write_index(tmp_path) - assert (r["charts"], r["models"]) == (0, 0) - assert not (tmp_path / "models.json").exists() - (tmp_path / "models").mkdir() - siteindex.write_index(tmp_path) - assert json.loads((tmp_path / "models.json").read_text(encoding="utf-8")) == { - "format": sitefiles.SITE_FORMAT, - "models": [], - } - - -def test_reingest_json_covers_models(tmp_path): - site = tmp_path / "site" - (site / sitefiles.MODELS_DIR).mkdir(parents=True) - store = sitefiles.Store(site) - loose = store.put("model.json", b'{ "a" : 1 }') - doc = {"id": "m", "key": KEY_A, "model": {}, "files": {"model.json": loose}} - (site / "models" / "m.json").write_text(json.dumps(doc), encoding="utf-8") - assert sitefiles.reingest_json(site) == {"changedEntries": 1} - man = json.loads((site / "models" / "m.json").read_text(encoding="utf-8")) - assert sitefiles.entry_text(site, man["files"]["model.json"]) == b'{"a":1}' - - -LIVE2D_BUNDLE = "/* live2d bundle */\n" - - -def fake_player(root, live2d=True, live2d_bundle=True): - files = { - sitefiles.READ_SET_SCRIPT: "// read set\n", - sitefiles.PLAYER_BUNDLES[0]: "/* bundle */\n", - f"{sitefiles.PLAYER_PAGE_DIR}/index.html": '\n', - f"{sitefiles.PLAYER_PAGE_DIR}/chart-list.js": 'import "../../src/element.js";\n', - } - if live2d: - files[f"{sitefiles.LIVE2D_PAGE_DIR}/index.html"] = '\n' - files[f"{sitefiles.LIVE2D_PAGE_DIR}/live2d.js"] = "import '../../src/live2d/define.js'; // @PLAYER_VERSION@\n" - if live2d_bundle: - files[sitefiles.LIVE2D_BUNDLES[0]] = LIVE2D_BUNDLE - files[sitefiles.LIVE2D_BUNDLES[0] + ".map"] = "{}" - for rel, text in files.items(): - p = root / rel - p.parent.mkdir(parents=True, exist_ok=True) - p.write_bytes(text.encode("utf-8")) - return root - - -def test_write_player_copies_the_live2d_page(tmp_path): - site = tmp_path / "site" - r = sitefiles.write_player(site, fake_player(tmp_path / "p")) - assert (r["pageFiles"], r["live2dPageFiles"]) == (2, 2) - version = hashlib.sha256(LIVE2D_BUNDLE.encode()).hexdigest()[:16] - page = site / sitefiles.LIVE2D_PAGE_SITE_DIR - assert (page / "live2d.js").read_text( - encoding="utf-8" - ) == f"import './ournotes-player.live2d.element.min.js'; // {version}\n" - assert (page / "ournotes-player.live2d.element.min.js").read_text(encoding="utf-8") == LIVE2D_BUNDLE - assert (page / "ournotes-player.live2d.element.min.js.map").is_file() - assert (site / "chart-list.js").read_text(encoding="utf-8") == 'import "./ournotes-player.element.min.js";\n' - r = sitefiles.write_player(tmp_path / "site2", fake_player(tmp_path / "p2", live2d=False, live2d_bundle=False)) - assert r["live2dPageFiles"] == 0 and not (tmp_path / "site2" / sitefiles.LIVE2D_PAGE_SITE_DIR).exists() - with pytest.raises(ConfigError, match="has the Live2D page but not"): - sitefiles.write_player(tmp_path / "site3", fake_player(tmp_path / "p3", live2d_bundle=False)) - player = fake_player(tmp_path / "p4") - (player / sitefiles.LIVE2D_PAGE_DIR / "extra.js").write_bytes(b'import "../../src/element.js";\n') - with pytest.raises(RuntimeError, match="beyond LIVE2D_PAGE_IMPORTS"): - sitefiles.write_player(tmp_path / "site4", player) - - -def test_player_pages_never_overwrite_site_data(tmp_path): - player = fake_player(tmp_path / "p") - (player / sitefiles.PLAYER_PAGE_DIR / "models.json").write_bytes(b"{}") - with pytest.raises(RuntimeError, match="would overwrite the site data"): - sitefiles.write_player(tmp_path / "site", player) - - -def put_manifest(site, mid, key, model=None, doc=None): - """A model manifest with a model.json only (`doc`, default a current one: MODEL_FORMAT, motionSync).""" - doc = {"format": webmodel.MODEL_FORMAT, "key": key, "motionSync": False} if doc is None else doc - entry = sitefiles.Store(site).put("model.json", json.dumps(doc).encode()) - (site / "models").mkdir(parents=True, exist_ok=True) - (site / "models" / f"{mid}.json").write_text( - json.dumps({"key": key, "model": model or {}, "files": {"model.json": entry}}), encoding="utf-8" - ) - - -def test_build_skips_existing_models_unless_forced(tmp_path, monkeypatch): - site = tmp_path / "site" - put_manifest(site, "adv_model_a", KEY_A) - done = [] - - def fake_task(mid, key, job, data=None): - done.append(mid) - put_manifest(site, mid, key) - return {"id": mid, "ok": True, "files": 0, "bytes": 0} - - monkeypatch.setattr(webmodel, "model_task", fake_task) - monkeypatch.setattr(webmodel, "_open", lambda cfg: (None, None)) - models = {"adv_model_a": KEY_A, "adv_model_b": KEY_B} - r = webmodel.build(site, models, Config(), fake_player(tmp_path / "p"), workers=1) - assert done == ["adv_model_b"] and r["modelsSkipped"] == ["adv_model_a"] and r["models"] == 2 - assert [m["id"] for m in r["modelsBuilt"]] == ["adv_model_b"] and r["live2dPageFiles"] == 2 - assert r["modelNames"] is None # no master data: no names - r = webmodel.build(site, models, Config(), fake_player(tmp_path / "p"), force=True, workers=1) - assert done == ["adv_model_b", "adv_model_a", "adv_model_b"] and r["modelsSkipped"] == [] - assert r["modelsRebuilt"] == [] # force: every model, none of them for a reason - with pytest.raises(ValueError, match="has the id"): - webmodel.build(site, {"other": KEY_A}, Config(), fake_player(tmp_path / "p"), workers=1) - - -def test_build_rebuilds_outdated_manifests(tmp_path, monkeypatch): - """An existing manifest whose model.json is older than MODEL_FORMAT, has no motionSync or cannot be read is built - again without force; a current one is skipped.""" - site = tmp_path / "site" - keys = {f"m{i}": f"Character/Live2D/g/m{i}/model/m{i}" for i in range(5)} - put_manifest(site, "m0", keys["m0"]) # current - put_manifest(site, "m1", keys["m1"], doc={"format": 1, "key": keys["m1"]}) # format 1 - put_manifest(site, "m2", keys["m2"], doc={"format": webmodel.MODEL_FORMAT, "key": keys["m2"]}) # no motionSync - (site / "models" / "m3.json").write_text( - json.dumps({"key": keys["m3"], "model": {}, "files": {}}), encoding="utf-8" - ) # no model.json - done = [] - - def fake_task(mid, key, job, data=None): - done.append(mid) - put_manifest(site, mid, key) - return {"id": mid, "ok": True, "files": 1, "bytes": 1} - - monkeypatch.setattr(webmodel, "model_task", fake_task) - monkeypatch.setattr(webmodel, "_open", lambda cfg: (None, None)) - assert webmodel.outdated(site, "m0") is None and webmodel.outdated(site, "m1") == "outdated" - r = webmodel.build(site, keys, Config(), fake_player(tmp_path / "p"), workers=1) - assert done == ["m1", "m2", "m3", "m4"] and r["modelsSkipped"] == ["m0"] - assert r["modelsRebuilt"] == [ - {"id": "m1", "reason": "outdated"}, - {"id": "m2", "reason": "outdated"}, - {"id": "m3", "reason": "unreadable"}, - ] - assert [m["id"] for m in r["modelsBuilt"]] == ["m1", "m2", "m3", "m4"] - r = webmodel.build(site, keys, Config(), fake_player(tmp_path / "p"), workers=1) # all current now - assert r["modelsSkipped"] == list(keys) and r["modelsRebuilt"] == [] and len(done) == 4 - - -# ---------------------------------------------------------------- command line -def run(argv, capsys): - code = 0 - try: - cli.main(argv) - except SystemExit as e: - code = e.code if isinstance(e.code, int) else 1 - out, err = capsys.readouterr() - return code, out, err - - -def test_web_argument_shape(): - p = cli.build_parser() - a = p.parse_args(["--json", "site", "build", "-o", "s", "--pair", "1:easy", "--live2d", "m1", "--live2d", KEY_B]) - assert a.pair == [(1, "easy")] and a.live2d == ["m1", KEY_B] and not a.all_live2d - a = p.parse_args(["--json", "site", "build", "-o", "s", "--all", "--all-live2d", "--name", "example"]) - assert a.all and a.all_live2d and a.live2d is None and a.name == "example" - for bad in (["--live2d", "m", "--all-live2d"], ["--pair", "1:easy", "--all"], ["--all", "--player-only"]): - with pytest.raises(SystemExit): - p.parse_args(["--json", "site", "build", "-o", "s", *bad]) - - -@pytest.mark.parametrize( - "argv, message", - [ - ([], "--all-live2d"), - (["--player-only", "--live2d", "m"], "cannot be combined"), - ], -) -def test_web_selection_errors(tmp_path, capsys, argv, message): - code, _, err = run(["--json", "site", "build", "-o", str(tmp_path / "s"), *argv], capsys) - assert code == 2 and message in err - - -class FakeCatalog: - def keys(self, prefix=""): - return [k for k in (KEY_A, KEY_B, "Live/Score/1") if k.startswith(prefix)] - - -# ---------------------------------------------------------------- the runtime export -def test_webmodel_uses_the_public_runtime_export(tmp_path, monkeypatch): - tree = ast.parse(inspect.getsource(webmodel)) - used = { - n.attr - for n in ast.walk(tree) - if isinstance(n, ast.Attribute) and isinstance(n.value, ast.Name) and n.value.id == "live2d" - } - assert "export_runtime" in used and not [a for a in used if a.startswith("_")] - assert not hasattr(live2d, "_extract_runtime") - rt = live2d.RuntimeExport({"name": "m", "moc3": "m.moc3"}, exporter=None, env=None, graph=None, classes={}) - monkeypatch.setattr(live2d, "export_runtime", lambda cat, key, out_dir: rt) - assert live2d.extract_runtime(None, KEY_A, tmp_path) == {"name": "m", "moc3": "m.moc3"} - - -def test_runtime_export_needs_the_apk(tmp_path): - with pytest.raises(ConfigError, match="paths.apk|NNNOTES_PATHS_APK"): - live2d.export_runtime(SimpleNamespace(apk=None), KEY_A, tmp_path) - - -# ---------------------------------------------------------------- names from the master data -def name_row(tid, ja, en, zh=None): - return { - "_id": tid, - "_japanese": ja, - "_english": en, - "_traditionalChinese": zh or ja, - "_simplifiedChinese": zh or ja, - "_korean": "", - } - - -def write_master(md, costumes, characters=None, texts=None): - md.mkdir(parents=True, exist_ok=True) - tables = { - "MasterCharacterCostume": costumes, - "MasterCharacter": characters - if characters is not None - else [ - {"_id": 1, "_nameTextID": "Name_A"}, - {"_id": 2, "_nameTextID": "Name_B"}, - {"_id": 3, "_nameTextID": "Gone"}, - ], - "MasterText": texts - if texts is not None - else [name_row("Name_A", "エー", "Ay", "艾"), name_row("Name_B", "ビー", "Bee")], - } - for name, rows in tables.items(): - (md / f"{name}.json").write_text(json.dumps({"_allData": rows}, ensure_ascii=False), encoding="utf-8") - return md - - -def costume(cid, path, char): - return {"_id": cid, "_characterID": char, "_costumeID": cid % 1000, "_isDefault": False, "_live2dPath": path} - - -PATH_A, PATH_B = KEY_A.removeprefix(webmodel.LIVE2D_PREFIX), KEY_B.removeprefix(webmodel.LIVE2D_PREFIX) - - -def test_model_names_from_the_costume_rows(tmp_path): - md = write_master( - tmp_path / "m", - [ - costume(1001, PATH_A, 1), - costume(1002, PATH_A, 1), # two costumes, one model - costume(2001, PATH_B, 2), - costume(2002, "002_adv/shared/model/shared", 2), - costume(3001, "002_adv/shared/model/shared", 1), - costume(3002, "003_adv/c/model/c", 3), # no name text - {"_id": 9, "_characterID": 1, "_live2dPath": ""}, - ], - ) - names = webmodel.model_names(md, "zh-Hant") - assert names == { - KEY_A: {"character": 1, "names": {"ja": "エー", "en": "Ay", "zh-Hant": "艾", "zh-Hans": "艾"}, "label": "艾"}, - KEY_B: { - "character": 2, - "names": {"ja": "ビー", "en": "Bee", "zh-Hant": "ビー", "zh-Hans": "ビー"}, - "label": "ビー", - }, - } - assert "label" not in webmodel.model_names(md)[KEY_A] - assert "label" not in webmodel.model_names(md, "ko")[KEY_A] # no Korean text - assert webmodel.model_facts(KEY_A, {"canvas": {}, "textures": [], "nodes": 0}, names[KEY_A]) == { - "group": "001_adv", - "canvas": {}, - "textures": 0, - "nodes": 0, - **names[KEY_A], - } - - -def test_master_names_follow_the_settings(tmp_path): - md = write_master(tmp_path / "m", [costume(1001, PATH_A, 1)]) - assert webmodel.master_names(Config()) is None - cfg = Config(overrides={("paths", "master"): str(md), ("catalog", "language"): "en"}) - assert webmodel.master_names(cfg) == { - KEY_A: {"character": 1, "names": {"ja": "エー", "en": "Ay", "zh-Hant": "艾", "zh-Hans": "艾"}, "label": "Ay"} - } - (md / "MasterCharacterCostume.json").unlink() - logged = [] - assert webmodel.master_names(cfg, log=logged.append) is None - assert logged == ["model names left out: MasterCharacterCostume.json not in the master data"] - with pytest.raises(ConfigError, match="paths.master"): - webmodel.master_names(Config(overrides={("paths", "master"): str(tmp_path / "absent")})) - - -def test_build_adds_names_and_refreshes_skipped_manifests(tmp_path, monkeypatch): - site = tmp_path / "site" - (site / "models").mkdir(parents=True) - put_manifest(site, "adv_model_a", KEY_A, model={"group": "001_adv", "character": 9, "label": "old"}) - jobs = [] - - def fake_task(mid, key, job, data=None): - jobs.append((mid, job["names"].get(key))) - put_manifest(site, mid, key) - return {"id": mid, "ok": True, "files": 0, "bytes": 0} - - monkeypatch.setattr(webmodel, "model_task", fake_task) - monkeypatch.setattr(webmodel, "_open", lambda cfg: (None, None)) - md = write_master(tmp_path / "m", [costume(1001, PATH_A, 1), costume(2001, PATH_B, 2)]) - cfg = Config(overrides={("paths", "master"): str(md), ("catalog", "language"): "ja"}) - r = webmodel.build(site, {"adv_model_a": KEY_A, "adv_model_b": KEY_B}, cfg, fake_player(tmp_path / "p"), workers=1) - assert r["modelNames"] == 2 and r["modelsSkipped"] == ["adv_model_a"] - assert jobs == [ - ( - "adv_model_b", - { - "character": 2, - "names": {"ja": "ビー", "en": "Bee", "zh-Hant": "ビー", "zh-Hans": "ビー"}, - "label": "ビー", - }, - ) - ] - man = json.loads((site / "models" / "adv_model_a.json").read_text(encoding="utf-8")) - assert man["model"] == { - "group": "001_adv", - "character": 1, - "names": {"ja": "エー", "en": "Ay", "zh-Hant": "艾", "zh-Hans": "艾"}, - "label": "エー", - } - index = json.loads((site / "models.json").read_text(encoding="utf-8")) - assert [(m["id"], m.get("label")) for m in index["models"]] == [("adv_model_a", "エー"), ("adv_model_b", None)] - # the same names again: the skipped manifest is not rewritten - before = (site / "models" / "adv_model_a.json").read_bytes() - assert webmodel.refresh_names(site, "adv_model_a", webmodel.model_names(md, "ja")[KEY_A]) is False - assert (site / "models" / "adv_model_a.json").read_bytes() == before - # a key the master data no longer maps loses the fields - assert webmodel.refresh_names(site, "adv_model_a", None) is True - man = json.loads((site / "models" / "adv_model_a.json").read_text(encoding="utf-8")) - assert man["model"] == {"group": "001_adv"} - - -# ---------------------------------------------------------------- model.json and the model sources of story.build -class FakeShader: - def __init__(self, name): - self.name = name - self.assets_file = SimpleNamespace(name=f"CAB-{name}") - - -def fake_export(monkeypatch, motion_sync=False): - """export_model's inputs made up: the runtime export writes fake_model's files, the player's mask materials are - MASKS and the shader dump indexes SHADER_INDEX (fake_model wrote its files); -> the player data.""" - - def export_runtime(cat, key, out_dir): - name = webmodel.model_id(key) - fake_model(out_dir, name, key=key, motion_sync=motion_sync) - prefab = json.loads((out_dir / f"{name}.prefab.json").read_text(encoding="utf-8")) - summary = { - "name": name, - "moc3": f"{name}.moc3", - "prefab": f"{name}.prefab.json", - "moc3Bytes": 10, - "textures": ["textures/p0-00.png", "textures/p1-11.png"], - "nodes": len(prefab["nodes"]), - } - return live2d.RuntimeExport(summary, SimpleNamespace(shaders={"S/Lit": FakeShader("S/Lit")}), None, None, {}) - - class FakeExporter: - def __init__(self, cat, out_dir, player=None): - self.shaders = {"S/Mask": FakeShader("S/Mask")} - - def material(self, path): - return MASKS[{v: k for k, v in webmodel.RESOURCES.items()}[path]] - - def dump_objects(objs, sdir, name, index): - index.extend(r for r in SHADER_INDEX if r["name"] == objs[0].name) - - def write_index(index, sdir): - jsonio.write_json(sdir / "shaders.json", index) - - monkeypatch.setattr(live2d, "export_runtime", export_runtime) - monkeypatch.setattr(webmodel, "Exporter", FakeExporter) - monkeypatch.setattr(webmodel.shader_mod, "dump_objects", dump_objects) - monkeypatch.setattr(webmodel.shader_mod, "write_index", write_index) - return SimpleNamespace(resource=lambda path: path) - - -def model_files(name): - return [f.replace("m.", f"{name}.") for f in MASKED_FILES] - - -@pytest.mark.parametrize("motion_sync", [False, True]) -def test_export_model_writes_model_json(tmp_path, monkeypatch, motion_sync): - player = fake_export(monkeypatch, motion_sync) - r = webmodel.export_model(None, player, KEY_A, tmp_path / "x") - doc = json.loads((tmp_path / "x" / webmodel.MODEL_INDEX).read_text(encoding="utf-8")) - assert doc == { - "format": 2, - "name": "adv_model_a", - "key": KEY_A, - "moc3": "adv_model_a.moc3", - "prefab": "adv_model_a.prefab.json", - "textures": ["textures/p0-00.png"], - "canvas": {"width": 1.0}, - "shaders": "shaders/shaders.json", - "resources": MASKS, - "motionSync": motion_sync, - } - assert r["files"] == model_files("adv_model_a") - - -def site_with(tmp_path, *models): - """A site with the model manifests of fake_model for each (key, motion_sync).""" - site = tmp_path / "site" - (site / sitefiles.MODELS_DIR).mkdir(parents=True, exist_ok=True) - for key, ms in models: - mid = webmodel.model_id(key) - webmodel.ingest( - sitefiles.Store(site), site, mid, key, *fake_model(tmp_path / mid, mid, key=key, motion_sync=ms) - ) - return site - - -def test_site_models_read_model_json(tmp_path): - site = site_with(tmp_path, (KEY_A, True), (KEY_B, False)) - src = webmodel.SiteModels(site) - assert src.ensure(KEY_A) == {"id": "adv_model_a", "motionSync": True} - assert src.ensure(KEY_B) == {"id": "adv_model_b", "motionSync": False} - assert src.story_fields(tmp_path / "story") == {} - with pytest.raises(RuntimeError, match="no model manifest models/c.json in the site"): - src.ensure("Character/Live2D/g/c/model/c") - # a model.json without motionSync (a model built before it had one) - man = json.loads((site / "models" / "adv_model_b.json").read_text(encoding="utf-8")) - doc = { - k: v for k, v in json.loads(sitefiles.entry_text(site, man["files"]["model.json"])).items() if k != "motionSync" - } - man["files"]["model.json"] = sitefiles.Store(site).put("model.json", json.dumps({**doc, "format": 1}).encode()) - (site / "models" / "adv_model_b.json").write_text(json.dumps(man), encoding="utf-8") - with pytest.raises(RuntimeError, match="format 1 has no motionSync; export the model again"): - src.ensure(KEY_B) - # the manifest of another key with the same id - with pytest.raises(RuntimeError, match="not of Character/Live2D/other/adv_model_a"): - src.ensure("Character/Live2D/other/adv_model_a/model/adv_model_a") From 80458d5e9a9a15a35615b6e7a3b6f0bbe0d80e8a Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 18 Oct 2026 16:26:28 +0800 Subject: [PATCH 71/91] refactor(fonts): open catalog inputs through the source session --- scripts/gen_font_settings.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/gen_font_settings.py b/scripts/gen_font_settings.py index 428ef86..e431593 100644 --- a/scripts/gen_font_settings.py +++ b/scripts/gen_font_settings.py @@ -27,10 +27,11 @@ import tempfile from pathlib import Path -from nnnotes.site.web import open_data from nnnotes.source.config import Config, use +from nnnotes.source.session import Session from nnnotes.unity import fontsettings, tmpfont from nnnotes.unity.export import Exporter +from nnnotes.unity.player import PlayerData def main(argv=None) -> None: @@ -42,7 +43,9 @@ def main(argv=None) -> None: p.add_argument("-o", "--out", type=Path, default=fontsettings.FILE, help="output file") args = p.parse_args(argv) cfg = use(Config.load()) - cat, _, player = open_data(cfg, args.region) + session = Session(cfg) + cat = session.catalog(region=args.region) + player = PlayerData(session.apk(args.region)) with tempfile.TemporaryDirectory(prefix="fontsettings-") as work: ex = Exporter(cat, Path(work), player=player, textures="deferred") doc = tmpfont.build_font_settings(ex, player, args.game_version or player.game_version) From 916ff41bc98041c643c630fbf1b13541dd1c23ed Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 18 Oct 2026 16:26:28 +0800 Subject: [PATCH 72/91] fix(site): include story lifecycle commands in listings --- src/nnnotes/cli/site.py | 10 ++++++++-- src/nnnotes/content/sitefacts.py | 8 ++++++-- src/nnnotes/site/build.py | 6 ++++++ tests/site/test_cli.py | 22 ++++++++++++++++++++++ 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/src/nnnotes/cli/site.py b/src/nnnotes/cli/site.py index d58f536..3f75ca3 100644 --- a/src/nnnotes/cli/site.py +++ b/src/nnnotes/cli/site.py @@ -54,7 +54,7 @@ def cmd_build(args, cfg): from ..pipeline import live as lp from ..pipeline import story as sp from ..site import catalog, pages - from ..site.build import Builder, document, documents, model_stats, size + from ..site.build import Builder, document, documents, model_stats, object_document, size from ..source.master import table charts = bool(args.pair or args.all) @@ -189,8 +189,14 @@ def cmd_build(args, cfg): source = sp.StoryStage().manifest(str(adv_id)) manifests = documents(builder.depot, entries) episode = document(builder.depot, manifests, source, "episode.json") + story_doc = document(builder.depot, manifests, source, "story.json") + scene_path = story_doc["parts"]["scene"] + scene = document(builder.depot, manifests, scene_path, "scene.json") + settings = object_document( + builder.depot, manifests, scene_path, scene["settings"]["playerSettings"] + ) facts = { - **sitefacts.story(episode, groups.get(adv_id, []), wanted, lang), + **sitefacts.story(episode, groups.get(adv_id, []), wanted, lang, settings), "audio": not args.no_audio, "audioFormat": args.format, "fonts": args.fonts, diff --git a/src/nnnotes/content/sitefacts.py b/src/nnnotes/content/sitefacts.py index af50544..388c10c 100644 --- a/src/nnnotes/content/sitefacts.py +++ b/src/nnnotes/content/sitefacts.py @@ -3,6 +3,7 @@ from pathlib import Path from ..source import languages, master +from .advcommand import name as command_name from .live2d import LIVE2D_PREFIX COSTUME_TABLE = "MasterCharacterCostume" @@ -56,8 +57,11 @@ def chart(live, music, language): } -def story(episode, groups, selected_languages, language): +def story(episode, groups, selected_languages, language, player_settings): title, row = episode.get("title") or {}, episode["master"] + commands = {"TalkWindow", *(c["cmd"] for c in episode["commands"] if not c.get("IgnoreData"))} + for field in ["_initializeEpisodes", "_finalizeEpisodes"]: + commands.update(command_name(r["Command"]) for r in player_settings[field]) return { "advId": episode["advId"], "asset": episode["asset"], @@ -69,7 +73,7 @@ def story(episode, groups, selected_languages, language): if isinstance(title.get(languages.column(c)[1:]), str) and title[languages.column(c)[1:]] }, "groups": groups, - "commands": sorted({c["cmd"] for c in episode["commands"] if not c.get("IgnoreData")}), + "commands": sorted(commands), "commandCount": episode["commandCount"], "language": language, "languages": list(selected_languages), diff --git a/src/nnnotes/site/build.py b/src/nnnotes/site/build.py index fd7b328..16b3375 100644 --- a/src/nnnotes/site/build.py +++ b/src/nnnotes/site/build.py @@ -44,6 +44,12 @@ def document(depot, manifests, path, logical): return json.loads(depot.read(entry["asset"].split("/")[-1].split(".")[0])) +def object_document(depot, manifests, path, oid): + """JSON of a standalone object in a decoded product layout.""" + entry = manifests[path]["objects"][oid]["files"]["json"] + return json.loads(depot.read(entry["asset"].split("/")[-1].split(".")[0])) + + def model_stats(snapshot, source): manifest = snapshot.documents[source] files = [*manifest["files"].values(), *(e for o in manifest["objects"].values() for e in o["files"].values())] diff --git a/tests/site/test_cli.py b/tests/site/test_cli.py index a07969f..4a92fbc 100644 --- a/tests/site/test_cli.py +++ b/tests/site/test_cli.py @@ -72,6 +72,26 @@ def outputs(args, regional, ws, expand=None): if hasattr(args, "adv_id"): assert expand == "expand" manifests = graph(depot) + manifests["storyscenes/default.json"] = product( + depot, + "story.scene", + "default", + {"scene.json": {"settings": {"playerSettings": "example:1"}}}, + { + "example:1": { + "class": "MonoBehaviour", + "files": { + "json": file( + depot, + { + "_initializeEpisodes": [{"Command": 15}], + "_finalizeEpisodes": [{"Command": 5}], + }, + ) + }, + } + }, + ) manifests["storyui/en.json"] = product(depot, "story.ui", "en", {"ui.json": {}}) manifests["storyfonts/1_en.json"] = product(depot, "story.fonts", "1/en", {"fonts.json": {}}) manifests["stories/1.json"]["files"]["episode.json"] = file( @@ -160,6 +180,8 @@ def test_cli_story_passes_fonts_languages_and_audio_policy(command, tmp_path): assert args.audio_format == "aac" and args.fonts == "open" man = json.loads((tmp_path / "site/stories/1.json").read_bytes()) assert man["audio"] is False and man["language"] == "ja" + listing = json.loads((tmp_path / "site/stories.json").read_bytes()) + assert listing["stories"][0]["commands"] == ["Bgm", "FadeOut", "TalkWindow"] def test_player_only_requires_only_depot_and_player_settings(command, tmp_path): From 7df32ff52f45939a8c3b7cfecd943b93538d8dc1 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 18 Oct 2026 16:31:01 +0800 Subject: [PATCH 73/91] refactor(cache): keep conversion memoization within each process --- CHANGELOG.md | 2 +- docs/architecture.md | 2 + src/nnnotes/content/storyfonts.py | 4 +- src/nnnotes/core/cache.py | 79 +++++-------------------------- src/nnnotes/formats/cri.py | 2 +- src/nnnotes/formats/shader.py | 2 +- src/nnnotes/unity/export.py | 4 +- tests/core/test_cache.py | 19 ++------ tests/unity/test_speed_exact.py | 12 +++-- 9 files changed, 34 insertions(+), 92 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 338640f..2ccbec8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor v ### Architecture - Site construction uses shared product requests, named depot catalogs and ordered layout publication. Player page collection and catalog model queries have dedicated interfaces. +- Conversion memoization is bounded within each process, with persistent reuse provided by depot task results. ## [0.2.1](https://github.com/MetaSekaiLab/nnnotes/compare/v0.2.0...v0.2.1) - 2026-10-13 @@ -207,4 +208,3 @@ schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor v - List MetaMiku as an author ([cc43e13](https://github.com/MetaSekaiLab/nnnotes/commit/cc43e132875206d02d8665b19c06b27255ab0037)) - Point the project links at MetaSekaiLab/nnnotes ([a2b8304](https://github.com/MetaSekaiLab/nnnotes/commit/a2b83040bcf06771028bbfd5e90c10c225bde3e9)) - English package readme; check commits on pull requests only ([e602dd5](https://github.com/MetaSekaiLab/nnnotes/commit/e602dd5ebf6dda0f18da5ace54a02dbb893a796f)) - diff --git a/docs/architecture.md b/docs/architecture.md index 3b5056c..385d351 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -44,6 +44,8 @@ decrypted bundles in the depot's `downloads/` directory. The site's `Builder` composes product snapshots, catalog indexes and player page files. Product assembly and conversion belong to the pipeline and content layers. `catalog_models` queries model subjects from a source catalog. +Converters use bounded process-local memoization for repeated reads and computations. Persistent reuse is keyed by depot tasks and their content inputs. Clearing a process memo releases its values; a later task can reuse its committed depot result. + `export episode`, `export shader`, `export audio` and `music-data` expose subject-specific data and conversions through the Session. All commands use the readers and converters below the pipeline layer. ## Outputs diff --git a/src/nnnotes/content/storyfonts.py b/src/nnnotes/content/storyfonts.py index 213af91..f60380f 100644 --- a/src/nnnotes/content/storyfonts.py +++ b/src/nnnotes/content/storyfonts.py @@ -41,7 +41,7 @@ has no sprite asset. Same inputs (font file, episode, game data) and library versions give the same bytes; generated glyph cells are kept -in the process cache (disk layer when the build configures one) by font file, sizes and glyph. +in the process cache by font file, sizes and glyph. """ from __future__ import annotations @@ -85,7 +85,7 @@ UI_RUBY_FIELDS = ("_rubyMarginTop",) KOREAN_ADJUST_CLASS = "LocalizeKoreanAdjust" # the font style a text gets in Korean (LocalizeKoreanAdjust.Apply) UI_TEXT_CLASSES = ("UIText", "UIRubyText") # Fwk.UI.UIText and its subclass -GLYPHS = cache.bucket("storyglyphs", disk=True) +GLYPHS = cache.bucket("storyglyphs") FACE_KEYS = ( "m_FaceIndex", "m_FamilyName", diff --git a/src/nnnotes/core/cache.py b/src/nnnotes/core/cache.py index e1ee210..6d98961 100644 --- a/src/nnnotes/core/cache.py +++ b/src/nnnotes/core/cache.py @@ -2,14 +2,11 @@ A bucket maps a content key to a value computed from exactly the inputs the key covers, so a hit returns what the computation would return: the extractors' outputs stay byte-identical with or without the cache. Buckets live in the -process (LRU, bounded by a byte budget, thread-safe). Buckets that hold bytes can also keep them on disk -(`configure(directory=...)`): processes of one build then share the results, and a later run with the same inputs -reuses them. Keys of the disk layer are salted with the source of the module that produces the value and the -versions of the libraries it calls, so a code or library change never reads an older result. +process (LRU, bounded by a byte budget, thread-safe). Persistent task results belong to the depot. from nnnotes.core import cache cache.configure(enabled=False) # every get misses, nothing is stored - cache.configure(memory_mb=512, directory="work/cache") + cache.configure(memory_mb=512) PNG = cache.bucket("png", salt=cache.source_salt(__file__)) k = PNG.key(data, width, height) png = PNG.get(k) @@ -30,7 +27,6 @@ from pathlib import Path from typing import Any, Protocol, TypedDict, cast -DISK_MAGIC = b"NNC1" DEFAULT_MEMORY_MB = 128 @@ -39,7 +35,6 @@ class Settings(TypedDict): enabled: bool memory: int # byte budget of each bucket - directory: Path | None # disk layer of the byte buckets closures: int # loaded bundle closures kept per thread closure_bytes: int # their bundle bytes @@ -47,7 +42,6 @@ class Settings(TypedDict): _settings: Settings = { "enabled": True, "memory": DEFAULT_MEMORY_MB << 20, - "directory": None, "closures": 4, "closure_bytes": 512 << 20, } @@ -59,20 +53,17 @@ def configure( *, enabled: bool | None = None, memory_mb: int | None = None, - directory: str | os.PathLike[str] | None = None, closures: int | None = None, closure_mb: int | None = None, ) -> Settings: - """Set the caches of this process: `enabled`; `memory_mb` (byte budget of each bucket); `directory` (disk layer - of the byte buckets; "" or None for none); `closures` / `closure_mb` (loaded bundle closures kept by + """Set the caches of this process: `enabled`; `memory_mb` (byte budget of each bucket); + `closures` / `closure_mb` (loaded bundle closures kept by unity.load_closure, per thread: count and bundle bytes). Clears the buckets. Returns the settings.""" with _lock: if enabled is not None: _settings["enabled"] = bool(enabled) if memory_mb is not None: _settings["memory"] = max(0, int(memory_mb)) << 20 - if directory is not None: - _settings["directory"] = Path(directory) if directory else None if closures is not None: _settings["closures"] = max(0, int(closures)) if closure_mb is not None: @@ -91,18 +82,18 @@ def enabled() -> bool: return _settings["enabled"] -def bucket(name: str, *, salt: str = "", disk: bool = False, max_item: int | None = None) -> Bucket: - """The process's bucket `name` (created on first use). `disk`: its values are bytes and may be kept on disk; +def bucket(name: str, *, salt: str = "", max_item: int | None = None) -> Bucket: + """The process's bucket `name` (created on first use). `max_item`: values larger than this many bytes are not kept (default: a quarter of the budget).""" with _lock: b = _buckets.get(name) if b is None: - b = _buckets[name] = Bucket(name, salt, disk, max_item) + b = _buckets[name] = Bucket(name, salt, max_item) return b def stats() -> dict[str, dict[str, int]]: - """Per bucket: items, bytes, hits, misses, disk hits, disk writes, disk write errors.""" + """Per bucket: items, bytes, hits and misses.""" return {n: b.stats() for n, b in sorted(_buckets.items())} @@ -174,12 +165,12 @@ def file_id(path: str | os.PathLike[str]) -> tuple[str, int, int]: class Bucket: """A memo cache of values by content key (see the module documentation).""" - def __init__(self, name: str, salt: str, disk: bool, max_item: int | None): - self.name, self.salt, self.disk, self._max_item = name, salt, disk, max_item + def __init__(self, name: str, salt: str, max_item: int | None): + self.name, self.salt, self._max_item = name, salt, max_item self._items: OrderedDict[str, tuple[Any, int]] = OrderedDict() # key -> (value, byte size) self._bytes = 0 self._lock = threading.Lock() - self.hits = self.misses = self.disk_hits = self.disk_writes = self.disk_errors = 0 + self.hits = self.misses = 0 def key(self, *parts: object) -> str: return key(self.name, self.salt, parts) @@ -198,32 +189,16 @@ def get(self, k: str) -> Any: self._items.move_to_end(k) self.hits += 1 return hit[0] - directory = _settings["directory"] - if self.disk and directory is not None: - data = _disk_read(self._path(directory, k)) - if data is not None: - with self._lock: - self.disk_hits += 1 - self._remember(k, data, len(data)) - return data with self._lock: self.misses += 1 return None def put(self, k: str, value: Any, size: int | None = None) -> None: - """Store `value` (bytes for a disk bucket) under `k`; `size`: its byte size (default len(value)).""" + """Store `value` under `k`; `size`: its byte size (default len(value)).""" if not _settings["enabled"]: return n = len(value) if size is None else int(size) self._remember(k, value, n) - directory = _settings["directory"] - if self.disk and directory is not None and n <= max(self._limit(), 64 << 20): - ok = _disk_write(self._path(directory, k), value) - with self._lock: - if ok: - self.disk_writes += 1 - else: - self.disk_errors += 1 def _remember(self, k: str, value: Any, n: int) -> None: if n > self._limit(): @@ -239,9 +214,6 @@ def _remember(self, k: str, value: Any, n: int) -> None: _, (_, m) = self._items.popitem(last=False) self._bytes -= m - def _path(self, directory: Path, k: str) -> Path: - return directory / self.name / k[:2] / k - def clear(self) -> None: with self._lock: self._items.clear() @@ -254,23 +226,9 @@ def stats(self) -> dict[str, int]: "bytes": self._bytes, "hits": self.hits, "misses": self.misses, - "diskHits": self.disk_hits, - "diskWrites": self.disk_writes, - "diskErrors": self.disk_errors, } -def _disk_read(p: Path) -> bytes | None: - try: - raw = p.read_bytes() - except OSError: - return None - if len(raw) < 12 or raw[:4] != DISK_MAGIC: - return None - (n,) = struct.unpack_from(" Path: """A new temporary file name next to `dst` for writing it atomically: short whatever `dst`'s name (a long name plus a suffix would pass the file system's name limit), random, so unique across processes and threads.""" @@ -293,20 +251,9 @@ def write_atomic(dst: Path, data: bytes) -> None: raise -def _disk_write(p: Path, data: bytes) -> bool: - """Write a disk entry atomically; False when the file system refused (counted as diskErrors: a cache that - cannot write only misses later, the result is the same).""" - try: - p.parent.mkdir(parents=True, exist_ok=True) - write_atomic(p, DISK_MAGIC + struct.pack(" bytes: - """One bytes value from a header and blobs (for disk buckets whose value is several files).""" + """One bytes value from a header and blobs.""" out = [struct.pack(" ACB / AWB bytes, layout _keys: dict[tuple, int] = {} _keys_lock = threading.Lock() diff --git a/src/nnnotes/formats/shader.py b/src/nnnotes/formats/shader.py index 85ad550..5f82343 100644 --- a/src/nnnotes/formats/shader.py +++ b/src/nnnotes/formats/shader.py @@ -32,7 +32,7 @@ from ..core import cache from ..core.jsonio import dumps, write_json -DUMPS = cache.bucket("shaders", salt=cache.source_salt(__file__, "UnityPy"), disk=True) +DUMPS = cache.bucket("shaders", salt=cache.source_salt(__file__, "UnityPy")) # UnityEngine.Rendering ShaderCompilerPlatform value -> tag PLATFORM = { diff --git a/src/nnnotes/unity/export.py b/src/nnnotes/unity/export.py index ef961ca..8ba2707 100644 --- a/src/nnnotes/unity/export.py +++ b/src/nnnotes/unity/export.py @@ -55,8 +55,8 @@ HEADER = ("m_GameObject", "m_Script") _SCALARS = (int, float, str, bool, type(None)) _LIBS = ("UnityPy", "Pillow", "texture2ddecoder", "astc-encoder-py", "etcpak", "numpy") -# decoded textures and their PNG encodings by content (export.py's source and the libraries salt the disk layer) -PNG = cache.bucket("png", salt=cache.source_salt(__file__, *_LIBS), disk=True) +# decoded textures and PNG encodings memoized by content within this process +PNG = cache.bucket("png", salt=cache.source_salt(__file__, *_LIBS)) PIXELS = cache.bucket("pixels", salt=cache.source_salt(__file__, *_LIBS)) diff --git a/tests/core/test_cache.py b/tests/core/test_cache.py index 314ebc2..4ab16db 100644 --- a/tests/core/test_cache.py +++ b/tests/core/test_cache.py @@ -9,12 +9,11 @@ def fresh_cache(): """Every test starts with the default settings and empty buckets, and leaves them so.""" before = cache.settings() - cache.configure(enabled=True, memory_mb=cache.DEFAULT_MEMORY_MB, directory="") + cache.configure(enabled=True, memory_mb=cache.DEFAULT_MEMORY_MB) yield cache.configure( enabled=before["enabled"], memory_mb=before["memory"] >> 20, - directory=before["directory"] or "", closures=before["closures"], closure_mb=before["closure_bytes"] >> 20, ) @@ -66,22 +65,14 @@ def test_non_bytes_values_with_size(): assert b.get(b.key("o")) == {"a": 1} -def test_disk_layer_shared_and_checked(tmp_path): - cache.configure(directory=tmp_path / "c") - b = cache.bucket("t-disk", salt="s1", disk=True) +def test_bucket_clear_releases_values(): + b = cache.bucket("t-clear", salt="s1") k = b.key("v") b.put(k, b"payload") - files = [p for p in (tmp_path / "c").rglob("*") if p.is_file()] - assert len(files) == 1 and not files[0].name.endswith(".part") - cache.clear() # memory gone: the disk answers - assert b.get(k) == b"payload" and b.stats()["diskHits"] == 1 - other = cache.bucket("t-disk-other-salt", salt="s2", disk=True) - assert other.get(other.key("v")) is None # another salt: another key - files[0].write_bytes(files[0].read_bytes()[:-1]) # a truncated file is a miss + assert b.get(k) == b"payload" and b.stats()["items"] == 1 cache.clear() assert b.get(k) is None - files[0].write_bytes(b"garbage") - assert b.get(k) is None + assert b.stats()["items"] == b.stats()["bytes"] == 0 def test_pack_round_trip(): diff --git a/tests/unity/test_speed_exact.py b/tests/unity/test_speed_exact.py index 45c9a4b..6f8c112 100644 --- a/tests/unity/test_speed_exact.py +++ b/tests/unity/test_speed_exact.py @@ -28,10 +28,10 @@ @pytest.fixture(autouse=True) def fresh_cache(): - cache.configure(enabled=True, memory_mb=cache.DEFAULT_MEMORY_MB, directory="") + cache.configure(enabled=True, memory_mb=cache.DEFAULT_MEMORY_MB) unity.clear_closures() yield - cache.configure(enabled=True, memory_mb=cache.DEFAULT_MEMORY_MB, directory="", closures=4, closure_mb=512) + cache.configure(enabled=True, memory_mb=cache.DEFAULT_MEMORY_MB, closures=4, closure_mb=512) unity.clear_closures() @@ -561,13 +561,15 @@ def test_decode_parallel_and_cached_equal_sequential(tmp_path, tools, fmt): assert (len(tools.calls) == n) == (run == "hit") # replayed: no tool ran -def test_decode_disk_cache_and_levels(tmp_path, tools): - cache.configure(directory=tmp_path / "c") +def test_decode_memo_and_levels(tmp_path, tools): cri.decode(None, "S", tmp_path / "a", key=0, flac_level=0) - cache.clear() # another process: the disk layer n = len(tools.calls) cri.decode(None, "S", tmp_path / "b", key=0, flac_level=0) assert len(tools.calls) == n and tree(tmp_path / "a") == tree(tmp_path / "b") + cache.clear() + cri.decode(None, "S", tmp_path / "fresh", key=0, flac_level=0) + assert len(tools.calls) > n and tree(tmp_path / "a") == tree(tmp_path / "fresh") + n = len(tools.calls) cri.decode(None, "S", tmp_path / "c12", key=0) # another level: decoded again assert len(tools.calls) > n assert tree(tmp_path / "a")["a.flac"] != tree(tmp_path / "c12")["a.flac"] From 89a0a1b431f4dcfd1394cbba248ea8990ceb7618 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 18 Oct 2026 16:33:53 +0800 Subject: [PATCH 74/91] test(cache): cover atomic writes through the shared file helpers --- tests/source/test_catalog_lazy.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/source/test_catalog_lazy.py b/tests/source/test_catalog_lazy.py index c061698..048de6d 100644 --- a/tests/source/test_catalog_lazy.py +++ b/tests/source/test_catalog_lazy.py @@ -171,14 +171,6 @@ def refuse(a, b): assert list(tmp_path.iterdir()) == [] -def test_disk_cache_entries_use_the_short_temporary_name(tmp_path): - from nnnotes.core import cache - - p = tmp_path / "ab" / ("c" * 64) - assert cache._disk_write(p, b"payload") and cache._disk_read(p) == b"payload" - assert sorted(x.name for x in p.parent.iterdir()) == [p.name] - - # ---------------------------------------------------------------- the command line def run(argv, capsys): code = 0 From 7e512172f583f76769f7dcad5743de29b81c31a5 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 18 Oct 2026 19:15:46 +0800 Subject: [PATCH 75/91] refactor(export): unify object products and font processing --- CHANGELOG.md | 4 + docs/architecture.md | 4 +- docs/products.md | 2 + scripts/atom_eval.py | 4 +- scripts/gen_font_settings.py | 69 +- src/nnnotes/content/advmedia.py | 186 --- src/nnnotes/content/advscene.py | 108 -- src/nnnotes/content/advui.py | 1348 +------------------ src/nnnotes/content/advvideo.py | 132 -- src/nnnotes/content/hostspot.py | 372 ------ src/nnnotes/content/live.py | 162 +-- src/nnnotes/content/liveaudio.py | 137 +- src/nnnotes/content/livenotes.py | 277 +--- src/nnnotes/content/livescene.py | 511 +------ src/nnnotes/content/liveui.py | 324 +---- src/nnnotes/content/story.py | 164 --- src/nnnotes/content/storyfontproduct.py | 2 +- src/nnnotes/content/storyfonts.py | 720 +--------- src/nnnotes/content/storyhost.py | 1506 --------------------- src/nnnotes/content/tmpassets.py | 28 +- src/nnnotes/musicdata/__init__.py | 8 +- src/nnnotes/unity/export.py | 1622 ----------------------- src/nnnotes/unity/fontsettings.py | 29 +- src/nnnotes/unity/live2d.py | 99 -- src/nnnotes/unity/reader.py | 14 + src/nnnotes/unity/texels.py | 182 +++ src/nnnotes/unity/textstyle.py | 134 +- src/nnnotes/unity/tmpfont.py | 195 +-- tests/cli/test_cli.py | 30 - tests/content/test_hostspot.py | 509 ------- tests/content/test_live_options.py | 126 +- tests/content/test_story_fonts.py | 620 +-------- tests/content/test_story_host.py | 798 ----------- tests/content/test_story_kinds.py | 141 -- tests/content/test_story_ui.py | 714 ---------- tests/content/test_tmpassets.py | 2 +- tests/formats/test_crilips.py | 28 - tests/formats/test_usm.py | 136 -- tests/pipeline/test_cristages.py | 2 +- tests/pipeline/test_objexport.py | 9 +- tests/pipeline/test_views.py | 2 +- tests/site/test_regions.py | 7 +- tests/unity/test_atoms_texture.py | 19 +- tests/unity/test_container.py | 66 + tests/unity/test_fontsettings.py | 12 +- tests/unity/test_mesh_texture_guards.py | 3 - tests/unity/test_speed_exact.py | 70 +- tests/unity/test_textstyle.py | 148 +-- 48 files changed, 366 insertions(+), 11419 deletions(-) delete mode 100644 src/nnnotes/content/advmedia.py delete mode 100644 src/nnnotes/content/advscene.py delete mode 100644 src/nnnotes/content/advvideo.py delete mode 100644 src/nnnotes/content/hostspot.py delete mode 100644 src/nnnotes/content/story.py delete mode 100644 src/nnnotes/content/storyhost.py delete mode 100644 src/nnnotes/unity/export.py delete mode 100644 src/nnnotes/unity/live2d.py create mode 100644 src/nnnotes/unity/texels.py delete mode 100644 tests/content/test_hostspot.py delete mode 100644 tests/content/test_story_host.py delete mode 100644 tests/content/test_story_kinds.py create mode 100644 tests/unity/test_container.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ccbec8..423fb79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor v ### Features +- Store content, task inputs, results and catalog versions in a persistent depot with optional remote storage, verified collection and bounded source downloads. +- Assemble Live2D, room, spot, live, story and UI products from shared object tables and explicit manifest references. +- Generate localized live and story text products with game or supplied fonts, shared glyph atlases and material records. - Build named sites through depot product, player read-set, asset and index tasks. - Retain unselected regional topics, publish immutable product dependencies in order, and restore complete sites from their saved depot layouts. - Offer Live variant products from chart manifests and select story UI/font languages explicitly. @@ -16,6 +19,7 @@ schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor v - Site construction uses shared product requests, named depot catalogs and ordered layout publication. Player page collection and catalog model queries have dedicated interfaces. - Conversion memoization is bounded within each process, with persistent reuse provided by depot task results. +- Product assembly consumes stored class-converter outputs. Font lookup, texel packing and material construction share object-based implementations. ## [0.2.1](https://github.com/MetaSekaiLab/nnnotes/compare/v0.2.0...v0.2.1) - 2026-10-13 diff --git a/docs/architecture.md b/docs/architecture.md index 385d351..d3e28b9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -12,7 +12,7 @@ subpackage of `nnnotes` and imports only the layers below it. | Products | `nnnotes.site`, `nnnotes.musicdata` | the ournotes-player site (charts, models, stories, indexes); the music data file and its deck statistics | | Pipeline | `nnnotes.pipeline` | stages, task descriptions and results, the content-addressed depot, plans, layouts, the catalog versions, the views of the master data | | Content | `nnnotes.content` | the game's subjects as exports: ADV episodes and stories, lives and scores, spots, rooms, the UI library | -| Unity objects | `nnnotes.unity` | bundle reading, object ids, the atoms (the pure functions objects are converted with: texture decoding, sprite images, mesh arrays, glTF, PNG, file sniffing), the converters (one per Unity class, `nnnotes.unity.convert`), clip bindings named in the hierarchy that plays a clip (`nnnotes.unity.bindings`), the type trees shipped for the player's script classes (`nnnotes.unity.scripttypes`), the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | +| Unity objects | `nnnotes.unity` | bundle reading, object ids, atomic texture/sprite/mesh/glTF/PNG conversions, class converters (`nnnotes.unity.convert`), clip bindings (`nnnotes.unity.bindings`), script type trees (`nnnotes.unity.scripttypes`), player data, text styles, font protocols and texel packing | | File formats | `nnnotes.formats` | formats read from bytes: CRI ACB/AWB and HCA, USM movies, CRI Lips data, shaders, motions | | Sources | `nnnotes.source` | the configuration, the APK, Addressables catalogs and bundles, the game API, master data, languages, `Session` | | Core | `nnnotes.core` | the one JSON writer, canonical JSON and its hashes, media types, the process caches and the downloads cache | @@ -40,6 +40,8 @@ decrypted bundles in the depot's `downloads/` directory. `export live`, `export story`, `export live2d`, `export spot`, `export room` and `export ui` request the products of selected subjects through that same pipeline. Stories first produce an episode, then request its referenced products. Shared objects and unchanged products reuse committed tasks. +Class converters emit object records and binary files. Product assemblers read those stored records through an object table and identify shared dependencies by object id and product manifest. Text products use the same font lookup, material and in-memory atlas functions. A product request contains its complete object and media dependencies before assembly. + `site build` uses the same product requests, then runs player read-set, asset, document and index tasks. A named catalog retains unselected topics and references immutable products. Publication writes dependencies before entry manifests and indexes. The successful run stores the complete layout so a new output directory can be restored through the depot. The site's `Builder` composes product snapshots, catalog indexes and player page files. Product assembly and conversion belong to the pipeline and content layers. `catalog_models` queries model subjects from a source catalog. diff --git a/docs/products.md b/docs/products.md index 6a76c50..95597d3 100644 --- a/docs/products.md +++ b/docs/products.md @@ -543,6 +543,8 @@ every episode whose key the selection holds. A stage with `INCLUDE_FONTS = True` requests traversal through its referenced font objects. The request records these roots in `font_roots`; unit selection and assembly follow the same rule. Other products keep their own font selection behavior. +The maintainer tool `scripts/gen_font_settings.py TASK.json --depot DEPOT -o OUTPUT.json` generates a font settings table from a saved text-product task. It reads the task's object table, font-bundle identities and APK manifest through the depot. The task must contain the complete font inputs, as used by `story.fonts` or `live.ui`. + ## Site snapshots `nnnotes.site.products.Snapshot` takes a depot and product-layout manifests keyed by path. It builds each product's diff --git a/scripts/atom_eval.py b/scripts/atom_eval.py index 1e23446..4ed1e51 100644 --- a/scripts/atom_eval.py +++ b/scripts/atom_eval.py @@ -1184,7 +1184,7 @@ def _apply_atoms(atoms) -> None: def _import_stage_modules() -> dict: """Import what a unity.export task needs, timed (wall, CPU).""" t0, c0 = time.perf_counter(), time.process_time() - for m in ("numpy", "PIL.Image", "nnnotes.pipeline.objexport", "nnnotes.unity.export"): + for m in ("numpy", "PIL.Image", "nnnotes.pipeline.objexport", "nnnotes.unity.texels"): importlib.import_module(m) return {"wall": round(time.perf_counter() - t0, 4), "cpu": round(time.process_time() - c0, 4)} @@ -1409,7 +1409,7 @@ def cmd_m3_one(a) -> None: # ---------------------------------------------------------------- imports def cmd_imports(a) -> dict: - sets = [m.split(",") for m in (a.module or ["numpy,PIL.Image,nnnotes.pipeline.objexport,nnnotes.unity.export"])] + sets = [m.split(",") for m in (a.module or ["numpy,PIL.Image,nnnotes.pipeline.objexport,nnnotes.unity.texels"])] code = ( "import resource, sys, time, json\n" "t = time.perf_counter(); c = time.process_time()\n" diff --git a/scripts/gen_font_settings.py b/scripts/gen_font_settings.py index e431593..0db28d3 100644 --- a/scripts/gen_font_settings.py +++ b/scripts/gen_font_settings.py @@ -1,54 +1,45 @@ -"""Regenerate the font settings table the open-font exports read (maintainer tool; not installed, not needed to use -nnnotes). - -With `--fonts open` (the default) the story, start canvas and talk window exports give each localized game font -asset an open font asset with that asset's point size, padding, render mode, style settings, line metrics, fallback -chain and text materials. nnnotes ships these values in `src/nnnotes/unity/fontsettings.json` (fontsettings.FORMAT), so -those exports read no game font asset; this script reads them from the game's font assets and writes the file. - -Supporting a new game version ------------------------------ -When the game changes its font assets or their materials (an open-font export stops with "... not in the font -settings", or the values differ from the game's), regenerate the table with a configured catalog and APK of that -version (`nnnotes config check`): - - python scripts/gen_font_settings.py [--region REGION] [--game-version VERSION] [-o PATH] - -It reads LocalizeManager's font names of every language and their material types from the player data, each of -those font assets with the assets of its fallback chain, their default materials and every localized text material -(" - ") the catalog has, and writes the table (`--game-version` default: the APK's versionName). The -`fonts` extra must be installed. +"""Generate a font settings table from the inputs of a saved text-product task. + +Usage: python scripts/gen_font_settings.py TASK.json --depot DEPOT [-o OUTPUT] + +The task's object table, font-bundle identities and APK manifest are read through the depot. The table contains +the font assets LocalizeManager names, their fallback chains, materials and object ids. Text products with these +inputs include story.fonts and live.ui. The fonts extra must be installed. """ from __future__ import annotations import argparse import sys -import tempfile from pathlib import Path +from nnnotes.content import storyui +from nnnotes.content.prefabs import mono_json +from nnnotes.pipeline import contract, textfonts +from nnnotes.pipeline.assemble import Objects +from nnnotes.pipeline.depot import open_depot from nnnotes.source.config import Config, use -from nnnotes.source.session import Session -from nnnotes.unity import fontsettings, tmpfont -from nnnotes.unity.export import Exporter -from nnnotes.unity.player import PlayerData +from nnnotes.unity import fontsettings, textstyle + + +def generate(task, depot) -> dict: + """Font settings from the complete stored inputs of a text product task.""" + objects = Objects(contract.loads(depot.input_bytes(task.input("objects"))), depot) + localize = storyui.script_root(objects, "LocalizeManager") + language = textstyle.language_fonts_json(mono_json(objects, localize)) + return textfonts.font_settings(task, objects, language).doc def main(argv=None) -> None: - p = argparse.ArgumentParser( - description="regenerate src/nnnotes/unity/fontsettings.json from the game's font assets" - ) - p.add_argument("--region", help="region of the catalog (default: [catalog] region)") - p.add_argument("--game-version", help="game version to record (default: the APK manifest's versionName)") - p.add_argument("-o", "--out", type=Path, default=fontsettings.FILE, help="output file") - args = p.parse_args(argv) - cfg = use(Config.load()) - session = Session(cfg) - cat = session.catalog(region=args.region) - player = PlayerData(session.apk(args.region)) - with tempfile.TemporaryDirectory(prefix="fontsettings-") as work: - ex = Exporter(cat, Path(work), player=player, textures="deferred") - doc = tmpfont.build_font_settings(ex, player, args.game_version or player.game_version) + parser = argparse.ArgumentParser(description="generate font settings from a saved text-product task") + parser.add_argument("task", type=Path, help="nnnotes.task/1 JSON with font inputs") + parser.add_argument("--depot", type=Path, help="depot holding the task inputs ([paths] depot)") + parser.add_argument("-o", "--out", type=Path, default=fontsettings.FILE, help="output table") + args = parser.parse_args(argv) + overrides = {("paths", "depot"): str(args.depot.resolve())} if args.depot else {} + cfg = use(Config.load(overrides=overrides)) + task = contract.Task.from_json(contract.loads(args.task.read_bytes())) + doc = generate(task, open_depot(cfg)) fontsettings.write(doc, args.out) print( f"{args.out}: {len(doc['fonts'])} font assets, {len(doc['materials'])} materials, " diff --git a/src/nnnotes/content/advmedia.py b/src/nnnotes/content/advmedia.py deleted file mode 100644 index 07e0e2a..0000000 --- a/src/nnnotes/content/advmedia.py +++ /dev/null @@ -1,186 +0,0 @@ -"""ADV episode assets beyond the scene: frames, particle effects, post-effect profiles, stills, chat and talk windows. - -One JSON file per kind at the story root, written only when the episode's resource closure (adv.closure) has that -kind; textures go to the story's `textures/` (PNG, named like the scene's) and every Shader object of the loaded -bundle closures is handed back to advscene, which dumps them with the scene's shaders into `shaders/`: - - frames.json Frame: "Adv/Frame/" + TargetAssetName (UIAdvWidget.SetFrame / AdvFrame), keyed by - TargetAssetName; each a prefab node list (AdvFrame, CanvasGroup, uGUI, Animator with its - controller and clips, UIParticle / ParticleSystem). Frame texts (TargetTextIDs) are episode - text ids (episode.json `text`). - effects.json Effect: "Adv/Effect/" + TargetAssetName, instantiated once per TargetName - (AdvEpisodeResourceLoader.LoadParticleEffect): `effects` by TargetAssetName (AdvParticleEffect, - ParticleSystem modules and renderers, Animator), `instances` TargetName -> TargetAssetName - posteffects.json PostEffect: "Adv/PostEffect/" + TargetAssetName, a URP VolumeProfile (not instantiated; - AdvGlobalVolume.ToggleVolume adds it as a child volume), keyed by TargetAssetName - stills.json Still: "Adv/Still/" + TargetAssetName (AdvStill, CanvasGroup, uGUI, DOTweenSequence - animations), keyed by TargetAssetName - chat.json ChatWindow / ChatTalk / ChatStamp: `chats` (the MasterAdvChat rows of the episode's chat ids), - `windows` ("Adv/Chat/Prefabs/" + _chatWindowAssetName, AdvChatWindow prefabs), `icons` - ("Adv/Chat/Icon/" + _chatIconAssetName) and `stamps` ("Adv/Chat/Stamp/" + TargetAssetName) as - sprites, `text` / `sounds` / `cueSheets`: the rows of the shared chat shards - (Adv/Chat/AdvChat-Text, -Sound, -SoundCueSheet); the cue sheet those rows name, AdvSe_Common, - is in no language's catalog and not in the APK (1.0.1 data), so nothing is decoded for them - talkwindows.json TalkWindow: "UI/Prefab/Parts/Adv/Talk/" + TargetAssetName (embedded content), keyed by window - name (the default window is also in ui/ui.json in its drawn form) - -Prefabs are exported with export.Exporter (references resolved, AnimatorControllers followed, clips in the -generic format). TextMeshPro text components always keep their layout and style (size, alignment, colours, -margins, text). The font data follows `fonts`: "open" (default) exports none of it -- font / sprite / style sheet -assets become {asset, name} and TextMeshPro materials {material, shader}, each with a note, so a viewer draws the -text with a font of its own; "game" exports the font assets and their materials with the atlas textures like any -other asset. A MonoBehaviour whose script is missing (Unity runs nothing for it) is recorded as such. -""" - -from __future__ import annotations - -from pathlib import Path - -from ..core.jsonio import write_json -from ..source import master as master_mod -from ..source.catalog import Catalog -from ..unity import fontsettings -from ..unity.export import HEADER, Exporter -from ..unity.player import PlayerData -from . import adv -from .storychat import SHARDS as CHAT_SHARDS - -TMP_SHADER_PREFIX = "TextMeshPro/" -TMP_ASSETS = ("TMP_FontAsset", "TMP_SpriteAsset", "TMP_StyleSheet") -FONT_CHOICES = ("open", "game") -FONT_NOTE = "font data not exported (fonts: open): draw the text with a font of your own" -# resource kind -> (story.json key = file key, file, address prefix removed from the entry keys; None: last segment) -FILES = { - "frame": ("frames", "frames.json", adv.RESOURCE_PREFIX["Frame"][1]), - "effect": ("effects", "effects.json", adv.RESOURCE_PREFIX["Effect"][1]), - "posteffect": ("postEffects", "posteffects.json", adv.RESOURCE_PREFIX["PostEffect"][1]), - "still": ("stills", "stills.json", adv.RESOURCE_PREFIX["Still"][1]), - "talkwindow": ("talkWindows", "talkwindows.json", None), -} -CHAT_KINDS = ("chatwindow", "chaticon", "chatstamp") -CHAT = ("chat", "chat.json") -KINDS = tuple(FILES) + CHAT_KINDS -INDEX_KEYS = tuple(k for k, _, _ in FILES.values()) + (CHAT[0],) - - -class MediaExporter(Exporter): - """export.Exporter for uGUI and particle prefabs: AnimatorControllers followed, missing-script MonoBehaviours - recorded; with fonts="open" the TextMeshPro font / sprite / style sheet assets and materials as name stubs.""" - - def __init__(self, cat: Catalog, out_dir: Path, player: PlayerData, fonts: str = "open"): - if fonts not in FONT_CHOICES: - raise ValueError(f"fonts={fonts!r}") - super().__init__( - cat, - out_dir, - player=player, - stub_assets=TMP_ASSETS if fonts == "open" else (), - follow=("AnimatorController",), - font_stubs=fontsettings.builtin() if fonts == "open" else None, - ) - self.fonts = fonts - - def scriptable(self, o) -> dict: - out = super().scriptable(o) - if self.fonts == "open" and out.get("asset") in TMP_ASSETS: - return {**out, "note": FONT_NOTE} - return out - - def material(self, o) -> dict: - if self.fonts == "game": - return super().material(o) - tt = o.read_typetree() - shader = self.deref(o, tt["m_Shader"]) - name = self.add_shader(shader) if shader is not None else None - if name is None or not name.startswith(TMP_SHADER_PREFIX): - return super().material(o) - return {"material": tt["m_Name"], "shader": {"shader": name}, "note": FONT_NOTE} - - def font_stub(self, o) -> dict: - """As material and scriptable: a TMP material or asset of a font bundle is only named, with FONT_NOTE.""" - out = super().font_stub(o) - if o.rec["type"] == "Material": - name = out["shader"]["shader"] - if not name.startswith(TMP_SHADER_PREFIX): - return out - return {"material": out["material"], "shader": {"shader": name}, "note": FONT_NOTE} - return {**out, "note": FONT_NOTE} if out["asset"] in TMP_ASSETS else out - - def component(self, o) -> dict: - if o.type.name == "MonoBehaviour": - tt = o.read_typetree() - if not tt["m_Script"]["m_PathID"]: - return { - "type": "MonoBehaviour", - "class": None, - "missingScript": True, - **{k: self.value(o, v) for k, v in tt.items() if k not in HEADER}, - } - return super().component(o) - - -def _suffix(address: str, prefix: str | None) -> str: - if prefix is None: - return address.rsplit("/", 1)[-1] - if not address.startswith(prefix): - raise ValueError(f"{address}: not under {prefix}") - return address[len(prefix) :] - - -def _chat(cat: Catalog, master: Path, ex: Exporter, episode: dict, resources: list[dict]) -> dict: - rows = {r["_id"]: r for r in master_mod.table(master, "MasterAdvChat")} - ids = sorted({c["TargetChatID"] for c in episode["commands"] if c.get("TargetChatID")}) - doc: dict = {"chats": {str(i): rows[i] for i in ids if i in rows}, "windows": {}, "icons": {}, "stamps": {}} - for r in resources: - if r["kind"] == "chatwindow": - doc["windows"][_suffix(r["address"], adv.CHAT_WINDOW_PREFIX)] = ex.export_key(r["address"]) - elif r["kind"] == "chaticon": - doc["icons"][_suffix(r["address"], adv.CHAT_ICON_PREFIX)] = ex.key_sprite(r["address"]) - elif r["kind"] == "chatstamp": - doc["stamps"][_suffix(r["address"], adv.RESOURCE_PREFIX["ChatStamp"][1])] = ex.key_sprite(r["address"]) - for name, key in CHAT_SHARDS.items(): - t = adv._text_of(cat, key) - doc[name] = adv.parse_shard(t)["rows"] if t else [] - return doc - - -def extract(cat: Catalog, player: PlayerData, master: Path, episode: dict, out_dir: Path, fonts: str = "open") -> dict: - """Write the kind files the episode needs (`fonts`: see the module docstring). Returns {"files": {index key: - file or None} for every INDEX_KEYS entry, "shaders": {name: Shader object}, "counts": {index key: entries}, - "textures": textures written}.""" - out_dir = Path(out_dir) - by_kind: dict[str, list[dict]] = {} - for r in episode["resources"]: - if r["kind"] in KINDS: - by_kind.setdefault(r["kind"], []).append(r) - files: dict[str, str | None] = dict.fromkeys(INDEX_KEYS) - shaders: dict = {} - counts: dict[str, int] = {} - textures = 0 - - def done(ex: Exporter, name: str, doc: dict) -> None: - nonlocal textures - ex.resolve_pending() - write_json(out_dir / name, doc) - for k, o in ex.shaders.items(): - shaders.setdefault(k, o) - textures += len(ex.textures) - - for kind, (key, name, prefix) in FILES.items(): - if kind not in by_kind: - continue - ex = MediaExporter(cat, out_dir, player, fonts) - entries = {_suffix(r["address"], prefix): ex.export_key(r["address"]) for r in by_kind[kind]} - doc: dict = {key: entries} - if kind == "effect": - doc["instances"] = adv.effect_instances(episode["commands"]) - done(ex, name, doc) - files[key] = name - counts[key] = len(entries) - chat = [r for k in CHAT_KINDS for r in by_kind.get(k, [])] - if chat: - ex = MediaExporter(cat, out_dir, player, fonts) - done(ex, CHAT[1], _chat(cat, master, ex, episode, chat)) - files[CHAT[0]] = CHAT[1] - counts[CHAT[0]] = len(chat) - return {"files": files, "shaders": shaders, "counts": counts, "textures": textures} diff --git a/src/nnnotes/content/advscene.py b/src/nnnotes/content/advscene.py deleted file mode 100644 index 26ac2a1..0000000 --- a/src/nnnotes/content/advscene.py +++ /dev/null @@ -1,108 +0,0 @@ -"""ADV presentation data -> scene.json + textures + shaders. - -Everything the game uses to present an ADV episode besides the episode script, -the Live2D models (with the materials of their mask pass: webmodel.py) and the -audio: - - player colour space, quality levels, URP pipelines, renderer data and - features, post-process data (APK boot data, see player.py) - cameraManager the app's Main/Sub camera prefab - advScene the ADV scene (AdvSceneRoot + scene camera) - characterField AdvCharacterField prefab (Field / Anchors / Stages) - backgroundField AdvBackgroundField prefab (backdrop mesh + sprite renderer) - globalVolume AdvGlobalVolume prefab (volumes + profiles) - settings AdvPlayerSettings (+ focus data), AdvMasterIdSettings - stages every stage prefab the episode uses (AdvStage, lights, - background sprite, volume profiles) - postTextures per camera renderer: post-process data textures used by the - URP chain (film grain textures, indexed by FilmGrain.type) - -Objects are exported with export.Exporter (references resolved, textures as -PNG). Shaders referenced anywhere, every Shader in the loaded bundle closures, -the ADV camera renderer's shaders and the shaders handed in by the caller (those -of advmedia's kind files) are dumped with shader.py into one `shaders/`. -""" - -from __future__ import annotations - -from pathlib import Path - -from ..core.jsonio import write_json -from ..formats import shader as shader_mod -from ..source.catalog import Catalog -from ..unity.export import Exporter -from ..unity.player import PlayerData - -ADV_KEYS = { - "cameraManager": "EmbCommon/Prefab/CameraManager", - "advScene": "EmbScene/Adv", - "characterField": "EmbCommon/Adv/Prefab/AdvCharacterField", - "backgroundField": "EmbCommon/Adv/Prefab/AdvBackgroundField", - "globalVolume": "EmbCommon/Adv/Prefab/AdvGlobalVolume", -} -SETTINGS_KEYS = { - "playerSettings": "EmbCommon/Adv/Settings/AdvPlayerSettings", - "masterIdSettings": "EmbCommon/Adv/Settings/AdvMasterIdSettings", -} -STAGE_PREFIX = "Adv/Stage/" - - -def camera_renderer(player_graphics: dict, pipeline: str, renderer_index: int) -> str: - rl = player_graphics["pipelines"][pipeline]["m_RendererDataList"] - i = player_graphics["pipelines"][pipeline]["m_DefaultRendererIndex"] if renderer_index < 0 else renderer_index - return rl[i] - - -def extract(cat: Catalog, player: PlayerData, episode: dict, out_dir: Path, shaders: dict | None = None) -> dict: - """Write scene.json, textures/ and shaders/. `shaders`: further {name: Shader object} to dump with the scene's - (the scene's object wins for a name both have).""" - out_dir = Path(out_dir) - out_dir.mkdir(parents=True, exist_ok=True) - ex = Exporter(cat, out_dir, player=player) - graphics = player.graphics() - doc: dict = {"player": graphics} - for name, key in ADV_KEYS.items(): - doc[name] = ex.prefab(key) - doc["settings"] = {name: ex.asset(key) for name, key in SETTINGS_KEYS.items()} - - stages = {} - for c in episode["commands"]: - if c["cmd"] == "Stage" and c.get("TargetAssetName"): - key = STAGE_PREFIX + c["TargetAssetName"] - stages.setdefault(c["TargetAssetName"], None) - if stages[c["TargetAssetName"]] is None: - stages[c["TargetAssetName"]] = ex.prefab(key) - doc["stages"] = stages - - # shaders: everything referenced above, the ADV camera renderer - renderers = set() - for node in doc["cameraManager"]["nodes"]: - for comp in node["components"]: - if comp.get("class") == "UniversalAdditionalCameraData": - renderers.add(camera_renderer(graphics, graphics["defaultPipeline"], comp["m_RendererIndex"])) - for rn in sorted(renderers): - for o in player.renderer_shaders(rn): - ex._shader(o) - for name, o in (shaders or {}).items(): - ex.shaders.setdefault(name, o) - # post-process textures the URP chain samples (film grain lookup by FilmGrain.type) - doc["postTextures"] = { - rn: {"filmGrainTex": [ex.texture(o) for o in player.post_textures(rn, "filmGrainTex")]} - for rn in sorted(renderers) - } - sdir = out_dir / "shaders" - index: list = [] - for name in sorted(ex.shaders): - shader_mod.dump_objects([ex.shaders[name]], sdir, ex.shaders[name].assets_file.name, index) - shader_summary = shader_mod.write_index(index, sdir) - doc["shaders"] = { - "index": "shaders/shaders.json", - "names": shader_summary["names"], - "cameraRenderers": sorted(renderers), - } - doc["convention"] = ( - "Unity space (left-handed, Y up); transforms are local TRS; " - "quaternions (x,y,z,w); colours in the project's Gamma space" - ) - write_json(out_dir / "scene.json", doc) - return {"stages": list(stages), "textures": len(ex.textures), **shader_summary} diff --git a/src/nnnotes/content/advui.py b/src/nnnotes/content/advui.py index 1a2f503..00beb55 100644 --- a/src/nnnotes/content/advui.py +++ b/src/nnnotes/content/advui.py @@ -1,1086 +1,35 @@ -"""ADV story UI -> /ui/ (ui.json, textures/, shaders/). - -Everything a player needs to draw the ADV front canvas (talk window, speaker -plate, location caption, episode title, rule transition cover, menu entry -button, menu panel and video buttons, backlog, choices, curtains, flash, -subtitles, next indicator, the tap areas), the screen canvases of stills, -frames and videos, the letterbox bands and the dialogs of the ADV screen, -taken from the game data: - - nodes RectTransform hierarchy of the drawn parts of UIAdvWidget - (DRAWN) with the talk windows under UIContainer/TalkView - (AdvTalkView.SetWindow): the default UIDefaultTalkWindow, then - the other windows the episode's TalkWindow rows attach, and the - front canvas' CenterTalkBackdrop when one of them uses the - backdrop filter (UIAdvTalkWindow._useBackdropFilter); per - node the uGUI components the runtime needs (Canvas and scaler, - Image, RawImage, CanvasGroup, UIGradientImage, layout groups, - DOTweenSequence durations, the views' serialized references as - node paths, `behaviours`: every serialized field of the other - game components of BEHAVIOURS, references as node paths and - sprites as names) and, for a TMP text, its text record `textStyle` - (textstyle.py: layout, colours, style flags, face / outline / - underlay values in em, font role) - videoAndStillCamera the camera the still and video canvases render with - (UIAdvWidget._videoAndStillCamera) and the screen image showing - its output (_videoAndStillScreenImage) - uiCamera the UI camera of the UIManager prefab, which renders the frame - canvas (its canvas camera is assigned at run time), in the form - of videoAndStillCamera - masterIdTexts the AdvMasterIdSettings texts of the skip and video skip - dialogs and the interruption dialog in the language (with the - master data) - dialogs the skip confirm dialog (UIAdvSkipConfirmDialogWidget) and - CommonDialogManager's dialog (UICommonDialogWidget) as node - records with text records - chatWidget the phone of the chat rows (UIAdvChatWidget): its canvas and - scaler, AdvChatView with its durations and eases, the window - parent rect - chatTexts per chat window of the episode (AdvChatWindow prefab), the text - record of each TMP text in the language, and chatStatusTexts: - the incoming call / lock screen status texts the windows set - blur with a backdrop-filter talk window: the UI blur pass settings - (the UIRendererFeature of renderer 0: Dual Kawase iterations, - offset, downsample, blend rate maximum, shader) - sprites sprite geometry (rect, border, pivot, pixels per unit, texture - rect/offset) remapped into packed textures - materials UI-Transition, the built-in Default UI Material, the materials - of the drawn images - clips Animator clips (NextIndicator, AutoNext, AdvLocation/AdvTitle Play) - transitions RuleTransitionSettings of every transition the episode or the - player settings reference - shaders UI/Default, UI/Transition, the blur shader with `blur` - -Language (`language`, a catalog language code; default `[catalog] language`): -the LanguageMode of the localized fonts, materials and line spacing -(LocalizeText) and the text field of the lines the episode shows. - -Fonts (`fonts`): "open" (default) writes no font data; a renderer draws the -text records with fonts of its own. "game" also writes the game's TMP font -data (tmpfont.py, optional `fonts` extra): per node the serialized TMP text -settings with the localized font / material (`text`), the TMP font assets -reached from the localized fonts (face info, style values, fallback chain) -with the character and glyph tables reduced to the characters the episode -shows (characters a dynamic font asset would add at runtime are generated the -way FontEngine renders them, see tmpfont.RuntimeGlyphs), their materials, the -glyph coverage, the TMP settings and the TextMeshPro/Mobile/Distance Field -shader. - -Textures: sprite atlases and SDF font atlases are large; only the texel blocks -a draw can sample are kept (export.Exporter deferred-texture mode with -export.TexelPacker). Each block is the sprite's texture rect or a glyph rect -grown by the widest footprint the shader samples, plus one texel for bilinear -filtering, read with the source wrap mode, and copied into a packed texture. -UVs are remapped by an integer translation, so sampling returns the same texel -values as the full atlas. -""" +"""TextMesh Pro sprite metadata, emoji sequence lookup and chat runtime text.""" from __future__ import annotations -import re -from collections.abc import Sequence -from pathlib import Path +from ..unity import textstyle -from ..core.jsonio import write_json -from ..source import languages -from ..source.catalog import Catalog -from ..unity import fontsettings, textstyle, tmpfont -from ..unity.export import Exporter, TexelPacker, _safe -from ..unity.player import PlayerData -from ..unity.textstyle import LANGUAGE_LINE_SPACING -from . import adv, advscene - -WIDGET_KEY = "EmbUI/Prefab/UIAdvWidget" # UIManager: UIManager._cameraController -> UICameraController._uiCamera / _uiCameraData -UI_MANAGER_KEY = "EmbCommon/Prefab/UIManager" -WINDOW_KEY = "EmbUI/Prefab/Parts/Adv/Talk/UIDefaultTalkWindow" -WINDOW_PARENT = "UIAdvWidget/FrontCanvas/UISafeArea/UIContainer/TalkView" # UIAdvWidget._centerTalkBackdropCanvasGroup / _centerTalkBackdropImage: the backdrop of a backdrop-filter talk window -BACKDROP_REFS = ("_centerTalkBackdropCanvasGroup", "_centerTalkBackdropImage") -LETTERBOX_KEY = "Textures/letterbox-image" -TRANSITION_PREFIX = "Adv/Transition/" -PLAYER_SETTINGS_KEY = "EmbCommon/Adv/Settings/AdvPlayerSettings" -MASTER_ID_SETTINGS_KEY = "EmbCommon/Adv/Settings/AdvMasterIdSettings" -CLIP_KEYS = { - "NextIndicator/Loop": "EmbUI/Animations/NextIndicator/Loop", - "AutoNext/Loop": "EmbUI/Animations/Auto/Loop", - "AdvLocation/Play": "EmbUI/Animations/AdvLocation/Play", - "AdvTitle/Play": "EmbUI/Animations/AdvTitle/Play", -} -CONTROLLER_KEYS = { - "NextIndicator": "EmbUI/Animations/NextIndicator/NextIndicator", - "AutoNext": "EmbUI/Animations/Auto/AutoNext", - "AdvLocation": "EmbUI/Animations/AdvLocation/AdvLocation", - "AdvTitle": "EmbUI/Animations/AdvTitle/AdvTitle", -} # Parts of the widget the runtime draws (and their ancestors). The block canvas # is not exported. -DRAWN = ( - "UIAdvWidget/VideoCanvas", - "UIAdvWidget/StillCanvas", - "UIAdvWidget/VideoAndStillRenderScreenCanvas", - "UIAdvWidget/FrameCanvas", - "UIAdvWidget/FrontCanvas/FlashView", - "UIAdvWidget/FrontCanvas/RuleTransition", - "UIAdvWidget/FrontCanvas/LocationVIew", - "UIAdvWidget/FrontCanvas/RightCurtain", - "UIAdvWidget/FrontCanvas/LeftCurtain", - "UIAdvWidget/FrontCanvas/CancelFullScreenButton", - "UIAdvWidget/FrontCanvas/TalkLogView", - "UIAdvWidget/FrontCanvas/UISafeArea/UIContainer/NextButton", - "UIAdvWidget/FrontCanvas/UISafeArea/UIContainer/ChoiceView", - "UIAdvWidget/FrontCanvas/UISafeArea/UIContainer/SubtitlesView", - "UIAdvWidget/FrontCanvas/UISafeArea/UIContainer/TalkView", - "UIAdvWidget/FrontCanvas/UISafeArea/UIContainer/NextIndicator", - "UIAdvWidget/FrontCanvas/UISafeArea/UIContainer/MenuView/MenuButtonsParent", - "UIAdvWidget/FrontCanvas/UISafeArea/UIContainer/MenuView/MenuEntryButton", - "UIAdvWidget/FrontCanvas/UISafeArea/UIContainer/MenuView/VideoButtons", - "UIAdvWidget/FrontCanvas/UISafeArea/UIContainer/TitleView", - "UIAdvWidget/AdvLetterBoxCanvas", -) # the view components of the drawn parts: record key and the serialized references kept (as node paths) -VIEWS = { - "AdvVideoView": ("videoView", ("_video", "_maskArea", "_curtainCanvasGroup", "_videoCanvasGroup")), - "AdvStillView": ("stillView", ("_overlay", "_background", "_target")), - "AdvFrameView": ("frameView", ()), - "AdvFlashView": ("flashView", ("_flash",)), - "AdvSubtitlesView": ("subtitlesView", ("_subtitles", "_subtitlesText")), - "AdvFrontScreenView": ( - "frontScreenView", - ("_nextButton", "_nextIndicator", "_cancelFullScreenButton", "_uiContainer"), - ), -} # views recorded with every serialized field (references as node paths) -FULL_VIEWS = {"AdvTalkLogView": "talkLogView", "AdvChoiceView": "choiceView"} # the other game and uGUI components recorded in `behaviours` (per class, every component of the node in order): # buttons and their state and animation parts, raycast targets, colour synchronizers, texts' UIText, safe areas, # scrolling, DOTween animations, localized sprites, the backlog entry and choice item views, the dialog parts -BEHAVIOURS = ( - "UIButton", - "ButtonActiveState", - "ButtonAnimationScale", - "ButtonSound", - "SimpleRaycastTarget", - "GraphicColorSynchronizer", - "UIText", - "AspectRatioFitter", - "UIToggle", - "UIToggleGroup", - "UIToggleButtonFrame", - "UIDecoratedNormalButton", - "UIDecoratedNormalButtonFrame", - "UIAddressableImage", - "UISafeArea", - "AdvViewportSafeArea", - "EnhancedScrollRect", - "EnhancedScroller", - "ScrollRect", - "RectMask2D", - "Scrollbar", - "DOTweenAnimation", - "LocalizeSpriteEvent", - "AdvTalkLogEntryView", - "AdvChoiceItem", - "DialogButtons", - "DialogAnimation", - "DialogSizeFitter", - "UIAdvSkipConfirmDialogWidget", - "UICommonDialogWidget", -) -SERIALIZED_SKIP = ( - "type", - "class", - "m_GameObject", - "m_Script", - "m_Name", - "m_EditorHideFlags", - "m_EditorClassIdentifier", -) # the dialogs of the ADV screen: the skip confirm dialog and CommonDialogManager's dialog widget -DIALOG_KEYS = { - "UIAdvSkipConfirmDialogWidget": "EmbUI/Prefab/UIAdvSkipConfirmDialogWidget", - "UICommonDialogWidget": "EmbUI/Prefab/UICommonDialogWidget", -} -CAMERA_FIELDS = ( - "m_Enabled", - "m_ClearFlags", - "m_BackGroundColor", - "m_NormalizedViewPortRect", - "near clip plane", - "far clip plane", - "field of view", - "orthographic", - "orthographic size", - "m_Depth", - "m_CullingMask", - "m_HDR", - "m_AllowMSAA", -) -CAMERA_DATA_FIELDS = ( - "m_RenderPostProcessing", - "m_VolumeLayerMask", - "m_RendererIndex", - "m_Antialiasing", - "m_AntialiasingQuality", - "m_RenderShadows", - "m_RequiresDepthTextureOption", - "m_RequiresOpaqueTextureOption", - "m_CameraType", - "m_VolumeFrameworkUpdateModeOption", - "m_Dithering", - "m_StopNaN", -) # AdvMasterIdSettings text ids the drawn parts show: the video skip dialog (AdvMenuView's skip video button), the # skip confirm dialog and the interruption dialog -MASTER_ID_TEXTS = ( - "_skipVideoMessageTextId", - "_skipButtonTextId", - "_cancelButtonTextId", - "_skipMessageTextId", - "_continuousSkipMessageTextId", - "_backEpisodeListButtonTextId", - "_continueEpisodeButtonTextId", - "_interruptionTitleTextId", - "_interruptionMessageTextId", - "_interruptionButtonTextId", -) -RUBY = re.compile(r"]*)>") # ruby reading markup of the text tables: base # the phone of the chat rows: UIAdvChatWidget's canvas, AdvChatView and the parent of the attached window -CHAT_WIDGET_KEY = "EmbUI/Prefab/UIAdvChatWidget" -CHAT_WIDGET_NODES = ( - "UIAdvChatWidget/ChatCanvas", - "UIAdvChatWidget/ChatCanvas/AdvChatView", - "UIAdvChatWidget/ChatCanvas/AdvChatView/Target", -) -CHAT_VIEW_FIELDS = ( - "_showEaseDuration", - "_hideEaseDuration", - "_showEase", - "_hideEase", - "_scrollDuration", - "_typingDelay", - "_typingTextBoxMinHeight", - "_screenModeTransitionDuration", - "_screenModeTransitionEase", - "_incomingCallPositionOffset", -) -CHAT_STATUS_KEYS = ("_incomingCallStatusTextKey", "_lockScreenStatusTextKey") # AdvChatWindow.RefreshStatusTexts CHAT_COMMANDS = ("ChatWindow", "ChatTalk", "ChatStamp", "ChatRead", "ChatTyping") # AdvMenuView: its serialized references (node paths), the fast-forward button sprites (AdvMenuView.SetFastForwardSpeed) -MENU_VIEW_REFS = ( - "_menuButtonsParent", - "_menuEntryButton", - "_skipButton", - "_autoButton", - "_fastForwardButton", - "_logButton", - "_continuousButton", - "_fullScreenButton", - "_interruptionButton", - "_menuButtonsBackground", - "_canvasGroup", - "_skipVideoButton", - "_pauseVideoButton", - "_subtitlesButton", - "_videoButtonParent", - "_videoFilterButton", - "_fastForwardButtonImage", -) -MENU_VIEW_SPRITES = ( - "_fastForwardNormalSprite", - "_fastForwardOnePointFiveSprite", - "_fastForwardOnePointSevenSprite", - "_fastForwardDoubleSprite", -) EMOJI_SPRITE_ASSET_KEY = "Font/SiriusEmoji/SiriusEmojiData" # LocalizeManager.EmojiSpriteAssetAddress -EMOJI_CLASSES = ("RubyEmojiTextMeshProUGUI",) # TMP_Text.GetUnderlineSpecialCharacter: the underline and the text highlight (DrawTextHighlight) draw with the '_' # of the text's font asset (no fallback), sampled on the asset's first atlas page UNDERLINE_CHARACTER = 0x5F TMP_CLASSES = ("TextMeshProUGUI", "RubyTextMeshProUGUI", "RubyEmojiTextMeshProUGUI") -TMP_FIELDS = ( - "m_fontSize", - "m_fontSizeBase", - "m_fontStyle", - "m_fontWeight", - "m_HorizontalAlignment", - "m_VerticalAlignment", - "m_characterSpacing", - "m_wordSpacing", - "m_lineSpacing", - "m_paragraphSpacing", - "m_characterHorizontalScale", - "m_TextWrappingMode", - "m_overflowMode", - "m_enableAutoSizing", - "m_fontSizeMin", - "m_fontSizeMax", - "m_enableKerning", - "m_ActiveFontFeatures", - "m_enableExtraPadding", - "m_isRichText", - "m_parseCtrlCharacters", - "m_isOrthographic", - "m_overrideHtmlColors", - "m_margin", - "m_fontColor", - "m_Color", - "m_colorMode", - "m_enableVertexGradient", - "m_TextStyleHashCode", - "m_useMaxVisibleDescender", - "m_horizontalMapping", - "m_verticalMapping", - "m_charWidthMaxAdj", - "m_lineSpacingMax", - "m_isRightToLeft", - "m_text", - "m_monospaceDistEm", # TMP_EmojiTextUGUI (the emoji text classes) -) -BLUR_FEATURE = "Fwk.UI.Rendering.UIRendererFeature" -FONT_MODES = ("open", "game") -IMAGE_FIELDS = ( - "m_Enabled", - "m_Color", - "m_Type", - "m_PreserveAspect", - "m_FillCenter", - "m_FillMethod", - "m_FillAmount", - "m_UseSpriteMesh", - "m_PixelsPerUnitMultiplier", - "m_Maskable", -) - - -def _strip_tags(s: str) -> str: - return re.sub(r"<[^<>]*>", "", s) - - -def _ref_path(v) -> str | None: - """The node path of a serialized reference as the exporter writes it ({component | transform | gameObject}).""" - if not v: - return None - return v.get("gameObject") or v.get("transform") - - -def _need_path(v) -> str: - """The node path of a serialized reference that must not be null (RuntimeError for a null one).""" - p = _ref_path(v) - if p is None: - raise RuntimeError(f"a required node reference is null ({v!r})") - return p - - -def _plain(v, sprites: list): - """A serialized value as plain data: a reference as its node path, a sprite as its name (its reference added to - `sprites`), a ScriptableObject or material as its name, a texture as its name; containers element-wise.""" - if isinstance(v, list): - return [_plain(x, sprites) for x in v] - if not isinstance(v, dict): - return v - if "spriteRef" in v: - sprites.append(v["spriteRef"]) - return v["name"] - if "gameObject" in v or "transform" in v: - return _ref_path(v) - if "asset" in v or ("material" in v and "shader" in v): - return v.get("name") or v.get("material") - if "textureRef" in v: - return v.get("name") - return {k: _plain(x, sprites) for k, x in v.items()} - - -def _serialized(c: dict, sprites: list) -> dict: - """Every serialized field of a component (SERIALIZED_SKIP left out), as _plain values.""" - return {k: _plain(v, sprites) for k, v in c.items() if k not in SERIALIZED_SKIP} - - -def _pictogram(c: dict) -> dict: - """App.UI.UIPictogram: its key, image and native-size flag and the catalog it draws from (the sprite is - resolved against the catalog in extract).""" - return { - "m_Enabled": c["m_Enabled"], - "_key": c["_key"], - "_image": _ref_path(c["_image"]), - "_setNativeSize": c["_setNativeSize"], - "catalog": (c["_spriteCatalog"] or {}).get("name"), - } - - -def _layout_components(node: dict) -> dict: - out: dict = {} - sprites: list = [] - behaviours: dict = {} - image_states = [c for c in node["components"] if c.get("class") == "ButtonImageState"] - for c in node["components"]: - cls = c.get("class") - t = c["type"] - if t == "CanvasGroup": - out["canvasGroup"] = {k: c[k] for k in ("m_Enabled", "m_Alpha", "m_IgnoreParentGroups")} - elif t == "Canvas": - out["canvas"] = { - k: c[k] - for k in ( - "m_Enabled", - "m_RenderMode", - "m_SortingOrder", - "m_VertexColorAlwaysGammaSpace", - "m_AdditionalShaderChannelsFlag", - "m_OverrideSorting", - "m_PixelPerfect", - ) - } - if c["m_RenderMode"] == 1: # screen space - camera: the plane it is drawn at, its camera if set - out["canvas"]["m_PlaneDistance"] = c["m_PlaneDistance"] - if c.get("m_Camera"): - out["canvas"]["camera"] = _ref_path(c["m_Camera"]) - elif t == "Animator": - out["animator"] = { - "controller": (c["m_Controller"] or {}).get("name"), - "enabled": c["m_Enabled"], - "updateMode": c["m_UpdateMode"], - "keepStateOnDisable": c["m_KeepAnimatorStateOnDisable"], - } - elif cls in ("CanvasScaler", "ClampedCanvasScaler"): - out["canvasScaler"] = { - k: c[k] - for k in ( - "m_Enabled", - "m_UiScaleMode", - "m_ReferencePixelsPerUnit", - "m_ScaleFactor", - "m_ReferenceResolution", - "m_ScreenMatchMode", - "m_MatchWidthOrHeight", - ) - } - if cls == "ClampedCanvasScaler": # Fwk.UI.ClampedCanvasScaler: the scale clamped past an aspect ratio - out["canvasScaler"].update({"class": cls, "_maxAspectThreshold": c["_maxAspectThreshold"]}) - elif cls == "Image": - img = {k: c[k] for k in IMAGE_FIELDS} - img["sprite"] = c["m_Sprite"]["name"] if c["m_Sprite"] else None - img["spriteRef"] = c["m_Sprite"]["spriteRef"] if c["m_Sprite"] else None - img["material"] = c["m_Material"]["material"] if c["m_Material"] else None - out["image"] = img - elif cls == "RawImage": - out["rawImage"] = {k: c[k] for k in ("m_Enabled", "m_Color", "m_UVRect", "m_RaycastTarget", "m_Maskable")} - out["rawImage"]["texture"] = (c["m_Texture"] or {}).get("name") - out["rawImage"]["material"] = c["m_Material"]["material"] if c["m_Material"] else None - elif cls == "UIGradientImage": - out["gradient"] = { - "enabled": c["m_Enabled"], - "gradient": c["_gradient"], - "angleDeg": c["_angleDeg"], - "splitAtKeysWhenAxisAligned": c["_splitAtKeysWhenAxisAligned"], - "blendMode": c["_blendMode"], - } - elif cls in TMP_CLASSES: - txt = {k: c[k] for k in TMP_FIELDS if k in c} - txt["class"] = cls - txt["enabled"] = c["m_Enabled"] - txt["fontAsset"] = c["m_fontAsset"]["name"] - txt["material"] = c["m_sharedMaterial"]["material"] - out["text"] = txt - elif cls == "LocalizeText": - out["localizeText"] = {"enabled": c["m_Enabled"], "localizeEnabled": c["_localizeEnabled"]} - elif cls in ("HorizontalLayoutGroup", "VerticalLayoutGroup"): - out["layoutGroup"] = {"class": cls, **{k: c[k] for k in c if k.startswith("m_")}} - elif cls == "ContentSizeFitter": - out["contentSizeFitter"] = {k: c[k] for k in ("m_Enabled", "m_HorizontalFit", "m_VerticalFit")} - elif cls == "LayoutElement": - out["layoutElement"] = {k: c[k] for k in c if k.startswith("m_")} - elif cls == "DOTweenSequence": - out["tweenSequence"] = { - "list": [{k: e[k] for k in ("_commandType", "_duration") if k in e} for e in c["_list"]], - "raw": {k: c[k] for k in c if k in ("updateType", "isSpeedBased")}, - } - elif cls == "SimpleAnimationTrigger": - out["animationTrigger"] = {"blendTime": c["_blendTime"]} - elif cls == "UIAdvTalkWindow": - out["talkWindow"] = { - k: c[k] - for k in ( - "_typingDelay", - "_safeAreaTalkBackgroundExpansionFactor", - "_useBackdropFilter", - "_backdropFilterColor", - "_talkTextColor", - "_talkTextOutlineColor", - ) - } - elif cls == "ButtonImageState" and len(image_states) > 1: - behaviours.setdefault(cls, []).append(_serialized(c, sprites)) - elif cls == "ButtonImageState": - out["buttonImageState"] = {"normal": c["_normaledSprite"]["name"] if c["_normaledSprite"] else None} - if c["_selectedSprite"]: # ButtonImageState.SetSelected: the sprite of the selected state - out["buttonImageState"].update( - { - "selected": c["_selectedSprite"]["name"], - "selectedRef": c["_selectedSprite"]["spriteRef"], - "target": _ref_path(c["_target"]), - } - ) - elif cls == "UISafeAreaEdgeAnchor": - out["safeAreaEdgeAnchor"] = { - "enabled": c["m_Enabled"], - **{k: c[k] for k in ("_left", "_right", "_top", "_bottom")}, - } - elif cls in VIEWS: - key, refs = VIEWS[cls] - out[key] = {"enabled": c["m_Enabled"], **{k: _ref_path(c[k]) for k in refs}} - elif cls == "AdvMenuView": - out["menuView"] = { - "enabled": c["m_Enabled"], - **{k: _ref_path(c[k]) for k in MENU_VIEW_REFS}, - **{k: c[k]["name"] if c[k] else None for k in MENU_VIEW_SPRITES}, - "_fadeDuration": c["_fadeDuration"], - } - sprites += [c[k]["spriteRef"] for k in MENU_VIEW_SPRITES if c[k]] - elif cls in FULL_VIEWS: - out[FULL_VIEWS[cls]] = {"enabled": c["m_Enabled"], **_serialized(c, sprites)} - elif cls == "UIPictogram": - behaviours.setdefault(cls, []).append(_pictogram(c)) - elif cls in BEHAVIOURS: - behaviours.setdefault(cls, []).append(_serialized(c, sprites)) - elif cls == "AdvChatView": - out["chatView"] = { - "enabled": c["m_Enabled"], - **{k: c[k] for k in CHAT_VIEW_FIELDS}, - "_windowParentRect": _ref_path(c["_windowParentRect"]), - } - elif cls == "Outline": - out["outline"] = {"enabled": c["m_Enabled"]} - if behaviours: - out["behaviours"] = behaviours - if sprites: - out["_spriteRefs"] = sprites # packed into the UI textures, removed in extract - return out # -------------------------------------------------------------------------- # main entry # -------------------------------------------------------------------------- -def _node_record(n: dict) -> dict: - """A node of ui.json: transform, rect and the uGUI component records (_layout_components).""" - rec = { - "path": n["path"], - "name": n["name"], - "active": n["active"], - "localPosition": n["localPosition"], - "localRotation": n["localRotation"], - "localScale": n["localScale"], - "rect": n.get("rect"), - } - rec.update(_layout_components(n)) - return rec - - -def _component(node: dict, classes: tuple) -> dict | None: - hit = [c for c in node["components"] if c.get("class") in classes] - if len(hit) > 1: - raise RuntimeError(f"{node['path']}: {len(hit)} components of {classes}") - return hit[0] if hit else None - - -def _need_component(node: dict, classes: tuple) -> dict: - """The one component of `classes` on `node` (RuntimeError when it has none).""" - c = _component(node, classes) - if c is None: - raise RuntimeError(f"{node['path']}: no component of {classes}") - return c - - -def extract( - cat: Catalog, - player: PlayerData, - episode: dict, - out_dir: Path, - fonts: str = "open", - *, - language: str, - master: Path | None = None, -) -> dict: - """Write /ui/. `fonts`: "open" (text records only) or "game" (also the game's TMP font data); - `language`: the client language (a languages.LANGUAGES code); - `master`: the decoded master data (MasterText of `masterIdTexts`; None: no `masterIdTexts`).""" - if fonts not in FONT_MODES: - raise ValueError(f"fonts must be one of {FONT_MODES}, not {fonts!r}") - language = languages.check(language) - mode, field = languages.mode(language), languages.column(language)[1:] - game = fonts == "game" - if game: - tmpfont.require_extra() - ui_dir = Path(out_dir) / "ui" - ui_dir.mkdir(parents=True, exist_ok=True) - ex = Exporter( - cat, - ui_dir, - player=player, - textures="deferred", - stub_assets=("TMP_FontAsset", "TMP_SpriteAsset", "TMP_StyleSheet"), - font_stubs=None if game else fontsettings.builtin(), - ) - packer = TexelPacker(ui_dir) - - # -- prefabs ------------------------------------------------------------- - widget = ex.prefab(WIDGET_KEY)["nodes"] - windows = talk_windows(episode) - window = window_nodes(ex, windows) - paths = {n["path"] for n in widget} - if WINDOW_PARENT not in paths: - raise RuntimeError(f"{WINDOW_PARENT} not in {WIDGET_KEY}") - widget_comp = next(c for c in widget[0]["components"] if c.get("class") == "UIAdvWidget") - drawn = DRAWN + ((backdrop_path(widget, widget_comp),) if uses_backdrop_filter(window) else ()) - - def kept(p: str) -> bool: - return any(p == d or p.startswith(d + "/") or d.startswith(p + "/") for d in drawn) - - nodes = [] - for n in widget: - if n["path"] == "UIAdvWidget" or not kept(n["path"]): - continue - nodes.append(n) - if n["path"] == WINDOW_PARENT: - nodes += window - for d in drawn: - if d not in {n["path"] for n in nodes}: - raise RuntimeError(f"{d} missing from the prefabs") - fonts_of = tmpfont.AssetFontSettings(ex) if game else fontsettings.builtin() - styles = textstyle.TextStyles(ex, player, mode, fonts_of) - dialog_nodes = {name: ex.prefab(key)["nodes"] for name, key in DIALOG_KEYS.items()} - catalogs = pictogram_catalogs(widget + window + [n for dn in dialog_nodes.values() for n in dn]) - - def records(prefab_nodes: list, keep_text: bool) -> list: - out = [] - for n in prefab_nodes: - rec = _node_record(n) - if "text" in rec: - if not (rec.get("localizeText") or {}).get("localizeEnabled"): - raise NotImplementedError(f"{n['path']}: text without LocalizeText") - rec["textStyle"] = styles.record( - n["path"], _need_component(n, TMP_CLASSES), _component(n, ("LocalizeText",)) - ) - if not keep_text: - del rec["text"] - resolve_pictograms(rec, catalogs) - out.append(rec) - return out - - out_nodes = records(nodes, game) - dialogs = {name: {"key": DIALOG_KEYS[name], "nodes": records(dn, False)} for name, dn in dialog_nodes.items()} - front = [n for n in widget if n["path"].startswith("UIAdvWidget/FrontCanvas/") and n["path"].count("/") == 2] - front_order = [n["name"] for n in front] - widget_rec = widget_canvases(widget, widget_comp) - camera = _camera(widget, _ref_path(widget_comp["_videoAndStillCamera"])) - screen_image = _ref_path(widget_comp["_videoAndStillScreenImage"]) - if screen_image not in {n["path"] for n in out_nodes}: - raise RuntimeError(f"{screen_image} (_videoAndStillScreenImage) is not a drawn node") - master_ids = ex.asset(MASTER_ID_SETTINGS_KEY) # exported once: a second export is a reference - id_texts = master_id_texts(master_ids, master, field) if master is not None else None - chat_nodes = {n["path"]: n for n in ex.prefab(CHAT_WIDGET_KEY)["nodes"]} - missing = [p for p in CHAT_WIDGET_NODES if p not in chat_nodes] - if missing: - raise RuntimeError(f"{missing} missing from {CHAT_WIDGET_KEY}") - chat_widget = {"nodes": [_node_record(chat_nodes[p]) for p in CHAT_WIDGET_NODES]} - chat, status_ids = chat_texts(ex, styles.fonts, styles.lang, mode, episode) - status_texts = master_texts(master, status_ids, field) if master is not None and status_ids else None - - materials: dict = {} - if game: - materials, font_doc = _game_fonts( - ex, - styles, - episode, - out_nodes, - field, - master_ids, - [t["text"] for t in (id_texts or {}).values()] + list((status_texts or {}).values()), - ) - rule = next(n for n in out_nodes if n["path"].endswith("FrontCanvas/RuleTransition")) - rt_comp = [ - c - for c in next(w for w in widget if w["path"] == rule["path"])["components"] - if c.get("class") == "UIAdvRuleTransitionView" - ][0] - materials[rt_comp["_material"]["material"]] = rt_comp["_material"] - rule["ruleTransition"] = {"material": rt_comp["_material"]["material"]} - # Graphic.defaultGraphicMaterial (Canvas.GetDefaultCanvasMaterial): the player - # ships the built-in UI/Default shader (unity_builtin_extra) but no serialized - # "Default UI Material", so its values are the shader's property defaults. - ui_default = player.shader("UI/Default", file="unity_builtin_extra") - materials["Default UI Material"] = { - "material": "Default UI Material", - "shader": {"shader": "UI/Default"}, - "keywords": [], - "textures": {}, - "ints": {}, - "floats": {}, - "colors": {}, - "shaderDefaults": True, - } - # the materials of the drawn images (the video mask and the curtain it masks) - for n in nodes + [n for dn in dialog_nodes.values() for n in dn]: - for c in n["components"]: - if c.get("class") in ("Image", "RawImage") and c["m_Material"]: - materials.setdefault(c["m_Material"]["material"], c["m_Material"]) - - # -- glyph blocks ------------------------------------------------------------- - if game: - font_out = tmpfont.export_fonts( - font_doc["fonts"], - font_doc["primaries"], - font_doc["needed"], - font_doc["runtime"], - font_doc["textMaterials"], - packer, - "characters shown by this episode (plus baked control characters); runtime " - "characters generated from the source font (runtimeGlyphs)", - ) - for name, mg in font_doc["missingGlyph"].items(): - font_out[name]["missingGlyph"] = mg - - # -- sprites ------------------------------------------------------------------- - sprite_refs = set() - all_records = out_nodes + [n for d in dialogs.values() for n in d["nodes"]] - for n in all_records: - if n.get("image") and n["image"]["spriteRef"]: - sprite_refs.add(n["image"]["spriteRef"]) - if (n.get("buttonImageState") or {}).get("selectedRef"): - sprite_refs.add(n["buttonImageState"]["selectedRef"]) - sprite_refs.update(n.get("_spriteRefs", ())) - for p in (n.get("behaviours") or {}).get("UIPictogram", ()): - sprite_refs.update(r for r in (p.pop("_spriteRef", None), p.pop("_defaultRef", None)) if r) - lb_sprite = ex.key_sprite(LETTERBOX_KEY) - sprite_refs.add(lb_sprite["spriteRef"]) - sprites_out = {} - for ref in sorted(sprite_refs): - s = ex.sprite_recs[ref] - tex_name = s["tex"].read_typetree()["m_Name"] - ex.pack_sprite(ref, packer, "sprites" if "FixUiSpriteAtlas" in tex_name else f"sprite_{s['name']}") - # transitions - ps = ex.asset(PLAYER_SETTINGS_KEY) - addresses = {ps["_defaultTransitionAssetAddress"]} - for c in list(ps["_initializeEpisodes"]) + list(ps["_finalizeEpisodes"]) + episode["commands"]: - cmd = c.get("Command") if "Command" in c else c.get("raw") - if cmd in (5, 6) and c.get("TargetAssetName"): - addresses.add(c["TargetAssetName"]) - transitions = {} - for addr in sorted(addresses): - a = ex.asset(TRANSITION_PREFIX + addr) - tex = a["_texture"] - if tex is None: - raise RuntimeError(f"transition {addr}: no rule texture") - if tex["mipCount"] > 1: - raise NotImplementedError(f"transition {addr}: mipmapped rule texture") - o = ex.tex_objs[tex["textureRef"]] - a["_block"] = packer.add(f"rule_{_safe(addr)}", o, 0, 0, tex["width"], tex["height"]) - transitions[addr] = a - - textures = packer.build() - - # resolve blocks - if game: - tmpfont.resolve_font_blocks(font_out) - for ref in sorted(sprite_refs): - sprites_out[ex.sprite_recs[ref]["name"]] = ex.packed_sprite(ref) - for n in all_records: - if n.get("image"): - n["image"].pop("spriteRef", None) - if n.get("buttonImageState"): - n["buttonImageState"].pop("selectedRef", None) - n.pop("_spriteRefs", None) - for addr, a in transitions.items(): - b = a.pop("_block") - t = textures[b["texture"]] - # rule textures are sampled with screen-space UVs over [0, 1]: they stay whole - if b["offset"] != (0, 0) or (t["width"], t["height"]) != (a["_texture"]["width"], a["_texture"]["height"]): - raise RuntimeError(f"transition {addr}: rule texture not copied whole") - a["_texture"] = {"texture": b["texture"], "name": a["_texture"]["name"]} - for m in materials.values(): - for _k, v in m["textures"].items(): - if v["texture"] is not None: - v["texture"] = { - "name": v["texture"]["name"], - "width": v["texture"]["width"], - "height": v["texture"]["height"], - "format": v["texture"]["format"], - } - - # -- clips ---------------------------------------------------------------------- - clips = {} - for name, key in CLIP_KEYS.items(): - o = ex.key_object(key) - if o.type.name != "AnimationClip": - raise RuntimeError(f"{key}: container asset is {o.type.name}") - clips[name] = ex.clip_curves(o) - controllers = {name: ex.controller_states(ex.key_object(key)) for name, key in CONTROLLER_KEYS.items()} - - # -- DOTween defaults (Resources/DOTweenSettings) ------------------------------------ - dts = player.mono(player.resource("DOTweenSettings")) - - # -- shaders ---------------------------------------------------------------------- - needed = {} - for name in sorted({m["shader"]["shader"] for m in materials.values()}): - if name == "UI/Default": - needed[name] = ui_default - elif name in ex.shaders: - needed[name] = ex.shaders[name] - else: - raise RuntimeError(f"shader {name} not in the loaded bundles") - blur = None - if uses_backdrop_filter(window): - blur, blur_shader = blur_settings(player) - needed[blur["shader"]] = blur_shader - index, shader_summary = ex.dump_shaders(ui_dir / "shaders", needed) - if blur is not None: - gles = [ - v - for r in index - if r["name"] == blur["shader"] - for v in r["variants"] - if v["platform"] == "gles3" and v["type"] == "GLES3" - ] - if not any(v["keywords"] == [] and v["stage"] == "vertex" for v in gles): - raise RuntimeError(f"{blur['shader']}: no GLES3 variant without keywords (the blur pass)") - variants_needed = {} - for m in materials.values(): - sh = m["shader"]["shader"] - rec = next(r for r in index if r["name"] == sh) - known = {k for v in rec["variants"] if v["platform"] == "gles3" and v["type"] == "GLES3" for k in v["keywords"]} - want = sorted(k for k in m["keywords"] if k in known) - ok = any( - v["platform"] == "gles3" - and v["type"] == "GLES3" - and sorted(v["keywords"]) == want - and v["stage"] == "vertex" - for v in rec["variants"] - ) - if not ok: - raise RuntimeError(f"{sh}: no GLES3 variant for material {m['material']} keywords {want}") - variants_needed[m["material"]] = want - # read last: loading the UIManager bundles registers their shaders - ui_cam = ui_camera(ex) - - doc = { - "about": about(language, windows), - "language": language_doc(language, styles if game else None), - "frontCanvasOrder": front_order, - "widget": widget_rec, - "nodes": out_nodes, - "videoAndStillCamera": camera, - "uiCamera": ui_cam, - "videoAndStillScreenImage": screen_image, - "chatWidget": chat_widget, - "sprites": sprites_out, - "letterBoxSprite": lb_sprite["name"], - "textures": textures, - "materials": materials, - "materialKeywords": variants_needed, - } - if blur is not None: - doc["blur"] = blur - if game: - tmp = player.resource("TMP Settings") - tmp_settings = {k: player.names_in(tmp, v) for k, v in player.mono(tmp).items()} - doc["fonts"] = font_out - doc["tmpSettings"] = { - k: tmp_settings[k] - for k in ( - "m_fallbackFontAssets", - "m_defaultFontAsset", - "m_missingGlyphCharacter", - "m_matchMaterialPreset", - "m_enableExtraPadding", - "m_warningsDisabled", - ) - } - doc.update( - { - "clips": clips, - "controllers": controllers, - "transitions": transitions, - "playerSettings": { - k: ps[k] - for k in ( - "_defaultTransitionAssetAddress", - "_waitAfterVoiceTime", - "_waitTalkTextUnitTime", - "_minTalkDisplayTime", - "_isAdvViewportFollowOnResolutionChanged", - ) - }, - "dotween": { - k: dts[k] - for k in ( - "defaultEaseType", - "defaultUpdateType", - "defaultTimeScaleIndependent", - "timeScale", - "defaultEaseOvershootOrAmplitude", - "defaultEasePeriod", - ) - }, - "shaders": {"index": "shaders/shaders.json", "names": shader_summary["names"]}, - } - ) - if game: - doc["glyphCoverage"] = font_doc["coverage"] - if id_texts is not None: - doc["masterIdTexts"] = id_texts - doc["dialogs"] = dialogs - if chat: - doc["chatTexts"] = chat - if status_texts: - doc["chatStatusTexts"] = status_texts - doc["textStyle"] = styles.summary() - write_json(ui_dir / "ui.json", doc) - out = {"textures": {k: (v["width"], v["height"]) for k, v in textures.items()}, "shaders": shader_summary["names"]} - if not game: - return {**out, "fonts": "open", "texts": sum(1 for n in out_nodes if "textStyle" in n)} - return { - **out, - "coverage": font_doc["coverage"]["counts"], - "fonts": sorted(font_out), - "runtimeGlyphs": { - k: { - "generated": len(v["runtimeGlyphs"]["generated"]), - "validation": { - kk: v["runtimeGlyphs"]["validation"][kk] - for kk in ("glyphs", "maxAbsError", "meanAbsError", "exactGlyphs") - }, - } - for k, v in font_out.items() - if v["runtimeGlyphs"] - }, - } - - -def talk_windows(episode: dict) -> list[str]: - """The talk windows the story attaches, in the order of their nodes under TalkView: UIDefaultTalkWindow (AdvPlayer - PlayInitialCommands attaches it), then by name the other talk windows of the episode's resources (the TalkWindow - rows, adv.closure kind "talkwindow").""" - names = {r["address"][r["address"].rindex("/") + 1 :] for r in episode["resources"] if r["kind"] == "talkwindow"} - return [adv.DEFAULT_TALK_WINDOW] + sorted(names - {adv.DEFAULT_TALK_WINDOW}) - - -def window_key(cat, name: str) -> str: - """The prefab key of the talk window `name`: WINDOW_KEY for the default window, else the TalkWindow address - ("UI/Prefab/Parts/Adv/Talk/" + name) resolved as adv.closure resolves it (embedded content: the "Emb" key when - the catalog lacks the plain one).""" - if name == adv.DEFAULT_TALK_WINDOW: - return WINDOW_KEY - address = adv.TALK_WINDOW_PREFIX + name - return address if cat.has(address) else adv.EMBEDDED_PREFIX + address - - -def window_nodes(ex: Exporter, names: list[str]) -> list[dict]: - """The prefab nodes of the talk windows `names`, in that order, with their paths under TalkView - (AdvTalkView.SetWindow parents the attached window there at localPosition 0).""" - out = [] - for name in names: - nodes = ex.prefab(window_key(ex.cat, name))["nodes"] - if not nodes or nodes[0]["path"] != name: - raise RuntimeError(f"talk window {name}: the prefab root is not {name}") - out += [{**w, "path": f"{WINDOW_PARENT}/{w['path']}"} for w in nodes] - return out - - -def uses_backdrop_filter(window: list[dict]) -> bool: - """Whether a talk window of the nodes (window_nodes) has UIAdvTalkWindow._useBackdropFilter set.""" - return any(c.get("class") == "UIAdvTalkWindow" and c["_useBackdropFilter"] for n in window for c in n["components"]) - - -def backdrop_path(widget: list[dict], comp: dict) -> str: - """The node of the centre-talk backdrop (UIAdvWidget BACKDROP_REFS: its CanvasGroup and Image, on one node).""" - paths = {_ref_path(comp[k]) for k in BACKDROP_REFS} - path = next(iter(paths)) if len(paths) == 1 else None - if path is None: - raise RuntimeError(f"UIAdvWidget {BACKDROP_REFS}: {sorted(map(str, paths))}") - node = next((n for n in widget if n["path"] == path), None) - if node is None: - raise RuntimeError(f"{path}: not a widget node") - if not any(c["type"] == "CanvasGroup" for c in node["components"]) or _component(node, ("Image",)) is None: - raise RuntimeError(f"{path}: no CanvasGroup or Image") - return path - - -def blur_record(graphics: dict) -> dict: - """blur: the UIRendererFeature of renderer 0 (UIBlurController.ExecBlur -> UIRenderPass CaptureAndBlur): its - Dual Kawase iterations, offset, downsample, blend rate maximum and shader name, read from the player data - (renderer 0 of every pipeline must be the same renderer).""" - first = {p["m_RendererDataList"][0] for p in graphics["pipelines"].values()} - if len(first) != 1: - raise NotImplementedError(f"renderer 0 differs between the pipelines: {sorted(first)}") - name = advscene.camera_renderer(graphics, graphics["defaultPipeline"], 0) - feats = [f for f in graphics["renderers"][name]["m_RendererFeatures"] if f["class"] == BLUR_FEATURE] - if len(feats) != 1: - raise RuntimeError(f"renderer {name}: {len(feats)} {BLUR_FEATURE}") - f = feats[0] - return { - "renderer": name, - "active": bool(f["m_Active"]), - "iterations": f["_blurIterations"], - "offset": f["_blurOffset"], - "downsample": f["_blurDownsample"], - "blendRateMax": f["_blurBlendRateMax"], - "shader": f["_dualKawaseBlurShader"], - } - - -def blur_settings(player: PlayerData) -> tuple[dict, object]: - """ui.json `blur` and its Shader object, for a backdrop-filter talk window (UIAdvWidget.UpdateAdvBlurAndBackdrop -> - UIManager.UseBlur): the UIRendererFeature of renderer 0 (blur_record; CameraManager and UICameraController set - renderer 0 on the main, sub and UI cameras, and the UI overlay camera runs the pass).""" - blur = blur_record(player.graphics()) - if not blur["active"]: - raise NotImplementedError(f"renderer {blur['renderer']}: the UIRendererFeature is not active") - objs = [o for o in player.renderer_shaders(blur["renderer"]) if o.read().m_ParsedForm.m_Name == blur["shader"]] - if len(objs) != 1: - raise RuntimeError(f"renderer {blur['renderer']}: {len(objs)} shaders {blur['shader']}") - return blur, objs[0] - - -def pictogram_catalogs(nodes: list) -> dict: - """{catalog name: its serialized record} of the UIPictogram sprite catalogs the prefab nodes name (the exporter - writes an asset in full at its first reference).""" - out = {} - for n in nodes: - for c in n["components"]: - cat = c.get("_spriteCatalog") if c.get("class") == "UIPictogram" else None - if cat and "_spritesByKey" in cat: - out.setdefault(cat["name"], cat) - return out - - -def resolve_pictograms(rec: dict, catalogs: dict) -> None: - """The sprite of each UIPictogram record of a node (UISpriteCatalogBase.GetSprite: the catalog's sprite of the - key, `sprite`; null when the catalog has none) and the catalog's default sprite (`defaultSprite`).""" - for p in (rec.get("behaviours") or {}).get("UIPictogram", ()): - cat = catalogs.get(p["catalog"]) - if cat is None: - raise RuntimeError(f"{rec['path']}: sprite catalog {p['catalog']} not exported") - by_key = {e["Key"]: e["Value"] for e in cat["_spritesByKey"]["_list"]} - hit, default = by_key.get(p["_key"]), cat.get("_defaultSprite") - p["sprite"] = hit["name"] if hit else None - p["defaultSprite"] = default["name"] if default else None - p["_spriteRef"] = hit["spriteRef"] if hit else None - p["_defaultRef"] = default["spriteRef"] if default and not hit else None - - -def widget_canvases(widget: list, comp: dict) -> dict: - """widget: UIWidget._canvasSortOrder and the sorting order of each canvas of UIWidget._canvases, in list order - (UIWidget.TrueCanvasSortOrder reads the first).""" - by_path = {n["path"]: n for n in widget} - canvases = [] - for ref in comp["_canvases"]: - path = _ref_path(ref) - hit = [c for c in (by_path.get(path) or {"components": []})["components"] if c["type"] == "Canvas"] - if len(hit) != 1: - raise RuntimeError(f"_canvases: {path} has {len(hit)} Canvas components") - canvases.append({"path": path, "sortingOrder": hit[0]["m_SortingOrder"]}) - return {"canvasSortOrder": comp["_canvasSortOrder"], "canvases": canvases} def emoji_sequence_key(name: str, unicode: int) -> str | None: @@ -1107,20 +56,6 @@ def emoji_sequence_key(name: str, unicode: int) -> str | None: return "".join(out) -def sprite_asset_record(ex: Exporter, key: str = EMOJI_SPRITE_ASSET_KEY) -> dict: - """The TMP sprite asset at `key` (LocalizeManager's emoji sprite asset by default) as the text layout reads it: - `name`, `faceInfo` (m_FaceInfo), `characters` (m_SpriteCharacterTable in order: `index`, `unicode`, `name`, - `glyph`, `scale`), `glyphs` ({glyph index: metrics, rect (in the sprite sheet), scale, atlasIndex}), - `sequences` (the entries of the legacy sprite list, spriteInfoList, whose name holds a '-', in order: `name`, - `unicode`; TMP_EmojiSearchEngine builds its sequence table from them), `sheet` (the sprite sheet's name, width, - height, mip count and texture settings) and `material` (its material record). Fallback sprite assets are not - implemented.""" - o = ex.key_object(key) - tt = o.read_typetree() - sheet = ex.referent(o, tt["spriteSheet"]).read_typetree() - return sprite_record(tt, sheet, ex.material(ex.deref(o, tt["m_Material"]))) - - def sprite_record(tt: dict, sheet: dict, material: dict) -> dict: """The text layout's sprite asset record from its fields, sheet metadata and material.""" if tt.get("fallbackSpriteAssets"): @@ -1220,91 +155,6 @@ def emoji_sequence_names(table: dict[tuple, str], fast: set[tuple], text: str) - return out -def _camera(widget: list, path: str | None, field: str = "_videoAndStillCamera") -> dict: - """videoAndStillCamera: the Camera (CAMERA_FIELDS) and UniversalAdditionalCameraData (CAMERA_DATA_FIELDS) of the - node at `path` (UIAdvWidget._videoAndStillCamera; `field` names the reference in errors).""" - node = next((n for n in widget if n["path"] == path), None) - if node is None: - raise RuntimeError(f"{field}: no node {path}") - cam = [c for c in node["components"] if c["type"] == "Camera"] - data = [c for c in node["components"] if c.get("class") == "UniversalAdditionalCameraData"] - if len(cam) != 1 or len(data) != 1: - raise RuntimeError(f"{path}: {len(cam)} Camera, {len(data)} UniversalAdditionalCameraData") - return { - "path": path, - "active": node["active"], - "camera": {k: cam[0][k] for k in CAMERA_FIELDS}, - "additionalCameraData": {k: data[0][k] for k in CAMERA_DATA_FIELDS}, - } - - -def ui_camera(ex: Exporter) -> dict: - """uiCamera: the camera UICameraController._uiCamera of the UIManager prefab (UIManager._cameraController), with - its UniversalAdditionalCameraData (_uiCameraData, on the same node), as _camera's record. The components are read - from their typetrees, so nothing they reference is exported.""" - go = ex.key_object(UI_MANAGER_KEY) - if go is None or go.type.name != "GameObject": - raise RuntimeError(f"{UI_MANAGER_KEY}: container asset is {go and go.type.name}") - _, graph = ex.load(UI_MANAGER_KEY) - - def mono(owner, pptr, cls: str, what: str): - o = ex.deref(owner, pptr) if pptr else None - if o is None or o.type.name != "MonoBehaviour" or ex.script_class(o) != cls: - raise RuntimeError(f"{UI_MANAGER_KEY}: {what} is not a {cls}") - return o - - comps = [ex.deref(go, c["component"]) for c in go.read_typetree()["m_Component"]] - managers = [ - c for c in comps if c is not None and c.type.name == "MonoBehaviour" and ex.script_class(c) == "UIManager" - ] - if len(managers) != 1: - raise RuntimeError(f"{UI_MANAGER_KEY}: {len(managers)} UIManager on the prefab root") - ctrl = mono( - managers[0], - managers[0].read_typetree()["_cameraController"], - "UICameraController", - "UIManager._cameraController", - ) - ctrl_tt = ctrl.read_typetree() - cam = ex.deref(ctrl, ctrl_tt["_uiCamera"]) if ctrl_tt["_uiCamera"] else None - if cam is None or cam.type.name != "Camera": - raise RuntimeError(f"{UI_MANAGER_KEY}: UICameraController._uiCamera is not a Camera") - data = mono(ctrl, ctrl_tt["_uiCameraData"], "UniversalAdditionalCameraData", "UICameraController._uiCameraData") - cam_tt, data_tt = cam.read_typetree(), data.read_typetree() - go_pid = cam_tt["m_GameObject"]["m_PathID"] - if data_tt["m_GameObject"]["m_PathID"] != go_pid: - raise RuntimeError(f"{UI_MANAGER_KEY}: _uiCamera and _uiCameraData are on different GameObjects") - path = graph.path(graph.tf_of_go[go_pid]) - node = { - "path": path, - "active": bool(graph.go[go_pid]["m_IsActive"]), - "components": [ - {"type": "Camera", **cam_tt}, - {"type": "MonoBehaviour", "class": "UniversalAdditionalCameraData", **data_tt}, - ], - } - return _camera([node], path, "_uiCamera") - - -def master_texts(master: Path, ids, field: str) -> dict[str, str]: - """{MasterText id: its text in the text field `field`} of `ids`; a missing id raises.""" - from ..source import master as master_mod - - ids = set(ids) - rows = {r["_id"]: r for r in master_mod.table(Path(master), "MasterText") if r["_id"] in ids} - missing = sorted(ids - set(rows)) - if missing: - raise KeyError(f"text ids not in MasterText: {missing}") - return {i: rows[i].get(f"_{field}") or "" for i in sorted(ids)} - - -def master_id_texts(settings: dict, master: Path, field: str) -> dict: - """masterIdTexts: {AdvMasterIdSettings field of MASTER_ID_TEXTS: {id (MasterText id), text (in `field`)}}.""" - ids = {k: settings[k] for k in MASTER_ID_TEXTS} - texts = master_texts(master, ids.values(), field) - return {k: {"id": i, "text": texts[i]} for k, i in ids.items()} - - def _lookup_lang(lang: dict) -> dict: """`lang` (textstyle.language_fonts) with the Japanese font names of LocalizeManager.InitJapaneseFontLookUpTable: the Japanese font names, then its additional font names (the table TryGetFontIndex searches).""" @@ -1313,71 +163,6 @@ def _lookup_lang(lang: dict) -> dict: return {**lang, "languages": {**lang["languages"], textstyle.JAPANESE_MODE: table}} -def chat_localize(font_asset: str, material: str, lang: dict, mode: int) -> dict: - """LocalizeText.Init -> OnFontChanged of a text with _localizeEnabled whose font may be an additional font (the - chat windows' font): TryGetFontIndex = its index in the lookup table (_lookup_lang), the font of that index among - the language's loaded fonts (its font names, then the additional fonts LocalizeManager.LoadAdditionalAsset - appends; an index out of range: font 0), GetFontMaterial of the material type, ApplyLanguageLineSpacing. - -> {fontAsset, material, lineSpacing}.""" - table = [f"{n} SDF" for n in _lookup_lang(lang)["languages"][textstyle.JAPANESE_MODE]["fontNames"]] - if font_asset not in table: - raise NotImplementedError(f"font {font_asset} not in the Japanese font lookup table") - own = lang["languages"][mode] - fonts = [f"{n} SDF" for n in own["fontNames"] + own["additionalFontNames"]] - i = table.index(font_asset) - font = fonts[i] if i < len(fonts) else fonts[0] - mat_type = textstyle.material_type(material) - if mat_type not in lang["materialTypes"]: - raise NotImplementedError(f"material type {mat_type} not in LocalizeManager._materialTypeList") - return {"fontAsset": font, "material": f"{font[:-4]} - {mat_type}", "lineSpacing": LANGUAGE_LINE_SPACING[mode]} - - -def chat_windows(episode: dict) -> dict[str, str]: - """{window name: address} of the chat windows of the episode's resources ("Adv/Chat/Prefabs/" + name).""" - return { - r["address"][len(adv.CHAT_WINDOW_PREFIX) :]: r["address"] - for r in episode["resources"] - if r["kind"] == "chatwindow" - } - - -def chat_text_nodes(ex: Exporter, episode: dict): - """(window name, node, TMP component, LocalizeText component or None) of every TMP text of the episode's chat - windows, and the status text ids their AdvChatWindow components set (CHAT_STATUS_KEYS).""" - out, status = [], set() - for name, address in sorted(chat_windows(episode).items()): - for n in ex.prefab(address)["nodes"]: - win = _component(n, ("AdvChatWindow",)) - if win is not None: - status |= {win[k] for k in CHAT_STATUS_KEYS if win.get(k)} - comp = _component(n, TMP_CLASSES) - if comp is not None: - out.append((name, n, comp, _component(n, ("LocalizeText",)))) - return out, sorted(status) - - -def chat_texts(ex: Exporter, fonts, lang: dict, mode: int, episode: dict) -> tuple[dict, list[str]]: - """chatTexts {window: {node path: {textStyle, localizeText}}} of the episode's chat windows in the language `mode` - (the text record of textstyle.text_style with the font and material chat_localize gives, their settings from - `fonts` (fontsettings.FontSettings); font roles are the slots of the lookup table) and the status text ids of the - windows.""" - table_lang, styles, out = _lookup_lang(lang), {}, {} - nodes, status = chat_text_nodes(ex, episode) - for name, n, comp, loc in nodes: - fa, mat = comp["m_fontAsset"]["name"], comp["m_sharedMaterial"]["material"] - if loc and loc["m_Enabled"] and loc["_localizeEnabled"]: - loc_font = chat_localize(fa, mat, lang, mode) - fa, mat = loc_font["fontAsset"], loc_font["material"] - if (fa, mat) not in styles: - face = fonts.font(fa)["faceInfo"] - styles[(fa, mat)] = textstyle.material_style(fonts.material(fa, mat), face["m_PointSize"], face["m_Scale"]) - out.setdefault(name, {})[n["path"]] = { - "textStyle": textstyle.text_style(comp, loc, table_lang, styles[(fa, mat)], mode), - "localizeText": {"enabled": loc["m_Enabled"], "localizeEnabled": loc["_localizeEnabled"]} if loc else None, - } - return out, status - - def _number(v) -> float | None: try: return float(v) @@ -1404,132 +189,3 @@ def chat_runtime_texts(episode: dict) -> list[str]: elif c["cmd"] in ("ChatTalk", "ChatStamp", "ChatRead") and p1 is not None and p1 == int(p1) and p1 >= 1: out.append(f" {int(p1)}") return out - - -def about(language: str, windows: Sequence[str] = (adv.DEFAULT_TALK_WINDOW,)) -> str: - """ui.json `about`: what the file holds (the widget and its talk windows), in which client language.""" - return f"ADV front canvas UI data (UIAdvWidget + {', '.join(windows)}), {language}" - - -def language_doc(language: str, styles: textstyle.TextStyles | None = None) -> dict: - """ui.json `language`: the LanguageMode and text field of `language`, with `styles` (fonts game) the - LocalizeManager font lists and the font swap, and the line spacing LocalizeText applies.""" - mode = languages.mode(language) - doc = {"mode": mode, "field": languages.column(language)[1:]} - if styles is not None: - doc.update({"fonts": styles.lang, "fontSwap": styles.swap}) - doc["lineSpacing"] = LANGUAGE_LINE_SPACING[mode] - return doc - - -def shown_texts(episode: dict, master_ids: dict, field: str) -> list[str]: - """The texts the episode shows in the text field `field` (e.g. "traditionalChinese"), tags stripped: Talk, - Location and Subtitles lines with their ruby readings (), the speaker names of the Talk rows - (TargetTextIDs; the unknown-speaker text for TargetStatus 1, joined names with the split text; none for - TargetStatus 2) and the title. `master_ids`: AdvMasterIdSettings.""" - lines = [] - for c in episode["commands"]: - if c["cmd"] in ("Talk", "Location", "Subtitles", "ChoiceSet") and c.get("lines"): - lines.append(c["lines"][field]) - lines += RUBY.findall(c["lines"][field] or "") - names = set() - for c in episode["commands"]: - if c["cmd"] == "Talk" and c.get("TargetName") and c.get("TargetStatus", 0) != 2 and c.get("AdvTextID"): - ids = list(c.get("TargetTextIDs") or []) - if c.get("TargetStatus", 0) == 1: - ids = [master_ids["_unknownCharacterNameTextId"]] - elif len(ids) > 1: - ids.append(master_ids["_splitCharacterNameTextId"]) - for i in ids: - row = episode["text"].get(i) - if row is None: - raise KeyError(f"speaker text id {i} not in the episode text shard") - names.add(row[field]) - for c in episode["commands"]: # chat rows: lines, window / sender names, stamp logs - if c["cmd"] not in CHAT_COMMANDS or c.get("IgnoreData"): - continue - if c.get("lines"): - lines.append(c["lines"][field]) - ids = list(c.get("TargetTextIDs") or []) - if c["cmd"] == "ChatStamp" and c.get("TargetAssetName"): # AdvChatStampCommand.GetStampLogText - ids.append(c["TargetAssetName"].rsplit("/", 1)[-1] + "_log") - for i in ids: - row = episode["text"].get(i) - if row is not None: - names.add(row[field]) - if any(c["cmd"] in CHAT_COMMANDS for c in episode["commands"]): - for k in ("_chatReadTextId", "_chatStampLogTextId"): - row = episode["text"].get(master_ids.get(k)) - if row is not None: - names.add(row[field]) - title = episode.get("title") - if not title: - raise RuntimeError("episode.json has no title lines (adv.extract)") - return [_strip_tags(s) for s in lines + sorted(names) + [title[field]]] + chat_runtime_texts(episode) - - -def _game_fonts( - ex: Exporter, - styles: textstyle.TextStyles, - episode: dict, - out_nodes: list, - field: str, - master_ids: dict, - ui_texts: Sequence[str] = (), -) -> tuple[dict, dict]: - """`--fonts game`: the localized font / material of every text node (`text.localized`), the TMP font assets, - the glyph coverage of the characters the episode shows (in the text field `field`; `master_ids`: - AdvMasterIdSettings; `ui_texts`: texts of the UI itself) and the text materials. - -> (materials, the inputs of tmpfont.export_fonts + coverage).""" - fonts = tmpfont.FontSet(ex) - used_text_nodes = [n for n in out_nodes if "text" in n] - primaries = {} - for n in used_text_nodes: - t = n["text"] - t["localized"] = textstyle.localize_text( - n["path"], t["fontAsset"], t["material"], styles.swap, styles.lang, styles.mode - ) - primaries[t["localized"]["fontAsset"]] = None - for fname in primaries: - primaries[fname] = fonts.by_key(fontsettings.font_dir(fname) + fname) - - # -- characters the episode shows ------------------------------------------ - shown = shown_texts(episode, master_ids, field) + list(ui_texts) - chars = sorted({ord(ch) for s in shown for ch in s}) - - primary = [primaries[p] for p in primaries] - if len(primary) != 1: - raise NotImplementedError(f"texts use {len(primary)} primary fonts") - primary = primary[0] - coverage, needed, runtime_units = tmpfont.glyph_coverage(fonts, primary, chars) - # the underline character from the primary asset itself (baked, or generated as its dynamic asset would) - kind, font = fonts.lookup(primary, UNDERLINE_CHARACTER) or (None, None) - in_asset = font is primary and kind in ("baked", "runtime") - if in_asset: - (needed if kind == "baked" else runtime_units).setdefault(primary.name, set()).add(UNDERLINE_CHARACTER) - coverage["underline"] = {"font": primary.name, "character": chr(UNDERLINE_CHARACTER), "inAsset": in_asset} - # the missing glyph (tmpfont.missing_glyph): its substitute from the asset that has it - tmpfont.check_glyph_variants(fonts, primary, shown) - mg = tmpfont.missing_glyph(fonts, primary, chars, tmpfont.missing_glyph_character(ex.player)) - if mg: - found = fonts.lookup(primary, mg["unicode"]) - if found is None: - raise RuntimeError(f"missing glyph U+{mg['unicode']:04X}: in no font asset of {primary.name}") - kind, f = found - if kind != "synthesized": - (needed if kind == "baked" else runtime_units).setdefault(f.name, set()).add(mg["unicode"]) - coverage["missingGlyph"] = [chr(u) for u in mg["characters"]] - - # -- materials --------------------------------------------------------------- - materials, text_materials = tmpfont.text_materials( - ex, fonts, [n["text"]["localized"] for n in used_text_nodes], {primary.name}, needed, runtime_units - ) - return materials, { - "fonts": fonts, - "primaries": {primary.name}, - "needed": needed, - "runtime": runtime_units, - "textMaterials": text_materials, - "coverage": coverage, - "missingGlyph": {primary.name: mg} if mg else {}, - } diff --git a/src/nnnotes/content/advvideo.py b/src/nnnotes/content/advvideo.py deleted file mode 100644 index ee0cfc0..0000000 --- a/src/nnnotes/content/advvideo.py +++ /dev/null @@ -1,132 +0,0 @@ -"""ADV videos: Movie / Clip VideoID -> the episode's -Video row -> `Cri/Video/<_assetName>` (a USM, usm.py) -> -/videos/.webm + videos/videos.json. - -An addressable key `Cri/Video/` depends on a small bundle holding the CriWare.Assets MonoBehaviour of the -movie (`movieInfo`: size, frame rate, frame count, codec, alpha / audio / subtitle stream counts; `assetInfo`) and on -the USM itself, stored as-is on the CDN like the cue sheets' raw data (VideoManager.Prepare hands it to CRI Mana). A -movie asset may instead hold the USM bytes itself (a CriSerializedBytesAssetImpl). -""" - -from __future__ import annotations - -from pathlib import Path -from typing import TYPE_CHECKING - -from ..core.jsonio import write_json -from ..formats.usm import AUDIO_BITRATE, VP9_ENCODE, adx_info, demux, hca_info, webm_reencoded, webm_streams, write_webm - -if TYPE_CHECKING: - from ..source.catalog import Catalog - -VIDEO_DIR = "videos" -INDEX = "videos/videos.json" -# movieInfo.codecType (CriMana.CodecType) -CODECS = {1: "sofdec.prime", 5: "h264", 9: "vp9"} - - -def usm_asset(cat: Catalog, key: str) -> tuple[dict, Path]: - """(the movie's CriWare.Assets MonoBehaviour typetree, the USM file) of a `Cri/Video/...` key.""" - import UnityPy - - from ..source.addressables import remote_path - - info, raw = None, None - for e in cat.dependencies(key): - if remote_path(e["internal_id"]) is None: - continue - bundle = cat.as_bundle(e) - if bundle is not None: - env = UnityPy.load(str(cat.fetch(bundle))) - for o in env.objects: - if o.type.name == "MonoBehaviour": - tt = o.read_typetree() - if "movieInfo" in tt: - info = tt - else: - raw = cat.fetch_raw(e) - if info is None or raw is None: - raise RuntimeError(f"{key}: movie asset or USM data not among the dependencies") - return info, raw - - -def export(cat: Catalog, key: str, dst: Path, cri_key: int, work: Path) -> dict: - """One USM -> dst (.webm, write_webm: VP9 copied, or re-encoded when the movie's video is not VP9 or has an alpha - stream; the first audio stream as Opus). Returns its record (movie info, the audio's source; `encode` when the - video was re-encoded).""" - from ..source.config import tool - - info, raw = usm_asset(cat, key) - mi = info["movieInfo"] - codec = CODECS.get(mi["codecType"], f"codecType {mi['codecType']}") - streams = demux(raw.read_bytes(), cri_key) - if mi["numAudioStreams"] and not any(s.kind == "audio" for s in streams): - raise RuntimeError(f"{key}: movie info lists an audio stream the USM does not carry") - hca = any(s.kind == "audio" and s.codec == "hca" for s in webm_streams(streams)) - picked = write_webm( - streams, dst, work, tool("ffmpeg", "ffmpeg"), cri_key, tool("vgmstream", "vgmstream-cli") if hca else None - ) - rec = { - "key": key, - "codec": codec, - "width": mi["width"], - "height": mi["height"], - "displayWidth": mi["dispWidth"], - "displayHeight": mi["dispHeight"], - "frameRate": [mi["framerateN"], mi["framerateD"]], - "frames": mi["totalFrames"], - "assetInfo": info.get("assetInfo"), - } - if webm_reencoded(picked): - rec["encode"] = { - "codec": "vp9", - "crf": int(VP9_ENCODE[VP9_ENCODE.index("-crf") + 1]), - "alpha": any(s.kind == "alpha" for s in picked), - } - audio = next((s for s in picked if s.kind == "audio"), None) - if audio is not None: - rec["audio"] = { - "codec": "opus", - "bitrate": AUDIO_BITRATE, - "source": adx_info(audio.data) if audio.codec == "adx" else hca_info(audio.data), - } - return rec - - -def extract(cat: Catalog, episode: dict, out_dir: Path, cri_key: int | None = None) -> dict | None: - """Every video of the episode's closure -> videos/.webm; videos/videos.json maps each VideoID of the - Movie / Clip rows to its file, its -Video row (`master`: size, autoStop, hasAudio) and the stream facts. - Returns the index, or None when the episode has no video.""" - from ..formats import cri - from . import adv - - keys = sorted(r["address"] for r in episode["resources"] if r["kind"] == "video") - if not keys: - return None - if cri_key is None: - if cat.apk is None: - raise RuntimeError("USM key: pass cri_key= or open the Catalog with apk=") - cri_key = cri.hca_key(cat.apk) - out_dir = Path(out_dir) - files: dict[str, dict] = {} - names: dict[str, str] = {} - for key in keys: - name = key.rsplit("/", 1)[-1] - if name in names: - raise RuntimeError(f"videos {names[name]} and {key} share the file name {name}") - names[name] = key - rel = f"{VIDEO_DIR}/{name}.webm" - files[key] = {"file": rel, **export(cat, key, out_dir / rel, cri_key, out_dir / VIDEO_DIR / "_work")} - videos = {} - for c in episode["commands"]: - vid = c.get("VideoID") - if c["cmd"] not in adv.VIDEO_COMMANDS or not vid or str(vid) in videos: - continue - row = episode["videos"].get(vid) or episode["videos"].get(str(vid)) - if not row: - continue - key = adv.VIDEO_ADDRESS.format(row["_assetName"]) - if key in files: - videos[str(vid)] = {"master": row, **files[key]} - index = {"videos": videos, "format": "WebM: VP9 as stored in the USM, audio as Opus"} - write_json(out_dir / INDEX, index) - return index diff --git a/src/nnnotes/content/hostspot.py b/src/nnnotes/content/hostspot.py deleted file mode 100644 index 0f74f88..0000000 --- a/src/nnnotes/content/hostspot.py +++ /dev/null @@ -1,372 +0,0 @@ -"""The spot files a story's home host writes from the bundles of a spot (content.storyhost): the background as -binary glTF and the situation's spot.json with its Spine files, read with UnityPy from the bundle closures of the -spot's catalog keys. - -`extract_room` writes .glb and .json as content.room makes room.glb and room.json from a product's objects -(the same material translation, sampler and glTF writer): every MeshFilter of the background's bundle closure, in the -bundles' object order; the empty meshes listed apart (`skippedEmptyMeshes`: no vertex data, -`meshesWithoutTriangles`: vertices only). `extract_spot` writes spot.json and spine/ as content.spot makes them, -the skeletons in the bundles' object order, and `resources`: the bundle names of the background and situation keys. -""" - -from __future__ import annotations - -import io -import json -from pathlib import Path - -import numpy as np -import UnityPy - -from ..core.jsonio import write_json -from ..formats.shader import parsed_summary -from ..source.catalog import Catalog -from ..source.config import apk_missing -from ..unity.convert.material import material_doc -from ..unity.reader import SceneGraph, deref, mesh_arrays, referent, script_class, strip_pptrs, texture_image -from . import room -from .prefabs import mat4 -from .spot import LANGS, atlas_pages - -__all__ = [ - "drawn_meshes", - "extract_room", - "extract_spot", - "main_textures", - "mesh_components", - "object_key", - "submesh_primitives", - "text_bytes", -] - - -def object_key(obj) -> tuple[str, int]: - """(serialized file name, path id) of an object: a path id is unique only within its file.""" - return obj.assets_file.name, obj.path_id - - -def main_textures(owner, mat_tt: dict, shader_name: str) -> list: - """[(texture object, texture env)] of a material's main texture slot (room.main_tex_slot) when it holds a texture; - `owner` is the material's object, whose file the env's PPtr is read from.""" - slot = room.main_tex_slot(shader_name) - return [ - (deref(owner, env["m_Texture"]), env) - for key, env in mat_tt["m_SavedProperties"]["m_TexEnvs"] - if key == slot and env["m_Texture"]["m_PathID"] - ] - - -# ---- extraction --------------------------------------------------------- -def submesh_primitives(tris: list, mats: list, material_for, on_null) -> list: - """[(glTF triangle indices, material index)] of a mesh's submeshes: `tris` (per submesh, Unity winding), - `mats` the renderer's material PPtrs of the same index. An empty submesh gives nothing; a submesh whose material - slot is empty (path id 0) gives nothing and is reported to `on_null(submesh index)`.""" - prims = [] - for i, t in enumerate(tris): - if not len(t): - continue - if not mats[i].path_id: # empty slot: no material to draw the submesh with - on_null(i) - continue - prims.append((t[:, [0, 2, 1]].reshape(-1).astype(np.uint32), material_for(mats[i]))) - return prims - - -def mesh_components(env) -> tuple[dict, dict]: - """({GameObject key: MeshFilter}, {GameObject key: MeshRenderer}) of an environment. The key is (file name, - GameObject path id), as object_key: a component is in its GameObject's file.""" - mf_by_go, mr_by_go = {}, {} - for o in env.objects: - if o.type.name == "MeshFilter": - mf_by_go[(o.assets_file.name, o.read_typetree()["m_GameObject"]["m_PathID"])] = o - elif o.type.name == "MeshRenderer": - mr_by_go[(o.assets_file.name, o.read_typetree()["m_GameObject"]["m_PathID"])] = o - return mf_by_go, mr_by_go - - -def drawn_meshes(mf_by_go: dict, mr_by_go: dict, graph, on_null_mesh): - """(transform path id, Mesh, material PPtrs) of each GameObject with a MeshFilter and an enabled MeshRenderer - (mesh_components maps). A MeshFilter whose mesh reference is empty (path id 0) draws nothing: its object path - goes to `on_null_mesh(path)`.""" - for key, mf in mf_by_go.items(): - tf_pid = graph.tf_of_go.get(key[1]) - mr = mr_by_go.get(key) - if tf_pid is None or mr is None: - continue - if not mr.read_typetree().get("m_Enabled", 1): - continue - mesh_ref = mf.read().m_Mesh - if not mesh_ref.path_id: - on_null_mesh(graph.path(tf_pid)) - continue - yield tf_pid, mesh_ref.read(), mr.read().m_Materials - - -def extract_room(cat: Catalog, bg_key: str, out_path: Path) -> dict: - env = UnityPy.load(*[str(p) for p in cat.fetch_key(bg_key)]) - graph = SceneGraph(env) - mf_by_go, mr_by_go = mesh_components(env) - - glb = room.Glb() - tex_cache: dict[tuple[str, int], int] = {} # texture object_key -> glTF texture - mat_cache: dict[tuple[str, int], int] = {} # material object_key -> glTF material - - def material_for(pptr) -> int: - mat = pptr.deref() - mkey = object_key(mat) - if mkey in mat_cache: - return mat_cache[mkey] - mat_tt = mat.read_typetree() - shader_tt = referent(mat, mat_tt["m_Shader"]).read_typetree() - gm = room.translate_material(material_doc(mat_tt, lambda p: p), parsed_summary(shader_tt)) - for tex_obj, env_ in main_textures(mat, mat_tt, shader_tt["m_ParsedForm"]["m_Name"]): - tid = object_key(tex_obj) - if tid not in tex_cache: - tex_tt = tex_obj.read_typetree() - buf = io.BytesIO() - texture_image(tex_obj.read()).save(buf, format="PNG") - tex_cache[tid] = glb.texture(buf.getvalue(), tex_tt["m_Name"], room.sampler(tex_tt)) - info, transform = room.texture_info(tex_cache[tid], env_["m_Scale"], env_["m_Offset"]) - if transform: - glb.ext_used.add("KHR_texture_transform") - gm["pbrMetallicRoughness"]["baseColorTexture"] = info - mat_cache[mkey] = glb.material(gm) - return mat_cache[mkey] - - n_mesh = 0 - n_inactive = 0 - skipped = [] - null_slots = [] # submeshes whose material slot is empty - null_meshes = [] # objects whose MeshFilter has no mesh - no_triangles = [] # meshes with vertices but no triangles - for tf_pid, mesh, mats in drawn_meshes(mf_by_go, mr_by_go, graph, null_meshes.append): - arrays = mesh_arrays(mesh) - if arrays is None: - skipped.append(mesh.m_Name) # empty mesh (no vertex data) - continue - v, vn, uv0, tris = arrays - if not any(len(t) for t in tris): # vertices only (e.g. a placeholder mesh without index data) - no_triangles.append({"mesh": mesh.m_Name, "path": graph.path(tf_pid)}) - continue - if len(mats) < len(tris): - raise NotImplementedError(f"{mesh.m_Name}: {len(tris)} submeshes, {len(mats)} materials") - m = graph.world(tf_pid) - p = (np.hstack([v, np.ones((len(v), 1))]) @ m.T)[:, :3] - p[:, 2] *= -1.0 # Unity LH -> glTF RH - if len(vn): - nw = vn @ np.linalg.inv(m[:3, :3]).T - nw /= np.maximum(np.linalg.norm(nw, axis=1, keepdims=True), 1e-12) - nw[:, 2] *= -1.0 - else: - nw = np.empty((0, 3)) - uv = uv0.copy() - uv[:, 1] = 1.0 - uv[:, 1] # Unity UV origin bottom-left -> glTF top-left - prims = submesh_primitives( - tris, - mats, - material_for, - lambda i: null_slots.append({"mesh": mesh.m_Name, "path": graph.path(tf_pid), "submesh": i}), - ) - if not prims: - continue - is_active = graph.active(tf_pid) - glb.add_mesh(mesh.m_Name, p, nw, uv, prims, active=is_active) - n_mesh += 1 - n_inactive += 0 if is_active else 1 - - out_path = Path(out_path) - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_bytes(glb.bytes({"backgroundKey": bg_key})) - doc = { - "backgroundKey": bg_key, - "glb": out_path.name, - "meshCount": n_mesh, - "inactiveMeshes": n_inactive, - "skippedEmptyMeshes": skipped, - "nullMaterialSubmeshes": null_slots, - "nullMeshFilters": null_meshes, - "meshesWithoutTriangles": no_triangles, - **glb.summary(), - } - write_json(out_path.with_suffix(".json"), doc) - return doc - - -def _master(md: Path, name: str) -> list[dict]: - return json.loads((md / f"{name}.json").read_text(encoding="utf-8"))["_allData"] - - -def text_bytes(ta) -> bytes: - s = ta.m_Script - return s.encode("utf-8", "surrogateescape") if isinstance(s, str) else bytes(s) - - -def _target(ref): - """The object a PPtr refers to, None for an empty reference (path id 0).""" - return ref.read() if ref.path_id else None - - -def write_skeleton(o, write, written_atlases: dict[int, str] | None = None) -> dict: - """The Spine files of the SkeletonDataAsset object `o` (module docstring), each given to `write(file name, - bytes)` in this order: the skeleton, then for each of its atlases not in `written_atlases` ({SpineAtlasAsset - path id: atlas file name}, updated here) the atlas text as stored and one PNG per page. Returns the skeleton's - record of spot.json: name, skeleton file, atlas files, scale, default mix, mixes.""" - written = {} if written_atlases is None else written_atlases - sda = o.read_typetree() - sk = o.read().skeletonJSON.read() - raw = text_bytes(sk) - is_json = raw.lstrip()[:1] == b"{" - ext = ".json" if is_json else ".skel" - sk_file = sk.m_Name if sk.m_Name.endswith(ext) else f"{sk.m_Name}{ext}" # as the atlas below - write(sk_file, raw) - atlases = [] - for ap in o.read().atlasAssets: - atlas_asset = ap.read() - if ap.path_id not in written: - af = atlas_asset.atlasFile.read() - araw = text_bytes(af) - atext = araw.decode("utf-8") - a_file = af.m_Name if af.m_Name.endswith(".atlas") else f"{af.m_Name}.atlas" - write(a_file, araw) # the atlas text exactly as the game stores it - pages = atlas_pages(atext) - mats = list(atlas_asset.materials) - if len(mats) != len(pages): - raise RuntimeError(f"{a_file}: {len(pages)} pages vs {len(mats)} materials") - for page, mp in zip(pages, mats): - mat = mp.read() - tex = next(e.m_Texture.read() for k, e in mat.m_SavedProperties.m_TexEnvs if k == "_MainTex") - buf = io.BytesIO() - texture_image(tex).save(buf, format="PNG") - write(page, buf.getvalue()) - written[ap.path_id] = a_file - atlases.append(written[ap.path_id]) - return { - "name": sda["m_Name"], - "skeleton": sk_file, - "atlases": atlases, - "scale": sda["scale"], - "defaultMix": sda["defaultMix"], - "mixes": [ - {"from": f, "to": t, "duration": d} - for f, t, d in zip(sda.get("fromAnimation", []), sda.get("toAnimation", []), sda.get("duration", [])) - ], - } - - -def spine_characters(objs: list, graph, skeletons: dict, world_of) -> list[dict]: - """SpotSpineCharacter components: object path, the SkeletonAnimation they drive (its skeleton data name, - initial animation settings and world matrix). `skeletons`: SkeletonDataAsset path id -> skeleton record. One - whose `_animation` is empty has those three null.""" - out = [] - for o in objs: - ssc = o.read_typetree() - path = graph.path(graph.tf_of_go[ssc["m_GameObject"]["m_PathID"]]) - anim_o = _target(o.read()._animation) - if anim_o is None: - out.append({"path": path, "skeletonData": None, "animation": None, "world": None}) - continue - at = anim_o.object_reader.read_typetree() - out.append( - { - "path": path, - "skeletonData": skeletons[anim_o.skeletonDataAsset.path_id]["name"], - "animation": { - k: at[k] - for k in ( - "_animationName", - "loop", - "timeScale", - "initialSkinName", - "initialFlipX", - "initialFlipY", - "pmaVertexColors", - "tintBlack", - "zSpacing", - ) - }, - "world": world_of(at["m_GameObject"]), - } - ) - return out - - -def tap_targets(objs: list, graph, world_of) -> list[dict]: - """SpotCharacter components (tap targets) by character id: object path, their fields, world matrix and the world - matrix of their `_focus` transform (null when the reference is empty).""" - out = [] - for o in objs: - sc = o.read_typetree() - focus_o = _target(o.read()._focus) - out.append( - { - "path": graph.path(graph.tf_of_go[sc["m_GameObject"]["m_PathID"]]), - **strip_pptrs({k: v for k, v in sc.items() if k != "_focus"}), - "world": world_of(sc["m_GameObject"]), - "focusWorld": None - if focus_o is None - else world_of(focus_o.object_reader.read_typetree()["m_GameObject"]), - } - ) - out.sort(key=lambda c: c["_characterId"]) - return out - - -def extract_spot(cat: Catalog, md: Path, spot_id: int, out_dir: Path) -> dict: - if cat.apk is None: - raise apk_missing("resolving the Spot / Spine component classes") - spots = {r["_id"]: r for r in _master(md, "MasterHomeSpot")} - sp = spots[spot_id] - txt = {r["_id"]: r for r in _master(md, "MasterText")} - taps: dict[int, list[int]] = {} - for r in _master(md, "MasterStoryHomeSpotTapTalkEpisode"): - if r.get("_spotId") == spot_id: - taps.setdefault(r["_characterId"], []).append(r["_advId"]) - - out_dir = Path(out_dir) - spine_dir = out_dir / "spine" - spine_dir.mkdir(parents=True, exist_ok=True) - - env = UnityPy.load(*[str(p) for p in cat.fetch_key(sp["_situationAssetPath"])]) - graph = SceneGraph(env) - by_cls: dict[str | None, list] = {} - for o in env.objects: - if o.type.name == "MonoBehaviour": - by_cls.setdefault(script_class(o), []).append(o) - - def world_of(go_pptr) -> list[float]: - return mat4(graph.world_of_go(go_pptr["m_PathID"])) - - settings = strip_pptrs(by_cls["SpotSituationSettings"][0].read_typetree()) - controller_o = by_cls["SpotCharacterController"][0] - controller = strip_pptrs(controller_o.read_typetree()) - - # Spine skeleton data -> files - def write(name: str, data: bytes) -> None: - (spine_dir / name).write_bytes(data) - - skeleton_files: dict[int, dict] = {} - written_atlases: dict[int, str] = {} - for o in by_cls.get("SkeletonDataAsset", []): - skeleton_files[o.path_id] = write_skeleton(o, write, written_atlases) - - spine_chars = spine_characters(by_cls.get("SpotSpineCharacter", []), graph, skeleton_files, world_of) - characters = tap_targets(by_cls.get("SpotCharacter", []), graph, world_of) - - doc = { - "spotId": spot_id, - "master": sp, - "name": {lang[1:]: txt.get(sp.get("_nameTextId"), {}).get(lang, "") for lang in LANGS}, - "tapTalk": {str(k): v for k, v in sorted(taps.items())}, - "situationSettings": settings, - "controller": {k: v for k, v in controller.items() if k not in ("_spotSpineCharacters", "_characters")}, - "characters": characters, - "spineCharacters": spine_chars, - "skeletons": list(skeleton_files.values()), - "resources": { - "background": [b.name for b in cat.resolve(sp["_backgroundAssetPath"])], - "situation": [b.name for b in cat.resolve(sp["_situationAssetPath"])], - }, - "matrixConvention": "Unity world space (left-handed), 4x4 column-major", - } - write_json(out_dir / "spot.json", doc) - return doc diff --git a/src/nnnotes/content/live.py b/src/nnnotes/content/live.py index 470902c..0865498 100644 --- a/src/nnnotes/content/live.py +++ b/src/nnnotes/content/live.py @@ -1,64 +1,12 @@ -"""A live (a music's chart at a difficulty): the `live` product, and the self-contained live directory of `build`. - -The `live` product (live.json format LIVE_FORMAT), read from a product's object table (`Objects`, -pipeline.assemble), never from a bundle, holds what belongs to one chart and names the products it plays with by -their manifest paths: - - chart.json the chart as shipped (the TextAsset of chart_key, un-gzipped) - notes.json the runtime notes the game converts it to (score.convert), with `source` - notes.mirror.json the same with the MirrorChart option on - live.json musicId, difficulty, `band` (band_choice: the band whose start timeline and background the - live plays), the files above, the counts of the converted chart (judgementNoteCount, - fullComboCount, noteCount, lastNoteTimeMs), `master` (music_record: the music's rows, title - and band names), `jacket` (the Sprite of jacket_key) and `parts` {part: manifest path}: the - scene (live.scene), the note highway (live.notes) with the default note skin and effect set - (live.noteskin, live.noteeffect), the default sounds (live.sounds) and the music - (live.music) - -A subject names a chart and optionally the band: `/[/]` (live_subject). Without a band -the live takes the band of DEFAULT_BAND_RULE. - -`build` writes the live directory instead, the data format the player reads from a directory: - - /score/ chart as shipped, converted runtime notes, master rows (score.py) - /audio// the live BGM cue sheet decoded per cue + cues.json + streams.json (score.py) - /audio/live-audio.json the live's sounds (BGM, note SE, live SE) with their CRI routing, and the - note SE / live SE cue sheets decoded next to the BGM (liveaudio.py) - /livescene/ scene graph, cameras, lane, LightWeight background, start timeline, - textures and shaders (livescene.py) - /livenotes/ note / line / effect prefabs, skins, clips, particle systems, - textures and shaders (livenotes.py) - /liveui/ the start canvas (music title, credits, difficulty) in the client language: - strings, text records or the game's fonts, sprites (liveui.py) - /live.json index of the above - -`options` (liveoptions.LiveOptions, from `--live-option`) adds the files of the option variants it offers: the -mirrored score (score.py, live.json `notesMirror`), the other note skins and effect sets, the bar line view and the -option tables (livenotes.py), the other note sound sets (liveaudio.py). Without it the directory holds what the -default options read. - -The band of the LightWeight background and of the start timeline is the band of the player's deck centre -(`SelfMemberList[2]`: LightWeightBackgroundLoadStep.d__3 reads its BandID, LiveResourceBandResolver.Resolve -its MemberCard's Character.Band; MasterMemberCard.get_BandID = Character._bandID). The preview has no deck: -`band_choice` takes it from a leader card or a band given, else from DEFAULT_BAND_RULE. -""" +"""Live products assembled from chart objects, master records and shared part manifests.""" from __future__ import annotations import gzip -from pathlib import Path -from typing import TYPE_CHECKING -from ..core.jsonio import write_json -from ..formats.encoders import FLAC_LEVEL from ..source import languages -from .liveoptions import LiveOptions from .prefabs import Objects, key_object -if TYPE_CHECKING: - from ..source.catalog import Catalog - from ..unity.player import PlayerData - LIVE_FORMAT = 1 # live.json "format" DIFFICULTIES = ("easy", "normal", "hard", "expert") # _easyID.._expertID -> file suffix _00.._03 # the master tables of a music record (music_record) and of a live record (live_record) @@ -242,111 +190,3 @@ def live(objects: Objects, record: dict, parts: dict) -> tuple[dict, bytes, dict chart = gzip.decompress(raw) if raw[:2] == b"\x1f\x8b" else raw keep = {o for o in objects.ids() if o != text and objects.info(o).get("class") != "MonoScript"} return doc, chart, {"notes.json": notes, "notes.mirror.json": mirror}, keep - - -def resolve_band( - cat: Catalog, master: Path, music_id: int, band: int | None = None, leader_card: int | None = None -) -> dict: - """The preview's band (band_choice; recorded in livescene/scene.json slice.bandChoice) of a band whose start - timeline and background the catalog holds (BAND_KEYS).""" - from .score import master_table - - names = ("MasterCharacter", "MasterLiveMusic") + (("MasterMemberCard",) if leader_card is not None else ()) - choice = band_choice({n: master_table(master, n) for n in names}, music_id, band, leader_card) - b = choice["band"] - missing = [k.format(band=b) for k in BAND_KEYS if not cat.has(k.format(band=b))] - if missing: - raise KeyError(f"band {b}: {', '.join(missing)} not in the catalog") - return choice - - -def index_doc(music_id: int, difficulty: str, score_summary: dict, audio: dict, live_audio: str) -> dict: - """live.json: `score_summary` from score.extract (its `notesMirror` when the score was also converted - mirrored), `audio` its BGM entry, `live_audio` the path of the live sounds (liveaudio.extract).""" - s = score_summary - return { - "musicId": music_id, - "difficulty": difficulty, - "chart": s["chart"], - "notes": s["notes"], - **({"notesMirror": s["notesMirror"]} if "notesMirror" in s else {}), - "master": s["master"], - "audio": audio, - "liveAudio": live_audio, - "scene": "livescene/scene.json", - "noteAssets": "livenotes/notes.json", - "liveUi": "liveui/liveui.json", - } - - -def build( - cat: Catalog, - master: Path, - player: PlayerData, - music_id: int, - difficulty: str, - out_dir: Path, - audio_format: str = "flac", - band: int | None = None, - leader_card: int | None = None, - *, - language: str, - fonts: str = "open", - options: LiveOptions = LiveOptions(), - flac_level: int = FLAC_LEVEL, -) -> dict: - """One live directory. `language`: the client language of the start canvas (a languages.LANGUAGES code); - `fonts`: "open" or "game" (liveui.extract); `options`: the option variants whose files it carries as well - (liveoptions.resolve); `flac_level`: ffmpeg's compression level of the FLAC audio (cri.decode).""" - from . import liveaudio, livenotes, livescene, liveui, score - - out_dir = Path(out_dir) - out_dir.mkdir(parents=True, exist_ok=True) - choice = resolve_band(cat, Path(master), music_id, band=band, leader_card=leader_card) - s = score.extract( - cat, - master, - music_id, - difficulty, - out_dir, - audio=True, - audio_fmt=audio_format, - flac_level=flac_level, - mirror=options.mirror, - ) - la = liveaudio.extract( - cat, - Path(master), - player, - music_id, - out_dir, - fmt=audio_format, # reuses the BGM decode - options=options, - flac_level=flac_level, - ) - sc = livescene.extract( - cat, player, out_dir, master=Path(master), music_id=music_id, band=choice["band"], band_choice=choice - ) - lu = liveui.extract( - cat, - player, - out_dir, - master=Path(master), - music_id=music_id, # reads livescene's shaders - difficulty=difficulty, - language=language, - fonts=fonts, - ) - nt = livenotes.extract(cat, player, out_dir, master=Path(master), options=options) - index = index_doc(music_id, difficulty, s, s["audio"], la["index"]) - write_json(out_dir / "live.json", index) - return { - **index, - "band": choice, - "sceneSummary": sc, - "noteAssetSummary": nt, - "liveAudioSummary": la, - "liveUiSummary": lu, - "judgementNoteCount": s["judgementNoteCount"], - "fullComboCount": s["fullComboCount"], - } diff --git a/src/nnnotes/content/liveaudio.py b/src/nnnotes/content/liveaudio.py index b491023..9b6f193 100644 --- a/src/nnnotes/content/liveaudio.py +++ b/src/nnnotes/content/liveaudio.py @@ -1,56 +1,12 @@ -"""Live sounds: the cue structure of the live's cue sheets, the game's sound settings and the live-audio documents. - -What a live plays besides its chart, from the master data (MasterSound, MasterSoundCueSheet, MasterLiveMusic, -MasterLiveNoteSe, MasterLiveSe, MasterOptionDefault), the cue sheets' ACBs and their decoded streams (cri.decode or -the cri.audio stage), the ACF of the APK (ACF_IN_APK) and two MonoBehaviours of the APK's built-in files -(SETTINGS_SCRIPTS). A sound entry (sound_entry): - - {row, sheet, cue, category, categories, volume, busSends, lengthMs, other, layers[{file, stream, awbId, - sampleRate, channels, samples, loopStart, loopEnd, loopFlag, volume, busSends, other}]} - -the cue structure (sequence -> tracks -> synth -> waveform) read from the ACB (acb_cues: command 65 categories, 146 -volume, 111 bus send), each layer the decoded stream `audio//`. - -music_doc (the live.music product, format MUSIC_FORMAT): `music` {soundId, volume, startTimeSec} the BGM sound, and -`sounds` {its id: its entry}, each layer in the product's audio format. - -sounds_doc (the live.sounds product, format SOUNDS_FORMAT), the sounds of the default options (auto play at Perfect, -fresh profile) and of the note sound sets offered: - - sounds MasterSound id -> entry: the note SE of the default preset and of every offered set, and the live - SE of LIVE_SE_SOUNDS (the start and finish cheers and the finish direction of every result: the one a - live ends with depends on how it is played) - categories CRI category volumes for a fresh profile (SoundVolumeSettings x AppConfigDefaultData) - acf, react the ACF's buses and categories, its REACT (ducking) entries between live categories - noteSe LiveNoteSeType -> sound id / volume / mute for the default preset (LiveSettingCreator.CreateSESettings); - with sets offered also `groups`: set id -> LiveNoteSeType -> sound id (MasterLiveNoteSe) - liveSe LiveSeType -> sound id (MasterLiveSe) - timeline intro / end timings the runtime needs - -A live.sounds subject names the sets offered: `default`, or `NoteSePatternId=,,...` (sounds_subject). - -extract() writes the same for one music into a live directory (audio/live-audio.json: the fields of both documents -in one, the BGM sound among the sounds) from a Catalog, decoding each cue sheet with cri.decode into -`/audio//` (a sheet already decoded there, the BGM by score.py, is reused). -""" +"""Music and live sound product records from explicit master data, cue sheets and player settings.""" from __future__ import annotations import struct -from pathlib import Path -from typing import TYPE_CHECKING -from ..core.jsonio import write_json from ..formats.acb import commands as _commands from ..formats.acb import tables as _tables from ..formats.acb import u16s as _u16s -from ..formats.encoders import FLAC_LEVEL -from ..source.apkset import ApkSet - -if TYPE_CHECKING: - from ..source.catalog import Catalog - from ..unity.player import PlayerData - from .liveoptions import LiveOptions ACF_IN_APK = "assets/Cri/Sound/Sirius.acf" MUSIC_FORMAT, SOUNDS_FORMAT = 1, 1 # the "format" of music_doc / sounds_doc @@ -548,94 +504,3 @@ def sounds_doc(tables: dict, subject: str, sheet, svs: dict, acd: dict, acf: byt # --------------------------------------------------------------------------- extraction -def _decode(cat: Catalog, sheet: str, out_dir: Path, fmt: str, flac_level: int) -> list[dict]: - """Decode a cue sheet into /audio// (reusing an existing decode) and return its stream list.""" - from ..formats import cri - - d = out_dir / "audio" / sheet - if not (d / "streams.json").exists(): - cri.decode(cat, sheet, d, fmt=fmt, flac_level=flac_level) - import json - - return json.loads((d / "streams.json").read_text(encoding="utf-8")) - - -def _player_mono(player: PlayerData, cls: str) -> dict: - hits = [] - for o in player.env.objects: - if o.type.name != "MonoBehaviour": - continue - try: - if player.script(o)[2] != cls: - continue - except Exception: # MonoBehaviours without a resolvable script are not ours - continue - hits.append(player.mono(o)) - if len(hits) != 1: - raise RuntimeError(f"{cls}: {len(hits)} objects in the player data") - return hits[0] - - -def extract( - cat: Catalog, - master: Path, - player: PlayerData, - music_id: int, - out_dir: Path, - fmt: str = "flac", - options: LiveOptions | None = None, - *, - flac_level: int = FLAC_LEVEL, -) -> dict: - """Decode the live's sound cue sheets and write /audio/live-audio.json (module documentation). Returns a - summary. `options`: the note sound sets it offers are written as noteSe.groups, their sounds decoded after the - default ones. `flac_level`: ffmpeg's compression level of the FLAC files (cri.decode).""" - from ..formats import cri - from .score import master_table - - out_dir = Path(out_dir) - if cat.apk is None: - raise RuntimeError("liveaudio needs the Catalog opened with apk= (ACF, HCA key)") - tables = {t: master_table(master, t) for t in dict.fromkeys((*MUSIC_TABLES, *SOUND_TABLES))} - music_row, _ = music_sound(tables, music_id) - live = live_sounds(tables, options.se_patterns if options is not None else ()) - layouts = {} - - def sheet(name: str): - acb, layouts[name] = cri.acb_data(cat, name) - return acb["acb"], _decode(cat, name, out_dir, fmt, flac_level) - - entries, read = sound_entries(tables, list(dict.fromkeys([music_row["_id"], *live["ids"]])), sheet) - with ApkSet(cat.apk) as z: - acf = acf_info(z.read(ACF_IN_APK)) - svs, acd = (_player_mono(player, s) for s in SETTINGS_SCRIPTS) - doc = { - "musicId": music_id, - "spec": "nnnotes.content.liveaudio", - "sounds": entries, - "categories": category_volumes(svs, acd), - "acf": {"name": acf["name"], "buses": acf["buses"], "categories": acf["categories"]}, - "react": acf["react"], - "music": {"soundId": music_row["_id"], "volume": 1.0, "startTimeSec": 0.0}, - "noteSe": live["noteSe"], - "liveSe": {str(k): v for k, v in sorted(live["liveSe"].items())}, - "liveSeNames": LIVE_SE_TYPES, - "timeline": TIMELINE, - "voice": { - "decision": "caller", - "character": None, - "rule": "LotteryStartVoice: random deck member, random MasterLiveStartCharacterVoice row of that " - "character; finish voice from LotteryFinishVoice; no deck -> undefined", - "startVoiceSoundIds": [], - }, - "decoded": { - n: {"layout": layouts[n], "cues": len(cues), "streams": len(streams)} for n, (cues, streams) in read.items() - }, - } - write_json(out_dir / "audio" / "live-audio.json", doc) - return { - "index": "audio/live-audio.json", - "sheets": sorted(read), - "sounds": len(entries), - "musicSoundId": music_row["_id"], - } diff --git a/src/nnnotes/content/livenotes.py b/src/nnnotes/content/livenotes.py index 01e136f..3d35d62 100644 --- a/src/nnnotes/content/livenotes.py +++ b/src/nnnotes/content/livenotes.py @@ -1,48 +1,11 @@ -"""The note highway of a live: what is spawned or animated per note, and its settings. - -Three products, each read from a product's object table (`Objects`, pipeline.assemble), never from a bundle, and each -shared by every live that loads it: - -notes the highway itself (`notes`, notes.json format NOTES_FORMAT): the note view prefabs (tap, flick - +left/right, trace, slide/slide_end/connection, guide, slide_combo(+skip), none), the long-note body - prefabs (slide_line_view, guide_line_view), the pair line and the groove loop (NOTE_PREFABS), the lane - effect settings (LANE_EFFECT_KEY, the set the game always loads: LaneEffectLoadStep.GetAssetPath), and - `settings`: the option defaults (preset 1) and ranges of the items the note runtime and the player read, - the MasterLiveSettings values, the note display time, the lane geometry, the default note skin and effect - set and the names of every note skin and effect set by option value (notes_settings) -noteskin a note skin (`noteskin`, MasterLiveNoteSkin._assetName: the LiveNoteSkinAsset of its key skin_key), with - the bindings of the arrow loops its flick units give the flick views (FLICK_SKIN_UNITS) -noteeffect a note effect set (`noteeffect`, MasterLiveNoteEffectSkin._assetName or its `Light` variant, which the - game loads at quality Low where the catalog has it: the LiveNoteEffectAssetSettings of its key - effect_settings_key and the effect prefabs it references) - -Each document names objects by id: notes.json `prefabs` {name: GameObject} and `laneEffect`, noteskin.json `skin`, -noteeffect.json `settings`; and each has - - bindings {prefab GameObject id: {Animator node path: {clip id: [{path, attribute}]}}}: the clips each Animator of - the table's prefabs plays (its controller's clips, the `_loopAnim` of a LoopAnimationTimeSync on its - `_animator`; for a skin, the arrow loop of each flick unit on the flick view's LoopAnimationTimeSync, - LiveFlickNoteView.OnSetupNoteSkin), named from the Animator's node (prefabs.clip_bindings) - shaders {shader name: object id}: what a lookup by name finds (prefabs.shader_table; a material names its - shader by id) - -Of the objects a product keeps all but the MonoScripts; of a shader its parsed form, variant index and the programs of -the player's graphics API; of a Mesh its typetree JSON and its `arrays` document (prefabs.mesh_roles). - -`extract` writes the note runtime of a live directory instead (livenotes/notes.json of the default options and the -variants a liveoptions.LiveOptions offers, with textures and shaders) through export.Exporter, following references -to AnimatorControllers and SpriteAtlases. -""" +"""Note, skin and effect products assembled from shared object tables and master records.""" from __future__ import annotations import re from collections.abc import Iterable, Mapping -from pathlib import Path -from typing import TYPE_CHECKING -from ..core.jsonio import write_json -from .liveoptions import OPTION_ITEM_TYPES, PLAYER_ITEMS, LiveOptions +from .liveoptions import OPTION_ITEM_TYPES, PLAYER_ITEMS from .prefabs import ( Objects, Prefab, @@ -57,11 +20,6 @@ shader_table, ) -if TYPE_CHECKING: - from ..source.catalog import Catalog - from ..unity.export import Exporter - from ..unity.player import PlayerData - __all__ = [ "EFFECT_KEY", "FLICK_SKIN_UNITS", @@ -74,7 +32,6 @@ "asset_name", "default_assets", "effect_settings_key", - "extract", "notes", "notes_keys", "notes_record", @@ -82,7 +39,6 @@ "noteeffect", "noteskin", "noteskin_keys", - "settings", "skin_key", ] @@ -189,8 +145,6 @@ "MasterLiveNoteSkin", "MasterLiveNoteEffectSkin", ) -DEFAULT_SKIN = "skin001" # MasterLiveNoteSkin._id 1 (NoteDesignId default 1) -DEFAULT_EFFECT = "effect001" # MasterLiveNoteEffectSkin._id 1 (NoteEffectId default 1) def skin_key(name: str) -> str: @@ -383,72 +337,12 @@ def noteeffect(objects: Objects, name: str) -> tuple[dict, set[str], dict[str, s return _product(objects, doc) -# ------------------------------------------------------------------ live directory (export.Exporter) -PREFABS = {**NOTE_PREFABS, "judge_effect_view": "EmbLive/Prefabs/UILiveNoteJudgeEffectView"} # Catalog key families exported whole (every key under the prefix): the note effect settings of the NoteEffectId # option's set ({effect}) and the lane effect settings (LANE_EFFECT), with the effect prefabs they reference -KEY_PREFIXES = [ - "Effect/Live/NoteEffect/{effect}/", - f"Effect/Live/LaneEffect/{LANE_EFFECT}/", -] -SINGLE_KEYS = [ - "EmbLive/Animation/Combo/UILiveCombo", # UIComboCounterView's animator controller - "Live/UiSpriteAssets/LiveComboSpriteAsset", - "Live/UiSpriteAssets/LiveJudgementSpriteAsset", -] - - -def settings(master: Path | None, options: LiveOptions = LiveOptions()) -> dict: - """The live directory's notes.json `settings`: option defaults and master values the note runtime reads; with - `options` also the offered note skins (`skins`: NoteDesignId -> MasterLiveNoteSkin._assetName) and effect sets - (`effects`: NoteEffectId -> MasterLiveNoteEffectSkin._assetName; also when only other qualities are offered, - whose effect set name the player derives from it), and with its option tables the items of PLAYER_ITEMS as - well.""" - from .score import master_table - - out: dict = {"skin": DEFAULT_SKIN, "effect": DEFAULT_EFFECT, **_lane_settings()} - if master is None: - return out - items = OPTION_ITEMS - if options.tables: - items = {**OPTION_ITEMS, **{k: OPTION_ITEM_TYPES[k] for k in PLAYER_ITEMS}} - opts = {} - for r in master_table(master, "MasterOptionDefault"): - if r["_presetId"] == 1 and r["_optionItemType"] in items: - opts[items[r["_optionItemType"]]] = r["_valueString"] - out["optionDefaults"] = opts - out["optionRanges"] = { - items.get(r["_optionItemType"], r["_optionItemType"]): [r["_minValue"], r["_maxValue"]] - for r in master_table(master, "MasterOptionRange") - if r["_optionItemType"] in items - } - ls = {r["_key"]: r["_value"] for r in master_table(master, "MasterLiveSettings")} - out["liveSettings"] = {k: ls[k] for k in LIVE_SETTING_KEYS if k in ls} - skins = {r["_id"]: r["_assetName"] for r in master_table(master, "MasterLiveNoteSkin")} - effects = {r["_id"]: r["_assetName"] for r in master_table(master, "MasterLiveNoteEffectSkin")} - out["skin"] = skins[int(opts["NoteDesignId"])] - out["effect"] = effects[int(opts["NoteEffectId"])] - out["judgementSprites"] = { - r["_noteSimulateJudgement"]: r["_spriteName"] for r in master_table(master, "MasterLiveJudgementSprite") - } - out["noteDisplayTimeMs"] = display_time_ms( - float(opts["NoteSpeed"]), - float(ls["note_speed_min"]), - float(ls["note_speed_max"]), - float(ls["note_speed_view_min"]), - float(ls["note_speed_view_max"]), - ) - if options.designs: - out["skins"] = {str(i): skins[i] for i in options.designs} - if options.effects or options.qualities: - out["effects"] = {str(i): effects[i] for i in options.values("NoteEffectId")} - return out # UI subtrees of [EmbScene/Live] animated by exported clips (combo counter, judgement text); used only to name # clip binding paths, the scene itself is exported by livescene. -SCENE_KEY = "EmbScene/Live" -SCENE_SUBTREES = ("UILiveCombo", "UILiveJudgement") # The bar line view (MeasureLineDisplay): the scene's LiveAllNoteView -> _barLineView (LiveAllBarLineView) -> # _container (LiveBarLineViewContainer) -> _elementPrefab (LiveBarLineView, the prefab the container instantiates) NOTE_VIEW = ("LiveGameView/LiveGameCamera/screen_root/LiveGameAllNoteView", "LiveAllNoteView") @@ -457,170 +351,3 @@ def settings(master: Path | None, options: LiveOptions = LiveOptions()) -> dict: ("_container", "LiveBarLineViewContainer"), ("_elementPrefab", "LiveBarLineView"), ) -BAR_LINE_VIEW = "bar_line_view" - - -TMP_FONT_NOTE = ( - "not exported: the note highway draws no TextMeshPro text with default options (the judge view's " - "fast / slow and assist texts stay hidden)" -) - - -def _drop_tmp_fonts(doc: dict, root: Path) -> list[str]: - """Reduce the font asset and material of every TextMeshProUGUI record to a name stub and delete the texture - files no record references any more (the TMP font atlases). Returns the deleted files.""" - - def walk(v): - if isinstance(v, dict): - if v.get("class") == "TextMeshProUGUI" and "m_fontAsset" in v: - fa, mat = v["m_fontAsset"], v.get("m_sharedMaterial") - if isinstance(fa, dict): - v["m_fontAsset"] = {"asset": fa.get("asset"), "name": fa.get("name"), "note": TMP_FONT_NOTE} - if isinstance(mat, dict): - v["m_sharedMaterial"] = { - "material": mat.get("material"), - "shader": mat.get("shader"), - "note": TMP_FONT_NOTE, - } - for x in v.values(): - walk(x) - elif isinstance(v, list): - for x in v: - walk(x) - - walk(doc) - used: set[str] = set() - - def refs(v): - if isinstance(v, dict): - t = v.get("texture") - if isinstance(t, str): - used.add(t) - for x in v.values(): - refs(x) - elif isinstance(v, list): - for x in v: - refs(x) - - refs(doc) - dropped = [] - for f in sorted((root / "textures").glob("*.png")): - rel = f"textures/{f.name}" - if rel not in used: - f.unlink() - dropped.append(rel) - return dropped - - -def effect_keys(cat: Catalog, effect: str) -> list[str]: - """The catalog keys of note effect set `effect` and of the lane effects (KEY_PREFIXES), in prefix and catalog - order.""" - keys: list[str] = [] - for p in KEY_PREFIXES: - keys += [k for k in cat.keys(p.format(effect=effect)) if k not in keys] - return keys - - -def _scene_mono(ex: Exporter, o, cls: str): - if o is None or o.type.name != "MonoBehaviour" or ex.script_class(o) != cls: - raise RuntimeError(f"expected a {cls} MonoBehaviour, got {o and o.type.name}") - return o, o.read_typetree() - - -def bar_line_prefab(ex: Exporter) -> dict: - """The bar line view as a node list: the LiveBarLineView prefab the live scene's bar line container - instantiates (BAR_LINE_CHAIN from the scene's LiveAllNoteView), its root first.""" - env, graph = ex.load(SCENE_KEY) - path, cls = NOTE_VIEW - gos = set(graph.gos_at_path(path)) - comps = [c["component"] for go in gos for c in graph.go[go]["m_Component"]] - owners = [o for o in env.objects if o.type.name == "GameObject" and o.path_id in gos] - if len(gos) != 1 or len(owners) != 1: - raise RuntimeError(f"{SCENE_KEY}: {len(gos)} GameObjects at {path}") - hits = [ - c - for c in (ex.deref(owners[0], p) for p in comps) - if c is not None and c.type.name == "MonoBehaviour" and ex.script_class(c) == cls - ] - if len(hits) != 1: - raise RuntimeError(f"{SCENE_KEY}: {len(hits)} {cls} at {path}") - o, tt = _scene_mono(ex, hits[0], cls) - for field, want in BAR_LINE_CHAIN: - o, tt = _scene_mono(ex, ex.deref(o, tt[field]), want) - root = graph.tf_of_go.get(tt["m_GameObject"]["m_PathID"]) - if root is None or graph.tf[root]["m_Father"]["m_PathID"]: - raise RuntimeError(f"{SCENE_KEY}: the bar line view is not a prefab root of the scene's bundles") - return { - "key": SCENE_KEY, - "reference": ".".join(f for f, _ in BAR_LINE_CHAIN), - "nodes": ex.hierarchy(env, graph, root), - } - - -def extract( - cat: Catalog, player: PlayerData, out_dir: Path, master: Path | None = None, options: LiveOptions = LiveOptions() -) -> dict: - """livenotes/ for the default options, plus the files of the variants `options` offers (the module - docstring). The variants' objects are exported after the default ones.""" - from ..unity.export import Exporter - - out_dir = Path(out_dir) - root = out_dir / "livenotes" - root.mkdir(parents=True, exist_ok=True) - ex = Exporter(cat, root, player=player, clip_format="livenotes", follow=("AnimatorController", "SpriteAtlas")) - doc: dict = {"settings": settings(master, options)} - skin = doc["settings"]["skin"] - - # note / line prefabs first: their hierarchies resolve clip binding paths of skin/effect assets - doc["prefabs"] = {name: ex.export_key(key) for name, key in PREFABS.items()} - doc["noteSkin"] = ex.export_key(skin_key(skin)) - keys = effect_keys(cat, doc["settings"]["effect"]) - keys += [k for k in SINGLE_KEYS if cat.has(k) and k not in keys] - assets, failures = {}, {} - - def export(ks): - for k in ks: - try: - assets[k] = ex.export_key(k) - except NotImplementedError as e: - failures[k] = str(e) - - export(keys) - other_skins = [n for n in doc["settings"].get("skins", {}).values() if n != skin] - if other_skins: - doc["noteSkins"] = {n: ex.export_key(skin_key(n)) for n in other_skins} - names = {int(i): n for i, n in doc["settings"].get("effects", {}).items()} - for effect, light in options.effect_sets(names): - if not cat.has(effect_settings_key(effect)): - if light: - continue # the game loads the set itself at quality Low (LiveOptions.effect_sets) - raise KeyError(f"effect set {effect}: {effect_settings_key(effect)} not in the catalog") - export([k for k in effect_keys(cat, effect) if k not in assets and k not in failures]) - if options.bar_lines: - doc["prefabs"][BAR_LINE_VIEW] = bar_line_prefab(ex) - doc["assets"] = assets - ex.add_path_source(SCENE_KEY, SCENE_SUBTREES) - unresolved = ex.resolve_pending() - doc["clips"] = ex.clips - doc["controllers"] = ex.controllers - doc["exportFailures"] = failures - - dropped = _drop_tmp_fonts(doc, root) - _, shader_summary = ex.dump_shaders(root / "shaders") - doc["shaders"] = {"index": "shaders/shaders.json", "names": shader_summary["names"]} - doc["convention"] = ( - "Unity space (left-handed, Y up); transforms are local TRS; quaternions (x,y,z,w); " - "colours as serialized (project colour space: see player.json)" - ) - write_json(root / "notes.json", doc) - return { - "prefabs": len(doc["prefabs"]), - "assets": len(assets), - "clips": len(ex.clips), - "controllers": len(ex.controllers), - "textures": len(ex.textures) - len(dropped), - "droppedFontTextures": dropped, - "failures": failures, - "unresolvedBindings": unresolved, - **shader_summary, - } diff --git a/src/nnnotes/content/livescene.py b/src/nnnotes/content/livescene.py index 092870a..29c79a9 100644 --- a/src/nnnotes/content/livescene.py +++ b/src/nnnotes/content/livescene.py @@ -1,71 +1,8 @@ -"""The scene of a live: what exists on the live screen before the first note, and the start timeline of each band. - -One product (`scene`, scene.json format SCENE_FORMAT), read from a product's object table (`Objects`, -pipeline.assemble), never from a bundle, and shared by every live: - - scene the root GameObjects of the addressable scene EmbScene/Live, each a prefab document (the Live - prefab instance with the cameras, render canvas, background view and UI canvases; LiveGameView - with the lane camera, the lane and the note and effect containers; LiveStartTimelineDirector); - their references to each other are `$ref`s to components (prefabs.component_ref) - renderSettings the scene's RenderSettings - graphics the built-in settings the render path reads, by id: `graphics` (GraphicsSettings: the render - pipeline asset, its renderers and their post-process data), `quality` (QualitySettings), `player` - (PlayerSettings: the colour space) - laneSkins {lane skin asset name: {part: Sprite}} of the lane skins of MasterLiveLaneSkin (LANE_SKIN_PARTS) - prefabs `laneLine` (LiveLaneLineView._linePrefab), `judgeEffectView` (UILiveNoteJudgeEffectView) and - `barLineView` (the LiveBarLineView prefab the scene's bar line container instantiates, - livenotes.BAR_LINE_CHAIN) - spriteAssets the combo and judgement sprite assets of the live UI (SPRITE_ASSETS) - bands {band id: {timeline, background, tracks}} of the bands of MasterBand whose start timeline the - catalog holds: the live_start_playable_timeline prefab (LiveStartTimelineLoadStep), the - LightWeight background sprite (LightWeightBackgroundLoadStep) and {AnimationTrack id: {name, - prefab, animator}}, the Animator each track of the timeline plays on (TIMELINE_TRACK_ANIMATORS) - settings the option defaults (preset 1) and ranges of the items the render path reads (OPTION_ITEMS), the - MasterLiveQualitySettings rows, the default lane skin and every lane skin by option value, and - the judgement sprite names by judgement (scene_settings) - bindings {prefab GameObject id: {Animator node path: {clip id: [{path, attribute}]}}}: the clips each - Animator plays through its controller and LoopAnimationTimeSyncs (livenotes.animator_bindings) - and the clips of each band's timeline tracks on theirs - shaders {shader name: object id}: what a lookup by name finds (prefabs.shader_table; of different - shaders of one name the one GraphicsSettings always includes), holding the shaders the live - screen finds by name (PLAYER_SHADERS); a material names its shader by id - -A live plays the timeline and the background of one band: the band of the player's deck centre, which the live -chooses. Of the objects the product keeps all but the MonoScripts; of a shader its parsed form, variant index and the -programs of the player's graphics API; of a Mesh its typetree JSON and its `arrays` document (prefabs.mesh_roles). - -`extract` writes the scene of a live directory instead (livescene/scene.json with textures and shaders) through -export.Exporter in its "livescene" clip format (clips inlined once, bindings with crc32 keys, path candidates and -curve counts): - - player colour space, quality levels, URP pipelines, renderer data (player.py) - scene the scene as a node list - laneSkin lane skin sprites of the LiveSkinId option's MasterLiveLaneSkin row - assets further prefabs by key: the start timeline of the band, the lane line prefab - master master rows the render path reads (music, quality settings, option defaults) - derived lane geometry computed with the game's formulas for the default options - (each entry names the function it mirrors) - postTextures film grain textures of the live renderers' post-process data - shaders every Shader in the loaded bundle closures, everything referenced, and the - shaders of the live camera renderers (GLES3 + other platforms, shader.py layout) - -Only what the LightWeight screen (mode 3) loads is exported (LiveResourceLoadPipelineFactory.Create): the mode-1 -stage (Band/{b}/live_stage/live_stage, its Volume Profile, StageBlinkLight), the background director and camera -blends are not loaded in mode 3; EmbLive/Volume/LiveStageVolume is referenced by nothing; the LiveGameVolume -profile is part of the scene (LiveEffectCamera's Volume). Only the music row, the slice and the jacket sprite depend -on the music: `band_part` exports the rest once for a band and `music_part` completes it for each music of that band -(the same document as `extract`). -""" +"""Live scene products assembled from shared prefab, shader and master records.""" from __future__ import annotations -import json -import math -from pathlib import Path -from typing import TYPE_CHECKING - -from ..core.jsonio import write_json -from .livenotes import BAR_LINE_CHAIN, LANE_COUNT, LANE_VIEW, NOTE_VIEW, animator_bindings +from .livenotes import BAR_LINE_CHAIN, NOTE_VIEW, animator_bindings from .liveoptions import OPTION_ITEM_TYPES from .prefabs import ( Objects, @@ -80,9 +17,6 @@ shader_table, ) -if TYPE_CHECKING: - from ..unity.export import Exporter - # --- keys ------------------------------------------------------------------- SCENE_KEY = "EmbScene/Live" LANE_SKIN_PARTS = ("lane_base", "lane_tap_area", "out_side_line") @@ -112,7 +46,6 @@ # Scene nodes whose textures the LightWeight preview draws (Live/RenderCanvas images); # every other texture of the scene prefab belongs to UI the preview does not show (headers, gauges, pause, gekisou, # skill cut-in, finish direction, mode-1 background) and is not written: its record stays, with "exported": false. -SCENE_DRAWN_TEXTURE_NODES = ("Live/RenderCanvas/LightWeightBackgroundShadowImage",) # shaders the live screen finds by name (GraphicsSettings always-included, APK player data): # LightWeight composite and blur, MV/VJ video PLAYER_SHADERS = ( @@ -123,7 +56,6 @@ "UI/Default", ) # URP renderer indices used by the scene's live cameras -LIVE_RENDERERS = (2, 3, 4, 5) # option items the render path reads OPTION_ITEMS = { k: OPTION_ITEM_TYPES[k] @@ -158,166 +90,9 @@ def lane_skin_key(name: str, part: str) -> str: return f"Live/Lane/{name}/{part}" -def master_rows(master: Path, music_id: int) -> dict: - from .score import master_table - - music = next(r for r in master_table(master, "MasterLiveMusic") if r["_id"] == music_id) - opts = { - r["_optionItemType"]: r["_valueString"] - for r in master_table(master, "MasterOptionDefault") - if r["_presetId"] == 1 and r["_optionItemType"] in OPTION_ITEMS - } - return { - "liveMusic": music, - "liveQualitySettings": master_table(master, "MasterLiveQualitySettings"), - "liveLaneSkin": master_table(master, "MasterLiveLaneSkin"), - "liveStageVideo": master_table(master, "MasterLiveStageVideo"), - "optionDefaultsPreset1": {OPTION_ITEMS[k]: {"id": k, "value": v} for k, v in sorted(opts.items())}, - "optionRanges": { - OPTION_ITEMS[r["_optionItemType"]]: [r["_minValue"], r["_maxValue"]] - for r in master_table(master, "MasterOptionRange") - if r["_optionItemType"] in OPTION_ITEMS - }, - } - - # --- derived lane geometry (mirrors the game functions named per entry) --- # LiveLaneLineView's constant tables: split border main / sub counts per LiveLaneSplitCountType # (None, Lane4, Lane6, Lane8, Lane12) -SPLIT_MAIN = (0, 4, 6, 8, 12) -SPLIT_SUB = (12, 12, 12, 0, 0) -REF_ASPECT = 1.7777778 # LiveGameView.FullInitialize -REF_FOV = 54.0 - - -def vertical_fov(width: int, height: int) -> float: - """LiveGameView.CalcVerticalFovKeepingHorizontal, aspect clamped to <= 16:9 by the caller.""" - a = min(width / height, REF_ASPECT) - h = math.atan(math.tan(math.radians(REF_FOV) * 0.5) * REF_ASPECT) - return math.degrees(2 * math.atan(math.tan(h) / a)) - - -def _node(nodes: list, path: str) -> dict: - hit = [n for n in nodes if n["path"] == path] - if len(hit) != 1: - raise KeyError(f"{path}: {len(hit)} nodes") - return hit[0] - - -def _comp(node: dict, cls: str) -> dict: - hit = [c for c in node["components"] if c.get("class", c["type"]) == cls] - if len(hit) != 1: - raise KeyError(f"{node['path']}: {len(hit)} {cls}") - return hit[0] - - -def derive_lane(nodes: list, options: dict) -> dict: - """Lane geometry for the given option values (MasterOptionDefault preset 1 strings).""" - import numpy as np - - lv = _comp(_node(nodes, "LiveGameView/root/LiveGameLane"), "LiveLaneView") - llv = _comp(_node(nodes, "LiveGameView/root/LiveGameLane/lines"), "LiveLaneLineView") - jr = _node(nodes, "LiveGameView/root/LiveGameLane/judgement_root") - n = LANE_COUNT - width = float(np.float32(lv["_laneWidth"])) # double field, passed on as (float) - length = lv["_lineLength"] - unit = width / n - judge_z = jr["localPosition"]["z"] # parents are identity: world z of position 0 - split = int(options["GuidelineCount"]) - main, sub = SPLIT_MAIN[split], SPLIT_SUB[split] - settings = {s["LineType"]: s for s in llv["_settings"]} - lines = [] - for i in range(n + 1): # LiveLaneLineView.UpdateProperties - if i == 0 or i == n: - t = 1 # GetSettings: OutSide - elif main and i % (n // main) == 0: - t = 0 # Normal - elif sub and i % (n // sub) == 0: - t = 2 # Space - else: - lines.append({"index": i, "active": False}) - continue - st = settings[t] - x = -width * 0.5 + i * unit - if t == 2: - ln, z0 = llv["_spaceLineLength"], judge_z - llv["_spaceLineLength"] * 0.5 - else: - ln, z0 = length, 0.0 - to_w = st["LineWidthTo"] * 2 if (i == n // 2 and t != 2) else st["LineWidthTo"] - # LiveLaneLine.SetLine: _anchor 2 (left side line) x -= startWidth/2, _anchor 1 (right) x += startWidth/2 - if i == 0: - x -= st["LineWidthFrom"] * 0.5 - elif i == n: - x += st["LineWidthFrom"] * 0.5 - lines.append( - { - "index": i, - "active": True, - "type": ("Normal", "OutSide", "Space")[t], - "start": [x, 0.0, z0], - "end": [x, 0.0, z0 + ln], - "startWidth": st["LineWidthFrom"], - "endWidth": to_w, - "gradient": st["LineColor"], - "alphaControlled": t != 1, - } - ) - return { - "laneCount": n, - "laneWidth": width, - "unitWidth": unit, - "lineLength": length, - "judgementZ": judge_z, - "unitCentersX": [unit * (i + 0.5) - width * 0.5 for i in range(n)], - "spawnRoots": [[unit * (i + 0.5) - width * 0.5, 0.0, length] for i in range(n)], - "judgementPositions": [[unit * (i + 0.5) - width * 0.5, 0.0, judge_z] for i in range(n)], - "splitCountType": split, - "splitBorderMainCount": main, - "splitBorderSubCount": sub, - "lines": lines, - "lineAlpha": min(max(int(options["GuidelineOpacity"]) / 100.0, 0.0), 1.0), - "laneBaseAlpha": min(max(int(options["LaneOpacity"]) / 100.0, 0.0), 1.0), - "showJudgementLine": options["JudgePositionDisplay"].upper() == "TRUE", - "laneJudgementPosOffset": (int(options["JudgePosition"]) + 5) / -10.0 + 1.0, - "screenPlane": lane_2d(), - "sources": { - "lines": "LiveLaneLineView.UpdateProperties, GetSettings, LiveLaneLine.SetLine", - "spawnRoots": "LiveNoteSpawnRoot.UpdateProperties", - "judgementPositions": "LiveNoteJudgementRoot3D.UpdateProperties", - "options": "LiveBootDataCreator.CreateViewData", - "screenPlane": "LiveSettingCreator.CreateLaneViewSettings, JudgementPositionLogic.GetJudgementPositions, " - "LiveViewUtility.GetNoteSpawnPosition", - }, - } - - -def lane_2d(lane_count: int = LANE_COUNT) -> dict: - """LiveSettingCreator.CreateLaneViewSettings as CreateViewData calls it (LANE_VIEW: size 1920x1080, - topRange 0.05, bottomRange 1, judgementScreenBottomPosition 2.24, topPosition 0) and - JudgementPositionLogic.GetJudgementPositions; units of 100 reference pixels.""" - w, h = LANE_VIEW["size"][0] / 100.0, LANE_VIEW["size"][1] / 100.0 - top, bottom = LANE_VIEW["topRange"], LANE_VIEW["bottomRange"] - lt, rt = [-w * top * 0.5, h * 0.5, 0.0], [w * top * 0.5, h * 0.5, 0.0] - lb, rb = [-w * bottom * 0.5, -h * 0.5, 0.0], [w * bottom * 0.5, -h * 0.5, 0.0] - t = min(max(LANE_VIEW["judgementScreenBottomPosition"] / (lt[1] - lb[1]), 0.0), 1.0) - jp = [] - for i in range(lane_count): - f = min(max(i / lane_count, 0.0), 1.0) - xb = (rb[0] - lb[0]) / lane_count * 0.5 + lb[0] + (rb[0] - lb[0]) * f - xt = (rt[0] - lt[0]) / lane_count * 0.5 + lt[0] + (rt[0] - lt[0]) * f - yb, yt = lb[1] + (rb[1] - lb[1]) * f, lt[1] + (rt[1] - lt[1]) * f - jp.append([xb + t * (xt - xb), yb + (yt - yb) * t, 0.0]) - # LiveViewUtility.GetNoteSpawnPosition: intersection of the two side edges - s1 = (lb[1] - lt[1]) / (lb[0] - lt[0]) - s2 = (rb[1] - rt[1]) / (rb[0] - rt[0]) - c1 = lt[1] - lt[0] * s1 - sx = ((rt[1] - rt[0] * s2) - c1) / (s1 - s2) - return { - "vertices": [lt, rt, lb, rb], - "judgementPositions": jp, - "spawnPosition": [sx, c1 + s1 * sx], - "pixelsPerUnit": 100.0, - } # --- product ---------------------------------------------------------------- @@ -499,285 +274,3 @@ def scene(objects: Objects, record: dict) -> tuple[dict, set[str], dict[str, set # --- extraction ------------------------------------------------------------- -def _component(nodes: list, cls: str) -> dict: - hit = [c for n in nodes for c in n["components"] if c.get("class", c["type"]) == cls] - if len(hit) != 1: - raise KeyError(f"{len(hit)} {cls} components") - return hit[0] - - -def bind_timeline(ex: Exporter, doc: dict, key: str) -> dict: - """Bind the clips of the start timeline's tracks to the Animators the game binds them to - (TIMELINE_TRACK_ANIMATORS): binding paths resolve against that GameObject's subtree, and each track records - `boundAnimator` {key, gameObject, field}. Returns binding counts per track.""" - tl = doc["assets"]["startTimeline"] - env, tl_graph = ex.load(key) - scene_graph = ex.load(SCENE_KEY)[1] - dir_json = _component(tl["nodes"], "PlayableDirector") - dir_path = next(n["path"] for n in tl["nodes"] if any(c is dir_json for c in n["components"])) - directors = [ - o - for o in env.objects - if o.type.name == "PlayableDirector" - and tl_graph.path(tl_graph.tf_of_go[o.read_typetree()["m_GameObject"]["m_PathID"]]) == dir_path - ] - if len(directors) != 1: - raise RuntimeError(f"{key}: {len(directors)} PlayableDirectors at {dir_path}") - asset_o = ex.deref(directors[0], directors[0].read_typetree()["m_PlayableAsset"]) - tracks_json = {t["m_Name"]: t for t in dir_json["m_PlayableAsset"]["m_Tracks"]} - counts = {} - for _track_o, tt, clips in ex.timeline_tracks(asset_o): - name = tt["m_Name"] - if name not in TIMELINE_TRACK_ANIMATORS: - if clips: - raise RuntimeError(f"{key}: track {name} with clips has no known binding") - continue - where, cls, field = TIMELINE_TRACK_ANIMATORS[name] - nodes, graph, owner = ( - (doc["scene"]["nodes"], scene_graph, SCENE_KEY) if where == "scene" else (tl["nodes"], tl_graph, key) - ) - ref = _component(nodes, cls)[field] - if not ref or ref.get("component") != "Animator": - raise RuntimeError(f"{cls}.{field}: {ref}") - root = ref["gameObject"] - tracks_json[name]["boundAnimator"] = {"key": owner, "gameObject": root, "field": f"{cls}.{field}"} - counts[name] = [ex.bind_clip(c, graph, root) for c in clips] - return counts - - -def lane_skin(rows: dict) -> str: - """Lane skin asset name: option LiveSkinId (305) -> MasterLiveLaneSkin row of that id (LaneSkinLoadStep).""" - sid = int(rows["optionDefaultsPreset1"]["LiveSkinId"]["value"]) - hit = [r for r in rows["liveLaneSkin"] if r["_id"] == sid] - if len(hit) != 1: - raise KeyError(f"MasterLiveLaneSkin: {len(hit)} rows with id {sid} (option LiveSkinId)") - return hit[0]["_assetName"] - - -def _texture_records(v, out: list) -> None: - if isinstance(v, dict): - if isinstance(v.get("texture"), str) and v["texture"].startswith("textures/"): - out.append(v) - for x in v.values(): - _texture_records(x, out) - elif isinstance(v, list): - for x in v: - _texture_records(x, out) - - -def _drop_scene_textures(doc: dict, base: Path, unlink: bool = True) -> list[str]: - """The texture files only the undrawn part of the scene prefab references (SCENE_DRAWN_TEXTURE_NODES): their - records marked "exported": false, the files deleted (`unlink`; else the caller does not write them). Returns - the files.""" - keep: list = [] - for k, v in doc.items(): - if k != "scene": - _texture_records(v, keep) - for n in doc["scene"]["nodes"]: - if n["path"] in SCENE_DRAWN_TEXTURE_NODES: - _texture_records(n["components"], keep) - kept = {r["texture"] for r in keep} - scene: list = [] - _texture_records(doc["scene"], scene) - dropped = sorted({r["texture"] for r in scene} - kept) - for r in scene: - if r["texture"] in dropped: - r["exported"] = False - if unlink: - for rel in dropped: - (base / rel).unlink() - return dropped - - -class SharedSceneMismatch(RuntimeError): - """A music's scene cannot be composed from its band's part (see music_part); export it whole.""" - - -def _export( - cat, - player, - base: Path, - rows: dict | None, - band: int, - skin: str, - extra_keys: dict | None, - slice_: dict | None, - jacket: str | None, -): - """The export sequence of extract (jacket None: without the jacket sprite, the slice and the music rows left as - placeholders for music_part). -> (exporter, doc, shader index, shader summary, timeline counts, dropped).""" - from ..unity.export import Exporter - - ex = Exporter(cat, base, player=player, clip_format="livescene", defer_writes=True) - graphics = player.graphics() - doc: dict = {"player": graphics} - doc["slice"] = slice_ - doc["scene"] = ex.prefab(SCENE_KEY) - doc["laneSkin"] = {part: ex.key_sprite(f"Live/Lane/{skin}/{part}") for part in LANE_SKIN_PARTS} - keys = dict(ASSET_KEYS) - keys.update(extra_keys or {}) - doc["assets"] = {name: ex.export_key(key.format(band=band)) for name, key in keys.items()} - timeline = bind_timeline(ex, doc, keys["startTimeline"].format(band=band)) - doc["sprites"] = { - name: ex.key_sprite(key.format(band=band, jacket=jacket)) - for name, key in SPRITE_KEYS.items() - if jacket is not None or name != "jacket" - } - if rows: - doc["master"] = rows if jacket is not None else None - opts = {k: v["value"] for k, v in rows["optionDefaultsPreset1"].items()} - doc["derived"] = { - "lane": derive_lane(doc["scene"]["nodes"], opts), - "fovRule": { - "refFov": REF_FOV, - "refAspect": REF_ASPECT, - "source": "LiveGameView.FullInitialize / CalcVerticalFovKeepingHorizontal", - "examples": { - f"{w}x{h}": vertical_fov(w, h) for w, h in ((2340, 1080), (1920, 1080), (1440, 1080), (2048, 1536)) - }, - }, - } - for name in PLAYER_SHADERS: - ex.add_shader(player.shader(name)) - rl = graphics["pipelines"][graphics["defaultPipeline"]]["m_RendererDataList"] - renderers = [rl[i] for i in LIVE_RENDERERS] - for rn in renderers: - for o in player.renderer_shaders(rn): - ex.add_shader(o) - doc["postTextures"] = {} - for rn in renderers: - if graphics["renderers"][rn].get("postProcessData"): - doc["postTextures"][rn] = { - "filmGrainTex": [ex.texture(o) for o in player.post_textures(rn, "filmGrainTex")] - } - index, summary = ex.dump_shaders(base / "shaders") - doc["shaders"] = {"index": "shaders/shaders.json", "names": summary["names"], "cameraRenderers": renderers} - doc["convention"] = ( - "Unity space (left-handed, Y up); transforms are local TRS; " - "quaternions (x,y,z,w); colours in the project's Gamma space" - ) - ex.resolve_pending() # bindings whose target hierarchy / component class was exported after the clip - dropped = _drop_scene_textures(doc, base, unlink=False) - ex.write_textures(skip=dropped) - return ex, doc, index, summary, timeline, dropped - - -def _summary(doc: dict, index: list, summary: dict, textures: int, dropped: list, timeline) -> dict: - gles3 = sum(1 for r in index for v in r["variants"] if v["platform"] == "gles3") - return { - "scene": "livescene/scene.json", - "nodes": len(doc["scene"]["nodes"]), - "assets": list(doc["assets"]), - "textures": textures - len(dropped), - "droppedTextures": len(dropped), - "shaders": summary["count"], - "variants": summary["variants"], - "gles3Variants": gles3, - "timelineBindings": timeline, - } - - -def extract( - cat, - player, - out_dir: Path, - master: Path | None = None, - music_id: int = 100001, - band: int | None = None, - band_choice: dict | None = None, - extra_keys: dict | None = None, -) -> dict: - """Write /livescene/scene.json (+ textures/, shaders/) and return a summary. - - `band`: the band of the LightWeight background (LightWeightBackgroundLoadStep.d__3: - SelfMemberList[2].BandID) and of the start timeline (LiveResourceBandResolver.Resolve). The game - takes it from the player's deck; the caller chooses it (live.resolve_band) and `band_choice` records how. - `master` (decoded master JSON dir) adds the music/option rows and the derived lane geometry; with `master` - the band is required. Without `master`: band 1 unless given, lane skin skin001.""" - out_dir = Path(out_dir) - rows = master_rows(master, music_id) if master else None - if band is None: - if rows: - raise ValueError( - f"music {music_id}: no band given; the LightWeight background and the start timeline " - "take the band of the deck centre (SelfMemberList[2]), see live.resolve_band" - ) - band = 1 - skin = lane_skin(rows) if rows else "skin001" - base = out_dir / "livescene" - base.mkdir(parents=True, exist_ok=True) - slice_ = {"musicId": music_id, "band": band, "laneSkin": skin} - if band_choice: - slice_["bandChoice"] = band_choice - jacket = rows["liveMusic"]["_jacketAssetName"] if rows else "jkt_001_100001" - ex, doc, index, summary, timeline, dropped = _export( - cat, player, base, rows, band, skin, extra_keys, slice_, jacket - ) - write_json(base / "scene.json", doc) - return _summary(doc, index, summary, len(ex.textures), dropped, timeline) - - -BAND_DOC = "band.json" - - -def band_part( - cat, player, out_dir: Path, master: Path, music_id: int, band: int, extra_keys: dict | None = None -) -> dict: - """What `extract` writes for any music of `band`: /livescene/ (textures, shaders; no scene.json) and - /band.json (the document with the slice, the jacket sprite and the music rows left out). `music_id`: - any music of the band (the option rows it reads are the same for every music).""" - out_dir = Path(out_dir) - rows = master_rows(master, music_id) - base = out_dir / "livescene" - base.mkdir(parents=True, exist_ok=True) - ex, doc, index, summary, timeline, dropped = _export( - cat, player, base, rows, band, lane_skin(rows), extra_keys, None, None - ) - part = { - "band": band, - "doc": doc, - "summary": _summary(doc, index, summary, len(ex.textures), dropped, timeline), - "textures": sorted({r["texture"] for r in _records(doc["scene"])}), - } - write_json(out_dir / BAND_DOC, part) - return part - - -def _records(v) -> list: - out: list = [] - _texture_records(v, out) - return out - - -def music_part( - cat, player, out_dir: Path, part: dict, master: Path, music_id: int, band_choice: dict | None = None -) -> dict: - """/livescene/scene.json of one music from its band's part (band_part; the part's livescene/ files - already in /livescene): the document extract writes. Raises SharedSceneMismatch when the jacket's - bundle closure holds shaders or a hierarchy (extract would dump or resolve against them) or the jacket texture - is one of the scene's (extract would keep it).""" - from ..unity.export import Exporter - - out_dir = Path(out_dir) - band = part["band"] - rows = master_rows(master, music_id) - doc = json.loads(json.dumps(part["doc"])) - key = SPRITE_KEYS["jacket"].format(band=band, jacket=rows["liveMusic"]["_jacketAssetName"]) - ex = Exporter(cat, out_dir / "livescene", player=player, clip_format="livescene") - env, _ = ex.load(key) - kinds = {o.type.name for o in env.objects} & {"Shader", "GameObject", "Transform", "RectTransform"} - if kinds: - raise SharedSceneMismatch(f"{key}: its closure holds {', '.join(sorted(kinds))}") - jacket = ex.key_sprite(key) - if {r["texture"] for r in _records(jacket)} & set(part["textures"]): - raise SharedSceneMismatch(f"{key}: its texture is one of the scene's") - slice_ = {"musicId": music_id, "band": band, "laneSkin": lane_skin(rows)} - if band_choice: - slice_["bandChoice"] = band_choice - doc["slice"] = slice_ - doc["sprites"]["jacket"] = jacket - doc["master"] = rows - write_json(out_dir / "livescene" / "scene.json", doc) - s = dict(part["summary"]) - s["textures"] += len(ex.textures) - return s diff --git a/src/nnnotes/content/liveui.py b/src/nnnotes/content/liveui.py index 0bf9d1d..a2732d6 100644 --- a/src/nnnotes/content/liveui.py +++ b/src/nnnotes/content/liveui.py @@ -1,42 +1,8 @@ -"""Live start canvas (Live/UILiveStartCanvas, LightWeight mode) -> /liveui/ (liveui.json, textures/). - -livescene/scene.json already carries the canvas hierarchy with every component (serialized TMP texts, inline -sprite records, the start timeline and its UILiveStartCanvasTrack clips) and livescene/shaders the shaders it -draws with. This module adds what a renderer needs beyond that to draw the canvas the way -UILiveStartCanvas.Initialize leaves it: - - strings the texts Initialize sets: music title, singer, the credit lines (LocalizeText master text id + - replacement word), music level, high score, resolved as the game resolves them - texts per TMP text of the drawn part: the LocalizeText font / material / line spacing overrides; with - fonts "open" also its text record (textstyle: layout, colours, style values of its material) - fonts fonts "game": TMP font assets reduced to the characters shown (tmpfont); FZLTH's runtime glyphs - generated from its source font; VibeMOPro's glyph pair adjustment lookup (kerning). fonts "open" - (default): the font asset names only, no font data and no atlases - materials fonts "game": the localized TMP materials (" - Default") and the fonts' default materials; - fonts "open": none - sprites WhiteRect, Songtitle_base, the Difficulty_* icons of _difficultyPairs and the music jacket; their - texel blocks copied into textures/ (export.TexelPacker, deferred-texture mode) - controller the root Animator's controller (evidence only: the start timeline drives the canvas) - -Texts and fonts follow the client language `language` (a catalog language code, languages.LANGUAGES): its -LanguageMode selects the text table column, the LocalizeManager fonts and the line spacing. -""" +"""Live start-canvas paths and localized strings from explicit master records.""" from __future__ import annotations -import json -from pathlib import Path -from typing import TYPE_CHECKING - -from ..core.jsonio import write_json from ..source import languages -from ..unity import fontsettings, textstyle -from ..unity.textstyle import LANGUAGE_LINE_SPACING - -if TYPE_CHECKING: - from ..source.catalog import Catalog - from ..unity.export import Exporter, TexelPacker - from ..unity.player import PlayerData SCENE_KEY = "EmbScene/Live" CANVAS = "Live/UILiveStartCanvas" @@ -44,39 +10,15 @@ # Parts drawn in LightWeight mode: root/base (a _simpleModeOnlyObjects entry), center_bottom and # simple_music_jacket_panel (left_top is _normalModeOnlyObjects, right_top/battle_panel is _battleLiveGameObjects). DRAWN = (f"{CANVAS}/root/base", f"{CONTENT}/center_bottom", f"{CONTENT}/simple_music_jacket_panel") -JACKET_KEY = "Image/Jacket/{jacket}" # App.LiveBase.LiveDifficulty LIVE_DIFFICULTY = {"easy": 0, "normal": 1, "hard": 2, "expert": 3, "master": 4} -TMP_STUBS = ("TMP_FontAsset", "TMP_SpriteAsset", "TMP_StyleSheet") -TMP_CLASS = "TextMeshProUGUI" JAPANESE_COLUMN = languages.column("ja") -FONT_CHOICES = ("open", "game") -FONT_NOTE = 'not exported (fonts "open"): the text record in `texts` describes the text without font data' def _drawn(path: str) -> bool: return any(path == d or path.startswith(d + "/") for d in DRAWN) -def _comp(node: dict, cls: str) -> dict | None: - hit = [c for c in node["components"] if c.get("class", c["type"]) == cls] - if len(hit) > 1: - raise RuntimeError(f"{node['path']}: {len(hit)} {cls}") - return hit[0] if hit else None - - -def resolve_strings(master: Path, music_id: int, difficulty: str, column: str) -> tuple[dict, dict]: - """The strings UILiveStartCanvas.Initialize sets (IReadOnlyLiveMusicScore slots 2..8), for a fresh profile, with - the text table column `column` of the current LanguageMode. -> (strings, sources).""" - from .score import master_table - - tables = { - name: master_table(master, name) - for name in ("MasterLiveMusic", "MasterText", "MasterBand", "MasterLiveMusicScore") - } - return record_strings(tables, music_id, difficulty, column) - - def record_strings(tables: dict, music_id: int, difficulty: str, column: str) -> tuple[dict, dict]: """The live start strings and their sources from explicit master-table rows.""" music = next(r for r in tables["MasterLiveMusic"] if r["_id"] == music_id) @@ -153,267 +95,3 @@ def credit(label_id: str, word_id: str) -> str: "_musicLevelTexts": "musicLevel", "_highScoreTexts": "highScore", } - - -def _game_fonts(ex: Exporter, fontset, texts: dict, chars_by_font: dict, packer: TexelPacker): - """The game's TMP fonts of the texts, reduced to the characters shown (tmpfont), their atlas blocks added to - `packer`. -> (glyph coverage, materials, fonts).""" - from ..unity import tmpfont - - coverage: dict[str, dict] = {} - needed: dict[str, set] = {} - runtime_units: dict[str, set] = {} - primaries = set() - for fname in sorted(chars_by_font): - primary = fontset.by_key(fontsettings.font_dir(fname) + fname) - primaries.add(primary.name) - cov, nd, rt = tmpfont.glyph_coverage(fontset, primary, sorted(chars_by_font[fname])) - if cov["missing"]: - raise RuntimeError(f"{fname}: characters {cov['missing']} missing from the font chain") - coverage[fname] = cov - for k, v in nd.items(): - needed.setdefault(k, set()).update(v) - for k, v in rt.items(): - runtime_units.setdefault(k, set()).update(v) - materials, text_mats = tmpfont.text_materials( - ex, fontset, [t["localized"] for t in texts.values()], primaries, needed, runtime_units - ) - font_out = tmpfont.export_fonts( - fontset, - primaries, - needed, - runtime_units, - text_mats, - packer, - "characters shown by the live start canvas (plus baked control characters); " - "runtime characters generated from the source font (runtimeGlyphs)", - ) - return coverage, materials, font_out - - -def extract( - cat: Catalog, - player: PlayerData, - out_dir: Path, - master: Path, - music_id: int, - difficulty: str, - *, - language: str, - fonts: str = "open", -) -> dict: - """Write /liveui/ (liveui.json, textures/) for one live and return a summary. Needs - /livescene/shaders/shaders.json (livescene.extract) for the shader check. `language`: the client - language (a languages.LANGUAGES code); `fonts`: "open" (text records, font names only) or "game" (the game's - TMP fonts reduced to the characters shown; needs the `fonts` extra).""" - from ..unity import tmpfont - from ..unity.export import Exporter, TexelPacker, _safe - from .score import master_table - - out_dir = Path(out_dir) - base = out_dir / "liveui" - base.mkdir(parents=True, exist_ok=True) - if difficulty not in LIVE_DIFFICULTY: - raise KeyError(f"difficulty {difficulty}") - if fonts not in FONT_CHOICES: - raise ValueError(f"fonts {fonts}: one of {', '.join(FONT_CHOICES)}") - mode, column = languages.mode(language), languages.column(language) - ex = Exporter( - cat, - base, - player=player, - textures="deferred", - stub_assets=TMP_STUBS, - follow=("AnimatorController",), - font_stubs=fontsettings.builtin() if fonts == "open" else None, - ) - packer = TexelPacker(base) - - # -- the canvas subtree of the scene (deferred: sprite / texture references) ------------------------- - env, graph = ex.load(SCENE_KEY) - roots = [p for p in graph.tf if graph.path(p) == CANVAS] - if len(roots) != 1: - raise RuntimeError(f"{CANVAS}: {len(roots)} transforms") - nodes = ex.hierarchy(env, graph, roots[0]) - by_path: dict[str, list] = {} - for n in nodes: - by_path.setdefault(n["path"], []).append(n) - canvas = _comp(by_path[CANVAS][0], "UILiveStartCanvas") - if canvas is None: - raise RuntimeError(f"{CANVAS}: no UILiveStartCanvas") - - # -- strings, and which texts Initialize writes them to --------------------------------------------- - strings, sources = resolve_strings(master, music_id, difficulty, column) - text_of: dict[str, str] = {} - for field, key in TEXT_FIELDS.items(): - refs = canvas[field] if isinstance(canvas[field], list) else [canvas[field]] - for r in refs: - text_of[r["gameObject"]] = strings[key] - - # -- TMP texts of the drawn part: LocalizeText overrides and the characters they show ------------- - styles = textstyle.TextStyles(ex, player, mode, fontsettings.builtin()) if fonts == "open" else None - lang = styles.lang if styles else textstyle.language_fonts(player) - swap = styles.swap if styles else textstyle.font_swap(lang, mode) - fontset = tmpfont.FontSet(ex) if fonts == "game" else None - texts: dict[str, dict] = {} - chars_by_font: dict[str, set] = {} - for n in nodes: - if not _drawn(n["path"]): - continue - t = _comp(n, TMP_CLASS) - if t is None: - continue - if len(by_path[n["path"]]) != 1: - raise RuntimeError(f"{n['path']}: text node path not unique") - lt = _comp(n, "LocalizeText") - if not (lt and lt["m_Enabled"] and lt["_localizeEnabled"]): - raise NotImplementedError(f"{n['path']}: text without an enabled LocalizeText") - loc = textstyle.localize_text( - n["path"], t["m_fontAsset"]["name"], t["m_sharedMaterial"]["material"], swap, lang, mode - ) - shown = text_of[n["path"]] if n["path"] in text_of else t["m_text"] - texts[n["path"]] = {"localized": loc, "text": shown, "setByInitialize": n["path"] in text_of} - if styles: - record = styles.record(n["path"], t, lt) - record.pop("text") # the serialized text; `text` above is the one shown - texts[n["path"]]["style"] = record - chars_by_font.setdefault(loc["fontAsset"], set()).update(ord(ch) for ch in shown) - for p in text_of: - if _drawn(p) and p not in texts: - raise RuntimeError(f"{p}: Initialize writes a text that is not exported") - - if fontset is not None: - coverage, materials, font_out = _game_fonts(ex, fontset, texts, chars_by_font, packer) - else: # name stubs: no font data, no atlases - coverage, materials = None, {} - font_out: dict = {f: {"name": f, "note": FONT_NOTE} for f in sorted(chars_by_font)} - - # -- sprites: drawn Images, the difficulty icons, the jacket ---------------------------------------- - replaced = {r["gameObject"] for f in ("_jacketImages", "_difficultyImages") for r in canvas[f]} - sprite_refs: dict[str, str] = {} # sprite name -> spriteRef - for n in nodes: - im = _comp(n, "Image") - if not _drawn(n["path"]) or im is None or n["path"] in replaced or not im["m_Sprite"]: - continue - sprite_refs.setdefault(im["m_Sprite"]["name"], im["m_Sprite"]["spriteRef"]) - difficulty_icons = {} - for p in canvas["_difficultyPairs"]: - s = p["Icon"] - difficulty_icons[str(p["Difficulty"])] = s["name"] - sprite_refs.setdefault(s["name"], s["spriteRef"]) - music = next(r for r in master_table(master, "MasterLiveMusic") if r["_id"] == music_id) - jacket = ex.key_sprite(JACKET_KEY.format(jacket=music["_jacketAssetName"])) - sprite_refs.setdefault(jacket["name"], jacket["spriteRef"]) - for name, ref in sorted(sprite_refs.items()): - s = ex.sprite_recs[ref] - if s["name"] != name: - raise RuntimeError(f"sprite {name}: record named {s['name']}") - ex.pack_sprite(ref, packer, f"sprites_{_safe(s['tex'].read_typetree()['m_Name'])}") - - textures = packer.build() - if fontset is not None: - tmpfont.resolve_font_blocks(font_out) - sprites_out = {name: ex.packed_sprite(ref) for name, ref in sorted(sprite_refs.items())} - for m in materials.values(): - for v in m["textures"].values(): - if v["texture"] is not None: - v["texture"] = { - "name": v["texture"]["name"], - "width": v["texture"]["width"], - "height": v["texture"]["height"], - "format": v["texture"]["format"], - } - - # -- shaders: livescene's dump (same name and source bundle), GLES3 variant per material ------------ - index = json.loads((out_dir / "livescene" / "shaders" / "shaders.json").read_text(encoding="utf-8")) - by_name = {r["name"]: r for r in index} - keywords = {} - for m in materials.values(): - sh = m["shader"]["shader"] - rec = by_name.get(sh) - if rec is None: - raise RuntimeError(f"{m['material']}: shader {sh} not in livescene/shaders") - if sh not in ex.shaders or ex.shaders[sh].assets_file.name != rec["source"]: - raise RuntimeError(f"{m['material']}: shader {sh} comes from another bundle than livescene's") - known = {k for v in rec["variants"] if v["platform"] == "gles3" and v["type"] == "GLES3" for k in v["keywords"]} - want = sorted(k for k in m["keywords"] if k in known) - if not any( - v["platform"] == "gles3" - and v["type"] == "GLES3" - and v["stage"] == "vertex" - and sorted(v["keywords"]) == want - for v in rec["variants"] - ): - raise RuntimeError(f"{sh}: no GLES3 variant for material {m['material']} keywords {want}") - keywords[m["material"]] = want - if "UI/Default" not in by_name: - raise RuntimeError("UI/Default not in livescene/shaders") - - # -- the root Animator's controller (evidence) ----------------------------------------------------- - an = _comp(by_path[f"{CANVAS}/root"][0], "Animator") - controller = an["m_Controller"] if an else None - tmp = player.resource("TMP Settings") - tmp_settings = {k: player.names_in(tmp, v) for k, v in player.mono(tmp).items()} - - doc = { - "about": f"Live start canvas (Live/UILiveStartCanvas, LightWeight mode): what livescene/scene.json lacks, " - f"{language}", - "canvas": CANVAS, - "slice": { - "musicId": music_id, - "difficulty": difficulty, - "liveDifficulty": LIVE_DIFFICULTY[difficulty], - "lightweight": True, - "profile": "fresh (no play records)", - }, - "language": { - "mode": mode, - "field": column[1:], - "fonts": lang, - "fontSwap": swap, - "lineSpacing": LANGUAGE_LINE_SPACING[mode], - }, - "strings": strings, - "stringSources": sources, - "texts": texts, - "sprites": sprites_out, - "difficultyIcons": difficulty_icons, - "jacket": jacket["name"], - "textures": textures, - "materials": materials, - "materialKeywords": keywords, - "fonts": font_out, - "glyphCoverage": coverage, - "tmpSettings": { - k: tmp_settings[k] - for k in ( - "m_GetFontFeaturesAtRuntime", - "m_matchMaterialPreset", - "m_enableExtraPadding", - "m_missingGlyphCharacter", - ) - }, - # images draw with Graphic.defaultGraphicMaterial (built-in UI/Default: livescene's dump of it has the - # same variants as the unity_builtin_extra one the ADV exports, checked byte for byte) - "shaders": { - "index": "../livescene/shaders/shaders.json", - "names": sorted({m["shader"]["shader"] for m in materials.values()} | {"UI/Default"}), - }, - "controller": controller, - } - if styles: - doc["fontData"] = "open" - doc["textStyle"] = styles.summary() - write_json(base / "liveui.json", doc) - return { - "liveui": str(base / "liveui.json"), - "texts": len(texts), - "sprites": sorted(sprites_out), - "textures": {k: (v["width"], v["height"]) for k, v in textures.items()}, - "fonts": sorted(font_out), - "coverage": {k: v["counts"] for k, v in (coverage or {}).items()}, - "strings": strings, - "runtimeGlyphs": { - k: len(v["runtimeGlyphs"]["generated"]) for k, v in font_out.items() if v.get("runtimeGlyphs") - }, - } diff --git a/src/nnnotes/content/story.py b/src/nnnotes/content/story.py deleted file mode 100644 index ffb6934..0000000 --- a/src/nnnotes/content/story.py +++ /dev/null @@ -1,164 +0,0 @@ -"""One ADV episode -> a self-contained story directory that a player can load without game code. - - /episode.json command list + text/sound/video shards + resource closure (adv.py) - /audio// every cue sheet in the episode's -SoundCueSheet - shard, decoded per cue + cues.json (cri.py) - /scene.json, textures/, shaders/ - player graphics, cameras, ADV fields, volumes, - settings and stages (advscene.py); textures/ and - shaders/ also hold those of the kind files below - /frames.json, effects.json, posteffects.json, stills.json, talkwindows.json, chat.json - the episode's frames, particle effects, post-effect - profiles, stills, talk windows and chat assets - (advmedia.py; each written only when used) - /videos/ Movie / Clip videos as WebM + videos.json (advvideo.py) - /ui/ ADV front canvas UI: ui.json, packed textures, - UI shaders; also the rule transitions (advui.py) - /crilips/ crilips.json + crilips.bin: the CRI Lips analysis data read from the - APK (crilips.py), when a voice reaches the CRI Lips path (needs_cri_lips) - /story.json index of the above (a kind file the episode does not use is null) - -The Live2D models the episode uses come from a model source (webmodel: SiteModels), `ensure(address) -> -{"id", "motionSync"}`: story.json `models` maps each model's address to its id, plus the fields of the source's -`story_fields` (SiteModels adds none). Every resource kind of the closure has an exporter (KIND_OUTPUT); the build -stops before writing anything when a resource is missing from the catalog or of a kind without one. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -from ..core.jsonio import write_json -from ..formats import cri, crilips -from ..source.catalog import Catalog -from ..unity.player import PlayerData -from . import adv, advmedia, advscene, advui, advvideo - -# resource kind -> where the story directory holds it -KIND_OUTPUT = { - "live2d": "story.json models (the model source)", - "stage": "scene.json stages", - "transition": "ui/ui.json transitions", - "video": advvideo.INDEX, - **{k: advmedia.FILES[k][1] for k in advmedia.FILES}, - **{k: advmedia.CHAT[1] for k in advmedia.CHAT_KINDS}, -} - - -CRILIPS_DIR = "crilips" -AIR_LIP_SYNC = ("airlipsync", "airlipsync_holdopen") # Talk Parameter1 modes without a voice analysis - - -def needs_cri_lips(episode: dict, split_key: str, motion_sync: list[bool]) -> bool: - """A voice of the episode reaches the CRI Lips analysis: a lip-synced voice row (Talk or Voice with TargetName and - VoiceIDs; IgnoreLipSync and IgnoreData not set) when a model of the episode has no MotionSync controller - (`motion_sync`: per model, model.json `motionSync`), or a Talk row whose voice also drives other speakers, which - use the analysis whatever their model (StartSharedAnalyzerLipSync: the EveryoneLipSync mode, or more speakers than - voices).""" - for c in episode["commands"]: - if c["cmd"] not in ("Talk", "Voice") or c.get("IgnoreData") or c.get("IgnoreLipSync"): - continue - if not c.get("TargetName") or not c.get("VoiceIDs"): - continue - if not all(motion_sync): - return True - if c["cmd"] == "Talk": - mode = (c.get("Parameter1") or "").strip().lower() - if mode == "everyonelipsync": - return True - if mode not in AIR_LIP_SYNC and len(c["TargetName"].split(split_key)) > len(c["VoiceIDs"]): - return True - return False - - -def check_kinds(resources: list[dict]) -> None: - """Raise for the first resource missing from the catalog or of a kind the story export does not handle.""" - missing = [r["address"] for r in resources if not r["present"]] - if missing: - raise RuntimeError(f"resources not in catalog: {missing}") - for r in resources: - if r["kind"] not in KIND_OUTPUT: - raise NotImplementedError(f"resource kind {r['kind']}: {r['address']}") - - -def build( - cat: Catalog, - master: Path, - player: PlayerData, - adv_id: int, - out_dir: Path, - audio_format: str = "flac", - audio: bool = True, - fonts: str = "open", - *, - models, - ui: bool = True, - language: str | None = None, - audio_options: dict | None = None, -) -> dict: - """Build the story directory with the models of the model source `models` (module docstring). `audio=False` - leaves the cue sheets undecoded (story.json `audio` is empty); `fonts`: "open" exports no font data of the game, - "game" exports it (advmedia, advui). `ui=False` writes no ui/ (story.json still names ui/ui.json: the caller - writes it, e.g. per language as storysite does); `language`: the UI's client language (a languages.LANGUAGES - code, needed with the UI); `audio_options`: further cri.decode options of every cue sheet - (flac_level, also). With audio, the CRI Lips data is read from the catalog's APK when a voice reaches it - (needs_cri_lips; story.json `crilips`, else null).""" - if ui and language is None: - raise ValueError("language: the client language of the UI is needed with ui=True") - out_dir = Path(out_dir) - out_dir.mkdir(parents=True, exist_ok=True) - episode = adv.to_json(adv.extract(cat, master, adv_id)) - check_kinds(episode["resources"]) - write_json(out_dir / "episode.json", episode) - - used = {r["address"]: models.ensure(r["address"]) for r in episode["resources"] if r["kind"] == "live2d"} - - sheets = {} - if audio: - for sheet in sorted({c["_cueSheetName"] for c in episode["cuesheets"].values()}): - cri.decode( - cat, sheet, out_dir / "audio" / sheet, fmt=audio_format, categories=True, **(audio_options or {}) - ) - sheets[sheet] = f"audio/{sheet}" - - media = advmedia.extract(cat, player, master, episode, out_dir, fonts=fonts) - scene = advscene.extract(cat, player, episode, out_dir, shaders=media["shaders"]) - ui_summary = ( - advui.extract(cat, player, episode, out_dir, fonts=fonts, language=language, master=master) - if ui and language is not None - else None - ) - videos = advvideo.extract(cat, episode, out_dir) - lips = None - if audio: - split = json.loads((out_dir / "scene.json").read_text(encoding="utf-8"))["settings"]["playerSettings"][ - "_targetNameSplitKey" - ] - if needs_cri_lips(episode, split, [m["motionSync"] for m in used.values()]): - if cat.apk is None: - raise crilips.CriLipsError("the CRI Lips data is read from the APK: set [paths] apk") - crilips.write_data(cat.apk, out_dir / CRILIPS_DIR) - lips = {"descriptor": f"{CRILIPS_DIR}/crilips.json", "data": f"{CRILIPS_DIR}/crilips.bin"} - index = { - "advId": adv_id, - "episode": "episode.json", - "scene": "scene.json", - "ui": "ui/ui.json", - "models": {a: m["id"] for a, m in used.items()}, - **models.story_fields(out_dir), - "audio": sheets, - **media["files"], - "videos": advvideo.INDEX if videos else None, - "crilips": lips, - } - write_json(out_dir / "story.json", index) - return { - **index, - "stages": scene["stages"], - "shaders": scene["count"], - "textures": scene["textures"], - "uiSummary": ui_summary, - "media": {"entries": media["counts"], "textures": media["textures"]}, - "videoCount": len(videos["videos"]) if videos else 0, - } diff --git a/src/nnnotes/content/storyfontproduct.py b/src/nnnotes/content/storyfontproduct.py index dace25a..3ec9ea9 100644 --- a/src/nnnotes/content/storyfontproduct.py +++ b/src/nnnotes/content/storyfontproduct.py @@ -6,8 +6,8 @@ from ..source import languages from ..unity import fontsettings, textstyle, tmpfont -from ..unity.export import MemoryPacker, _safe, packed_png from ..unity.ids import object_id, parse_object_id +from ..unity.texels import MemoryPacker, _safe, packed_png from . import advui, storyfonts, storysimpleui, storyui, tmpassets from .prefabs import component_class, key_object, mono_json, ref_id, shader_table diff --git a/src/nnnotes/content/storyfonts.py b/src/nnnotes/content/storyfonts.py index f60380f..3637038 100644 --- a/src/nnnotes/content/storyfonts.py +++ b/src/nnnotes/content/storyfonts.py @@ -1,48 +1,4 @@ -"""The TextMeshPro font data of a story's texts in one language -> /ui/fonts.json, ui/fonts/*.png and -ui/languages.json (the site's per-language files, storysite.py; format in ournotes-player docs/story-data-format.md). - - ui/fonts.json font assets reduced to the characters the episode shows in the language, their glyph pages - (ui/fonts/*.png), the text materials, their GLES3 keyword sets and the text binding of every - text node of ui/ui.json: its serialized TextMeshPro fields and the font asset, material and - line spacing it uses in the language (LocalizeText); with open fonts also `chatTexts` and - `dialogTexts`, the bindings of the chat window texts of ui.json `chatTexts` and of the text - nodes of its `dialogs` - ui/languages.json the language's LanguageMode, text field, line spacing and the font asset and line metrics of - each font role - ui/shaders/ the shader the text materials name (TextMeshPro/Mobile/Distance Field) is added to the UI's - -Two sources give the same format: - - open_fonts (default) glyphs generated from a font file per language: each text's localized game font asset - (LocalizeText) is replaced by an asset of the font file with the game asset's point size, padding, - style settings and render mode, holding the characters of the episode's text table, its title, the - static labels of the UI and the chat windows' status and run-time texts in that language plus - TextMeshPro's synthesized control characters and U+005F (UNDERLINE_CHARACTER). Face info and glyph - metrics come from the font file at that point size (FreeType, no hinting); the distance field is - tmpfont.RuntimeGlyphs' (the supersampled render modes rasterize at up to MAX_OVERSAMPLE times the size, - recorded in atlasRenderMode; the anti-aliased distance-field modes are generated as the 1X distance - field and recorded as SDF). Glyphs are shelf-packed, U+005F first (on the first page), then in code - point order of their first character into pages of PAGE_WIDTH texels, at most PAGE_MAX_HEIGHT high; - every page of an asset has the same size. The text materials are the game's localized materials with - the page as _MainTex and the page size as _TextureWidth / _TextureHeight. The game asset's fallback - chain is mirrored (open_font_set): each game fallback asset stands for an asset of the font file of - the language LocalizeManager lists it for, which holds the characters the assets before it in the - chain lack. The game assets' settings, character coverage, fallback lists and text materials come from - nnnotes' table of them (fontsettings.builtin()): no game font asset, material, glyph or font bundle is - read. - game_fonts the game's TMP font assets (advui.extract with fonts="game", tmpfont.export_fonts): the same records - moved from its ui.json into fonts.json, the glyph pages into ui/fonts/. - -Emoji: every text a Fwk.UI.UIText drives has LocalizeManager's emoji sprite asset (UIText.Awake), which draws the -characters its font assets lack and the tags the emoji search writes for emoji sequences. With open -fonts the sprite asset holds the sprites the episode's texts can draw (sprite_needs), each image drawn from the emoji -font (EmojiFont; none: empty glyphs) at the game sprite's size, and the open font assets of those texts leave out the -code points a sprite draws; with game fonts it is the game's asset with its sheet. Without a sprite to draw, fonts.json -has no sprite asset. - -Same inputs (font file, episode, game data) and library versions give the same bytes; generated glyph cells are kept -in the process cache by font file, sizes and glyph. -""" +"""Font file readers, glyph generation, atlas packing and material records for text products.""" from __future__ import annotations @@ -50,30 +6,22 @@ import hashlib import json import math -import shutil import unicodedata from pathlib import Path import numpy as np -from ..core import cache, jsonio -from ..formats import shader as shader_mod -from ..source import languages, master +from ..core import cache +from ..source import languages from ..source.config import ConfigError from ..unity import fontsettings, textstyle, tmpfont -from ..unity.export import Exporter, _safe, packed_png, texture_png +from ..unity.texels import _safe, packed_png from . import adv, advui -FONTS_FORMAT = "ournotes.story-fonts/1" -LANGUAGE_FORMAT = "ournotes.story-language/1" -FONTS_DOC = "fonts.json" -LANGUAGE_DOC = "languages.json" PAGES_DIR = "fonts" PAGE_WIDTH = 2048 PAGE_MAX_HEIGHT = 4096 MAX_OVERSAMPLE = 8 -SOURCES = ("open", "game") -TEXT_SHADER_PREFIX = "TextMeshPro/" # UnityEngine.TextCore.LowLevel.GlyphRasterModes flags of the supersampled render modes -> raster pixels per texel OVERSAMPLE_FLAGS = {tmpfont.RASTER_MODE["8X"]: 8, tmpfont.RASTER_MODE["16X"]: 16, tmpfont.RASTER_MODE["32X"]: 32} ELEMENT_CHARACTER = 1 # TMP TextElementType.Character @@ -86,39 +34,12 @@ KOREAN_ADJUST_CLASS = "LocalizeKoreanAdjust" # the font style a text gets in Korean (LocalizeKoreanAdjust.Apply) UI_TEXT_CLASSES = ("UIText", "UIRubyText") # Fwk.UI.UIText and its subclass GLYPHS = cache.bucket("storyglyphs") -FACE_KEYS = ( - "m_FaceIndex", - "m_FamilyName", - "m_StyleName", - "m_PointSize", - "m_Scale", - "m_UnitsPerEM", - "m_LineHeight", - "m_AscentLine", - "m_CapLine", - "m_MeanLine", - "m_Baseline", - "m_DescentLine", - "m_SuperscriptOffset", - "m_SuperscriptSize", - "m_SubscriptOffset", - "m_SubscriptSize", - "m_UnderlineOffset", - "m_UnderlineThickness", - "m_StrikethroughOffset", - "m_StrikethroughThickness", - "m_TabWidth", -) STYLE_KEYS = fontsettings.STYLE_KEYS # OpenType name table ids NAME_FAMILY, NAME_STYLE, NAME_VERSION, NAME_LICENSE, NAME_LICENSE_URL = 1, 2, 5, 13, 14 NAME_TYPO_FAMILY, NAME_TYPO_STYLE = 16, 17 -def _dump(obj) -> bytes: - return jsonio.dumps(obj, ensure_ascii=False, indent=1, sort_keys=True).encode("utf-8") + b"\n" - - def _salt() -> str: return cache.key(cache.source_salt(__file__), cache.source_salt(tmpfont.__file__, "freetype-py", "scipy", "numpy")) @@ -279,69 +200,6 @@ def pack_cells(cells: list[tuple[int, int, int]]) -> tuple[dict[int, tuple[int, # ---------------------------------------------------------------- texts -def _ui_prefab_nodes(ex: Exporter, ui_doc: dict) -> tuple[dict[str, dict], dict[str, dict]]: - """({path: node} of UIAdvWidget's prefab, {path: node} of the talk windows of ui.json under TalkView).""" - widget = {n["path"]: n for n in ex.prefab(advui.WIDGET_KEY)["nodes"]} - names = [rec["name"] for rec in ui_doc["nodes"] if rec["path"] == f"{advui.WINDOW_PARENT}/{rec['name']}"] - return widget, {n["path"]: n for n in advui.window_nodes(ex, names)} - - -def ui_text_targets(nodes, root: str = "") -> set[str]: - """The paths (`root` + the prefab path) of the texts the Fwk.UI.UIText components (and its subclass UIRubyText) of - the prefab `nodes` drive (_targetText, on any node): UIText.Awake registers OnEmojiSpriteAssetLoaded with - LocalizeManager, which sets the text's sprite asset (TMP_Text.spriteAsset) to LocalizeManager's emoji sprite - asset, over any serialized one (the asset is loaded at the title, before any story). A UIText without a target - text drives none.""" - out = set() - for n in nodes: - for c in n["components"]: - if c.get("class") in UI_TEXT_CLASSES: - target = (c.get("_targetText") or {}).get("gameObject") - if target is not None: - out.add(root + target) - return out - - -def story_ui_text_targets(ex: Exporter, ui_doc: dict) -> set[str]: - """ui_text_targets of UIAdvWidget and of the talk windows of ui.json (paths under TalkView).""" - names = [rec["name"] for rec in ui_doc["nodes"] if rec["path"] == f"{advui.WINDOW_PARENT}/{rec['name']}"] - out = ui_text_targets(ex.prefab(advui.WIDGET_KEY)["nodes"]) - for name in names: - out |= ui_text_targets(ex.prefab(advui.window_key(ex.cat, name))["nodes"], f"{advui.WINDOW_PARENT}/") - return out - - -def sprite_binding(path: str, node: list[dict], driven: bool, sprite: str | None) -> dict: - """The sprite settings of a text binding: for a text a UIText drives (`driven`), `spriteAsset` = `sprite` (the - emoji sprite asset's name in fonts.json) and the text's serialized `m_tintAllSprites`; {} for any other text, which - keeps its serialized sprite asset (only none is implemented).""" - comps = [c for c in node if c.get("class") in advui.TMP_CLASSES] - if len(comps) != 1: - raise RuntimeError(f"{path}: {len(comps)} text components") - c = comps[0] - if not driven: - if c.get("m_spriteAsset"): - raise NotImplementedError(f"{path}: a text with its own sprite asset") - return {} - if sprite is None: - raise RuntimeError(f"{path}: a UIText text without the emoji sprite asset") - return {"spriteAsset": sprite, "m_tintAllSprites": c["m_tintAllSprites"]} - - -def text_components(ex: Exporter, ui_doc: dict) -> dict[str, list[dict]]: - """{node path: the node's components} of every text node of the story UI (ui.json nodes with a text record), from - the prefabs advui assembles (UIAdvWidget with the talk windows of ui.json, the children of TalkView, under it).""" - widget, window = _ui_prefab_nodes(ex, ui_doc) - out = {} - for rec in ui_doc["nodes"]: - if "textStyle" not in rec: - continue - path = rec["path"] - hits = [n for n in (widget.get(path), window.get(path)) if n is not None] - if len(hits) != 1: - raise RuntimeError(f"{path}: {len(hits)} prefab nodes") - out[path] = hits[0]["components"] - return out def text_extras(path: str, comps: list[dict]) -> dict: @@ -362,145 +220,11 @@ def text_extras(path: str, comps: list[dict]) -> dict: return out -def _binding( - path: str, node: list[dict], swap: dict, lang: dict, mode: int, driven: bool = False, sprite: str | None = None -) -> dict: - """The text binding of a text node (its components `node`): the serialized TextMeshPro fields of its text - component, as advui writes them with fonts="game", the settings of its other text components (text_extras), - `localized` = textstyle.localize_text (LocalizeText) in the language `mode` and its sprite settings - (sprite_binding: `driven` by a UIText, `sprite` the emoji sprite asset's name).""" - comps = [c for c in node if c.get("class") in advui.TMP_CLASSES] - if len(comps) != 1: - raise RuntimeError(f"{path}: {len(comps)} text components") - c = comps[0] - t = {k: c[k] for k in advui.TMP_FIELDS if k in c} - t.update( - { - "class": c["class"], - "enabled": c["m_Enabled"], - "fontAsset": c["m_fontAsset"]["name"], - "material": c["m_sharedMaterial"]["material"], - } - ) - t.update(text_extras(path, node)) - t["localized"] = textstyle.localize_text(path, t["fontAsset"], t["material"], swap, lang, mode) - t.update(sprite_binding(path, node, driven, sprite)) - return t - - -def text_bindings(ex: Exporter, ui_doc: dict, lang: dict, mode: int, sprite: str | None = None) -> dict[str, dict]: - """{node path: text binding (_binding)} of every text node of the story UI (text_components); `sprite` = the - emoji sprite asset's name for the texts a UIText drives (story_ui_text_targets).""" - swap = textstyle.font_swap(lang, mode) - driven = story_ui_text_targets(ex, ui_doc) - return { - path: _binding(path, node, swap, lang, mode, path in driven, sprite) - for path, node in text_components(ex, ui_doc).items() - } - - -def dialog_bindings( - ex: Exporter, ui_doc: dict, lang: dict, mode: int, sprite: str | None = None -) -> dict[str, dict[str, dict]]: - """{dialog: {node path: text binding (_binding)}} of the text nodes of the dialogs of ui.json (`dialogs`, each - from its prefab `key`); `sprite` as text_bindings.""" - swap = textstyle.font_swap(lang, mode) - out: dict[str, dict[str, dict]] = {} - for name, d in sorted((ui_doc.get("dialogs") or {}).items()): - nodes = ex.prefab(d["key"])["nodes"] - prefab, driven = {n["path"]: n["components"] for n in nodes}, ui_text_targets(nodes) - for rec in d["nodes"]: - if "textStyle" in rec: - out.setdefault(name, {})[rec["path"]] = _binding( - rec["path"], prefab[rec["path"]], swap, lang, mode, rec["path"] in driven, sprite - ) - return out - - -def chat_bindings( - ex: Exporter, episode: dict, lang: dict, mode: int, sprite: str | None = None -) -> dict[str, dict[str, dict]]: - """{chat window: {node path: text binding}} of the TMP texts of the episode's chat windows (advui.chat_text_nodes): - the fields text_bindings gives, `localized` from advui.chat_localize for a text with LocalizeText enabled, else - the serialized font asset, material and line spacing; `sprite` as text_bindings.""" - out: dict[str, dict[str, dict]] = {} - nodes, _ = advui.chat_text_nodes(ex, episode) - driven = set().union( - *(ui_text_targets(ex.prefab(address)["nodes"]) for address in advui.chat_windows(episode).values()) - ) - for window, node, c, loc in nodes: - t = {k: c[k] for k in advui.TMP_FIELDS if k in c} - t.update( - { - "class": c["class"], - "enabled": c["m_Enabled"], - "fontAsset": c["m_fontAsset"]["name"], - "material": c["m_sharedMaterial"]["material"], - } - ) - t.update(text_extras(node["path"], node["components"])) - if loc and loc["m_Enabled"] and loc["_localizeEnabled"]: - t["localized"] = advui.chat_localize(t["fontAsset"], t["material"], lang, mode) - else: - t["localized"] = {"fontAsset": t["fontAsset"], "material": t["material"], "lineSpacing": c["m_lineSpacing"]} - t.update(sprite_binding(node["path"], node["components"], node["path"] in driven, sprite)) - out.setdefault(window, {})[node["path"]] = t - return out - - # the IAdvFrameTextReceiver implementations (AdvSystem.Asset.UI): a frame with one gets its texts at run time FRAME_TEXT_RECEIVERS = ("AdvSlanderCommentFrame",) FRAME_PREFIX = adv.RESOURCE_PREFIX["Frame"][1] -def frame_prefabs(ex: Exporter, episode: dict) -> dict[str, list[dict]]: - """{frame name (its address without the Frame prefix, the key of story/frames.json): its prefab node list} of the - episode's Frame resources in the catalog.""" - return { - r["address"][len(FRAME_PREFIX) :]: ex.prefab(r["address"])["nodes"] - for r in episode["resources"] - if r["kind"] == "frame" and r.get("present", True) - } - - -def frame_bindings( - frames: dict[str, list[dict]], lang: dict, mode: int, sprite: str | None = None -) -> dict[str, dict[str, dict]]: - """{frame name: {node path: text binding}} of the TMP texts of the episode's frames (`frames`: frame_prefabs), as - chat_bindings: `localized` from LocalizeText (advui.chat_localize) when it is enabled, else the serialized font - asset, material and line spacing; `sprite` as text_bindings (the texts a UIText drives).""" - out: dict[str, dict[str, dict]] = {} - for name, nodes in frames.items(): - driven = ui_text_targets(nodes) - for node in nodes: - comps = [c for c in node["components"] if c.get("class") in advui.TMP_CLASSES] - if not comps: - continue - c = comps[0] - loc = next((x for x in node["components"] if x.get("class") == "LocalizeText"), None) - t = {k: c[k] for k in advui.TMP_FIELDS if k in c} - t.update( - { - "class": c["class"], - "enabled": c["m_Enabled"], - "fontAsset": c["m_fontAsset"]["name"], - "material": c["m_sharedMaterial"]["material"], - } - ) - t.update(text_extras(node["path"], node["components"])) - if loc and loc["m_Enabled"] and loc["_localizeEnabled"]: - t["localized"] = advui.chat_localize(t["fontAsset"], t["material"], lang, mode) - else: - t["localized"] = { - "fontAsset": t["fontAsset"], - "material": t["material"], - "lineSpacing": c["m_lineSpacing"], - } - t.update(sprite_binding(node["path"], node["components"], node["path"] in driven, sprite)) - out.setdefault(name, {})[node["path"]] = t - return out - - def frame_texts(frames: dict[str, list[dict]], episode: dict, field: str) -> list[str]: """The texts the episode's frames (`frames`: frame_prefabs) show: the serialized texts of their text nodes, and the texts AdvFrameCommand.SetFrameTexts gives the frames with an IAdvFrameTextReceiver (FRAME_TEXT_RECEIVERS): per @@ -532,35 +256,6 @@ def frame_texts(frames: dict[str, list[dict]], episode: dict, field: str) -> lis return serialized + (out + ["@\n"] if out else out) -def shown_texts(episode: dict, ui_doc: dict, master_dir: Path | None, field: str) -> list[str]: - """The texts the episode shows in the text field `field`: every text of its text table (lines, speaker names, - frame and choice texts), its title, the static labels of the UI (the MasterText rows of the text keys of the text - nodes and of the dialogs' text nodes, the `masterIdTexts` of ui.json) and of the chat windows - (`chatStatusTexts`), the serialized texts of the chat windows' text nodes (`chatTexts`, shown until the player - sets them) and the texts the chat windows format at run time (advui.chat_runtime_texts), markup included.""" - texts = [row.get(field) for row in episode["text"].values()] - texts.append((episode.get("title") or {}).get(field)) - texts += [t["text"] for t in (ui_doc.get("masterIdTexts") or {}).values()] - texts += list((ui_doc.get("chatStatusTexts") or {}).values()) + advui.chat_runtime_texts(episode) - texts += [rec["textStyle"].get("text") for w in (ui_doc.get("chatTexts") or {}).values() for rec in w.values()] - nodes = list(ui_doc["nodes"]) + [n for d in (ui_doc.get("dialogs") or {}).values() for n in d["nodes"]] - keys = {rec["textStyle"]["textKey"] for rec in nodes if "textStyle" in rec and rec["textStyle"].get("textKey")} - if keys: - if master_dir is None: - raise RuntimeError(f"UI text keys {sorted(keys)} need the master data") - rows = {r["_id"]: r for r in master.table(master_dir, "MasterText") if r["_id"] in keys} - missing = sorted(keys - set(rows)) - if missing: - raise KeyError(f"UI text keys not in MasterText: {missing}") - texts += [rows[k].get(f"_{field}") for k in sorted(keys)] - return [s for s in texts if isinstance(s, str)] - - -def shown_characters(episode: dict, ui_doc: dict, master_dir: Path | None, field: str) -> list[int]: - """The code points of shown_texts, sorted.""" - return sorted({ord(ch) for s in shown_texts(episode, ui_doc, master_dir, field) for ch in s}) - - def _material_textures(m: dict) -> None: """Texture references of a material as advui writes them: {name, width, height, format}.""" for v in m["textures"].values(): @@ -569,76 +264,6 @@ def _material_textures(m: dict) -> None: v["texture"] = {"name": t["name"], "width": t["width"], "height": t["height"], "format": t.get("format")} -def text_shaders(ex: Exporter, ui_dir: Path, materials: dict) -> dict[str, list[str]]: - """Add the shaders the text `materials` name to ui/shaders (index kept in name order) and return each - material's GLES3 keyword set (its keywords that the shader's GLES3 variants use; a vertex program with exactly - those must exist).""" - sdir = Path(ui_dir) / "shaders" - index = json.loads((sdir / "shaders.json").read_text(encoding="utf-8")) - have = {r["name"] for r in index} - for name in sorted({m["shader"]["shader"] for m in materials.values()}): - if name in have: - continue - if name not in ex.shaders: - raise RuntimeError(f"shader {name} not in the loaded bundles") - shader_mod.dump_objects([ex.shaders[name]], sdir, ex.shaders[name].assets_file.name, index) - index.sort(key=lambda r: r["name"]) - shader_mod.write_index(index, sdir) - recs = {r["name"]: r for r in index} - out = {} - for name, m in sorted(materials.items()): - rec = recs[m["shader"]["shader"]] - gles = [v for v in rec["variants"] if v["platform"] == "gles3" and v["type"] == "GLES3"] - known = {k for v in gles for k in v["keywords"]} - want = sorted(k for k in m["keywords"] if k in known) - if not any(sorted(v["keywords"]) == want and v["stage"] == "vertex" for v in gles): - raise RuntimeError(f"{rec['name']}: no GLES3 variant for material {name} keywords {want}") - out[name] = want - return out - - -def language_doc(language: str, source: str, fonts: dict, texts: dict, lang: dict) -> dict: - """ui/languages.json of `language`: LanguageMode, text field, line spacing and per font role the font asset - the role's texts use with its line metrics (textstyle.face_metrics).""" - mode = languages.mode(language) - roles: dict[str, dict] = {} - for _path, t in sorted(texts.items()): - role = textstyle.font_role(t["fontAsset"], lang) - fa = t["localized"]["fontAsset"] - if roles.get(role, {}).get("fontAsset", fa) != fa: - raise NotImplementedError(f"font role {role} uses two font assets ({roles[role]['fontAsset']}, {fa})") - f = fonts[fa] - roles[role] = { - "fontAsset": fa, - **textstyle.face_metrics(f["faceInfo"], {k: f[k] for k in ("normalSpacingOffset", "boldSpacing")}), - } - return { - "format": LANGUAGE_FORMAT, - "language": language, - "mode": mode, - "field": languages.column(language)[1:], - "lineSpacing": textstyle.LANGUAGE_LINE_SPACING[mode], - "fonts": source, - "roles": roles, - } - - -def _write(ui_dir: Path, fonts_doc: dict, lang_doc: dict) -> None: - (Path(ui_dir) / FONTS_DOC).write_bytes(_dump(fonts_doc)) - (Path(ui_dir) / LANGUAGE_DOC).write_bytes(_dump(lang_doc)) - - -def _exporter(cat, player, work: Path, font_stubs=None) -> Exporter: - return Exporter( - cat, - Path(work), - player=player, - textures="deferred", - stub_assets=("TMP_FontAsset", "TMP_SpriteAsset", "TMP_StyleSheet"), - font_stubs=font_stubs, - ) - - # ---------------------------------------------------------------- open fonts PAGE_SETTINGS = {"m_FilterMode": 1, "m_Aniso": 1, "m_MipBias": 0.0, "m_WrapU": 1, "m_WrapV": 1, "m_WrapW": 1} SUBSET = ( @@ -784,23 +409,6 @@ def open_asset( } -def line_breaking(player) -> dict: - """TMP_Settings' line breaking rules: the text of its leading and following characters TextAssets (as stored) - and m_UseModernHangulLineBreakingRules.""" - o = player.resource("TMP Settings") - tt = player.mono(o) - - def text(key: str) -> str: - t = player.deref(o, tt[key]) - return t.read_typetree()["m_Script"] if t is not None else "" - - return { - "leading": text("m_leadingCharacters"), - "following": text("m_followingCharacters"), - "useModernHangulLineBreakingRules": bool(tt["m_UseModernHangulLineBreakingRules"]), - } - - def search_order(fallbacks_of, font: str, exported: set) -> list[str]: """The font assets TMP searches after `font` for a character `font` lacks, as tmpfont.FontSet.lookup (TMP_FontAssetUtilities.GetCharacterFromFontAsset_Internal: each fallback, then that fallback's own fallbacks, @@ -822,20 +430,6 @@ def walk(f: str) -> None: return out -def fallback_chains(ex: Exporter, primaries: set, exported: set) -> dict[str, list[str]]: - """{exported font asset: its search_order}, over the game's font assets reachable from the `primaries` (font - asset names): the fallback list of a font record once the assets that serve no shown character are left out.""" - fs = tmpfont.FontSet(ex) - for n in sorted(primaries): - fs.by_key(fontsettings.font_dir(n) + n) - missing = sorted(exported - set(fs.fonts)) - if missing: - raise RuntimeError(f"font assets {missing} are not reachable from {sorted(primaries)}") - return { - n: search_order(lambda f: [fb.name for fb in fs.fallbacks(fs.fonts[f])], n, exported) for n in sorted(exported) - } - - FALLBACK_FLOATS = ("_GradientScale", "_TextureWidth", "_TextureHeight", "_WeightNormal", "_WeightBold") SCALE_CLAMP = 1.0 # ShaderUtilities.m_clamp @@ -912,61 +506,6 @@ def fallback_material(m: dict, target: dict, name: str) -> dict: return out -def open_font_set( - cat, - player, - ex: Exporter, - language: str, - font: FontFile, - bindings: list[dict], - shown: list[str], - ui_dir: Path, - font_of=no_font_file, - sprite_drawn: dict | None = None, - margin_materials: dict | None = None, - settings=None, -) -> dict: - """The open font assets of the text `bindings` (their `localized` game font asset and material, replaced in - place by the open asset and its material) in `language`, for the texts `shown`, with the game font assets of - `settings` (fontsettings.FontSettings; default: the built-in table): - - - per localized game font asset (a primary) an asset of `font` (the language's font file) with the game asset's - point size, padding, style settings, render mode and text materials (open_asset); - - the game asset's fallback chain mirrored: each asset of its search order (search_order) stands for an asset of - the font file of its counterpart_language (`font_of(language)` -> FontFile, asked for when a character - reaches it) with that game asset's settings; the game's own fallback assets no language lists are left out; - - each shown code point goes to the first asset of its primary's chain whose font file maps it, as - TMP_Text.GetTextElement finds a character; one no asset maps stays with the primary as `missing`. The code - points the game's chain lacks take the missing glyph (tmpfont.missing_glyph: the primary gets `missingGlyph`; - its substitute goes through the chain like any character) and those a sprite draws for the primary's texts - (`sprite_drawn` {game font asset: code points}) are left out; - - an asset's `fallbacks` are the assets of its chain that hold a character (in search order), and each text - material M of a primary has, per fallback of the primary, the fallback material "M + " - (fallback_material). - - `margin_materials` {game font asset: game material names}: further text materials of that asset whose underlay - the glyph margin covers (so the pages equal another document's with the same characters). The pages go to - `ui_dir`/fonts/. -> {fonts, textures, materials, missing (code points no asset maps), substituted (code points - drawn as a missing glyph)}.""" - result = build_open_font_set( - language, - font, - bindings, - shown, - textstyle.language_fonts(player), - tmpfont.missing_glyph_character(player), - font_of, - sprite_drawn, - margin_materials, - settings, - ) - for path, body in result.pop("pages").items(): - dest = ui_dir / path - dest.parent.mkdir(parents=True, exist_ok=True) - dest.write_bytes(body) - return result - - def build_open_font_set( language: str, font: FontFile, @@ -1105,105 +644,6 @@ def name_of(key: tuple[str, str]) -> str: } -def open_fonts( - cat, - player, - episode: dict, - ui_dir: Path, - language: str, - font: FontFile, - master_dir: Path | None = None, - emoji: EmojiFont | None = None, - font_of=no_font_file, -) -> dict: - """ui/fonts.json, ui/fonts/*.png and ui/languages.json of `language` from the font file `font` and the emoji font - `emoji` (module docstring; without one the sprites keep their metrics and have no texels), next to the - ui/ui.json advui.extract wrote for that language in `ui_dir`; the text shaders go to ui/shaders. `font_of`: - the font file of another language for the fallback chains (open_font_set). Returns a summary.""" - tmpfont.require_extra() - ui_dir = Path(ui_dir) - ui_doc = json.loads((ui_dir / "ui.json").read_text(encoding="utf-8")) - mode, field = languages.mode(language), languages.column(language)[1:] - settings = fontsettings.builtin() - ex = _exporter(cat, player, ui_dir / "_fonts", settings) - lang = textstyle.language_fonts(player) - sprite_name = emoji.asset_name if emoji is not None else NO_EMOJI_FONT - texts = text_bindings(ex, ui_doc, lang, mode, sprite_name) - dialogs = dialog_bindings(ex, ui_doc, lang, mode, sprite_name) - chat = chat_bindings(ex, episode, lang, mode, sprite_name) if ui_doc.get("chatTexts") else {} - frames = frame_prefabs(ex, episode) - frame = frame_bindings(frames, lang, mode, sprite_name) - shown = shown_texts(episode, ui_doc, master_dir, field) + frame_texts(frames, episode, field) - chars = sorted({ord(ch) for s in shown for ch in s}) - - bindings = list(texts.values()) + [ - t for group in (dialogs, chat, frame) for w in group.values() for t in w.values() - ] - # the code points the sprite asset draws for the texts with it (sprite_drawn); the open font asset of those - # texts leaves them out - sprite_drawn: dict[str, set[int]] = {} - rec = None - if any("spriteAsset" in t for t in bindings): - rec = advui.sprite_asset_record(ex) - sprite_drawn = drawn_by_sprites(rec, [t for t in bindings if "spriteAsset" in t], chars, settings.lookup) - # the open font assets with the mirrored fallback chains, the missing glyph and the fallback materials - fset = open_font_set( - cat, player, ex, language, font, bindings, shown, ui_dir, font_of, sprite_drawn, settings=settings - ) - fonts, textures, materials = fset["fonts"], fset["textures"], fset["materials"] - missing, substituted = fset["missing"], fset["substituted"] - coverage = {"characters": len(chars), "missing": [chr(u) for u in sorted(missing)]} - if substituted: - coverage["missingGlyph"] = [chr(u) for u in sorted(substituted)] - sprites = None - indices = sprite_needs(rec, shown, set().union(*sprite_drawn.values())) if rec is not None else [] - if rec is None or not indices: # no sprite to draw: the data holds no sprite asset - for t in bindings: - t.pop("spriteAsset", None) - t.pop("m_tintAllSprites", None) - else: - sprites = open_sprite_asset(emoji, rec, indices, sprite_name) - for pname, px in sprites["pages"]: - (ui_dir / PAGES_DIR).mkdir(parents=True, exist_ok=True) - (ui_dir / sprites["textures"][pname]["texture"]).write_bytes(packed_png(px)) - textures.update(sprites["textures"]) - materials[sprites["material"]["material"]] = sprites["material"] - coverage["sprites"] = {"characters": len(sprites["asset"]["characters"]), "missing": sprites["missing"]} - keywords = text_shaders(ex, ui_dir, materials) - fonts_doc = { - "format": FONTS_FORMAT, - "language": language, - "source": "open", - "fonts": fonts, - "textures": textures, - "materials": materials, - "materialKeywords": keywords, - "texts": texts, - "coverage": coverage, - "lineBreaking": line_breaking(player), - } - if sprites is not None: - fonts_doc["spriteAssets"] = {sprite_name: sprites["asset"]} - fonts_doc["emojiSpriteAsset"] = sprite_name - if dialogs: - fonts_doc["dialogTexts"] = dialogs - if chat: - fonts_doc["chatTexts"] = chat - if frame: - fonts_doc["frameTexts"] = frame - _write(ui_dir, fonts_doc, language_doc(language, "open", fonts, texts, lang)) - shutil.rmtree(ui_dir / "_fonts", ignore_errors=True) - return { - "fonts": sorted(fonts), - "characters": len(chars), - "missing": len(missing), - "glyphs": sum(len(f["glyphs"]) for f in fonts.values()), - "pages": len(textures), - "sprites": len(sprites["asset"]["characters"]) if sprites else 0, - "missingSprites": len(sprites["missing"]) if sprites else 0, - } - - # ---------------------------------------------------------------- emoji sprites (open) EMOJI_VS16 = 0xFE0F SPRITE_MARGIN = 1 # transparent texels around each sprite cell (a bilinear quad edge samples half a texel) @@ -1433,155 +873,3 @@ def open_sprite_asset(emoji: EmojiFont | None, game: dict, indices: list[int], n # ---------------------------------------------------------------- game fonts -def game_sprite_asset(ex: Exporter, ui_dir: Path, rec: dict) -> dict: - """LocalizeManager's emoji sprite asset from the game data (`rec`, advui.sprite_asset_record): every character and - glyph, the sprite sheet as one page (ui/fonts/, the glyph rects unchanged) and its material sampling it. - -> {asset_name, asset, textures, material}.""" - sh = rec["sheet"] - if sh["mipCount"] != 1: - raise NotImplementedError(f"{rec['name']}: sprite sheet with {sh['mipCount']} mip levels") - o = ex.key_object(advui.EMOJI_SPRITE_ASSET_KEY) - sheet = ex.referent(o, o.read_typetree()["spriteSheet"]) - page = f"sprite_{rec['name']}" - dst = f"{PAGES_DIR}/{_safe(page)}.png" - (Path(ui_dir) / PAGES_DIR).mkdir(parents=True, exist_ok=True) - (Path(ui_dir) / dst).write_bytes(texture_png(sheet.read())) - glyphs = { - k: { - **g, - **( - {"packed": {"texture": page, "dx": 0, "dy": 0}} - if g["rect"]["m_Width"] > 0 and g["rect"]["m_Height"] > 0 - else {} - ), - } - for k, g in rec["glyphs"].items() - } - material = _sheet_material(rec["material"], rec["material"]["material"], page, sh["width"], sh["height"]) - asset = { - "faceInfo": rec["faceInfo"], - "characters": rec["characters"], - "glyphs": glyphs, - "sequences": rec["sequences"], - "material": material["material"], - } - textures = { - page: { - "texture": dst, - "name": sh["name"], - "width": sh["width"], - "height": sh["height"], - "mipCount": 1, - "settings": sh["settings"], - } - } - return {"asset_name": rec["name"], "asset": asset, "textures": textures, "material": material} - - -def game_fonts( - cat, - player, - game_ui_dir: Path, - ui_dir: Path, - language: str, - episode: dict | None = None, - master_dir: Path | None = None, -) -> dict: - """ui/fonts.json, ui/fonts/*.png and ui/languages.json of `language` from the ui/ directory advui.extract wrote - with fonts="game" for that language (`game_ui_dir`): its font records (each font's `fallbacks` reduced to the - exported assets in search order, fallback_chains), glyph pages, text materials, keyword sets, text records, glyph - coverage and TMP settings, next to the fonts-open ui/ui.json in `ui_dir`; the text shaders go to ui/shaders as - with open fonts. With the `episode` (and the master data for its UI labels), the texts a UIText drives get the - game's emoji sprite asset when they draw a sprite (sprite_needs; its code points leave coverage.missing). Returns a - summary.""" - game_ui_dir, ui_dir = Path(game_ui_dir), Path(ui_dir) - doc = json.loads((game_ui_dir / "ui.json").read_text(encoding="utf-8")) - ui_doc = json.loads((ui_dir / "ui.json").read_text(encoding="utf-8")) - texts = {n["path"]: n["text"] for n in doc["nodes"] if "text" in n} - open_paths = {n["path"] for n in ui_doc["nodes"] if "textStyle" in n} - if set(texts) != open_paths: - raise RuntimeError("the text nodes of the fonts-game and fonts-open UI differ") - fonts = doc["fonts"] - pages = sorted({g["packed"]["texture"] for f in fonts.values() for g in f["glyphs"].values() if "packed" in g}) - textures = {} - (ui_dir / PAGES_DIR).mkdir(parents=True, exist_ok=True) - for p in pages: - t = dict(doc["textures"][p]) - dst = f"{PAGES_DIR}/{Path(t['texture']).name}" - shutil.copyfile(game_ui_dir / t["texture"], ui_dir / dst) - t["texture"] = dst - textures[p] = t - names = {t["localized"]["material"] for t in texts.values()} | {f["material"] for f in fonts.values()} - materials = {n: doc["materials"][n] for n in sorted(names)} - ex = _exporter(cat, player, ui_dir / "_fonts") - chains = fallback_chains(ex, {t["localized"]["fontAsset"] for t in texts.values()}, set(fonts)) - fonts = {n: {**f, "fallbacks": chains[n]} for n, f in fonts.items()} - for t in texts.values(): # TMP's fallback materials of the text materials (fallback_material) - loc = t["localized"] - for fb in fonts[loc["fontAsset"]]["fallbacks"]: - name = f"{loc['material']} + {fb}" - if name not in materials: - materials[name] = fallback_material(materials[loc["material"]], materials[fonts[fb]["material"]], name) - driven = story_ui_text_targets(ex, ui_doc) & set(texts) - sprites, drawn = None, set() - if episode is not None and driven: - rec = advui.sprite_asset_record(ex) - shown = shown_texts(episode, ui_doc, master_dir, languages.column(language)[1:]) - chars = sorted({ord(ch) for s in shown for ch in s}) - fs = tmpfont.FontSet(ex) - - def lookup(name, u): - return fs.lookup(fs.by_key(fontsettings.font_key(ex.cat, name, name)), u) - - drawn = set().union(*drawn_by_sprites(rec, [texts[p] for p in sorted(driven)], chars, lookup).values()) - if sprite_needs(rec, shown, drawn): - sprites = game_sprite_asset(ex, ui_dir, rec) - if sprites is None: - driven, drawn = set(), set() - sprite_name = sprites["asset_name"] if sprites else None - for path, comps in text_components(ex, ui_doc).items(): - texts[path] = { - **texts[path], - **text_extras(path, comps), - **sprite_binding(path, comps, path in driven, sprite_name), - } - if sprites is not None: - textures.update(sprites["textures"]) - materials[sprites["material"]["material"]] = sprites["material"] - for n in sorted({t["localized"]["material"] for t in texts.values()}): # the text shaders for text_shaders - fa = next(t["localized"]["fontAsset"] for t in texts.values() if t["localized"]["material"] == n) - ex.material(ex.key_object(fontsettings.font_dir(fa) + n)) - keywords = text_shaders(ex, ui_dir, materials) - cov = doc["glyphCoverage"] - if sprites is not None: - cov = { - **cov, - "missing": [c for c in cov["missing"] if ord(c) not in drawn], - "sprites": {"characters": len(sprites["asset"]["characters"]), "missing": []}, - } - fonts_doc = { - "format": FONTS_FORMAT, - "language": language, - "source": "game", - "fonts": fonts, - "textures": textures, - "materials": materials, - "materialKeywords": keywords, - "texts": texts, - "coverage": cov, - "tmpSettings": doc["tmpSettings"], - "lineBreaking": line_breaking(player), - } - if sprites is not None: - fonts_doc["spriteAssets"] = {sprite_name: sprites["asset"]} - fonts_doc["emojiSpriteAsset"] = sprite_name - lang = textstyle.language_fonts(player) - _write(ui_dir, fonts_doc, language_doc(language, "game", fonts, texts, lang)) - shutil.rmtree(ui_dir / "_fonts", ignore_errors=True) - return { - "fonts": sorted(fonts), - "characters": cov["characters"], - "missing": len(cov["missing"]), - "glyphs": sum(len(f["glyphs"]) for f in fonts.values()), - "pages": len(textures), - } diff --git a/src/nnnotes/content/storyhost.py b/src/nnnotes/content/storyhost.py deleted file mode 100644 index 8183627..0000000 --- a/src/nnnotes/content/storyhost.py +++ /dev/null @@ -1,1506 +0,0 @@ -"""The host screen of an Overlay story (MasterAdv._playbackMode 1) -> /host/ and /ui/simple/. - -The game plays these episodes with App.Adv.SimpleAdvPlayer inside the screen that starts them, not in the ADV scene: -SimpleAdvView lays the characters out in the slots of the host's SimpleAdvLayoutRoot (each drawn by its own -CameraTargetRenderer) and attaches the UISimpleAdvTalkWindow under the root's TalkRoot. The host is the story group -of the episode: a tap talk (MasterStoryHomeSpotTapTalkEpisode) or an area talk (MasterHomeSpot._advId) of a home -spot (SpotManager), or the reward phase of a live result (LiveResultDisplayPresenter). - - host/host.json kind, the capture camera of a slot (CameraTargetRenderer prefab), the SimpleAdv root and its - SimpleAdvLayoutRoot fields, the sequences the host screen has played when the talk starts, - and per kind: - home spot.json + Spine files (hostspot.extract_spot) and room.glb + room.json - (hostspot.extract_room) under host/spot/; per placed glb node its object path, its - state after SpotBackground.Prepare and the SpotSceneRoot.SetObject "floor" render - queue; the room materials as saved; the "Spot" scene's object root, lights, - volumes and RenderSettings; the lights of the placed prefabs; the background - Volume; the Spine materials; the blur of renderer 0's UIRendererFeature; the - situation's ambient sound; shaders in host/shaders/ - afterlive the result background node and sprite, the reward panel, the substitutes - host/ui/ui.json the host UI in the advui record format (no text nodes) - ui/simple/ui.json per language: UISimpleAdvTalkWindow (and UISystemMessageWidget for an area talk) in the - advui record format with text records, fonts.json and glyph pages (storyfonts format, open - fonts) - -Same inputs give the same bytes. -""" - -from __future__ import annotations - -import json -import shutil -import struct -from pathlib import Path - -import numpy as np -import UnityPy - -from ..core.jsonio import dumps, write_json -from ..formats import shader as shader_mod -from ..source import languages, master -from ..unity import fontsettings, textstyle, tmpfont -from ..unity.bindings import RECT_PROPS -from ..unity.convert.animation import streamed_frames -from ..unity.export import Exporter, TexelPacker -from ..unity.reader import SceneGraph, deref, mesh_arrays, referent, script_class -from . import advui, hostspot, storyfonts -from .spot import atlas_pages - -FORMAT = "ournotes.story-host/1" -OVERLAY = 1 # MasterAdv._playbackMode of the episodes SimpleAdvPlayer plays -HOST_DIR = "host" -HOST_DOC = f"{HOST_DIR}/host.json" -HOST_UI_DOC = f"{HOST_DIR}/ui/ui.json" -SIMPLE_DIR = "simple" -SIMPLE_DOC = f"ui/{SIMPLE_DIR}/ui.json" -FONT_MODES = ("open", "game") -TMP_STUBS = ("TMP_FontAsset", "TMP_SpriteAsset", "TMP_StyleSheet") -# game addresses; a missing one is loaded from "Emb" + address (emb_key) -ADDRESSES = { - "cameraTarget": "Common/Prefab/CameraTargetRenderer", - "talkWindow": "UI/Prefab/Parts/Adv/Talk/UISimpleAdvTalkWindow", - "systemMessage": "UI/Prefab/UISystemMessageWidget", - "spotWidget": "UI/Prefab/UISpotWidget", - "spotLetterbox": "UI/Prefab/Parts/Spot/UISpotLetterbox", - "spotScene": "Scene/Spot", - "resultWidget": "UI/Prefab/UILiveResultWidget", - "resultBackground": "UI/Prefab/UILiveResultBackgroundWidget", - "resultBackgroundSprite": "LowImage/LiveResult/live_result_background", -} -KIND_KEYS = { - "home": ("spotWidget", "spotLetterbox", "spotScene"), - "afterlive": ("resultWidget", "resultBackground", "resultBackgroundSprite"), -} -# story group kind (storysite.story_groups) -> (host kind, talk) -HOST_GROUPS = {"spotTalk": ("home", "tap"), "spot": ("home", "area"), "liveResult": ("afterlive", None)} -SUBSTITUTES = ["musicResultPanel", "rewardItems", "memberCard", "navigation", "gradeUpEffect"] -CLONE = "(Clone)" # Object.Instantiate's name suffix, removed by SpotBackground.Prepare -FLOOR_NAME, FLOOR_EXCLUDE, FLOOR_QUEUE = "floor", "shadow", 2000 # SpotSceneRoot.SetObject -REWARD_PANEL = "RewardPanel" -COMPONENT_HEADER = ("type", "class", "m_Name") -NO_EPISODE = {"text": {}, "commands": []} # the episode argument of storyfonts.shown_characters for a UI alone -DOTWEEN_FIELDS = ( - "defaultEaseType", - "defaultUpdateType", - "defaultTimeScaleIndependent", - "timeScale", - "defaultEaseOvershootOrAmplitude", - "defaultEasePeriod", -) -# components that light the renderers of a scene (lighting_components) -LIGHTING_TYPES = ("Light", "ReflectionProbe") -# RenderSettings values of the active scene the renderers' lighting depends on; object references by name -# (render_settings) -RENDER_SETTINGS_FIELDS = ( - "m_Fog", - "m_AmbientMode", - "m_AmbientSkyColor", - "m_AmbientEquatorColor", - "m_AmbientGroundColor", - "m_AmbientIntensity", - "m_SubtractiveShadowColor", - "m_DefaultReflectionMode", - "m_DefaultReflectionResolution", - "m_ReflectionBounces", - "m_ReflectionIntensity", - "m_UseRadianceAmbientProbe", - "m_IndirectSpecularColor", -) -RENDER_SETTINGS_REFS = ("m_SkyboxMaterial", "m_CustomReflection", "m_Sun") - - -# ---------------------------------------------------------------- host, keys -def host_of(groups: list[dict]) -> dict: - """The host of an Overlay story from its story groups (storysite.story_groups): a spotTalk group is a home tap - talk (SpotManager.OnTalkCharacter: spot and tapped SpotCharacter id), a spot group a home area talk - (SpotManager.PlayAreaTalkAsync), a liveResult group the live result reward phase. Exactly one such group.""" - hits = [g for g in groups if g.get("kind") in HOST_GROUPS] - if len(hits) != 1: - raise ValueError( - f"an Overlay story needs exactly one spotTalk, spot or liveResult group, not {len(hits)} " - f"(groups: {[g.get('kind') for g in groups]})" - ) - g = hits[0] - kind, talk = HOST_GROUPS[g["kind"]] - out = {"kind": kind, "group": {"kind": g["kind"], "id": g["id"]}} - if kind == "home": - chars = g.get("characters") or [] - out.update(spotId=g["spot"]["id"], talk=talk, characterId=chars[0]["id"] if talk == "tap" and chars else None) - return out - - -def emb_key(cat, address: str) -> str: - """The catalog key of a game address: the address itself, else the embedded copy "Emb" + address.""" - for key in (address, f"Emb{address}"): - if cat.has(key): - return key - raise KeyError(f"{address}: neither it nor Emb{address} is in the catalog") - - -def _parent(path: str) -> str: - return path.rsplit("/", 1)[0] if "/" in path else "" - - -def _trs(node: dict) -> dict: - return {k: node[k] for k in ("localPosition", "localRotation", "localScale")} - - -def _first(nodes: list[dict], path: str) -> dict: - hits = [n for n in nodes if n["path"] == path] - if len(hits) != 1: - raise RuntimeError(f"{path}: {len(hits)} nodes") - return hits[0] - - -# ---------------------------------------------------------------- records -def camera_target(nodes: list[dict]) -> dict: - """cameraTarget: the CameraTargetRenderer prefab (SimpleAdvView slot capture): the transforms of the parent of - `_stage` and `_captureCamera` (Container), of the stage and of the capture camera with its Camera and - UniversalAdditionalCameraData settings (the slot's RenderTexture, layer and stage offset are code).""" - crt = advui._component(nodes[0], ("CameraTargetRenderer",)) - if crt is None: - raise RuntimeError(f"{nodes[0]['path']}: no CameraTargetRenderer") - stage, cam = advui._ref_path(crt["_stage"]), advui._ref_path(crt["_captureCamera"]) - if not stage or not cam or _parent(cam) != _parent(stage): - raise RuntimeError("CameraTargetRenderer: stage and capture camera not under one parent") - container = _parent(stage) - node = _first(nodes, cam) - c = [x for x in node["components"] if x["type"] == "Camera"] - d = [x for x in node["components"] if x.get("class") == "UniversalAdditionalCameraData"] - if len(c) != 1 or len(d) != 1: - raise RuntimeError(f"{cam}: {len(c)} Camera, {len(d)} UniversalAdditionalCameraData") - c, d = c[0], d[0] - return { - "container": {"path": container, **_trs(_first(nodes, container))}, - "stage": {"path": stage, **_trs(_first(nodes, stage))}, - "camera": { - "path": cam, - "localPosition": node["localPosition"], - "localRotation": node["localRotation"], - "fieldOfView": c["field of view"], - "near": c["near clip plane"], - "far": c["far clip plane"], - "orthographic": bool(c["orthographic"]), - "orthographicSize": c["orthographic size"], - "clearFlags": c["m_ClearFlags"], - "backgroundColor": c["m_BackGroundColor"], - "hdr": bool(c["m_HDR"]), - "allowMSAA": bool(c["m_AllowMSAA"]), - "rendererIndex": d["m_RendererIndex"], - "renderPostProcessing": bool(d["m_RenderPostProcessing"]), - "antialiasing": d["m_Antialiasing"], - }, - } - - -def _fields(c: dict) -> dict: - return {k: v for k, v in c.items() if k not in COMPONENT_HEADER} - - -def layout_root(nodes: list[dict], path: str) -> dict: - """layoutRoot: the serialized fields of the SimpleAdvLayoutRoot on the node at `path` (the profile - SimpleAdvLayoutRoot.CreateProfile makes).""" - c = advui._component(_first(nodes, path), ("SimpleAdvLayoutRoot",)) - if c is None: - raise RuntimeError(f"{path}: no SimpleAdvLayoutRoot") - return _fields(c) - - -def kept(path: str, subtrees=(), singles=()) -> bool: - """A node kept in a host UI: inside one of `subtrees` (root included), one of `singles`, or an ancestor of - either.""" - return any(path == s or path.startswith(s + "/") for s in subtrees) or any( - s == path or s.startswith(path + "/") for s in (*subtrees, *singles) - ) - - -# serialized references kept as node paths: component class -> (record key, fields) -REF_RECORDS = { - "UIAdvTalkWindow": ( - "talkWindowRefs", - ( - "SpeakerText", - "TalkText", - "TalkArea", - "_speakerNameObjects", - "_talkNextIndicator", - "_talkBackground", - "_autoIcon", - "_fastIcon", - ), - ), - "UISystemMessage": ("systemMessage", ("_message", "_showAnimation", "_canvasGroup", "_animator")), - "SpotLetterboxBands": ("letterboxBands", ("_topBand", "_bottomBand", "_leftBand", "_rightBand")), -} -WIDGET_FIELDS = ("_canvasSortOrder", "_isMainWidget", "_useBlur", "_blurDuration", "_useGlassMorphism") -TWEEN_EVENT = 1 # DOTweenSequence command type of an event entry (persistent calls) - - -def _plain(v): - """A value with serialized references ({gameObject | transform}) as node paths.""" - if isinstance(v, list): - return [_plain(x) for x in v] - if isinstance(v, dict) and ("gameObject" in v or "transform" in v): - return advui._ref_path(v) - return v - - -def tween_calls(entry: dict) -> list[dict]: - """The persistent calls of a DOTweenSequence event entry (UnityEvent m_PersistentCalls).""" - calls = (((entry.get("_tweenEvent") or {}).get("m_PersistentCalls") or {}).get("m_Calls")) or [] - out = [] - for x in calls: - t, a = x.get("m_Target") or {}, x.get("m_Arguments") or {} - out.append( - { - "target": advui._ref_path(t), - "class": t.get("class") or t.get("component"), - "method": x.get("m_MethodName"), - "string": a.get("m_StringArgument"), - "float": a.get("m_FloatArgument"), - "int": a.get("m_IntArgument"), - "bool": a.get("m_BoolArgument"), - "mode": x.get("m_Mode"), - } - ) - return out - - -def node_record(node: dict) -> dict: - """The advui record of a prefab node (transform, rect, advui._layout_components) with the records of the - components the host screens add: aspectRatioFitter, simpleAdvLayoutRoot, safeArea (UISafeArea), the - references of UIAdvTalkWindow / UISystemMessage / SpotLetterboxBands as node paths, `widget` (the UIManager - widget fields of a *Widget component) and, per DOTweenSequence event entry, its persistent `calls`.""" - rec = { - "path": node["path"], - "name": node["name"], - "active": node["active"], - "localPosition": node["localPosition"], - "localRotation": node["localRotation"], - "localScale": node["localScale"], - "rect": node.get("rect"), - } - rec.update(advui._layout_components(node)) - for c in node["components"]: - cls = c.get("class") - if cls == "AspectRatioFitter": - rec["aspectRatioFitter"] = {k: c[k] for k in ("m_Enabled", "m_AspectMode", "m_AspectRatio")} - elif cls == "SimpleAdvLayoutRoot": - rec["simpleAdvLayoutRoot"] = _fields(c) - elif cls == "UISafeArea": - rec["safeArea"] = {k: _plain(v) for k, v in _fields(c).items()} - elif cls in REF_RECORDS: - key, names = REF_RECORDS[cls] - rec[key] = {k: _plain(c.get(k)) for k in names} - elif cls == "DOTweenSequence": - for entry, out in zip(c["_list"], rec["tweenSequence"]["list"]): - if entry.get("_commandType") == TWEEN_EVENT: - out["calls"] = tween_calls(entry) - elif cls and cls.endswith("Widget") and "_canvasSortOrder" in c: - rec["widget"] = {"class": cls, **{k: c[k] for k in WIDGET_FIELDS if k in c}} - return rec - - -# ---------------------------------------------------------------- clips, shaders -TRANSFORM_CURVES = {"m_LocalPosition": "xyz", "m_LocalScale": "xyz", "m_LocalRotation": "xyzw"} - - -def clip_view(ex: Exporter, o, graph, root: str) -> dict: - """An AnimationClip played by the Animator at `root` as Exporter.clip_curves writes it, bindings below the - animated root included: their properties are keyed ":". Properties: - RectTransform driven fields, CanvasGroup.m_Alpha, Transform local position / rotation / scale / euler angles, - GameObject.m_IsActive and MonoBehaviour fields ("."). Streamed curves as keys, dense curves as - {samples, sampleRate, beginTime} (the dense clip's frames), constant curves as {constant}; the discrete curves - that follow the streamed float curves in the stream (a sprite reference's ".m_Sprite": the value indexes - `pptrCurveMapping`, the names of the referenced objects) as {discrete: [[time, value]]}, the value held until - the next key. A binding whose path names no object of the prefab (the Animator ignores it) is keyed - "#:" and its crc listed in `unresolvedPaths`.""" - rec = ex.clip_record(o) - ex.bind_clip(o, graph, root) - tt, mc, st, de = rec["tt"], rec["muscle"], rec["streamed"], rec["dense"] - name, const = rec["name"], rec["constant"] - n_st, n_disc, n_de = st["curveCount"], st.get("discreteCurveCount", 0), de["m_CurveCount"] - if n_disc and n_de: - raise NotImplementedError(f"clip {name}: discrete and dense curves") - props, lost = [], [] - for r in rec["bindings"]: - if r["int"] or r["serializeReference"]: - raise NotImplementedError(f"clip {name}: binding {r['raw']}") - if r["path"] is None: - lost.append(r["pathCrc"]) - pre = f"#{r['pathCrc']}:" if r["path"] is None else f"{r['path']}:" if r["path"] else "" - tid, attr = r["typeID"], r["attribute"] - if tid == 224 and attr in RECT_PROPS: - names = ["RectTransform." + attr] - elif tid == 225 and attr == "m_Alpha": - names = ["CanvasGroup.m_Alpha"] - elif tid == 4 and r["attributeCrc"] == 4: # localEulerAnglesRaw - names = [f"Transform.localEulerAngles.{a}" for a in "xyz"] - elif tid == 4 and r["transformAttr"] and attr in TRANSFORM_CURVES: - names = [f"Transform.{attr}.{a}" for a in TRANSFORM_CURVES[attr]] - elif tid == 1 and attr == "m_IsActive": - names = ["GameObject.m_IsActive"] - elif tid == 114 and r["script"] and attr: - names = [f"{r['script'][1]}.{attr}"] - elif r["pptr"] and attr == "m_Sprite" and tid == 212: - names = ["SpriteRenderer.m_Sprite"] - else: - raise NotImplementedError(f"clip {name}: binding {r['raw']}") - if r["pptr"] != (n_st <= len(props) < n_st + n_disc): - raise NotImplementedError(f"clip {name}: reference curve {r['raw']} outside the discrete curves") - props += [pre + n for n in names] - if n_st + n_disc + n_de + len(const) != len(props): - raise RuntimeError(f"clip {name}: curve count != bound properties") - curves = {p: {"keys": []} for p in props[:n_st]} - curves.update({p: {"discrete": []} for p in props[n_st : n_st + n_disc]}) - for t, keys in streamed_frames(st["data"]): - for ci, c0, c1, c2, c3 in keys: - if ci < n_st: - curves[props[ci]]["keys"].append([t, c0, c1, c2, c3]) - else: - curves[props[ci]]["discrete"].append([t, c3]) - samples = list(de["m_SampleArray"]) - for j in range(n_de): - curves[props[n_st + j]] = { - "samples": samples[j::n_de][: de["m_FrameCount"]], - "sampleRate": de["m_SampleRate"], - "beginTime": de["m_BeginTime"], - } - for i, v in enumerate(const): - curves[props[n_st + n_disc + n_de + i]] = {"constant": v} - out = { - "name": name, - "sampleRate": tt["m_SampleRate"], - "startTime": mc["m_StartTime"], - "stopTime": mc["m_StopTime"], - "loopTime": bool(mc["m_LoopTime"]), - "events": tt["m_Events"], - "curves": curves, - "keyFormat": "[time, c0, c1, c2, c3]; value(t) = ((c0*x + c1)*x + c2)*x + c3, x = t - time, " - "from the last key with time <= t (first key at -FLT_MAX)", - } - if rec["pptrCurveMapping"]: - refs = [ex.deref(o, p) for p in rec["pptrCurveMapping"]] - out["pptrCurveMapping"] = [x.read_typetree().get("m_Name") if x is not None else None for x in refs] - if lost: - out["unresolvedPaths"] = lost - return out - - -def gles3_keywords(index: list, shader: str, keywords, what: str) -> list[str]: - """The keywords of `keywords` that the GLES3 variants of `shader` use; a GLES3 vertex program with exactly - those must exist (as advui checks its materials).""" - rec = next((r for r in index if r["name"] == shader), None) - if rec is None: - raise RuntimeError(f"shader {shader} of {what} not dumped") - gles = [v for v in rec["variants"] if v["platform"] == "gles3" and v["type"] == "GLES3"] - known = {k for v in gles for k in v["keywords"]} - want = sorted(k for k in keywords if k in known) - if not any(sorted(v["keywords"]) == want and v["stage"] == "vertex" for v in gles): - raise RuntimeError(f"{shader}: no GLES3 variant for {what} keywords {want}") - return want - - -def dump_gles3(objects: dict, out_dir: Path) -> tuple[list, dict]: - """Dump the Shader objects {name: object} in name order (shader.py layout) keeping only the GLES3 programs: - the index lists those, the other programs are not kept. -> (index, write_index summary).""" - index: list = [] - for name in sorted(objects): - shader_mod.dump_objects([objects[name]], out_dir, objects[name].assets_file.name, index) - for rec in index: - keep = [v for v in rec["variants"] if v["platform"] == "gles3" and v["type"] == "GLES3"] - for v in rec["variants"]: - if v not in keep: - (Path(out_dir) / v["file"]).unlink(missing_ok=True) - rec["variants"] = keep - for d in sorted(Path(out_dir).rglob("*"), reverse=True): - if d.is_dir() and not any(d.iterdir()): - d.rmdir() - return index, shader_mod.write_index(index, out_dir) - - -# ---------------------------------------------------------------- UI documents -class UiDoc: - """A ui.json of prefab parts in the advui record format, written into `out_dir`: node records (node_record; - a TMP text gets its advui text record `textStyle` in the LanguageMode `mode`, None: no text allowed), the packed - sprites and textures, the materials with their GLES3 keyword sets, the UI shaders, the DOTween defaults, and - per Animator its controller (Exporter.controller_states) and clips (clip_view); a node whose controller cannot - be exported loses its `animator` record and is listed in `animatorsDropped`.""" - - def __init__(self, cat, player, out_dir: Path, mode: int | None = None): - self.player, self.dir = player, Path(out_dir) - self.dir.mkdir(parents=True, exist_ok=True) - self.ex = Exporter( - cat, self.dir, player=player, textures="deferred", stub_assets=TMP_STUBS, font_stubs=fontsettings.builtin() - ) - self.packer = TexelPacker(self.dir) - self.styles = textstyle.TextStyles(self.ex, player, mode, fontsettings.builtin()) if mode is not None else None - self.nodes: list[dict] = [] - self.sources: list[tuple[str, dict]] = [] # (key, prefab node) of each record - self.roots: list[dict] = [] - self.doc: dict | None = None - - def prefab(self, key: str) -> list[dict]: - return self.ex.prefab(key)["nodes"] - - def add(self, key: str, nodes: list[dict], subtrees=None, singles=()) -> list[dict]: - """Records of the prefab `nodes` of `key` (every node, or those `kept` by subtrees / singles).""" - out = [] - for n in nodes: - if subtrees is not None and not kept(n["path"], subtrees, singles): - continue - rec = node_record(n) - if "text" in rec: - if self.styles is None: - raise RuntimeError(f"{n['path']}: a text node in a UI without texts") - if not (rec.get("localizeText") or {}).get("localizeEnabled"): - raise NotImplementedError(f"{n['path']}: text without LocalizeText") - rec["textStyle"] = self.styles.record( - n["path"], advui._need_component(n, advui.TMP_CLASSES), advui._component(n, ("LocalizeText",)) - ) - del rec["text"] - self.nodes.append(rec) - self.sources.append((key, n)) - out.append(rec) - self.roots.append({"root": nodes[0]["path"], "key": key}) - return out - - def sprite(self, key: str) -> dict: - """{spriteRef, name} of the sprite a key names (a Sprite, or a Texture2D's sprite sub-asset).""" - o = self.ex.key_object(key) - return self.ex.sprite(o) if o.type.name == "Sprite" else self.ex.key_sprite(key) - - def _controller(self, key: str, path: str): - env, graph = self.ex.load(key) - hits = [] - for o in env.objects: - if o.type.name == "Animator": - tf = graph.tf_of_go.get(o.read_typetree()["m_GameObject"]["m_PathID"]) - if tf is not None and graph.path(tf) == path: - hits.append(o) - if len(hits) != 1: - return None - ctrl = self.ex.deref(hits[0], hits[0].read_typetree()["m_Controller"]) - if ctrl is None: - return None - try: - states = self.ex.controller_states(ctrl) - clips = {} - for p in ctrl.read_typetree()["m_AnimationClips"]: - c = self.ex.deref(ctrl, p) - if c is None or c.type.name != "AnimationClip": - raise NotImplementedError("not an AnimationClip") - v = clip_view(self.ex, c, graph, path) - clips[f"{states['name']}/{v['name']}"] = v - except NotImplementedError: - return None - return states, clips - - def write(self, about: str, head: dict | None = None, tail: dict | None = None) -> dict: - clips, controllers, dropped = {}, {}, [] - for rec, (key, n) in zip(self.nodes, self.sources): - if "animator" in rec: - got = self._controller(key, n["path"]) - if got is None: - del rec["animator"] - dropped.append(rec["path"]) - else: - controllers[got[0]["name"]] = got[0] - clips.update(got[1]) - ex, refs = self.ex, set() - for rec in self.nodes: - if (rec.get("image") or {}).get("spriteRef"): - refs.add(rec["image"]["spriteRef"]) - if (rec.get("buttonImageState") or {}).get("selectedRef"): - refs.add(rec["buttonImageState"]["selectedRef"]) - for ref in sorted(refs): - s = ex.sprite_recs[ref] - tex = s["tex"].read_typetree()["m_Name"] - ex.pack_sprite(ref, self.packer, "sprites" if "FixUiSpriteAtlas" in tex else f"sprite_{s['name']}") - textures = self.packer.build() - sprites = {ex.sprite_recs[ref]["name"]: ex.packed_sprite(ref) for ref in sorted(refs)} - for rec in self.nodes: - (rec.get("image") or {}).pop("spriteRef", None) - (rec.get("buttonImageState") or {}).pop("selectedRef", None) - materials = { - "Default UI Material": { - "material": "Default UI Material", - "shader": {"shader": "UI/Default"}, - "keywords": [], - "textures": {}, - "ints": {}, - "floats": {}, - "colors": {}, - "shaderDefaults": True, - } - } - for _, n in self.sources: - for c in n["components"]: - if c.get("class") in ("Image", "RawImage") and c.get("m_Material"): - materials.setdefault(c["m_Material"]["material"], json.loads(dumps(c["m_Material"]))) - for m in materials.values(): - storyfonts._material_textures(m) - needed = {} - for name in sorted({m["shader"]["shader"] for m in materials.values()}): - if name == "UI/Default": - needed[name] = self.player.shader("UI/Default", file="unity_builtin_extra") - elif name in ex.shaders: - needed[name] = ex.shaders[name] - else: - raise RuntimeError(f"shader {name} not in the loaded bundles") - index, summary = ex.dump_shaders(self.dir / "shaders", needed) - keywords = { - k: gles3_keywords(index, m["shader"]["shader"], m["keywords"], f"material {k}") - for k, m in materials.items() - } - dts = self.player.mono(self.player.resource("DOTweenSettings")) - doc = { - "about": about, - **(head or {}), - "roots": self.roots, - "nodes": self.nodes, - "sprites": sprites, - "textures": textures, - "materials": materials, - "materialKeywords": keywords, - "clips": clips, - "controllers": controllers, - "animatorsDropped": dropped, - "dotween": {k: dts[k] for k in DOTWEEN_FIELDS}, - "shaders": {"index": "shaders/shaders.json", "names": summary["names"]}, - } - if self.styles is not None: - doc["textStyle"] = self.styles.summary() - doc.update(tail or {}) - write_json(self.dir / "ui.json", doc) - self.doc = doc - return doc - - -# ---------------------------------------------------------------- home -def container_go(env, internal_id: str): - """The GameObject an AssetBundle container entry names (a prefab's root).""" - for o in env.objects: - if o.type.name == "AssetBundle": - for path, info in o.read_typetree()["m_Container"]: - if path.lower() == internal_id.lower(): - go = deref(o, info["asset"]) - if go is not None and go.type.name == "GameObject": - return go - raise KeyError(f"{internal_id}: no GameObject in the AssetBundle containers") - - -def _go_ref(owner, pptr: dict) -> int | None: - if not pptr["m_PathID"]: - return None - if pptr["m_FileID"] == 0: - return pptr["m_PathID"] - o = deref(owner, pptr) - return o.path_id if o is not None else None - - -def background(cat, key: str, situation_name: str, ex: Exporter) -> dict: - """The placed background prefab `key` as SpotSceneRoot.SetObject leaves it, over the nodes and materials of - hostspot.extract_room (the same bundle load, glb_order): SpotBackground.Prepare(situation_name) on the prefab - root, the "floor" render queue (FindFirstChildWithName(background, "floor", "shadow"), its - GetComponentInChildren().material.renderQueue = 2000, before the root is activated), then - SetActiveFast(background, true). Objects outside the placed prefab (other prefabs of the bundle closure) are - not placed: inactive.""" - env = UnityPy.load(*[str(p) for p in cat.fetch_key(key)]) - graph = SceneGraph(env) - mf_by_go, mr_by_go = hostspot.mesh_components(env) - rendered = {go for _, go in mr_by_go} # SceneGraph: a GameObject path id is in one file only - root_go = container_go(env, cat.entry(key)["internal_id"]).path_id - root = graph.tf_of_go[root_go] - spot_bg, volumes = [], [] - for o in env.objects: - if o.type.name != "MonoBehaviour": - continue - tf = graph.tf_of_go.get(o.read_typetree()["m_GameObject"]["m_PathID"]) - if tf is None or not under(graph, tf, root): - continue - cls = script_class(o) - if cls == "SpotBackground" and tf == root: # background.GetComponent() - spot_bg.append(o) - elif cls == "Volume": - volumes.append((o, tf)) - if spot_bg: - tt = spot_bg[0].read_typetree() - entries = [ - (e.get("SituationName"), [_go_ref(spot_bg[0], h) for h in e.get("HideObjects") or []]) - for e in tt.get("_situationObjects") or [] - ] - prep = prepare(graph, root, situation_name, entries) - else: - prep = { - "name": situation_name.replace(CLONE, ""), - "matched": False, - "activate": [], - "hide": [], - "overrides": {}, - } - overrides = prep["overrides"] - floor = find_first_child_with_name(graph, root, FLOOR_NAME, FLOOR_EXCLUDE) - renderer = ( - None - if floor is None - else component_in_children( - graph, floor, lambda t: _go(graph, t) in rendered, lambda t: active_in_hierarchy(graph, t, overrides) - ) - ) - shown = {**overrides, root_go: True} - nodes, mats = glb_order(hostspot.drawn_meshes(mf_by_go, mr_by_go, graph, lambda p: None)) - rows = [] - for p in mats: - mat = p.deref() - tt = mat.read_typetree() - sh = referent(mat, tt["m_Shader"]) - rows.append((mat, tt, sh.read_typetree()["m_ParsedForm"]["m_Name"], sh)) - tex_index = glb_textures([(mat, tt, name) for mat, tt, name, _ in rows]) - if len(volumes) > 1: - raise NotImplementedError(f"{key}: {len(volumes)} Volume components") - volume = None - if volumes: - o, tf = volumes[0] - volume = { - "path": graph.path(tf), - "active": active_in_hierarchy(graph, tf, shown), - **volume_record(ex.component(o), graph.go[_go(graph, tf)]["m_Layer"]), - } - t = graph.tf[root] - return { - "roomRoot": { - "path": graph.path(root), - "localPosition": t["m_LocalPosition"], - "localRotation": t["m_LocalRotation"], - "localScale": t["m_LocalScale"], - }, - "roomNodes": [ - { - "path": graph.path(tf), - "placed": under(graph, tf, root), - "active": under(graph, tf, root) and active_in_hierarchy(graph, tf, shown), - "renderQueue": FLOOR_QUEUE if tf == renderer else None, - } - for tf, _ in nodes - ], - "roomMaterials": [room_material(mat, tt, name, tex_index) for mat, tt, name, _ in rows], - "situation": { - "name": prep["name"], - "matched": prep["matched"], - "activate": [graph.path(x) for x in prep["activate"]], - "hide": [graph.path(graph.tf_of_go[g]) if g in graph.tf_of_go else None for g in prep["hide"]], - }, - "floor": { - "object": graph.path(floor) if floor is not None else None, - "renderer": graph.path(renderer) if renderer is not None else None, - "renderQueue": FLOOR_QUEUE, - }, - "volume": volume, - "lights": [ - {"prefab": "background", **r} - for r in lighting_components(env, graph, root, lambda tf: active_in_hierarchy(graph, tf, shown)) - ], - "shaders": {name: sh for _, _, name, sh in reversed(rows)}, # the first material's object - "check": { - "nodes": [n for _, n in nodes], - "textures": len(tex_index), - "mirrored": sum( - 1 for tf, _ in nodes if under(graph, tf, root) and np.linalg.det(graph.world(tf)[:3, :3]) < 0 - ), - }, - } - - -def spine_records(env, ex: Exporter) -> tuple[dict, list, dict]: - """The Spine data of a situation prefab's bundle closure (loaded as hostspot.extract_spot loads it): the material of - every atlas page (Exporter.material records by name), per SpotSpineCharacter in spot.json's order its - SkeletonAnimation's MeshRenderer sorting order / layer and {atlas page: material name} (SpineAtlasAsset - materials in page order), and the shaders the materials name. -> (materials, characters, {name: shader}).""" - graph = SceneGraph(env) - by_cls: dict[str | None, list] = {} - mr_by_go = {} - for o in env.objects: - if o.type.name == "MonoBehaviour": - by_cls.setdefault(script_class(o), []).append(o) - elif o.type.name == "MeshRenderer": - mr_by_go[o.read_typetree()["m_GameObject"]["m_PathID"]] = o - materials, shaders, pages_of = {}, {}, {} - - def pages(sda) -> dict: - if sda.path_id not in pages_of: - out = {} - for ap in sda.read_typetree()["atlasAssets"]: - a = deref(sda, ap) - if a is None: - continue - att = a.read_typetree() - names = atlas_pages(hostspot.text_bytes(referent(a, att["atlasFile"]).read()).decode("utf-8")) - mats = [deref(a, m) for m in att["materials"]] - if len(names) != len(mats): - raise RuntimeError(f"atlas {att.get('m_Name')}: {len(names)} pages vs {len(mats)} materials") - for page, m in zip(names, mats): - rec = None if m is None else ex.material(m) - if rec is not None: - storyfonts._material_textures(rec) - materials.setdefault(rec["material"], rec) - sh = rec["shader"]["shader"] - shaders.setdefault(sh, ex.shaders[sh]) - out[page] = rec and rec["material"] - pages_of[sda.path_id] = out - return pages_of[sda.path_id] - - for o in by_cls.get("SkeletonDataAsset", []): - pages(o) - chars = [] - for o in by_cls.get("SpotSpineCharacter", []): - tt = o.read_typetree() - rec = { - "path": graph.path(graph.tf_of_go[tt["m_GameObject"]["m_PathID"]]), - "sortingOrder": None, - "sortingLayer": None, - "pageMaterials": {}, - } - anim = deref(o, tt["_animation"]) - if anim is not None: - at = anim.read_typetree() - mr = mr_by_go.get(at["m_GameObject"]["m_PathID"]) - if mr is not None: - mt = mr.read_typetree() - rec.update(sortingOrder=mt.get("m_SortingOrder"), sortingLayer=mt.get("m_SortingLayerID")) - sda = deref(anim, at["skeletonDataAsset"]) - rec["pageMaterials"] = dict(pages(sda)) if sda is not None else {} - chars.append(rec) - return {k: materials[k] for k in sorted(materials)}, chars, shaders - - -def render_settings(env) -> dict: - """renderSettings: the scene's RenderSettings (RENDER_SETTINGS_FIELDS; RENDER_SETTINGS_REFS as {"name"} or null) and - `lightmaps`, the lightmap count of its LightmapSettings. SpotManager.ActiveSpotScene makes the "Spot" scene the - active one, whose settings light the spot.""" - rs = [o for o in env.objects if o.type.name == "RenderSettings"] - lm = [o for o in env.objects if o.type.name == "LightmapSettings"] - if len(rs) != 1 or len(lm) != 1: - raise RuntimeError(f"scene: {len(rs)} RenderSettings and {len(lm)} LightmapSettings") - tt = rs[0].read_typetree() - out = {k: _plain(tt[k]) for k in RENDER_SETTINGS_FIELDS if k in tt} - for k in RENDER_SETTINGS_REFS: - ref = tt.get(k) - if not ref or not ref["m_PathID"]: - out[k] = None - continue - target = deref(rs[0], ref) - out[k] = {"name": target.read_typetree().get("m_Name") if target is not None else None} - out["lightmaps"] = len(lm[0].read_typetree().get("m_Lightmaps") or []) - return out - - -def lighting_components(env, graph, root_tf: int, active) -> list[dict]: - """The Light / ReflectionProbe components under `root_tf` in path order: path, type, m_Enabled, `active(tf)` - (activeInHierarchy as placed) and a Light's m_Type.""" - out = [] - for o in env.objects: - if o.type.name not in LIGHTING_TYPES: - continue - tt = o.read_typetree() - tf = graph.tf_of_go.get(tt["m_GameObject"]["m_PathID"]) - if tf is None or not under(graph, tf, root_tf): - continue - rec = { - "path": graph.path(tf), - "type": o.type.name, - "m_Enabled": int(tt.get("m_Enabled", 1)), - "active": bool(active(tf)), - } - if o.type.name == "Light": - rec["m_Type"] = tt.get("m_Type") - out.append(rec) - return sorted(out, key=lambda r: (r["path"], r["type"])) - - -def _active_path(nodes: list[dict], path: str) -> bool: - """activeInHierarchy of the node at `path` in a node list (it and its ancestors active).""" - return all(n["active"] for n in nodes if n["path"] == path or path.startswith(n["path"] + "/")) - - -def scene_root(ex: Exporter, key: str | None) -> dict: - """sceneRoot: the "Spot" scene's SpotSceneRoot: world TRS of `_objRoot` (the parent SetObject puts the - background and the situation under), its Light / ReflectionProbe components (with `active`, the object's - activeInHierarchy), its Volumes, the scene camera (`_sceneCamera`, advui camera record) and the scene's - RenderSettings (render_settings); null / [] when the scene or the component is not there.""" - out = {"key": key, "objRoot": None, "lights": [], "volumes": [], "sceneCamera": None, "renderSettings": None} - if key is None: - return out - nodes = ex.prefab(key)["nodes"] - out["renderSettings"] = render_settings(ex.load(key)[0]) - roots = [(n, c) for n in nodes for c in n["components"] if c.get("class") == "SpotSceneRoot"] - if len(roots) != 1: - return out - c = roots[0][1] - obj, cam = advui._ref_path(c.get("_objRoot")), advui._ref_path(c.get("_sceneCamera")) - out["objRoot"] = {"path": obj, **world_trs(nodes, obj)} if obj else None - out["lights"] = [ - { - "path": n["path"], - "type": x["type"], - "active": _active_path(nodes, n["path"]), - **{k: _plain(v) for k, v in x.items() if k != "type"}, - } - for n in nodes - for x in n["components"] - if x["type"] in LIGHTING_TYPES - ] - out["volumes"] = [ - {"path": n["path"], **volume_record(x, n["layer"])} - for n in nodes - for x in n["components"] - if x.get("class") == "Volume" - ] - out["sceneCamera"] = advui._camera(nodes, cam) if cam else None - return out - - -def home_host(cat, master_dir, player, host: dict, hdir: Path, keys: dict) -> dict: - """host.json `home`: hostspot.extract_spot and hostspot.extract_room into host/spot/, the placed background - (background), the Spine records, the Spot scene root, the blur, the ambient sound and the shaders (host/shaders/, - GLES3 programs only) with each material's GLES3 keyword set.""" - sdir = hdir / "spot" - doc = hostspot.extract_spot(cat, Path(master_dir), host["spotId"], sdir) - row = doc["master"] - # A tap talk's character need not be among the spot's SpotCharacters (spot.json `characters`): the game then has - # no tap target that starts the talk (HomeSpotAfterTalkRequestFactory.TrySelectAdv matches the tapped - # SpotCharacter's _characterId), and the player plays it without the camera focus - bg_key, sit_key = row["_backgroundAssetPath"], row["_situationAssetPath"] - room_doc = hostspot.extract_room(cat, bg_key, sdir / "room.glb") - ex = Exporter(cat, hdir, player=player, textures="deferred", inline_meshes=False) - sit_env = UnityPy.load(*[str(p) for p in cat.fetch_key(sit_key)]) - sit_go = container_go(sit_env, cat.entry(sit_key)["internal_id"]) - sit_name = sit_go.read_typetree()["m_Name"] - sit_graph = SceneGraph(sit_env) - sit_lights = [ - {"prefab": "situation", **r} - for r in lighting_components( - sit_env, - sit_graph, - sit_graph.tf_of_go[sit_go.path_id], - lambda tf: active_in_hierarchy(sit_graph, tf, {sit_go.path_id: True}), - ) - ] - bg = background(cat, bg_key, sit_name + CLONE, ex) - check_room(bg_key, glb_json(sdir / "room.glb"), room_doc, bg) - spine_mats, spine_chars, spine_shaders = spine_records(sit_env, ex) - if [c["path"] for c in spine_chars] != [c["path"] for c in doc["spineCharacters"]]: - raise RuntimeError(f"{sit_key}: Spine characters differ from spot.json's") - blur = advui.blur_record(player.graphics()) - blur_obj = [o for o in player.renderer_shaders(blur["renderer"]) if o.read().m_ParsedForm.m_Name == blur["shader"]] - if len(blur_obj) != 1: - raise RuntimeError(f"renderer {blur['renderer']}: {len(blur_obj)} shaders {blur['shader']}") - needed = dict(bg["shaders"]) - for name, o in [*spine_shaders.items(), (blur["shader"], blur_obj[0])]: - needed.setdefault(name, o) - index, summary = dump_gles3(needed, hdir / "shaders") - keywords = { - "room": [ - gles3_keywords(index, m["shader"], m["keywords"], f"room material {m['name']}") for m in bg["roomMaterials"] - ], - "spine": { - k: gles3_keywords(index, m["shader"]["shader"], m["keywords"], f"Spine material {k}") - for k, m in spine_mats.items() - }, - } - gles3_keywords(index, blur["shader"], [], "the blur pass") - return { - "spotId": host["spotId"], - "talk": host["talk"], - "characterId": host["characterId"], - "titleTextId": row.get("_advNameTextId") if host["talk"] == "area" else None, - "spot": f"{HOST_DIR}/spot/spot.json", - "room": f"{HOST_DIR}/spot/room.glb", - "roomRoot": bg["roomRoot"], - "roomNodes": bg["roomNodes"], - "roomMaterials": bg["roomMaterials"], - "situation": bg["situation"], - "floor": bg["floor"], - "sceneRoot": scene_root(ex, keys.get("spotScene")), - "volume": bg["volume"], - "lights": bg["lights"] + sit_lights, - "spineMaterials": spine_mats, - "spineCharacters": spine_chars, - "shaders": {"index": f"{HOST_DIR}/shaders/shaders.json", "names": summary["names"]}, - "materialKeywords": keywords, - "blur": blur, - "ambient": ambient_record(doc["situationSettings"], master_dir), - } - - -def home_ui(ui: UiDoc, keys: dict) -> tuple[str, dict, list]: - """host/ui nodes of a home talk: UISpotWidget's root, View, View/Fade, its DOTweenSequence_* nodes (and the - nodes their calls target), the SimpleAdv root (UISpotWidget._afterTalkContents) subtree; UISpotLetterbox. - The opened sequence is _fadeIn: the spot appears through SpotManager.PlayInAnimation -> SpotPresenter.FadeIn - (the Fade cover from 1 to 0); a spot change (ChangeSpotAsync) covers with _transitionIn and reveals with - _transitionOut, which ends with the cover at 0 as well. - -> (overlay root, layout root, opened sequences).""" - nodes = ui.prefab(keys["spotWidget"]) - w = advui._need_component(nodes[0], ("UISpotWidget",)) - overlay = advui._need_path(w["_afterTalkContents"]) - seqs = [advui._need_path(w[k]) for k in ("_transitionIn", "_transitionOut", "_fadeIn", "_fadeOut") if w.get(k)] - fade = f"{_parent(overlay)}/Fade" - ui.add(keys["spotWidget"], nodes, [overlay, fade, *seqs], _call_targets(nodes, seqs)) - ui.add(keys["spotLetterbox"], ui.prefab(keys["spotLetterbox"])) - return overlay, layout_root(nodes, overlay), [advui._need_path(w["_fadeIn"])] - - -def _call_targets(nodes: list[dict], seqs: list[str]) -> list[str]: - """The nodes the DOTweenSequence event entries of the nodes at `seqs` call.""" - out = [] - for n in nodes: - if n["path"] in seqs: - for c in n["components"]: - if c.get("class") == "DOTweenSequence": - for e in c["_list"]: - out += [x["target"] for x in tween_calls(e) if x["target"] and x["target"] not in out] - return out - - -def afterlive_ui(ui: UiDoc, keys: dict) -> tuple[str, dict, list, dict]: - """host/ui nodes of a live result talk: UILiveResultBackgroundWidget (its _backgroundImage showing the - live_result_background sprite); UILiveResultWidget's chain to the SimpleAdv root (_advSimpleContents) and its - subtree, the widget's and UISoloLiveMusicResultView's _inAnimation sequences and the nodes their calls target. - -> (overlay root, layout root, opened sequences, host.json `afterlive`).""" - bnodes = ui.prefab(keys["resultBackground"]) - bw = advui._need_component(bnodes[0], ("UILiveResultBackgroundWidget",)) - bg_path = advui._ref_path(bw["_backgroundImage"]) - recs = [r for r in ui.add(keys["resultBackground"], bnodes) if r["path"] == bg_path and r.get("image")] - if len(recs) != 1: - raise RuntimeError(f"{bg_path}: {len(recs)} image nodes") - sprite = ui.sprite(keys["resultBackgroundSprite"]) - recs[0]["image"].update(sprite=sprite["name"], spriteRef=sprite["spriteRef"]) - nodes = ui.prefab(keys["resultWidget"]) - rw = advui._need_component(nodes[0], ("UILiveResultWidget",)) - overlay = advui._need_path(rw["_advSimpleContents"]) - view = [ - (n, c) - for n in nodes - for c in n["components"] - if c.get("class") == "UISoloLiveMusicResultView" and overlay.startswith(n["path"] + "/") - ] - if len(view) != 1: - raise RuntimeError(f"{overlay}: {len(view)} UISoloLiveMusicResultView ancestors") - seqs = [ - advui._need_path(bw["_inAnimation"]), - advui._need_path(rw["_inAnimation"]), - advui._need_path(view[0][1]["_inAnimation"]), - ] - ui.add(keys["resultWidget"], nodes, [overlay, *seqs[1:]], _call_targets(nodes, seqs[1:])) - parts = overlay.split("/") - reward = next(("/".join(parts[: i + 1]) for i in range(len(parts) - 2, 0, -1) if parts[i] == REWARD_PANEL), None) - if reward is None: - raise RuntimeError(f"{overlay}: no {REWARD_PANEL} ancestor") - return ( - overlay, - layout_root(nodes, overlay), - seqs, - { - "backgroundNode": bg_path, - "backgroundSprite": sprite["name"], - "rewardPanel": reward, - "substitutes": list(SUBSTITUTES), - }, - ) - - -# ---------------------------------------------------------------- background (SpotBackground, SpotSceneRoot) -def _go(graph, tf: int) -> int: - return graph.tf[tf]["m_GameObject"]["m_PathID"] - - -def _children(graph, tf: int) -> list[int]: - return [c["m_PathID"] for c in graph.tf[tf]["m_Children"]] - - -def prepare(graph, background_tf: int, situation_name: str, entries: list) -> dict: - """SpotBackground.Prepare(situationName) on the background at `background_tf` (entries: its _situationObjects - as [(SituationName, [HideObjects GameObject path ids | None])]): "(Clone)" removed from the name; the first - entry of that name activates every child of transform.GetChild(0) and deactivates its HideObjects; no entry of - that name keeps the prefab state. -> {name, matched, activate [transform ids], hide [GameObject ids | None], - overrides {GameObject id: activeSelf}}.""" - name = situation_name.replace(CLONE, "") - kids = _children(graph, background_tf) - if not kids: - raise RuntimeError(f"{graph.path(background_tf)}: SpotBackground without a child (GetChild(0))") - for entry_name, hide in entries: - if entry_name == name: - activate = _children(graph, kids[0]) - overrides = {_go(graph, t): True for t in activate} - overrides.update({g: False for g in hide if g}) - return {"name": name, "matched": True, "activate": activate, "hide": list(hide), "overrides": overrides} - return {"name": name, "matched": False, "activate": [], "hide": [], "overrides": {}} - - -def active_in_hierarchy(graph, tf: int, overrides: dict) -> bool: - """GameObject.activeInHierarchy of the object at `tf`, with `overrides` {GameObject id: activeSelf} over the - serialized m_IsActive.""" - while tf in graph.tf: - go = _go(graph, tf) - if not overrides.get(go, bool(graph.go.get(go, {}).get("m_IsActive", 1))): - return False - tf = graph.tf[tf]["m_Father"]["m_PathID"] - return True - - -def under(graph, tf: int, root: int) -> bool: - """The transform `tf` is `root` or below it.""" - while tf in graph.tf: - if tf == root: - return True - tf = graph.tf[tf]["m_Father"]["m_PathID"] - return False - - -def find_first_child_with_name(graph, tf: int, contains: str, excludes: str) -> int | None: - """SpotSceneRoot.SetObject's FindFirstChildWithName: depth first over the descendants of `tf` (itself - excluded), children in order, the first whose name contains `contains` and not `excludes` (ordinal).""" - for c in _children(graph, tf): - name = graph.go.get(_go(graph, c), {}).get("m_Name", "") - if contains in name and excludes not in name: - return c - hit = find_first_child_with_name(graph, c, contains, excludes) - if hit is not None: - return hit - return None - - -def component_in_children(graph, tf: int, has, active) -> int | None: - """GameObject.GetComponentInChildren() (inactive objects skipped): depth first from `tf` itself, the first - transform whose object is active in the hierarchy (`active(tf)`) and has the component (`has(tf)`).""" - if active(tf) and has(tf): - return tf - for c in _children(graph, tf): - hit = component_in_children(graph, c, has, active) - if hit is not None: - return hit - return None - - -def glb_order(drawn, arrays_of=mesh_arrays) -> tuple[list, list]: - """The nodes and materials hostspot.extract_room writes, in its order, from its hostspot.drawn_meshes items - [(transform id, Mesh, material PPtrs)]: a mesh without vertex data or triangles, or whose submeshes all lack - a material, is no node; a material is numbered at its first use (hostspot.submesh_primitives order, by - hostspot.object_key). -> ([(transform id, mesh name)], [material PPtr]).""" - index: dict[tuple[str, int], int] = {} - mats, nodes = [], [] - - def material_for(p) -> int: - key = hostspot.object_key(p.deref()) - if key not in index: - index[key] = len(mats) - mats.append(p) - return index[key] - - for tf, mesh, pptrs in drawn: - arrays = arrays_of(mesh) - if arrays is None: - continue - tris = arrays[3] - if not any(len(t) for t in tris): - continue - if len(pptrs) < len(tris): - raise NotImplementedError(f"{mesh.m_Name}: {len(tris)} submeshes, {len(pptrs)} materials") - if hostspot.submesh_primitives(tris, pptrs, material_for, lambda i: None): - nodes.append((tf, mesh.m_Name)) - return nodes, mats - - -def glb_textures(materials: list) -> dict[tuple[str, int], int]: - """{texture hostspot.object_key: glb texture index} of hostspot.extract_room for its materials [(material object, - typetree, shader name)] in glb order: the main texture slot of each (hostspot.main_textures), numbered at first - use.""" - index: dict[tuple[str, int], int] = {} - for owner, tt, shader_name in materials: - for tex, _ in hostspot.main_textures(owner, tt, shader_name): - index.setdefault(hostspot.object_key(tex), len(index)) - return index - - -def _texture_key(owner, ref: dict) -> tuple[str, int] | None: - """hostspot.object_key of the texture a material's PPtr `ref` names (read from `owner`'s typetree); None for an - empty slot or a texture outside the loaded files (so not in the glb).""" - if not ref["m_PathID"]: - return None - try: - return hostspot.object_key(deref(owner, ref)) - except FileNotFoundError: - return None - - -def room_material(owner, tt: dict, shader_name: str, tex_index: dict) -> dict: - """A room material (`owner` its object, `tt` its typetree) as saved (its shader runs as the game's): name, - shader, keywords, float / int / colour properties, custom render queue, and per texture slot the glb texture - index (null when the texture is not in the glb), scale and offset.""" - sp = tt["m_SavedProperties"] - keywords = tt.get("m_ValidKeywords") - if keywords is None: - keywords = (tt.get("m_ShaderKeywords") or "").split() - return { - "name": tt["m_Name"], - "shader": shader_name, - "keywords": list(keywords), - "floats": dict(sp.get("m_Floats", [])), - "ints": dict(sp.get("m_Ints", [])), - "colors": dict(sp.get("m_Colors", [])), - "renderQueue": tt.get("m_CustomRenderQueue"), - "texEnvs": { - k: { - "texture": tex_index.get(_texture_key(owner, v["m_Texture"])), - "scale": [v["m_Scale"]["x"], v["m_Scale"]["y"]], - "offset": [v["m_Offset"]["x"], v["m_Offset"]["y"]], - } - for k, v in sp["m_TexEnvs"] - }, - } - - -def glb_json(path) -> dict: - """The JSON chunk of a binary glTF file.""" - data = Path(path).read_bytes() - (n,) = struct.unpack_from(" None: - """hostspot.extract_room's room.glb (its JSON chunk `glb`) and room.json (`room_doc`) have the nodes, materials and - glTF textures of the background record `bg` (background), in its order.""" - if ( - [n["name"] for n in glb["nodes"]] != bg["check"]["nodes"] - or room_doc["meshCount"] != len(bg["roomNodes"]) - or [m["name"] for m in glb.get("materials", [])] != [m["name"] for m in bg["roomMaterials"]] - or len(glb.get("textures", [])) != bg["check"]["textures"] - ): - raise RuntimeError(f"{key}: room.glb nodes / materials / textures differ from the background's order") - - -def volume_record(comp: dict, layer: int) -> dict: - """A Volume component (Exporter record: sharedProfile and its components inlined) -> enabled, isGlobal, - weight, priority, blendDistance, layer, profile name and per profile component its class, active flag and the - parameters it overrides (m_OverrideState set) with their values.""" - prof = comp.get("sharedProfile") or {} - comps = [] - for c in prof.get("components") or []: - if not c: - continue - over = { - k: _plain(v["m_Value"]) - for k, v in c.items() - if isinstance(v, dict) and "m_OverrideState" in v and v["m_OverrideState"] - } - comps.append({"class": c.get("asset"), "active": bool(c.get("active", 1)), **over}) - return { - "enabled": bool(comp.get("m_Enabled", 1)), - "isGlobal": bool(comp["m_IsGlobal"]), - "weight": comp["weight"], - "priority": comp["priority"], - "blendDistance": comp["blendDistance"], - "layer": layer, - "profile": prof.get("name"), - "components": comps, - } - - -def _qmul(a, b): - ax, ay, az, aw = a - bx, by, bz, bw = b - return ( - aw * bx + ax * bw + ay * bz - az * by, - aw * by - ax * bz + ay * bw + az * bx, - aw * bz + ax * by - ay * bx + az * bw, - aw * bw - ax * bx - ay * by - az * bz, - ) - - -def _rotate(q, v): - x, y, z, _ = _qmul(_qmul(q, (v[0], v[1], v[2], 0.0)), (-q[0], -q[1], -q[2], q[3])) - return (x, y, z) - - -def world_trs(nodes: list[dict], path: str) -> dict: - """World position, rotation and (lossy) scale of the node at `path` from the local TRS of it and its - ancestors (position through the parents' rotation and scale, rotation the product of the local rotations, - scale the product of the local scales).""" - parts = path.split("/") - pos, rot, scl = (0.0, 0.0, 0.0), (0.0, 0.0, 0.0, 1.0), (1.0, 1.0, 1.0) - for i in range(1, len(parts) + 1): - n = _first(nodes, "/".join(parts[:i])) - p, r, s = n["localPosition"], n["localRotation"], n["localScale"] - d = _rotate(rot, (scl[0] * p["x"], scl[1] * p["y"], scl[2] * p["z"])) - pos = (pos[0] + d[0], pos[1] + d[1], pos[2] + d[2]) - rot = _qmul(rot, (r["x"], r["y"], r["z"], r["w"])) - scl = (scl[0] * s["x"], scl[1] * s["y"], scl[2] * s["z"]) - return {"position": dict(zip("xyz", pos)), "rotation": dict(zip("xyzw", rot)), "scale": dict(zip("xyz", scl))} - - -def _rows(master_dir, name: str) -> dict: - try: - return {r["_id"]: r for r in master.table(Path(master_dir), name)} - except FileNotFoundError: - return {} - - -def ambient_record(settings: dict, master_dir) -> dict | None: - """ambient: the situation's ambient sound (SpotSituationSettings._soundId, ambientSoundVolume) with its cue - sheet and cue (MasterSound, MasterSoundCueSheet; null when the master data lacks the row); null without one.""" - sid = settings.get("_soundId") or 0 - if not sid: - return None - snd = _rows(master_dir, "MasterSound").get(sid) or {} - sheet = _rows(master_dir, "MasterSoundCueSheet").get(snd.get("_soundCueSheetID")) or {} - return { - "soundId": sid, - "volume": settings.get("ambientSoundVolume"), - "cueSheet": sheet.get("_cueSheetName"), - "cue": snd.get("_cueName"), - } - - -# ---------------------------------------------------------------- per language: ui/simple/ -def text_bindings(ui: UiDoc, lang: dict, mode: int) -> dict[str, dict]: - """{node path: text binding} of the text nodes of `ui`, as storyfonts.text_bindings makes them for the story - UI: the serialized TextMeshPro fields, storyfonts.text_extras and `localized` (LocalizeText in `mode`).""" - swap = textstyle.font_swap(lang, mode) - out = {} - for rec, (_, n) in zip(ui.nodes, ui.sources): - if "textStyle" not in rec: - continue - path = rec["path"] - if path in out: - raise RuntimeError(f"{path}: two text nodes") - c = advui._need_component(n, advui.TMP_CLASSES) - t = {k: c[k] for k in advui.TMP_FIELDS if k in c} - t.update( - { - "class": c["class"], - "enabled": c["m_Enabled"], - "fontAsset": c["m_fontAsset"]["name"], - "material": c["m_sharedMaterial"]["material"], - } - ) - t.update(storyfonts.text_extras(path, n["components"])) - t["localized"] = textstyle.localize_text(path, t["fontAsset"], t["material"], swap, lang, mode) - out[path] = t - return out - - -def story_text_materials(story_fonts: dict | None, lang: dict, mode: int) -> dict[str, set]: - """{localized game font asset: its localized game text materials} of the story UI's texts (the `texts` of - the story's ui/fonts.json, open fonts).""" - if not story_fonts or story_fonts.get("source") != "open": - return {} - swap, out = textstyle.font_swap(lang, mode), {} - for path, t in story_fonts.get("texts", {}).items(): - loc = textstyle.localize_text(path, t["fontAsset"], t["material"], swap, lang, mode) - out.setdefault(loc["fontAsset"], set()).add(loc["material"]) - return out - - -def open_fonts( - ui: UiDoc, - cat, - player, - episode: dict, - story_ui: dict, - language: str, - font, - master_dir, - texts: list[str], - story_fonts: dict | None = None, - font_of=storyfonts.no_font_file, -) -> dict: - """ui/simple/fonts.json and ui/simple/fonts/*.png from the font file `font` for the text nodes of `ui`, with the - story UI's font logic (storyfonts.open_font_set: the fallback chains, whose other font files `font_of` gives, - the missing glyph, the fallback materials): the texts are storyfonts.shown_texts over the story and its own - ui/ui.json (`story_ui`), the MasterText rows of the simple UI's text keys and `texts`. The glyph margin also - covers the story UI's text materials of the same font (`story_fonts`, the story's ui/fonts.json), so the glyph - pages equal the story UI's when the characters do.""" - tmpfont.require_extra() - mode, field = languages.mode(language), languages.column(language)[1:] - lang = textstyle.language_fonts(player) - bindings = text_bindings(ui, lang, mode) - if ui.doc is None: - raise RuntimeError("the simple UI's texts are read from its written ui.json") - shown = ( - storyfonts.shown_texts(episode, story_ui, master_dir, field) - + storyfonts.shown_texts(NO_EPISODE, ui.doc, master_dir, field) - + [s for s in texts if isinstance(s, str)] - ) - fset = storyfonts.open_font_set( - cat, - player, - ui.ex, - language, - font, - list(bindings.values()), - shown, - ui.dir, - font_of, - margin_materials=story_text_materials(story_fonts, lang, mode), - ) - materials = fset["materials"] - keywords = storyfonts.text_shaders(ui.ex, ui.dir, materials) - chars = {ord(ch) for s in shown for ch in s} - coverage = {"characters": len(chars), "missing": [chr(u) for u in sorted(fset["missing"])]} - if fset["substituted"]: - coverage["missingGlyph"] = [chr(u) for u in sorted(fset["substituted"])] - doc = { - "format": storyfonts.FONTS_FORMAT, - "language": language, - "source": "open", - "fonts": fset["fonts"], - "textures": fset["textures"], - "materials": materials, - "materialKeywords": keywords, - "texts": bindings, - "coverage": coverage, - "lineBreaking": storyfonts.line_breaking(player), - } - (ui.dir / storyfonts.FONTS_DOC).write_bytes( - dumps(doc, ensure_ascii=False, indent=1, sort_keys=True).encode("utf-8") + b"\n" - ) - return { - "fonts": sorted(fset["fonts"]), - "characters": len(chars), - "missing": len(fset["missing"]), - "pages": len(fset["textures"]), - } - - -def simple_ui( - cat, - master_dir, - player, - episode: dict, - ldir: Path, - language: str, - keys: dict, - host: dict, - title_id: str | None, - font, - font_of=storyfonts.no_font_file, -) -> dict: - """/ui/simple/: ui.json (UISimpleAdvTalkWindow, the window SimpleAdvPlayer.LoadInitialTalkWindow - attaches; for an area talk UISystemMessageWidget, which SystemMessageManager.ShowMessageByTextIdAsync shows the - spot's title with, and `texts.areaTitle`) and its open fonts.""" - sdir = Path(ldir) / "ui" / SIMPLE_DIR - story_ui = json.loads((Path(ldir) / "ui" / "ui.json").read_text(encoding="utf-8")) - fonts_doc = Path(ldir) / "ui" / storyfonts.FONTS_DOC - story_fonts = json.loads(fonts_doc.read_text(encoding="utf-8")) if fonts_doc.is_file() else None - ui = UiDoc(cat, player, sdir, languages.mode(language)) - ui.add(keys["talkWindow"], ui.prefab(keys["talkWindow"])) - texts = {} - if host.get("talk") == "area": - ui.add(keys["systemMessage"], ui.prefab(keys["systemMessage"])) - field = languages.column(language) - row = _rows(master_dir, "MasterText").get(title_id) - if row is None: - raise KeyError(f"area talk title {title_id} not in MasterText") - texts["areaTitle"] = row.get(field) or "" - ui.write( - f"SimpleAdvPlayer talk window UI data, {language}", - head={"language": advui.language_doc(language)}, - tail={"texts": texts} if texts else None, - ) - return open_fonts( - ui, cat, player, episode, story_ui, language, font, master_dir, list(texts.values()), story_fonts, font_of - ) - - -# ---------------------------------------------------------------- entry -def build( - cat, - master_dir, - player, - episode: dict, - story_dir, - language_dirs: dict, - groups: list, - *, - fonts: str = "open", - font_files: dict | None = None, - font_of=None, -) -> dict | None: - """The host data of `episode` (episode.json of story.build) when it is an Overlay episode (master - `_playbackMode` 1; else None and nothing written): /host/ (host.json, ui/, spot/, shaders/) and in - each language directory of `language_dirs` {language: dir with the story's ui/ui.json} its ui/simple/. - `groups`: the story groups of the episode (storysite.story_groups), which name the host (host_of); `fonts`: - "open" with `font_files` {language: storyfonts.FontFile} and `font_of` (language -> FontFile, the fallback - chains' other font files; default: those of `font_files`). -> {kind, doc, ui} (the story manifest's `host`).""" - if fonts not in FONT_MODES: - raise ValueError(f"fonts must be one of {FONT_MODES}, not {fonts!r}") - if (episode.get("master") or {}).get("_playbackMode") != OVERLAY: - return None - host = host_of(groups) - if fonts == "game": - raise NotImplementedError("the talk window UI of an Overlay story has open fonts only (fonts='open')") - font_files = dict(font_files or {}) - lost = [lang for lang in language_dirs if lang not in font_files] - if lost: - raise ValueError(f"no font file for {lost}") - - def own_font(code): - return font_files[code] if code in font_files else storyfonts.no_font_file(code) - - font_of = font_of or own_font - keys: dict = {k: emb_key(cat, ADDRESSES[k]) for k in ("cameraTarget", "talkWindow")} - if host.get("talk") == "area": - keys["systemMessage"] = emb_key(cat, ADDRESSES["systemMessage"]) - for k in KIND_KEYS[host["kind"]]: - try: - keys[k] = emb_key(cat, ADDRESSES[k]) - except KeyError: - if k != "spotScene": - raise - keys[k] = None # scene root: null / [] (scene_root) - hdir = Path(story_dir) / HOST_DIR - shutil.rmtree(hdir, ignore_errors=True) - ui = UiDoc(cat, player, hdir / "ui") - cam = camera_target(ui.prefab(keys["cameraTarget"])) - home = afterlive = None - if host["kind"] == "home": - overlay, layout, opened = home_ui(ui, keys) - home = home_host(cat, master_dir, player, host, hdir, keys) - else: - overlay, layout, opened, afterlive = afterlive_ui(ui, keys) - ui.write(f"host screen UI data of an Overlay story ({host['kind']}), no texts") - doc = { - "format": FORMAT, - "about": "Host screen of an Overlay story (SimpleAdvPlayer): the slot capture camera, the SimpleAdv " - "root and its layout, the host UI and the home spot or live result screen it is shown over", - "advId": episode.get("advId"), - "kind": host["kind"], - "group": host["group"], - "keys": keys, - "cameraTarget": cam, - "overlayRoot": overlay, - "layoutRoot": layout, - "openedSequences": opened, - "ui": HOST_UI_DOC, - "home": home, - "afterlive": afterlive, - } - write_json(hdir / "host.json", doc) - for language, ldir in language_dirs.items(): - shutil.rmtree(Path(ldir) / "ui" / SIMPLE_DIR, ignore_errors=True) - simple_ui( - cat, - master_dir, - player, - episode, - Path(ldir), - language, - keys, - host, - (home or {}).get("titleTextId"), - font_files[language], - font_of, - ) - return {"kind": host["kind"], "doc": HOST_DOC, "ui": SIMPLE_DOC} diff --git a/src/nnnotes/content/tmpassets.py b/src/nnnotes/content/tmpassets.py index 8f11b4d..d134750 100644 --- a/src/nnnotes/content/tmpassets.py +++ b/src/nnnotes/content/tmpassets.py @@ -1,19 +1,7 @@ -"""TextMesh Pro font assets read from an object table (content/prefabs.Objects). +"""TextMesh Pro fonts, materials and texture pages read from shared object tables. -The records tmpfont.py reads through an Exporter, read from the object layer instead: a font asset's fields from its -`json`, its source font file from its `font` file, its atlas pages from their `image` files (PNG of the decoded image), -its materials from their `json` (the shader by name, the textures as texture records). - - Texture a Texture2D of the table: its record and its RGBA texels - Font a TMP_FontAsset of the table (the fields tmpfont.Font reads) - FontSet the font assets reached from a font asset, with the TMP character lookup of tmpfont.FontLookup - build_font_settings the font settings table (fontsettings.FORMAT) read from the table - missing_glyph_character - the game's TMP Settings resource (tmpfont.missing_glyph_character) - -The glyph tables and runtime glyphs are the tmpfont functions (export_fonts, runtime_glyphs), which take an -export.MemoryPacker to hold the packed pages in memory. Reading a font needs the `fonts` extra (tmpfont.require_extra). -""" +Font and FontSet implement the glyph-generation protocols in unity.tmpfont. Texture supplies decoded RGBA +texels to unity.texels.MemoryPacker. build_font_settings assembles the localized font settings table.""" from __future__ import annotations @@ -26,9 +14,9 @@ import numpy as np from PIL import Image -from ..unity.export import texel_rows from ..unity.fontsettings import FONT_CLASS, font_key from ..unity.ids import parse_object_id +from ..unity.texels import texel_rows from ..unity.tmpfont import ( MISSING_GLYPH_CHARACTER, BuiltFontSettings, @@ -98,7 +86,7 @@ def pixels(self) -> np.ndarray: return self._pixels def record(self) -> dict: - """The record of a material's texture reference (export.Exporter.texture in deferred mode).""" + """The texture reference and its dimensions and sampler settings.""" return { "textureRef": self.oid, "name": self.name, @@ -157,7 +145,7 @@ def atlas_settings(self) -> list[dict]: return [a.settings for a in self.atlases if a is not None] def default_material(self) -> dict: - """The record of the default material (export.Exporter.material).""" + """The font asset's default material record.""" return self._fonts.material(self.material_oid) @@ -198,7 +186,7 @@ def of(self, oid: str) -> Font: return self._by_oid[oid] def material(self, oid: str) -> dict: - """The record of the Material `oid` (export.Exporter.material): its shader by name, its textures as texture + """The record of the Material `oid`: its shader by name, its textures as texture records, its other fields as in its `json`.""" doc = self.objects.json(oid) shader = _ref(doc["shader"]) @@ -234,7 +222,7 @@ def named_material(self, font: FontAsset, name: str) -> dict: class TableFontSettings(BuiltFontSettings): """FontSettings read from the font assets of an object table (FontSet): each font asset's bundle from - `bundle_of(name)`, the records as tmpfont.AssetFontSettings reads them from the game's assets.""" + `bundle_of(name)`, the records required by the font settings table.""" def __init__(self, fonts: FontSet, bundle_of: Callable[[str], str]): super().__init__() diff --git a/src/nnnotes/musicdata/__init__.py b/src/nnnotes/musicdata/__init__.py index 53e6fce..74cf1c5 100644 --- a/src/nnnotes/musicdata/__init__.py +++ b/src/nnnotes/musicdata/__init__.py @@ -244,16 +244,12 @@ def jacket_webp(image, size: int = JACKET_SIZE) -> bytes: def catalog_jacket(cat, size: int = JACKET_SIZE) -> Callable[[str], bytes]: """jacket(name) for a catalog: the WebP bytes (jacket_webp) of the Texture2D `Image/Jacket/`.""" - import tempfile - - from ..unity.export import Exporter - - ex = Exporter(cat, Path(tempfile.gettempdir()), textures="deferred") + from ..unity.reader import container_asset, load_closure def jacket(name: str) -> bytes: key = JACKET_KEY.format(jacket=name) try: - o = ex.key_object(key) + o = container_asset(load_closure(cat.fetch_key(key)), cat.entry(key)["internal_id"]) except KeyError: raise MusicDataError(f"jacket {name}: no asset {key}") from None except Exception as e: diff --git a/src/nnnotes/unity/export.py b/src/nnnotes/unity/export.py deleted file mode 100644 index 8ba2707..0000000 --- a/src/nnnotes/unity/export.py +++ /dev/null @@ -1,1622 +0,0 @@ -"""Typetree export with every reference resolved. - -`Exporter` turns Unity objects into JSON-ready values: textures become PNG -files (or, in deferred mode, references whose texels a `TexelPacker` copies -later), sprites carry their mesh and UVs (texture-bound or packed in a -SpriteAtlas), materials carry shader name, keywords and properties, meshes carry -vertex data (optional), ScriptableObject MonoBehaviours are inlined once, and -references to GameObjects, Transforms and components become hierarchy paths. -Prefabs and scenes are exported as ordered node lists (hierarchy order, -components in GameObject order). AnimationClips carry the Mecanim clip data the -Animator samples (streamed cubic keys, dense samples, constants) with their -bindings resolved to relative paths and property names; AnimatorControllers -carry layers, states, transitions and parameters. `export_key` exports any -addressable key by the type of its container asset. - -Clip, binding and controller JSON comes in several shapes (`clip_format`: -"generic", or "livescene" / "livenotes" for the live scene and live notes exports; -`clip_curves` and `controller_states` for the ADV UI). All of them are views of one record -(`clip_record` / `controller_record`), so resolution lives here only. -""" - -from __future__ import annotations - -import hashlib -import io -import json -import math -import re -from pathlib import Path -from typing import Literal - -import numpy as np -from PIL import Image -from UnityPy.files import ObjectReader -from UnityPy.helpers import MeshHelper - -from ..core import cache -from ..source.catalog import Catalog -from .bindings import ENGINE_PROPS, RECT_PROPS, crc, flat_names, material_attribute, material_prop_names -from .convert.animation import CLASS_ID, TRANSFORM_ATTR, controller_doc, streamed_frames -from .convert.material import material_doc -from .reader import ( - DEFAULT_RESOURCES, - SceneGraph, - check_texture, - deref, - external_path, - is_pptr, - load_closure, - mesh_arrays, - script_class, - texture_image, -) - -HEADER = ("m_GameObject", "m_Script") -_SCALARS = (int, float, str, bool, type(None)) -_LIBS = ("UnityPy", "Pillow", "texture2ddecoder", "astc-encoder-py", "etcpak", "numpy") -# decoded textures and PNG encodings memoized by content within this process -PNG = cache.bucket("png", salt=cache.source_salt(__file__, *_LIBS)) -PIXELS = cache.bucket("pixels", salt=cache.source_salt(__file__, *_LIBS)) - - -def _decode_inputs(tex) -> tuple: - """What UnityPy's Texture2D.image depends on (Texture2DConverter.get_image_from_texture2d): image data, size, - format, version, platform, platform blob.""" - r = tex.object_reader - blob = getattr(tex, "m_PlatformBlob", None) - return ( - bytes(tex.get_image_data()), - tex.m_Width, - tex.m_Height, - int(tex.m_TextureFormat), - tuple(getattr(r, "version", (0, 0, 0, 0))), - int(getattr(r, "platform", 0) or 0), - None if blob is None else bytes(blob), - ) - - -def texture_png(tex) -> bytes: - """PNG bytes of a Texture2D's decoded image as Pillow writes it by default (`tex.image.save(PNG)`), cached by - the decoder's inputs.""" - check_texture(tex) - k = PNG.key("texture", _decode_inputs(tex)) - data = PNG.get(k) - if data is None: - buf = io.BytesIO() - tex.image.save(buf, format="PNG") - data = buf.getvalue() - PNG.put(k, data) - return data - - -def packed_png(rgba: np.ndarray) -> bytes: - """PNG bytes (Pillow, optimize=True) of an RGBA array (bottom row first), cached by the array's content.""" - rgba = np.ascontiguousarray(rgba, np.uint8) - k = PNG.key("packed", rgba.shape, rgba.data) - data = PNG.get(k) - if data is None: - buf = io.BytesIO() - Image.fromarray(rgba[::-1].copy(), "RGBA").save(buf, format="PNG", optimize=True) - data = buf.getvalue() - PNG.put(k, data) - return data - - -def _safe(s: str) -> str: - return re.sub(r"[^A-Za-z0-9_.-]+", "_", s) - - -def _key(o) -> tuple[str, int]: - return (o.assets_file.name, o.path_id) - - -def _rid(o, name: str) -> str: - """Stable id for an object exported into a registry: name + hash of (file, path id).""" - return f"{name}#{hashlib.sha1(f'{o.assets_file.name}:{o.path_id}'.encode()).hexdigest()[:8]}" - - -def _events(tt: dict) -> list[dict]: - return [ - {x: e[x] for x in ("time", "functionName", "data", "floatParameter", "intParameter", "messageOptions")} - for e in tt["m_Events"] - ] - - -# --- texel blocks -> packed textures --------------------------------------------- -def grow_box(x0: float, y0: float, x1: float, y1: float, margin: int) -> tuple[int, int, int, int]: - """Texel box covering [x0, x1) x [y0, y1) grown by `margin` texels on every side.""" - return (math.floor(x0) - margin, math.floor(y0) - margin, math.ceil(x1) + margin, math.ceil(y1) + margin) - - -def wrap_mode(mode: int) -> Literal["wrap", "clip"]: - """The np.take mode of a texture's wrap mode (m_WrapU / m_WrapV: 0 repeat, 1 clamp).""" - if mode == 0: - return "wrap" - if mode == 1: - return "clip" - raise NotImplementedError(f"texture wrap mode {mode}") - - -def texel_rows(img, texture_format: int, width: int, height: int, name: str) -> np.ndarray: - """The RGBA texels of a decoded Texture2D image `img` (a PIL image; the texture's format, size and name), bottom - row first. Alpha8 keeps its value in the alpha channel.""" - if texture_format == 1: # Alpha8: value in the alpha channel - if img.mode == "L": - a = np.asarray(img, np.uint8) - elif img.mode == "RGBA": - a = np.asarray(img, np.uint8)[:, :, 3] - else: - raise NotImplementedError(f"Alpha8 decoded as {img.mode}") - arr = np.zeros(a.shape + (4,), np.uint8) - arr[:, :, 3] = a # rgb 0 (GL alpha texture); TMP reads .a only - else: - arr = np.asarray(img.convert("RGBA"), np.uint8) - if arr.shape[:2] != (height, width): - raise RuntimeError(f"{name}: decoded size {arr.shape}") - return arr[::-1].copy() # bottom row first (Unity v = 0) - - -def pack_group(group: str, handles: list[dict], pixels, width_limit: int) -> tuple[dict, np.ndarray]: - """The layout and the texels of one packed texture `group` of the blocks `handles` (TexelPacker.build). Returns its - record without its file (name, width, height, mipCount, settings, sources, blocks) and its RGBA array, bottom row - first. `pixels(tex)` gives the texels of a block's source texture. Every handle gets its texture (`group`) and its - integer `offset`.""" - settings = {json.dumps(h["settings"], sort_keys=True) for h in handles} - if len(settings) != 1: - raise RuntimeError(f"{group}: blocks from textures with different sampler settings") - # dedupe identical blocks of the same texture - uniq: dict[tuple, dict] = {} - for h in handles: - uniq.setdefault((h["key"], h["box"]), h) - blocks = sorted(uniq.values(), key=lambda h: (-(h["box"][3] - h["box"][1]), h["box"])) - x = y = shelf = 0 - width = max(width_limit, max(b["box"][2] - b["box"][0] for b in blocks)) - for b in blocks: - w, hgt = b["box"][2] - b["box"][0], b["box"][3] - b["box"][1] - if x + w > width: - x, y, shelf = 0, y + shelf, 0 - b["dst"] = (x, y) - x += w - shelf = max(shelf, hgt) - height = y + shelf - if len(blocks) == 1: # a single block keeps its own size - width = blocks[0]["box"][2] - blocks[0]["box"][0] - out = np.zeros((height, width, 4), np.uint8) - for b in blocks: - x0, y0, x1, y1 = b["box"] - dx, dy = b["dst"] - if "array" in b: - out[dy : dy + (y1 - y0), dx : dx + (x1 - x0)] = b["array"] - continue - src = pixels(b["tex"]) - st = b["settings"] - rows = np.take(np.arange(src.shape[0]), np.arange(y0, y1), mode=wrap_mode(st["m_WrapV"])) - cols = np.take(np.arange(src.shape[1]), np.arange(x0, x1), mode=wrap_mode(st["m_WrapU"])) - out[dy : dy + (y1 - y0), dx : dx + (x1 - x0)] = src[np.ix_(rows, cols)] - for h in handles: - d = uniq[(h["key"], h["box"])]["dst"] - h["offset"] = (d[0] - h["box"][0], d[1] - h["box"][1]) - h["texture"] = group - s = next(iter(handles))["settings"] - rec = { - "name": group, - "width": width, - "height": height, - "mipCount": 1, - "settings": s, - "sources": sorted({h["srcName"] for h in handles}), - "blocks": len(blocks), - } - return rec, out - - -def texture_block( - key: tuple, settings: dict, name: str, size: tuple[int, int], box: tuple[int, int, int, int], tex -) -> dict: - """A block that copies the texel box `box` of the texture `tex` (its texels come from the packer's `pixels`). - `key` identifies the texture, `settings` is its m_TextureSettings, `name` and `size` (width, height) its name and - size.""" - return {"tex": tex, "key": key, "settings": settings, "box": box, "srcName": name, "srcSize": size} - - -def array_block(name: str, rgba: np.ndarray, origin: tuple[int, int], settings: dict) -> dict: - """A block of generated texels: `rgba` (RGBA, bottom row first) placed at `origin` of its virtual texture.""" - ox, oy = origin - return { - "array": rgba, - "key": ("generated", name), - "settings": settings, - "box": (ox, oy, ox + rgba.shape[1], oy + rgba.shape[0]), - "srcName": name, - "srcSize": (rgba.shape[1], rgba.shape[0]), - } - - -class TexelPacker: - """Copies texel blocks of source textures into packed textures. - - Block coordinates are Unity texel coordinates (origin bottom-left). A block - [x0, x1) x [y0, y1) is read with the source texture's wrap mode, so texels - a sampler would fetch across an edge are the ones copied. Blocks keep their - size; a block's UVs move by the integer translation `offset` (in texels), so - sampling a packed texture returns the same texel values as the source. - """ - - WIDTH = 2048 - - def __init__(self, out_dir: Path): - self.out = Path(out_dir) - self._pixels: dict[tuple, np.ndarray] = {} - self.groups: dict[str, list] = {} - - def pixels(self, tex_obj) -> np.ndarray: - """The texture's texels as RGBA, bottom row first (read-only; decoded once per content, see PIXELS).""" - k = _key(tex_obj) - if k not in self._pixels: - tt = tex_obj.read_typetree() - tex = tex_obj.read() - ck = PIXELS.key(_decode_inputs(tex), tt["m_TextureFormat"] == 1, tt["m_Width"], tt["m_Height"]) - arr = PIXELS.get(ck) - if arr is None: - arr = texel_rows(texture_image(tex), tt["m_TextureFormat"], tt["m_Width"], tt["m_Height"], tt["m_Name"]) - arr.setflags(write=False) - PIXELS.put(ck, arr, arr.nbytes) - self._pixels[k] = arr - return self._pixels[k] - - def add(self, group: str, tex_obj, x0: int, y0: int, x1: int, y1: int) -> dict: - """Register a block; returns a handle filled in by build().""" - tt = tex_obj.read_typetree() - size = (tt["m_Width"], tt["m_Height"]) - h = texture_block(_key(tex_obj), tt["m_TextureSettings"], tt["m_Name"], size, (x0, y0, x1, y1), tex_obj) - self.groups.setdefault(group, []).append(h) - return h - - def add_array(self, group: str, name: str, rgba: np.ndarray, origin: tuple[int, int], settings: dict) -> dict: - """Register generated texels (RGBA, bottom row first) covering [origin, origin + shape) of a - virtual texture as one block; the whole array is copied (no wrap).""" - h = array_block(name, rgba, origin, settings) - self.groups.setdefault(group, []).append(h) - return h - - def build(self) -> dict: - textures = {} - for group, handles in self.groups.items(): - rec, out = pack_group(group, handles, self.pixels, self.WIDTH) - fn = f"textures/{_safe(group)}.png" - (self.out / "textures").mkdir(parents=True, exist_ok=True) - (self.out / fn).write_bytes(packed_png(out)) - textures[group] = {"texture": fn, **rec} - return textures - - -class MemoryPacker: - """TexelPacker over texture objects held in memory instead of Unity objects: a block's texture has `key`, `name`, - `settings` (m_TextureSettings), `width`, `height` and `pixels` (its RGBA texels, bottom row first). `build()` gives - the packed textures' records and their PNG bytes by file name, the same records and bytes TexelPacker writes.""" - - WIDTH = TexelPacker.WIDTH - - def __init__(self): - self.groups: dict[str, list] = {} - - @staticmethod - def pixels(tex) -> np.ndarray: - return tex.pixels - - def add(self, group: str, tex, x0: int, y0: int, x1: int, y1: int) -> dict: - """Register a block; returns a handle filled in by build().""" - h = texture_block(tex.key, tex.settings, tex.name, (tex.width, tex.height), (x0, y0, x1, y1), tex) - self.groups.setdefault(group, []).append(h) - return h - - def add_array(self, group: str, name: str, rgba: np.ndarray, origin: tuple[int, int], settings: dict) -> dict: - """Register generated texels as TexelPacker.add_array does.""" - h = array_block(name, rgba, origin, settings) - self.groups.setdefault(group, []).append(h) - return h - - def build(self) -> tuple[dict, dict[str, bytes]]: - """The records of the packed textures by group, and the PNG bytes of their pages by file name.""" - textures: dict[str, dict] = {} - pages: dict[str, bytes] = {} - for group, handles in self.groups.items(): - rec, out = pack_group(group, handles, self.pixels, self.WIDTH) - fn = f"textures/{_safe(group)}.png" - pages[fn] = packed_png(out) - textures[group] = {"texture": fn, **rec} - return textures, pages - - -CLIP_FORMATS = ("generic", "livescene", "livenotes") - - -class FontStub: - """A reference into a font bundle left out of a closure (Exporter `font_stubs`): the record of its object in - the font settings' `objects` ({type, class, name}).""" - - def __init__(self, rec: dict, path_id: int): - self.rec, self.path_id = rec, path_id - - -class Exporter: - """`player` resolves references into `unity default resources` (PlayerData); - `inline_meshes=False` emits meshes as name + vertex count; MonoBehaviour - classes in `stub_assets` (assets exported elsewhere, e.g. motions) are - emitted as {asset, name} only. - - `textures="deferred"`: textures and sprites become references - ({textureRef} / {spriteRef}) and nothing is written; `tex_objs` / `sprite_recs` - hold the objects, and `pack_sprite` / `packed_sprite` with a `TexelPacker` copy - only the texel blocks a draw can sample. - `clip_format`: the JSON shape of clips and bindings ("generic": inlined once, - bindings {path, typeID, class, attribute}; "livescene": inlined once, bindings with - crc32 keys, candidates and curve counts; "livenotes": clips and controllers in the - `clips` / `controllers` registries, referenced by id). - `follow`: referenced kinds exported instead of named ("AnimatorController", - "SpriteAtlas"). - `defer_writes` (textures "write"): texture records are made as usual but their PNG files are written only by - `write_textures`, which can leave out the files a caller would delete. - `class_of(o)`: the class name of a MonoBehaviour (default: unity.script_class, through its m_Script with the - script bundle loaded). - `font_stubs` (fontsettings.FontSettings): the bundles of its font assets are left out of every closure; a - reference into them resolves through its `objects` (a font asset to {asset, name}, a material to its record).""" - - def __init__( - self, - cat: Catalog, - out_dir: Path, - player=None, - inline_meshes: bool = True, - stub_assets: tuple[str, ...] = (), - textures: str = "write", - clip_format: str = "generic", - follow: tuple[str, ...] = (), - defer_writes: bool = False, - class_of=None, - font_stubs=None, - ): - if textures not in ("write", "deferred"): - raise ValueError(f"textures={textures!r}") - if clip_format not in CLIP_FORMATS: - raise ValueError(f"clip_format={clip_format!r}") - self.cat = cat - self.player = player - self.out = Path(out_dir) - self.inline_meshes = inline_meshes - self.stub_assets = stub_assets - self.deferred = textures == "deferred" - self.clip_format = clip_format - self.follow = tuple(follow) - self.defer_writes = defer_writes - self.script_class = class_of or script_class - self.font_stubs = font_stubs - self._font_bundles: frozenset | None = None - self._unwritten: dict[str, object] = {} # defer_writes: file -> Texture2D - self.textures: dict[tuple, dict] = {} - self.shaders: dict[str, ObjectReader] = {} - self.tex_objs: dict[str, object] = {} # deferred: "file:pathID" -> Texture2D - self.sprite_recs: dict[str, dict] = {} # deferred: "file:pathID" -> sprite render record - self.clips: dict[str, dict] = {} # livenotes format: id -> clip - self.controllers: dict[str, dict] = {} # livenotes format: id -> controller - self.material_props: set[str] = set() # float/colour property names of exported materials - self._graph_of_file: dict[int, SceneGraph] = {} - self._done: dict[tuple, dict] = {} # MonoBehaviour/clip/controller/atlas -> reference once exported - self._component_ref: dict[tuple, str] = {} - self._objects: dict[tuple, ObjectReader] = {} - self._go_file: dict[int, str] = {} - self._loaded: dict[str, tuple] = {} - self._clip_recs: dict[tuple, dict] = {} - self._atlas_maps: dict[tuple, tuple] = {} - self._names_of: dict[tuple, list] = {} # object -> flat field names - self._names_by_class: dict = {} # typeID | ("MonoBehaviour", class) -> names of exported components - self._class_of: dict[tuple, str | None] = {} - self._rel_tables: dict[tuple, dict] = {} - self._gos_at_cache: dict[tuple, list] = {} - self._path_sources: list[tuple[str, dict]] = [] - self._crcs: dict[str, int] = {} - - # --- loading --------------------------------------------------------- - def load(self, key: str): - """(env, graph) of a key's bundle closure, loaded once.""" - if key in self._loaded: - return self._loaded[key] - if self.font_stubs is None: - env = load_closure(self.cat.fetch_key(key)) - else: - env = load_closure(self.cat.fetch_key(key, self.font_bundles())) - graph = SceneGraph(env) - go_file = self._go_file - for o in env.objects: - self._graph_of_file.setdefault(id(o.assets_file), graph) - self._objects[_key(o)] = o - if o.type.name == "Shader": - self._shader(o) - elif o.type.name == "GameObject": - go_file[o.path_id] = o.assets_file.name - # Components living on GameObjects are exported where they sit in the - # hierarchy; references to them elsewhere become {component, gameObject}. - for go_pid, go in graph.go.items(): - path = graph.path(graph.tf_of_go[go_pid]) if go_pid in graph.tf_of_go else go["m_Name"] - for c in go["m_Component"]: - self._component_ref[(go_file[go_pid], c["component"]["m_PathID"])] = path - self._loaded[key] = (env, graph) - return env, graph - - def font_bundles(self) -> frozenset: - """The names of the bundles that hold the font assets of `font_stubs` (none without it).""" - if self._font_bundles is None: - self._font_bundles = frozenset() if self.font_stubs is None else self.font_stubs.bundles(self.cat) - return self._font_bundles - - def font_stub(self, o: FontStub) -> dict: - """The value of a reference into a font bundle left out (`font_stubs`): a material's record, a font asset's - {asset, name}.""" - if o.rec["type"] == "Material" and self.font_stubs is not None: - return self.font_stubs.material(None, o.rec["name"]) - return {"asset": o.rec["class"], "name": o.rec["name"]} - - def deref(self, owner, pptr: dict) -> ObjectReader | None: - """The object `pptr` (read from `owner`'s typetree) names; None for a null reference. A reference into a font - bundle left out (`font_stubs`) is an error here: only `ref` writes such a reference, as its record.""" - o = self._lookup(owner, pptr) - if isinstance(o, FontStub): - raise FileNotFoundError( - f"path id {o.path_id} is in a font bundle left out of the closure ({o.rec['name']})" - ) - return o - - def referent(self, owner, pptr: dict) -> ObjectReader: - """The object a reference that must not be null names (KeyError naming `owner` for a null one).""" - o = self.deref(owner, pptr) - if o is None: - raise KeyError(f"{owner.type.name} {owner.path_id}: a required reference is null") - return o - - def _lookup(self, owner, pptr: dict) -> ObjectReader | FontStub | None: - if not pptr["m_PathID"]: - return None - if external_path(owner, pptr) == DEFAULT_RESOURCES: - if self.player is None: - raise RuntimeError("reference into unity default resources needs player data") - return self.player.deref(owner, pptr) - try: - return deref(owner, pptr) - except FileNotFoundError: - rec = self.font_stubs.objects.get(pptr["m_PathID"]) if self.font_stubs is not None else None - if rec is None: - raise - return FontStub(rec, pptr["m_PathID"]) - - def graph(self, o) -> SceneGraph: - return self._graph_of_file[id(o.assets_file)] - - # --- values ---------------------------------------------------------- - def value(self, owner, v): - if is_pptr(v): - return self.ref(owner, v) - if isinstance(v, dict): - return {k: self.value(owner, x) for k, x in v.items()} - if isinstance(v, list): - if all(type(x) in _SCALARS for x in v): # numbers / strings: nothing to resolve - return list(v) - return [self.value(owner, x) for x in v] - if isinstance(v, (bytes, bytearray)): - return v.hex() - return v - - def ref(self, owner, pptr: dict): - o = self._lookup(owner, pptr) - if o is None: - return None - if isinstance(o, FontStub): - return self.font_stub(o) - t = o.type.name - if t in ("AnimatorController", "AnimatorOverrideController") and "AnimatorController" in self.follow: - return self.controller(o) - if t == "SpriteAtlas" and "SpriteAtlas" in self.follow: - return self.atlas(o) - if t == "Texture2D": - return self.texture(o) - if t == "Sprite": - return self.sprite(o) - if t == "Material": - return self.material(o) - if t == "Shader": - return {"shader": self._shader(o)} - if t == "Mesh": - return self.mesh(o) - if t == "AnimationClip": - return self.clip(o) - if t == "GameObject": - g = self.graph(o) - return {"gameObject": g.path(g.tf_of_go[o.path_id])} - if t in ("Transform", "RectTransform"): - return {"transform": self.graph(o).path(o.path_id)} - if _key(o) in self._component_ref: - desc: dict[str, str | None] = {"component": t, "gameObject": self._component_ref[_key(o)]} - if t == "MonoBehaviour": - desc["class"] = self.script_class(o) - return desc - if t == "MonoBehaviour": - return self.scriptable(o) - if t == "MonoScript": - ms = o.read() - return {"script": f"{ms.m_Namespace}.{ms.m_ClassName}".lstrip(".")} - return {"object": t, "name": getattr(o.read(), "m_Name", None)} - - # --- shaders, textures, sprites, materials, meshes --------------------- - def add_shader(self, o) -> str: - """Register a Shader object (dumped by the caller from `shaders`); returns its name.""" - name = o.read().m_ParsedForm.m_Name - self.shaders.setdefault(name, o) - return name - - _shader = add_shader - - def texture(self, o) -> dict: - tt = o.read_typetree() - if self.deferred: - ref = f"{o.assets_file.name}:{o.path_id}" - self.tex_objs[ref] = o - return { - "textureRef": ref, - "name": tt["m_Name"], - "width": tt["m_Width"], - "height": tt["m_Height"], - "format": tt["m_TextureFormat"], - "mipCount": tt.get("m_MipCount", 1), - "settings": tt.get("m_TextureSettings"), - } - k = _key(o) - if k not in self.textures: - tex = o.read() - h = hashlib.sha1(f"{k[0]}:{k[1]}".encode()).hexdigest()[:8] - fn = f"textures/{_safe(tt['m_Name'])}-{h}.png" - (self.out / "textures").mkdir(parents=True, exist_ok=True) - if self.defer_writes: - self._unwritten[fn] = tex - else: - (self.out / fn).write_bytes(texture_png(tex)) - self.textures[k] = { - "texture": fn, - "name": tt["m_Name"], - "width": tt["m_Width"], - "height": tt["m_Height"], - "format": tt["m_TextureFormat"], - "mipCount": tt.get("m_MipCount", 1), - "colorSpace": tt.get("m_ColorSpace"), - "settings": tt.get("m_TextureSettings"), - } - return self.textures[k] - - def write_textures(self, skip=()) -> list[str]: - """defer_writes: write the PNG file of every texture recorded since the last call, except the files in - `skip` (paths relative to the output directory, "textures/.png"). Returns the files written.""" - skip, written = set(skip), [] - for fn, tex in sorted(self._unwritten.items()): - if fn not in skip: - (self.out / fn).write_bytes(texture_png(tex)) - written.append(fn) - self._unwritten.clear() - return written - - def sprite_render_data(self, o, tt: dict | None = None): - """(render data, texture owner, atlas typetree) of a Sprite: its own m_RD when it is bound to a - texture; for a sprite packed into a SpriteAtlas (m_RD.texture null) the atlas' m_RenderDataMap entry - under the sprite's m_RenderDataKey, which holds texture, textureRect and uvTransform (Unity - SpriteAtlas runtime binding). (None, None, None) when the sprite has neither.""" - tt = tt or o.read_typetree() - rd = tt["m_RD"] - if rd["texture"]["m_PathID"]: - return rd, o, None - atlas_o = self.deref(o, tt["m_SpriteAtlas"]) - if atlas_o is None: - return None, None, None - k = _key(atlas_o) - if k not in self._atlas_maps: - at = atlas_o.read_typetree() - index: dict[tuple, list] = {} - for rk, d in at["m_RenderDataMap"]: - index.setdefault((json.dumps(rk[0], sort_keys=True), rk[1]), []).append(d) - self._atlas_maps[k] = (at, index) - at, index = self._atlas_maps[k] - key = tt["m_RenderDataKey"] - hits = index.get((json.dumps(key[0], sort_keys=True), key[1]), []) - if len(hits) != 1: - raise RuntimeError(f"sprite {tt['m_Name']}: {len(hits)} atlas render data entries") - return hits[0], atlas_o, at - - def sprite(self, o) -> dict: - tt = o.read_typetree() - src, owner, at = self.sprite_render_data(o, tt) - if self.deferred: - return self._deferred_sprite(o, tt, src, owner) - if src is None: - return {"sprite": tt["m_Name"], "texture": None, "unboundAtlasTags": tt.get("m_AtlasTags")} - tex = self.texture(self.referent(owner, src["texture"])) - rd = tt["m_RD"] - vd = rd["m_VertexData"] - raw = vd["m_DataSize"] - raw = raw if isinstance(raw, (bytes, bytearray)) else bytes(raw) - n = vd["m_VertexCount"] - ch0 = vd["m_Channels"][0] - if ch0["format"] != 0 or ch0["dimension"] != 3 or ch0["stream"] != 0 or ch0["offset"] != 0: - raise NotImplementedError(f"sprite {tt['m_Name']}: position channel {ch0}") - pos = np.frombuffer(raw, " dict: - if src is None: - raise RuntimeError(f"sprite {tt['m_Name']}: no texture and no atlas") - tex = self.referent(owner, src["texture"]) - if src["alphaTexture"]["m_PathID"] or src.get("secondaryTextures"): - raise NotImplementedError(f"sprite {tt['m_Name']}: alpha/secondary textures") - raw = src["settingsRaw"] - if (raw >> 2) & 0xF: - raise NotImplementedError(f"sprite {tt['m_Name']}: packing rotation {(raw >> 2) & 0xF}") - if src.get("downscaleMultiplier", 1.0) != 1.0: - raise NotImplementedError(f"sprite {tt['m_Name']}: downscaleMultiplier") - ref = f"{o.assets_file.name}:{o.path_id}" - self.tex_objs[f"{tex.assets_file.name}:{tex.path_id}"] = tex - self.sprite_recs[ref] = { - "name": tt["m_Name"], - "tex": tex, - "rect": tt["m_Rect"], - "border": tt["m_Border"], - "pivot": tt["m_Pivot"], - "pixelsPerUnit": tt["m_PixelsToUnits"], - "textureRect": src["textureRect"], - "textureRectOffset": src["textureRectOffset"], - "settingsRaw": raw, - } - return {"spriteRef": ref, "name": tt["m_Name"]} - - def pack_sprite(self, ref: str, packer: TexelPacker, group: str, margin: int = 1) -> None: - """Deferred mode: register the texels a sprite draw samples (its texture rect grown by `margin` - texels for bilinear filtering) as a block of `group`.""" - s = self.sprite_recs[ref] - tr = s["textureRect"] - s["_block"] = packer.add( - group, s["tex"], *grow_box(tr["x"], tr["y"], tr["x"] + tr["width"], tr["y"] + tr["height"], margin) - ) - - def packed_sprite(self, ref: str) -> dict: - """Deferred mode, after packer.build(): the sprite with its texture rect moved into the packed texture.""" - s = self.sprite_recs[ref] - b = s.pop("_block") - tr = s["textureRect"] - return { - "texture": b["texture"], - "rect": s["rect"], - "border": s["border"], - "pivot": s["pivot"], - "pixelsPerUnit": s["pixelsPerUnit"], - "settingsRaw": s["settingsRaw"], - "textureRect": { - "x": tr["x"] + b["offset"][0], - "y": tr["y"] + b["offset"][1], - "width": tr["width"], - "height": tr["height"], - }, - "textureRectOffset": s["textureRectOffset"], - "source": b["srcName"], - } - - def material(self, o) -> dict: - m = material_doc(o.read_typetree(), lambda p: self.ref(o, p)) - self.material_props.update(m["floats"]) - self.material_props.update(m["colors"]) - return m - - def mesh(self, o) -> dict: - m = o.read() - if not self.inline_meshes: - return {"mesh": m.m_Name, "vertexCount": m.m_VertexData.m_VertexCount} - arrays = mesh_arrays(m) - if arrays is None: - return {"mesh": m.m_Name, "empty": True} - v, n, uv, tris = arrays - out = { - "mesh": m.m_Name, - "vertices": v.tolist(), - "normals": n.tolist(), - "uv0": uv.tolist(), - "submeshes": [t.reshape(-1).tolist() for t in tris], - } - tt = o.read_typetree() - if tt.get("m_BindPose"): - out.update(self._skin(m, tt, len(v))) - return out - - @staticmethod - def _skin(m, tt: dict, vertex_count: int) -> dict: - """A skinned mesh's skin as serialized: `m_Skin` (BoneWeights4 per vertex: weight[4], boneIndex[4], from - the BlendWeight / BlendIndices vertex channels, or m_Skin in older layouts; unused slots 0), - `m_BindPose` (Matrix4x4f per bone, e00..e33), `m_BoneNameHashes`, `m_RootBoneNameHash`. Bone index i is - the renderer's m_Bones[i].""" - h = MeshHelper.MeshHandler(m) - h.process() - w, ix = h.m_BoneWeights, h.m_BoneIndices - if not w or not ix or len(w) != vertex_count or len(ix) != vertex_count: - raise RuntimeError(f"mesh {tt['m_Name']}: {len(w or [])} bone weights for {vertex_count} vertices") - skin = [] - for ws, bs in zip(w, ix): - if len(ws) > 4 or len(bs) != len(ws): - raise NotImplementedError(f"mesh {tt['m_Name']}: {len(ws)} weights / {len(bs)} bone indices per vertex") - pad = 4 - len(ws) - skin.append({"weight": [float(x) for x in ws] + [0.0] * pad, "boneIndex": [int(x) for x in bs] + [0] * pad}) - return { - "m_Skin": skin, - "m_BindPose": tt["m_BindPose"], - "m_BoneNameHashes": tt.get("m_BoneNameHashes", []), - "m_RootBoneNameHash": tt.get("m_RootBoneNameHash"), - } - - # --- field names (crc32 resolution) ------------------------------------ - def _crc_of(self, name: str) -> int: - c = self._crcs.get(name) - if c is None: - c = self._crcs[name] = crc(name) - return c - - def _names(self, o, tt: dict | None = None) -> list[str]: - """Flat field paths of an object's typetree, computed once.""" - k = _key(o) - if k not in self._names_of: - self._names_of[k] = flat_names(tt if tt is not None else o.read_typetree()) - return self._names_of[k] - - def _mono_class(self, o) -> str | None: - k = _key(o) - if k not in self._class_of: - try: - self._class_of[k] = self.script_class(o) - except Exception: # script not loadable from this closure - self._class_of[k] = None - return self._class_of[k] - - def _register_names(self, o, tt: dict) -> None: - """Field paths of an exported component, by class (names a clip binding can target).""" - k = ("MonoBehaviour", self._mono_class(o)) if o.type.name == "MonoBehaviour" else o.type.value - self._names_by_class.setdefault(k, set()).update(self._names(o, tt)) - - # --- clip bindings ----------------------------------------------------- - def _rel_table(self, graph: SceneGraph) -> dict[int, set[str]]: - """crc32 -> transform paths relative to any of their ancestors (or themselves: ""). - - An Animator binding path is relative to the Animator's GameObject, which need not be - the hierarchy root, and the clip does not say which GameObject plays it.""" - t = self._rel_tables.get(("relative", id(graph))) - if t is None: - t = {0: {""}} - for tf_pid in graph.tf: - parts = graph.path(tf_pid).split("/") - for i in range(1, len(parts)): - s = "/".join(parts[i:]) - t.setdefault(self._crc_of(s), set()).add(s) - self._rel_tables[("relative", id(graph))] = t - return t - - def _all_graphs(self) -> list[SceneGraph]: - return list({id(g): g for g in self._graph_of_file.values()}.values()) - - def add_path_source(self, key: str, roots: tuple[str, ...] = ()) -> int: - """Name binding paths from a hierarchy that is not exported (e.g. the scene whose UI a clip animates): - paths relative to any transform inside the subtrees rooted at transforms named in `roots` (the whole - hierarchy when empty). Used after the loaded hierarchies; resolved bindings record the key. Returns - the number of subtrees.""" - env = load_closure(self.cat.fetch_key(key)) - g = SceneGraph(env) - paths = [g.path(pid) for pid in g.tf] - bases = [p for p in paths if not roots or p.rsplit("/", 1)[-1] in roots] - table: dict[int, set[str]] = {} - for base in bases: - d = base.count("/") - for full in paths: - if full == base or full.startswith(base + "/"): - parts = full.split("/") - for i in range(d, len(parts)): - s = "/".join(parts[i + 1 :]) - table.setdefault(self._crc_of(s), set()).add(s) - self._path_sources.append((key, table)) - return len(bases) - - def _gos_at(self, graph: SceneGraph, rel: str) -> list[int]: - """GameObjects a relative binding path can name ("" = any).""" - k = (id(graph), rel) - if k not in self._gos_at_cache: - self._gos_at_cache[k] = [ - go_pid - for go_pid, tf in graph.tf_of_go.items() - if rel == "" or graph.path(tf) == rel or graph.path(tf).endswith("/" + rel) - ] - return self._gos_at_cache[k] - - def _binding_gos(self, r: dict): - """(graph, GameObject) pairs a resolved binding path can name: exactly root/path when the binding was - found below its animated root (bind_clip), else every GameObject whose path ends with it.""" - for g in r["graphs"]: - if r["root"] is not None: - full = r["root"] if r["path"] == "" else f"{r['root']}/{r['path']}" - yield from ((g, go) for go in g.gos_at_path(full)) - else: - yield from ((g, go) for go in self._gos_at(g, r["path"])) - - def _bound_names(self, r: dict) -> set[str]: - """Field paths of the components a resolved binding path can name (and the bound component kind).""" - tid, names = r["typeID"], set() - for g, go_pid in self._binding_gos(r): - if tid == 1: - r["component"] = "GameObject" - names.update(flat_names(g.go[go_pid])) - for c in g.go[go_pid]["m_Component"]: - co = self._objects.get((self._go_file[go_pid], c["component"]["m_PathID"])) - if co is None: - continue - if tid == 114: - if ( - r["script"] is not None - and co.type.name == "MonoBehaviour" - and self._mono_class(co) == r["script"][1] - ): - names.update(self._names(co)) - elif co.type.value == tid: - r["component"] = co.type.name - names.update(self._names(co)) - return names - - def _material_attribute(self, r: dict) -> str | None: - """Attribute of a renderer material property binding (customType 22): a property of the materials of the - renderer the binding names (bindings.material_attribute).""" - kinds: dict[str, str] = {} - for g, go_pid in self._binding_gos(r): - for c in g.go[go_pid]["m_Component"]: - co = self._objects.get((self._go_file[go_pid], c["component"]["m_PathID"])) - if co is None or co.type.value != r["typeID"]: - continue - for mp in co.read_typetree().get("m_Materials", []): - mo = self.deref(co, mp) - if mo is None: - continue - sp = mo.read_typetree()["m_SavedProperties"] - kinds.update({k: "float" for k, _ in sp.get("m_Floats", [])}) - kinds.update({k: "vector" for k, _ in sp.get("m_Colors", [])}) - return material_attribute(r["attributeCrc"], kinds) - - def _class_names(self, r: dict) -> set[str]: - """Names any binding of the class can target: engine properties, material properties of the - exported materials, and field paths of every exported component of the class.""" - k = ("MonoBehaviour", r["script"][1]) if r["typeID"] == 114 and r["script"] else r["typeID"] - return ( - set(ENGINE_PROPS) - | set(RECT_PROPS) - | set(material_prop_names(self.material_props)) - | self._names_by_class.get(k, set()) - ) - - def _path_tiers(self, owner): - """(graphs, source, crc32 -> paths | None for the union of `graphs`) in resolution order: the clip's - own hierarchy, every loaded hierarchy, then the path sources (after bind_clip, the animated root's - subtree comes first: see _resolve).""" - own = self._graph_of_file.get(id(owner.assets_file)) - if own is not None: - yield [own], None, self._rel_table(own) - yield self._all_graphs(), None, None - for key, table in self._path_sources: - yield [], key, table - - def _resolve(self, r: dict) -> None: - """Name a binding's path and attribute from their crc32: first tier with exactly one hit.""" - crc = r["pathCrc"] - if r["path"] is None and r["bound"]: - graph, root = r["bound"] - cands = self._subtree_table(graph, root).get(crc, set()) - if len(cands) == 1: - r.update(path=next(iter(cands)), pathSource=None, graphs=[graph], pathCandidates=None, root=root) - if r["path"] is None: - first = None - for graphs, source, table in self._path_tiers(r["owner"]): - if table is not None: - cands = table.get(crc, set()) - else: - cands = set().union(*(self._rel_table(g).get(crc, ()) for g in graphs)) - if cands and first is None: - first = cands - if len(cands) == 1: - r.update(path=next(iter(cands)), pathSource=source, graphs=graphs, pathCandidates=None) - break - else: - r["pathCandidates"] = sorted(first) if first and len(first) > 1 else None - if r["attribute"] is None and r["customType"] == 22: # RendererMaterial: not a field path crc - if r["path"] is not None: - self._bound_names(r) # names the bound component - r["attribute"] = self._material_attribute(r) - elif r["attribute"] is None: - a = r["attributeCrc"] - hit = [x for x in (self._bound_names(r) if r["path"] is not None else ()) if self._crc_of(x) == a] - if len(hit) != 1: - hit = [x for x in self._class_names(r) if self._crc_of(x) == a] - if len(hit) == 1: - r["attribute"] = hit[0] - - def _binding_record(self, owner, b: dict) -> dict: - tid = b["typeID"] - r = { - "owner": owner, - "raw": b, - "pathCrc": b["path"], - "typeID": tid, - "attributeCrc": b["attribute"], - "pptr": bool(b["isPPtrCurve"]), - "int": bool(b["isIntCurve"]), - "customType": b.get("customType"), - "serializeReference": bool(b.get("isSerializeReferenceCurve")), - "path": None, - "pathCandidates": None, - "pathSource": None, - "graphs": [], - "attribute": None, - "curveCount": 1, - "transformAttr": False, - "component": None, - "script": None, - "bound": False, - "root": None, - } - if tid == 4 and b["attribute"] in TRANSFORM_ATTR and not r["pptr"]: - r["attribute"], r["curveCount"] = TRANSFORM_ATTR[b["attribute"]] - r["transformAttr"] = True - if tid == 114: - so = self.deref(owner, b["script"]) - if so is not None: - ms = so.read() - r["script"] = (ms.m_Namespace, ms.m_ClassName) - self._resolve(r) - return r - - def resolve_pending(self) -> int: - """Resolve bindings left unresolved when their clip was exported (target hierarchy, material or - component class seen later, path sources added since) and update the emitted binding JSON in - place. Returns the number of paths and attributes still unresolved.""" - left = 0 - for rec in self._clip_recs.values(): - changed = False - for r in rec["bindings"]: - if r["path"] is None or r["attribute"] is None: - before = (r["path"], r["attribute"], r["component"]) - self._resolve(r) - changed |= before != (r["path"], r["attribute"], r["component"]) - left += (r["path"] is None) + (r["attribute"] is None) - if changed: - self._refresh_views(rec) - return left - - @staticmethod - def _refresh_views(rec: dict) -> None: - """Re-render a clip's emitted binding JSON in place after its records changed.""" - for view, dicts in rec["views"]: - for r, d in zip(rec["bindings"], dicts): - new = view(r) - if new != d: - d.clear() - d.update(new) - - def _subtree_table(self, graph: SceneGraph, root: str) -> dict[int, set[str]]: - """crc32 -> paths relative to the transform at `root` of every transform in its subtree ("" = root).""" - k = ("subtree", id(graph), root) - if k not in self._rel_tables: - t: dict[int, set[str]] = {} - for tf_pid in graph.tf: - full = graph.path(tf_pid) - if full == root or full.startswith(root + "/"): - rel = full[len(root) + 1 :] - t.setdefault(self._crc_of(rel), set()).add(rel) - if not t: - raise KeyError(f"animated root {root} not in the hierarchy") - self._rel_tables[k] = t - return self._rel_tables[k] - - def bind_clip(self, clip_o, graph: SceneGraph, root: str) -> dict: - """The clip is played by the Animator on the GameObject at `root` in `graph` (e.g. the object a timeline - track is bound to): binding paths are relative to it. A path found exactly once in root's subtree replaces - the guess of the generic tiers; a path not in the subtree keeps it (objects parented below root at - runtime). In the livescene format the clip's bindings also get the livenotes binding fields (`class`, - `curves`). The clip must have been exported. Returns {"subtree": n, "outside": n, "unresolved": n}.""" - rec = self._clip_recs[_key(clip_o)] - table = self._subtree_table(graph, root) - counts = {"subtree": 0, "outside": 0, "unresolved": 0} - for r in rec["bindings"]: - r["bound"] = (graph, root) - cands = table.get(r["pathCrc"], set()) - if len(cands) == 1: - path = next(iter(cands)) - if r["path"] != path: # the generic guess named another object: re-resolve below root - r.update(component=None, attribute=r["attribute"] if r["transformAttr"] else None) - r.update(path=path, pathSource=None, graphs=[graph], pathCandidates=None, root=root) - counts["subtree"] += 1 - self._resolve(r) - if r["root"] is None: - counts["outside" if r["path"] is not None else "unresolved"] += 1 - self._refresh_views(rec) - return counts - - def timeline_tracks(self, asset_o) -> list[tuple]: - """(track object, track typetree, [AnimationClip objects]) of every track of a TimelineAsset (depth first - through group tracks): the clips of its TimelineClips' AnimationPlayableAssets and its infinite clip.""" - out = [] - - def walk(pptrs, owner): - for p in pptrs: - t = self.deref(owner, p) - if t is None: - continue - tt = t.read_typetree() - clips = [] - for c in tt.get("m_Clips", []): - a = self.deref(t, c["m_Asset"]) - at = a.read_typetree() if a is not None else {} - if isinstance(at.get("m_Clip"), dict) and at["m_Clip"].get("m_PathID"): - clips.append(self.deref(a, at["m_Clip"])) - if isinstance(tt.get("m_AnimClip"), dict) and tt["m_AnimClip"].get("m_PathID"): - clips.append(self.deref(t, tt["m_AnimClip"])) - out.append((t, tt, clips)) - walk(tt.get("m_Children", []), t) - - walk(asset_o.read_typetree()["m_Tracks"], asset_o) - return out - - # --- clips ------------------------------------------------------------- - def clip_record(self, o) -> dict: - """Clip data and resolved binding records, built once (every clip view reads this).""" - k = _key(o) - rec = self._clip_recs.get(k) - if rec is None: - tt = o.read_typetree() - if tt["m_Legacy"] or tt["m_Compressed"]: - raise NotImplementedError(f"clip {tt['m_Name']}: legacy/compressed") - mc = tt["m_MuscleClip"] - data = mc["m_Clip"]["data"] - cbc = tt["m_ClipBindingConstant"] - rec = { - "object": o, - "tt": tt, - "name": tt["m_Name"], - "muscle": mc, - "streamed": data["m_StreamedClip"], - "dense": data["m_DenseClip"], - "constant": list(data["m_ConstantClip"]["data"]), - "pptrCurveMapping": list(cbc.get("pptrCurveMapping", [])), - "bindings": [self._binding_record(o, b) for b in cbc["genericBindings"]], - "views": [], - } - self._clip_recs[k] = rec - return rec - - @staticmethod - def _curve_total(rec: dict) -> int: - return rec["streamed"]["curveCount"] + rec["dense"]["m_CurveCount"] + len(rec["constant"]) - - @staticmethod - def _clip_header(rec: dict) -> dict: - tt, mc = rec["tt"], rec["muscle"] - return { - "sampleRate": tt["m_SampleRate"], - "wrapMode": tt["m_WrapMode"], - "startTime": mc["m_StartTime"], - "stopTime": mc["m_StopTime"], - "loopTime": bool(mc["m_LoopTime"]), - "cycleOffset": mc["m_CycleOffset"], - "events": _events(tt), - } - - @staticmethod - def _dense(rec: dict) -> dict: - de = rec["dense"] - return { - "curveCount": de["m_CurveCount"], - "frameCount": de["m_FrameCount"], - "sampleRate": de["m_SampleRate"], - "beginTime": de["m_BeginTime"], - "samples": list(de["m_SampleArray"]), - } - - def clip(self, o) -> dict: - """An AnimationClip with the data the Animator samples, in `clip_format`. - - Curve indices run over streamed curves, then dense, then constant curves; each - binding takes its curve count of them in `bindings` order.""" - k = _key(o) - if k in self._done: - return self._done[k] - rec = self.clip_record(o) - return {"generic": self._clip_generic, "livescene": self._clip_livescene, "livenotes": self._clip_livenotes}[ - self.clip_format - ](rec) - - def _views(self, rec: dict, view) -> list[dict]: - dicts = [view(r) for r in rec["bindings"]] - rec["views"].append((view, dicts)) - return dicts - - def _clip_generic(self, rec: dict) -> dict: - o, st = rec["object"], rec["streamed"] - bindings = self._views(rec, self._binding_generic) - floats = sum(r["curveCount"] for r in rec["bindings"] if not r["pptr"]) - if self._curve_total(rec) != floats: - raise RuntimeError(f"clip {rec['name']}: curve count != binding curve count") - ref = {"clip": rec["name"]} - self._done[_key(o)] = ref - streamed = {"curveCount": st["curveCount"], "frames": streamed_frames(st["data"])} - if st.get("discreteCurveCount"): - streamed["discreteCurveCount"] = st["discreteCurveCount"] - out = { - **ref, - **self._clip_header(rec), - "bindings": bindings, - "streamed": streamed, - "dense": self._dense(rec), - "constant": rec["constant"], - } - if rec["pptrCurveMapping"]: - out["pptrCurveMapping"] = [self.ref(o, p) for p in rec["pptrCurveMapping"]] - return out - - @staticmethod - def _binding_generic(r: dict) -> dict: - if r["serializeReference"]: - raise NotImplementedError(f"clip binding kind {r['raw']}") - tid = r["typeID"] - cls = r["script"][1] if tid == 114 and r["script"] else CLASS_ID.get(tid, f"classID {tid}") - d = {"path": r["path"], "typeID": tid, "class": cls, "attribute": r["attribute"]} - if r["curveCount"] != 1: - d["curves"] = r["curveCount"] - if r["pptr"]: - d["pptr"] = True - if r["int"]: - d["int"] = True - if r["path"] is None: - d["pathCrc"] = r["pathCrc"] - if r["attribute"] is None: - d["attributeCrc"] = r["attributeCrc"] - return d - - def _clip_livescene(self, rec: dict) -> dict: - o, st = rec["object"], rec["streamed"] - bindings = self._views(rec, self._binding_livescene) - ref = {"clip": rec["name"]} - self._done[_key(o)] = ref - float_curves = sum(b["curveCount"] for b in bindings if not b["isPPtrCurve"]) - pptr = [self.value(o, p) for p in rec["pptrCurveMapping"]] - out = { - **ref, - **self._clip_header(rec), - "curveOrder": "streamed, dense, constant (each binding takes curveCount curves in order)", - "bindings": bindings, - "streamed": { - "curveCount": st["curveCount"], - "discreteCurveCount": st.get("discreteCurveCount", 0), - "frames": streamed_frames(st["data"]), - }, - "dense": self._dense(rec), - "constant": rec["constant"], - "pptrCurveMapping": pptr, - } - total = self._curve_total(rec) - npptr = sum(1 for b in bindings if b["isPPtrCurve"]) - if total != float_curves + npptr: - out["curveCountMismatch"] = {"data": total, "bindings": float_curves, "pptrBindings": npptr} - return out - - @staticmethod - def _binding_livescene(r: dict) -> dict: - tid = r["typeID"] - d = { - "pathCrc": r["pathCrc"], - "path": r["path"], - "typeID": tid, - "attributeCrc": r["attributeCrc"], - "isPPtrCurve": r["pptr"], - "isIntCurve": r["int"], - "customType": r["customType"], - } - if r["pathCandidates"]: - d["pathCandidates"] = r["pathCandidates"] - if r["transformAttr"]: - d["attribute"] = r["attribute"] - else: - if tid == 114 and r["script"]: - d["class"] = f"{r['script'][0]}.{r['script'][1]}".lstrip(".") - if tid != 114 and r["component"]: - d["component"] = r["component"] - if r["attribute"] is not None: - d["attribute"] = r["attribute"] - d["curveCount"] = r["curveCount"] - if r["bound"]: # timeline clips also carry the livenotes binding fields (class, curves) - if "class" not in d: - d["class"] = CLASS_ID.get(tid, f"classID {tid}") - d["curves"] = r["curveCount"] - return d - - def _clip_livenotes(self, rec: dict) -> dict: - o, st = rec["object"], rec["streamed"] - bindings = self._views(rec, self._binding_livenotes) - float_curves = sum(r["curveCount"] for r in rec["bindings"] if not r["pptr"]) - de = rec["dense"] - if self._curve_total(rec) != float_curves: - raise RuntimeError( - f"clip {rec['name']}: curve count != binding curve count " - f"({st['curveCount']}+{de['m_CurveCount']}+{len(rec['constant'])} vs {float_curves})" - ) - pptr = [self.ref(o, p) for p in rec["pptrCurveMapping"]] - rid = _rid(o, rec["name"]) - ref = {"clip": rid} - self._done[_key(o)] = ref - try: - frames = streamed_frames(st["data"]) - except ValueError as e: - raise ValueError(f"clip {rec['name']}: {e}") from None - self.clips[rid] = { - "name": rec["name"], - **self._clip_header(rec), - "bindings": bindings, - "streamed": {"curveCount": st["curveCount"], "frames": frames}, - "dense": self._dense(rec), - "constant": rec["constant"], - "pptrCurveMapping": pptr, - "discreteCurveCount": st.get("discreteCurveCount", 0), - } - return ref - - @staticmethod - def _binding_livenotes(r: dict) -> dict: - tid = r["typeID"] - if tid == 114 and not r["pptr"] and r["script"] and r["path"] is not None and r["attribute"] is not None: - d = {"path": r["path"], "typeID": tid, "class": r["script"][1], "attribute": r["attribute"], "curves": 1} - else: - d = { - "path": r["path"] if r["path"] is not None else f"crc32:{r['pathCrc']}", - "typeID": tid, - "class": CLASS_ID.get(tid, f"classID {tid}"), - } - if r["pptr"]: - d["pptr"] = True - if r["transformAttr"]: - d.update(attribute=r["attribute"], curves=r["curveCount"]) - else: - if tid == 114: - d["script"] = r["script"][1] if r["script"] else None - d["attribute"] = r["attribute"] if r["attribute"] is not None else f"crc32:{r['attributeCrc']}" - d["curves"] = r["curveCount"] - if r["int"]: - d["int"] = True - if r["pathSource"]: - d["scene"] = r["pathSource"] - return d - - def clip_curves(self, o) -> dict: - """A clip as per-property streamed cubic keys, for runtimes that animate RectTransform, - CanvasGroup alpha and Transform euler angles of the animated root only (ADV UI).""" - rec = self.clip_record(o) - tt, mc, st, de = rec["tt"], rec["muscle"], rec["streamed"], rec["dense"] - name, const = rec["name"], rec["constant"] - if st["discreteCurveCount"] or de["m_CurveCount"]: - raise NotImplementedError(f"clip {name}: discrete/dense curves") - props = [] - for r in rec["bindings"]: - if r["pathCrc"] != 0: - raise NotImplementedError(f"clip {name}: binding below the animated root") - if r["pptr"] or r["int"] or r["serializeReference"]: - raise NotImplementedError(f"clip {name}: binding kind {r['raw']}") - if r["typeID"] == 224 and r["attribute"] in RECT_PROPS: - props.append("RectTransform." + r["attribute"]) - elif r["typeID"] == 225 and r["attribute"] == "m_Alpha": - props.append("CanvasGroup.m_Alpha") - elif r["typeID"] == 4 and r["attributeCrc"] == 4: # localEulerAnglesRaw (x, y, z) - props += [ - "Transform.localEulerAngles.x", - "Transform.localEulerAngles.y", - "Transform.localEulerAngles.z", - ] - else: - raise NotImplementedError(f"clip {name}: binding {r['raw']}") - if st["curveCount"] + len(const) != len(props): - raise RuntimeError(f"clip {name}: curve count != bound properties") - curves = {p: {"keys": []} for p in props[: st["curveCount"]]} - for t, keys in streamed_frames(st["data"]): - for ci, c0, c1, c2, c3 in keys: - curves[props[ci]]["keys"].append([t, c0, c1, c2, c3]) - for i, v in enumerate(const): - curves[props[st["curveCount"] + i]] = {"constant": v} - return { - "name": name, - "sampleRate": tt["m_SampleRate"], - "startTime": mc["m_StartTime"], - "stopTime": mc["m_StopTime"], - "loopTime": bool(mc["m_LoopTime"]), - "events": tt["m_Events"], - "curves": curves, - "keyFormat": "[time, c0, c1, c2, c3]; value(t) = ((c0*x + c1)*x + c2)*x + c3, x = t - time, " - "from the last key with time <= t (first key at -FLT_MAX)", - } - - # --- animator controllers, sprite atlases -------------------------------- - def controller_record(self, o) -> dict: - tt = o.read_typetree() - if o.type.name != "AnimatorController": - raise NotImplementedError(f"{o.type.name} {tt.get('m_Name')}") - return {"object": o, "tt": tt, "name": tt["m_Name"], "tos": dict(tt["m_TOS"])} - - def controller(self, o) -> dict: - """AnimatorController: layers, state machines (states with speed, loop, blend trees, transitions), - parameters and default values; clips exported in `clip_format`. Inlined once, or in the - `controllers` registry for the livenotes format.""" - k = _key(o) - if k in self._done: - return self._done[k] - rec = self.controller_record(o) - registry = self.clip_format == "livenotes" - cid = _rid(o, rec["name"]) if registry else rec["name"] - ref = {"controller": cid} - self._done[k] = ref - body = controller_doc(rec["tt"], lambda p: self.ref(o, p), lambda v: self.value(o, v)) - if registry: - self.controllers[cid] = body - return ref - return {**ref, **body} - - def controller_states(self, o) -> dict: - """A single-layer AnimatorController without transitions, blend trees or parameter-driven states - as its state list (name, speed, loop, cycle offset, clip name) -- what the ADV UI runtime plays.""" - rec = self.controller_record(o) - tt, tos = rec["tt"], rec["tos"] - clips = [self.referent(o, p).read().m_Name for p in tt["m_AnimationClips"]] - ctrl = tt["m_Controller"] - if len(ctrl["m_StateMachineArray"]) != 1 or len(ctrl["m_LayerArray"]) != 1: - raise NotImplementedError(f"controller {tt['m_Name']}: several layers") - sm = ctrl["m_StateMachineArray"][0]["data"] - states = [] - for st in sm["m_StateConstantArray"]: - d = st["data"] - motion = [] - for bt in d["m_BlendTreeConstantArray"]: - for node in bt["data"]["m_NodeArray"]: - n = node["data"] - if n["m_ChildIndices"]: - raise NotImplementedError(f"controller {tt['m_Name']}: blend trees") - motion.append(clips[n["m_ClipID"]] if n["m_ClipID"] != 0xFFFFFFFF else None) - if d["m_SpeedParamID"] or d["m_TimeParamID"] or d["m_CycleOffsetParamID"]: - raise NotImplementedError(f"controller {tt['m_Name']}: parameter-driven state") - if d["m_TransitionConstantArray"]: - raise NotImplementedError(f"controller {tt['m_Name']}: state transitions") - states.append( - { - "name": tos[d["m_NameID"]], - "speed": d["m_Speed"], - "loop": d["m_Loop"], - "cycleOffset": d["m_CycleOffset"], - "clip": motion[0] if motion else None, - } - ) - if sm["m_AnyStateTransitionConstantArray"]: - raise NotImplementedError(f"controller {tt['m_Name']}: any-state transitions") - return {"name": tt["m_Name"], "defaultState": states[sm["m_DefaultState"]]["name"], "states": states} - - def atlas(self, o) -> dict: - """SpriteAtlas: every packed sprite (name -> sprite with atlas texture and UVs), inlined once.""" - k = _key(o) - if k in self._done: - return self._done[k] - tt = o.read_typetree() - ref = {"atlas": tt["m_Name"]} - self._done[k] = ref - sprites = {} - for p in tt["m_PackedSprites"]: - s = self.ref(o, p) - if s is not None: - sprites[s["sprite"]] = s - return {**ref, "tag": tt.get("m_Tag"), "isVariant": tt.get("m_IsVariant"), "sprites": sprites} - - # --- MonoBehaviours, components ------------------------------------------ - def scriptable(self, o) -> dict: - """A MonoBehaviour that is not a component (ScriptableObject asset), inlined once.""" - k = _key(o) - if k in self._done: - return self._done[k] - cls = self.script_class(o) - tt = o.read_typetree() - ref = {"asset": cls, "name": tt.get("m_Name", "")} - if cls in self.stub_assets: - return ref - self._done[k] = ref - return {**ref, **{x: self.value(o, y) for x, y in tt.items() if x not in HEADER}} - - def component(self, o) -> dict: - tt = o.read_typetree() - self._register_names(o, tt) - out = {"type": o.type.name} - if o.type.name == "MonoBehaviour": - out["class"] = self.script_class(o) - out.update({k: self.value(o, v) for k, v in tt.items() if k not in HEADER}) - return out - - # --- hierarchies ----------------------------------------------------- - def hierarchy(self, env, graph: SceneGraph, root_tf: int) -> list[dict]: - by_pid = {} - for o in env.objects: - by_pid.setdefault(o.path_id, []).append(o) - nodes = [] - - def walk(tf_pid): - t = graph.tf[tf_pid] - go = graph.go[t["m_GameObject"]["m_PathID"]] - comps = [] - for c in go["m_Component"]: - objs = [x for x in by_pid.get(c["component"]["m_PathID"], []) if x.type.name not in ("GameObject",)] - if len(objs) != 1: - raise RuntimeError(f"{graph.path(tf_pid)}: component {c} resolves to {len(objs)}") - co = objs[0] - if co.type.name in ("Transform", "RectTransform"): - continue - comps.append(self.component(co)) - node = { - "path": graph.path(tf_pid), - "name": go["m_Name"], - "active": bool(go["m_IsActive"]), - "layer": go["m_Layer"], - "tag": go.get("m_Tag"), - "localPosition": t["m_LocalPosition"], - "localRotation": t["m_LocalRotation"], - "localScale": t["m_LocalScale"], - "components": comps, - } - if "m_AnchorMin" in t: - node["rect"] = { - k: t[k] for k in ("m_AnchorMin", "m_AnchorMax", "m_AnchoredPosition", "m_SizeDelta", "m_Pivot") - } - nodes.append(node) - for ch in t["m_Children"]: - walk(ch["m_PathID"]) - - walk(root_tf) - return nodes - - # --- keys -------------------------------------------------------------- - def _container_asset(self, env, internal_id: str) -> ObjectReader: - for o in env.objects: - if o.type.name != "AssetBundle": - continue - tt = o.read_typetree() - for path, info in tt["m_Container"]: - if path.lower() == internal_id.lower(): - asset = self.deref(o, info["asset"]) - if asset is None: - raise KeyError(f"{internal_id}: the container entry is a null reference") - return asset - raise KeyError(f"{internal_id} not in any AssetBundle container") - - def key_object(self, key: str) -> ObjectReader: - """The object an addressable key names (its bundle's container asset).""" - env, _ = self.load(key) - return self._container_asset(env, self.cat.entry(key)["internal_id"]) - - def prefab(self, key: str, loaded=None) -> dict: - """A prefab or scene as a node list; `loaded` = (env, graph) from load(key).""" - env, graph = loaded or self.load(key) - iid = self.cat.entry(key)["internal_id"] - if iid.endswith(".unity"): - scene_files = set() - for o in env.objects: - if o.type.name == "AssetBundle": - for path, cab in o.read_typetree().get("m_SceneHashes", []): - if path.lower() == iid.lower(): - scene_files.add(cab.lower()) - roots = [ - o.path_id - for o in env.objects - if o.type.name in ("Transform", "RectTransform") - and o.assets_file.name.lower().startswith(tuple(scene_files)) - and not graph.tf[o.path_id]["m_Father"]["m_PathID"] - ] - if not roots: - raise RuntimeError(f"{key}: no scene roots") - return { - "key": key, - "scene": True, - "nodes": [n for r in sorted(roots, key=graph.path) for n in self.hierarchy(env, graph, r)], - } - go = self._container_asset(env, iid) - if go is None or go.type.name != "GameObject": - raise RuntimeError(f"{key}: container asset is {go and go.type.name}") - return {"key": key, "nodes": self.hierarchy(env, graph, graph.tf_of_go[go.path_id])} - - def asset(self, key: str) -> dict: - o = self.key_object(key) - if o.type.name != "MonoBehaviour": - raise RuntimeError(f"{key}: asset is {o.type.name}") - return {"key": key, **self.scriptable(o)} - - def export_key(self, key: str) -> dict: - """Any addressable key by its container asset: prefab or scene (node list), ScriptableObject, - AnimatorController, AnimationClip, SpriteAtlas, Material, Texture2D, Sprite, Mesh; other kinds as - {object, value} (typetree with references resolved).""" - iid = self.cat.entry(key)["internal_id"] - if iid.endswith(".unity"): - return self.prefab(key) - env, graph = self.load(key) - o = self._container_asset(env, iid) - t = o.type.name - if t == "GameObject": - return self.prefab(key, (env, graph)) - kinds = { - "MonoBehaviour": self.scriptable, - "AnimatorController": self.controller, - "AnimatorOverrideController": self.controller, - "AnimationClip": self.clip, - "SpriteAtlas": self.atlas, - "Material": self.material, - "Texture2D": self.texture, - "Sprite": self.sprite, - "Mesh": self.mesh, - } - if t in kinds: - return {"key": key, **kinds[t](o)} - return {"key": key, "object": t, "value": self.value(o, o.read_typetree())} - - def key_sprite(self, key: str) -> dict: - """The Sprite that a texture key's bundle holds for its texture (Addressables sprite sub-asset).""" - env, _ = self.load(key) - tex = self.key_object(key) - hits = [] - for o in env.objects: - if o.type.name == "Sprite": - t = self.deref(o, o.read_typetree()["m_RD"]["texture"]) - if t is not None and t.path_id == tex.path_id: - hits.append(o) - if len(hits) != 1: - raise RuntimeError(f"{key}: {len(hits)} sprites for the texture") - return {"key": key, **self.sprite(hits[0])} - - # --- shaders --------------------------------------------------------- - def dump_shaders(self, out_dir: Path, objects: dict | None = None) -> tuple[list, dict]: - """Dump shaders (name -> Shader object; default every registered one) in name order with - shader.py's layout into `out_dir`; returns (index records, write_index summary).""" - from ..formats import shader as shader_mod - - objects = self.shaders if objects is None else objects - index: list = [] - for name in sorted(objects): - shader_mod.dump_objects([objects[name]], out_dir, objects[name].assets_file.name, index) - return index, shader_mod.write_index(index, out_dir) diff --git a/src/nnnotes/unity/fontsettings.py b/src/nnnotes/unity/fontsettings.py index 316baca..fd316b2 100644 --- a/src/nnnotes/unity/fontsettings.py +++ b/src/nnnotes/unity/fontsettings.py @@ -1,26 +1,9 @@ -"""The settings of the game's TextMeshPro font assets that the exports with open fonts (`--fonts open`) take. - -An open font asset stands for a game font asset: it gets that asset's point size, padding, render mode, style -settings, line metrics and text materials, mirrors its fallback chain, and the characters the game asset lacks with -its fallbacks take TextMeshPro's missing glyph (storyfonts.open_font_set). These values come from FontSettings: - - builtin() the table shipped with nnnotes (fontsettings.json, FORMAT), so the open-font exports read no - game font asset, material or font bundle - tmpfont.AssetFontSettings - the same values read from the game's font assets through an Exporter (`--fonts game`, and - tmpfont.build_font_settings, which makes the table) - -The table holds, per font asset LocalizeManager lists for a language and per asset of their fallback chains: -`faceInfo` (m_FaceInfo), `atlasPadding`, `atlasRenderMode`, the style settings (STYLE_KEYS), `material` (the name of -its default material), `fallbacks` (the names of its fallback font asset table, in order), `coverage` (the code -points the asset has: its character table, and for a dynamic asset the character map of its source font file, as a -bit set: bit u of the zlib-compressed bytes, base64), `variants` (the [code point, variation selector] pairs its -source font file maps to a glyph other than the code point's) and `bundle` (the file name of the bundle that holds -it); per material the material record (Exporter.material, its texture references reduced: table_material) of every -font asset's default material and of every localized text material (" - " for the types of -LocalizeManager); and `objects`, the path id, type and name of each of those font assets and materials, by which -references into the font bundles resolve without loading them (export.Exporter `font_stubs`). -""" +"""Font settings tables for TextMesh Pro assets and open-font generation. + +A table records each font's face metrics, atlas parameters, style settings, fallback order, character coverage, +variation sequences and bundle identity. Materials carry shader names and texture settings. The object index +maps font and material path ids to their names. FontSettings reads either the bundled table or a table assembled +from product objects by content.tmpassets.build_font_settings.""" from __future__ import annotations diff --git a/src/nnnotes/unity/live2d.py b/src/nnnotes/unity/live2d.py deleted file mode 100644 index e426c65..0000000 --- a/src/nnnotes/unity/live2d.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Live2D (Cubism) model: Cubism-for-Unity prefab -> prefab.json + Cubism runtime files. - -Components are identified by their MonoScript class (resolved through the APK's -shared_monoscripts bundle, so the Catalog must be opened with `apk=`). - -`extract_runtime` writes what a re-implementation of the game's own runtime needs (`export_runtime` writes the -same and also returns the exporter state, for callers that export more of the model's bundles): - - .moc3 CubismMoc._bytes - textures/*.png atlas pages (Texture2D) - .prefab.json the whole model prefab exported by export.Exporter: - every GameObject with every component (CubismRenderer/ - MeshRenderer + materials per drawable, controllers, - Live2DCharacter, eye blink, harmonic motion, motion - sync, anchors ...), with the fade motion list, the - expression list and every AnimationClip (Mecanim clip - data + resolved bindings) inlined; plus `canvas` (moc3) - -The texture each drawable is drawn with is its CubismRenderer._mainTexture in prefab.json, keyed by -CubismDrawable._unmanagedIndex (the Cubism Core drawable index). -""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - -from ..core.jsonio import write_json -from ..formats.cubism import moc3_canvas -from ..source.catalog import Catalog -from ..source.config import apk_missing -from .export import Exporter -from .reader import SceneGraph, script_class - -# the moc is written as .moc3; prefab.json names it only -STUB_ASSETS = ("CubismMoc",) - - -def _classes(env) -> dict[str | None, list]: - by_cls: dict[str | None, list] = {} - for o in env.objects: - if o.type.name == "MonoBehaviour": - by_cls.setdefault(script_class(o), []).append(o) - return by_cls - - -def _only(by_cls, cls: str, name: str) -> dict: - objs = by_cls.get(cls, []) - if len(objs) != 1: - raise RuntimeError(f"{name}: {len(objs)} {cls}") - return objs[0].read_typetree() - - -@dataclass -class RuntimeExport: - """What `export_runtime` wrote and read: `summary` (the result of `extract_runtime`), the Exporter it wrote with - (its shaders, textures, ...), the loaded bundle environment, its scene graph and the MonoBehaviours by script - class.""" - - summary: dict - exporter: Exporter - env: object - graph: SceneGraph - classes: dict[str | None, list] - - -def export_runtime(cat: Catalog, key: str, out_dir: Path) -> RuntimeExport: - """Write the runtime files of the model `key` into `out_dir` (module docstring) and return them with the - exporter state.""" - if cat.apk is None: - raise apk_missing("resolving the Cubism component classes") - out_dir = Path(out_dir) - out_dir.mkdir(parents=True, exist_ok=True) - name = key.rsplit("/", 1)[-1] - ex = Exporter(cat, out_dir, inline_meshes=False, stub_assets=STUB_ASSETS) - env, graph = ex.load(key) - by_cls = _classes(env) - moc = bytes(_only(by_cls, "CubismMoc", name)["_bytes"]) - (out_dir / f"{name}.moc3").write_bytes(moc) - prefab = ex.prefab(key, loaded=(env, graph)) - prefab["canvas"] = moc3_canvas(moc) - write_json(out_dir / f"{name}.prefab.json", prefab, indent=None) - pages = sorted((o for o in env.objects if o.type.name == "Texture2D"), key=lambda o: o.read().m_Name) - tex_files = [ex.texture(o)["texture"] for o in pages] - res = { - "name": name, - "moc3": f"{name}.moc3", - "prefab": f"{name}.prefab.json", - "moc3Bytes": len(moc), - "textures": tex_files, - "nodes": len(prefab["nodes"]), - } - return RuntimeExport(res, ex, env, graph, by_cls) - - -def extract_runtime(cat: Catalog, key: str, out_dir: Path) -> dict: - """Write the runtime files of the model `key` into `out_dir`; returns the summary: name, moc3, prefab, - moc3Bytes, textures, nodes.""" - return export_runtime(cat, key, out_dir).summary diff --git a/src/nnnotes/unity/reader.py b/src/nnnotes/unity/reader.py index 962e10c..9fb3c13 100644 --- a/src/nnnotes/unity/reader.py +++ b/src/nnnotes/unity/reader.py @@ -165,6 +165,20 @@ def first_mono(env, must_have: str | None = None) -> dict | None: return None +def container_asset(env, internal_id: str) -> ObjectReader: + """The object named by a bundle's container path, matched without case sensitivity.""" + for owner in env.objects: + if owner.type.name != "AssetBundle": + continue + for path, info in owner.read_typetree()["m_Container"]: + if path.lower() == internal_id.lower(): + asset = deref(owner, info["asset"]) + if asset is None: + raise KeyError(f"{internal_id}: the container entry is a null reference") + return asset + raise KeyError(f"{internal_id} not in any AssetBundle container") + + def textassets(env) -> dict[str, str]: out = {} for o in env.objects: diff --git a/src/nnnotes/unity/texels.py b/src/nnnotes/unity/texels.py new file mode 100644 index 0000000..d815803 --- /dev/null +++ b/src/nnnotes/unity/texels.py @@ -0,0 +1,182 @@ +"""In-memory texel block packing and PNG encoding for object-based text products.""" + +from __future__ import annotations + +import io +import json +import math +import re +from typing import Literal + +import numpy as np +from PIL import Image + +from ..core import cache + +_LIBS = ("UnityPy", "Pillow", "texture2ddecoder", "astc-encoder-py", "etcpak", "numpy") +# decoded textures and PNG encodings memoized by content within this process +PNG = cache.bucket("png", salt=cache.source_salt(__file__, *_LIBS)) + + +def packed_png(rgba: np.ndarray) -> bytes: + """PNG bytes (Pillow, optimize=True) of an RGBA array (bottom row first), cached by the array's content.""" + rgba = np.ascontiguousarray(rgba, np.uint8) + k = PNG.key("packed", rgba.shape, rgba.data) + data = PNG.get(k) + if data is None: + buf = io.BytesIO() + Image.fromarray(rgba[::-1].copy(), "RGBA").save(buf, format="PNG", optimize=True) + data = buf.getvalue() + PNG.put(k, data) + return data + + +def _safe(s: str) -> str: + return re.sub(r"[^A-Za-z0-9_.-]+", "_", s) + + +# --- texel blocks -> packed textures --------------------------------------------- +def grow_box(x0: float, y0: float, x1: float, y1: float, margin: int) -> tuple[int, int, int, int]: + """Texel box covering [x0, x1) x [y0, y1) grown by `margin` texels on every side.""" + return (math.floor(x0) - margin, math.floor(y0) - margin, math.ceil(x1) + margin, math.ceil(y1) + margin) + + +def wrap_mode(mode: int) -> Literal["wrap", "clip"]: + """The np.take mode of a texture's wrap mode (m_WrapU / m_WrapV: 0 repeat, 1 clamp).""" + if mode == 0: + return "wrap" + if mode == 1: + return "clip" + raise NotImplementedError(f"texture wrap mode {mode}") + + +def texel_rows(img, texture_format: int, width: int, height: int, name: str) -> np.ndarray: + """The RGBA texels of a decoded Texture2D image `img` (a PIL image; the texture's format, size and name), bottom + row first. Alpha8 keeps its value in the alpha channel.""" + if texture_format == 1: # Alpha8: value in the alpha channel + if img.mode == "L": + a = np.asarray(img, np.uint8) + elif img.mode == "RGBA": + a = np.asarray(img, np.uint8)[:, :, 3] + else: + raise NotImplementedError(f"Alpha8 decoded as {img.mode}") + arr = np.zeros(a.shape + (4,), np.uint8) + arr[:, :, 3] = a # rgb 0 (GL alpha texture); TMP reads .a only + else: + arr = np.asarray(img.convert("RGBA"), np.uint8) + if arr.shape[:2] != (height, width): + raise RuntimeError(f"{name}: decoded size {arr.shape}") + return arr[::-1].copy() # bottom row first (Unity v = 0) + + +def pack_group(group: str, handles: list[dict], pixels, width_limit: int) -> tuple[dict, np.ndarray]: + """The layout and the texels of one packed texture `group` of the blocks `handles` (MemoryPacker.build). Returns its + record without its file (name, width, height, mipCount, settings, sources, blocks) and its RGBA array, bottom row + first. `pixels(tex)` gives the texels of a block's source texture. Every handle gets its texture (`group`) and its + integer `offset`.""" + settings = {json.dumps(h["settings"], sort_keys=True) for h in handles} + if len(settings) != 1: + raise RuntimeError(f"{group}: blocks from textures with different sampler settings") + # dedupe identical blocks of the same texture + uniq: dict[tuple, dict] = {} + for h in handles: + uniq.setdefault((h["key"], h["box"]), h) + blocks = sorted(uniq.values(), key=lambda h: (-(h["box"][3] - h["box"][1]), h["box"])) + x = y = shelf = 0 + width = max(width_limit, max(b["box"][2] - b["box"][0] for b in blocks)) + for b in blocks: + w, hgt = b["box"][2] - b["box"][0], b["box"][3] - b["box"][1] + if x + w > width: + x, y, shelf = 0, y + shelf, 0 + b["dst"] = (x, y) + x += w + shelf = max(shelf, hgt) + height = y + shelf + if len(blocks) == 1: # a single block keeps its own size + width = blocks[0]["box"][2] - blocks[0]["box"][0] + out = np.zeros((height, width, 4), np.uint8) + for b in blocks: + x0, y0, x1, y1 = b["box"] + dx, dy = b["dst"] + if "array" in b: + out[dy : dy + (y1 - y0), dx : dx + (x1 - x0)] = b["array"] + continue + src = pixels(b["tex"]) + st = b["settings"] + rows = np.take(np.arange(src.shape[0]), np.arange(y0, y1), mode=wrap_mode(st["m_WrapV"])) + cols = np.take(np.arange(src.shape[1]), np.arange(x0, x1), mode=wrap_mode(st["m_WrapU"])) + out[dy : dy + (y1 - y0), dx : dx + (x1 - x0)] = src[np.ix_(rows, cols)] + for h in handles: + d = uniq[(h["key"], h["box"])]["dst"] + h["offset"] = (d[0] - h["box"][0], d[1] - h["box"][1]) + h["texture"] = group + s = next(iter(handles))["settings"] + rec = { + "name": group, + "width": width, + "height": height, + "mipCount": 1, + "settings": s, + "sources": sorted({h["srcName"] for h in handles}), + "blocks": len(blocks), + } + return rec, out + + +def texture_block( + key: tuple, settings: dict, name: str, size: tuple[int, int], box: tuple[int, int, int, int], tex +) -> dict: + """A block that copies the texel box `box` of the texture `tex` (its texels come from the packer's `pixels`). + `key` identifies the texture, `settings` is its m_TextureSettings, `name` and `size` (width, height) its name and + size.""" + return {"tex": tex, "key": key, "settings": settings, "box": box, "srcName": name, "srcSize": size} + + +def array_block(name: str, rgba: np.ndarray, origin: tuple[int, int], settings: dict) -> dict: + """A block of generated texels: `rgba` (RGBA, bottom row first) placed at `origin` of its virtual texture.""" + ox, oy = origin + return { + "array": rgba, + "key": ("generated", name), + "settings": settings, + "box": (ox, oy, ox + rgba.shape[1], oy + rgba.shape[0]), + "srcName": name, + "srcSize": (rgba.shape[1], rgba.shape[0]), + } + + +class MemoryPacker: + """Pack in-memory texture blocks into PNG pages. Textures supply a key, name, settings, width, height and + bottom-first RGBA pixels. build() returns page records and encoded bytes by file name.""" + + WIDTH = 2048 + + def __init__(self): + self.groups: dict[str, list] = {} + + @staticmethod + def pixels(tex) -> np.ndarray: + return tex.pixels + + def add(self, group: str, tex, x0: int, y0: int, x1: int, y1: int) -> dict: + """Register a block; returns a handle filled in by build().""" + h = texture_block(tex.key, tex.settings, tex.name, (tex.width, tex.height), (x0, y0, x1, y1), tex) + self.groups.setdefault(group, []).append(h) + return h + + def add_array(self, group: str, name: str, rgba: np.ndarray, origin: tuple[int, int], settings: dict) -> dict: + """Register generated RGBA texels at their virtual texture origin.""" + h = array_block(name, rgba, origin, settings) + self.groups.setdefault(group, []).append(h) + return h + + def build(self) -> tuple[dict, dict[str, bytes]]: + """The records of the packed textures by group, and the PNG bytes of their pages by file name.""" + textures: dict[str, dict] = {} + pages: dict[str, bytes] = {} + for group, handles in self.groups.items(): + rec, out = pack_group(group, handles, self.pixels, self.WIDTH) + fn = f"textures/{_safe(group)}.png" + pages[fn] = packed_png(out) + textures[group] = {"texture": fn, **rec} + return textures, pages diff --git a/src/nnnotes/unity/textstyle.py b/src/nnnotes/unity/textstyle.py index dba56e8..7db9a2e 100644 --- a/src/nnnotes/unity/textstyle.py +++ b/src/nnnotes/unity/textstyle.py @@ -1,34 +1,9 @@ -"""TextMesh Pro text layout and style as plain values (shared by the UI exports). - -A text record describes what a TMP text draws with any font: its layout settings (size, auto-size range, -alignment, wrapping, overflow, spacing, margins), its colours and style flags, and the look of its TMP -distance-field material turned into face / outline / underlay values in em. `fontRole` names which of the -game's localized font slots the text uses, so a renderer maps roles to fonts of its own. - - language_fonts Fwk.Localization.LocalizeManager font names per LanguageMode (level0) - language_fonts_json the same from the object's json (the shipped typetree of the class) - localize_text LocalizeText.Init / OnFontChanged: the font asset, material and line spacing a TMP text ends up - with in a language - -The language is a Fwk.Localization.LanguageMode (`mode`), the value languages.mode gives for a catalog language code; -the callers take it from the client language they export (`[catalog] language`). - font_role the role of a font asset: its slot in LocalizeManager's font name list - material_style TMP distance-field material properties -> face / outline / underlay values in em - text_style one TMP text component -> its text record - TextStyles text records of an exporter's TMP texts (reads the localized materials and face info) -""" +"""TextMesh Pro layout, localization, font roles and material style records.""" from __future__ import annotations -import struct -from typing import TYPE_CHECKING - from ..source.languages import LANGUAGES -if TYPE_CHECKING: # annotations only: the module loads without UnityPy - from .export import Exporter - from .player import PlayerData - # Fwk.Localization.LanguageMode / language field names of the text tables. JAPANESE_MODE = 0 LANGUAGE_FIELDS = {mode: column[1:] for mode, column in LANGUAGES.values()} # LanguageMode -> text field @@ -73,75 +48,6 @@ } -class _Reader: - """Little-endian serialized-field reader with 4-byte alignment.""" - - def __init__(self, raw: bytes, pos: int = 0): - self.raw, self.pos = raw, pos - - def align(self): - self.pos = (self.pos + 3) & ~3 - - def i32(self) -> int: - v = struct.unpack_from(" int: - v = struct.unpack_from(" int: - v = self.raw[self.pos] - self.pos += 1 - return v - - def string(self) -> str: - n = self.i32() - s = self.raw[self.pos : self.pos + n].decode("utf-8") - self.pos += n - self.align() - return s - - def strings(self) -> list[str]: - return [self.string() for _ in range(self.i32())] - - -def language_fonts(player: PlayerData) -> dict: - """Fwk.Localization.LocalizeManager (level0): font names per LanguageMode. - - Read field by field from the serialized bytes; language_fonts_json gives the same dictionary from the object's - json. Layout: m_GameObject, m_Enabled, m_Script, m_Name, _dontDestroyOnload, _languageDataList[{_languageMode, - _fontNames[], _additionalFontNames[]}], _materialTypeList[]. - """ - hits = [ - o - for o in player._by_type.get("MonoBehaviour", []) - if player.script(o)[1:] == ("Fwk.Localization", "LocalizeManager") - ] - if len(hits) != 1: - raise RuntimeError(f"LocalizeManager: {len(hits)} objects") - r = _Reader(hits[0].get_raw_data()) - r.i32() - r.i64() # m_GameObject - r.u8() - r.align() # m_Enabled - r.i32() - r.i64() # m_Script - r.string() # m_Name - r.u8() - r.align() # _dontDestroyOnload - langs = {} - for _ in range(r.i32()): - mode = r.i32() - langs[mode] = {"fontNames": r.strings(), "additionalFontNames": r.strings()} - materials = r.strings() - if r.pos != len(r.raw): - raise RuntimeError(f"LocalizeManager: {len(r.raw) - r.pos} trailing bytes") - return {"languages": langs, "materialTypes": materials} - - def language_fonts_json(localize: dict) -> dict: """language_fonts of the LocalizeManager's object json (its shipped type tree: the fields as serialized), the same {"languages": {mode: {"fontNames", "additionalFontNames"}}, "materialTypes"} dictionary.""" @@ -356,41 +262,3 @@ def text_style(comp: dict, localize: dict | None, lang: dict, style: dict, mode: **style, } return rec - - -class TextStyles: - """Text records of TMP texts read through an Exporter: the material each text draws with in the language - `mode` (LocalizeText) and the face info of its font asset, from `fonts` (fontsettings.FontSettings: the table - shipped with nnnotes, fontsettings.builtin(), or the game's font assets, tmpfont.AssetFontSettings); no font data - is kept.""" - - def __init__(self, ex: Exporter, player: PlayerData, mode: int, fonts): - self.ex, self.mode = ex, mode - self.fonts = fonts - self.lang = language_fonts(player) - self.swap = font_swap(self.lang, mode) - self.roles: dict[str, dict] = {} - self._styles: dict[tuple, dict] = {} - - def _face(self, font_asset: str) -> tuple[dict, dict]: - f = self.fonts.font(font_asset) - return f["faceInfo"], {k: f[k] for k in ("normalSpacingOffset", "boldSpacing")} - - def record(self, path: str, comp: dict, localize: dict | None) -> dict: - """The text record of `comp` (TMP text component) with its LocalizeText component `localize`.""" - fa, mat = comp["m_fontAsset"]["name"], comp["m_sharedMaterial"]["material"] - if localize and localize["m_Enabled"] and localize["_localizeEnabled"]: - loc = localize_text(path, fa, mat, self.swap, self.lang, self.mode) - fa, mat = loc["fontAsset"], loc["material"] - if (fa, mat) not in self._styles: - face, extra = self._face(fa) - self._styles[(fa, mat)] = material_style(self.fonts.material(fa, mat), face["m_PointSize"], face["m_Scale"]) - rec = text_style(comp, localize, self.lang, self._styles[(fa, mat)], self.mode) - if rec["fontRole"] not in self.roles: - face, extra = self._face(fa) - self.roles[rec["fontRole"]] = face_metrics(face, extra) - return rec - - def summary(self) -> dict: - """The document-level part: the language's text field, units and the line metrics of each role used.""" - return {"language": LANGUAGE_FIELDS[self.mode], "units": UNITS, "roles": self.roles} diff --git a/src/nnnotes/unity/tmpfont.py b/src/nnnotes/unity/tmpfont.py index 241cddc..b780dd7 100644 --- a/src/nnnotes/unity/tmpfont.py +++ b/src/nnnotes/unity/tmpfont.py @@ -1,20 +1,7 @@ -"""TextMeshPro font assets for the UI exports with game fonts (`--fonts game`). - - FontSet TMP font assets reachable from the localized fonts (fallback chains) with TMP - character lookup; RuntimeGlyphs generates the SDF glyphs a dynamic font asset adds - at runtime (FontEngine) from the source font file, validated against the baked ones - glyph_coverage / export_fonts / resolve_font_blocks - the character and glyph tables reduced to the characters a page shows, their texel - blocks copied into packed textures (export.TexelPacker) - -The localized font / material / line spacing of a text (LocalizeText) and the text records are in textstyle.py. -Needs the optional dependencies of the `fonts` extra (require_extra). -""" +"""TextMesh Pro font lookup, glyph rendering and material records over font-asset protocols.""" from __future__ import annotations -import copy -import hashlib import importlib.util import io import math @@ -24,7 +11,6 @@ import numpy as np from ..source.config import ConfigError -from .export import Exporter, MemoryPacker, TexelPacker, _key, grow_box from .fontsettings import ( END_OF_TEXT, FONT_CLASS, @@ -34,12 +20,10 @@ TMP_SYNTHESIZED, FontSettings, encode_coverage, - font_dir, - font_key, is_variation_selector, table_material, ) -from .textstyle import language_fonts +from .texels import MemoryPacker, grow_box EXTRA = "fonts" EXTRA_MODULES = {"fontTools": "fonttools", "freetype": "freetype-py", "scipy": "scipy"} @@ -77,8 +61,7 @@ def font_cmap(data: bytes, face_index: int) -> tuple[set[int], dict[tuple[int, i class FontAsset(Protocol): - """A TextMesh Pro font asset as the lookup, the glyph tables and the runtime glyphs read it: Font (read through an - Exporter) and tmpassets.Font (read from an object table).""" + """A TextMesh Pro font asset supplying character lookup, glyph tables and runtime glyph generation.""" name: str tt: dict @@ -98,50 +81,6 @@ def atlas_settings(self) -> list[dict]: ... def default_material(self) -> dict: ... -class Font: - def __init__(self, ex: Exporter, obj): - self.obj = obj - self._ex = ex - tt = obj.read_typetree() - self.tt = tt - self.name = tt["m_Name"] - self.chars = {c["m_Unicode"]: c for c in tt["m_CharacterTable"]} - self.glyphs = {g["m_Index"]: g for g in tt["m_GlyphTable"]} - self.dynamic = tt["m_AtlasPopulationMode"] == 1 - self.atlases = [ex.deref(obj, p) for p in tt["m_AtlasTextures"]] - self.material_obj = ex.referent(obj, tt["m_Material"]) - self.fallback_objs = [ex.deref(obj, p) for p in tt["m_FallbackFontAssetTable"]] - if any(p["regularTypeface"]["m_PathID"] or p["italicTypeface"]["m_PathID"] for p in tt["m_FontWeightTable"]): - raise NotImplementedError(f"{self.name}: font weight table typefaces") - self.cmap: set[int] = set() - self.variants: dict[tuple[int, int], str] = {} # (base, selector) -> a variant glyph other than the base's - self.font_data: bytes | None = None - src = ex.deref(obj, tt["m_SourceFontFile"]) if tt["m_SourceFontFile"]["m_PathID"] else None - if src is not None: - data = bytes(src.read_typetree()["m_FontData"]) - self.cmap, self.variants = font_cmap(data, tt["m_FaceInfo"]["m_FaceIndex"]) - self.font_data = data - self.source_font = { - "name": src.read_typetree()["m_Name"], - "bytes": len(data), - "sha256": hashlib.sha256(data).hexdigest(), - } - else: - self.source_font = None - - def atlas_names(self) -> list[str | None]: - """The names of the atlas textures (None for a missing page).""" - return [a.read_typetree()["m_Name"] if a is not None else None for a in self.atlases] - - def atlas_settings(self) -> list[dict]: - """The sampler settings of the atlas textures that are there.""" - return [a.read_typetree()["m_TextureSettings"] for a in self.atlases if a is not None] - - def default_material(self) -> dict: - """The record of the default material (Exporter.material).""" - return self._ex.material(self.material_obj) - - class FontLookup: """TMP character lookup over font assets: a font's own characters, then its fallback chain (`fallbacks(f)`, as TextMeshPro searches it; see lookup). Results are (kind, font asset) with kind "baked", "runtime" (a dynamic asset @@ -197,49 +136,12 @@ def baked_anywhere(self, u: int) -> list[str]: return sorted(f.name for f in self.fonts.values() if u in f.chars) -class FontSet(FontLookup): - """Font assets reachable from the localized fonts, with TMP character lookup.""" - - def __init__(self, ex: Exporter): - super().__init__() - self.ex = ex - self._by_obj: dict[tuple, Font] = {} - - def by_key(self, key: str) -> Font: - return self.of(self.ex.key_object(key)) - - def loaded(self, name: str) -> Font: - """The font asset `name`, read by `of` or `by_key`.""" - return cast(Font, self.fonts[name]) - - def of(self, obj) -> Font: - k = _key(obj) - if k not in self._by_obj: - f = Font(self.ex, obj) - if f.name in self.fonts: - raise RuntimeError(f"two font assets named {f.name}") - self._by_obj[k] = f - self.fonts[f.name] = f - self._fallback_names[f.name] = [self.of(fb).name for fb in f.fallback_objs if fb is not None] - return self._by_obj[k] - - # -------------------------------------------------------------------------- # missing glyph # -------------------------------------------------------------------------- MISSING_GLYPH_CHARACTER = 0x25A1 # TMP_Settings.missingGlyphCharacter 0: U+25A1 (TMP_Text.SetArraySizes) -def missing_glyph_character(player) -> int: - """TMP_Settings.missingGlyphCharacter of the game's TMP Settings (0: U+25A1). FontSet.lookup searches no global - fallback: the settings' fallback font assets and default font asset must be unset.""" - o = player.resource("TMP Settings") - tt = player.mono(o) - if tt["m_fallbackFontAssets"] or tt["m_defaultFontAsset"]["m_PathID"]: - raise NotImplementedError("TMP Settings fallback font assets / default font asset") - return tt["m_missingGlyphCharacter"] or MISSING_GLYPH_CHARACTER - - def missing_glyph(fonts: FontLookup, font: FontAsset, chars, character: int = MISSING_GLYPH_CHARACTER) -> dict | None: """The missing glyph of a text of `font` (TMP_Text.SetArraySizes, a character GetTextElement finds in no font asset and no sprite asset): {unicode: the substitute, characters: the code points of `chars` that `font` and its @@ -253,13 +155,6 @@ def missing_glyph(fonts: FontLookup, font: FontAsset, chars, character: int = MI return {"unicode": sub, "characters": miss} -def underline_character(fonts: FontLookup, font: FontAsset) -> bool: - """Whether `font` itself has U+005F (TMP_Text.GetUnderlineSpecialCharacter: GetCharacterFromFontAsset without - fallbacks; a dynamic asset adds it from its source font): the glyph its texts' underline and highlight draw with.""" - r = fonts._in_font(font, 0x5F) - return r is not None and r[0] in ("baked", "runtime") - - def check_glyph_variants(fonts: FontLookup, font: FontAsset, texts) -> None: """Raise NotImplementedError when a character of `texts` that `font` (or a fallback) has is followed by a variation selector its source font maps to a variant glyph (SetArraySizes: GetGlyphVariantIndex -> the alternative glyph); @@ -460,7 +355,7 @@ def sdf(self, gi: int) -> tuple[dict, tuple, np.ndarray]: a = np.floor(255.0 * np.clip(0.5 + d / (2.0 * self.spread), 0.0, 1.0) + 0.5).astype(np.uint8) return m, (xmin, ymin, w, h), a - def validate(self, packer: TexelPacker | MemoryPacker, units) -> dict: + def validate(self, packer: MemoryPacker, units) -> dict: """Generate the baked characters `units` of the same font asset and compare with its atlas.""" f, p = self.font, self.pad errs, per, metric_diff = [], {}, {k: 0.0 for k in GLYPH_METRIC_FIELDS} @@ -516,13 +411,6 @@ def rgba_alpha(a: np.ndarray) -> np.ndarray: return out -def material_by_name(ex: Exporter, font: FontAsset, mat_name: str): - key = font_dir(font.name) + mat_name - if not ex.cat.has(key): - raise KeyError(f"material key {key}") - return ex.material(ex.key_object(key)) - - def shared_settings(name: str, settings: list[dict]) -> dict: """The sampler settings that all the atlas pages of the font asset `name` have (RuntimeError when they differ or there is no page).""" @@ -536,7 +424,7 @@ def runtime_glyphs( extra: set, baked: set, margin: int, - packer: TexelPacker | MemoryPacker, + packer: MemoryPacker, chars_out: dict, glyphs_out: dict, ) -> dict: @@ -683,22 +571,13 @@ def text_material_records( return materials, text_mats -def text_materials( - ex: Exporter, fonts: FontSet, localized: list[dict], primaries: set, needed: dict, runtime_units: dict -) -> tuple[dict, dict]: - """text_material_records for the font assets of `fonts`, their materials read through `ex`.""" - return text_material_records( - fonts, localized, primaries, needed, runtime_units, lambda f, name: material_by_name(ex, f, name) - ) - - def export_fonts( fonts: FontLookup, primaries: set, needed: dict, runtime_units: dict, text_mats: dict, - packer: TexelPacker | MemoryPacker, + packer: MemoryPacker, subset: str, ) -> dict: """Font asset records with the character / glyph tables reduced to the `needed` baked characters (plus the @@ -809,7 +688,7 @@ def value(r: dict) -> dict: def resolve_font_blocks(font_out: dict) -> None: - """After TexelPacker.build(): glyph texel blocks -> {texture, dx, dy} (integer offset into the packed texture).""" + """After MemoryPacker.build(): glyph texel blocks -> {texture, dx, dy} (integer offset into the packed texture).""" for fo in font_out.values(): for g in fo["glyphs"].values(): b = g.pop("_block", None) @@ -844,57 +723,6 @@ def __init__(self): self.material_ids: dict[str, int] = {} # material name -> its object's path id -class AssetFontSettings(BuiltFontSettings): - """FontSettings read from the game's font assets (`ex`: an Exporter of the catalog and player data): a font asset - at its address (fontsettings.font_key), its fallbacks through their references (FontSet), its materials at their - addresses. Needs the `fonts` extra for the source font files of dynamic assets.""" - - def __init__(self, ex): - super().__init__() - self.ex = ex - self._set = None - - def _fontset(self): - if self._set is None: - self._set = FontSet(self.ex) - return self._set - - def _font_obj(self, name: str) -> Font: - fs = self._fontset() - if name not in fs.fonts: - fs.by_key(font_key(self.ex.cat, name, name)) - return fs.loaded(name) - - def font(self, name: str) -> dict: - if name not in self.fonts: - f = self._font_obj(name) - mat = f.default_material() - self.materials.setdefault(mat["material"], mat) - self.material_ids.setdefault(mat["material"], f.material_obj.path_id) - self.font_ids[name] = f.obj.path_id - fallbacks = [fb.name for fb in self._fontset().fallbacks(f)] - self.fonts[name] = font_record(f, mat["material"], fallbacks, f.obj.assets_file.parent.name) - return self.fonts[name] - - def has_material(self, font: str, name: str) -> bool: - return name in self.materials or self.ex.cat.has(font_key(self.ex.cat, font, name)) - - def material(self, font: str | None, name: str) -> dict: - if name not in self.materials: - if font is None: - raise KeyError(f"material {name}: not read yet, and no font asset to find it by") - key = font_key(self.ex.cat, font, name) - if not self.ex.cat.has(key): - raise KeyError(f"material key {key}") - obj = self.ex.key_object(key) - self.material_ids[name] = obj.path_id - self.materials[name] = self.ex.material(obj) - return copy.deepcopy(self.materials[name]) - - def where(self) -> str: - return "the game's font assets" - - def listed_fonts(lang: dict) -> list[str]: """The font assets LocalizeManager lists for the languages (`lang`: textstyle.language_fonts): " SDF" for every font name and additional font name, sorted.""" @@ -931,12 +759,3 @@ def font_document(settings: BuiltFontSettings, names: list[str], game_version: s "materials": {n: table_material(settings.materials[n]) for n in sorted(settings.materials)}, "objects": dict(sorted(objects.items(), key=lambda kv: int(kv[0]))), } - - -def build_font_settings(ex, player, game_version: str) -> dict: - """The font settings table (fontsettings.FORMAT) from the game's assets: every font asset LocalizeManager lists - for a language (font names and additional font names) with the assets of their fallback chains, their default - materials and the localized text materials of the listed assets that the catalog has (table_material), and the - objects index of those assets.""" - settings = AssetFontSettings(ex) - return font_document(settings, load_listed(settings, language_fonts(player)), game_version) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index e7dc822..a401ac4 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -465,36 +465,6 @@ def test_flac_level_reaches_the_decode(data, tmp_path, capsys, monkeypatch): assert code == 0 and seen == [level], (argv, err) -def test_live_passes_the_flac_level_to_its_decodes(tmp_path, monkeypatch): - from nnnotes.content import live, liveaudio, score - from nnnotes.formats import cri - - class Stop(Exception): - pass - - seen = {} - monkeypatch.setattr(live, "resolve_band", lambda *a, **kw: {"band": 1}) - monkeypatch.setattr(score, "extract", lambda *a, **kw: seen.update(score=kw["flac_level"]) or {}) - - def sounds(*a, **kw): - seen["liveaudio"] = kw["flac_level"] - raise Stop - - monkeypatch.setattr(liveaudio, "extract", sounds) - with pytest.raises(Stop): - live.build(None, tmp_path, None, 7, "expert", tmp_path / "o", language="ja", flac_level=3) - assert seen == {"score": 3, "liveaudio": 3} - - def decode(cat, sheet, d, **kw): - seen["decode"] = kw - d.mkdir(parents=True) - (d / "streams.json").write_text("[]", encoding="utf-8") - - monkeypatch.setattr(cri, "decode", decode) - assert liveaudio._decode(None, "s", tmp_path, "flac", 3) == [] - assert seen["decode"] == {"fmt": "flac", "flac_level": 3} - - def test_export_live_asks_for_the_live_and_the_products_it_plays_with(data, tmp_path, capsys, monkeypatch): from nnnotes.cli import assets from nnnotes.content import livenotes diff --git a/tests/content/test_hostspot.py b/tests/content/test_hostspot.py deleted file mode 100644 index cabd194..0000000 --- a/tests/content/test_hostspot.py +++ /dev/null @@ -1,509 +0,0 @@ -"""hostspot (the spot files of a story's home host, read from bundles): empty material slots and mesh filters, what -draws nothing, materials and textures of a bundle whose serialized files reuse path ids, Spine characters and tap -targets with and without the references the prefab leaves empty, and the Spine files of a skeleton (synthetic -objects).""" - -import copy -import io -import json -import struct -from types import SimpleNamespace - -import numpy as np -import pytest -from PIL import Image -from UnityPy.classes import PPtr as UnityPPtr - -from nnnotes.content import hostspot, room -from nnnotes.unity.reader import DEFAULT_RESOURCES -from tests.helpers.fakeunity import F32, POSITION, UV0, mesh_tt, submesh, unitypy_mesh -from tests.helpers.spotobjects import fv, material, shader - - -class PPtr: - def __init__(self, path_id): - self.path_id = path_id - - -def test_empty_material_slots_are_skipped_and_reported(): - tris = [np.array([[0, 1, 2]]), np.zeros((0, 3), int), np.array([[2, 3, 0], [0, 1, 3]])] - mats = [PPtr(0), PPtr(11), PPtr(12)] - nulls, used = [], [] - - def material_for(p): - used.append(p.path_id) - return p.path_id * 10 - - prims = hostspot.submesh_primitives(tris, mats, material_for, nulls.append) - assert nulls == [0] and used == [12] # the empty submesh 1 gives nothing, silently - assert len(prims) == 1 and prims[0][1] == 120 - assert prims[0][0].tolist() == [2, 0, 3, 0, 3, 1] # winding reversed for glTF - assert hostspot.submesh_primitives(tris[:1], mats[:1], material_for, nulls.append) == [] and nulls == [0, 0] - - -class Obj: - """A component: its typetree and the object UnityPy reads for it.""" - - def __init__(self, tt, **fields): - self.tt, self.fields = tt, fields - - def read_typetree(self): - return self.tt - - def read(self): - return type("Read", (), self.fields)() - - -class Ref(PPtr): - def __init__(self, path_id, target=None): - super().__init__(path_id) - self.target = target - - def read(self): - assert self.path_id, "an empty reference is not read" - return self.target - - -class Graph: - tf_of_go = {1: 11, 2: 12, 3: 13, 4: 14} - - def path(self, tf_pid): - return f"Room/obj{tf_pid}" - - -def test_a_mesh_filter_without_a_mesh_is_skipped_and_reported(): - mesh = object() - mfs = { - ("f", 1): Obj({}, m_Mesh=Ref(21, mesh)), - ("f", 2): Obj({}, m_Mesh=Ref(0)), - ("f", 3): Obj({}, m_Mesh=Ref(0)), - ("f", 4): Obj({}, m_Mesh=Ref(24, mesh)), - } - mrs = { - ("f", 1): Obj({"m_Enabled": 1}, m_Materials=["m1"]), - ("f", 2): Obj({"m_Enabled": 1}, m_Materials=[]), - ("f", 3): Obj({"m_Enabled": 0}, m_Materials=[]), # 3: disabled renderer, 4: no renderer - ("g", 4): Obj({"m_Enabled": 1}, m_Materials=["m4"]), - } # another file's GameObject 4 - nulls = [] - drawn = list(hostspot.drawn_meshes(mfs, mrs, Graph(), nulls.append)) - assert drawn == [(11, mesh, ["m1"])] - assert nulls == ["Room/obj12"] # the disabled one draws nothing either way - - -def test_extract_room_lists_what_draws_nothing(tmp_path, monkeypatch): - """A mesh without triangles, a MeshFilter without a mesh and a submesh without a material are not written but - listed in hostspot.json.""" - - class Mesh: - def __init__(self, name): - self.m_Name = name - - class Cat: - def fetch_key(self, key): - return [] - - class SceneGraphStub(Graph): - def __init__(self, env): - pass - - def world(self, tf_pid): - return np.eye(4) - - def active(self, tf_pid): - return True - - tri = np.array([[0, 1, 2]]) - arrays = { - "quad": (np.zeros((3, 3)), np.empty((0, 3)), np.zeros((3, 2)), [tri]), - "placeholder": (np.zeros((4, 3)), np.empty((0, 3)), np.zeros((4, 2)), [np.zeros((0, 3), int)]), - "bare": (np.zeros((4, 3)), np.empty((0, 3)), np.zeros((4, 2)), []), - } - - def drawn(mf_by_go, mr_by_go, graph, on_null_mesh): - on_null_mesh("Room/obj14") - yield 11, Mesh("quad"), [PPtr(0)] - yield 12, Mesh("placeholder"), [] - yield 13, Mesh("bare"), [] - - monkeypatch.setattr(hostspot.UnityPy, "load", lambda *a: type("Env", (), {"objects": []})()) - monkeypatch.setattr(hostspot, "SceneGraph", SceneGraphStub) - monkeypatch.setattr(hostspot, "drawn_meshes", drawn) - monkeypatch.setattr(hostspot, "mesh_arrays", lambda mesh: arrays[mesh.m_Name]) - doc = hostspot.extract_room(Cat(), "Spot/a/Background/b", tmp_path / "hostspot.glb") - assert doc["meshCount"] == 0 and doc["materials"] == [] - assert doc["nullMeshFilters"] == ["Room/obj14"] - assert doc["meshesWithoutTriangles"] == [ - {"mesh": "placeholder", "path": "Room/obj12"}, - {"mesh": "bare", "path": "Room/obj13"}, - ] - assert doc["nullMaterialSubmeshes"] == [{"mesh": "quad", "path": "Room/obj11", "submesh": 0}] - assert (tmp_path / "hostspot.glb").is_file() and (tmp_path / "hostspot.json").is_file() - - -# ---------------------------------------------------------------- a bundle whose serialized files reuse path ids -V0, V1 = {"x": 0.0, "y": 0.0, "z": 0.0}, {"x": 1.0, "y": 1.0, "z": 1.0} -Q0 = {"x": 0.0, "y": 0.0, "z": 0.0, "w": 1.0} -ROOM_KEY = "Spot/a/Background/b" -PREFAB = "Assets/Room.prefab" -# the quads under the prefab root: (object name, material path id, its file id in CAB-a: 1 is CAB-b) -ROOM_OBJECTS = [("wall", 5, 0), ("wall_b", 5, 1), ("floor", 6, 0), ("wall_c", 5, 0)] - - -def ref(pid, file_id=0): - return {"m_FileID": file_id, "m_PathID": pid} - - -class File: - """A serialized file as UnityPy's PPtr resolves it: name, objects by path id, externals (paths) and the bundle - holding it (`parent`); a file outside the loaded bundles is not found.""" - - def __init__(self, name, bundle, externals=()): - self.name, self.parent, self.objects = name, bundle, {} - self.externals = [SimpleNamespace(path=p) for p in externals] - self.environment = SimpleNamespace(find_file=lambda name: None) - bundle.files[name] = self - - def add(self, kind, pid, tt, obj=None): - o = SimpleNamespace( - type=SimpleNamespace(name=kind), - path_id=pid, - assets_file=self, - read_typetree=lambda: copy.deepcopy(tt), - read=lambda: obj, - ) - self.objects[pid] = o - return o - - def pptr(self, pid, file_id=0): - """A PPtr as UnityPy parses it from an object of this file.""" - return UnityPPtr(m_FileID=file_id, m_PathID=pid, assetsfile=self) - - -def tex_env(pid, file_id=0): - return {"m_Texture": ref(pid, file_id), "m_Scale": {"x": 1.0, "y": 1.0}, "m_Offset": {"x": 0.0, "y": 0.0}} - - -def textured(name, **slots): - """A material typetree of shader 3 with the texture envs `slots` {slot: tex_env}.""" - tt = material(name=name) - tt.update(m_Shader=ref(3), m_ValidKeywords=[], m_CustomRenderQueue=-1) - tt["m_SavedProperties"]["m_TexEnvs"] = [[k, v] for k, v in slots.items()] - return tt - - -def quad(name): - return unitypy_mesh( - mesh_tt(name, 3, {POSITION: (F32, 3, np.eye(3)), UV0: (F32, 2, np.zeros((3, 2)))}, [0, 1, 2], [submesh(0, 3)]) - ) - - -def two_file_room(): - """The environment of a bundle of two serialized files that reuse path ids. CAB-a holds the background prefab - (root `room`, GameObject 100, over ROOM_OBJECTS), shader 3, material 5 `a_wall` (main texture its texture 7, - `_Mask` in unity default resources), material 6 `a_floor` (main texture CAB-b's texture 7, `_Detail` its own - texture 7) and texture 7 `wall_a`. CAB-b holds shader 3, material 5 `b_wall` (main texture its texture 7) and - texture 7 `wall_b`: the pixels of `wall_a` with another filter.""" - bundle = SimpleNamespace(files={}) - a = File("CAB-a", bundle, ["archive:/CAB-b/CAB-b", DEFAULT_RESOURCES]) - b = File("CAB-b", bundle) - pixels = Image.new("RGBA", (2, 2), (200, 120, 40, 255)) - for f in (a, b): - f.add("Shader", 3, shader(name="Unlit/Transparent", srcBlend=fv(5.0), destBlend=fv(10.0))) - a.add("Material", 5, textured("a_wall", _MainTex=tex_env(7), _Mask=tex_env(9, 2))) - a.add("Material", 6, textured("a_floor", _MainTex=tex_env(7, 1), _Detail=tex_env(7))) - b.add("Material", 5, textured("b_wall", _MainTex=tex_env(7))) - for f, name, filter_mode in ((a, "wall_a", 1), (b, "wall_b", 0)): - f.add( - "Texture2D", - 7, - { - "m_Name": name, - "m_MipCount": 1, - "m_TextureSettings": {"m_FilterMode": filter_mode, "m_WrapU": 1, "m_WrapV": 1}, - }, - pixels.copy(), - ) - a.add("AssetBundle", 1, {"m_Container": [[PREFAB.lower(), {"asset": ref(100)}]]}) - a.add("GameObject", 100, {"m_Name": "room", "m_IsActive": 1, "m_Layer": 0}) - a.add( - "Transform", - 200, - { - "m_GameObject": ref(100), - "m_Father": ref(0), - "m_LocalPosition": V0, - "m_LocalRotation": Q0, - "m_LocalScale": V1, - "m_Children": [ref(201 + i) for i in range(len(ROOM_OBJECTS))], - }, - ) - for i, (name, mat, file_id) in enumerate(ROOM_OBJECTS): - go, mesh = 101 + i, quad(f"{name}_mesh") - a.add("GameObject", go, {"m_Name": name, "m_IsActive": 1, "m_Layer": 0}) - a.add( - "Transform", - 201 + i, - { - "m_GameObject": ref(go), - "m_Father": ref(200), - "m_Children": [], - "m_LocalPosition": V0, - "m_LocalRotation": Q0, - "m_LocalScale": V1, - }, - ) - a.add( - "MeshFilter", - 301 + i, - {"m_GameObject": ref(go)}, - SimpleNamespace(m_Mesh=SimpleNamespace(path_id=21 + i, read=lambda mesh=mesh: mesh)), - ) - a.add( - "MeshRenderer", - 401 + i, - {"m_GameObject": ref(go), "m_Enabled": 1}, - SimpleNamespace(m_Materials=[a.pptr(mat, file_id)]), - ) - return SimpleNamespace(objects=[o for f in (a, b) for o in f.objects.values()]) - - -class RoomCatalog: - def fetch_key(self, key): - return [] - - def entry(self, key): - return {"internal_id": PREFAB} - - -def load_room(monkeypatch, env): - """UnityPy.load gives `env`; a texture's image is the object it reads as.""" - monkeypatch.setattr(hostspot.UnityPy, "load", lambda *a: env) - monkeypatch.setattr(hostspot, "texture_image", lambda image: image) - - -def glb_json(path): - data = path.read_bytes() - (n,) = struct.unpack_from(" bytes: - buf = io.BytesIO() - image.save(buf, format="PNG") - return buf.getvalue() - - -def test_write_skeleton_writes_the_skeleton_atlas_and_pages_in_order(): - a, b = images() - shared = SpineRef(40, atlas_asset("chr", [Tex(a), Tex(b)])) - got = [] - written = {} - rec = hostspot.write_skeleton(skeleton("chr", b"\x00binary", [shared]), lambda n, d: got.append((n, d)), written) - assert [n for n, _ in got] == ["chr.skel", "chr.atlas", "p1.png", "p2.png"] - assert got[0][1] == b"\x00binary" and got[1][1] == ATLAS_TEXT.encode() - assert got[2][1] == png_of(a) and got[3][1] == png_of(b) - assert rec == { - "name": "chr_SkeletonData", - "skeleton": "chr.skel", - "atlases": ["chr.atlas"], - "scale": 0.01, - "defaultMix": 0.2, - "mixes": [{"from": "a", "to": "b", "duration": 0.5}], - } - assert written == {40: "chr.atlas"} - got.clear() # a second skeleton on the same atlas: written once - rec2 = hostspot.write_skeleton( - skeleton("chr2", ' {"skeleton": 1}', [shared]), lambda n, d: got.append((n, d)), written - ) - assert got == [("chr2.json", b' {"skeleton": 1}')] and rec2["atlases"] == ["chr.atlas"] - - -def test_write_skeleton_keeps_a_name_that_already_has_the_extension(): - a, b = images() - got = [] - shared = SpineRef(40, atlas_asset("chr.atlas", [Tex(a), Tex(b)])) - rec = hostspot.write_skeleton(skeleton("chr.skel", b"\x00binary", [shared]), lambda n, d: got.append((n, d))) - assert [n for n, _ in got] == ["chr.skel", "chr.atlas", "p1.png", "p2.png"] and rec["skeleton"] == "chr.skel" - - -def test_write_skeleton_checks_pages_against_materials(): - a, _ = images() - with pytest.raises(RuntimeError, match="2 pages vs 1 materials"): - hostspot.write_skeleton( - skeleton("x", b"s", [SpineRef(41, atlas_asset("x.atlas", [Tex(a)]))]), lambda n, d: None - ) diff --git a/tests/content/test_live_options.py b/tests/content/test_live_options.py index cc25e06..80b9d0b 100644 --- a/tests/content/test_live_options.py +++ b/tests/content/test_live_options.py @@ -4,7 +4,7 @@ import pytest -from nnnotes.content import live, liveaudio, livenotes, liveoptions, livescene, score +from nnnotes.content import liveaudio, liveoptions, score from nnnotes.content.liveoptions import LiveOptions, OptionSpecError, parse_specs, resolve from tests.helpers.live import CHART @@ -225,115 +225,10 @@ def test_effect_sets(tmp_path): assert LiveOptions().effect_sets(names) == [] -def test_option_names_follow_the_game_enum(): - assert livescene.OPTION_ITEM_TYPES is liveoptions.OPTION_ITEM_TYPES - t = liveoptions.OPTION_ITEM_TYPES - assert (t[2], t[421], t[436], t[465], t[999]) == ( - "NoteTiming", - "UseIndividualNoteSe", - "SideFlickSeId", - "TraceSeMute", - "SongTitleDisplay", - ) - assert set(liveoptions.PLAYER_ITEMS) <= set(t) and not set(liveoptions.PLAYER_ITEMS) & set(livenotes.OPTION_ITEMS) - - # ---------------------------------------------------------------- note assets settings -def test_livenotes_settings_default_unchanged(tmp_path): - m = write_master(tmp_path / "m") - a = livenotes.settings(m) - assert livenotes.settings(m, LiveOptions()) == a - assert json.dumps(a) == json.dumps(livenotes.settings(m, LiveOptions())) - assert set(a["optionDefaults"]) == set(livenotes.OPTION_ITEMS.values()) - assert "skins" not in a and "effects" not in a and "NoteTiming" not in a["optionRanges"] - - -def test_livenotes_settings_tables_and_variants(tmp_path): - m = write_master(tmp_path / "m") - a = livenotes.settings(m) - t = livenotes.settings(m, resolve({"defaults": True}, m)) - names = {liveoptions.OPTION_ITEM_TYPES[k] for k in liveoptions.PLAYER_ITEMS} - assert livenotes.settings(m, resolve(parse_specs(["MeasureLineDisplay"]), m)) == t # the tables only - assert set(t["optionDefaults"]) == set(a["optionDefaults"]) | names - assert {k: t["optionDefaults"][k] for k in a["optionDefaults"]} == a["optionDefaults"] - assert t["optionDefaults"]["NoteTiming"] == "0.00" and t["optionDefaults"]["TapSeVolume"] == "90" - assert t["optionRanges"]["NoteTiming"] == [-300, 300] and t["optionRanges"]["LiveMusicVolume"] == [0, 100] - assert "QualitySetting" not in t["optionRanges"] # not an item the player offers - o = livenotes.settings(m, resolve(parse_specs(["NoteDesignId=1,3", "LiveQuality=2"]), m)) - assert o["skins"] == {"1": "skin001", "3": "skin003"} and o["effects"] == {"1": "fxA"} - assert o["skin"] == "skin001" and o["effect"] == "fxA" - o = livenotes.settings(m, resolve(parse_specs(["NoteEffectId"]), m)) - assert o["effects"] == {"1": "fxA", "2": "fxASimple"} and "skins" not in o # ---------------------------------------------------------------- bar line view -class _Obj: - """A serialized object: type, path id, typetree, script class.""" - - def __init__(self, t, pid, tt, cls=None): - self.type = type("T", (), {"name": t})() - self.path_id, self._tt, self.cls = pid, tt, cls - - def read_typetree(self): - return self._tt - - -class _Graph: - def __init__(self, paths, fathers): - self.paths, self.tf = paths, {tf: {"m_Father": {"m_PathID": f}} for tf, f in fathers.items()} - self.tf_of_go = {go: tf for tf, (go, _) in paths.items()} - self.go = {} - - def gos_at_path(self, path): - return [go for tf, (go, p) in self.paths.items() if p == path] - - -class _Ex: - """The Exporter calls bar_line_prefab makes, over a scene with LiveAllNoteView -> LiveAllBarLineView -> - LiveBarLineViewContainer -> the LiveBarLineView prefab (GameObject 50, transform 51, a root).""" - - def __init__(self, element_cls="LiveBarLineView", prefab_father=0): - def p(pid): - return {"m_FileID": 0, "m_PathID": pid} - - objs = [ - _Obj("GameObject", 10, {}), - _Obj("MonoBehaviour", 11, {"m_GameObject": p(10), "_barLineView": p(21)}, "LiveAllNoteView"), - _Obj("MonoBehaviour", 21, {"m_GameObject": p(20), "_container": p(31)}, "LiveAllBarLineView"), - _Obj("MonoBehaviour", 31, {"m_GameObject": p(30), "_elementPrefab": p(52)}, "LiveBarLineViewContainer"), - _Obj("MonoBehaviour", 52, {"m_GameObject": p(50), "_renderer": p(62)}, element_cls), - ] - self.objs = {o.path_id: o for o in objs} - self.env = type("Env", (), {"objects": objs})() - self.graph = _Graph({12: (10, livenotes.NOTE_VIEW[0]), 51: (50, "LiveBarLineView")}, {12: 5, 51: prefab_father}) - self.graph.go = {10: {"m_Component": [{"component": p(11)}]}} - self.loaded, self.walked = [], [] - - def load(self, key): - self.loaded.append(key) - return self.env, self.graph - - def deref(self, owner, pptr): - return self.objs.get(pptr["m_PathID"]) - - def script_class(self, o): - return o.cls - - def hierarchy(self, env, graph, root_tf): - self.walked.append(root_tf) - return [{"path": "LiveBarLineView", "components": [{"type": "MonoBehaviour", "class": "LiveBarLineView"}]}] - - -def test_bar_line_prefab_follows_the_scene_references(): - ex = _Ex() - rec = livenotes.bar_line_prefab(ex) - assert ex.loaded == [livenotes.SCENE_KEY] and ex.walked == [51] - assert rec["key"] == "EmbScene/Live" and rec["reference"] == "_barLineView._container._elementPrefab" - assert rec["nodes"][0]["path"] == "LiveBarLineView" - with pytest.raises(RuntimeError, match="LiveBarLineView MonoBehaviour"): - livenotes.bar_line_prefab(_Ex(element_cls="Other")) - with pytest.raises(RuntimeError, match="not a prefab root"): - livenotes.bar_line_prefab(_Ex(prefab_father=12)) # ---------------------------------------------------------------- note sounds @@ -403,22 +298,3 @@ def test_mirrored_conversion(): assert y["laneIndexFloat"] == 24 - x["laneIndexFloat"] - x["widthFloat"] assert [n["direction"] for n in b["notes"]][:2] == ["Normal", "Right"] # ApplyMirrorDirection assert b["notes"][4]["laneIndex"] == 16 # 16.5 rounds to even - - -def test_live_index_doc(): - s = {"chart": "score/c.json", "notes": "score/c.notes.json", "master": "score/master.json"} - a = live.index_doc(1, "expert", s, {"cueSheet": "x"}, "audio/live-audio.json") - assert list(a) == [ - "musicId", - "difficulty", - "chart", - "notes", - "master", - "audio", - "liveAudio", - "scene", - "noteAssets", - "liveUi", - ] - b = live.index_doc(1, "expert", {**s, "notesMirror": "score/c.mirror.notes.json"}, {}, "audio/live-audio.json") - assert list(b)[3:5] == ["notes", "notesMirror"] and b["notesMirror"] == "score/c.mirror.notes.json" diff --git a/tests/content/test_story_fonts.py b/tests/content/test_story_fonts.py index 010b5ae..2e6fb9f 100644 --- a/tests/content/test_story_fonts.py +++ b/tests/content/test_story_fonts.py @@ -12,7 +12,7 @@ pytest.importorskip("scipy") pytest.importorskip("fontTools") -from nnnotes.content import advui, storyfonts +from nnnotes.content import storyfonts from nnnotes.core import cache from nnnotes.unity import fontsettings, tmpfont # noqa: E402 from tests.helpers.fonts import UPEM, make_font @@ -223,232 +223,6 @@ def test_open_asset_checks_the_gradient_scale(font): storyfonts.open_asset(font, "X SDF", [0x41], g, {}) -def test_language_doc(): - lang = {"languages": {0: {"fontNames": ["GameA", "GameNum"], "additionalFontNames": []}}, "materialTypes": []} - fonts = { - "Open SDF": { - "faceInfo": { - "m_PointSize": 40.0, - "m_Scale": 1.0, - "m_LineHeight": 40.0, - "m_AscentLine": 35.2, - "m_DescentLine": -4.8, - }, - "normalSpacingOffset": 0.0, - "boldSpacing": 7.0, - } - } - texts = { - "a/b": { - "fontAsset": "GameA SDF", - "localized": {"fontAsset": "Open SDF", "material": "m", "lineSpacing": -100.0}, - } - } - doc = storyfonts.language_doc("en", "open", fonts, texts, lang) - assert doc == { - "format": storyfonts.LANGUAGE_FORMAT, - "language": "en", - "mode": 1, - "field": "english", - "lineSpacing": -100.0, - "fonts": "open", - "roles": { - "primary": { - "fontAsset": "Open SDF", - "lineHeightEm": 1.0, - "ascentEm": 0.88, - "descentEm": -0.12, - "spacingOffset": 0.0, - "boldSpacing": 7.0, - } - }, - } - - -def test_line_breaking(): - class Obj: - def __init__(self, text): - self.text = text - - def read_typetree(self): - return {"m_Script": self.text} - - class Player: - def resource(self, path): - assert path == "TMP Settings" - return "settings" - - def mono(self, o): - return { - "m_leadingCharacters": {"m_PathID": 1}, - "m_followingCharacters": {"m_PathID": 2}, - "m_UseModernHangulLineBreakingRules": 0, - } - - def deref(self, owner, pptr): - return Obj({1: "([", 2: ")]"}[pptr["m_PathID"]]) - - assert storyfonts.line_breaking(Player()) == { - "leading": "([", - "following": ")]", - "useModernHangulLineBreakingRules": False, - } - - -def test_shown_characters(): - episode = { - "text": {"a": {"english": "Hi x", "japanese": "ね"}, "b": {"english": None}}, - "title": {"english": "T1"}, - "commands": [], - } - ui = {"nodes": [{"path": "n", "textStyle": {"textKey": None, "text": "placeholder"}}]} - got = storyfonts.shown_characters(episode, ui, None, "english") - assert got == sorted({ord(c) for c in "Hi xT1"}) - # chat windows: their status texts and the texts they format at run time - chat = {**episode, "commands": [{"cmd": "ChatWindow", "Parameter1": "57", "Parameter4": "4"}]} - got = storyfonts.shown_characters(chat, {**ui, "chatStatusTexts": {"k": "Call"}}, None, "english") - assert got == sorted({ord(c) for c in "Hi xT1Call...57%(4)"}) - ui["nodes"][0]["textStyle"]["textKey"] = "k" - with pytest.raises(RuntimeError, match="master data"): - storyfonts.shown_characters(episode, ui, None, "english") - - -def test_game_fonts_moves_the_records(tmp_path, monkeypatch): - game_ui, ui = tmp_path / "game" / "ui", tmp_path / "open" / "ui" - (game_ui / "textures").mkdir(parents=True) - ui.mkdir(parents=True) - (game_ui / "textures" / "font_G_SDF.png").write_bytes(b"png") - text = { - "m_fontSize": 30, - "m_text": "", - "class": "TextMeshProUGUI", - "enabled": 1, - "fontAsset": "G SDF", - "material": "G - Default", - "localized": {"fontAsset": "G SDF", "material": "G - Default", "lineSpacing": 0.0}, - } - font = { - "faceInfo": { - "m_PointSize": 40.0, - "m_Scale": 1.0, - "m_LineHeight": 40.0, - "m_AscentLine": 30.0, - "m_DescentLine": -10.0, - }, - "normalSpacingOffset": 0.0, - "boldSpacing": 7.0, - "material": "G SDF Material", - "fallbacks": ["Other SDF"], - "glyphs": {"5": {"packed": {"texture": "font_G SDF", "dx": 1, "dy": 2}}, "3": {}}, - } - doc = { - "nodes": [{"path": "t", "text": text}, {"path": "img"}], - "fonts": {"G SDF": font}, - "textures": { - "font_G SDF": { - "texture": "textures/font_G_SDF.png", - "name": "font_G SDF", - "width": 8, - "height": 8, - "mipCount": 1, - "settings": {}, - }, - "sprites": {"texture": "textures/sprites.png"}, - }, - "materials": { - "G - Default": {"material": "G - Default"}, - "G SDF Material": {"material": "G SDF Material"}, - "UI-Transition": {}, - }, - "glyphCoverage": {"characters": 2, "missing": ["?"]}, - "tmpSettings": {"m_warningsDisabled": 0}, - } - (game_ui / "ui.json").write_text(json.dumps(doc), encoding="utf-8") - (ui / "ui.json").write_text( - json.dumps({"nodes": [{"path": "t", "textStyle": {}}, {"path": "img"}]}), encoding="utf-8" - ) - - class Ex: - def key_object(self, key): - assert key == "EmbFont/G/G - Default" - return key - - def material(self, o): - return {} - - ruby = { - "class": "RubyTextMeshProUGUI", - "_rubyVerticalOffset": "1em", - "_rubyScale": 0.5, - "_rubyLineHeight": "", - "_rubyShowType": 0, - "_rubyMargin": 10.0, - "m_tintAllSprites": 0, - } - sprite = { - "asset_name": "Emoji", - "asset": {"characters": [], "glyphs": {}, "material": "Emoji Material"}, - "textures": {"sprite_Emoji": {"texture": "fonts/sprite_Emoji.png"}}, - "material": {"material": "Emoji Material"}, - } - monkeypatch.setattr(storyfonts, "_exporter", lambda cat, player, work: Ex()) - monkeypatch.setattr( - storyfonts, - "text_components", - lambda ex, ui_doc: {"t": [ruby, {"class": "UIRubyText", "_rubyMarginTop": -22.0}]}, - ) - monkeypatch.setattr(storyfonts, "story_ui_text_targets", lambda ex, ui_doc: {"t"}) - monkeypatch.setattr(storyfonts.advui, "sprite_asset_record", lambda ex: "rec") - monkeypatch.setattr(storyfonts, "shown_texts", lambda episode, ui_doc, master_dir, field: ["?\U0001f600"]) - monkeypatch.setattr(storyfonts, "drawn_by_sprites", lambda ex, rec, bindings, chars: {"G SDF": {ord("?")}}) - monkeypatch.setattr(storyfonts, "sprite_needs", lambda rec, texts, drawn: [0] if drawn == {ord("?")} else []) - monkeypatch.setattr(storyfonts, "game_sprite_asset", lambda ex, ui_dir, rec: sprite) - monkeypatch.setattr(storyfonts, "text_shaders", lambda ex, ui_dir, mats: {n: [] for n in mats}) - monkeypatch.setattr( - storyfonts, - "fallback_chains", - lambda ex, primaries, exported: ( - {n: [] for n in exported} if primaries == {"G SDF"} else pytest.fail(f"primaries {primaries}") - ), - ) - monkeypatch.setattr( - storyfonts, - "line_breaking", - lambda player: {"leading": "", "following": "", "useModernHangulLineBreakingRules": False}, - ) - monkeypatch.setattr( - storyfonts.textstyle, - "language_fonts", - lambda player: {"languages": {0: {"fontNames": ["G"]}}, "materialTypes": ["Default"]}, - ) - r = storyfonts.game_fonts(None, None, game_ui, ui, "ja", {"text": {}}) - out = json.loads((ui / "fonts.json").read_text(encoding="utf-8")) - assert r["pages"] == 2 and (ui / "fonts" / "font_G_SDF.png").read_bytes() == b"png" - assert out["source"] == "game" and out["fonts"] == {"G SDF": {**font, "fallbacks": []}} - assert out["texts"] == { - "t": { - **text, - "ruby": {k: ruby[k] for k in storyfonts.RUBY_FIELDS}, - "uiRubyText": {"_rubyMarginTop": -22.0}, - "spriteAsset": "Emoji", - "m_tintAllSprites": 0, - } - } - assert out["textures"] == { - "font_G SDF": {**doc["textures"]["font_G SDF"], "texture": "fonts/font_G_SDF.png"}, - **sprite["textures"], - } - assert set(out["materials"]) == {"G - Default", "G SDF Material", "Emoji Material"} - assert out["tmpSettings"] == doc["tmpSettings"] - assert out["spriteAssets"] == {"Emoji": sprite["asset"]} and out["emojiSpriteAsset"] == "Emoji" - assert out["coverage"] == {"characters": 2, "missing": [], "sprites": {"characters": 0, "missing": []}} - storyfonts.game_fonts(None, None, game_ui, ui, "ja") # without the episode: no sprite asset - out = json.loads((ui / "fonts.json").read_text(encoding="utf-8")) - assert "spriteAssets" not in out and "spriteAsset" not in out["texts"]["t"] and out["coverage"]["missing"] == ["?"] - lang = json.loads((ui / "languages.json").read_text(encoding="utf-8")) - assert lang["roles"]["primary"]["fontAsset"] == "G SDF" and lang["mode"] == 0 - - def test_text_extras(): comps = [ { @@ -481,71 +255,6 @@ def test_text_extras(): storyfonts.text_extras("p", [{"class": "RubyTextMeshProUGUI", "_rubyScale": 0.5}]) -def test_missing_glyph(): - """The game's missing glyph: the code points no font of the chain has, the substitute U+25A1, else space, else - U+0003; the TMP settings it needs; glyph variants of a variation selector raise.""" - - class F: - def __init__(self, name, variants=()): - self.name, self.variants = name, dict.fromkeys(variants, "v") - - class Fonts: - def __init__(self, have): - self.have = have - - def lookup(self, font, u): - return ("baked", self.have[u]) if u in self.have else None - - p, fb = F("P"), F("FB", [(0x2661, 0xFE0F)]) - fonts = Fonts({0x41: p, 0x25A1: fb, 0x20: p, 0x2661: fb, 0x03: p}) - assert tmpfont.missing_glyph(fonts, p, [0x41, 0x20]) is None - assert tmpfont.missing_glyph(fonts, p, [0x41, 0xFE0F, 0x26A1]) == { - "unicode": 0x25A1, - "characters": [0x26A1, 0xFE0F], - } - no_box = Fonts({0x41: p, 0x20: p, 0x03: p}) - assert tmpfont.missing_glyph(no_box, p, [0x42])["unicode"] == 0x20 - assert tmpfont.missing_glyph(Fonts({0x03: p}), p, [0x42])["unicode"] == 0x03 - assert tmpfont.missing_glyph(fonts, p, [0x42], 0x3F)["unicode"] == 0x20 # a set missingGlyphCharacter - assert [ - u for u in (0xFDFF, 0xFE00, 0xFE0F, 0xFE10, 0xE0100, 0xE01EF, 0xE01F0) if fontsettings.is_variation_selector(u) - ] == [0xFE00, 0xFE0F, 0xE0100, 0xE01EF] - tmpfont.check_glyph_variants(fonts, p, ["A\ufe0f", "\u2661", "\u26a1\ufe0f"]) - with pytest.raises(NotImplementedError, match="glyph variant of U\\+2661 U\\+FE0F"): - tmpfont.check_glyph_variants(fonts, p, ["x\u2661\ufe0f"]) - - class Own: - def __init__(self, r): - self.r = r - - def _in_font(self, font, u): - assert u == 0x5F - return self.r - - assert [tmpfont.underline_character(Own(r), p) for r in (("baked", p), ("runtime", p), None)] == [True, True, False] - - class Player: - def __init__(self, **tt): - self.tt = { - "m_fallbackFontAssets": [], - "m_defaultFontAsset": {"m_FileID": 0, "m_PathID": 0}, - "m_missingGlyphCharacter": 0, - **tt, - } - - def resource(self, path): - assert path == "TMP Settings" - return "settings" - - def mono(self, o): - return self.tt - - assert tmpfont.missing_glyph_character(Player()) == 0x25A1 - assert tmpfont.missing_glyph_character(Player(m_missingGlyphCharacter=0x3F)) == 0x3F - with pytest.raises(NotImplementedError, match="fallback font assets"): - tmpfont.missing_glyph_character(Player(m_fallbackFontAssets=[{"m_PathID": 5}])) - - def test_search_order(): """TMP's depth-first fallback search, every asset once, reduced to the exported assets.""" chain = {"P": ["A", "B"], "A": ["P", "C"], "B": ["C", "D"], "C": [], "D": ["A"]} @@ -558,234 +267,6 @@ def test_search_order(): assert storyfonts.search_order(fallbacks, "C", {"P"}) == [] -def test_chat_bindings(monkeypatch): - """A binding per chat window text: the serialized fields, the extras, `localized` from advui.chat_localize for - a localized text and the serialized font, material and line spacing for the others.""" - name = { - "class": "TextMeshProUGUI", - "m_Enabled": 1, - "m_fontAsset": {"name": "CHAT SDF"}, - "m_sharedMaterial": {"material": "CHAT - Default"}, - "m_fontSize": 30.0, - "m_text": "", - "m_lineSpacing": 2.0, - "m_monospaceDistEm": 0.0, - } - pct = { - **name, - "class": "RubyEmojiTextMeshProUGUI", - "m_text": "100%", - "_rubyVerticalOffset": "1em", - "_rubyScale": 0.5, - "_rubyLineHeight": "", - "_rubyShowType": 0, - "_rubyMargin": 0.0, - } - loc = {"class": "LocalizeText", "m_Enabled": 1, "_localizeEnabled": 1} - nodes = [ - ("Line", {"path": "Line/Name", "components": [name, loc]}, name, loc), - ("Line", {"path": "Line/Pct", "components": [pct]}, pct, None), - ] - monkeypatch.setattr(storyfonts.advui, "chat_text_nodes", lambda ex, episode: (nodes, [])) - monkeypatch.setattr(storyfonts.advui, "chat_windows", lambda episode: {}) - lang = { - "languages": { - 0: {"fontNames": ["J"], "additionalFontNames": ["CHAT"]}, - 1: {"fontNames": ["E"], "additionalFontNames": ["EC"]}, - }, - "materialTypes": ["Default"], - } - got = storyfonts.chat_bindings(None, {}, lang, 1) - assert set(got) == {"Line"} and set(got["Line"]) == {"Line/Name", "Line/Pct"} - n, p = got["Line"]["Line/Name"], got["Line"]["Line/Pct"] - assert n["localized"] == {"fontAsset": "EC SDF", "material": "EC - Default", "lineSpacing": -100.0} - assert (n["class"], n["fontAsset"], n["m_monospaceDistEm"]) == ("TextMeshProUGUI", "CHAT SDF", 0.0) - assert p["localized"] == {"fontAsset": "CHAT SDF", "material": "CHAT - Default", "lineSpacing": 2.0} - assert p["ruby"]["_rubyScale"] == 0.5 and "m_lineSpacing" in p - assert "m_monospaceDistEm" in advui.TMP_FIELDS - - -def test_frame_bindings_and_texts(): - """A binding per TMP text of each frame (as chat_bindings, a UIText's text with the sprite asset); the serialized - frame texts and the texts a frame with an IAdvFrameTextReceiver gets at run time from its Frame rows' - TargetTextIDs (NFC, then "@" and a line feed).""" - body = { - "class": "TextMeshProUGUI", - "m_Enabled": 1, - "m_fontAsset": {"name": "J SDF"}, - "m_sharedMaterial": {"material": "J - Default"}, - "m_fontSize": 52.0, - "m_text": "", - "m_lineSpacing": 0.0, - "m_tintAllSprites": 0, - } - loc = {"class": "LocalizeText", "m_Enabled": 1, "_localizeEnabled": 1} - ui = {"class": "UIText", "_targetText": {"gameObject": "Card/Body"}} - frames = { - "slander": [ - {"path": "Card", "components": [{"class": "AdvSlanderCommentFrame"}]}, - {"path": "Card/Body", "components": [body, loc, ui]}, - ], - "plain": [ - {"path": "Img", "components": [{"class": "Image"}]}, - {"path": "Label", "components": [{**body, "m_text": "Fixed"}]}, - ], - } - lang = { - "languages": { - 0: {"fontNames": ["J"], "additionalFontNames": []}, - 1: {"fontNames": ["E"], "additionalFontNames": []}, - }, - "materialTypes": ["Default"], - } - got = storyfonts.frame_bindings(frames, lang, 1, "Emoji") - assert set(got) == {"slander", "plain"} and set(got["slander"]) == {"Card/Body"} and set(got["plain"]) == {"Label"} - b = got["slander"]["Card/Body"] - assert b["localized"] == {"fontAsset": "E SDF", "material": "E - Default", "lineSpacing": -100.0} - assert (b["spriteAsset"], b["m_tintAllSprites"], b["m_fontSize"]) == ("Emoji", 0, 52.0) - episode = { - "text": {"t1": {"english": "Ame\u0301lie"}, "t2": {"english": "id"}, "t3": {"english": 5}}, - "commands": [ - {"cmd": "Frame", "TargetAssetName": "slander", "TargetTextIDs": ["t1", "t2", "t3", "nope"]}, - {"cmd": "Frame", "TargetAssetName": "plain", "TargetTextIDs": ["t2"]}, - {"cmd": "Frame", "TargetAssetName": "slander", "TargetTextIDs": ["t2"], "IgnoreData": 1}, - {"cmd": "Talk", "TargetTextIDs": ["t2"]}, - ], - } - assert storyfonts.frame_texts(frames, episode, "english") == ["Fixed", "Am\u00e9lie", "id", "@\n"] - assert storyfonts.frame_texts({"plain": frames["plain"]}, episode, "english") == ["Fixed"] - - -def test_dialog_bindings(): - """A binding per text node of each dialog of ui.json, from the components of the dialog's prefab; `localized` by - LocalizeText's font swap in the language.""" - text = { - "class": "TextMeshProUGUI", - "m_Enabled": 1, - "m_fontAsset": {"name": "J SDF"}, - "m_sharedMaterial": {"material": "J - Default"}, - "m_fontSize": 30.0, - "m_text": "", - "m_lineSpacing": 0.0, - } - - class Ex: - def prefab(self, key): - assert key == "EmbUI/Prefab/D" - return { - "nodes": [ - {"path": "D/Title", "components": [text, {"class": "LocalizeText", "m_Enabled": 1}]}, - { - "path": "D/Body", - "components": [ - {**text, "m_tintAllSprites": 0}, - {"class": "UIText", "_targetText": {"gameObject": "D/Body"}}, - ], - }, - {"path": "D/Image", "components": [{"class": "Image"}]}, - ] - } - - ui = { - "dialogs": { - "D": { - "key": "EmbUI/Prefab/D", - "nodes": [ - {"path": "D/Title", "textStyle": {"textKey": "ui_ok"}}, - {"path": "D/Body", "textStyle": {}}, - {"path": "D/Image"}, - ], - } - } - } - lang = {"languages": {0: {"fontNames": ["J"]}, 1: {"fontNames": ["E"]}}, "materialTypes": ["Default"]} - got = storyfonts.dialog_bindings(Ex(), ui, lang, 1, "Emoji") - assert set(got) == {"D"} and set(got["D"]) == {"D/Title", "D/Body"} - assert "spriteAsset" not in got["D"]["D/Title"] - assert (got["D"]["D/Body"]["spriteAsset"], got["D"]["D/Body"]["m_tintAllSprites"]) == ("Emoji", 0) - t = got["D"]["D/Title"] - assert (t["class"], t["fontAsset"], t["material"]) == ("TextMeshProUGUI", "J SDF", "J - Default") - assert t["m_fontSize"] == 30.0 - assert t["localized"]["fontAsset"] == "E SDF" and t["localized"]["material"] == "E - Default" - assert storyfonts.dialog_bindings(Ex(), {"nodes": []}, lang, 1) == {} - # the dialogs' text keys are shown characters - episode = {"text": {}, "title": {"english": ""}, "commands": []} - with pytest.raises(RuntimeError, match="master data"): - storyfonts.shown_characters(episode, {"nodes": [], **ui}, None, "english") - - -def test_text_components_of_every_talk_window(): - """The components of each text node of ui.json from the widget prefab or from the prefab of its talk window (the - TalkView children of ui.json, advui.window_nodes); a text node found in neither raises.""" - text = {"class": "TextMeshProUGUI", "m_Enabled": 1} - view = advui.WINDOW_PARENT - - class Cat: - def has(self, address): - return False - - class Ex: - cat = Cat() - - def prefab(self, key): - if key == advui.WIDGET_KEY: - return {"nodes": [{"path": "UIAdvWidget/FrontCanvas/Title", "components": [text, {"id": "title"}]}]} - name = key.rsplit("/", 1)[1] - return { - "nodes": [ - {"path": name, "components": []}, - {"path": f"{name}/TalkText", "components": [text, {"id": name}]}, - ] - } - - nodes = [ - {"path": "UIAdvWidget/FrontCanvas/Title", "name": "Title", "textStyle": {}}, - {"path": view, "name": "TalkView"}, - {"path": f"{view}/UIDefaultTalkWindow", "name": "UIDefaultTalkWindow"}, - {"path": f"{view}/UIDefaultTalkWindow/TalkText", "name": "TalkText", "textStyle": {}}, - {"path": f"{view}/UICenterTalkWindow", "name": "UICenterTalkWindow"}, - {"path": f"{view}/UICenterTalkWindow/TalkText", "name": "TalkText", "textStyle": {}}, - ] - got = storyfonts.text_components(Ex(), {"nodes": nodes}) - assert {p: c[1]["id"] for p, c in got.items()} == { - "UIAdvWidget/FrontCanvas/Title": "title", - f"{view}/UIDefaultTalkWindow/TalkText": "UIDefaultTalkWindow", - f"{view}/UICenterTalkWindow/TalkText": "UICenterTalkWindow", - } - lone = [n for n in nodes if "UICenterTalkWindow" not in n["path"]] + [ - {"path": f"{view}/UICenterTalkWindow/TalkText", "name": "TalkText", "textStyle": {}} - ] - with pytest.raises(RuntimeError, match="0 prefab nodes"): - storyfonts.text_components(Ex(), {"nodes": lone}) - - -def test_ui_text_targets_and_sprite_binding(): - """A text a UIText (or UIRubyText) drives, on its node or another, gets the emoji sprite asset; any other text - keeps its serialized sprite asset (only none is).""" - text = {"class": "TextMeshProUGUI", "m_tintAllSprites": 0, "m_spriteAsset": None} - nodes = [ - {"path": "W/A", "components": [text, {"class": "UIText", "_targetText": {"gameObject": "W/A"}}]}, - {"path": "W/B", "components": [text]}, - ] - assert storyfonts.ui_text_targets(nodes) == {"W/A"} - rerooted = [{"path": "P/W/A", "components": [text, {"class": "UIRubyText", "_targetText": {"gameObject": "W/A"}}]}] - assert storyfonts.ui_text_targets(rerooted, "P/") == {"P/W/A"} - other = [ - { - "path": "W", - "components": [ - {"class": "UIText", "_targetText": {"gameObject": "W/B"}}, - {"class": "UIText", "_targetText": None}, - ], - } - ] - assert storyfonts.ui_text_targets(other) == {"W/B"}, "the text of another node; none without a target" - assert storyfonts.sprite_binding("W/A", [text], True, "Emoji") == {"spriteAsset": "Emoji", "m_tintAllSprites": 0} - assert storyfonts.sprite_binding("W/B", [text], False, "Emoji") == {} - with pytest.raises(NotImplementedError, match="own sprite asset"): - storyfonts.sprite_binding("W/B", [{**text, "m_spriteAsset": {"name": "Own"}}], False, "Emoji") - - def sprite_record(): """A sprite asset record as advui.sprite_asset_record gives it: 32 x 32 glyphs in a 128 x 128 sheet.""" names = [ @@ -948,33 +429,6 @@ def __init__(self, name, have, fallbacks=(), pad=4, point=40): self.name, self.chars, self.fb, self.pad, self.point = name, set(have), list(fallbacks), pad, point -def chain_env(monkeypatch, fonts, lang): - """open_font_set's game side: the font settings of the ChainFont list `fonts` (their default materials and a - text material " - OutlineX" each) and the LocalizeManager table `lang`.""" - table = {"format": fontsettings.FORMAT, "fonts": {}, "materials": {}} - for f in fonts: - base = f.name[:-4] - default = json.loads(json.dumps(game(pad=f.pad)["material"])) - default["material"] = f"{base} SDF Material" - default["floats"].update({"_WeightNormal": 0.0, "_WeightBold": 0.5 + f.pad / 100, "_FaceDilate": 0.0}) - text = outline_material(pad=f.pad) - text["material"] = f"{base} - OutlineX" - text["floats"].update({"_WeightNormal": 0.0, "_WeightBold": 0.75, "_FaceDilate": 0.0}) - table["materials"].update({default["material"]: default, text["material"]: text}) - table["fonts"][f.name] = { - "faceInfo": {"m_PointSize": float(f.point)}, - "atlasPadding": f.pad, - "atlasRenderMode": 4134, - **game()["style"], - "material": default["material"], - "fallbacks": f.fb, - "coverage": fontsettings.encode_coverage(f.chars), - } - monkeypatch.setattr(tmpfont, "missing_glyph_character", lambda player: 0x25A1) - monkeypatch.setattr(storyfonts.textstyle, "language_fonts", lambda player: lang) - return fontsettings.FontSettings(table) - - # LocalizeManager fonts: ja and en J, zh Z, ko K CHAIN_LANG = { "languages": { @@ -1027,75 +481,3 @@ def test_fallback_material(): rng = f32(f32(f32(0.9) / f32(4)) * f32(4)) assert f["_ScaleRatioC"] == float(f32(f32(f32(4) - rng) / f32(5))) assert storyfonts.fallback_material({**m, "keywords": ["RATIOS_OFF"]}, t, "x")["floats"]["_ScaleRatioA"] == 1.0 - - -def test_open_font_set_mirrors_the_fallback_chain(tmp_path, monkeypatch): - """ko texts of K (fallback J, whose fallbacks are a game-only asset X and K): a character the ko font file - lacks goes to the asset of the ja font file standing for J; the missing glyph; the fallback materials; the - bindings.""" - ko = storyfonts.FontFile( - make_font(tmp_path / "Ko.ttf", family="Ko Sans", underscore=True, drop={0x53E3}, extra={0x25A1: "A"}) - ) - ja = storyfonts.FontFile(make_font(tmp_path / "Ja.ttf", family="Ja Sans", underscore=True)) - fonts = [ - ChainFont("K SDF", {0x41, 0x42, 0x20, 0x5F, 0x25A1, 0x78}, ["J SDF"], pad=15, point=100), - ChainFont("J SDF", {0x53E3, 0x48, 0x5F}, ["X SDF", "K SDF"]), - ChainFont("X SDF", {0x2606}), - ] - settings = chain_env(monkeypatch, fonts, CHAIN_LANG) - binding = {"localized": {"fontAsset": "K SDF", "material": "K - OutlineX", "lineSpacing": 0}} - asked = [] - - def font_of(code): - asked.append(code) - return {"ja": ja}[code] - - r = storyfonts.open_font_set( - None, "player", None, "ko", ko, [binding], ["AB \u53e3Hx\u2661_"], tmp_path, font_of, settings=settings - ) - prim, fb = "Ko Sans Regular SDF", "Ja Sans Regular SDF (J SDF)" - assert asked == ["ja"] and set(r["fonts"]) == {prim, fb} - p, f = r["fonts"][prim], r["fonts"][fb] - assert p["fallbacks"] == [fb] and f["fallbacks"] == [prim] - assert {ord(c) for c in "AB Hx_\u25a1"} <= {int(u) for u in p["characters"]} - assert str(0x53E3) not in p["characters"] and str(0x2661) not in p["characters"] - assert str(0x53E3) in f["characters"] and str(0x41) not in f["characters"] and str(0x5F) not in f["characters"] - assert p["missingGlyph"] == {"unicode": 0x25A1, "characters": [0x2661]} and "missingGlyph" not in f - assert r["missing"] == set() and r["substituted"] == {0x2661} - assert p["atlasPadding"] == 15 and f["atlasPadding"] == 4 and f["faceInfo"]["m_PointSize"] == 40 - text, fbm = "Ko Sans Regular - OutlineX", f"Ko Sans Regular - OutlineX + {fb}" - assert set(r["materials"]) == {f"{prim} Material", text, f"{fb} Material", fbm} - m, own = r["materials"][fbm], r["materials"][text] - assert m["floats"]["_GradientScale"] == 5.0 and own["floats"]["_GradientScale"] == 16.0 - assert m["textures"]["_MainTex"] == r["materials"][f"{fb} Material"]["textures"]["_MainTex"] - assert m["keywords"] == own["keywords"] and m["floats"]["_OutlineWidth"] == own["floats"]["_OutlineWidth"] - assert binding["localized"] == {"fontAsset": prim, "material": text, "lineSpacing": 0} - assert all((tmp_path / t["texture"]).is_file() for t in r["textures"].values()) - # nothing the ko font lacks: no fallback asset, no fallback material, no other font file asked for - asked.clear() - b2 = {"localized": {"fontAsset": "K SDF", "material": "K - OutlineX", "lineSpacing": 0}} - r2 = storyfonts.open_font_set( - None, "player", None, "ko", ko, [b2], ["ABx"], tmp_path / "b", font_of, settings=settings - ) - assert asked == [] and set(r2["fonts"]) == {prim} and r2["fonts"][prim]["fallbacks"] == [] - assert set(r2["materials"]) == {f"{prim} Material", text} and "missingGlyph" not in r2["fonts"][prim] - # a character only the ja file has, without it: a ConfigError naming the setting - b3 = {"localized": {"fontAsset": "K SDF", "material": "K - OutlineX", "lineSpacing": 0}} - with pytest.raises(storyfonts.ConfigError, match="needs the font file of ja"): - storyfonts.open_font_set(None, "player", None, "ko", ko, [b3], ["A\u53e3"], tmp_path / "c", settings=settings) - # a character no file of the chain maps stays with the primary as missing; a sprite-drawn one is left out - b4 = {"localized": {"fontAsset": "K SDF", "material": "K - OutlineX", "lineSpacing": 0}} - r4 = storyfonts.open_font_set( - None, - "player", - None, - "ko", - ko, - [b4], - ["A\u2606B"], - tmp_path / "d", - font_of, - sprite_drawn={"K SDF": {0x42}}, - settings=settings, - ) - assert r4["missing"] == {0x2606} and str(0x42) not in r4["fonts"][prim]["characters"] diff --git a/tests/content/test_story_host.py b/tests/content/test_story_host.py deleted file mode 100644 index 57e9462..0000000 --- a/tests/content/test_story_host.py +++ /dev/null @@ -1,798 +0,0 @@ -"""The host data of Overlay stories (storyhost.py) on synthetic prefab nodes, hierarchies and master rows: the host -of a story group, the slot capture camera and layout root records, SpotBackground.Prepare, the floor render queue -rule, the room node, material and texture order, the build's entry and return value. Nothing here comes from game -data.""" - -import json -from types import SimpleNamespace - -import numpy as np -import pytest - -from nnnotes.content import advui, hostspot, storyhost -from nnnotes.unity.reader import SceneGraph -from tests.content.test_hostspot import ROOM_KEY, File, RoomCatalog, load_room, two_file_room - -V0 = {"x": 0.0, "y": 0.0, "z": 0.0} -V1 = {"x": 1.0, "y": 1.0, "z": 1.0} -Q0 = {"x": 0.0, "y": 0.0, "z": 0.0, "w": 1.0} - - -# ---------------------------------------------------------------- synthetic hierarchies -def obj(kind, pid, tt): - return SimpleNamespace(type=SimpleNamespace(name=kind), path_id=pid, read_typetree=lambda: tt) - - -def graph(tree, inactive=()): - """A SceneGraph of `tree` {name: [children as (name, subtree) pairs]} given as nested lists [(name, [...])]: - GameObject ids 1000 + n, transform ids n, in depth-first order; names in `inactive` are saved inactive.""" - objs, ids = [], {} - - def walk(name, kids, father): - tf = len(ids) + 1 - ids[name] = tf - objs.append( - obj( - "GameObject", - 1000 + tf, - {"m_Name": name.split("#")[0], "m_IsActive": name not in inactive, "m_Layer": 0, "m_Component": []}, - ) - ) - entry = { - "m_GameObject": {"m_FileID": 0, "m_PathID": 1000 + tf}, - "m_Father": {"m_FileID": 0, "m_PathID": father}, - "m_Children": [], - "m_LocalPosition": V0, - "m_LocalRotation": Q0, - "m_LocalScale": V1, - } - objs.append(obj("Transform", tf, entry)) - for child, sub in kids: - entry["m_Children"].append({"m_FileID": 0, "m_PathID": walk(child, sub, tf)}) - return tf - - for name, kids in tree: - walk(name, kids, 0) - return SceneGraph(SimpleNamespace(objects=objs)), ids - - -BACKGROUND = [ - ( - "bg", - [ - ( - "light", - [ - ("table", []), - ("chair", []), - ("wall_floor_shadow", [("floor_mesh", [])]), - ("obj_floor", [("mesh_a", []), ("mesh_b", [])]), - ], - ) - ], - ), - ("model", [("table#m", [])]), -] - - -def test_prepare_matches_the_situation_without_clone_suffix(): - g, ids = graph(BACKGROUND, inactive=("chair",)) - entries = [("other", [1000 + ids["table"]]), ("sit_01", [1000 + ids["table"], None])] - got = storyhost.prepare(g, ids["bg"], "sit_01(Clone)", entries) - assert got["name"] == "sit_01" and got["matched"] - assert got["activate"] == [ids["table"], ids["chair"], ids["wall_floor_shadow"], ids["obj_floor"]] - assert got["hide"] == [1000 + ids["table"], None] - ov = got["overrides"] - assert ov[1000 + ids["chair"]] is True and ov[1000 + ids["table"]] is False # hidden after activation - assert not storyhost.active_in_hierarchy(g, ids["table"], ov) - assert storyhost.active_in_hierarchy(g, ids["chair"], ov) - assert not storyhost.active_in_hierarchy(g, ids["chair"], {}) # saved state - - -def test_prepare_without_a_matching_entry_keeps_the_prefab_state(): - g, ids = graph(BACKGROUND) - got = storyhost.prepare(g, ids["bg"], "sit_09(Clone)", [("sit_01", [1000 + ids["table"]])]) - assert got == {"name": "sit_09", "matched": False, "activate": [], "hide": [], "overrides": {}} - g2, ids2 = graph([("bg", [])]) - with pytest.raises(RuntimeError, match="GetChild"): - storyhost.prepare(g2, ids2["bg"], "x", []) - - -def test_floor_rule_skips_shadow_names_and_inactive_renderers(): - g, ids = graph(BACKGROUND, inactive=("mesh_a",)) - floor = storyhost.find_first_child_with_name(g, ids["bg"], "floor", "shadow") - assert floor == ids["floor_mesh"] # below the excluded "wall_floor_shadow", depth first - g2, ids2 = graph( - [("bg", [("a_shadow_floor", []), ("obj_floor", [("mesh_a", []), ("mesh_b", [])])])], inactive=("mesh_a",) - ) - floor = storyhost.find_first_child_with_name(g2, ids2["bg"], "floor", "shadow") - assert floor == ids2["obj_floor"] - renderers = {ids2["mesh_a"], ids2["mesh_b"]} - hit = storyhost.component_in_children( - g2, floor, renderers.__contains__, lambda t: storyhost.active_in_hierarchy(g2, t, {}) - ) - assert hit == ids2["mesh_b"] # GetComponentInChildren skips the inactive mesh_a - assert storyhost.component_in_children(g2, floor, set().__contains__, lambda t: True) is None - assert storyhost.find_first_child_with_name(g2, ids2["mesh_b"], "floor", "shadow") is None - - -def test_under_tells_the_placed_prefab_from_other_roots(): - g, ids = graph(BACKGROUND) - assert storyhost.under(g, ids["mesh_a"], ids["bg"]) and storyhost.under(g, ids["bg"], ids["bg"]) - assert not storyhost.under(g, ids["table#m"], ids["bg"]) - - -# ---------------------------------------------------------------- room order -class Mesh: - def __init__(self, name, tris): - self.m_Name, self.tris = name, [np.asarray(t, np.int64).reshape(-1, 3) for t in tris] - - -def pp(pid, file="CAB-a"): - """A material PPtr: its path id and the object it names in `file`.""" - return SimpleNamespace( - path_id=pid, deref=lambda: SimpleNamespace(assets_file=SimpleNamespace(name=file), path_id=pid) - ) - - -def arrays(mesh): - return None if mesh.m_Name == "empty" else (None, None, None, mesh.tris) - - -def test_glb_json_reads_the_json_chunk(tmp_path): - import struct - - js = json.dumps({"nodes": [{"name": "a"}]}).encode("utf-8") - js += b" " * (-len(js) % 4) - (tmp_path / "r.glb").write_bytes( - b"glTF" + struct.pack("読み"), - "TargetName": "a", - "AdvTextID": "t", - "TargetTextIDs": ["n"], - }, - {"cmd": "Subtitles", "lines": lines("字幕")}, - {"cmd": "Subtitles"}, - {"cmd": "Bgm", "lines": lines("no")}, - ], - "text": {"n": lines("名")}, - "title": lines("題"), - } - got = advui.shown_texts(episode, {"_unknownCharacterNameTextId": "u", "_splitCharacterNameTextId": "s"}, "japanese") - assert got == ["読み", "よみ", "字幕", "名", "題"] - - -def test_master_id_texts(tmp_path): - settings = {k: f"id_{i}" for i, k in enumerate(advui.MASTER_ID_TEXTS)} - settings.update( - { - "_skipVideoMessageTextId": "ui_skip_movie", - "_skipButtonTextId": "ui_skip", - "_cancelButtonTextId": "ui_cancel", - "_unknownCharacterNameTextId": "x", - } - ) - rows = [{"_id": i, "_japanese": f"ja {i}", "_english": f"en {i}"} for i in [*settings.values(), "y"]] - (tmp_path / "MasterText.json").write_text(json.dumps({"_allData": rows}), encoding="utf-8") - got = advui.master_id_texts(settings, tmp_path, "english") - assert set(got) == set(advui.MASTER_ID_TEXTS) and "_unknownCharacterNameTextId" not in got - assert got["_skipVideoMessageTextId"] == {"id": "ui_skip_movie", "text": "en ui_skip_movie"} - assert got["_interruptionButtonTextId"]["text"].startswith("en id_") - with pytest.raises(KeyError, match="not in MasterText"): - advui.master_id_texts({**settings, "_skipButtonTextId": "gone"}, tmp_path, "english") - - def ref(path, component="MonoBehaviour", cls="UIButton"): return {"component": component, "gameObject": path, "class": cls} -def test_menu_and_chat_view_records(): - menu = { - "type": "MonoBehaviour", - "class": "AdvMenuView", - "m_Enabled": 1, - "_fadeDuration": 0.2, - **{k: ref(f"M/{k}") for k in advui.MENU_VIEW_REFS}, - "_menuButtonsBackground": {"transform": "M/Back"}, - "_fastForwardNormalSprite": {"spriteRef": "cab:5", "name": "fast_off"}, - "_fastForwardOnePointFiveSprite": {"spriteRef": "cab:6", "name": "fast15"}, - "_fastForwardOnePointSevenSprite": None, - "_fastForwardDoubleSprite": {"spriteRef": "cab:8", "name": "fast2"}, - } - node = advui._layout_components({"components": [menu]}) - rec = node["menuView"] - assert rec["_skipButton"] == "M/_skipButton" and rec["_menuButtonsBackground"] == "M/Back" - assert (rec["_fastForwardNormalSprite"], rec["_fastForwardOnePointSevenSprite"]) == ("fast_off", None) - assert rec["_fadeDuration"] == 0.2 and node["_spriteRefs"] == ["cab:5", "cab:6", "cab:8"] - assert set(rec) == {"enabled", "_fadeDuration", *advui.MENU_VIEW_REFS, *advui.MENU_VIEW_SPRITES} - chat = { - "type": "MonoBehaviour", - "class": "AdvChatView", - "m_Enabled": 1, - **{k: 0.25 for k in advui.CHAT_VIEW_FIELDS}, - "_windowParentRect": {"transform": "C/Canvas/View/Target"}, - } - rec = advui._layout_components({"components": [chat]})["chatView"] - assert rec == { - "enabled": 1, - **{k: 0.25 for k in advui.CHAT_VIEW_FIELDS}, - "_windowParentRect": "C/Canvas/View/Target", - } - - -def test_widget_canvases(): - widget = [ - {"path": "W", "components": []}, - {"path": "W/Block", "components": [canvas(order=300)]}, - {"path": "W/Front", "components": [canvas(mode=1, order=304)]}, - {"path": "W/Letter", "components": [canvas(mode=0, order=0)]}, - ] - comp = { - "_canvasSortOrder": 99999, - "_canvases": [{"component": "Canvas", "gameObject": p} for p in ("W/Block", "W/Front", "W/Letter")], - } - assert advui.widget_canvases(widget, comp) == { - "canvasSortOrder": 99999, - "canvases": [ - {"path": "W/Block", "sortingOrder": 300}, - {"path": "W/Front", "sortingOrder": 304}, - {"path": "W/Letter", "sortingOrder": 0}, - ], - } - with pytest.raises(RuntimeError, match="0 Canvas"): - advui.widget_canvases(widget, {**comp, "_canvases": [{"component": "Canvas", "gameObject": "W"}]}) - - def test_emoji_sequence_keys(): key = advui.emoji_sequence_key assert key("1f636-200d-1f32b-fe0f", 0x1F636) == "\U0001f636\u200d\U0001f32b\ufe0f" @@ -387,65 +91,6 @@ def read_typetree(self): return self.tree -def test_sprite_asset_record(): - """The emoji sprite asset as the layout reads it: face info, the character table in order, the glyphs, the - sequence entries of the legacy sprite list, the sheet and the material.""" - metrics = {"m_Width": 32.0, "m_Height": 32.0} - tree = { - "m_Name": "Emoji", - "fallbackSpriteAssets": [], - "m_FaceInfo": {"m_PointSize": 0.0}, - "spriteSheet": "sheet", - "m_Material": "mat", - "spriteInfoList": [ - {"name": "1f600", "unicode": 0x1F600}, - {"name": "2764-fe0f", "unicode": 0x2764}, - {"name": "", "unicode": 0}, - ], - "m_SpriteCharacterTable": [ - {"m_Unicode": 0x1F600, "m_Name": "1f600", "m_GlyphIndex": 0, "m_Scale": 1.0}, - {"m_Unicode": 0x2764, "m_Name": "2764-fe0f", "m_GlyphIndex": 1, "m_Scale": 1.0}, - ], - "m_GlyphTable": [ - {"m_Index": i, "m_Metrics": metrics, "m_GlyphRect": {"m_X": 32 * i}, "m_Scale": 1.0, "m_AtlasIndex": 0} - for i in (0, 1) - ], - } - sheet = { - "m_Name": "Sheet", - "m_Width": 64, - "m_Height": 32, - "m_MipCount": 1, - "m_TextureSettings": {"m_FilterMode": 1}, - } - - class Ex: - def key_object(self, key): - assert key == advui.EMOJI_SPRITE_ASSET_KEY - return FakeObj(tree) - - def deref(self, o, ref): - return FakeObj(sheet) if ref == "sheet" else ref - - referent = deref - - def material(self, o): - return {"material": f"{o} record"} - - rec = advui.sprite_asset_record(Ex()) - assert rec["characters"] == [ - {"index": 0, "unicode": 0x1F600, "name": "1f600", "glyph": 0, "scale": 1.0}, - {"index": 1, "unicode": 0x2764, "name": "2764-fe0f", "glyph": 1, "scale": 1.0}, - ] - assert rec["glyphs"]["1"] == {"metrics": metrics, "rect": {"m_X": 32}, "scale": 1.0, "atlasIndex": 0} - assert rec["sequences"] == [{"name": "2764-fe0f", "unicode": 0x2764}] - assert rec["sheet"] == {"name": "Sheet", "width": 64, "height": 32, "mipCount": 1, "settings": {"m_FilterMode": 1}} - assert rec["material"] == {"material": "mat record"} and rec["faceInfo"] == {"m_PointSize": 0.0} - tree["fallbackSpriteAssets"] = [{"m_PathID": 1}] - with pytest.raises(NotImplementedError, match="fallback sprite assets"): - advui.sprite_asset_record(Ex()) - - def test_emoji_sequence_parse(): """TMP_EmojiSearchEngine: the table (the first entry of a key wins) and the names a parse writes: the longest run of UTF-16 units whose prefixes are all in the fast set, no backtracking, the scan going on after a failed run.""" @@ -483,26 +128,6 @@ def test_tmp_hash(): } -def test_chat_localize(): - """TryGetFontIndex over the Japanese font names and additional fonts, the font of that index among the - language's loaded fonts (index out of range: font 0), the material of the type.""" - assert advui.chat_localize("CHAT SDF", "CHAT - Default", LANG, 1) == { - "fontAsset": "EN-CHAT SDF", - "material": "EN-CHAT - Default", - "lineSpacing": -100.0, - } - assert advui.chat_localize("CHAT SDF", "CHAT - Outline", LANG, 4)["fontAsset"] == "KO-R SDF" - assert advui.chat_localize("JA-N SDF", "JA-N - Outline", LANG, 0) == { - "fontAsset": "JA-N SDF", - "material": "JA-N - Outline", - "lineSpacing": 0.0, - } - with pytest.raises(NotImplementedError, match="lookup table"): - advui.chat_localize("Other SDF", "Other - Default", LANG, 1) - with pytest.raises(NotImplementedError, match="material type"): - advui.chat_localize("CHAT SDF", "CHAT - Glow", LANG, 1) - - def lines(s): return {"japanese": s, "english": s} @@ -525,24 +150,6 @@ def chat_episode(): } -def test_chat_runtime_texts_and_shown_chat_rows(): - ep = chat_episode() - assert advui.chat_runtime_texts(ep) == ["...", "80%", "(3)", " 2", " 1"] - assert advui.chat_runtime_texts({"commands": [{"cmd": "Talk"}]}) == [] - got = advui.shown_texts( - ep, - { - "_unknownCharacterNameTextId": "u", - "_splitCharacterNameTextId": "p", - "_chatReadTextId": "60001", - "_chatStampLogTextId": "60004", - }, - "japanese", - ) - assert got == ["やあ", *sorted(["窓", "送", "スタンプ", "既読"]), "題", "...", "80%", "(3)", " 2", " 1"] - assert advui.chat_windows(ep) == {"Line": "Adv/Chat/Prefabs/Line"} - - def tmp_text(font, material, text=""): return { "type": "MonoBehaviour", @@ -588,324 +195,3 @@ def localize(enabled=1, key="0"): "_localizeEnabled": enabled, "_masterTextID": key, } - - -def test_chat_texts(monkeypatch): - window = [ - { - "path": "Line", - "components": [ - {"class": "AdvChatWindow", "_incomingCallStatusTextKey": "call", "_lockScreenStatusTextKey": ""} - ], - }, - {"path": "Line/Top/Name", "components": [tmp_text("CHAT SDF", "CHAT - Default"), localize()]}, - {"path": "Line/Top/Percent", "components": [tmp_text("JA-N SDF", "JA-N - Outline", "100%")]}, - ] - - class Ex: - def prefab(self, address): - assert address == "Adv/Chat/Prefabs/Line" - return {"nodes": window} - - face = {"m_PointSize": 40.0, "m_Scale": 1.0} - fonts = fontsettings.FontSettings( - { - "format": fontsettings.FORMAT, - "fonts": {"EN-CHAT SDF": {"faceInfo": face}, "JA-N SDF": {"faceInfo": face}}, - "materials": { - "EN-CHAT - Default": {"floats": {}, "colors": {}, "keywords": []}, - "JA-N - Outline": {"floats": {}, "colors": {}, "keywords": []}, - }, - } - ) - monkeypatch.setattr(advui.textstyle, "material_style", lambda m, point, scale: {"face": {"point": point}}) - got, status = advui.chat_texts(Ex(), fonts, LANG, 1, chat_episode()) - assert status == ["call"] - name, pct = got["Line"]["Line/Top/Name"], got["Line"]["Line/Top/Percent"] - assert name["localizeText"] == {"enabled": 1, "localizeEnabled": 1} - assert name["textStyle"]["fontRole"] == "font2" and name["textStyle"]["localized"] is True - assert name["textStyle"]["lineSpacing"]["applied"] == -100.0 and name["textStyle"]["face"] == {"point": 40.0} - assert pct["localizeText"] is None and pct["textStyle"]["fontRole"] == "number" - assert pct["textStyle"]["lineSpacing"]["applied"] == 3.0 - # a font's folder: the embedded copy when the catalog has it, else LocalizeManager's Font/ - no_emb = type("C", (), {"has": lambda self, k: False})() - emb = type("C", (), {"has": lambda self, k: True})() - assert fontsettings.font_key(no_emb, "EN-CHAT SDF", "EN-CHAT - Default") == "Font/EN-CHAT/EN-CHAT - Default" - assert fontsettings.font_key(emb, "A SDF", "A - Default") == "EmbFont/A/A - Default" - - -def test_behaviours_record_every_serialized_field(): - """BEHAVIOURS components: every field but the object header, references as node paths, sprites as names (the - references collected for the UI textures), assets and materials as names; several components of a class in - order; a node with several ButtonImageState components has them there instead of `buttonImageState`.""" - sprite = {"spriteRef": "cab:9", "name": "sp_voice"} - comps = [ - { - "type": "MonoBehaviour", - "class": "UIButton", - "m_Enabled": 1, - "m_GameObject": {"m_PathID": 1}, - "m_Script": {"m_PathID": 2}, - "m_Name": "", - "_initialButtonState": 0, - "_buttonStates": [ref("B/State"), ref("B/State2", cls="ButtonImageState")], - "_buttonAnimation": ref("B", cls="ButtonAnimationScale"), - "m_OnClick": { - "m_PersistentCalls": {"m_Calls": [{"m_Target": ref("B/T", cls="Image"), "m_MethodName": "x"}]} - }, - }, - { - "type": "MonoBehaviour", - "class": "ButtonActiveState", - "m_Enabled": 1, - "_target": {"gameObject": "B/On"}, - "_selectedActive": 1, - }, - {"type": "MonoBehaviour", "class": "ButtonActiveState", "m_Enabled": 0, "_target": None, "_selectedActive": 0}, - { - "type": "MonoBehaviour", - "class": "ButtonImageState", - "m_Enabled": 1, - "_target": ref("B/Icon", cls="Image"), - "_normaledSprite": sprite, - "_selectedSprite": None, - }, - { - "type": "MonoBehaviour", - "class": "ButtonImageState", - "m_Enabled": 1, - "_target": ref("B/Frame", cls="Image"), - "_normaledSprite": {"spriteRef": "cab:10", "name": "frame"}, - "_selectedSprite": sprite, - }, - { - "type": "MonoBehaviour", - "class": "SimpleRaycastTarget", - "m_Enabled": 1, - "m_Material": None, - "m_RaycastPadding": {"x": 0.0, "y": 0.0, "z": 0.0, "w": 0.0}, - }, - { - "type": "MonoBehaviour", - "class": "LocalizeSpriteEvent", - "m_Enabled": 1, - "m_LocalizedAssetReference": { - "m_TableReference": {"m_TableCollectionName": "GUID:abc"}, - "m_TableEntryReference": {"m_KeyId": 7, "m_Key": ""}, - }, - }, - {"type": "MonoBehaviour", "class": "GraphicRaycaster", "m_Enabled": 1}, - ] - rec = advui._layout_components({"components": comps}) - b = rec["behaviours"] - assert set(b) == {"UIButton", "ButtonActiveState", "ButtonImageState", "SimpleRaycastTarget", "LocalizeSpriteEvent"} - assert b["UIButton"] == [ - { - "m_Enabled": 1, - "_initialButtonState": 0, - "_buttonStates": ["B/State", "B/State2"], - "_buttonAnimation": "B", - "m_OnClick": {"m_PersistentCalls": {"m_Calls": [{"m_Target": "B/T", "m_MethodName": "x"}]}}, - } - ] - assert [x["_target"] for x in b["ButtonActiveState"]] == ["B/On", None] - assert [x["_normaledSprite"] for x in b["ButtonImageState"]] == ["sp_voice", "frame"] - assert "buttonImageState" not in rec and rec["_spriteRefs"] == ["cab:9", "cab:10", "cab:9"] - assert b["LocalizeSpriteEvent"][0]["m_LocalizedAssetReference"]["m_TableEntryReference"]["m_KeyId"] == 7 - one = advui._layout_components({"components": comps[3:4]}) - assert one["buttonImageState"] == {"normal": "sp_voice"} and "behaviours" not in one - assert ( - advui._plain({"asset": "UIPictogramCatalog", "name": "UIPictogramCatalog", "_list": []}, []) - == "UIPictogramCatalog" - ) - assert advui._plain({"material": "M", "shader": {"shader": "UI/Default"}}, []) == "M" - - -def test_full_views_and_pictograms(): - view = { - "type": "MonoBehaviour", - "class": "AdvChoiceView", - "m_Enabled": 1, - "m_Script": {}, - "_choiceItems": [ref("C/Item", cls="AdvChoiceItem"), ref("C/Item (1)", cls="AdvChoiceItem")], - "_choiceItemParent": {"transform": "C/Choices"}, - "_animationIntervalIn": 0.1, - } - rec = advui._layout_components({"components": [view]}) - assert rec["choiceView"] == { - "enabled": 1, - "m_Enabled": 1, - "_choiceItems": ["C/Item", "C/Item (1)"], - "_choiceItemParent": "C/Choices", - "_animationIntervalIn": 0.1, - } - catalog = { - "asset": "Cat", - "name": "Cat", - "m_Enabled": 1, - "_spritesByKey": {"_list": [{"Key": 98, "Value": {"spriteRef": "cab:98", "name": "IconClose"}}]}, - "_defaultSprite": {"spriteRef": "cab:1", "name": "IconSetting"}, - } - picto = { - "type": "MonoBehaviour", - "class": "UIPictogram", - "m_Enabled": 1, - "_key": 98, - "_spriteCatalog": catalog, - "_image": ref("C/Close", cls="Image"), - "_setNativeSize": 0, - } - nodes = [ - {"path": "C/Close", "components": [picto]}, - {"path": "C/Other", "components": [{**picto, "_key": 5, "_spriteCatalog": {"asset": "Cat", "name": "Cat"}}]}, - ] - cats = advui.pictogram_catalogs(nodes) - assert set(cats) == {"Cat"} - recs = [] - for n in nodes: - r = {"path": n["path"], **advui._layout_components(n)} - advui.resolve_pictograms(r, cats) - recs.append(r["behaviours"]["UIPictogram"][0]) - assert recs[0] == { - "m_Enabled": 1, - "_key": 98, - "_image": "C/Close", - "_setNativeSize": 0, - "catalog": "Cat", - "sprite": "IconClose", - "defaultSprite": "IconSetting", - "_spriteRef": "cab:98", - "_defaultRef": None, - } - assert (recs[1]["sprite"], recs[1]["_defaultRef"]) == (None, "cab:1") - with pytest.raises(RuntimeError, match="not exported"): - advui.resolve_pictograms({"path": "x", **advui._layout_components(nodes[1])}, {}) - - -def test_talk_windows_and_their_nodes(): - """The default talk window first, then the episode's other talk windows by name; each window's prefab nodes under - TalkView, the default from WINDOW_KEY and the others from the TalkWindow address as adv.closure resolves it.""" - res = [ - {"kind": "talkwindow", "address": "EmbUI/Prefab/Parts/Adv/Talk/UIDefaultTalkWindow"}, - {"kind": "talkwindow", "address": "EmbUI/Prefab/Parts/Adv/Talk/UICenterTalkWindow"}, - {"kind": "chatwindow", "address": "Adv/Chat/Prefabs/Zed"}, - {"kind": "talkwindow", "address": "UI/Prefab/Parts/Adv/Talk/UIBTalkWindow"}, - ] - assert advui.talk_windows({"resources": res}) == ["UIDefaultTalkWindow", "UIBTalkWindow", "UICenterTalkWindow"] - assert advui.talk_windows({"resources": []}) == ["UIDefaultTalkWindow"] - - class Cat: - def has(self, address): - return address == "UI/Prefab/Parts/Adv/Talk/UIBTalkWindow" - - class Ex: - cat = Cat() - - def __init__(self): - self.keys = [] - - def prefab(self, key): - self.keys.append(key) - name = key.rsplit("/", 1)[1] - return {"nodes": [{"path": name, "components": []}, {"path": f"{name}/TalkArea", "components": []}]} - - assert advui.window_key(Cat(), "UIDefaultTalkWindow") == advui.WINDOW_KEY - assert advui.window_key(Cat(), "UIBTalkWindow") == "UI/Prefab/Parts/Adv/Talk/UIBTalkWindow" - assert advui.window_key(Cat(), "UICenterTalkWindow") == "EmbUI/Prefab/Parts/Adv/Talk/UICenterTalkWindow" - ex = Ex() - got = advui.window_nodes(ex, ["UIDefaultTalkWindow", "UICenterTalkWindow"]) - assert [n["path"] for n in got] == [ - f"{advui.WINDOW_PARENT}/{p}" - for p in ( - "UIDefaultTalkWindow", - "UIDefaultTalkWindow/TalkArea", - "UICenterTalkWindow", - "UICenterTalkWindow/TalkArea", - ) - ] - assert ex.keys == [advui.WINDOW_KEY, "EmbUI/Prefab/Parts/Adv/Talk/UICenterTalkWindow"] - - class Wrong(Ex): - def prefab(self, key): - return {"nodes": [{"path": "Other", "components": []}]} - - with pytest.raises(RuntimeError, match="prefab root"): - advui.window_nodes(Wrong(), ["UICenterTalkWindow"]) - assert ( - advui.about("en", ["UIDefaultTalkWindow", "UICenterTalkWindow"]) - == "ADV front canvas UI data (UIAdvWidget + UIDefaultTalkWindow, UICenterTalkWindow), en" - ) - - -def test_backdrop_filter_and_the_backdrop_node(): - """The centre-talk backdrop is kept when a talk window uses the backdrop filter: the node UIAdvWidget's backdrop - CanvasGroup and Image references name (one node with both components).""" - - def win(on): - return [ - {"path": "W", "components": [{"class": "UIAdvTalkWindow", "_useBackdropFilter": on}]}, - {"path": "W/TalkArea", "components": [image()]}, - ] - - assert advui.uses_backdrop_filter(win(0) + win(1)) and not advui.uses_backdrop_filter(win(0)) - path = "UIAdvWidget/FrontCanvas/CenterTalkBackdrop" - widget = [ - {"path": "UIAdvWidget", "components": []}, - {"path": path, "components": [{"type": "CanvasGroup", "m_Alpha": 0.0}, image()]}, - {"path": "UIAdvWidget/FrontCanvas/Plain", "components": [image()]}, - ] - comp = {"_centerTalkBackdropCanvasGroup": ref(path, "CanvasGroup", None), "_centerTalkBackdropImage": ref(path)} - assert advui.backdrop_path(widget, comp) == path - with pytest.raises(RuntimeError, match="BACKDROP|_centerTalkBackdrop"): - advui.backdrop_path(widget, {**comp, "_centerTalkBackdropImage": ref("UIAdvWidget/FrontCanvas/Plain")}) - with pytest.raises(RuntimeError, match="no CanvasGroup"): - advui.backdrop_path(widget, {k: ref("UIAdvWidget/FrontCanvas/Plain") for k in comp}) - - -def test_blur_settings(): - """The blur of a backdrop-filter talk window: renderer 0's UIRendererFeature record and its shader object.""" - feature = { - "class": "Fwk.UI.Rendering.UIRendererFeature", - "m_Active": 1, - "_blurIterations": 3, - "_blurOffset": 1.0, - "_blurDownsample": 1, - "_blurBlendRateMax": 0.3, - "_dualKawaseBlurShader": "Hidden/UI/DualKawaseBlur", - } - - class Shader: - def __init__(self, name): - self.m_ParsedForm = type("P", (), {"m_Name": name}) - - def read(self): - return self - - class Player: - def __init__(self, active=1): - self.active = active - - def graphics(self): - return { - "defaultPipeline": "P", - "pipelines": {"P": {"m_RendererDataList": ["R0"], "m_DefaultRendererIndex": 0}}, - "renderers": {"R0": {"m_RendererFeatures": [{**feature, "m_Active": self.active}]}}, - } - - def renderer_shaders(self, name): - assert name == "R0" - return [Shader("Hidden/Other"), Shader("Hidden/UI/DualKawaseBlur")] - - blur, shader = advui.blur_settings(Player()) - assert blur == { - "renderer": "R0", - "active": True, - "iterations": 3, - "offset": 1.0, - "downsample": 1, - "blendRateMax": 0.3, - "shader": "Hidden/UI/DualKawaseBlur", - } - assert shader.m_ParsedForm.m_Name == "Hidden/UI/DualKawaseBlur" - with pytest.raises(NotImplementedError, match="not active"): - advui.blur_settings(Player(0)) diff --git a/tests/content/test_tmpassets.py b/tests/content/test_tmpassets.py index 4886590..1136118 100644 --- a/tests/content/test_tmpassets.py +++ b/tests/content/test_tmpassets.py @@ -11,7 +11,7 @@ from nnnotes.content import tmpassets from nnnotes.unity import fontsettings -from nnnotes.unity.export import MemoryPacker, packed_png +from nnnotes.unity.texels import MemoryPacker, packed_png from tests.helpers.fonts import make_font FILE = "CAB-tmpassets" diff --git a/tests/formats/test_crilips.py b/tests/formats/test_crilips.py index e799fc9..a169d06 100644 --- a/tests/formats/test_crilips.py +++ b/tests/formats/test_crilips.py @@ -184,31 +184,3 @@ def test_supported_versions_carry_the_player_configuration(): def _talk(names, voices, **kw): return {"cmd": "Talk", "TargetName": names, "VoiceIDs": voices, **kw} - - -def test_needs_cri_lips(): - from nnnotes.content import story - - def ep(*rows): - return {"commands": list(rows)} - - one = _talk("a", [1]) - # a voiced row reaches the analysis on a model without MotionSync - assert story.needs_cri_lips(ep(one), "・", [True, False]) - assert not story.needs_cri_lips(ep(one), "・", [True, True]) - # rows without lip sync never do - for row in ( - _talk("a", [1], IgnoreLipSync=1), - _talk("a", [1], IgnoreData=1), - _talk("", [1]), - _talk("a", []), - {"cmd": "Bgm", "VoiceIDs": [1], "TargetName": "a"}, - ): - assert not story.needs_cri_lips(ep(row), "・", [False]) - # a Voice row on a model without MotionSync - assert story.needs_cri_lips(ep({"cmd": "Voice", "TargetName": "a", "VoiceIDs": [1]}), "・", [False]) - # a voice shared by more speakers, or every showing character, uses the analysis on any model - assert story.needs_cri_lips(ep(_talk("a・b", [1])), "・", [True]) - assert not story.needs_cri_lips(ep(_talk("a・b", [1, 2])), "・", [True]) - assert story.needs_cri_lips(ep(_talk("a", [1], Parameter1=" EveryoneLipSync ")), "・", [True]) - assert not story.needs_cri_lips(ep(_talk("a・b", [1], Parameter1="AirLipSync")), "・", [True]) diff --git a/tests/formats/test_usm.py b/tests/formats/test_usm.py index fb9a22c..2396773 100644 --- a/tests/formats/test_usm.py +++ b/tests/formats/test_usm.py @@ -1,7 +1,6 @@ """usm: USM masks, demux of every stream kind, subtitles, the movie files' ffmpeg arguments; advvideo: the video index (synthetic data, made-up key).""" -import json import random import shutil import struct @@ -10,7 +9,6 @@ import pytest -from nnnotes.content import advvideo from nnnotes.formats import usm from nnnotes.formats.usm import Stream @@ -670,140 +668,6 @@ def test_webm_arguments(): ] -def test_story_export_keeps_its_command(tmp_path, monkeypatch): - """The story's WebM of a VP9 + ADX movie: the command it always ran (plus -hide_banner), the same record.""" - info = { - "movieInfo": { - "codecType": 9, - "width": 320, - "height": 180, - "dispWidth": 320, - "dispHeight": 180, - "framerateN": 30, - "framerateD": 1, - "totalFrames": 3, - "numAlphaStreams": 0, - "numSubtitleChannels": 0, - "numAudioStreams": 1, - }, - "assetInfo": {"loop": 0}, - } - movie = tmp_path / "m.usm" - movie.write_bytes( - build([(b"@SFV", 0, None, VP9, "video"), (b"@SFA", 0, None, [adx(samples=100) + rand(4 * 36, 1)], "audio")]) - ) - monkeypatch.setattr(advvideo, "usm_asset", lambda cat, key: (info, movie)) - monkeypatch.setattr("nnnotes.source.config.tool", lambda name, exe: name) - calls = [] - - def fake_run(args, **kw): - calls.append(args) - Path(args[-1]).write_bytes(b"webm") - return subprocess.CompletedProcess(args, 0, "", "") - - monkeypatch.setattr(usm.subprocess, "run", fake_run) - work = tmp_path / "videos" / "_work" - rec = advvideo.export(None, "Cri/Video/adv/m/m", tmp_path / "videos" / "m.webm", KEY, work) - assert calls == [ - [ - "ffmpeg", - "-hide_banner", - "-y", - "-loglevel", - "error", - "-f", - "ivf", - "-i", - str(work / "video.ivf"), - "-f", - "adx", - "-i", - str(work / "audio.adx"), - "-map", - "0:v:0", - "-map", - "1:a:0", - "-c:a", - "libopus", - "-b:a", - "192k", - "-flags:a", - "+bitexact", - "-c:v", - "copy", - "-map_metadata", - "-1", - "-fflags", - "+bitexact", - "-f", - "webm", - str(tmp_path / "videos" / "m.webm"), - ] - ] - assert list(rec) == [ - "key", - "codec", - "width", - "height", - "displayWidth", - "displayHeight", - "frameRate", - "frames", - "assetInfo", - "audio", - ] - assert rec["audio"] == { - "codec": "opus", - "bitrate": "192k", - "source": {"codec": "adx", "channels": 2, "sampleRate": 48000, "samples": 100}, - } - assert not work.exists() - movie.write_bytes( - build( - [(b"@SFV", 0, video_header(9), VP9, "video"), (b"@ALP", 0, video_header(1, alpha_type=1), MPEG1, "video")] - ) - ) - info["movieInfo"]["numAudioStreams"] = 0 - rec = advvideo.export(None, "Cri/Video/x", tmp_path / "videos" / "x.webm", KEY, work) - assert rec["encode"] == {"codec": "vp9", "crf": 20, "alpha": True} and "audio" not in rec - assert "alphamerge" in calls[-1][calls[-1].index("-filter_complex") + 1] - - -def test_video_index(tmp_path, monkeypatch): - def fake_export(cat, key, dst, cri_key, work): - assert cri_key == KEY - dst.parent.mkdir(parents=True, exist_ok=True) - dst.write_bytes(b"webm") - return {"key": key, "codec": "vp9", "frames": 10} - - monkeypatch.setattr(advvideo, "export", fake_export) - episode = { - "resources": [ - {"kind": "video", "address": "Cri/Video/adv/m_b/m_b", "present": True}, - {"kind": "video", "address": "Cri/Video/adv/m_a/m_a", "present": True}, - {"kind": "stage", "address": "Adv/Stage/s/s", "present": True}, - ], - "commands": [ - {"cmd": "Movie", "VideoID": 2}, - {"cmd": "Clip", "VideoID": 1}, - {"cmd": "Clip"}, - {"cmd": "Clip", "VideoID": 2}, - {"cmd": "Movie", "VideoID": 9}, - {"cmd": "Stage", "VideoID": 1}, - ], - "videos": { - 1: {"_id": 1, "_assetName": "adv/m_a/m_a", "_hasAudio": True}, - 2: {"_id": 2, "_assetName": "adv/m_b/m_b", "_hasAudio": False}, - }, - } - index = advvideo.extract(None, episode, tmp_path, cri_key=KEY) - assert list(index["videos"]) == ["2", "1"] - assert index["videos"]["1"]["file"] == "videos/m_a.webm" and index["videos"]["2"]["master"]["_hasAudio"] is False - assert (tmp_path / "videos" / "m_a.webm").read_bytes() == b"webm" - assert json.loads((tmp_path / "videos" / "videos.json").read_text(encoding="utf-8")) == index - assert advvideo.extract(None, {"resources": [], "commands": [], "videos": {}}, tmp_path / "none") is None - - # ---------------------------------------------------------------- with ffmpeg (skipped without it) def _encoders(ffmpeg): r = subprocess.run([ffmpeg, "-hide_banner", "-encoders"], capture_output=True, text=True) diff --git a/tests/pipeline/test_cristages.py b/tests/pipeline/test_cristages.py index fee08ce..0698408 100644 --- a/tests/pipeline/test_cristages.py +++ b/tests/pipeline/test_cristages.py @@ -603,7 +603,7 @@ def test_movie_parameters(): def test_stage_modules_import_nothing_heavy(): code = ( "import sys; import nnnotes.pipeline.cristages as c; c.AudioStage().normalize(None); " - "heavy = ('UnityPy', 'numpy', 'PIL', 'nnnotes.formats.cri', 'nnnotes.formats.usm', 'nnnotes.unity.export'); " + "heavy = ('UnityPy', 'numpy', 'PIL', 'nnnotes.formats.cri', 'nnnotes.formats.usm', 'nnnotes.unity.texels'); " "print(sorted(m for m in heavy if m in sys.modules))" ) out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, cwd=ROOT, check=True) diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index 1a1cb2b..6316257 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -815,10 +815,9 @@ def test_textures(exported): assert img.size == (W, H) and img.mode == "RGBA" expected = Image.frombytes("RGBA", (W, H), TEX_PIXELS).transpose(Image.Transpose.FLIP_TOP_BOTTOM) assert img.tobytes() == expected.tobytes() - from nnnotes.unity.export import texture_png - - tex = bundle_all().assets[0].objects[10].obj - assert png == texture_png(tex) # the other commands' bytes + encoded = __import__("io").BytesIO() + expected.save(encoded, format="PNG") + assert png == encoded.getvalue() facts = art(doc, f"{oid(10)}#image")["semantics"]["facts"] assert facts == { "width": W, @@ -1596,7 +1595,7 @@ def test_sprite_crop_waits_for_exports(tmp_path): def test_modules_import_without_unitypy(): code = ( "import sys; import nnnotes.pipeline.objexport as o; o.ExportStage().ATOMS; o.SpriteCropStage().ATOMS; " - "print(sorted(m for m in ('UnityPy', 'PIL', 'nnnotes.unity.export') if m in sys.modules))" + "print(sorted(m for m in ('UnityPy', 'PIL', 'nnnotes.unity.texels') if m in sys.modules))" ) out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, cwd=ROOT, check=True) assert out.stdout.strip() == "[]" diff --git a/tests/pipeline/test_views.py b/tests/pipeline/test_views.py index 78a02dc..9485fde 100644 --- a/tests/pipeline/test_views.py +++ b/tests/pipeline/test_views.py @@ -1000,7 +1000,7 @@ def test_views_plan_and_run_after_the_address_table(tmp_path): def test_views_import_neither_unitypy_nor_numpy(): code = ( "import sys, nnnotes.pipeline.views; " - "print(sorted(m for m in ('UnityPy', 'numpy', 'PIL', 'nnnotes.unity.export') if m in sys.modules))" + "print(sorted(m for m in ('UnityPy', 'numpy', 'PIL', 'nnnotes.unity.texels') if m in sys.modules))" ) out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, cwd=ROOT, check=True).stdout assert out.strip() == "[]" diff --git a/tests/site/test_regions.py b/tests/site/test_regions.py index d604ccd..c4ab20e 100644 --- a/tests/site/test_regions.py +++ b/tests/site/test_regions.py @@ -89,7 +89,12 @@ def test_music_rows_have_every_language(tmp_path): @pytest.mark.parametrize("code", LANGS) def test_start_canvas_strings_follow_the_language(tmp_path, code): - strings, sources = liveui.resolve_strings(write_master(tmp_path / "m"), 1, "expert", languages.column(code)) + master = write_master(tmp_path / "m") + tables = { + name: score.master_table(master, name) + for name in ("MasterLiveMusic", "MasterText", "MasterBand", "MasterLiveMusicScore") + } + strings, sources = liveui.record_strings(tables, 1, "expert", languages.column(code)) assert strings["musicTitle"] == "Song-ja" # the game shows the Japanese title by default assert strings["singerName"] == f"Band-{code}" assert strings["lyricsWriter"] == f"L:Lyricist-{code}" diff --git a/tests/unity/test_atoms_texture.py b/tests/unity/test_atoms_texture.py index a87d90b..28b9f79 100644 --- a/tests/unity/test_atoms_texture.py +++ b/tests/unity/test_atoms_texture.py @@ -1,6 +1,4 @@ -"""texture.decode, png.encode and astc.container on synthetic textures (decode = UnityPy's parse_image_data; PNG bytes -= export.texture_png's), the reader's objects, containers and texture inputs through a fake environment, and the -atom registry.""" +"""Texture decoding, lossless PNG output and container inputs over synthetic textures.""" import io import re @@ -13,7 +11,6 @@ from PIL import Image from UnityPy.export.Texture2DConverter import parse_image_data -from nnnotes.unity import export from nnnotes.unity.atoms import ATOMS, Impl, Unsupported, astc, env_name, ids, impl_id, png, resolve from nnnotes.unity.atoms.reader import ObjectInfo, UnityBundle from nnnotes.unity.atoms.texture import ASTC_BLOCKS, TextureInput, decode, is_hdr_astc @@ -69,9 +66,12 @@ def test_decode_is_parse_image_data(fmt): @pytest.mark.parametrize("fmt", FORMATS) -def test_png_bytes_equal_export_texture_png(fmt): +def test_png_bytes_match_the_decoded_image(fmt): tex = fake(fmt, seed=fmt) - assert png.encode(decode(inp(tex))) == export.texture_png(tex) + image = parse_image_data(tex.get_image_data(), tex.m_Width, tex.m_Height, fmt, VERSION, ANDROID, [], True) + expected = io.BytesIO() + image.save(expected, format="PNG") + assert png.encode(decode(inp(tex))) == expected.getvalue() @pytest.mark.parametrize("fmt", [RGBA32, ASTC_6x6]) @@ -81,11 +81,6 @@ def test_further_mip_levels_do_not_change_the_image(fmt): assert decode(inp(mips)).tobytes() == decode(inp(base)).tobytes() -def test_decode_input_key_is_export_decode_inputs(): - tex = fake(RGBA32) - assert inp(tex).key() == export._decode_inputs(tex) - - @pytest.mark.parametrize("fmt", [f for f in range(66, 72)]) def test_hdr_astc_is_refused(fmt): assert is_hdr_astc(fmt) @@ -204,7 +199,7 @@ def test_reader_texture_inputs(): env, view, inline, streamed_data = bundle() objs = {o.path_id: o for o in view.objects()} ti = view.texture_input(objs[-5]) - assert ti.key() == export._decode_inputs(inline) + assert ti.key() == inp(inline).key() assert view.texture_input(objs[7]).data == streamed_data and env.lookups == ["CAB-aa.resS"] env.lookups.clear() empty = view.texture_input(objs[9]) diff --git a/tests/unity/test_container.py b/tests/unity/test_container.py new file mode 100644 index 0000000..d6f9b67 --- /dev/null +++ b/tests/unity/test_container.py @@ -0,0 +1,66 @@ +"""Container-path lookup and jacket decoding over synthetic bundle objects.""" + +from types import SimpleNamespace + +import pytest +from PIL import Image + +from nnnotes.musicdata import MusicDataError, catalog_jacket +from nnnotes.unity import reader + + +def environment(monkeypatch, asset): + pointer = {"m_FileID": 1, "m_PathID": 7} + owner = SimpleNamespace( + type=SimpleNamespace(name="AssetBundle"), + read_typetree=lambda: {"m_Container": [("Assets/Jacket.PNG", {"asset": pointer})]}, + ) + + def dereference(source, ref): + assert source is owner and ref == pointer + return asset + + monkeypatch.setattr(reader, "deref", dereference) + return SimpleNamespace(objects=[SimpleNamespace(type=SimpleNamespace(name="Texture2D")), owner]) + + +def test_container_matches_full_paths_and_resolves_external_references(monkeypatch): + asset = object() + env = environment(monkeypatch, asset) + assert reader.container_asset(env, "assets/jacket.png") is asset + with pytest.raises(KeyError, match="not in any AssetBundle container"): + reader.container_asset(env, "jacket.png") + with pytest.raises(KeyError, match="null reference"): + reader.container_asset(environment(monkeypatch, None), "assets/jacket.png") + + +def test_jacket_reads_and_resizes_the_named_texture(monkeypatch): + import io + + asset = SimpleNamespace( + type=SimpleNamespace(name="Texture2D"), + read=lambda: SimpleNamespace(image=Image.new("RGBA", (20, 8), (64, 128, 192, 255))), + ) + env = environment(monkeypatch, asset) + requested = [] + + class Catalog: + def fetch_key(self, key): + requested.append(key) + return ["bundle"] + + def entry(self, key): + return {"internal_id": "Assets/Jacket.PNG"} + + def load(paths): + assert paths == ["bundle"] + return env + + monkeypatch.setattr(reader, "load_closure", load) + get = catalog_jacket(Catalog(), size=10) + image = Image.open(io.BytesIO(get("sample"))) + assert image.format == "WEBP" and image.size == (10, 4) + assert requested == ["Image/Jacket/sample"] + asset.type.name = "Sprite" + with pytest.raises(MusicDataError, match="is not a Texture2D"): + get("sample") diff --git a/tests/unity/test_fontsettings.py b/tests/unity/test_fontsettings.py index 2a10e0e..42da8ff 100644 --- a/tests/unity/test_fontsettings.py +++ b/tests/unity/test_fontsettings.py @@ -4,7 +4,7 @@ import pytest -from nnnotes.unity import export, fontsettings +from nnnotes.unity import fontsettings from nnnotes.unity.fontsettings import END_OF_TEXT, SPACE FACE = {"m_PointSize": 40.0, "m_Scale": 1.0} @@ -147,16 +147,6 @@ def test_write_one_entry_per_line(tmp_path): assert sum(ln.startswith('"7":') or ln.startswith('"8":') for ln in lines) == 2 -def test_font_stub_resolves_from_the_objects(): - """A reference into a font bundle left out of the closure: a font asset is named, a material is its record.""" - s = fontsettings.FontSettings(table(**{"A SDF": font("A", material="A SDF Material")})) - ex = export.Exporter.__new__(export.Exporter) - ex.font_stubs = s - asset = export.FontStub(s.objects[7], 7) - assert ex.font_stub(asset) == {"asset": fontsettings.FONT_CLASS, "name": "A SDF"} - assert ex.font_stub(export.FontStub(s.objects[8], 8))["floats"] == {"_GradientScale": 10.0} - - def test_builtin_table_is_complete(): """Every fallback, default material and object the shipped table names is in it.""" s = fontsettings.builtin() diff --git a/tests/unity/test_mesh_texture_guards.py b/tests/unity/test_mesh_texture_guards.py index 2a42cc8..8d7e383 100644 --- a/tests/unity/test_mesh_texture_guards.py +++ b/tests/unity/test_mesh_texture_guards.py @@ -6,7 +6,6 @@ import pytest from UnityPy.enums import TextureFormat -from nnnotes.unity import export from nnnotes.unity import reader as unity @@ -106,8 +105,6 @@ def test_hdr_astc_textures_are_refused_before_decoding(fmt): tex = FakeTexture(int(fmt)) # UnityPy may give the format as a plain int with pytest.raises(unity.UnsupportedTexture, match=rf"'number1_glow': {fmt.name} \(HDR ASTC\)"): unity.texture_image(tex) - with pytest.raises(unity.UnsupportedTexture): - export.texture_png(tex) assert tex.decoded == 0 diff --git a/tests/unity/test_speed_exact.py b/tests/unity/test_speed_exact.py index 6f8c112..1e89b98 100644 --- a/tests/unity/test_speed_exact.py +++ b/tests/unity/test_speed_exact.py @@ -20,9 +20,8 @@ from nnnotes.core import cache, jsonio from nnnotes.formats import cri, encoders, shader -from nnnotes.unity import export from nnnotes.unity import reader as unity -from nnnotes.unity.export import Exporter, TexelPacker +from nnnotes.unity import texels as export from nnnotes.unity.reader import SceneGraph @@ -217,26 +216,6 @@ def ref_value(ex, owner, v): return v -def test_value_fast_path(tmp_path): - ex = Exporter(None, tmp_path) - ex.ref = lambda owner, p: {"ref": p["m_PathID"]} - rng = random.Random(5) - for _ in range(200): - v = [rng.random() for _ in range(rng.randrange(5))] - doc = { - "a": v, - "b": [1, "x", None, True], - "c": [{"m_FileID": 0, "m_PathID": 3}, 2.0], - "d": [b"\x01", 1], - "e": [[1, 2], [3]], - "f": [], - "g": {"h": [{"m_FileID": 1, "m_PathID": 0}]}, - } - out = ex.value(None, doc) - assert out == ref_value(ex, None, doc) - assert out["a"] is not v and out["f"] == [] # new lists, as before - - # ---------------------------------------------------------------- textures class FakeTex: decodes = 0 @@ -283,33 +262,6 @@ def png_of(img, **kw): return b.getvalue() -def test_texture_png_cached_and_equal(): - t = FakeTex(b"abcdefgh" * 10, 16, 8) - FakeTex.decodes = 0 - want = png_of(t.image) - assert export.texture_png(t) == want - assert export.texture_png(FakeTex(b"abcdefgh" * 10, 16, 8)) == want # same inputs: no decode - assert FakeTex.decodes == 2 - assert export.texture_png(FakeTex(b"abcdefgh" * 10, 8, 16)) != want # other size: decoded - cache.configure(enabled=False) - assert export.texture_png(t) == want - - -def test_exporter_texture_files_immediate_and_deferred(tmp_path): - objs = [tex_obj(FakeTex(bytes([i]) * 64, 8, 4), name=f"t{i}", pid=i) for i in range(4)] - a = Exporter(None, tmp_path / "a") - recs_a = [a.texture(o) for o in objs] - b = Exporter(None, tmp_path / "b", defer_writes=True) - recs_b = [b.texture(o) for o in objs] - assert recs_a == recs_b - assert not list((tmp_path / "b" / "textures").iterdir()) - skip = {recs_b[1]["texture"]} - assert b.write_textures(skip=skip) == sorted(r["texture"] for r in recs_b if r["texture"] not in skip) - ta = tree(tmp_path / "a") - assert tree(tmp_path / "b") == {k: v for k, v in ta.items() if k not in skip} - assert ta[recs_a[0]["texture"]] == png_of(objs[0].read().image) - - def ref_pixels(tex_obj): tt = tex_obj.read_typetree() img = tex_obj.read().image @@ -322,26 +274,6 @@ def ref_pixels(tex_obj): return arr[::-1].copy() -@pytest.mark.parametrize("fmt,mode", [(4, "RGBA"), (3, "RGB"), (1, "L"), (1, "RGBA")]) -def test_packer_pixels_and_png(tmp_path, fmt, mode): - o = tex_obj(FakeTex(b"seed%d" % fmt + mode.encode(), 12, 6, fmt, mode)) - FakeTex.decodes = 0 - want = ref_pixels(o) - p1, p2 = TexelPacker(tmp_path / "1"), TexelPacker(tmp_path / "2") - a = p1.pixels(o) - assert np.array_equal(a, want) and not a.flags.writeable - assert np.array_equal(p2.pixels(o), want) - assert FakeTex.decodes == 2 # the reference, then one decode for both packers - for p in (p1, p2): - p.add("g", o, 0, 0, 12, 6) - p.add("g", o, -1, -1, 3, 3) - r1, r2 = p1.build(), p2.build() - assert r1 == r2 and tree(tmp_path / "1") == tree(tmp_path / "2") - out = tree(tmp_path / "1")["textures/g.png"] - arr = np.asarray(Image.open(io.BytesIO(out))) - assert out == png_of(Image.fromarray(arr.copy(), "RGBA"), optimize=True) - - def test_packed_png_matches_pillow(): rgba = np.random.default_rng(1).integers(0, 256, (5, 7, 4), dtype=np.uint8) assert export.packed_png(rgba) == png_of(Image.fromarray(rgba[::-1].copy(), "RGBA"), optimize=True) diff --git a/tests/unity/test_textstyle.py b/tests/unity/test_textstyle.py index 0be5a53..2b7a2f7 100644 --- a/tests/unity/test_textstyle.py +++ b/tests/unity/test_textstyle.py @@ -5,14 +5,9 @@ from types import SimpleNamespace import pytest -from UnityPy.helpers.TypeTreeHelper import read_typetree -from UnityPy.streams.EndianBinaryReader import EndianBinaryReader -from nnnotes.content import advui from nnnotes.source import languages -from nnnotes.source.config import ConfigError -from nnnotes.unity import fontsettings, textstyle, tmpfont -from nnnotes.unity.scripttypes import load_typetrees, typetree_key, typetree_node +from nnnotes.unity import fontsettings, textstyle WHITE = {"r": 1.0, "g": 1.0, "b": 1.0, "a": 1.0} DARK = {"r": 0.2, "g": 0.2, "b": 0.2, "a": 0.8} @@ -261,121 +256,6 @@ def test_no_font_data_in_the_record(): } -def test_text_styles_reads_the_localized_material(monkeypatch): - """The face info and the localized material come from the font settings (no font asset read).""" - monkeypatch.setattr(textstyle, "language_fonts", lambda player: LANG) - face = {"m_Scale": 1.0, "m_PointSize": 100.0, "m_LineHeight": 120.0, "m_AscentLine": 90.0, "m_DescentLine": -30.0} - fonts = fontsettings.FontSettings( - { - "format": fontsettings.FORMAT, - "fonts": {"MainT SDF": {"faceInfo": face, "normalSpacingOffset": 0.0, "boldSpacing": 7.0}}, - "materials": {"MainT - Outline": material(["OUTLINE_ON"], _OutlineWidth=0.25)}, - } - ) - styles = textstyle.TextStyles(None, player=None, mode=TW, fonts=fonts) - rec = styles.record("Canvas/Text", component(), LOCALIZE) - assert rec["outline"]["widthEm"] == pytest.approx(0.25 * 15 / 16 * 0.16, abs=1e-6) - assert styles.summary()["roles"] == { - "primary": {"lineHeightEm": 1.2, "ascentEm": 0.9, "descentEm": -0.3, "spacingOffset": 0.0, "boldSpacing": 7.0} - } - assert styles.summary()["language"] == "traditionalChinese" - with pytest.raises(KeyError, match="MainT - Default of MainT SDF not in the font settings"): - styles.record("Canvas/Text", component(m_sharedMaterial={"material": "MainJ - Default"}), LOCALIZE) - - -def test_font_modes_and_extra(monkeypatch): - assert advui.FONT_MODES == ("open", "game") - with pytest.raises(ValueError, match="fonts must be one of"): - advui.extract(None, None, {}, ".", fonts="other", language="ja") - monkeypatch.setattr(tmpfont.importlib.util, "find_spec", lambda name: None) - with pytest.raises(tmpfont.ConfigError, match=r"nnnotes\[fonts\]") as e: - tmpfont.require_extra() - assert "\n" not in str(e.value) and "fonttools" in str(e.value) - - -def test_the_language_is_an_argument(): - assert not hasattr(textstyle, "LANGUAGE_MODE") and not hasattr(textstyle, "LANGUAGE_FIELD") - en = languages.mode("en") - swap = textstyle.font_swap(LANG, en) - assert swap == {"MainJ SDF": "MainE SDF", "Digits SDF": "DigitsE SDF"} - assert textstyle.localize_text("p", "MainJ SDF", "MainJ - Outline", swap, LANG, en) == { - "fontAsset": "MainE SDF", - "material": "MainE - Outline", - "lineSpacing": -100.0, - } - assert textstyle.font_swap(LANG, languages.mode("ja")) == {"MainJ SDF": "MainJ SDF", "Digits SDF": "Digits SDF"} - style = textstyle.material_style(material(), point_size=100.0) - assert textstyle.text_style(component(), LOCALIZE, LANG, style, en)["lineSpacing"]["applied"] == -100.0 - - -def test_story_ui_language_records(tmp_path): - assert advui.about("en") == "ADV front canvas UI data (UIAdvWidget + UIDefaultTalkWindow), en" - assert advui.language_doc("en") == {"mode": 1, "field": "english", "lineSpacing": -100.0} - assert advui.language_doc("zh-Hant") == {"mode": 2, "field": "traditionalChinese", "lineSpacing": 5.0} - assert list(advui.language_doc("ko")) == ["mode", "field", "lineSpacing"] - # an unknown language is refused before anything is read, naming the setting - with pytest.raises(ConfigError, match="catalog.language: not one of the game's languages"): - advui.extract(None, None, {}, tmp_path, language="xx") - - -def test_shown_texts_take_the_language_field(): - def lines(ja, en): - return {"japanese": ja, "english": en} - - episode = { - "commands": [ - { - "cmd": "Talk", - "TargetName": "a", - "AdvTextID": "t1", - "TargetTextIDs": ["n1"], - "lines": lines("ja1", "en1"), - }, - {"cmd": "Talk", "TargetName": "b", "AdvTextID": "t2", "TargetStatus": 1, "lines": lines("ja2", "en2")}, - { - "cmd": "Talk", - "TargetName": "c", - "AdvTextID": "t3", - "TargetStatus": 2, - "TargetTextIDs": ["n3"], - "lines": lines("ja3", "en3"), - }, - {"cmd": "Location", "lines": lines("jaL", "enL")}, - { - "cmd": "Talk", - "TargetName": "d", - "AdvTextID": "t4", - "TargetTextIDs": ["n1", "n2"], - "lines": lines("ja4", "en4"), - }, - ], - "text": { - "n1": lines("名1", "Name1"), - "n2": lines("名2", "Name2"), - "n3": lines("名3", "Name3"), - "unk": lines("？", "???"), - "split": lines("・", "&"), - }, - "title": lines("題", "Title"), - } - ids = {"_unknownCharacterNameTextId": "unk", "_splitCharacterNameTextId": "split"} - assert advui.shown_texts(episode, ids, "english") == [ - "en1", - "en2", - "en3", - "enL", - "en4", - "&", - "???", - "Name1", - "Name2", - "Title", - ] - assert advui.shown_texts(episode, ids, "japanese")[:2] == ["ja1", "ja2"] - with pytest.raises(RuntimeError, match="no title"): - advui.shown_texts({**episode, "title": None}, ids, "english") - - # -------------------------------------------------------------------------- # LocalizeManager: the serialized bytes, the object json of the shipped type tree # -------------------------------------------------------------------------- @@ -456,17 +336,6 @@ def script(self, o) -> tuple: return ("Fwk", "Fwk.Localization", "LocalizeManager") -def test_language_fonts_reads_the_serialized_fields(): - raw = localize_manager_bytes(LOCALIZE_LANGUAGES, LOCALIZE_MATERIALS) - assert textstyle.language_fonts(_Boot(raw)) == EXPECTED_LANGUAGES - - -def test_language_fonts_refuses_trailing_bytes(): - raw = localize_manager_bytes(LOCALIZE_LANGUAGES, LOCALIZE_MATERIALS) + b"\0\0\0\0" - with pytest.raises(RuntimeError, match="4 trailing bytes"): - textstyle.language_fonts(_Boot(raw)) - - def test_language_fonts_json_is_the_same_dictionary_from_the_object_json(): doc = { "_dontDestroyOnload": 1, @@ -477,18 +346,3 @@ def test_language_fonts_json_is_the_same_dictionary_from_the_object_json(): "_materialTypeList": LOCALIZE_MATERIALS, } assert textstyle.language_fonts_json(doc) == EXPECTED_LANGUAGES - - -def test_the_shipped_type_tree_reads_the_localize_manager_bytes(): - entry = load_typetrees("6000.3.12f1")["classes"][typetree_key("Fwk", "Fwk.Localization", "LocalizeManager")] - assert entry["typeHash"] == LOCALIZE_HASH - assert [1, "UInt8", "_dontDestroyOnload", ALIGN] in entry["nodes"] - raw = localize_manager_bytes(LOCALIZE_LANGUAGES, LOCALIZE_MATERIALS) - tt = read_typetree( - typetree_node(entry["nodes"]), - EndianBinaryReader(raw, "<"), - as_dict=True, - byte_size=len(raw), - check_read=True, - ) - assert textstyle.language_fonts_json(tt) == EXPECTED_LANGUAGES == textstyle.language_fonts(_Boot(raw)) From 44d6a4a6f66f719dfa55e787161db704efa036a5 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 18 Oct 2026 19:45:00 +0800 Subject: [PATCH 76/91] feat(depot): release completed bundle downloads during exports --- CHANGELOG.md | 1 + docs/architecture.md | 5 ++ docs/commands.md | 8 ++ docs/configuration.md | 9 ++- src/nnnotes/cli/assets.py | 10 +++ src/nnnotes/core/downloads.py | 97 ++++++++++++++++------- src/nnnotes/pipeline/downloads.py | 56 ++++++++++++++ src/nnnotes/pipeline/orchestrate.py | 34 ++++++++- tests/cli/test_assets.py | 34 ++++++++- tests/core/test_downloads.py | 65 ++++++++++++++++ tests/pipeline/test_downloads.py | 104 +++++++++++++++++++++++++ tests/pipeline/test_orchestrate.py | 114 +++++++++++++++++++++++++++- 12 files changed, 498 insertions(+), 39 deletions(-) create mode 100644 src/nnnotes/pipeline/downloads.py create mode 100644 tests/pipeline/test_downloads.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 423fb79..ac9277d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor v ### Features - Store content, task inputs, results and catalog versions in a persistent depot with optional remote storage, verified collection and bounded source downloads. +- Release recoverable bundle downloads after successful conversion batches under the configured budget and command leases. - Assemble Live2D, room, spot, live, story and UI products from shared object tables and explicit manifest references. - Generate localized live and story text products with game or supplied fonts, shared glyph atlases and material records. - Build named sites through depot product, player read-set, asset and index tasks. diff --git a/docs/architecture.md b/docs/architecture.md index d3e28b9..7fb53b9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -48,6 +48,11 @@ The site's `Builder` composes product snapshots, catalog indexes and player page Converters use bounded process-local memoization for repeated reads and computations. Persistent reuse is keyed by depot tasks and their content inputs. Clearing a process memo releases its values; a later task can reuse its committed depot result. +The coordinator observes successful object batches after worker readers close. Their recoverable bundle downloads +can then be released toward the configured budget under the command's exclusive lease. Preliminary episode +requests retain inputs until the expanded product request is known. Stored conversion results remain available +for assembly and layouts. + `export episode`, `export shader`, `export audio` and `music-data` expose subject-specific data and conversions through the Session. All commands use the readers and converters below the pipeline layer. ## Outputs diff --git a/docs/commands.md b/docs/commands.md index 87f2979..6969eef 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -69,6 +69,14 @@ Catalog files are kept. A removed file is downloaded again when a command needs identity (sha256 and size) of every bundle and raw file it reads, so `export assets` plans with files whose name carries the catalog's hash without downloading them again. +During product and asset requests, a completed object-conversion batch makes its bundle downloads eligible for +release toward the same budget. Readers finish before release; active batches, failed batches and preliminary +episode requests stay protected. A batch already present in the depot can release its downloaded input as well. +Only files recoverable through an explicit catalog or depot locator are released, and their contents are checked +against the input identity. `all` retains downloads throughout the run; `0` releases each eligible batch without +scanning the download directory. With a finite size, finished files are evicted in LRU order and active files can +temporarily exceed the budget. Later requests can download a released bundle again. + Commands may run at the same time on one depot: no command removes downloads while another one holds a lease, and a command that starts while one trims waits for it. The lease of a process that ended without releasing it is dropped by the next command; a lease written on another host counts for a day. diff --git a/docs/configuration.md b/docs/configuration.md index f36ad31..a5419a3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -110,7 +110,7 @@ exist when they are given. | `[paths] player` | `NNNOTES_PATHS_PLAYER` | `--player` (of `site build`) | ournotes-player checkout (built) or installed package | | `[paths] fonts.` | `NNNOTES_PATHS_FONTS_` (e.g. `NNNOTES_PATHS_FONTS_ZH_HANT`) | `--font =PATH` (of `site build`) | font file (OpenType or TrueType) the story text of a language is drawn with (`--fonts open`); also read for the other story languages whose game fonts fall back to that language's font | | `[paths] fonts.emoji` | `NNNOTES_PATHS_FONTS_EMOJI` | `--font emoji=PATH` (of `site build`) | colour emoji font (PNG bitmap glyphs: CBDT or sbix) the stories' emoji sprites are drawn from (`--fonts open`); Noto Color Emoji (SIL Open Font License 1.1) is the tested font. Optional: without it the sprites keep their layout with empty glyphs, listed in `ui/fonts.json` `coverage.sprites.missing` | -| `[depot] downloads` | `NNNOTES_DEPOT_DOWNLOADS` | — | how much of the depot's downloaded bundles and raw files to keep once no command uses the depot: a size (`500MB`, `20GiB`; units B, KB, MB, GB, TB and KiB, MiB, GiB, TiB), `all`, or `0` (unset) ([commands.md](commands.md#downloads)) | +| `[depot] downloads` | `NNNOTES_DEPOT_DOWNLOADS` | — | download-cache budget, applied to finished conversion batches and when the last command ends: a size (`500MB`, `20GiB`; units B, KB, MB, GB, TB and KiB, MiB, GiB, TiB), `all`, or `0` (unset) ([commands.md](commands.md#downloads)) | | `[depot] remote` | `NNNOTES_DEPOT_REMOTE` | — | a remote of the depot shared by the machines that use it: `s3:///` (S3-compatible object storage; `pip install 'nnnotes[remote]'`) or `file:///` ([contracts.md](contracts.md#depot-remote)) | | `[depot] endpoint` | `NNNOTES_DEPOT_ENDPOINT` | — | endpoint URL of the S3-compatible object storage; unset: AWS S3 | | `[depot] access_key_id` | `NNNOTES_DEPOT_ACCESS_KEY_ID` | — | access key id of the object storage; unset: boto3 looks for credentials itself (`AWS_ACCESS_KEY_ID`, `~/.aws`) | @@ -188,9 +188,10 @@ story text); it is not a setting. | `catalog versions` / `import` / `diff`, `depot verify` | the depot; `import` reads the APK's catalog from `[paths] apk` when it is set | | `catalog fetch` | the depot, `[catalog] region` and `language`, that region's `cdn`; its `api` (optional) for the resource version label | -The region, its `cdn`, `[bundle] key` and `nonce_seed` are read only when a file must be downloaded: when the -catalog and every bundle a command needs are downloaded, none of them is needed. A download that needs a missing -one stops the command with one line naming the file and the setting (see Errors). +The region, its `cdn`, `[bundle] key` and `nonce_seed` are read only when a file must be downloaded. A product +request can read a downloaded catalog and its bundle files without these settings. Use `[depot] downloads = all` +for an offline sequence of requests; a bounded budget can release a bundle that a later request needs again. +A download that needs a missing setting stops the command with one line naming the file and setting (see Errors). ## Errors diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index d77d215..d36ca5a 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -1317,8 +1317,10 @@ def cmd_export(args, cfg): def request_orchestrator(ws, cfg): """The shared executor for product requests.""" + from ..pipeline.downloads import BundleDownloads from ..pipeline.orchestrate import Orchestrator + lease, budget = getattr(ws.args, "lease", None), Session(cfg).downloads_budget() return Orchestrator( ws.depot, ws.scheduled, @@ -1329,12 +1331,14 @@ def request_orchestrator(ws, cfg): initializer=_worker_init, initargs=(cfg,), log=ws.log, + on_batch_done=BundleDownloads(ws.depot, lease, budget) if lease is not None and budget is not None else None, ) def run_request(args, cfg, ws: Workspace, expand=None, prepared=None): """Execute a product request and optional expansion without materializing an output layout.""" from ..pipeline import builtins + from ..pipeline.downloads import BundleDownloads from ..pipeline.orchestrate import failure if prepared is None: @@ -1343,6 +1347,8 @@ def run_request(args, cfg, ws: Workspace, expand=None, prepared=None): orch = request_orchestrator(ws, cfg) else: pipeline, params, orch = prepared + if expand is not None: + orch.on_batch_done = None # the episode supplies the roots of the complete product request ws.run_index() facts = ws.facts(pipeline, fetch=True, background=True) sel = ws.selected @@ -1374,6 +1380,10 @@ def run_request(args, cfg, ws: Workspace, expand=None, prepared=None): aborted = aborted or str(e) if isinstance(facts["raw"], Later) and aborted is None: ws.log(f"{sum(p['kind'] == 'raw' for p in ws.problems())} raw files not readable") + released = orch.on_batch_done + if isinstance(released, BundleDownloads) and released.removed: + run.log.append({"event": "downloads.released", "files": released.removed, "bytes": released.bytes_removed}) + ws.log(f"downloads: released {released.removed} bundle files ({released.bytes_removed} bytes)") if expand is not None and aborted is None: more = expand(run) if more is not None: diff --git a/src/nnnotes/core/downloads.py b/src/nnnotes/core/downloads.py index 732afcc..f1a79a1 100644 --- a/src/nnnotes/core/downloads.py +++ b/src/nnnotes/core/downloads.py @@ -1,11 +1,12 @@ -"""The downloads of a depot as a cache bounded in size: what a command reads again stays while commands run, and the -bundles and raw files are trimmed to the budget when the last command using the depot ends. +"""Budgeted depot downloads, with finished-batch release and command leases protecting active readers. /downloads/ the game files as downloaded (source.catalog) /leases/.json one per running command: {"host", "pid", "started"} /leases/.exclusive held by a command that must be the only one (trimming, `depot gc`) -A command holds a lease while it runs (`lease`). When it ends and no other lease is live (a lease of this host whose +A command holds a lease while it runs (`lease`). A successful object conversion batch may release its recoverable +bundle downloads toward the budget (`release`) while it is the only command. When the command ends and no other +lease is live (a lease of this host whose process runs, or of another host younger than a day), the files under the `bundles/` and `raw/` directories of downloads/ are removed, least recently used first (a file's modification time: the catalog sets it when it hands the file out), until they fit the budget. Catalog files stay. A removed file is downloaded again when a command needs @@ -25,12 +26,12 @@ import secrets import socket import time -from collections.abc import Generator, Iterator +from collections.abc import Generator, Iterable, Iterator from contextlib import contextmanager from pathlib import Path from typing import Any -__all__ = ["evictable", "exclusive", "lease", "parse_budget", "trim"] +__all__ = ["evictable", "exclusive", "lease", "parse_budget", "release", "trim"] FOREIGN_LEASE_SECONDS = 24 * 3600 # a lease of another host counts as live this long EXCLUSIVE = ".exclusive" @@ -71,39 +72,79 @@ def evictable(downloads: Path) -> Iterator[Path]: yield p -def trim(downloads: Path, budget: int | None, *, dry_run: bool = False) -> dict[str, int]: - """Remove the least recently used bundles and raw files until the rest fit `budget` bytes (None: nothing). - -> {"kept", "removed", "bytesKept", "bytesRemoved"}.""" - files: list[tuple[int, int, Path]] = [] - for p in evictable(downloads): +def _files(paths: Iterable[Path]) -> list[tuple[int, int, Path]]: + out: list[tuple[int, int, Path]] = [] + for path in paths: try: - st = p.stat() + stat = path.stat() except OSError: continue - files.append((st.st_mtime_ns, st.st_size, p)) + out.append((stat.st_mtime_ns, stat.st_size, path)) + return out + + +def _remove(files: list[tuple[int, int, Path]], needed: int, dry_run: bool) -> tuple[list[Path], int]: + removed: list[Path] = [] + removed_bytes = 0 + for _, size, path in sorted(files, key=lambda f: (f[0], str(f[2]))): + if removed_bytes >= needed: + break + if not dry_run: + try: + path.unlink() + except OSError: + continue + removed.append(path) + removed_bytes += size + return removed, removed_bytes + + +def trim(downloads: Path, budget: int | None, *, dry_run: bool = False) -> dict[str, int]: + """Remove the least recently used bundles and raw files until the rest fit `budget` bytes (None: nothing). + -> {"kept", "removed", "bytesKept", "bytesRemoved"}.""" + files = _files(evictable(downloads)) total = sum(f[1] for f in files) - removed = removed_bytes = 0 - if budget is not None: - for _, size, p in sorted(files, key=lambda f: (f[0], str(f[2]))): - if total - removed_bytes <= budget: - break - if not dry_run: - try: - p.unlink() - except OSError: # open elsewhere (Windows): kept - continue - removed += 1 - removed_bytes += size - if not dry_run and removed: - _prune_dirs(downloads) + removed, removed_bytes = _remove(files, 0 if budget is None else total - budget, dry_run) + if not dry_run and removed: + _prune_dirs(downloads) return { - "kept": len(files) - removed, - "removed": removed, + "kept": len(files) - len(removed), + "removed": len(removed), "bytesKept": total - removed_bytes, "bytesRemoved": removed_bytes, } +def release(downloads: Path, ready: Iterable[Path], budget: int | None) -> dict[str, int]: + """Trim only finished downloads, in LRU order, toward the cache budget. The caller holds an exclusive lease. + + Active files remain protected even when they exceed the budget. A zero budget examines only finished files. + Paths must resolve below a bundles/ or raw/ directory within downloads; partial files are retained. + """ + if budget is None: + return {"removed": 0, "bytesRemoved": 0} + root = downloads.resolve() + paths: set[Path] = set() + for path in ready: + path = path.resolve() + if not path.is_relative_to(root) or not {"bundles", "raw"} & set(path.relative_to(root).parts[:-1]): + raise ValueError("released file must be a bundle or raw file within the downloads directory") + if path.is_file() and not path.name.endswith(".part"): + paths.add(path) + files = _files(paths) + total = sum(f[1] for f in files) if budget == 0 else sum(f[1] for f in _files(evictable(root))) + removed, size = _remove(files, total - budget, False) + for path in removed: + parent = path.parent + while parent != root: + try: + parent.rmdir() + except OSError: + break + parent = parent.parent + return {"removed": len(removed), "bytesRemoved": size} + + def _prune_dirs(root: Path) -> None: for d in sorted((p for p in root.rglob("*") if p.is_dir()), key=lambda p: len(p.parts), reverse=True): try: diff --git a/src/nnnotes/pipeline/downloads.py b/src/nnnotes/pipeline/downloads.py new file mode 100644 index 0000000..24d4397 --- /dev/null +++ b/src/nnnotes/pipeline/downloads.py @@ -0,0 +1,56 @@ +"""Budgeted release of recoverable bundle downloads after complete conversion batches.""" + +from pathlib import Path + +from ..core import downloads +from .contract import Input + + +class BundleDownloads: + """A main-process batch observer; only closed, recoverable bundle inputs become eligible for release.""" + + def __init__(self, depot, lease: Path | None, budget: int | None): + self.depot, self.lease, self.budget = depot, Path(lease) if lease is not None else None, budget + self.ready: dict[Path, Input] = {} + self.removed = self.bytes_removed = 0 + + def __call__(self, tasks): + if self.budget is None or self.lease is None or not self.lease.is_file(): + return + if self.lease.parent.resolve() != (self.depot.root / "leases").resolve() or self.lease.suffix != ".json": + return + if not tasks or any(task.stage != "unity.export" for task in tasks): + return + root = self.depot.downloads.resolve() + for task in tasks: + inp = task.input("bundle") + kinds = {loc["kind"] for loc in inp.locators} + if "catalog" not in kinds and not ("depot" in kinds and self.depot.has(inp.sha256, inp.size)): + continue + for loc in inp.locators: + if loc["kind"] != "download": + continue + path = (root / loc["path"]).resolve() + if path.is_relative_to(root) and "bundles" in path.relative_to(root).parts[:-1]: + self.ready[path] = inp + if not self.ready: + return + with downloads.exclusive(self.depot.root, self.lease) as alone: + if not alone: + return + valid = [] + for path, inp in list(self.ready.items()): + try: + matches = path.is_file() and self.depot.identify(path) == (inp.sha256, inp.size) + except OSError: + matches = False + if matches: + valid.append(path) + else: + self.ready.pop(path) + result = downloads.release(root, valid, self.budget) + self.removed += result["removed"] + self.bytes_removed += result["bytesRemoved"] + for path in valid: + if not path.exists(): + self.ready.pop(path, None) diff --git a/src/nnnotes/pipeline/orchestrate.py b/src/nnnotes/pipeline/orchestrate.py index 8006c1b..dd48d8c 100644 --- a/src/nnnotes/pipeline/orchestrate.py +++ b/src/nnnotes/pipeline/orchestrate.py @@ -510,7 +510,9 @@ def coverage(results, census: dict | None = None) -> dict: class Orchestrator: """Runs pipelines of stages into a depot (see the module documentation). `initializer(depot, *initargs)` runs - first in every worker (settings, caches, the depot's fetcher); it must be importable (spawned processes).""" + first in every worker (settings, caches, the depot's fetcher); it must be importable (spawned processes). + `on_batch_done(tasks)` observes successful batches in the coordinator after readers close, including batches + entirely satisfied by committed results. Failed batches and incomplete task descriptions are not observed.""" def __init__( self, @@ -525,12 +527,16 @@ def __init__( initializer=None, initargs: tuple = (), log=_log, + on_batch_done=None, ): self.depot, self.stages = depot, stages self.workers, self.memory, self.worker_bytes = max(0, int(workers)), memory, int(worker_bytes) self.recycle_tasks, self.recycle_rss = recycle_tasks, recycle_rss self.initializer, self.initargs = initializer, tuple(initargs) self.log = log + self.on_batch_done = on_batch_done + self._batch_errors: set[tuple] = set() + self._observe_batches = True self._pool: list[_Worker] = [] self._started = 0 @@ -595,7 +601,7 @@ def _stage(self, stage: Stage, params: dict, env: Env, force: set, run: Run) -> t0 = time.perf_counter() tasks: list[Task] = [] try: - subjects = stage.subjects(env) + subjects = list(stage.subjects(env)) except Pending as p: self._fail( run, @@ -623,6 +629,7 @@ def _stage(self, stage: Stage, params: dict, env: Env, force: set, run: Run) -> ) except Exception as e: self._fail(run, env, tid, None, stage.name, "task.describe", f"{type(e).__name__}: {e}"[:2000]) + self._observe_batches = len(tasks) == len(subjects) forced = "*" in force or stage.name in force todo = [] for t in sorted(tasks, key=lambda t: t.id): @@ -631,6 +638,11 @@ def _stage(self, stage: Stage, params: dict, env: Env, force: set, run: Run) -> env.done(t.id, t.key) else: todo.append(t) + pending = {t.id for t in todo} + if self.on_batch_done is not None and self._observe_batches: + for job in jobs(tasks, self.stages): + if job.batch is not None and not any(t.id in pending for t in job.tasks): + self.on_batch_done(job.tasks) replies = self._execute(todo, force=lambda t: forced or t.id in force, run=run) stop = next((r for r in replies.values() if r.get("code") == "config"), None) for t in todo: @@ -657,6 +669,7 @@ def _fail(run: Run, env: Env, tid: str, key, stage: str, code: str, message: str # ---------------------------------------------------------------- execution def _execute(self, tasks: list[Task], force, run: Run) -> dict[str, dict]: """Run `tasks`; {task id: reply} (none for a task not started because the run stopped).""" + self._batch_errors.clear() if not tasks: return {} sched = Scheduler(jobs(tasks, self.stages), self.task_budget()) @@ -675,6 +688,7 @@ def _execute(self, tasks: list[Task], force, run: Run) -> dict[str, dict]: break if j.batch is not None: self.stages[j.batch[0]].end_batch(j.batch[1]) + self._batch_done(j, replies) sched.finished(j.id) else: self._pool_run(sched, force, replies, run) @@ -724,6 +738,7 @@ def _collect(self, w: _Worker, sched: Scheduler, replies: dict, run: Run) -> Non except (EOFError, OSError): break if msg.get("end"): + self._batch_done(job, replies) sched.finished(job.id) w.job = None w.done += w.replied @@ -733,10 +748,14 @@ def _collect(self, w: _Worker, sched: Scheduler, replies: dict, run: Run) -> Non t = job.tasks[w.replied] w.replied += 1 replies[t.id] = msg + if not msg["ok"] and job.batch is not None: + self._batch_errors.add(job.batch) self._done(run, t, msg, w) if msg.get("code") == "config": sched.stop() w.proc.join(5) + if job.batch is not None: + self._batch_errors.add(job.batch) if w.replied < len(job.tasks): t = job.tasks[w.replied] replies[t.id] = { @@ -754,6 +773,17 @@ def _collect(self, w: _Worker, sched: Scheduler, replies: dict, run: Run) -> Non w.job = None self._replace(w) + def _batch_done(self, job: Job, replies: dict) -> None: + """Observe a successful batch after its reader is dropped; cache-only batches are also observed.""" + if ( + self.on_batch_done is not None + and self._observe_batches + and job.batch is not None + and job.batch not in self._batch_errors + and all(replies.get(t.id, {}).get("ok") for t in job.tasks) + ): + self.on_batch_done(job.tasks) + def _recycle(self, w: _Worker, r: int | None) -> bool: if self.recycle_tasks and w.done >= self.recycle_tasks: return True diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index cc9e3eb..e3e29f1 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -214,19 +214,24 @@ def depends(self, subject, env): return [self._census(subject, env), "link.scripts:all"] def inputs(self, subject, env): - return [env.input_of(self._census(subject, env), "census"), env.input_of("link.scripts:all", "scripts")] + return [ + env.fact("bundles")[units.table(env).unit(subject).bundle], + env.input_of(self._census(subject, env), "census"), + env.input_of("link.scripts:all", "scripts"), + ] def context(self, subject, env): unit = units.table(env).unit(subject) return {"unit": unit.root, "members": list(unit.members)} def batch(self, task): - return task.input("census").sha256 + return task.input("bundle").sha256 def estimate(self, subject, env, inputs): return Cost(0.01, 1 << 20) def run(self, task, depot): + assert depot.input_bytes(task.input("bundle")).startswith(b"UnityFS") doc = contract.loads(depot.input_bytes(task.input("census"))) scripts = contract.loads(depot.input_bytes(task.input("scripts")))["scripts"] container = {(e["file"], e["pathId"]): e["path"] for b in doc["assetBundles"] for e in b["container"]} @@ -1586,10 +1591,12 @@ def test_export_live2d_writes_the_cubism_files_of_one_model(tmp_path, capsys, mo assert f"live2d.cubism:{MODEL_KEY}" in ids and "fake.product:Bg/x" not in ids -def test_export_runs_the_request_expand_gives_and_places_the_last_run(tmp_path, capsys, monkeypatch): +@pytest.mark.parametrize("downloads_budget", ["all", "0"]) +def test_export_runs_the_request_expand_gives_and_places_the_last_run(tmp_path, capsys, monkeypatch, downloads_budget): from urllib.parse import quote monkeypatch.setattr(assets, "STAGE_SOURCES", PRODUCT_SOURCES) + monkeypatch.setenv("NNNOTES_DEPOT_DOWNLOADS", downloads_budget) monkeypatch.setattr("nnnotes.pipeline.catalogdb.apk_version", lambda apk: None) d = setup_data(tmp_path, censuses(broken=False), extra=[(MODEL_KEY, "Assets/Game/Bg/x.prefab", [3])]) apk = builtin_apk(tmp_path / "base.apk", {"assets/aa/catalog.bin": synth.CatalogWriter().build([])}) @@ -1601,6 +1608,7 @@ def hook(run): # called once, with the first run (the command's own request: th if len(calls) > 1: pytest.fail("expand was called again after its second answer") assert "fake.product:Bg/x" in calls[0] + assert (d["downloads"] / "bundles" / NAMES["bg_x"]).exists() return {"live2d.cubism": [MODEL_KEY]}, [f"key:{MODEL_KEY}"] def hooked(args, cfg, ws, expand=None): # the command's export takes the hook; its own re-run takes none @@ -1634,6 +1642,26 @@ def hooked(args, cfg, ws, expand=None): # the command's export takes the hook; assert not (tmp_path / "out" / "fake.product").exists() # the first run's product is not placed +def test_completed_bundle_downloads_are_released_before_layout_publication(tmp_path, capsys, fake, monkeypatch): + from nnnotes.pipeline import layout + + monkeypatch.setenv("NNNOTES_DEPOT_DOWNLOADS", "0") + d = setup_data(tmp_path, censuses(broken=False)) + place = layout.materialize + observed = [] + + def publish(*args, **kwargs): + assert list((tmp_path / "s/leases").glob("*.json")) + assert not (d["downloads"] / "bundles" / NAMES["bg_x"]).exists() + observed.append(True) + return place(*args, **kwargs) + + monkeypatch.setattr(layout, "materialize", publish) + code, out, err = export(capsys, tmp_path, d, "s", "out") + assert code == 0 and observed, err + assert "downloads: released" in err + + def test_every_product_a_story_plays_with_is_installed(): from nnnotes.pipeline import story diff --git a/tests/core/test_downloads.py b/tests/core/test_downloads.py index 656645a..e14751b 100644 --- a/tests/core/test_downloads.py +++ b/tests/core/test_downloads.py @@ -137,3 +137,68 @@ def test_exclusive_is_granted_to_the_only_command(tmp_path): with downloads.exclusive(tmp_path, own) as alone: assert not alone # another lease is live assert not list((tmp_path / "leases").iterdir()) + + +def test_release_respects_the_budget_and_protects_active_files(tmp_path): + root = tmp_path / "downloads" + old = put(root, "bundles/old.bundle", 10, 1) + ready = put(root, "bundles/ready.bundle", 10, 2) + active = put(root, "bundles/active.bundle", 20, 3) + assert downloads.release(root, [old, ready], 30) == {"removed": 1, "bytesRemoved": 10} + assert not old.exists() and ready.exists() and active.exists() + assert downloads.release(root, [ready], 5) == {"removed": 1, "bytesRemoved": 10} + assert active.exists() + + +def test_zero_budget_release_examines_only_finished_paths(tmp_path, monkeypatch): + root = tmp_path / "downloads" + ready = put(root, "bundles/ready.bundle", 10, 1) + active = put(root, "raw/active.bin", 20, 2) + partial = put(root, "bundles/fetch.part", 5, 3) + + def scan(_): + raise AssertionError("zero-budget release does not need a directory inventory") + + monkeypatch.setattr(downloads, "evictable", scan) + assert downloads.release(root, [ready, ready, partial], 0) == {"removed": 1, "bytesRemoved": 10} + assert active.exists() and partial.exists() + + +def test_release_validates_paths_and_keeps_unlimited_downloads(tmp_path): + root = tmp_path / "downloads" + file = put(root, "bundles/one.bundle", 10, 1) + assert downloads.release(root, [file], None) == {"removed": 0, "bytesRemoved": 0} + for path in [tmp_path / "outside", root / "catalog.bin"]: + with pytest.raises(ValueError, match="within the downloads"): + downloads.release(root, [path], 0) + assert file.exists() + + +def test_release_keeps_locked_files_and_continues(tmp_path, monkeypatch): + root = tmp_path / "downloads" + locked = put(root, "bundles/locked.bundle", 10, 1) + other = put(root, "bundles/other.bundle", 20, 2) + unlink = type(locked).unlink + + def remove(path, *args, **kwargs): + if path == locked: + raise PermissionError("open") + return unlink(path, *args, **kwargs) + + monkeypatch.setattr(type(locked), "unlink", remove) + assert downloads.release(root, [locked, other], 0) == {"removed": 1, "bytesRemoved": 20} + assert locked.exists() and not other.exists() + + +def test_release_rejects_symlinks_to_outside_files(tmp_path): + root = tmp_path / "downloads" + target = put(tmp_path, "outside.bin", 10, 1) + link = root / "bundles/link.bundle" + link.parent.mkdir(parents=True) + try: + link.symlink_to(target) + except OSError: + pytest.skip("symlink creation is unavailable") + with pytest.raises(ValueError, match="within the downloads"): + downloads.release(root, [link], 0) + assert target.exists() and link.is_symlink() diff --git a/tests/pipeline/test_downloads.py b/tests/pipeline/test_downloads.py new file mode 100644 index 0000000..ce38c73 --- /dev/null +++ b/tests/pipeline/test_downloads.py @@ -0,0 +1,104 @@ +"""Bundle release uses completed task inputs, recoverable locators and command leases.""" + +import hashlib +from dataclasses import replace + +from nnnotes.core import downloads +from nnnotes.pipeline.contract import Input, Task +from nnnotes.pipeline.depot import Depot +from nnnotes.pipeline.downloads import BundleDownloads + + +def bundle(depot, data=b"bundle", name="one.bundle", recover=True): + path = depot.downloads / "bundles" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + locators = [{"kind": "download", "path": "bundles/" + name}] + if recover: + locators.append({"kind": "catalog", "catalog": "example", "location": name}) + inp = Input("bundle", hashlib.sha256(data).hexdigest(), len(data), name, tuple(locators)) + return path, Task("unity.export", 1, name, inputs=(inp,)) + + +def test_other_leases_protect_files_until_a_later_batch(tmp_path): + depot = Depot(tmp_path) + path, task = bundle(depot) + with downloads.lease(tmp_path, None) as own: + release = BundleDownloads(depot, own, 0) + with downloads.lease(tmp_path, None): + release([task]) + assert path.exists() and release.removed == 0 + release([task]) + assert not path.exists() and release.removed == 1 and release.bytes_removed == 6 + + +def test_only_matching_recoverable_bundle_inputs_are_released(tmp_path): + depot = Depot(tmp_path) + changed, task = bundle(depot) + changed.write_bytes(b"another") + private, private_task = bundle(depot, name="private.bundle", recover=False) + other, other_task = bundle(depot, name="other.bundle") + with downloads.lease(tmp_path, None) as own: + release = BundleDownloads(depot, own, 0) + release([task, private_task]) + release([replace(other_task, stage="unity.census")]) + assert changed.exists() and private.exists() and other.exists() + release([other_task]) + assert not other.exists() and release.removed == 1 + + +def test_missing_lease_and_unlimited_budget_keep_files(tmp_path): + depot = Depot(tmp_path) + path, task = bundle(depot) + BundleDownloads(depot, None, 0)([task]) + BundleDownloads(depot, tmp_path / "absent", 0)([task]) + invalid = tmp_path / "outside.json" + invalid.write_text("{}") + BundleDownloads(depot, invalid, 0)([task]) + with downloads.lease(tmp_path, None) as own: + BundleDownloads(depot, own, None)([task]) + assert path.exists() + + +def test_cas_recovery_requires_an_explicit_depot_locator(tmp_path): + depot = Depot(tmp_path) + path, task = bundle(depot, recover=False) + inp = task.inputs[0] + depot.put(path.read_bytes()) + with downloads.lease(tmp_path, None) as own: + release = BundleDownloads(depot, own, 0) + release([task]) + assert path.exists() + task = replace(task, inputs=(replace(inp, locators=(*inp.locators, {"kind": "depot"})),)) + release([task]) + assert not path.exists() and depot.input_bytes(task.inputs[0]) == b"bundle" + + +def test_released_input_can_be_fetched_for_an_expanded_request(tmp_path): + fetched = [] + + def fetch(inp, locator): + fetched.append(locator) + path = tmp_path / "downloads/bundles/one.bundle" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"bundle") + return path + + depot = Depot(tmp_path, fetch=fetch) + path, task = bundle(depot) + with downloads.lease(tmp_path, None) as own: + BundleDownloads(depot, own, 0)([task]) + assert not path.exists() + assert depot.input_bytes(task.inputs[0]) == b"bundle" and len(fetched) == 1 + + +def test_download_locators_cannot_release_files_outside_the_depot(tmp_path): + depot = Depot(tmp_path / "depot") + outside = tmp_path / "bundles/outside" + outside.parent.mkdir() + outside.write_bytes(b"bundle") + _, task = bundle(depot) + inp = replace(task.inputs[0], locators=({"kind": "download", "path": str(outside)}, {"kind": "catalog"})) + with downloads.lease(depot.root, None) as own: + BundleDownloads(depot, own, 0)([replace(task, inputs=(inp,))]) + assert outside.exists() diff --git a/tests/pipeline/test_orchestrate.py b/tests/pipeline/test_orchestrate.py index 47c2914..4b3533e 100644 --- a/tests/pipeline/test_orchestrate.py +++ b/tests/pipeline/test_orchestrate.py @@ -172,11 +172,11 @@ def _note(self, line: str) -> None: f.write(line + "\n") -def batched(tmp_path, name, texts, workers): +def batched(tmp_path, name, texts, workers, **kwargs): log = tmp_path / f"{name}-log" log.mkdir() stages = registry(Batched(log), Count(), Total()) - depot, run, orch = orchestrate(tmp_path, name, texts, workers=workers, stages=stages) + depot, run, orch = orchestrate(tmp_path, name, texts, workers=workers, stages=stages, **kwargs) return depot, run, orch, [" ".join(f.read_text(encoding="utf-8").split()) for f in sorted(log.iterdir())] @@ -214,6 +214,116 @@ def test_a_worker_dying_in_a_job_fails_its_task_and_the_rest_runs_in_another(tmp assert orch._started >= 2 +@pytest.mark.parametrize("workers", [0, 2]) +def test_batch_observer_runs_after_readers_close_and_observes_cache_hits(tmp_path, workers): + seen = [] + log = tmp_path / "s-log" + + def observe(tasks): + batch = tasks[0].subject[0] + text = "\n".join(p.read_text(encoding="utf-8") for p in log.glob("*.log")) + assert f"drop {batch}" in text + seen.append([t.subject for t in tasks]) + + depot, first, _, logs = batched(tmp_path, "s", {"a1": "x", "a2": "y", "b1": "z"}, workers, on_batch_done=observe) + assert first.exit_code == 0 and sorted(seen) == [["a1", "a2"], ["b1"]] + seen.clear() + stages = registry(Batched(log), Count(), Total()) + again = Orchestrator(depot, stages, workers=workers, on_batch_done=observe, log=lambda _: None) + run = again.run(PIPELINE, facts={"files": toy_files(tmp_path / "in", {"a1": "x", "a2": "y", "b1": "z"})}) + assert all(t["status"] == "hit" for t in run.tasks.values()) + assert sorted(seen) == [["a1", "a2"], ["b1"]] + assert [" ".join(p.read_text(encoding="utf-8").split()) for p in sorted(log.iterdir())] == logs + + +@pytest.mark.parametrize("workers", [0, 1]) +def test_a_mixed_cached_batch_is_observed_after_its_pending_tasks(tmp_path, workers): + depot, _, _, _ = batched(tmp_path, "s", {"a1": "x"}, workers) + seen = [] + orch = Orchestrator( + depot, + registry(Batched(tmp_path / "s-log"), Count(), Total()), + workers=workers, + on_batch_done=lambda tasks: seen.append([t.subject for t in tasks]), + log=lambda _: None, + ) + run = orch.run(PIPELINE, facts={"files": toy_files(tmp_path / "in", {"a1": "x", "a2": "y"})}) + assert run.tasks["toy.src:a1"]["status"] == "hit" + assert run.tasks["toy.src:a2"]["status"] == "ran" + assert seen == [["a2"]] + + +@pytest.mark.parametrize("workers, failure", [(0, "CRASH"), (2, "CRASH"), (1, "DIE")]) +def test_failed_batches_are_kept_even_when_their_remaining_tasks_succeed(tmp_path, workers, failure): + seen = [] + _, run, _, _ = batched( + tmp_path, + "s", + {"a1": "x", "a2": failure, "a3": "y", "b1": "z"}, + workers, + on_batch_done=lambda tasks: seen.append(tasks[0].subject[0]), + ) + assert run.exit_code == 1 + assert seen == ["b"] + + +def test_incomplete_task_descriptions_defer_batch_observation(tmp_path): + class Missing(Batched): + def inputs(self, subject, env): + if subject == "a2": + raise ValueError("unavailable input") + return super().inputs(subject, env) + + log = tmp_path / "log" + log.mkdir() + seen = [] + _, run, _ = orchestrate( + tmp_path, + "s", + {"a1": "x", "a2": "y"}, + stages=registry(Missing(log), Count(), Total()), + on_batch_done=seen.append, + ) + assert run.exit_code == 1 and seen == [] + + +@pytest.mark.parametrize("workers", [0, 2]) +def test_interruption_before_batch_end_does_not_notify(tmp_path, monkeypatch, workers): + depot = Depot(tmp_path / "s") + log = tmp_path / "log" + log.mkdir() + seen = [] + orch = Orchestrator(depot, registry(Batched(log)), workers=workers, on_batch_done=seen.append, log=lambda _: None) + done = orch._done + + def interrupt(*args): + done(*args) + raise KeyboardInterrupt + + monkeypatch.setattr(orch, "_done", interrupt) + with pytest.raises(KeyboardInterrupt): + orch.run(["toy.src"], facts={"files": toy_files(tmp_path / "in", {"a1": "x", "a2": "y"})}) + assert seen == [] and orch._pool == [] + + +def test_configuration_stop_keeps_the_incomplete_batch(tmp_path): + from nnnotes.source.config import ConfigError + + class Stops(Batched): + def run(self, task, depot): + if task.subject == "a2": + raise ConfigError("setting paths.tool is not set") + return super().run(task, depot) + + log = tmp_path / "log" + log.mkdir() + seen = [] + orch = Orchestrator(Depot(tmp_path / "s"), registry(Stops(log)), on_batch_done=seen.append, log=lambda _: None) + with pytest.raises(ConfigError): + orch.run(["toy.src"], facts={"files": toy_files(tmp_path / "in", {"a1": "x", "a2": "y", "a3": "z"})}) + assert seen == [] + + def test_jobs_group_a_stages_batches(): def task(stage, name, cpu, peak): return Task(stage, 1, name, cost=Cost(cpu, peak)) From bec1dc50253ee81156d28a74abcbf7fb1c8f1a48 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 18 Oct 2026 20:18:30 +0800 Subject: [PATCH 77/91] fix(release): import the packaged deck data module --- .github/scripts/smoke_wheel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/smoke_wheel.py b/.github/scripts/smoke_wheel.py index 84aa7a2..354258c 100644 --- a/.github/scripts/smoke_wheel.py +++ b/.github/scripts/smoke_wheel.py @@ -14,7 +14,8 @@ import nnnotes import ournotes_sim -from nnnotes import deckdata, musicdata +from nnnotes import musicdata +from nnnotes.musicdata import deckdata assert Path(nnnotes.__file__).resolve().is_relative_to(Path(sys.prefix).resolve()) deck = musicdata.Deck(workers=1, aptitude=False) From 6eb4296a2a5169878dcc91829620c9f759e31d25 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 18 Oct 2026 20:18:30 +0800 Subject: [PATCH 78/91] chore(release): prepare nnnotes v0.3.0 --- CHANGELOG.md | 9 ++++++++- release-notes/v0.3.0.txt | 14 ++++++++++++++ src/nnnotes/__init__.py | 2 +- 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 release-notes/v0.3.0.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index ac9277d..f5e04a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,14 @@ All notable changes to this project are listed here. Versions follow [Semantic V over the command line (commands, options, exit statuses and `--json` reports), the configuration keys, the JSON schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor version can change any of them. -## Unreleased +## [0.3.0](https://github.com/MetaSekaiLab/nnnotes/compare/v0.2.1...v0.3.0) - 2026-10-18 + +### Compatibility + +- Configure persistent storage with `[paths] depot` and optional `[depot] remote`. Manage it with `depot plan`, `depot run`, `depot verify` and `depot gc`. +- Exported products use `nnnotes.product/1` manifests and shared object references. Browser playback uses ournotes-player 0.2.x. +- Build named sites with `site build --name NAME -o DIR`. Copy selected source bundles with `bundle pull -o DIR`. +- The `deck` extra uses ournotes-sim 0.0.9. ### Features diff --git a/release-notes/v0.3.0.txt b/release-notes/v0.3.0.txt new file mode 100644 index 0000000..fd56451 --- /dev/null +++ b/release-notes/v0.3.0.txt @@ -0,0 +1,14 @@ +nnnotes 0.3.0 + +Persistent depots store task inputs, content-addressed results, catalog versions and retained runs, with optional S3-compatible remote storage. Commands reuse verified task results and manage source downloads under a configurable budget and command leases. + +Live2D, room, spot, live, story and UI exports use shared object products. Named site builds publish assets, product manifests and indexes in dependency order, preserve unselected regional topics and restore complete layouts from depot records. + +Configuration and command interface: +- Set [paths] depot and, when needed, [depot] remote. +- Use depot plan, depot run, depot verify and depot gc to manage stored work. +- Use site build --name NAME -o DIR for a named site and bundle pull -o DIR for source bundle copies. +- Product playback uses ournotes-player 0.2.x. Story and UI data use the documented nnnotes.product/1 formats. +- The deck extra pins ournotes-sim 0.0.9. + +See the configuration, depot, products and site references for command options and data contracts. diff --git a/src/nnnotes/__init__.py b/src/nnnotes/__init__.py index 7d6ba71..c111a90 100644 --- a/src/nnnotes/__init__.py +++ b/src/nnnotes/__init__.py @@ -19,7 +19,7 @@ import importlib from typing import TYPE_CHECKING -__version__ = "0.2.1" +__version__ = "0.3.0" _LAZY = { "Catalog": "nnnotes.source.catalog", From 19decb95a87e207150e151fc250fb754c244d0e2 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 18 Oct 2026 20:12:23 +0800 Subject: [PATCH 79/91] feat(depot): share catalog versions through remotes --- CHANGELOG.md | 1 + docs/contracts.md | 11 +++- src/nnnotes/cli/assets.py | 5 +- src/nnnotes/cli/catalog.py | 4 +- src/nnnotes/pipeline/catalogdb.py | 63 ++++++++++++++++++-- src/nnnotes/pipeline/depot.py | 7 ++- src/nnnotes/pipeline/remote.py | 6 +- tests/cli/test_assets.py | 28 +++++++++ tests/pipeline/test_catalog_remote.py | 85 +++++++++++++++++++++++++++ 9 files changed, 194 insertions(+), 16 deletions(-) create mode 100644 tests/pipeline/test_catalog_remote.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f5e04a4..42c555e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor v ### Features +- Share catalog versions and labels through depot remotes, with verified catalog files fetched on demand. - Store content, task inputs, results and catalog versions in a persistent depot with optional remote storage, verified collection and bounded source downloads. - Release recoverable bundle downloads after successful conversion batches under the configured budget and command leases. - Assemble Live2D, room, spot, live, story and UI products from shared object tables and explicit manifest references. diff --git a/docs/contracts.md b/docs/contracts.md index d323c51..ed0bf6c 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -295,6 +295,8 @@ ac/ab/.json results inputs//ab/.json input identities by name runs/.json run manifests runs/selections/ the id of the latest run of a selection +catalogs/versions/.json one catalog version and its labels +catalogs/.bin catalog files ``` The depot directory is the remote's cache. A content, result, input identity or run the depot lacks is read from the @@ -302,10 +304,15 @@ remote and kept; a content read from the remote is checked against its content i remote as well: the bytes a result names before the result, a run manifest before the pointer to it. A result that differs from the remote's result of the same key is a conflict, as in the depot itself. The latest run of a selection is the one any machine wrote last, and the results of its tasks are known to be on the remote without asking for each -of them. Downloads, file identities by path (`inputs/paths/`), catalog versions, measured costs, run logs and leases +of them. Downloads, file identities by path (`inputs/paths/`), measured costs, run logs and leases stay in the depot directory. -Every request names one object; only `depot gc --remote` lists the remote. +Catalog discovery lists the version records and merges them into the local catalog index. Catalog bytes are fetched +on demand and verified by size and SHA-256 before use. Each import publishes its catalog files before its version +record. `--stored-catalog` can select a version held by the remote on a fresh depot. Import sequences order the +versions, with version ids breaking ties between independent imports. + +Object reads and writes name individual keys. Catalog discovery and `depot gc --remote` use listings. ## UI contracts diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index d36ca5a..25f7c4d 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -333,12 +333,13 @@ def __setstate__(self, state): def catalog(self, vid: str): """(Catalog, {location id: location}) of a stored catalog version.""" from ..pipeline import catalogdb + from ..pipeline.depot import open_depot from ..source.catalog import Catalog with self._lock: hit = self._catalogs.get(vid) if hit is None: - db = catalogdb.CatalogDB(self.root) + db = catalogdb.CatalogDB(self.root, remote=open_depot(self.cfg, root=self.root).remote) v = next((x for x in db.versions() if x["id"] == vid), None) if v is None: raise FileNotFoundError(f"catalog version {vid[:12]} is not in the depot") @@ -524,7 +525,7 @@ def __init__(self, args, cfg, log=_log, layouts=()): self.apk = cfg.path("paths", "apk") self.fetcher = CatalogFetcher(self.root, cfg) self.depot = open_depot(cfg, fetch=self.fetcher) - self.db = CatalogDB(self.root) + self.db = CatalogDB(self.root, remote=self.depot.remote) try: self.selection = parse_selection(getattr(args, "select", None)) except ValueError as e: diff --git a/src/nnnotes/cli/catalog.py b/src/nnnotes/cli/catalog.py index 67273b6..d49fd60 100644 --- a/src/nnnotes/cli/catalog.py +++ b/src/nnnotes/cli/catalog.py @@ -46,8 +46,10 @@ def cmd_browse(args, cfg): # ---------------------------------------------------------------- catalog versions of the depot def _db(cfg): from ..pipeline.catalogdb import CatalogDB + from ..pipeline.depot import open_depot - return CatalogDB(Session(cfg).depot_root()) + depot = open_depot(cfg) + return CatalogDB(depot.root, remote=depot.remote) def version_line(v: dict) -> str: diff --git a/src/nnnotes/pipeline/catalogdb.py b/src/nnnotes/pipeline/catalogdb.py index fd3655d..01b37b5 100644 --- a/src/nnnotes/pipeline/catalogdb.py +++ b/src/nnnotes/pipeline/catalogdb.py @@ -16,6 +16,10 @@ catalogs (sha256, size, buildResultHash), the CDN's `.hash` text when it was served (an opaque change signal), resource version, APK version name +With a depot remote, each version has its own `catalogs/versions/.json` object. Version discovery merges these +records into the local index; catalog bytes are fetched and verified only when read. Import sequences order the +versions, with version ids breaking ties between independent imports. + `diff` compares two indexes: keys (the primary keys of asset locations: internal id, provider, resource type and dependencies, bundles and raw files named by their stable names), bundles and raw files matched by stable name (the name without its content hash: added, removed, changed with old and new name, size, hash, CRC) and the keys whose @@ -36,6 +40,7 @@ from . import contract from .contract import Cost, Input from .depot import write_file +from .remote import Remote from .stages import Output, Stage CATALOGS = "nnnotes.catalogs/1" @@ -413,8 +418,9 @@ def fetch(cdn: str, language: str, timeout: float = 120, *, version: str = "main class CatalogDB: """The catalog versions of a depot (/catalogs/).""" - def __init__(self, depot_root): + def __init__(self, depot_root, *, remote: Remote | None = None): self.root = Path(depot_root) / "catalogs" + self.remote = remote def path(self, sha: str) -> Path: if not contract.is_sha256(sha): @@ -425,10 +431,32 @@ def _load(self) -> list[dict]: try: doc = contract.loads((self.root / "index.json").read_bytes()) except FileNotFoundError: - return [] + doc = {"schema": CATALOGS, "versions": []} if doc.get("schema") != CATALOGS: raise ValueError(f"{self.root / 'index.json'} is not a {CATALOGS} document") - return doc["versions"] + versions = {v["id"]: v for v in doc["versions"]} + if self.remote is not None: + for key, _, _ in self.remote.list("catalogs/versions/"): + if not key.endswith(".json"): + continue + data = self.remote.get(key) + if data is None: + continue + v = contract.loads(data) + expected = version_id(v["remote"]["sha256"], v["apk"]["sha256"] if v["apk"] else None, v.get("source")) + if v["id"] != expected or key != f"catalogs/versions/{expected}.json": + raise ValueError(f"catalog version identity differs: {key}") + local = versions.get(expected) + if local is not None: + v["labels"] = sorted(set(v["labels"]) | set(local["labels"])) + for name, value in local.items(): + if v.get(name) is None: + v[name] = value + versions[expected] = v + merged = sorted(versions.values(), key=lambda v: (v["seq"], v["id"])) + if self.remote is not None: + self._save(merged) + return merged def _save(self, versions: list[dict]) -> None: write_file(self.root / "index.json", contract.encode({"schema": CATALOGS, "versions": versions})) @@ -444,8 +472,12 @@ def versions(self, region: str | None = None, language: str | None = None) -> li def _put(self, data: bytes) -> dict: sha = contract.sha256(data) p = self.path(sha) - if not (p.is_file() and p.stat().st_size == len(data)): + if not (p.is_file() and p.read_bytes() == data): write_file(p, data) + if self.remote is not None: + key = f"catalogs/{sha}.bin" + if self.remote.size(key) != len(data): + self.remote.put(key, data) return {"sha256": sha, "size": len(data), "buildResultHash": parse_header(data)["buildResultHash"]} def add( @@ -507,6 +539,8 @@ def add( raise ValueError(f"label {label!r} already names catalog version {other['id'][:12]}") if label not in v["labels"]: v["labels"].append(label) + if self.remote is not None: + self.remote.put(f"catalogs/versions/{vid}.json", contract.encode(v)) self._save(versions) return v @@ -532,10 +566,26 @@ def get(self, ref: str, region: str | None = None, language: str | None = None) def catalog_bytes(self, version: dict) -> tuple[bytes, bytes | None]: """(remote catalog, APK catalog or None) of a version.""" - remote = self.path(version["remote"]["sha256"]).read_bytes() - apk = self.path(version["apk"]["sha256"]).read_bytes() if version["apk"] else None + remote = self._bytes(version["remote"]) + apk = self._bytes(version["apk"]) if version["apk"] else None return remote, apk + def _bytes(self, record: dict) -> bytes: + sha, size = record["sha256"], record["size"] + path = self.path(sha) + try: + data = path.read_bytes() + except FileNotFoundError: + data = self.remote.get(f"catalogs/{sha}.bin") if self.remote is not None else None + if data is None: + raise FileNotFoundError(f"catalog content {sha} is not in the depot") from None + if len(data) != size or contract.sha256(data) != sha: + raise ValueError(f"catalog content {sha} differs from its identity") from None + write_file(path, data) + if len(data) != size or contract.sha256(data) != sha: + raise ValueError(f"catalog content {sha} differs from its identity") + return data + def index(self, version: dict) -> dict: return index(*self.catalog_bytes(version)) @@ -545,6 +595,7 @@ def inputs(self, version: dict) -> list[Input]: for role in CATALOG_NAMES: rec = version[role] if rec: + self._bytes(rec) out.append( Input( role, diff --git a/src/nnnotes/pipeline/depot.py b/src/nnnotes/pipeline/depot.py index 0bd5b5b..5d7dfc9 100644 --- a/src/nnnotes/pipeline/depot.py +++ b/src/nnnotes/pipeline/depot.py @@ -725,13 +725,14 @@ def _gc_remote(self, remote: Remote, keep: int | None, dry_run: bool, grace: flo return out -def open_depot(cfg, *, fetch=None) -> Depot: +def open_depot(cfg, *, fetch=None, root=None) -> Depot: """The depot of a configuration: `[paths] depot`, with the remote `[depot] remote` names (its `endpoint`, - `access_key_id`, `secret_access_key` and `signing_region`) when it is set.""" + `access_key_id`, `secret_access_key` and `signing_region`) when it is set. `root` selects an explicit depot + directory when the caller already holds one.""" from ..source.config import ConfigError from ..source.session import Session - root = Session(cfg).depot_root() + root = Session(cfg).depot_root() if root is None else Path(root) url = cfg.get("depot", "remote") if not url: return Depot(root, fetch=fetch) diff --git a/src/nnnotes/pipeline/remote.py b/src/nnnotes/pipeline/remote.py index ffa50c6..9e89c9a 100644 --- a/src/nnnotes/pipeline/remote.py +++ b/src/nnnotes/pipeline/remote.py @@ -7,12 +7,14 @@ sha256 of the name) /runs/.json run manifests /runs/selections/ the id of the latest run of a selection + /catalogs/versions/.json one catalog version and its labels + /catalogs/.bin catalog bytes, verified when fetched A remote is `s3:///` (S3-compatible object storage: path-style requests, checksums only where the request needs them, so stores other than AWS answer them) or `file://` (a shared directory). Every operation but `list` names one object, so a remote behind a gateway that caches listings answers them all; `list` is -used by `depot gc` only. Contents and results are written once: a write is skipped when the object is there with the -same size. The local depot is the remote's cache (pipeline.depot). +used for catalog version discovery and `depot gc`. Contents and results are written once: a write is skipped when +the object is there with the same size. The local depot is the remote's cache (pipeline.depot). """ from __future__ import annotations diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index e3e29f1..1f6175c 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -485,6 +485,34 @@ def test_a_second_depot_with_the_same_remote_hits_without_downloads(tmp_path, ca assert not [p for p in (tmp_path / "b").glob("downloads/**/*") if p.is_file()] # nothing downloaded +def test_a_fresh_depot_exports_a_stored_catalog_from_its_remote(tmp_path, capsys, fake, monkeypatch): + monkeypatch.setenv("NNNOTES_DEPOT_REMOTE", (tmp_path / "remote").as_uri()) + d = setup_data(tmp_path, depots=("a",)) + code, out, err = export(capsys, tmp_path, d, "a", "out-a") + assert code == 1, err + first = json.loads(out) + code, out, err = nn( + capsys, + "export", + "assets", + "--stored-catalog", + "latest", + "--depot", + tmp_path / "b", + "-o", + tmp_path / "out-b", + "--workers", + "0", + "--json", + ) + assert code == 1 and out, err + again = json.loads(out) + assert again["tasks"]["ran"] == 0 and again["run"] == first["run"], err + assert tree(tmp_path / "out-a", [f"{assets.REPORTS}/run.json"]) == tree( + tmp_path / "out-b", [f"{assets.REPORTS}/run.json"] + ) + + def test_export_is_the_same_with_worker_processes(tmp_path, capsys, fake): d = setup_data(tmp_path, depots=("s0", "s2")) export(capsys, tmp_path, d, "s0", "o0") diff --git a/tests/pipeline/test_catalog_remote.py b/tests/pipeline/test_catalog_remote.py new file mode 100644 index 0000000..30f4736 --- /dev/null +++ b/tests/pipeline/test_catalog_remote.py @@ -0,0 +1,85 @@ +"""Catalog metadata and content shared by independent depots over a directory remote.""" + +import pytest + +from nnnotes.pipeline import contract +from nnnotes.pipeline.catalogdb import CatalogDB +from nnnotes.pipeline.remote import open_remote +from tests.pipeline.test_catalogdb import APK, H3, H4, build, remote_entries + + +def databases(tmp_path): + remote = open_remote((tmp_path / "remote").as_uri()) + return remote, CatalogDB(tmp_path / "a", remote=remote), CatalogDB(tmp_path / "b", remote=remote) + + +def test_catalog_versions_are_shared_and_content_is_fetched_on_demand(tmp_path): + remote, first, second = databases(tmp_path) + body, apk = build(remote_entries()), build(APK) + version = first.add(body, apk, label="stable", region="one", language="en") + assert remote.get(f"catalogs/versions/{version['id']}.json") == contract.encode(version) + assert remote.get("catalogs/index.json") is None + assert second.get("stable") == version + assert not second.path(version["remote"]["sha256"]).exists() + assert second.catalog_bytes(version) == (body, apk) + assert second.path(version["apk"]["sha256"]).read_bytes() == apk + assert CatalogDB(tmp_path / "b").get("stable") == version + assert CatalogDB(tmp_path / "b").catalog_bytes(version) == (body, apk) + + +def test_independent_imports_preserve_versions_and_refresh_known_metadata(tmp_path): + _, first, second = databases(tmp_path) + body = build(remote_entries()) + one = first.add(body, label="one", region="r", language="en") + assert second.get("one")["id"] == one["id"] + two = second.add(build(remote_entries(h_a=H3)), label="two", region="r", language="en") + three = first.add(build(remote_entries(h_a=H4)), label="three", region="r", language="en") + assert [v["id"] for v in second.versions()] == [one["id"], two["id"], three["id"]] + first.add(body, label="extra", region="r", language="en", resource_version="1.2.3") + assert second.get("extra")["resourceVersion"] == "1.2.3" + assert second.get("latest")["id"] == three["id"] + with pytest.raises(ValueError, match="already names"): + second.add(build(remote_entries(h_a=H3)), label="one", region="r", language="en") + + +def test_remote_catalog_bytes_are_verified_before_local_storage(tmp_path): + remote, first, second = databases(tmp_path) + body = build(remote_entries()) + version = first.add(body) + sha = version["remote"]["sha256"] + remote.put(f"catalogs/{sha}.bin", b"x" * len(body)) + with pytest.raises(ValueError, match="differs from its identity"): + second.catalog_bytes(version) + assert not second.path(sha).exists() + + +def test_remote_catalog_record_must_match_its_identity(tmp_path): + remote, first, second = databases(tmp_path) + version = first.add(build(remote_entries())) + version["id"] = "0" * 64 + remote.put(f"catalogs/versions/{version['id']}.json", contract.encode(version)) + with pytest.raises(ValueError, match="version identity differs"): + second.versions() + + +def test_catalog_task_inputs_materialize_shared_catalogs(tmp_path): + _, first, second = databases(tmp_path) + body, apk = build(remote_entries()), build(APK) + version = first.add(body, apk) + inputs = second.inputs(second.get(version["id"])) + assert [i.sha256 for i in inputs] == [contract.sha256(body), contract.sha256(apk)] + assert second.catalog_bytes(version) == (body, apk) + + +def test_catalog_locator_recovery_reads_shared_version_metadata(tmp_path): + from nnnotes.cli.assets import CatalogFetcher + from nnnotes.source.config import Config + + remote, first, _ = databases(tmp_path) + body = build(remote_entries()) + version = first.add(body) + cfg = Config({"depot": {"remote": remote.url}}, environ={}) + fetcher = CatalogFetcher(tmp_path / "fresh", cfg) + catalog, locations = fetcher.catalog(version["id"]) + assert catalog.sources()["remote"] == body + assert locations From d97b076fe9d90900a98999fd166a32d3b53042d3 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 18 Oct 2026 20:26:26 +0800 Subject: [PATCH 80/91] fix(depot): retain planned and standalone task results --- CHANGELOG.md | 1 + docs/assets.md | 9 +++++++- docs/contracts.md | 4 ++++ src/nnnotes/cli/assets.py | 29 ++++++++++++++++------- src/nnnotes/pipeline/orchestrate.py | 7 ++++-- src/nnnotes/pipeline/plan.py | 25 ++++++++++++++++---- src/nnnotes/schemas/layout.schema.json | 12 ++++++++++ tests/cli/test_assets.py | 11 ++++++++- tests/pipeline/test_orchestrate.py | 9 ++++++++ tests/pipeline/test_plan.py | 32 ++++++++++++++++++++++++-- 10 files changed, 120 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42c555e..a7276fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor v ### Features - Share catalog versions and labels through depot remotes, with verified catalog files fetched on demand. +- Retain census planning results, emitted task descriptions and standalone task runs as depot collection roots. - Store content, task inputs, results and catalog versions in a persistent depot with optional remote storage, verified collection and bounded source downloads. - Release recoverable bundle downloads after successful conversion batches under the configured budget and command leases. - Assemble Live2D, room, spot, live, story and UI products from shared object tables and explicit manifest references. diff --git a/docs/assets.md b/docs/assets.md index e3fb2ff..f7743a1 100644 --- a/docs/assets.md +++ b/docs/assets.md @@ -91,7 +91,7 @@ gives per stage its counts and its first 20 tasks that are not hits; `--json` ha | `--json` | the plan document (`nnnotes.plan/1`) | | `--check` | exit 1 when anything would run or a layout file would change, else 0 | | `--census` | first fetch the bundles without a census and take it, so the plan knows every bundle's classes | -| `--emit-tasks DIR` | write the tasks that can run now as task descriptions `DIR/.json` | +| `--emit-tasks DIR` | write the tasks that can run now as task descriptions `DIR/.json`; retain their layout and stored input dependencies in the depot | | `--why TASK` | one task: its status, reasons, and its key parts beside those of the previous run | Reasons: `new-input`, `input-changed`, `stage-version`, `atom-changed`, `params-changed`, `context-changed`, @@ -99,6 +99,10 @@ Reasons: `new-input`, `input-changed`, `stage-version`, `atom-changed`, `params- another run); unknown tasks are `pending` on a task, or on `fetch:` for a bundle whose content is not known yet. See [contracts.md](contracts.md#plan-nnnotesplan1). +Census planning writes a run record for its completed tasks. Emitted descriptions have a saved task layout whose +input references keep stored inputs reachable during collection. Interrupted pipeline runs also retain completed +task results in their run records. + ### depot run Runs task descriptions (`nnnotes.task/1`, as `plan --emit-tasks` writes them) into a depot: each is checked (its key @@ -107,6 +111,9 @@ locators (the depot, its downloads, a file, or with `--fetch` the catalog), and `export assets` would store it. Several tasks run one after another in one process; stages that do not read bundles never load UnityPy. `--force` runs tasks whose result is stored again (the new result must be identical). +Each attempted task has a run record, selected by `depot.run` and its task id. The JSON report's `runs` array names +these records. Collection applies `--keep` separately to the recorded versions of each task. + ### catalog versions, import, fetch, diff The catalog versions of a depot: `import` a catalog file (with the APK's catalog from `[paths] apk` or diff --git a/docs/contracts.md b/docs/contracts.md index ed0bf6c..0e5f779 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -214,6 +214,10 @@ bytes nor keys; the file names of `original` and `cas` both follow from the reco `original` manifest alone. Documents derived for `original` from records and paths (the views with the files of their objects) are the exception: `original` has the derived documents, `cas` the artifacts they are made from. +Task-description layouts use the ordinary layout contract. Each task file entry may include `inputs`, an array of +content records (`sha256`, `size`) for the task's input dependencies. Collection follows those references as well as +the task-description file itself. + ## Product manifest: nnnotes.product/1 A product layout writes, for each product of `pipeline.assemble`, a manifest diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index 25f7c4d..fdb9915 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -1533,7 +1533,7 @@ def cmd_plan(args, cfg): if args.census: ws.run_index() facts = ws.facts(["catalog.index", "unity.census"], fetch=True) - Orchestrator( + run = Orchestrator( ws.depot, ws.scheduled, workers=ws.workers(), @@ -1543,10 +1543,16 @@ def cmd_plan(args, cfg): initializer=_worker_init, initargs=(cfg,), log=ws.log, - ).run(["catalog.index", "unity.census"], facts=facts) + ).run( + ["catalog.index", "unity.census"], + facts=facts, + context=ws.context(), + selection=["depot.plan.census", *ws.selection], + ) + run.write() p = ws.plan(pipeline, params, since=args.since, out=args.out, layouts=layouts) if args.emit_tasks: - files = p.emit(args.emit_tasks) + files = p.emit(args.emit_tasks, depot=ws.depot, context=ws.context(), selection=ws.selection) ws.log(f"{len(files)} task descriptions in {args.emit_tasks}") if args.why: common.report(args, why(ws, p, args.why, args.since), why_lines) @@ -1617,7 +1623,7 @@ def _task_files(items) -> list[Path]: def cmd_run_stage(args, cfg): from ..pipeline.depot import open_depot - from ..pipeline.orchestrate import run_one + from ..pipeline.orchestrate import Run, run_one problems, tasks = [], [] for f in _task_files(args.tasks): @@ -1644,7 +1650,7 @@ def cmd_run_stage(args, cfg): root = Session(cfg).depot_root() depot = open_depot(cfg, fetch=CatalogFetcher(root, cfg) if args.fetch else None) use(cfg) - out, bad = [], 0 + out, bad, runs = [], 0, [] for t in tasks: r = run_one(t, depot, inst.stages, args.force) if r["ok"]: @@ -1653,12 +1659,19 @@ def cmd_run_stage(args, cfg): else: out.append({"id": t.id, "key": t.key, "status": "failed", "code": r["code"], "message": r["message"]}) bad += 1 - if r["code"] == "config": - break + run = Run(depot, dict(t.context), ["depot.run", t.id], {t.stage: dict(t.params)}, {t.stage: t.version}) + run.tasks[t.id] = {k: out[-1][k] for k in ("id", "key", "status")} + runs.append(run.write()["id"]) + if not r["ok"] and r["code"] == "config": + break n = Counter(o["status"] for o in out) common.report( args, - {"tasks": out, "summary": {"tasks": len(tasks), "hit": n["hit"], "ran": n["ran"], "failed": n["failed"]}}, + { + "tasks": out, + "runs": runs, + "summary": {"tasks": len(tasks), "hit": n["hit"], "ran": n["ran"], "failed": n["failed"]}, + }, run_lines, ) if bad: diff --git a/src/nnnotes/pipeline/orchestrate.py b/src/nnnotes/pipeline/orchestrate.py index dd48d8c..eb973f4 100644 --- a/src/nnnotes/pipeline/orchestrate.py +++ b/src/nnnotes/pipeline/orchestrate.py @@ -590,8 +590,10 @@ def run( try: for name in pipeline: self._stage(self.stages[name], norm[name], env, force, run) - except ConfigError as e: - e.run = run # pyright: ignore[reportAttributeAccessIssue] (what ran before the stop: its results stay valid) + except BaseException as e: + if isinstance(e, ConfigError): + e.run = run # pyright: ignore[reportAttributeAccessIssue] + run.write() raise finally: self.close() @@ -813,6 +815,7 @@ def _record(self, run: Run, j: Job, sched: Scheduler, w) -> None: ) def _done(self, run: Run, t: Task, reply: dict, w) -> None: + run.tasks[t.id] = {"id": t.id, "key": t.key, "status": reply["status"] if reply["ok"] else "failed"} e = { "event": "end", "task": t.id, diff --git a/src/nnnotes/pipeline/plan.py b/src/nnnotes/pipeline/plan.py index e5184ea..7e8121e 100644 --- a/src/nnnotes/pipeline/plan.py +++ b/src/nnnotes/pipeline/plan.py @@ -28,7 +28,7 @@ from . import contract from .depot import write_file -from .orchestrate import WORKER_BYTES, jobs +from .orchestrate import WORKER_BYTES, Run, jobs from .stages import Env, Pending, Stage, describe @@ -76,15 +76,30 @@ def check(self) -> int: return 1 return int(any(v["write"] or v["remove"] for v in self.doc["summary"]["layouts"].values())) - def emit(self, directory) -> list[Path]: - """Write the runnable tasks as task descriptions /.json (one per distinct key).""" + def emit(self, directory, *, depot=None, context=None, selection=()) -> list[Path]: + """Write runnable tasks as /.json (one per key). With a depot, retain a task layout and + the stored inputs of these descriptions as a run root.""" d = Path(directory) d.mkdir(parents=True, exist_ok=True) - out = {} + out, entries = {}, [] for t in self.tasks: if t.key not in out: out[t.key] = d / f"{t.key}.json" - write_file(out[t.key], contract.encode(t.to_json())) + body = contract.encode(t.to_json()) + write_file(out[t.key], body) + if depot is not None: + entries.append( + { + "path": out[t.key].name, + "id": t.id, + "sha256": depot.put(body), + "size": len(body), + "inputs": [{"sha256": i.sha256, "size": i.size} for i in t.inputs], + } + ) + if depot is not None and entries: + layout = depot.put(contract.encode(contract.layout_doc("tasks", {}, entries))) + Run(depot, dict(context or {}), ["depot.plan.tasks", *selection], {}, {}).write({"tasks": layout}) return sorted(out.values()) def text(self, limit: int | None = 20) -> str: diff --git a/src/nnnotes/schemas/layout.schema.json b/src/nnnotes/schemas/layout.schema.json index 1d296f0..ec97a97 100644 --- a/src/nnnotes/schemas/layout.schema.json +++ b/src/nnnotes/schemas/layout.schema.json @@ -44,6 +44,18 @@ }, "id": { "type": "string" + }, + "inputs": { + "description": "Input content dependencies retained by an emitted task description.", + "type": "array", + "items": { + "type": "object", + "required": ["sha256", "size"], + "properties": { + "sha256": {"$ref": "urn:nnnotes:schema:common#/$defs/sha256"}, + "size": {"type": "integer", "minimum": 0} + } + } } } } diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index 1f6175c..ab15a5e 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -552,12 +552,16 @@ def test_plan_emit_tasks_and_run_stage_give_the_exported_depot(tmp_path, capsys, doc = json.loads(out) if not any(n["status"] == "run" for n in doc["nodes"]): break + Depot(tmp_path / "q").gc() code, out, err = nn(capsys, *flags(d), "depot", "run", tasks, "--depot", tmp_path / "q") assert code in (0, 1), err + Depot(tmp_path / "q").gc() rounds += 1 assert rounds == 3 # index + census; scripts + addresses; export skip = ["inputs", "runs", "costs", "catalogs", "downloads"] - assert task_tree(tmp_path / "orch", skip) == task_tree(tmp_path / "q", skip) + expected, actual = task_tree(tmp_path / "orch", skip), task_tree(tmp_path / "q", skip) + # The task runner also retains emitted descriptions and their input references. + assert {p: actual.get(p) for p in expected} == expected def test_plan_explains_and_waits_for_bundles_not_fetched(tmp_path, capsys, fake): @@ -1224,6 +1228,11 @@ def test_plan_census_takes_the_censuses_first(tmp_path, capsys, fake): census = [n for n in nodes if n["stage"] == "unity.census"] assert census and all(n["status"] == "hit" for n in census) assert {n["status"] for n in nodes if n["stage"] == "link.scripts"} == {"run"} + depot = Depot(tmp_path / "s") + run = depot.latest_run(["depot.plan.census", "all"]) + assert run and {t["id"] for t in run["tasks"]} >= {n["id"] for n in census} + depot.gc(keep=1) + assert all(depot.has_result(n["key"]) for n in census) class NamedByExport(Named): diff --git a/tests/pipeline/test_orchestrate.py b/tests/pipeline/test_orchestrate.py index 4b3533e..1b61955 100644 --- a/tests/pipeline/test_orchestrate.py +++ b/tests/pipeline/test_orchestrate.py @@ -304,6 +304,10 @@ def interrupt(*args): with pytest.raises(KeyboardInterrupt): orch.run(["toy.src"], facts={"files": toy_files(tmp_path / "in", {"a1": "x", "a2": "y"})}) assert seen == [] and orch._pool == [] + saved = depot.latest_run([]) + assert saved and any(t["status"] == "ran" for t in saved["tasks"]) + depot.gc(keep=1) + assert all(depot.has_result(t["key"]) for t in saved["tasks"] if t["key"]) def test_configuration_stop_keeps_the_incomplete_batch(tmp_path): @@ -323,6 +327,11 @@ def run(self, task, depot): orch.run(["toy.src"], facts={"files": toy_files(tmp_path / "in", {"a1": "x", "a2": "y", "a3": "z"})}) assert seen == [] + saved = orch.depot.latest_run([]) + assert saved and any(t["status"] == "ran" for t in saved["tasks"]) + orch.depot.gc(keep=1) + assert all(orch.depot.has_result(t["key"]) for t in saved["tasks"] if t["status"] == "ran") + def test_jobs_group_a_stages_batches(): def task(stage, name, cpu, peak): diff --git a/tests/pipeline/test_plan.py b/tests/pipeline/test_plan.py index 76cb105..f98693d 100644 --- a/tests/pipeline/test_plan.py +++ b/tests/pipeline/test_plan.py @@ -2,11 +2,14 @@ from pathlib import Path +import pytest + from nnnotes.pipeline import contract -from nnnotes.pipeline.contract import Task +from nnnotes.pipeline.contract import Input, Task from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.orchestrate import Orchestrator -from nnnotes.pipeline.plan import plan, why +from nnnotes.pipeline.plan import Plan, plan, why +from nnnotes.pipeline.remote import open_remote from nnnotes.pipeline.stages import registry from tests.pipeline.test_stages import PIPELINE, Count, Src, Total, toy_files, toy_stages @@ -149,6 +152,31 @@ def test_emitted_tasks_are_runnable_descriptions(tmp_path): assert f.read_bytes() == contract.encode(t.to_json()) +@pytest.mark.parametrize("shared", [False, True]) +def test_emitted_tasks_and_stored_inputs_survive_collection(tmp_path, shared): + remote = open_remote((tmp_path / "remote").as_uri()) if shared else None + depot = Depot(tmp_path / "depot", remote=remote) + sha = depot.put(b"input") + unused = depot.put(b"unreferenced") + task = Task("toy.src", 1, "a", inputs=(Input("source", sha, 5, locators=({"kind": "depot"},)),)) + files = Plan({}, [task]).emit(tmp_path / "tasks", depot=depot, selection=["example"]) + run = depot.latest_run(["depot.plan.tasks", "example"]) + from tests.pipeline.test_contract import validators + + schemas = validators() + assert not list(schemas[contract.RUN].iter_errors(run)) + layout = depot.document(run["layouts"]["tasks"]) + assert not list(schemas[contract.LAYOUT].iter_errors(layout)) + depot.gc(keep=1, remote=shared, grace=0) + assert not depot.has(unused) + reader = Depot(tmp_path / "fresh", remote=remote) if shared else depot + saved = reader.document(reader.latest_run(["depot.plan.tasks", "example"])["layouts"]["tasks"]) + body = reader.read(saved["entries"][0]["sha256"]) + assert body == files[0].read_bytes() + restored = Task.from_json(contract.loads(body)) + assert reader.input_bytes(restored.inputs[0]) == b"input" + + def test_peak_estimate_oversize_and_layout_counts(tmp_path): depot, facts = setup(tmp_path, {"a": "x" * 100, "b": "y" * 10, "c": "z" * 5}) # peaks 1000, 100, 50 p = plan( From 8e17b9124d8a13e1e13dffe7a3e35e1520db87a2 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 18 Oct 2026 20:27:58 +0800 Subject: [PATCH 81/91] fix(depot): publish stored inputs with emitted tasks --- src/nnnotes/pipeline/plan.py | 5 +++++ tests/pipeline/test_plan.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/nnnotes/pipeline/plan.py b/src/nnnotes/pipeline/plan.py index 7e8121e..82869ca 100644 --- a/src/nnnotes/pipeline/plan.py +++ b/src/nnnotes/pipeline/plan.py @@ -88,6 +88,11 @@ def emit(self, directory, *, depot=None, context=None, selection=()) -> list[Pat body = contract.encode(t.to_json()) write_file(out[t.key], body) if depot is not None: + if depot.remote is not None: + for inp in t.inputs: + path = depot.path(inp.sha256) + if path.is_file() and depot.put_file(path) != (inp.sha256, inp.size): + raise ValueError(f"task {t.id}: stored input {inp.role} differs from its identity") entries.append( { "path": out[t.key].name, diff --git a/tests/pipeline/test_plan.py b/tests/pipeline/test_plan.py index f98693d..6ab42c8 100644 --- a/tests/pipeline/test_plan.py +++ b/tests/pipeline/test_plan.py @@ -155,9 +155,10 @@ def test_emitted_tasks_are_runnable_descriptions(tmp_path): @pytest.mark.parametrize("shared", [False, True]) def test_emitted_tasks_and_stored_inputs_survive_collection(tmp_path, shared): remote = open_remote((tmp_path / "remote").as_uri()) if shared else None - depot = Depot(tmp_path / "depot", remote=remote) + depot = Depot(tmp_path / "depot") sha = depot.put(b"input") unused = depot.put(b"unreferenced") + depot.remote = remote task = Task("toy.src", 1, "a", inputs=(Input("source", sha, 5, locators=({"kind": "depot"},)),)) files = Plan({}, [task]).emit(tmp_path / "tasks", depot=depot, selection=["example"]) run = depot.latest_run(["depot.plan.tasks", "example"]) From e559c736c90452548d4bf8341f2a7722439ee4f3 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 18 Oct 2026 20:43:12 +0800 Subject: [PATCH 82/91] chore(release): align the dependency lock and depot notes --- release-notes/v0.3.0.txt | 2 ++ uv.lock | 14 +++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/release-notes/v0.3.0.txt b/release-notes/v0.3.0.txt index fd56451..c6a55fd 100644 --- a/release-notes/v0.3.0.txt +++ b/release-notes/v0.3.0.txt @@ -2,6 +2,8 @@ nnnotes 0.3.0 Persistent depots store task inputs, content-addressed results, catalog versions and retained runs, with optional S3-compatible remote storage. Commands reuse verified task results and manage source downloads under a configurable budget and command leases. +Catalog versions and labels are shared through the remote. Census planning, emitted task descriptions and standalone task results retain the content they need through collection, including completed results from interrupted pipelines. + Live2D, room, spot, live, story and UI exports use shared object products. Named site builds publish assets, product manifests and indexes in dependency order, preserve unselected regional topics and restore complete layouts from depot records. Configuration and command interface: diff --git a/uv.lock b/uv.lock index b7f6467..327e072 100644 --- a/uv.lock +++ b/uv.lock @@ -1127,7 +1127,7 @@ requires-dist = [ { name = "mkdocs-material", marker = "extra == 'docs'", specifier = ">=9.5" }, { name = "mkdocstrings", extras = ["python"], marker = "extra == 'docs'", specifier = ">=0.29" }, { name = "numpy", specifier = ">=2.0" }, - { name = "ournotes-sim", marker = "extra == 'deck'", specifier = "==0.0.8" }, + { name = "ournotes-sim", marker = "extra == 'deck'", specifier = "==0.0.9" }, { name = "pillow", specifier = ">=10.0" }, { name = "pycryptodome", specifier = ">=3.20" }, { name = "pyright", marker = "extra == 'test'", specifier = ">=1.1.414" }, @@ -1307,14 +1307,14 @@ wheels = [ [[package]] name = "ournotes-sim" -version = "0.0.8" +version = "0.0.9" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/42/d5/ec0ab100c4a3bde81283645df66ee7a816fe40ac3a8ee79fdd2dcb9861b8/ournotes_sim-0.0.8-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:6cb6f32f6c00e343e9de37022ffd1d90f6954f1b69019ba25157b50336dd6f03", size = 1134062, upload-time = "2026-10-13T16:16:34.889Z" }, - { url = "https://files.pythonhosted.org/packages/2a/af/eaa943b68e6435887fddd8327f5c391e66e5aeffd492c7137b188ed6bb81/ournotes_sim-0.0.8-cp311-abi3-macosx_11_0_x86_64.whl", hash = "sha256:21532a8b66300b5b68f3f63f479a68f44c68b178f8b6b4bea53e9320c8c864da", size = 1218956, upload-time = "2026-10-13T16:16:36.526Z" }, - { url = "https://files.pythonhosted.org/packages/c2/37/9559de8d086a56af01fb5789535154586f4279a297cc37437c9fa9d4feeb/ournotes_sim-0.0.8-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f07de96c82330012847f3cb730e1aba1ee5e2de3645a03bb4c342397adf071d9", size = 1202150, upload-time = "2026-10-13T16:16:38.009Z" }, - { url = "https://files.pythonhosted.org/packages/e9/6f/23ddd3f76644b629546b0a9a386e40da0048662500b509c29e068988e126/ournotes_sim-0.0.8-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0e59b5bde6cadc318dc9daa7430d10c1520d2d113fea0729c97bbab6fe42d91b", size = 1279737, upload-time = "2026-10-13T16:16:39.429Z" }, - { url = "https://files.pythonhosted.org/packages/97/74/7ff602125b2f99aeefe48573beb642dfa81fa7d15eee24f64422e8674bcd/ournotes_sim-0.0.8-cp311-abi3-win_amd64.whl", hash = "sha256:06948cac00294f2686fc53bd35f96d59186f9d579b6ca8bb1052976ebcaf2697", size = 1171400, upload-time = "2026-10-13T16:16:41.082Z" }, + { url = "https://files.pythonhosted.org/packages/f9/6f/cc7aa9280d3148ddcdf82c17635ee37c7aec445f36cdcc02e2678bfed0be/ournotes_sim-0.0.9-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:f38711f42603ca5dbac1080f0d86653cd2b252568f82a9adca69844349c19f01", size = 1188929, upload-time = "2026-10-14T16:04:17.831Z" }, + { url = "https://files.pythonhosted.org/packages/a4/d8/3dd1fce51ae87dbd5645d4ff17bcdd2744280c8386c3ffeefbb480003bb7/ournotes_sim-0.0.9-cp311-abi3-macosx_11_0_x86_64.whl", hash = "sha256:fc023ae1a396ea6ce395612632379338d4ababac7dbcaf999a18768e79a68ecb", size = 1278009, upload-time = "2026-10-14T16:04:19.963Z" }, + { url = "https://files.pythonhosted.org/packages/c4/91/bf82f3696f1619c0daabe38f8bf0b63fe3f90093343da1f067ce500d2504/ournotes_sim-0.0.9-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c16900b18417e0a23cfd3a31445a604044e75f8ffca3bae406436804140ae1e9", size = 1259438, upload-time = "2026-10-14T16:04:21.609Z" }, + { url = "https://files.pythonhosted.org/packages/24/b8/26a13c40e707fb36b3b673a909a7316b47d663400acf9b292e3e2cb83a2a/ournotes_sim-0.0.9-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f59a1ffa87299a290c2d1b39fde511697794155d5a50bb09b96957bf8093bc2e", size = 1340684, upload-time = "2026-10-14T16:04:23.624Z" }, + { url = "https://files.pythonhosted.org/packages/be/7c/365c01caa5390bae1a1c39036bf2abd9f9d02008e905b78af041db410349/ournotes_sim-0.0.9-cp311-abi3-win_amd64.whl", hash = "sha256:afa4211a89825e766526395fc3d00a9bad332c7d312bbb66eedaff15ae68556a", size = 1234199, upload-time = "2026-10-14T16:04:25.524Z" }, ] [[package]] From 472997d6519ead2f1c6e6e70a700f5fa93fb7af3 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 18 Oct 2026 21:04:01 +0800 Subject: [PATCH 83/91] feat(schema): ship shared product document contracts --- CHANGELOG.md | 1 + docs/contracts.md | 29 + docs/schemas.md | 13 + scripts/check_product_schemas.py | 76 ++ src/nnnotes/schemas/crilips.schema.json | 57 + src/nnnotes/schemas/episode.schema.json | 305 +++++ src/nnnotes/schemas/live-music.schema.json | 152 +++ .../schemas/live-noteeffect.schema.json | 84 ++ src/nnnotes/schemas/live-notes.schema.json | 229 ++++ src/nnnotes/schemas/live-noteskin.schema.json | 84 ++ src/nnnotes/schemas/live-scene.schema.json | 282 +++++ src/nnnotes/schemas/live-sounds.schema.json | 279 +++++ src/nnnotes/schemas/live.schema.json | 89 ++ src/nnnotes/schemas/model-json.schema.json | 141 +++ src/nnnotes/schemas/prefab.schema.json | 174 +++ src/nnnotes/schemas/product.schema.json | 41 +- src/nnnotes/schemas/room.schema.json | 81 ++ src/nnnotes/schemas/spot.schema.json | 59 + src/nnnotes/schemas/story-asset.schema.json | 76 ++ src/nnnotes/schemas/story-fonts.schema.json | 1037 ++++++++++++++++- src/nnnotes/schemas/story-host.schema.json | 196 +++- src/nnnotes/schemas/story-scene.schema.json | 146 +++ .../schemas/story-simple-ui.schema.json | 251 +++- src/nnnotes/schemas/story-sound.schema.json | 56 + src/nnnotes/schemas/story-ui.schema.json | 283 +++++ src/nnnotes/schemas/story-video.schema.json | 85 ++ src/nnnotes/schemas/story.schema.json | 176 +++ src/nnnotes/schemas/ui-library.schema.json | 2 +- src/nnnotes/schemas/ui-pack.schema.json | 2 +- tests/content/test_live.py | 13 +- tests/content/test_live2d.py | 2 + tests/content/test_livenotes.py | 10 + tests/content/test_livescene.py | 8 +- tests/content/test_room.py | 3 + tests/content/test_spot.py | 3 + tests/content/test_storyasset.py | 3 + tests/content/test_storyui.py | 2 + tests/content/test_storyvideo.py | 3 + tests/helpers/schemas.py | 9 + tests/helpers/storyui.py | 21 +- tests/pipeline/test_objexport.py | 3 + 41 files changed, 4491 insertions(+), 75 deletions(-) create mode 100644 scripts/check_product_schemas.py create mode 100644 src/nnnotes/schemas/crilips.schema.json create mode 100644 src/nnnotes/schemas/episode.schema.json create mode 100644 src/nnnotes/schemas/live-music.schema.json create mode 100644 src/nnnotes/schemas/live-noteeffect.schema.json create mode 100644 src/nnnotes/schemas/live-notes.schema.json create mode 100644 src/nnnotes/schemas/live-noteskin.schema.json create mode 100644 src/nnnotes/schemas/live-scene.schema.json create mode 100644 src/nnnotes/schemas/live-sounds.schema.json create mode 100644 src/nnnotes/schemas/live.schema.json create mode 100644 src/nnnotes/schemas/model-json.schema.json create mode 100644 src/nnnotes/schemas/prefab.schema.json create mode 100644 src/nnnotes/schemas/room.schema.json create mode 100644 src/nnnotes/schemas/spot.schema.json create mode 100644 src/nnnotes/schemas/story-asset.schema.json create mode 100644 src/nnnotes/schemas/story-scene.schema.json create mode 100644 src/nnnotes/schemas/story-sound.schema.json create mode 100644 src/nnnotes/schemas/story-ui.schema.json create mode 100644 src/nnnotes/schemas/story-video.schema.json create mode 100644 src/nnnotes/schemas/story.schema.json create mode 100644 tests/helpers/schemas.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a7276fa..7b1885e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor v ### Features +- Ship schemas for shared product root documents and prefab objects, with numeric font and transform contracts. - Share catalog versions and labels through depot remotes, with verified catalog files fetched on demand. - Retain census planning results, emitted task descriptions and standalone task runs as depot collection roots. - Store content, task inputs, results and catalog versions in a persistent depot with optional remote storage, verified collection and bounded source downloads. diff --git a/docs/contracts.md b/docs/contracts.md index 0e5f779..9c47051 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -328,3 +328,32 @@ Object reads and writes name individual keys. Catalog discovery and `depot gc -- ## Site catalog [`site.schema.json`](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/site.schema.json) describes `nnnotes.site/1`. A named site stores its complete region-topic catalog in `site.json`. Each topic names an immutable product manifest and the metadata of its listing entry. Partial selections retain other topics; complete selections replace the requested region and product kind. The successful run points at the complete site layout so depot collection retains every published dependency. + +## Product document schemas + +The fixed product documents have the following schemas, also available through `nnnotes.schemas.load`. + +| Schema | Document | +|---|---| +| [episode](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/episode.schema.json) | Story episode commands and resource rows | +| [model-json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/model-json.schema.json) | Live2D model objects, resources and animation bindings | +| [prefab](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/prefab.schema.json) | Prefab hierarchy nodes, transforms and components | +| [live](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/live.schema.json) | Live entry and shared product parts | +| [live-scene](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/live-scene.schema.json) | Live scene, lane resources and bands | +| [live-notes](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/live-notes.schema.json) | Note prefabs and shared animation bindings | +| [live-noteskin](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/live-noteskin.schema.json) | Note skin sprites and animation bindings | +| [live-noteeffect](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/live-noteeffect.schema.json) | Note effect prefabs and animation bindings | +| [live-music](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/live-music.schema.json) | Music layers and cue metadata | +| [live-sounds](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/live-sounds.schema.json) | Sound groups, categories and routing | +| [story](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/story.schema.json) | Story entry and shared product parts | +| [story-scene](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/story-scene.schema.json) | Story scene objects, settings and graphics | +| [story-ui](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/story-ui.schema.json) | Localized story UI objects and text settings | +| [story-asset](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/story-asset.schema.json) | Story media prefab objects and animation bindings | + +### Scene and media document schemas + +- [room](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/room.schema.json) +- [spot](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/spot.schema.json) +- [story-video](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/story-video.schema.json) +- [story-sound](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/story-sound.schema.json) +- [crilips](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/crilips.schema.json) diff --git a/docs/schemas.md b/docs/schemas.md index 8ed6374..367721d 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -45,3 +45,16 @@ schemas.load("plan") # the schema as a dict The documents of the asset export are described in [contracts](contracts.md), the views in [views](views.md). The schema files are in [src/nnnotes/schemas](https://github.com/MetaSekaiLab/nnnotes/tree/main/src/nnnotes/schemas). + +## Product documents + +The product root and prefab schemas are listed in [Product document schemas](contracts.md#product-document-schemas). Their fields match the shared object interfaces consumed by ournotes-player. + +Compare the shared constraints with an ournotes-player checkout or extracted package: + +```sh +python scripts/check_product_schemas.py /path/to/ournotes-player +``` + +The check covers product, live, model, story, prefab, room, spot, audio, video and UI documents. Schema identifiers, +titles and descriptions may differ by package; validation constraints must agree. diff --git a/scripts/check_product_schemas.py b/scripts/check_product_schemas.py new file mode 100644 index 0000000..0558f6b --- /dev/null +++ b/scripts/check_product_schemas.py @@ -0,0 +1,76 @@ +"""Compare the shared product contracts with an ournotes-player checkout or extracted package.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +NAMES = ( + "product", + "episode", + "model-json", + "prefab", + "live", + "live-scene", + "live-notes", + "live-noteskin", + "live-noteeffect", + "live-music", + "live-sounds", + "story", + "story-scene", + "story-ui", + "story-asset", + "story-fonts", + "story-host", + "story-simple-ui", + "ui-pack", + "ui-library", + "room", + "spot", + "story-video", + "story-sound", + "crilips", +) + + +def contract(value): + if isinstance(value, dict): + out = {} + for key, item in value.items(): + if key in {"$id", "title", "description"}: + continue + if key in {"properties", "patternProperties", "$defs", "dependentSchemas"}: + out[key] = {name: contract(schema) for name, schema in item.items()} + elif key in {"const", "enum"}: + out[key] = item + else: + out[key] = contract(item) + return out + if isinstance(value, list): + return [contract(item) for item in value] + return value + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("player", type=Path, help="ournotes-player root containing schema/") + args = parser.parse_args() + source = Path(__file__).resolve().parents[1] / "src" / "nnnotes" / "schemas" + errors = [] + for name in NAMES: + paths = [source / f"{name}.schema.json", args.player / "schema" / f"{name}.schema.json"] + try: + left, right = [contract(json.loads(path.read_text(encoding="utf-8"))) for path in paths] + if left != right: + errors.append(f"{name}: schema constraints differ") + except (OSError, ValueError) as error: + errors.append(f"{name}: {error}") + if errors: + parser.exit(1, "\n".join(errors) + "\n") + print(f"{len(NAMES)} shared product schemas match") + + +if __name__ == "__main__": + main() diff --git a/src/nnnotes/schemas/crilips.schema.json b/src/nnnotes/schemas/crilips.schema.json new file mode 100644 index 0000000..36ae156 --- /dev/null +++ b/src/nnnotes/schemas/crilips.schema.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "dtype", + "order", + "total_floats", + "blocks", + "constants", + "frontend" + ], + "properties": { + "dtype": { + "const": "float32" + }, + "order": { + "const": "LE" + }, + "total_floats": { + "type": "integer", + "minimum": 0 + }, + "blocks": { + "type": "array", + "items": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "name", + "offset", + "count" + ], + "properties": { + "name": { + "type": "string" + }, + "offset": { + "type": "integer", + "minimum": 0 + }, + "count": { + "type": "integer", + "minimum": 0 + } + } + } + }, + "constants": { + "type": "object" + }, + "frontend": { + "type": "object" + } + }, + "$id": "urn:nnnotes:schema:crilips", + "title": "crilips document" +} diff --git a/src/nnnotes/schemas/episode.schema.json b/src/nnnotes/schemas/episode.schema.json new file mode 100644 index 0000000..805248d --- /dev/null +++ b/src/nnnotes/schemas/episode.schema.json @@ -0,0 +1,305 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "advId", + "asset", + "commandCount", + "commands", + "text", + "sounds", + "cuesheets", + "videos", + "resources", + "master", + "title" + ], + "properties": { + "advId": { + "type": "integer", + "minimum": 0 + }, + "asset": { + "type": "string" + }, + "commandCount": { + "type": "integer", + "minimum": 0 + }, + "commands": { + "type": "array", + "items": { + "$ref": "#/$defs/command" + } + }, + "text": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/texts" + } + }, + "sounds": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "_cueName": { + "type": "string" + }, + "_soundCueSheetID": { + "type": "integer" + } + } + } + }, + "cuesheets": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "_cueSheetName" + ], + "properties": { + "_cueSheetName": { + "type": "string", + "minLength": 1 + } + } + } + }, + "videos": { + "type": "object", + "additionalProperties": { + "type": "object" + } + }, + "resources": { + "type": "array", + "items": { + "type": "object", + "required": [ + "kind", + "address" + ], + "properties": { + "kind": { + "type": "string", + "minLength": 1 + }, + "address": { + "type": "string", + "minLength": 1 + }, + "present": { + "type": "boolean" + } + } + } + }, + "master": { + "type": "object", + "required": [ + "_id", + "_advEpisodeAsset", + "_titleTextId", + "_playbackMode" + ], + "properties": { + "_id": { + "type": "integer", + "minimum": 0 + }, + "_sheetName": { + "type": "string" + }, + "_advEpisodeAsset": { + "type": "string" + }, + "_titleTextId": { + "type": "string" + }, + "_rubyTitleTextId": { + "type": "string" + }, + "_playbackMode": { + "enum": [ + 0, + 1 + ] + } + } + }, + "title": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/texts" + } + ] + } + }, + "$defs": { + "texts": { + "type": "object", + "properties": { + "japanese": { + "type": [ + "string", + "null" + ] + }, + "english": { + "type": [ + "string", + "null" + ] + }, + "traditionalChinese": { + "type": [ + "string", + "null" + ] + }, + "simplifiedChinese": { + "type": [ + "string", + "null" + ] + }, + "korean": { + "type": [ + "string", + "null" + ] + } + } + }, + "command": { + "type": "object", + "required": [ + "i", + "cmd", + "raw" + ], + "properties": { + "i": { + "type": "integer" + }, + "cmd": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9]*$" + }, + "raw": { + "type": "integer", + "minimum": 0 + }, + "TargetName": { + "type": "string" + }, + "TargetAssetName": { + "type": "string" + }, + "TargetAssetIndex": { + "type": "integer" + }, + "PositionType": { + "type": "integer" + }, + "CameraDistance": { + "type": "number" + }, + "Duration": { + "type": "number" + }, + "DelaySeconds": { + "type": "number" + }, + "IsNoWait": { + "type": [ + "boolean", + "integer" + ] + }, + "IgnoreData": { + "type": [ + "boolean", + "integer" + ] + }, + "IgnoreLipSync": { + "type": [ + "boolean", + "integer" + ] + }, + "Parameter1": { + "type": "string" + }, + "Parameter2": { + "type": "string" + }, + "Parameter3": { + "type": "string" + }, + "Parameter4": { + "type": "string" + }, + "TargetTextIDs": { + "type": "array", + "items": { + "type": "string" + } + }, + "TargetStatus": { + "type": "integer" + }, + "AdvTextID": { + "type": "string" + }, + "lines": { + "$ref": "#/$defs/texts" + }, + "VoiceIDs": { + "type": "array", + "items": { + "type": "integer" + } + }, + "voiceCues": { + "type": "array", + "items": { + "type": [ + "string", + "null" + ] + } + }, + "MotionName": { + "type": "string" + }, + "ExpressionName": { + "type": "string" + }, + "BgmID": { + "type": "integer" + }, + "SeID": { + "type": "integer" + }, + "Key": { + "type": "string" + }, + "VideoID": { + "type": "integer" + }, + "TargetChatID": { + "type": "integer" + } + } + } + }, + "$id": "urn:nnnotes:schema:episode", + "title": "episode document" +} diff --git a/src/nnnotes/schemas/live-music.schema.json b/src/nnnotes/schemas/live-music.schema.json new file mode 100644 index 0000000..56b3f87 --- /dev/null +++ b/src/nnnotes/schemas/live-music.schema.json @@ -0,0 +1,152 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "format", + "musicId", + "music", + "sounds" + ], + "properties": { + "format": { + "const": 1 + }, + "musicId": { + "type": "integer" + }, + "music": { + "type": "object", + "required": [ + "soundId" + ], + "properties": { + "soundId": { + "$ref": "#/$defs/soundId" + } + } + }, + "sounds": { + "type": "object", + "propertyNames": { + "pattern": "^[1-9][0-9]*$" + }, + "additionalProperties": { + "$ref": "#/$defs/sound" + } + } + }, + "$defs": { + "soundId": { + "type": "integer", + "minimum": 1 + }, + "sound": { + "type": "object", + "required": [ + "sheet", + "cue", + "category", + "categories", + "volume", + "row", + "layers" + ], + "properties": { + "sheet": { + "type": "string" + }, + "cue": { + "type": "string" + }, + "category": { + "type": "integer" + }, + "categories": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "volume": { + "type": "number", + "minimum": 0 + }, + "row": { + "type": "object", + "required": [ + "_isRandomPitch" + ], + "properties": { + "_isRandomPitch": { + "const": false + } + } + }, + "layers": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/layer" + } + } + } + }, + "layer": { + "type": "object", + "required": [ + "file", + "sampleRate", + "samples", + "loopFlag", + "loopStart", + "loopEnd", + "volume" + ], + "properties": { + "file": { + "type": "string", + "pattern": "^[A-Za-z0-9_][^\\\\]*\\.(flac|m4a)$" + }, + "sampleRate": { + "const": 48000 + }, + "channels": { + "type": "integer", + "minimum": 1 + }, + "samples": { + "type": "integer", + "minimum": 1 + }, + "loopFlag": { + "type": "integer" + }, + "loopStart": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "loopEnd": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "volume": { + "type": "number", + "minimum": 0 + }, + "encoderDelay": { + "type": "integer", + "minimum": 0 + } + } + } + }, + "$id": "urn:nnnotes:schema:live-music", + "title": "live-music document" +} diff --git a/src/nnnotes/schemas/live-noteeffect.schema.json b/src/nnnotes/schemas/live-noteeffect.schema.json new file mode 100644 index 0000000..c2731e7 --- /dev/null +++ b/src/nnnotes/schemas/live-noteeffect.schema.json @@ -0,0 +1,84 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "format", + "name", + "settings", + "bindings", + "shaders" + ], + "properties": { + "format": { + "const": 1 + }, + "name": { + "type": "string" + }, + "settings": { + "$ref": "#/$defs/objectId" + }, + "bindings": { + "$ref": "#/$defs/bindings" + }, + "shaders": { + "$ref": "#/$defs/shaders" + } + }, + "$defs": { + "objectId": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "objectIdText": { + "pattern": "^.+:-?[0-9]+$" + }, + "bindings": { + "type": "object", + "propertyNames": { + "$ref": "#/$defs/objectIdText" + }, + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "object", + "propertyNames": { + "$ref": "#/$defs/objectIdText" + }, + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "required": [ + "path", + "attribute" + ], + "properties": { + "path": { + "type": [ + "string", + "null" + ] + }, + "attribute": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } + }, + "shaders": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/objectId" + } + } + }, + "$id": "urn:nnnotes:schema:live-noteeffect", + "title": "live-noteeffect document" +} diff --git a/src/nnnotes/schemas/live-notes.schema.json b/src/nnnotes/schemas/live-notes.schema.json new file mode 100644 index 0000000..905c60f --- /dev/null +++ b/src/nnnotes/schemas/live-notes.schema.json @@ -0,0 +1,229 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "format", + "prefabs", + "laneEffect", + "settings", + "bindings", + "shaders" + ], + "properties": { + "format": { + "const": 1 + }, + "prefabs": { + "type": "object", + "required": [ + "tap_note_view", + "flick_note_view", + "flick_left_note_view", + "flick_right_note_view", + "slide_note_view", + "slide_end_note_view", + "connection_note_view", + "none_note_view", + "slide_line_view", + "pair_note_line" + ], + "additionalProperties": { + "$ref": "#/$defs/objectId" + } + }, + "laneEffect": { + "$ref": "#/$defs/objectId" + }, + "settings": { + "type": "object", + "required": [ + "laneCount", + "laneSize", + "laneTopRange", + "laneBottomRange", + "laneTopPosition", + "judgementScreenBottomPosition", + "optionDefaults", + "optionRanges", + "liveSettings" + ], + "properties": { + "laneCount": { + "type": "integer", + "minimum": 1 + }, + "laneSize": { + "type": "array", + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "minItems": 2, + "maxItems": 2 + }, + "laneTopRange": { + "type": "number" + }, + "laneBottomRange": { + "type": "number" + }, + "laneTopPosition": { + "type": "number" + }, + "judgementScreenBottomPosition": { + "type": "number" + }, + "tiltCenterLane": { + "type": "number" + }, + "noteDisplayTimeMs": { + "type": "number" + }, + "optionDefaults": { + "type": "object", + "required": [ + "NoteSpeed", + "SlideOpacity", + "GuideOpacity" + ], + "additionalProperties": { + "type": "string" + } + }, + "optionRanges": { + "type": "object", + "required": [ + "SlideOpacity", + "GuideOpacity" + ], + "additionalProperties": { + "$ref": "#/$defs/range" + } + }, + "liveSettings": { + "type": "object", + "required": [ + "note_speed_min", + "note_speed_max", + "note_speed_view_min", + "note_speed_view_max" + ], + "properties": { + "note_speed_min": { + "type": "string" + }, + "note_speed_max": { + "type": "string" + }, + "note_speed_view_min": { + "type": "string" + }, + "note_speed_view_max": { + "type": "string" + } + } + }, + "skins": { + "type": "object", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "type": "string" + } + }, + "effects": { + "type": "object", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "type": "string" + } + }, + "effect": { + "type": "string" + } + } + }, + "bindings": { + "$ref": "#/$defs/bindings" + }, + "shaders": { + "$ref": "#/$defs/shaders" + } + }, + "$defs": { + "objectId": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "objectIdText": { + "pattern": "^.+:-?[0-9]+$" + }, + "bindings": { + "type": "object", + "propertyNames": { + "$ref": "#/$defs/objectIdText" + }, + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "object", + "propertyNames": { + "$ref": "#/$defs/objectIdText" + }, + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "required": [ + "path", + "attribute" + ], + "properties": { + "path": { + "type": [ + "string", + "null" + ] + }, + "attribute": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } + }, + "shaders": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/objectId" + } + }, + "range": { + "type": "array", + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "minItems": 2, + "maxItems": 2 + } + }, + "$id": "urn:nnnotes:schema:live-notes", + "title": "live-notes document" +} diff --git a/src/nnnotes/schemas/live-noteskin.schema.json b/src/nnnotes/schemas/live-noteskin.schema.json new file mode 100644 index 0000000..4077f29 --- /dev/null +++ b/src/nnnotes/schemas/live-noteskin.schema.json @@ -0,0 +1,84 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "format", + "name", + "skin", + "bindings", + "shaders" + ], + "properties": { + "format": { + "const": 1 + }, + "name": { + "type": "string" + }, + "skin": { + "$ref": "#/$defs/objectId" + }, + "bindings": { + "$ref": "#/$defs/bindings" + }, + "shaders": { + "$ref": "#/$defs/shaders" + } + }, + "$defs": { + "objectId": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "objectIdText": { + "pattern": "^.+:-?[0-9]+$" + }, + "bindings": { + "type": "object", + "propertyNames": { + "$ref": "#/$defs/objectIdText" + }, + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "object", + "propertyNames": { + "$ref": "#/$defs/objectIdText" + }, + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "required": [ + "path", + "attribute" + ], + "properties": { + "path": { + "type": [ + "string", + "null" + ] + }, + "attribute": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } + }, + "shaders": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/objectId" + } + } + }, + "$id": "urn:nnnotes:schema:live-noteskin", + "title": "live-noteskin document" +} diff --git a/src/nnnotes/schemas/live-scene.schema.json b/src/nnnotes/schemas/live-scene.schema.json new file mode 100644 index 0000000..9666abf --- /dev/null +++ b/src/nnnotes/schemas/live-scene.schema.json @@ -0,0 +1,282 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "format", + "scene", + "graphics", + "laneSkins", + "prefabs", + "spriteAssets", + "bands", + "settings", + "bindings", + "shaders" + ], + "properties": { + "format": { + "const": 1 + }, + "scene": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/objectId" + } + }, + "renderSettings": { + "$ref": "#/$defs/optionalObjectId" + }, + "graphics": { + "type": "object", + "required": [ + "graphics" + ], + "properties": { + "graphics": { + "$ref": "#/$defs/objectId" + }, + "quality": { + "$ref": "#/$defs/objectId" + }, + "player": { + "$ref": "#/$defs/objectId" + } + } + }, + "laneSkins": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "lane_base", + "lane_tap_area", + "out_side_line" + ], + "properties": { + "lane_base": { + "$ref": "#/$defs/objectId" + }, + "lane_tap_area": { + "$ref": "#/$defs/objectId" + }, + "out_side_line": { + "$ref": "#/$defs/objectId" + } + } + } + }, + "prefabs": { + "type": "object", + "required": [ + "laneLine", + "judgeEffectView" + ], + "properties": { + "laneLine": { + "$ref": "#/$defs/objectId" + }, + "judgeEffectView": { + "$ref": "#/$defs/objectId" + }, + "barLineView": { + "$ref": "#/$defs/optionalObjectId" + } + } + }, + "spriteAssets": { + "type": "object", + "required": [ + "combo", + "judgement" + ], + "properties": { + "combo": { + "$ref": "#/$defs/objectId" + }, + "judgement": { + "$ref": "#/$defs/objectId" + } + } + }, + "bands": { + "type": "object", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "type": "object", + "required": [ + "timeline", + "background", + "tracks" + ], + "properties": { + "timeline": { + "$ref": "#/$defs/objectId" + }, + "background": { + "$ref": "#/$defs/objectId" + }, + "tracks": { + "type": "object", + "propertyNames": { + "$ref": "#/$defs/objectIdText" + }, + "additionalProperties": { + "type": "object", + "required": [ + "prefab", + "animator" + ], + "properties": { + "name": { + "type": [ + "string", + "null" + ] + }, + "prefab": { + "$ref": "#/$defs/objectId" + }, + "animator": { + "type": "string" + } + } + } + } + } + } + }, + "settings": { + "type": "object", + "required": [ + "optionDefaults", + "qualitySettings", + "laneSkin" + ], + "properties": { + "optionDefaults": { + "type": "object", + "required": [ + "JudgePositionDisplay", + "NoteStartPosition" + ], + "additionalProperties": { + "type": "string" + } + }, + "optionRanges": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/range" + } + }, + "qualitySettings": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": [ + "_quality", + "_effectRenderingScale" + ], + "properties": { + "_quality": { + "type": "integer" + }, + "_effectRenderingScale": { + "type": "number" + } + } + } + }, + "laneSkin": { + "type": "string" + } + } + }, + "bindings": { + "$ref": "#/$defs/bindings" + }, + "shaders": { + "$ref": "#/$defs/shaders" + } + }, + "$defs": { + "objectId": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "objectIdText": { + "pattern": "^.+:-?[0-9]+$" + }, + "bindings": { + "type": "object", + "propertyNames": { + "$ref": "#/$defs/objectIdText" + }, + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "object", + "propertyNames": { + "$ref": "#/$defs/objectIdText" + }, + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "required": [ + "path", + "attribute" + ], + "properties": { + "path": { + "type": [ + "string", + "null" + ] + }, + "attribute": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } + }, + "shaders": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/objectId" + } + }, + "range": { + "type": "array", + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "minItems": 2, + "maxItems": 2 + }, + "optionalObjectId": { + "type": [ + "string", + "null" + ], + "pattern": "^.+:-?[0-9]+$" + } + }, + "$id": "urn:nnnotes:schema:live-scene", + "title": "live-scene document" +} diff --git a/src/nnnotes/schemas/live-sounds.schema.json b/src/nnnotes/schemas/live-sounds.schema.json new file mode 100644 index 0000000..a7420ef --- /dev/null +++ b/src/nnnotes/schemas/live-sounds.schema.json @@ -0,0 +1,279 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "format", + "sounds", + "categories", + "react", + "noteSe", + "liveSe", + "timeline" + ], + "properties": { + "format": { + "const": 1 + }, + "sounds": { + "type": "object", + "propertyNames": { + "pattern": "^[1-9][0-9]*$" + }, + "additionalProperties": { + "$ref": "#/$defs/sound" + } + }, + "categories": { + "type": "object", + "additionalProperties": { + "type": "number", + "minimum": 0 + } + }, + "react": { + "type": "array", + "items": { + "$ref": "#/$defs/react" + } + }, + "noteSe": { + "type": "object", + "required": [ + "types", + "volumes", + "mutes" + ], + "properties": { + "types": { + "$ref": "#/$defs/byType", + "additionalProperties": { + "$ref": "#/$defs/soundId" + } + }, + "volumes": { + "$ref": "#/$defs/byType", + "additionalProperties": { + "type": "number", + "minimum": 0 + } + }, + "mutes": { + "$ref": "#/$defs/byType", + "additionalProperties": { + "type": "boolean" + } + }, + "patternId": { + "type": "integer", + "minimum": 1 + }, + "groups": { + "type": "object", + "propertyNames": { + "pattern": "^[1-9][0-9]*$" + }, + "additionalProperties": { + "$ref": "#/$defs/byType", + "additionalProperties": { + "$ref": "#/$defs/soundId" + } + } + } + } + }, + "liveSe": { + "$ref": "#/$defs/byType", + "additionalProperties": { + "$ref": "#/$defs/soundId" + } + }, + "voice": { + "type": "object", + "properties": { + "startVoiceSoundId": { + "$ref": "#/$defs/optionalSoundId" + }, + "clearVoiceSoundId": { + "$ref": "#/$defs/optionalSoundId" + }, + "fullComboVoiceSoundId": { + "$ref": "#/$defs/optionalSoundId" + }, + "allPerfectVoiceSoundId": { + "$ref": "#/$defs/optionalSoundId" + } + } + }, + "timeline": { + "type": "object", + "required": [ + "startCheerDelaySec" + ], + "properties": { + "startCheerDelaySec": { + "type": "number", + "minimum": 0 + } + } + } + }, + "$defs": { + "soundId": { + "type": "integer", + "minimum": 1 + }, + "optionalSoundId": { + "type": "integer", + "minimum": -1 + }, + "byType": { + "type": "object", + "propertyNames": { + "pattern": "^[0-9]+$" + } + }, + "sound": { + "type": "object", + "required": [ + "sheet", + "cue", + "category", + "categories", + "volume", + "row", + "layers" + ], + "properties": { + "sheet": { + "type": "string" + }, + "cue": { + "type": "string" + }, + "category": { + "type": "integer" + }, + "categories": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "volume": { + "type": "number", + "minimum": 0 + }, + "row": { + "type": "object", + "required": [ + "_isRandomPitch" + ], + "properties": { + "_isRandomPitch": { + "const": false + } + } + }, + "layers": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/layer" + } + } + } + }, + "layer": { + "type": "object", + "required": [ + "file", + "sampleRate", + "samples", + "loopFlag", + "loopStart", + "loopEnd", + "volume" + ], + "properties": { + "file": { + "type": "string", + "pattern": "^[A-Za-z0-9_][^\\\\]*\\.(flac|m4a)$" + }, + "sampleRate": { + "const": 48000 + }, + "channels": { + "type": "integer", + "minimum": 1 + }, + "samples": { + "type": "integer", + "minimum": 1 + }, + "loopFlag": { + "type": "integer" + }, + "loopStart": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "loopEnd": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "volume": { + "type": "number", + "minimum": 0 + }, + "encoderDelay": { + "type": "integer", + "minimum": 0 + } + } + }, + "react": { + "type": "object", + "required": [ + "src", + "dest", + "level", + "decrementMs", + "incrementMs", + "holdMs" + ], + "properties": { + "src": { + "type": "string" + }, + "dest": { + "type": "string" + }, + "level": { + "type": "number", + "minimum": 0 + }, + "decrementMs": { + "type": "number", + "minimum": 0 + }, + "incrementMs": { + "type": "number", + "minimum": 0 + }, + "holdMs": { + "type": "number", + "minimum": 0 + } + } + } + }, + "$id": "urn:nnnotes:schema:live-sounds", + "title": "live-sounds document" +} diff --git a/src/nnnotes/schemas/live.schema.json b/src/nnnotes/schemas/live.schema.json new file mode 100644 index 0000000..e638703 --- /dev/null +++ b/src/nnnotes/schemas/live.schema.json @@ -0,0 +1,89 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "format", + "musicId", + "difficulty", + "band", + "jacket", + "notes", + "parts" + ], + "properties": { + "format": { + "const": 1 + }, + "musicId": { + "type": "integer" + }, + "difficulty": { + "type": "string" + }, + "band": { + "type": "object", + "required": [ + "band" + ], + "properties": { + "band": { + "type": "integer" + } + } + }, + "jacket": { + "$ref": "#/$defs/objectId" + }, + "notes": { + "$ref": "#/$defs/file" + }, + "notesMirror": { + "$ref": "#/$defs/file" + }, + "parts": { + "type": "object", + "required": [ + "scene", + "notes", + "noteskin", + "noteeffect", + "sounds", + "music" + ], + "additionalProperties": { + "type": "string", + "pattern": "^[^/\\\\]+/[^/\\\\]+\\.json$" + }, + "properties": { + "ui": { + "type": "object", + "propertyNames": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] + }, + "additionalProperties": { + "type": "string", + "pattern": "^[^/\\\\]+/[^/\\\\]+\\.json$" + } + } + } + } + }, + "$defs": { + "objectId": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "file": { + "type": "string", + "pattern": "^[A-Za-z0-9_][^\\\\]*\\.json$" + } + }, + "$id": "urn:nnnotes:schema:live", + "title": "live document" +} diff --git a/src/nnnotes/schemas/model-json.schema.json b/src/nnnotes/schemas/model-json.schema.json new file mode 100644 index 0000000..84ef221 --- /dev/null +++ b/src/nnnotes/schemas/model-json.schema.json @@ -0,0 +1,141 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "format", + "name", + "key", + "prefab", + "moc", + "canvas", + "textures", + "shaders", + "resources", + "motionSync", + "bindings" + ], + "additionalProperties": false, + "properties": { + "format": { + "const": 3 + }, + "name": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" + }, + "key": { + "type": "string" + }, + "prefab": { + "$ref": "#/$defs/objectId" + }, + "moc": { + "$ref": "#/$defs/objectId" + }, + "canvas": { + "type": "object", + "required": [ + "pixelsPerUnit", + "originX", + "originY", + "width", + "height", + "mocVersion" + ], + "additionalProperties": false, + "properties": { + "pixelsPerUnit": { + "type": "number" + }, + "originX": { + "type": "number" + }, + "originY": { + "type": "number" + }, + "width": { + "type": "number" + }, + "height": { + "type": "number" + }, + "mocVersion": { + "type": "integer" + } + } + }, + "textures": { + "type": "array", + "items": { + "$ref": "#/$defs/objectId" + } + }, + "shaders": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/objectId" + } + }, + "resources": { + "type": "object", + "required": [ + "cubismMask", + "cubismMaskCulling" + ], + "additionalProperties": false, + "properties": { + "cubismMask": { + "$ref": "#/$defs/objectId" + }, + "cubismMaskCulling": { + "$ref": "#/$defs/objectId" + } + } + }, + "motionSync": { + "type": "boolean" + }, + "bindings": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "required": [ + "path", + "attribute" + ], + "additionalProperties": false, + "properties": { + "path": { + "type": [ + "string", + "null" + ] + }, + "attribute": { + "type": [ + "string", + "null" + ] + }, + "pathCandidates": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + }, + "$defs": { + "objectId": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "$id": "urn:nnnotes:schema:model-json", + "title": "model-json document" +} diff --git a/src/nnnotes/schemas/prefab.schema.json b/src/nnnotes/schemas/prefab.schema.json new file mode 100644 index 0000000..a3af7c7 --- /dev/null +++ b/src/nnnotes/schemas/prefab.schema.json @@ -0,0 +1,174 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "nodes" + ], + "properties": { + "name": { + "type": "string" + }, + "scene": { + "type": "string" + }, + "nodes": { + "type": "array", + "items": { + "$ref": "#/$defs/node" + } + } + }, + "$defs": { + "vec2": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + }, + "vec3": { + "type": "object", + "required": [ + "x", + "y", + "z" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + }, + "z": { + "type": "number" + } + } + }, + "quat": { + "type": "object", + "required": [ + "x", + "y", + "z", + "w" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + }, + "z": { + "type": "number" + }, + "w": { + "type": "number" + } + } + }, + "component": { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "pathId": { + "type": "string", + "pattern": "^-?[0-9]+$" + }, + "script": { + "type": [ + "string", + "null" + ] + }, + "fields": { + "type": "object" + } + } + }, + "node": { + "type": "object", + "required": [ + "path", + "localPosition", + "localRotation", + "localScale", + "components" + ], + "properties": { + "path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "pathId": { + "type": "string", + "pattern": "^-?[0-9]+$" + }, + "transformPathId": { + "type": "string", + "pattern": "^-?[0-9]+$" + }, + "active": { + "type": "boolean" + }, + "layer": { + "type": "integer" + }, + "localPosition": { + "$ref": "#/$defs/vec3" + }, + "localRotation": { + "$ref": "#/$defs/quat" + }, + "localScale": { + "$ref": "#/$defs/vec3" + }, + "components": { + "type": "array", + "items": { + "$ref": "#/$defs/component" + } + }, + "rect": { + "type": "object", + "properties": { + "m_AnchorMin": { + "$ref": "#/$defs/vec2" + }, + "m_AnchorMax": { + "$ref": "#/$defs/vec2" + }, + "m_AnchoredPosition": { + "$ref": "#/$defs/vec2" + }, + "m_SizeDelta": { + "$ref": "#/$defs/vec2" + }, + "m_Pivot": { + "$ref": "#/$defs/vec2" + } + } + } + } + } + }, + "$id": "urn:nnnotes:schema:prefab", + "title": "prefab document" +} diff --git a/src/nnnotes/schemas/product.schema.json b/src/nnnotes/schemas/product.schema.json index 21c4678..2c510f9 100644 --- a/src/nnnotes/schemas/product.schema.json +++ b/src/nnnotes/schemas/product.schema.json @@ -1,8 +1,5 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:nnnotes:schema:product", - "title": "nnnotes.product/1", - "description": "The manifest of a product in a product layout: its files and the files of the objects it uses, each a content-addressed asset whose path is relative to the directory above the manifest's.", "type": "object", "required": [ "schema", @@ -11,29 +8,44 @@ "files", "objects" ], + "additionalProperties": false, "properties": { "schema": { "const": "nnnotes.product/1" }, "product": { - "$ref": "urn:nnnotes:schema:common#/$defs/stageName" + "type": "string", + "pattern": "^[a-z][a-z0-9]*(\\.[a-z][a-z0-9-]*)*$" }, "subject": { - "type": "string" + "type": "string", + "minLength": 1 }, "files": { "type": "object", + "propertyNames": { + "pattern": "^[A-Za-z0-9_][^\\\\]*$" + }, "additionalProperties": { "$ref": "#/$defs/file" } }, "objects": { "type": "object", + "propertyNames": { + "pattern": "^.+:-?[0-9]+$" + }, "additionalProperties": { "type": "object", + "required": [ + "class", + "files" + ], + "additionalProperties": false, "properties": { "class": { - "type": "string" + "type": "string", + "minLength": 1 }, "name": { "type": "string" @@ -44,11 +56,7 @@ "$ref": "#/$defs/file" } } - }, - "required": [ - "files" - ], - "additionalProperties": false + } } }, "root": { @@ -66,7 +74,6 @@ }, "options": { "type": "object", - "description": "Optional: Live option name -> the values the files support (options that select files).", "properties": { "LiveQuality": { "type": "array", @@ -93,8 +100,7 @@ } }, "audio": { - "type": "boolean", - "description": "Whether audio streams are included." + "type": "boolean" } }, "$defs": { @@ -116,17 +122,14 @@ "pattern": "^assets/[0-9a-f]{64}\\.[a-z0-9]+$" }, "encodedAsset": { - "description": "An asset stored gzip- or brotli-encoded: the SHA-256 of its decoded bytes, the file's extension, then .gz or .br.", "type": "string", "pattern": "^assets/[0-9a-f]{64}\\.[a-z0-9]+\\.(gz|br)$" }, "size": { - "description": "The byte count of the file (decoded).", "type": "integer", "minimum": 0 }, "stored": { - "description": "The byte count of an encoded asset as stored (less than size).", "type": "integer", "minimum": 0 }, @@ -222,5 +225,7 @@ } } } - } + }, + "$id": "urn:nnnotes:schema:product", + "title": "nnnotes.product/1" } diff --git a/src/nnnotes/schemas/room.schema.json b/src/nnnotes/schemas/room.schema.json new file mode 100644 index 0000000..3f4dac2 --- /dev/null +++ b/src/nnnotes/schemas/room.schema.json @@ -0,0 +1,81 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "backgroundKey", + "glb", + "meshCount", + "inactiveMeshes", + "nodes", + "materialIds", + "materials", + "textures", + "samplers" + ], + "properties": { + "backgroundKey": { + "type": "string" + }, + "glb": { + "type": "string" + }, + "meshCount": { + "type": "integer", + "minimum": 0 + }, + "inactiveMeshes": { + "type": "integer", + "minimum": 0 + }, + "nullMaterialSubmeshes": { + "type": "array", + "items": { + "type": "object" + } + }, + "nullMeshFilters": { + "type": "array", + "items": { + "type": "string" + } + }, + "emptyMeshes": { + "type": "array", + "items": { + "type": "object" + } + }, + "nodes": { + "type": "array", + "items": { + "type": "object" + } + }, + "materialIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "materials": { + "type": "array", + "items": { + "type": "object" + } + }, + "textures": { + "type": "array", + "items": { + "type": "string" + } + }, + "samplers": { + "type": "array", + "items": { + "type": "object" + } + } + }, + "$id": "urn:nnnotes:schema:room", + "title": "room document" +} diff --git a/src/nnnotes/schemas/spot.schema.json b/src/nnnotes/schemas/spot.schema.json new file mode 100644 index 0000000..eb658d5 --- /dev/null +++ b/src/nnnotes/schemas/spot.schema.json @@ -0,0 +1,59 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "spotId", + "master", + "name", + "tapTalk", + "situationSettings", + "controller", + "characters", + "spineCharacters", + "skeletons", + "matrixConvention" + ], + "properties": { + "spotId": { + "type": "integer" + }, + "master": { + "type": "object" + }, + "name": { + "type": "object" + }, + "tapTalk": { + "type": "object" + }, + "situationSettings": { + "type": "object" + }, + "controller": { + "type": "object" + }, + "characters": { + "type": "array", + "items": { + "type": "object" + } + }, + "spineCharacters": { + "type": "array", + "items": { + "type": "object" + } + }, + "skeletons": { + "type": "array", + "items": { + "type": "object" + } + }, + "matrixConvention": { + "type": "string" + } + }, + "$id": "urn:nnnotes:schema:spot", + "title": "spot document" +} diff --git a/src/nnnotes/schemas/story-asset.schema.json b/src/nnnotes/schemas/story-asset.schema.json new file mode 100644 index 0000000..699775e --- /dev/null +++ b/src/nnnotes/schemas/story-asset.schema.json @@ -0,0 +1,76 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "format", + "key", + "objects", + "bindings", + "shaders" + ], + "properties": { + "format": { + "const": 1 + }, + "key": { + "type": "string" + }, + "objects": { + "type": "array", + "items": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "bindings": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "required": [ + "path", + "attribute" + ], + "properties": { + "path": { + "type": [ + "string", + "null" + ] + }, + "attribute": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "propertyNames": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, + "propertyNames": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "shaders": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, + "$id": "urn:nnnotes:schema:story-asset", + "title": "story-asset document" +} diff --git a/src/nnnotes/schemas/story-fonts.schema.json b/src/nnnotes/schemas/story-fonts.schema.json index 6deecc2..dd81327 100644 --- a/src/nnnotes/schemas/story-fonts.schema.json +++ b/src/nnnotes/schemas/story-fonts.schema.json @@ -1,26 +1,1025 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:nnnotes:schema:story-fonts", - "title": "Episode font records and glyph pages", "type": "object", - "required": ["format", "language", "source", "fonts", "textures", "materials", "materialKeywords", "texts", "sprites", "coverage", "lineBreaking", "shaders"], + "required": [ + "format", + "language", + "source", + "fonts", + "textures", + "materials", + "materialKeywords", + "texts", + "coverage", + "shaders", + "sprites", + "lineBreaking" + ], "properties": { - "format": {"const": "nnnotes.story-fonts/1"}, - "language": {"enum": ["ja", "en", "zh-Hant", "zh-Hans", "ko"]}, - "source": {"enum": ["open", "game"]}, - "fonts": {"type": "object", "additionalProperties": {"type": "object", "required": ["characters", "glyphs", "material", "fallbacks"]}}, - "textures": {"type": "object", "additionalProperties": {"type": "object", "required": ["texture", "width", "height"], "properties": {"texture": {"type": "string"}, "width": {"type": "integer", "minimum": 1}, "height": {"type": "integer", "minimum": 1}}}}, - "materials": {"type": "object", "additionalProperties": {"type": "object"}}, - "materialKeywords": {"type": "object", "additionalProperties": {"type": "array", "items": {"type": "string"}}}, - "texts": {"type": "object", "additionalProperties": {"type": "object", "required": ["style", "localized", "font"], "properties": {"style": {"type": "object"}, "localized": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/font"}]}, "font": {"$ref": "#/$defs/font"}}}}, - "sprites": {"type": "object", "additionalProperties": {"type": "object", "required": ["spriteAsset", "m_tintAllSprites"]}}, - "coverage": {"type": "object", "required": ["characters", "missing"], "properties": {"characters": {"type": "integer", "minimum": 0}, "missing": {"type": "array", "items": {"type": "string"}}}}, - "lineBreaking": {"type": "object", "required": ["leading", "following", "useModernHangulLineBreakingRules"], "properties": {"leading": {"type": "string"}, "following": {"type": "string"}, "useModernHangulLineBreakingRules": {"type": "boolean"}}}, - "shaders": {"type": "object", "additionalProperties": {"type": "string"}}, - "spriteAssets": {"type": "object", "additionalProperties": {"type": "object"}}, - "emojiSpriteAsset": {"type": "string"} + "format": { + "const": "nnnotes.story-fonts/1" + }, + "language": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] + }, + "source": { + "enum": [ + "open", + "game" + ] + }, + "fonts": { + "type": "object", + "propertyNames": { + "minLength": 1 + }, + "additionalProperties": { + "$ref": "#/$defs/font" + } + }, + "textures": { + "type": "object", + "propertyNames": { + "minLength": 1 + }, + "additionalProperties": { + "$ref": "#/$defs/texture" + } + }, + "materials": { + "type": "object", + "propertyNames": { + "minLength": 1 + }, + "additionalProperties": { + "$ref": "#/$defs/material" + } + }, + "materialKeywords": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "texts": { + "type": "object", + "propertyNames": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "additionalProperties": { + "$ref": "#/$defs/text" + } + }, + "spriteAssets": { + "type": "object", + "propertyNames": { + "minLength": 1 + }, + "additionalProperties": { + "$ref": "#/$defs/spriteAsset" + } + }, + "emojiSpriteAsset": { + "type": "string", + "minLength": 1 + }, + "coverage": { + "type": "object", + "required": [ + "characters", + "missing" + ], + "properties": { + "characters": { + "type": "integer", + "minimum": 0 + }, + "missing": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "missingGlyph": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "sprites": { + "type": "object", + "required": [ + "characters", + "missing" + ], + "properties": { + "characters": { + "type": "integer", + "minimum": 0 + }, + "missing": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "underline": { + "type": "object", + "required": [ + "font", + "character", + "inAsset" + ], + "properties": { + "font": { + "type": "string", + "minLength": 1 + }, + "character": { + "const": "_" + }, + "inAsset": { + "type": "boolean" + } + } + } + } + }, + "lineBreaking": { + "type": "object", + "required": [ + "leading", + "following", + "useModernHangulLineBreakingRules" + ], + "properties": { + "leading": { + "type": "string" + }, + "following": { + "type": "string" + }, + "useModernHangulLineBreakingRules": { + "type": "boolean" + } + } + }, + "shaders": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "sprites": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "spriteAsset" + ], + "properties": { + "spriteAsset": { + "type": "string", + "minLength": 1 + } + } + }, + "propertyNames": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } }, "$defs": { - "font": {"type": "object", "required": ["fontAsset", "material", "lineSpacing"], "properties": {"fontAsset": {"type": "string"}, "material": {"type": "string"}, "lineSpacing": {"type": "number"}}} - } + "int": { + "type": "integer" + }, + "num": { + "type": "number" + }, + "faceInfo": { + "type": "object", + "required": [ + "m_PointSize", + "m_Scale", + "m_UnitsPerEM", + "m_LineHeight", + "m_AscentLine", + "m_CapLine", + "m_MeanLine", + "m_Baseline", + "m_DescentLine", + "m_SuperscriptOffset", + "m_SuperscriptSize", + "m_SubscriptOffset", + "m_SubscriptSize", + "m_UnderlineOffset", + "m_UnderlineThickness", + "m_StrikethroughOffset", + "m_StrikethroughThickness", + "m_TabWidth" + ], + "properties": { + "m_FaceIndex": { + "type": "integer", + "minimum": 0 + }, + "m_FamilyName": { + "type": "string" + }, + "m_StyleName": { + "type": "string" + }, + "m_PointSize": { + "type": "number", + "exclusiveMinimum": 0 + }, + "m_Scale": { + "type": "number", + "exclusiveMinimum": 0 + }, + "m_UnitsPerEM": { + "type": "integer", + "minimum": 1 + }, + "m_LineHeight": { + "$ref": "#/$defs/num" + }, + "m_AscentLine": { + "$ref": "#/$defs/num" + }, + "m_CapLine": { + "$ref": "#/$defs/num" + }, + "m_MeanLine": { + "$ref": "#/$defs/num" + }, + "m_Baseline": { + "$ref": "#/$defs/num" + }, + "m_DescentLine": { + "$ref": "#/$defs/num" + }, + "m_SuperscriptOffset": { + "$ref": "#/$defs/num" + }, + "m_SuperscriptSize": { + "$ref": "#/$defs/num" + }, + "m_SubscriptOffset": { + "$ref": "#/$defs/num" + }, + "m_SubscriptSize": { + "$ref": "#/$defs/num" + }, + "m_UnderlineOffset": { + "$ref": "#/$defs/num" + }, + "m_UnderlineThickness": { + "$ref": "#/$defs/num" + }, + "m_StrikethroughOffset": { + "$ref": "#/$defs/num" + }, + "m_StrikethroughThickness": { + "$ref": "#/$defs/num" + }, + "m_TabWidth": { + "$ref": "#/$defs/num" + } + } + }, + "metrics": { + "type": "object", + "required": [ + "m_Width", + "m_Height", + "m_HorizontalBearingX", + "m_HorizontalBearingY", + "m_HorizontalAdvance" + ], + "properties": { + "m_Width": { + "type": "number", + "minimum": 0 + }, + "m_Height": { + "type": "number", + "minimum": 0 + }, + "m_HorizontalBearingX": { + "$ref": "#/$defs/num" + }, + "m_HorizontalBearingY": { + "$ref": "#/$defs/num" + }, + "m_HorizontalAdvance": { + "$ref": "#/$defs/num" + } + } + }, + "rect": { + "type": "object", + "required": [ + "m_X", + "m_Y", + "m_Width", + "m_Height" + ], + "properties": { + "m_X": { + "$ref": "#/$defs/int" + }, + "m_Y": { + "$ref": "#/$defs/int" + }, + "m_Width": { + "type": "integer", + "minimum": 0 + }, + "m_Height": { + "type": "integer", + "minimum": 0 + } + } + }, + "glyph": { + "type": "object", + "required": [ + "metrics", + "rect", + "scale", + "atlasIndex" + ], + "properties": { + "metrics": { + "$ref": "#/$defs/metrics" + }, + "rect": { + "$ref": "#/$defs/rect" + }, + "scale": { + "type": "number", + "exclusiveMinimum": 0 + }, + "atlasIndex": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "packed": { + "type": "object", + "required": [ + "texture", + "dx", + "dy" + ], + "additionalProperties": false, + "properties": { + "texture": { + "type": "string", + "minLength": 1 + }, + "dx": { + "$ref": "#/$defs/int" + }, + "dy": { + "$ref": "#/$defs/int" + } + } + }, + "runtime": { + "type": "boolean" + } + } + }, + "character": { + "type": "object", + "required": [ + "glyph", + "scale", + "elementType" + ], + "properties": { + "glyph": { + "type": "integer", + "minimum": 0 + }, + "scale": { + "type": "number", + "exclusiveMinimum": 0 + }, + "elementType": { + "type": "integer" + } + } + }, + "valueRecord": { + "type": "object", + "required": [ + "xPlacement", + "yPlacement", + "xAdvance", + "yAdvance" + ], + "properties": { + "xPlacement": { + "$ref": "#/$defs/num" + }, + "yPlacement": { + "$ref": "#/$defs/num" + }, + "xAdvance": { + "$ref": "#/$defs/num" + }, + "yAdvance": { + "$ref": "#/$defs/num" + } + } + }, + "source": { + "type": "object", + "required": [ + "family", + "style", + "version", + "license", + "file", + "faceIndex", + "bytes", + "sha256", + "generator" + ], + "properties": { + "family": { + "type": "string" + }, + "style": { + "type": "string" + }, + "version": { + "type": "string" + }, + "license": { + "type": "string" + }, + "file": { + "type": "string", + "pattern": "^[^/\\\\]+$" + }, + "faceIndex": { + "type": "integer", + "minimum": 0 + }, + "bytes": { + "type": "integer", + "minimum": 1 + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "generator": { + "type": "object" + } + } + }, + "font": { + "type": "object", + "required": [ + "faceInfo", + "atlasWidth", + "atlasHeight", + "atlasPadding", + "atlasRenderMode", + "atlasPopulationMode", + "atlases", + "normalStyle", + "normalSpacingOffset", + "boldStyle", + "boldSpacing", + "italicStyle", + "tabSize", + "material", + "fallbacks", + "glyphPairAdjustmentRecords", + "characters", + "glyphs", + "runtimeCharacters", + "runtimeGlyphs", + "subset" + ], + "properties": { + "faceInfo": { + "$ref": "#/$defs/faceInfo" + }, + "atlasWidth": { + "type": "integer", + "minimum": 1 + }, + "atlasHeight": { + "type": "integer", + "minimum": 1 + }, + "atlasPadding": { + "type": "integer", + "minimum": 0 + }, + "atlasRenderMode": { + "$ref": "#/$defs/int" + }, + "atlasPopulationMode": { + "enum": [ + 0, + 1 + ] + }, + "atlases": { + "type": "array", + "items": { + "type": [ + "string", + "null" + ] + } + }, + "normalStyle": { + "$ref": "#/$defs/num" + }, + "normalSpacingOffset": { + "$ref": "#/$defs/num" + }, + "boldStyle": { + "$ref": "#/$defs/num" + }, + "boldSpacing": { + "$ref": "#/$defs/num" + }, + "italicStyle": { + "$ref": "#/$defs/int" + }, + "tabSize": { + "$ref": "#/$defs/int" + }, + "material": { + "type": "string", + "minLength": 1 + }, + "fallbacks": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "glyphPairAdjustmentRecords": { + "type": "integer", + "minimum": 0 + }, + "glyphPairAdjustments": { + "type": "object", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "type": "object", + "required": [ + "first", + "second", + "flags" + ], + "properties": { + "first": { + "$ref": "#/$defs/valueRecord" + }, + "second": { + "$ref": "#/$defs/valueRecord" + }, + "flags": { + "$ref": "#/$defs/int" + } + } + } + }, + "characters": { + "type": "object", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "$ref": "#/$defs/character" + } + }, + "missingGlyph": { + "type": "object", + "required": [ + "unicode", + "characters" + ], + "additionalProperties": false, + "properties": { + "unicode": { + "type": "integer", + "minimum": 0 + }, + "characters": { + "type": "array", + "minItems": 1, + "items": { + "type": "integer", + "minimum": 0 + } + } + } + }, + "glyphs": { + "type": "object", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "$ref": "#/$defs/glyph" + } + }, + "runtimeCharacters": { + "type": "array", + "items": { + "type": "integer", + "minimum": 0 + } + }, + "runtimeGlyphs": { + "type": [ + "object", + "null" + ] + }, + "sourceFont": { + "type": [ + "object", + "null" + ] + }, + "source": { + "$ref": "#/$defs/source" + }, + "browser": { + "type": "object", + "required": [ + "family" + ], + "additionalProperties": false, + "properties": { + "family": { + "type": "string", + "minLength": 1 + }, + "weight": { + "type": "string", + "minLength": 1 + }, + "style": { + "type": "string", + "minLength": 1 + }, + "pageWidth": { + "type": "integer", + "minimum": 1 + }, + "pageHeight": { + "type": "integer", + "minimum": 1 + }, + "underlayOffset": { + "type": "number", + "minimum": 0 + }, + "gameFont": { + "type": "string", + "minLength": 1 + } + } + }, + "subset": { + "type": "string" + } + } + }, + "spriteAsset": { + "type": "object", + "required": [ + "faceInfo", + "characters", + "glyphs", + "sequences", + "material" + ], + "properties": { + "faceInfo": { + "type": "object", + "required": [ + "m_PointSize", + "m_Scale", + "m_Baseline", + "m_AscentLine", + "m_DescentLine" + ], + "properties": { + "m_PointSize": { + "type": "number", + "minimum": 0 + }, + "m_Scale": { + "$ref": "#/$defs/num" + }, + "m_Baseline": { + "$ref": "#/$defs/num" + }, + "m_AscentLine": { + "$ref": "#/$defs/num" + }, + "m_DescentLine": { + "$ref": "#/$defs/num" + } + } + }, + "characters": { + "type": "array", + "items": { + "type": "object", + "required": [ + "index", + "unicode", + "name", + "glyph", + "scale" + ], + "properties": { + "index": { + "type": "integer", + "minimum": 0 + }, + "unicode": { + "type": "integer", + "minimum": 0 + }, + "name": { + "type": "string" + }, + "glyph": { + "type": "integer", + "minimum": 0 + }, + "scale": { + "$ref": "#/$defs/num" + } + } + } + }, + "glyphs": { + "type": "object", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "$ref": "#/$defs/glyph" + } + }, + "sequences": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "unicode" + ], + "properties": { + "name": { + "type": "string", + "pattern": "-" + }, + "unicode": { + "type": "integer", + "minimum": 0 + } + } + } + }, + "material": { + "type": "string", + "minLength": 1 + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/source" + }, + { + "type": "null" + } + ] + }, + "subset": { + "type": "string" + } + } + }, + "texture": { + "type": "object", + "required": [ + "texture", + "name", + "width", + "height", + "mipCount", + "settings" + ], + "properties": { + "texture": { + "type": "string", + "pattern": "^(fonts|textures)/[^/\\\\]+\\.png$" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "width": { + "type": "integer", + "minimum": 1 + }, + "height": { + "type": "integer", + "minimum": 1 + }, + "mipCount": { + "const": 1 + }, + "settings": { + "type": "object", + "required": [ + "m_FilterMode", + "m_WrapU", + "m_WrapV" + ], + "properties": { + "m_FilterMode": { + "enum": [ + 0, + 1, + 2 + ] + }, + "m_WrapU": { + "enum": [ + 0, + 1, + 2 + ] + }, + "m_WrapV": { + "enum": [ + 0, + 1, + 2 + ] + } + } + } + } + }, + "material": { + "type": "object", + "required": [ + "material", + "shader", + "keywords", + "textures", + "floats", + "colors" + ], + "properties": { + "material": { + "type": "string", + "minLength": 1 + }, + "shader": { + "type": "object", + "required": [ + "shader" + ], + "properties": { + "shader": { + "type": "string", + "minLength": 1 + } + } + }, + "keywords": { + "type": "array", + "items": { + "type": "string" + } + }, + "textures": { + "type": "object" + }, + "floats": { + "type": "object", + "additionalProperties": { + "type": "number" + } + }, + "ints": { + "type": "object", + "additionalProperties": { + "type": "number" + } + }, + "colors": { + "type": "object" + } + } + }, + "text": { + "type": "object", + "required": [ + "font", + "localized", + "style" + ], + "properties": { + "font": { + "type": "object", + "required": [ + "fontAsset", + "material", + "lineSpacing" + ], + "properties": { + "fontAsset": { + "type": "string", + "minLength": 1 + }, + "material": { + "type": "string", + "minLength": 1 + }, + "lineSpacing": { + "type": "number" + } + } + }, + "localized": { + "type": [ + "object", + "null" + ], + "required": [ + "fontAsset", + "material", + "lineSpacing" + ], + "properties": { + "fontAsset": { + "type": "string", + "minLength": 1 + }, + "material": { + "type": "string", + "minLength": 1 + }, + "lineSpacing": { + "type": "number" + } + } + }, + "style": { + "type": "object" + }, + "localizeKoreanAdjust": { + "type": "object" + } + } + } + }, + "$id": "urn:nnnotes:schema:story-fonts", + "title": "Episode font records and glyph pages" } diff --git a/src/nnnotes/schemas/story-host.schema.json b/src/nnnotes/schemas/story-host.schema.json index d39bc06..99a158f 100644 --- a/src/nnnotes/schemas/story-host.schema.json +++ b/src/nnnotes/schemas/story-host.schema.json @@ -1,23 +1,185 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:nnnotes:schema:story-host", - "title": "Overlay story host screen", "type": "object", - "required": ["format", "kind", "prefabs", "cameraTarget", "overlayRoot", "layoutRoot", "openedSequences", "home", "afterlive", "graphics", "dotween", "bindings", "shaders"], + "required": [ + "format", + "kind", + "cameraTarget", + "dotween", + "prefabs", + "overlayRoot", + "layoutRoot", + "openedSequences", + "home", + "afterlive", + "graphics", + "bindings", + "shaders" + ], "properties": { - "format": {"const": "nnnotes.story-host/1"}, - "kind": {"enum": ["home", "afterlive"]}, - "prefabs": {"type": "object", "additionalProperties": {"type": "string"}}, - "cameraTarget": {"type": "string"}, - "overlayRoot": {"$ref": "#/$defs/location"}, - "layoutRoot": {"allOf": [{"$ref": "#/$defs/location"}, {"type": "object", "required": ["fields"], "properties": {"fields": {"type": "object"}}}]}, - "openedSequences": {"type": "array", "items": {"$ref": "#/$defs/location"}}, - "home": {"type": ["object", "null"]}, - "afterlive": {"type": ["object", "null"]}, - "graphics": {"type": "object", "required": ["graphics"], "additionalProperties": {"type": "string"}}, - "dotween": {"type": "string"}, - "bindings": {"type": "object"}, - "shaders": {"type": "object", "additionalProperties": {"type": "string"}} + "format": { + "const": "nnnotes.story-host/1" + }, + "kind": { + "enum": [ + "home", + "afterlive" + ] + }, + "cameraTarget": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "dotween": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "prefabs": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "overlayRoot": { + "type": "object", + "required": [ + "prefab", + "path" + ], + "properties": { + "prefab": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "path": { + "type": "string", + "minLength": 1 + } + } + }, + "layoutRoot": { + "type": "object", + "required": [ + "prefab", + "path", + "fields" + ], + "properties": { + "prefab": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "path": { + "type": "string", + "minLength": 1 + }, + "fields": { + "type": "object" + } + } + }, + "openedSequences": { + "type": "array", + "items": { + "type": "object", + "required": [ + "prefab", + "path" + ], + "properties": { + "prefab": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "path": { + "type": "string", + "minLength": 1 + } + } + } + }, + "home": { + "type": [ + "object", + "null" + ] + }, + "afterlive": { + "type": [ + "object", + "null" + ] + }, + "graphics": { + "type": "object", + "required": [ + "graphics" + ], + "properties": { + "graphics": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "player": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "quality": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, + "bindings": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "required": [ + "path", + "attribute" + ], + "properties": { + "path": { + "type": [ + "string", + "null" + ] + }, + "attribute": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "propertyNames": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, + "propertyNames": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "shaders": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } }, - "$defs": {"location": {"type": "object", "required": ["prefab", "path"], "properties": {"prefab": {"type": "string"}, "path": {"type": "string"}}}} + "$id": "urn:nnnotes:schema:story-host", + "title": "Overlay story host screen" } diff --git a/src/nnnotes/schemas/story-scene.schema.json b/src/nnnotes/schemas/story-scene.schema.json new file mode 100644 index 0000000..c5e66ce --- /dev/null +++ b/src/nnnotes/schemas/story-scene.schema.json @@ -0,0 +1,146 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "format", + "scene", + "renderSettings", + "prefabs", + "settings", + "graphics", + "bindings", + "shaders" + ], + "properties": { + "format": { + "const": 1 + }, + "scene": { + "type": "array", + "items": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "renderSettings": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "prefabs": { + "type": "object", + "required": [ + "cameraManager", + "characterField", + "backgroundField", + "globalVolume" + ], + "properties": { + "cameraManager": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "characterField": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "backgroundField": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "globalVolume": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, + "settings": { + "type": "object", + "required": [ + "playerSettings", + "masterIdSettings" + ], + "properties": { + "playerSettings": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "masterIdSettings": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, + "graphics": { + "type": "object", + "required": [ + "graphics", + "player", + "quality" + ], + "properties": { + "graphics": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "player": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "quality": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, + "bindings": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "required": [ + "path", + "attribute" + ], + "properties": { + "path": { + "type": [ + "string", + "null" + ] + }, + "attribute": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "propertyNames": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, + "propertyNames": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "shaders": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, + "$id": "urn:nnnotes:schema:story-scene", + "title": "story-scene document" +} diff --git a/src/nnnotes/schemas/story-simple-ui.schema.json b/src/nnnotes/schemas/story-simple-ui.schema.json index 5b8b2ee..344036f 100644 --- a/src/nnnotes/schemas/story-simple-ui.schema.json +++ b/src/nnnotes/schemas/story-simple-ui.schema.json @@ -1,20 +1,241 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:nnnotes:schema:story-simple-ui", - "title": "Overlay story talk-window and system-message UI", "type": "object", - "required": ["format", "language", "talkWindow", "systemMessage", "uiManager", "settings", "graphics", "texts", "textStyle", "bindings", "shaders"], + "required": [ + "uiManager", + "language", + "graphics", + "texts", + "textStyle", + "bindings", + "shaders", + "format", + "settings", + "talkWindow", + "systemMessage" + ], "properties": { - "format": {"const": "nnnotes.story-simple-ui/1"}, - "language": {"type": "object", "required": ["code", "mode", "field", "lineSpacing", "fontNames", "fontSwap"], "properties": {"code": {"enum": ["ja", "en", "zh-Hant", "zh-Hans", "ko"]}}}, - "talkWindow": {"type": "string"}, - "systemMessage": {"type": "string"}, - "uiManager": {"type": "string"}, - "settings": {"type": "object", "required": ["localizeManager", "tmpSettings", "dotween"], "additionalProperties": {"type": "string"}}, - "graphics": {"type": "object", "required": ["graphics"], "additionalProperties": {"type": "string"}}, - "texts": {"type": "object", "additionalProperties": {"type": "object", "required": ["style", "localized"], "properties": {"style": {"type": "object"}, "localized": {"type": ["object", "null"]}}}}, - "textStyle": {"type": "object"}, - "bindings": {"type": "object"}, - "shaders": {"type": "object", "additionalProperties": {"type": "string"}} - } + "uiManager": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "language": { + "type": "object", + "required": [ + "code", + "mode", + "field", + "lineSpacing", + "fontNames", + "additionalFontNames", + "materialTypes", + "fontSwap" + ], + "properties": { + "code": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] + }, + "mode": { + "enum": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + "field": { + "enum": [ + "japanese", + "english", + "traditionalChinese", + "simplifiedChinese", + "korean" + ] + }, + "lineSpacing": { + "type": "number" + }, + "fontNames": { + "type": "array", + "items": { + "type": "string" + } + }, + "additionalFontNames": { + "type": "array", + "items": { + "type": "string" + } + }, + "materialTypes": { + "type": "array", + "items": { + "type": "string" + } + }, + "fontSwap": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, + "graphics": { + "type": "object", + "required": [ + "graphics" + ], + "properties": { + "graphics": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "player": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "quality": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, + "texts": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "localized", + "style" + ], + "properties": { + "localized": { + "type": [ + "object", + "null" + ], + "required": [ + "fontAsset", + "material", + "lineSpacing" + ], + "properties": { + "fontAsset": { + "type": "string", + "minLength": 1 + }, + "material": { + "type": "string", + "minLength": 1 + }, + "lineSpacing": { + "type": "number" + } + } + }, + "style": { + "type": "object" + } + } + }, + "propertyNames": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "textStyle": { + "type": "object" + }, + "bindings": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "required": [ + "path", + "attribute" + ], + "properties": { + "path": { + "type": [ + "string", + "null" + ] + }, + "attribute": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "propertyNames": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, + "propertyNames": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "shaders": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "format": { + "const": "nnnotes.story-simple-ui/1" + }, + "settings": { + "type": "object", + "required": [ + "dotween", + "tmpSettings", + "localizeManager" + ], + "properties": { + "dotween": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "tmpSettings": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "localizeManager": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, + "talkWindow": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "systemMessage": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "$id": "urn:nnnotes:schema:story-simple-ui", + "title": "Overlay story talk-window and system-message UI" } diff --git a/src/nnnotes/schemas/story-sound.schema.json b/src/nnnotes/schemas/story-sound.schema.json new file mode 100644 index 0000000..ad750b8 --- /dev/null +++ b/src/nnnotes/schemas/story-sound.schema.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "file", + "sampleRate", + "samples" + ], + "properties": { + "file": { + "type": "string" + }, + "sampleRate": { + "type": "integer", + "minimum": 1 + }, + "samples": { + "type": "integer", + "minimum": 0 + }, + "channels": { + "type": "integer", + "minimum": 1 + }, + "categories": { + "type": "array", + "items": { + "type": "string" + } + }, + "encoderDelay": { + "type": "integer", + "minimum": 0 + }, + "loopStart": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "loopEnd": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + } + } + }, + "$id": "urn:nnnotes:schema:story-sound", + "title": "story-sound document" +} diff --git a/src/nnnotes/schemas/story-ui.schema.json b/src/nnnotes/schemas/story-ui.schema.json new file mode 100644 index 0000000..56e3d0c --- /dev/null +++ b/src/nnnotes/schemas/story-ui.schema.json @@ -0,0 +1,283 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "format", + "widget", + "chatWidget", + "uiManager", + "letterbox", + "talkWindows", + "dialogs", + "clips", + "controllers", + "language", + "settings", + "graphics", + "texts", + "textStyle", + "bindings", + "shaders" + ], + "properties": { + "format": { + "const": 1 + }, + "widget": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "chatWidget": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "uiManager": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "letterbox": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "talkWindows": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "dialogs": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "clips": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "controllers": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "language": { + "type": "object", + "required": [ + "code", + "mode", + "field", + "lineSpacing", + "fontNames", + "additionalFontNames", + "materialTypes", + "fontSwap" + ], + "properties": { + "code": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] + }, + "mode": { + "enum": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + "field": { + "enum": [ + "japanese", + "english", + "traditionalChinese", + "simplifiedChinese", + "korean" + ] + }, + "lineSpacing": { + "type": "number" + }, + "fontNames": { + "type": "array", + "items": { + "type": "string" + } + }, + "additionalFontNames": { + "type": "array", + "items": { + "type": "string" + } + }, + "materialTypes": { + "type": "array", + "items": { + "type": "string" + } + }, + "fontSwap": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, + "settings": { + "type": "object", + "required": [ + "dotween", + "tmpSettings", + "localizeManager", + "masterIdSettings" + ], + "properties": { + "dotween": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "tmpSettings": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "localizeManager": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "masterIdSettings": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, + "graphics": { + "type": "object", + "required": [ + "graphics" + ], + "properties": { + "graphics": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "player": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "quality": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, + "texts": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "localized", + "style" + ], + "properties": { + "localized": { + "type": [ + "object", + "null" + ], + "required": [ + "fontAsset", + "material", + "lineSpacing" + ], + "properties": { + "fontAsset": { + "type": "string", + "minLength": 1 + }, + "material": { + "type": "string", + "minLength": 1 + }, + "lineSpacing": { + "type": "number" + } + } + }, + "style": { + "type": "object" + } + } + }, + "propertyNames": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "textStyle": { + "type": "object" + }, + "bindings": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "required": [ + "path", + "attribute" + ], + "properties": { + "path": { + "type": [ + "string", + "null" + ] + }, + "attribute": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "propertyNames": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, + "propertyNames": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + }, + "shaders": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, + "$id": "urn:nnnotes:schema:story-ui", + "title": "story-ui document" +} diff --git a/src/nnnotes/schemas/story-video.schema.json b/src/nnnotes/schemas/story-video.schema.json new file mode 100644 index 0000000..1416adf --- /dev/null +++ b/src/nnnotes/schemas/story-video.schema.json @@ -0,0 +1,85 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "format", + "file", + "codec", + "width", + "height", + "displayWidth", + "displayHeight", + "frameRate", + "frames", + "tracks" + ], + "properties": { + "format": { + "const": 1 + }, + "file": { + "type": "string" + }, + "codec": { + "type": "string" + }, + "width": { + "type": "integer", + "minimum": 1 + }, + "height": { + "type": "integer", + "minimum": 1 + }, + "displayWidth": { + "type": "integer", + "minimum": 1 + }, + "displayHeight": { + "type": "integer", + "minimum": 1 + }, + "frameRate": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "type": "integer", + "minimum": 1 + } + }, + "frames": { + "type": "integer", + "minimum": 0 + }, + "tracks": { + "type": "array", + "items": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "kind", + "channel", + "codec" + ], + "properties": { + "kind": { + "type": "string" + }, + "channel": { + "type": "integer", + "minimum": 0 + }, + "codec": { + "type": "string" + }, + "source": { + "type": "string" + } + } + } + } + }, + "$id": "urn:nnnotes:schema:story-video", + "title": "story-video document" +} diff --git a/src/nnnotes/schemas/story.schema.json b/src/nnnotes/schemas/story.schema.json new file mode 100644 index 0000000..f000dd4 --- /dev/null +++ b/src/nnnotes/schemas/story.schema.json @@ -0,0 +1,176 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "format", + "advId", + "episode", + "parts" + ], + "properties": { + "format": { + "const": 1 + }, + "advId": { + "type": "integer", + "minimum": 0 + }, + "episode": { + "const": "episode.json" + }, + "chats": { + "type": "object" + }, + "effects": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "host": { + "type": [ + "object", + "null" + ] + }, + "parts": { + "type": "object", + "required": [ + "scene", + "assets", + "models", + "sounds", + "ui", + "fonts" + ], + "properties": { + "scene": { + "type": "string", + "pattern": "^[^/\\\\?#]+(?:/[^/\\\\?#]+)+\\.json$" + }, + "chat": { + "anyOf": [ + { + "type": "string", + "pattern": "^[^/\\\\?#]+(?:/[^/\\\\?#]+)+\\.json$" + }, + { + "type": "null" + } + ] + }, + "crilips": { + "anyOf": [ + { + "type": "string", + "pattern": "^[^/\\\\?#]+(?:/[^/\\\\?#]+)+\\.json$" + }, + { + "type": "null" + } + ] + }, + "assets": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^[^/\\\\?#]+(?:/[^/\\\\?#]+)+\\.json$" + } + }, + "models": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^[^/\\\\?#]+(?:/[^/\\\\?#]+)+\\.json$" + } + }, + "sounds": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^[^/\\\\?#]+(?:/[^/\\\\?#]+)+\\.json$" + } + }, + "videos": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^[^/\\\\?#]+(?:/[^/\\\\?#]+)+\\.json$" + } + }, + "host": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^[^/\\\\?#]+(?:/[^/\\\\?#]+)+\\.json$" + } + }, + "spots": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^[^/\\\\?#]+(?:/[^/\\\\?#]+)+\\.json$" + } + }, + "rooms": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^[^/\\\\?#]+(?:/[^/\\\\?#]+)+\\.json$" + } + }, + "ui": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^[^/\\\\?#]+(?:/[^/\\\\?#]+)+\\.json$" + }, + "propertyNames": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] + } + }, + "fonts": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^[^/\\\\?#]+(?:/[^/\\\\?#]+)+\\.json$" + }, + "propertyNames": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] + } + }, + "simpleui": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^[^/\\\\?#]+(?:/[^/\\\\?#]+)+\\.json$" + }, + "propertyNames": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] + } + } + }, + "additionalProperties": false + } + }, + "$id": "urn:nnnotes:schema:story", + "title": "story document" +} diff --git a/src/nnnotes/schemas/ui-library.schema.json b/src/nnnotes/schemas/ui-library.schema.json index e130916..963e743 100644 --- a/src/nnnotes/schemas/ui-library.schema.json +++ b/src/nnnotes/schemas/ui-library.schema.json @@ -1,6 +1,5 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:nnnotes:schema:ui-library", "type": "object", "required": [ "format", @@ -144,5 +143,6 @@ } } }, + "$id": "urn:nnnotes:schema:ui-library", "title": "UI product library" } diff --git a/src/nnnotes/schemas/ui-pack.schema.json b/src/nnnotes/schemas/ui-pack.schema.json index ed6b3cc..1c48070 100644 --- a/src/nnnotes/schemas/ui-pack.schema.json +++ b/src/nnnotes/schemas/ui-pack.schema.json @@ -1,6 +1,5 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:nnnotes:schema:ui-pack", "type": "object", "required": [ "format", @@ -125,5 +124,6 @@ } } }, + "$id": "urn:nnnotes:schema:ui-pack", "title": "UI object product" } diff --git a/tests/content/test_live.py b/tests/content/test_live.py index 6e81030..3111bba 100644 --- a/tests/content/test_live.py +++ b/tests/content/test_live.py @@ -7,8 +7,16 @@ from nnnotes.pipeline.assemble import Objects from nnnotes.pipeline.depot import Depot from tests.helpers import live as lh +from tests.helpers.schemas import validate -PARTS = {"scene": "scenes/default.json", "music": "music/7.json"} +PARTS = { + "scene": "scenes/default.json", + "music": "music/7.json", + "notes": "notes/default.json", + "noteskin": "noteskins/skin001.json", + "noteeffect": "noteeffects/effect001.json", + "sounds": "sounds/default.json", +} def objects_of(tmp_path, **kw) -> Objects: @@ -61,6 +69,7 @@ def test_the_record_holds_the_musics_rows_the_difficulty_and_the_band(): def test_the_live_holds_its_chart_converted_and_names_its_parts(tmp_path, gz): record = live.live_record(lh.master(), "7/expert") doc, chart, converted, keep = live.live(objects_of(tmp_path, gz=gz), record, PARTS) + validate("live", doc) notes, mirror = score.convert(lh.CHART), score.convert(lh.CHART, mirror=True) assert doc == { "format": live.LIVE_FORMAT, @@ -76,7 +85,7 @@ def test_the_live_holds_its_chart_converted_and_names_its_parts(tmp_path, gz): "lastNoteTimeMs": notes["lastNoteTimeMs"], "master": record["music"], "jacket": lh.JACKET, - "parts": {"music": "music/7.json", "scene": "scenes/default.json"}, + "parts": PARTS, } assert chart == contract.encode(lh.CHART) # as shipped, un-gzipped source = { diff --git a/tests/content/test_live2d.py b/tests/content/test_live2d.py index 0a82828..a098bdb 100644 --- a/tests/content/test_live2d.py +++ b/tests/content/test_live2d.py @@ -8,6 +8,7 @@ from nnnotes.pipeline.assemble import Objects from nnnotes.pipeline.depot import Depot from tests.helpers import live2dmodel as lm +from tests.helpers.schemas import validate KEY_A = "Character/Live2D/001_adv/adv_model_a/model/adv_model_a" KEY_B = "Character/Live2D/sub_x/adv_model_b/model/adv_model_b" @@ -20,6 +21,7 @@ def objects_of(tmp_path, objs=None, roots=(lm.ROOT, lm.MASK_MAT, lm.CULL_MAT)) - def test_model_document(tmp_path): doc, keep, roles = live2d.model(objects_of(tmp_path), lm.KEY) + validate("model-json", doc) assert doc == { "format": live2d.MODEL_FORMAT, "name": "m001", diff --git a/tests/content/test_livenotes.py b/tests/content/test_livenotes.py index 55d25ad..18a3213 100644 --- a/tests/content/test_livenotes.py +++ b/tests/content/test_livenotes.py @@ -6,6 +6,7 @@ from nnnotes.pipeline.assemble import Objects from nnnotes.pipeline.depot import Depot from tests.helpers import livenotes as ln +from tests.helpers.schemas import validate FLICKS = sorted(livenotes.FLICK_SKIN_UNITS) ARROW = [{"path": "arrow", "attribute": "m_Color.a"}] @@ -19,7 +20,14 @@ def objects_of(tmp_path, keys, objs=None) -> Objects: def test_the_notes_document(tmp_path): record = livenotes.notes_record(ln.master()) + for row in record["MasterLiveSettings"]: + row["_value"] = str(row["_value"]) + for name in ("SlideOpacity", "GuideOpacity"): + item = next(i for i, key in livenotes.NOTE_ITEMS.items() if key == name) + record["MasterOptionDefault"].append({"_presetId": 1, "_optionItemType": item, "_valueString": "100"}) + record["MasterOptionRange"].append({"_optionItemType": item, "_minValue": 0, "_maxValue": 100}) doc, keep, roles = livenotes.notes(objects_of(tmp_path, ln.NOTES_KEYS), record) + validate("live-notes", doc) assert doc["format"] == livenotes.NOTES_FORMAT assert doc["prefabs"] == {n: ln.prefab_id(n) for n in livenotes.NOTE_PREFABS} assert doc["laneEffect"] == ln.LANE @@ -61,6 +69,7 @@ def test_the_notes_settings(tmp_path): def test_a_skin_binds_the_arrow_loop_of_its_flick_units(tmp_path): doc, keep, _ = livenotes.noteskin(objects_of(tmp_path, ln.SKIN_KEYS), "skin001") + validate("live-noteskin", doc) assert (doc["format"], doc["name"], doc["skin"]) == (livenotes.NOTES_FORMAT, "skin001", ln.SKIN) # the flick unit's loop plays on the flick view's time sync Animator; the left unit has none, the right is unset assert doc["bindings"] == {ln.prefab_id("flick_note_view"): {"flick_note_view/anim_root": {ln.SKIN_CLIP: ARROW}}} @@ -69,6 +78,7 @@ def test_a_skin_binds_the_arrow_loop_of_its_flick_units(tmp_path): def test_an_effect_set(tmp_path): doc, _, roles = livenotes.noteeffect(objects_of(tmp_path, ln.EFFECT_KEYS), "effect001Light") + validate("live-noteeffect", doc) assert (doc["name"], doc["settings"]) == ("effect001Light", ln.EFFECT) assert doc["bindings"][ln.EFFECT_FX] == {"note_normal/root": {ln.FX_CLIP: GLOW}} assert doc["shaders"] == {"Sirius/Sprite": ln.SHADER} and set(roles) == {ln.SHADER} diff --git a/tests/content/test_livescene.py b/tests/content/test_livescene.py index e413932..7451377 100644 --- a/tests/content/test_livescene.py +++ b/tests/content/test_livescene.py @@ -6,6 +6,7 @@ from nnnotes.pipeline.assemble import Objects from nnnotes.pipeline.depot import Depot from tests.helpers import livescene as ls +from tests.helpers.schemas import validate LANE = [{"path": "LiveGameLane", "attribute": "m_Color.a"}] @@ -20,7 +21,12 @@ def record(): def test_the_scene_document(tmp_path): - doc, keep, roles = livescene.scene(objects_of(tmp_path, record()), record()) + settings = record() + for name in ("JudgePositionDisplay", "NoteStartPosition"): + item = next(i for i, key in livescene.OPTION_ITEMS.items() if key == name) + settings["MasterOptionDefault"].append({"_presetId": 1, "_optionItemType": item, "_valueString": "0"}) + doc, keep, roles = livescene.scene(objects_of(tmp_path, settings), settings) + validate("live-scene", doc) assert doc["format"] == livescene.SCENE_FORMAT assert doc["scene"] == [ls.LIVE, ls.GAME, ls.DIRECTOR] and doc["renderSettings"] == ls.RENDER assert doc["graphics"] == {"graphics": ls.GRAPHICS, "quality": ls.QUALITY, "player": ls.PLAYER} diff --git a/tests/content/test_room.py b/tests/content/test_room.py index 3331bf6..6e0a7d2 100644 --- a/tests/content/test_room.py +++ b/tests/content/test_room.py @@ -166,6 +166,9 @@ def objects_of(tmp_path, objs=None) -> Objects: def test_room_bakes_the_drawn_meshes_and_lists_what_draws_nothing(tmp_path): glb, doc = room.room(objects_of(tmp_path), so.BACKGROUND) + from tests.helpers.schemas import validate + + validate("room", doc) gdoc, body = gltf.read(glb) assert [n["name"] for n in gdoc["nodes"]] == ["quad", "quad2", "quad"] # wall, floor, wall2: hierarchy order assert [n["extras"]["unityActive"] for n in gdoc["nodes"]] == [True, False, True] diff --git a/tests/content/test_spot.py b/tests/content/test_spot.py index 19d928c..841c0d0 100644 --- a/tests/content/test_spot.py +++ b/tests/content/test_spot.py @@ -36,6 +36,9 @@ def test_master_record(): def test_spot_document(tmp_path): doc, files = spot.spot(objects_of(tmp_path), so.SITUATION, spot.master_record(so.MASTER, so.SITUATION)) + from tests.helpers.schemas import validate + + validate("spot", doc) assert doc == { "spotId": 3, "master": so.MASTER["MasterHomeSpot"][0], diff --git a/tests/content/test_storyasset.py b/tests/content/test_storyasset.py index d5ddae8..3b8d9f8 100644 --- a/tests/content/test_storyasset.py +++ b/tests/content/test_storyasset.py @@ -41,6 +41,9 @@ def test_manifest_name_rejects_other_keys(key): def test_the_document_of_a_prefab_key(tmp_path): keys = {KEY: [PREFAB, "CAB-n:999"]} # 999 is not in the table: not one of the key's objects doc, _, _ = storyasset.asset(objects_of(tmp_path, keys, SMALL), KEY) + from tests.helpers.schemas import validate + + validate("story-asset", doc) assert doc == { "format": storyasset.ASSET_FORMAT, "key": KEY, diff --git a/tests/content/test_storyui.py b/tests/content/test_storyui.py index 8042e54..29b0b21 100644 --- a/tests/content/test_storyui.py +++ b/tests/content/test_storyui.py @@ -9,6 +9,7 @@ from nnnotes.pipeline.depot import Depot from tests.helpers import storyui as sui from tests.helpers.livenotes import comp +from tests.helpers.schemas import validate def ui(objects, language, master): @@ -37,6 +38,7 @@ def test_the_story_ui_keys_are_the_keys_of_its_table(): def test_the_document_has_the_story_ui_fields_in_the_language(tmp_path): doc, keep, roles = ui(story(tmp_path), "zh-Hant", sui.MASTER_RECORD) + validate("story-ui", doc) assert set(doc) == { "format", "language", diff --git a/tests/content/test_storyvideo.py b/tests/content/test_storyvideo.py index 8921311..f17f202 100644 --- a/tests/content/test_storyvideo.py +++ b/tests/content/test_storyvideo.py @@ -75,6 +75,9 @@ def test_video_describes_the_movie_and_carries_the_tracks_of_the_webm(): key = "Cri/Video/adv/m/m" objects = table(key, {"movieInfo": movie_info()}) doc = storyvideo.video(objects, key, TRACKS) + from tests.helpers.schemas import validate + + validate("story-video", doc) assert doc == { "format": 1, "file": "movie.webm", diff --git a/tests/helpers/schemas.py b/tests/helpers/schemas.py new file mode 100644 index 0000000..0c29aff --- /dev/null +++ b/tests/helpers/schemas.py @@ -0,0 +1,9 @@ +"""Validate synthetic product documents against the schemas shipped in nnnotes.""" + +from jsonschema import Draft202012Validator + +from nnnotes import schemas + + +def validate(name: str, document: dict) -> None: + Draft202012Validator(schemas.load(name)).validate(document) diff --git a/tests/helpers/storyui.py b/tests/helpers/storyui.py index abb9176..3b94258 100644 --- a/tests/helpers/storyui.py +++ b/tests/helpers/storyui.py @@ -219,7 +219,26 @@ def table_doc(depot, objs: dict | None = None, keys: dict | None = None, extra_r def font_settings() -> fontsettings.FontSettings: """The font settings the texts draw with: the Japanese and the traditional Chinese font assets, and the default and outline materials of the Japanese font and the outline material of the traditional Chinese one.""" - face = {"m_Scale": 1.0, "m_PointSize": 100.0, "m_LineHeight": 120.0, "m_AscentLine": 90.0, "m_DescentLine": -30.0} + face = { + "m_Scale": 1.0, + "m_PointSize": 100.0, + "m_UnitsPerEM": 1000, + "m_LineHeight": 120.0, + "m_AscentLine": 90.0, + "m_DescentLine": -30.0, + "m_CapLine": 70.0, + "m_MeanLine": 50.0, + "m_Baseline": 0.0, + "m_SuperscriptOffset": 60.0, + "m_SuperscriptSize": 0.5, + "m_SubscriptOffset": -20.0, + "m_SubscriptSize": 0.5, + "m_UnderlineOffset": -10.0, + "m_UnderlineThickness": 2.0, + "m_StrikethroughOffset": 30.0, + "m_StrikethroughThickness": 2.0, + "m_TabWidth": 40.0, + } fonts = { name: {"faceInfo": face, "normalSpacingOffset": 0.0, "boldSpacing": 7.0} for name in ("MainJ SDF", "MainT SDF") } diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index 6316257..633f302 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -976,6 +976,9 @@ def test_items_name_the_objects_they_reference(exported): def test_prefab(exported): depot, doc = exported p = obj(depot, doc, f"{oid(60)}#json") + from tests.helpers.schemas import validate + + validate("prefab", p) assert p["name"] == "root" and [n["path"] for n in p["nodes"]] == ["root", "root/child"] root, child = p["nodes"] assert root["pathId"] == "60" and root["transformPathId"] == "61" and root["layer"] == 5 From b0454bb12da3397633da38fa4f9758b6a680af42 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 18 Oct 2026 21:07:40 +0800 Subject: [PATCH 84/91] docs(schema): list every packaged product contract --- docs/schemas.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/schemas.md b/docs/schemas.md index 367721d..003c67e 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -23,25 +23,45 @@ schemas.load("plan") # the schema as a dict | `common` | | definitions shared by the other schemas | | `cost` | `nnnotes.cost/1` | the measured cost of a task's last run | | `coverage` | `nnnotes.coverage/1` | the coverage of a run per class and per reason code | +| `crilips` | product document | CRI Lips tensor blocks and frontend constants | +| `episode` | product document | story commands and resource tables | | `failures` | `nnnotes.failures/1` | the failed tasks and items of a run | | `layout` | `nnnotes.layout/1` | the files a layout owns in its output directory | +| `live` | `1` | live chart entry and shared parts | +| `live-music` | `1` | music layers and cue metadata | +| `live-noteeffect` | `1` | note effect resources and animation bindings | +| `live-notes` | `1` | note prefabs, settings and animations | +| `live-noteskin` | `1` | note skin resources and animation bindings | +| `live-scene` | `1` | scene objects, settings and band timelines | +| `live-sounds` | `1` | sound groups and category routing | | `live-ui` | `nnnotes.live-ui/1` (`format`) | live start strings, text styles, glyph pages and image bindings | +| `model-json` | `3` | Live2D model objects and animation bindings | | `music-data` | `nnnotes.music-data/2` | every song and chart of one master data version ([music data](music-data.md)) | | `objects` | `nnnotes.objects/1` | the objects a product reaches: their class, item status and files ([products](products.md)) | | `plan` | `nnnotes.plan/1` | what a run would do and why (`depot plan`) | +| `prefab` | product document | prefab nodes, transforms and components | | `product` | `nnnotes.product/1` | a product's manifest: its files and the files of the objects it uses, as content-addressed assets ([products](products.md)) | | `result` | `nnnotes.result/1` | the result of a task, a function of its key alone | +| `room` | product document | room geometry, materials and textures | | `run` | `nnnotes.run/1` | a run manifest: context, selection, tasks with keys and statuses, layouts | | `scripts` | `nnnotes.scripts/1` | every MonoScript of the censuses, by object id | | `site` | `nnnotes.site/1` | a named site's complete region-topic catalog and immutable product references | +| `spot` | product document | spot settings, characters and Spine data | +| `story` | `1` | story entry and shared parts | +| `story-asset` | `1` | story media prefab objects | | `story-fonts` | `nnnotes.story-fonts/1` (`format`) | an episode's text bindings, font records, glyph pages and text shaders | | `story-host` | `nnnotes.story-host/1` (`format`) | an Overlay story's host screen, layout and initial state | +| `story-scene` | `1` | story scene objects and graphics settings | | `story-simple-ui` | `nnnotes.story-simple-ui/1` (`format`) | localized Overlay talk-window and system-message prefabs | +| `story-sound` | product document | cue metadata and waveform files | +| `story-ui` | `1` | localized story UI and text bindings | +| `story-video` | `1` | video dimensions, timing and media tracks | | `task` | `nnnotes.task/1` | one stage applied to one subject, with everything needed to run it (`depot run`) | | `ui-library` | `nnnotes.ui-library/1` | selected UI keys and their product manifests | | `ui-pack` | `nnnotes.ui-pack/1` | native UI roots, prefab and controller identities, animation bindings | | `view` | `nnnotes.view/1` | a semantic view of a master table, each row with its names and the objects it names | + The documents of the asset export are described in [contracts](contracts.md), the views in [views](views.md). The schema files are in [src/nnnotes/schemas](https://github.com/MetaSekaiLab/nnnotes/tree/main/src/nnnotes/schemas). From 1ae3260db8f4e1ee6f7a9b3619866d0ffd0af507 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 19 Oct 2026 01:40:23 +0800 Subject: [PATCH 85/91] fix: retain depot results and align product exports --- CHANGELOG.md | 7 +- README.md | 2 +- docs/api.md | 1 - docs/architecture.md | 5 +- docs/assets.md | 7 +- docs/cli.md | 1 + docs/depot.md | 51 +++++++ docs/formats.md | 37 +++++ docs/index.md | 2 +- mkdocs.yml | 2 + pyproject.toml | 2 +- release-notes/v0.3.0.txt | 4 +- src/nnnotes/cli/__init__.py | 13 +- src/nnnotes/cli/assets.py | 16 +- src/nnnotes/cli/depot.py | 15 +- src/nnnotes/cli/export.py | 149 ++++++++++++++++--- src/nnnotes/cli/site.py | 7 +- src/nnnotes/content/storyepisode.py | 4 +- src/nnnotes/core/downloads.py | 15 +- src/nnnotes/pipeline/catalogdb.py | 12 ++ src/nnnotes/pipeline/depot.py | 131 ++++++++++++++--- src/nnnotes/pipeline/exportfiles.py | 141 ++++++++++++++++++ src/nnnotes/pipeline/live.py | 12 +- src/nnnotes/pipeline/siteindex.py | 4 +- src/nnnotes/pipeline/siteproducts.py | 6 +- src/nnnotes/pipeline/story.py | 19 ++- src/nnnotes/schemas/product.schema.json | 117 +++++++++++++++ src/nnnotes/schemas/story-ui.schema.json | 9 +- src/nnnotes/site/catalog.py | 23 ++- src/nnnotes/site/pages.py | 15 +- src/nnnotes/site/publish.py | 2 + src/nnnotes/site/webmodel.py | 10 -- tests/cli/test_assets.py | 6 +- tests/cli/test_cli.py | 19 ++- tests/content/test_storysimpleui.py | 2 +- tests/core/test_downloads.py | 12 ++ tests/pipeline/test_catalog_remote.py | 12 ++ tests/pipeline/test_depot_gc.py | 55 ++++++- tests/pipeline/test_depot_remote.py | 55 ++++++- tests/pipeline/test_exportfiles.py | 179 +++++++++++++++++++++++ tests/pipeline/test_live.py | 6 +- tests/pipeline/test_orchestrate.py | 4 +- tests/pipeline/test_story.py | 18 +-- tests/site/test_catalog.py | 37 ++++- tests/site/test_cli.py | 56 ++++++- tests/site/test_pages.py | 25 +++- tests/site/test_publish.py | 8 + tests/source/test_catalog_versions.py | 9 +- tests/test_api.py | 1 - uv.lock | 14 +- 50 files changed, 1224 insertions(+), 135 deletions(-) create mode 100644 docs/depot.md create mode 100644 docs/formats.md create mode 100644 src/nnnotes/pipeline/exportfiles.py delete mode 100644 src/nnnotes/site/webmodel.py create mode 100644 tests/pipeline/test_exportfiles.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b1885e..4aeb4a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,17 +4,20 @@ All notable changes to this project are listed here. Versions follow [Semantic V over the command line (commands, options, exit statuses and `--json` reports), the configuration keys, the JSON schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor version can change any of them. -## [0.3.0](https://github.com/MetaSekaiLab/nnnotes/compare/v0.2.1...v0.3.0) - 2026-10-18 +## [0.3.0](https://github.com/MetaSekaiLab/nnnotes/compare/v0.2.5...v0.3.0) - 2026-10-18 ### Compatibility - Configure persistent storage with `[paths] depot` and optional `[depot] remote`. Manage it with `depot plan`, `depot run`, `depot verify` and `depot gc`. - Exported products use `nnnotes.product/1` manifests and shared object references. Browser playback uses ournotes-player 0.2.x. - Build named sites with `site build --name NAME -o DIR`. Copy selected source bundles with `bundle pull -o DIR`. -- The `deck` extra uses ournotes-sim 0.0.9. +- The `deck` extra uses ournotes-sim 0.0.12. ### Features +- Export episode documents, shader files and cue-sheet audio through depot tasks and retained file layouts. +- Share cached results when adding remote storage, verify remote-only content, and retain recent results with their dependencies during collection. +- Validate player package compatibility and publish chart metadata and requested language references consistently. - Ship schemas for shared product root documents and prefab objects, with numeric font and transform contracts. - Share catalog versions and labels through depot remotes, with verified catalog files fetched on demand. - Retain census planning results, emitted task descriptions and standalone task runs as depot collection roots. diff --git a/README.md b/README.md index ac6fe38..15c1a27 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ pip install 'nnnotes[deck]' # the deck model's chart statistics of musi ``` nnnotes also needs the game's `base.apk`, decoded master data for the commands that read it, and the external tools -[vgmstream](https://vgmstream.org/) and [FFmpeg](https://ffmpeg.org/) for audio and video (Node.js 20+ and a built +[vgmstream](https://vgmstream.org/) and [FFmpeg](https://ffmpeg.org/) for audio and video (Node.js 22+ and a built ournotes-player for `site build`). ## Quick start diff --git a/docs/api.md b/docs/api.md index 9b04a4d..ecb2d1a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -38,6 +38,5 @@ raises `GameApiError`. Other names are internal and can change in any version. ## Site and schemas -::: nnnotes.site.webmodel ::: nnnotes.schemas diff --git a/docs/architecture.md b/docs/architecture.md index 7fb53b9..a257b19 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -53,7 +53,10 @@ can then be released toward the configured budget under the command's exclusive requests retain inputs until the expanded product request is known. Stored conversion results remain available for assembly and layouts. -`export episode`, `export shader`, `export audio` and `music-data` expose subject-specific data and conversions through the Session. All commands use the readers and converters below the pipeline layer. +`export episode`, `export shader` and `export audio` request depot tasks and project their results into files. +Episode assembly reads the common object table, shader files reuse converted Shader artifacts, and audio files +reuse `cri.audio` streams. `music-data` uses Session to obtain catalog and master inputs for its tabular export. +Commands record their accessed depot content and results as retention roots; dry runs leave no command record. ## Outputs diff --git a/docs/assets.md b/docs/assets.md index f7743a1..33c23a5 100644 --- a/docs/assets.md +++ b/docs/assets.md @@ -137,10 +137,11 @@ parallel (default: the CPUs the process may use; `0`: one at a time). Removes what no kept run reaches. The kept runs are, for each selection, the `--keep N` latest (default: every run; the latest run of a selection is always kept). A kept run reaches the results of its tasks, the contents those results name (their artifacts, and the inputs their task keys name when the depot holds them) and its layout -manifests with the files they list. Every other run, result (with its measured cost) and content is removed. +manifests with the files they list. Results and contents younger than `--grace SECONDS` (default 86400) also stay; +a recent result keeps its older inputs and artifacts. Other runs, results (with their measured costs) and contents are removed. `--dry-run` counts what would be removed. A layout made of hard links or clones keeps its files. `--remote` -collects the depot's remote the same way, by the runs on the remote; its results and contents younger than a day stay, -since a run on another machine may be writing them. gc runs only when +collects the depot's remote the same way, by the runs on the remote and the same grace period. Remote collection +does not populate the local cache. gc runs only when no other command uses the depot (it exits with an error otherwise), and commands that start meanwhile wait for it. The downloads are bounded separately, by `[depot] downloads` ([commands.md](commands.md#downloads)). diff --git a/docs/cli.md b/docs/cli.md index 7f16ea9..e7de8bd 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -386,6 +386,7 @@ remove the runs, results and contents no kept run reaches | --- | --- | | `--keep` `N` | keep the N latest runs of each selection (default: every run; the latest run of a selection is kept) | | `--dry-run` | count what would be removed, remove nothing | +| `--grace` `SECONDS` | keep recent results and their inputs and contents in both depots (default: 86400 seconds) | | `--remote` | also collect the depot's remote, by its runs (its results and contents younger than a day stay) | ## nnnotes voices list diff --git a/docs/depot.md b/docs/depot.md new file mode 100644 index 0000000..b34cf36 --- /dev/null +++ b/docs/depot.md @@ -0,0 +1,51 @@ +# Depot reference + +The depot stores task results and their content-addressed files. Set `[paths] depot` to its directory. +Commands share it across executions; an output layout is a separately materialized selection of its results. + +| Directory | Contents | +|---|---| +| `cas/sha256/` | File bytes indexed by SHA-256 | +| `ac/` | Task result documents indexed by task key | +| `runs/` | Run manifests, logs and latest-selection pointers | +| `catalogs/` | Catalog versions, labels and verified catalog bytes | +| `inputs/` | File identity records and named input identities | +| `costs/` | Measured task costs used by scheduling | +| `downloads/` | Recoverable source downloads | +| `leases/` | Active command leases and the exclusive collection lock | + +## Remote storage + +Set `[depot] remote` to a `file:///...` directory or an `s3://bucket/prefix` URL. S3 storage requires +the `remote` extra. See [configuration](configuration.md) for endpoint and credential settings. + +Results and content are fetched when needed. Publishing a run first uploads its completed results, available +stored inputs and layout contents. This also shares results computed before remote storage was configured. +A run pointer alone is not evidence that a result exists: planning checks the result itself. +Catalog discovery merges version metadata and shares locally imported versions with their catalog bytes. + +`depot verify` checks local content and results. When a local result refers to content held only by the remote, +verification checks that remote object without adding it to the local cache. Full verification checks hashes; +`--quick` checks sizes. + +## Retention + +Each command using the depot records its outcome and the stored content and task results it accessed. These +command runs are independent of product and site selections, so they retain inputs without replacing those +selections' latest layout. Commands that only use downloads or catalog files still leave an execution record; +those source directories are outside CAS collection. Dry runs leave no execution record. + +`depot gc` retains the results, stored inputs and layouts reachable from kept runs. By default all runs are kept; +`--keep N` retains the latest N runs per selection and always the selection's latest run. + +Results and content younger than `--grace SECONDS` are also retained. The default is 86400 seconds, in both local +and remote storage. A recent result protects even older inputs and artifacts, covering the interval before its +run record is written. `--grace 0` removes this time-based protection. `--remote` additionally collects remote +storage from its own run records. `--dry-run` only counts removals and does not fetch remote content into the cache. + +Collection requires an exclusive lease. A command waiting for an exclusive holder reports the wait and fails +after 30 seconds if the holder is still active; it leaves that holder's lock intact. Download retention is +separate: `[depot] downloads` sets its byte budget, with `all` keeping downloads. + +For task planning and execution, see [asset commands](assets.md). For identities and document shapes, see +[contracts](contracts.md) and [formats](formats.md). diff --git a/docs/formats.md b/docs/formats.md new file mode 100644 index 0000000..1a67c49 --- /dev/null +++ b/docs/formats.md @@ -0,0 +1,37 @@ +# Format reference + +nnnotes publishes versioned JSON documents. The `schema` or `format` field identifies the document contract; +installed schemas are available through `nnnotes.schemas`. See the [schema index](schemas.md) for individual files. + +| Document | Identity | Purpose | +|---|---|---| +| Task | `nnnotes.task/1` | Stage, version, parameters, inputs and task key | +| Result | `nnnotes.result/1` | Artifacts and item outcomes of a task | +| Run | `nnnotes.run/1` | Requested tasks, outcomes and retained layouts | +| Object table | `nnnotes.objects/1` | Converted objects, roles and reference closure | +| Product manifest | `nnnotes.product/1` | A product's files and the objects it uses | +| Layout | `nnnotes.layout/1` | Owned output paths and their stored content | + +## Objects and references + +Objects have stable addresses within their serialized file and are referred to by `:`. +`{"$ref": ""}` names an object in the product's object table. Its roles identify JSON, images, +shader programs and binary files. `{"$hex": "..."}` carries inline hexadecimal bytes; blob descriptors identify +separate binary roles. Full representations and examples are in [products](products.md) and [contracts](contracts.md). + +## Products + +A product manifest names its `product`, `subject`, `files` and `objects`. Files point to content-addressed assets +and record decoded sizes. Encoded files additionally record stored sizes; split JSON files list their parts. +The manifest's optional `root` locates these paths relative to the manifest URL. + +Story and live language parameters are part of their task identities. Their language-indexed part references +name the requested UI and font products. Site snapshots retain the selected available languages. + +A site's chart alias may additionally carry `musicId`, `difficulty`, `chart`, `regions`, `quality`, `flows`, +`audio` and `audioFormat`. The chart facts agree with `charts.json`; its byte count covers unique assets in the +product closure, including variants. The supported start flow is `direct`. With audio disabled, `audio` is false +and `audioFormat` is null. Standalone live products provide chart facts through their `live.json` and music part. + +For product composition see [products](products.md); for compressed files, regional aliases and publication +ordering see [site layout](site.md). The [depot reference](depot.md) describes storage and retention. diff --git a/docs/index.md b/docs/index.md index faf386f..1fbbb30 100644 --- a/docs/index.md +++ b/docs/index.md @@ -46,7 +46,7 @@ pip install 'nnnotes[deck]' # the deck model's chart statistics of musi Besides the package, nnnotes reads the game's `base.apk` and, for the commands that need it, decoded master data; audio and video need [vgmstream](https://vgmstream.org/) and [FFmpeg](https://ffmpeg.org/), and `site build` also -Node.js 20+ and a built ournotes-player. [Configuration](configuration.md#what-each-command-needs) lists what each +Node.js 22+ and a built ournotes-player. [Configuration](configuration.md#what-each-command-needs) lists what each command needs. ## Versions diff --git a/mkdocs.yml b/mkdocs.yml index 75c65d5..a388376 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -45,6 +45,8 @@ nav: - Contracts: contracts.md - Stages: stages.md - Products: products.md + - Depot: depot.md + - Formats: formats.md - Views: views.md - Replay data: replay.md - UI library: ui.md diff --git a/pyproject.toml b/pyproject.toml index 231f7b8..8f9e997 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ docs = ["mkdocs>=1.6,<2", "mkdocs-material>=9.5", "mkdocstrings[python]>=0.29"] remote = ["boto3>=1.35"] # the deck model of music-data: one nnnotes version carries one ournotes-deck release (.github/workflows/deck.yml moves # the pin when ournotes-deck publishes a newer one) -deck = ["ournotes-sim==0.0.9"] +deck = ["ournotes-sim==0.0.12"] [project.scripts] nnnotes = "nnnotes.cli:main" diff --git a/release-notes/v0.3.0.txt b/release-notes/v0.3.0.txt index c6a55fd..955dfec 100644 --- a/release-notes/v0.3.0.txt +++ b/release-notes/v0.3.0.txt @@ -6,11 +6,13 @@ Catalog versions and labels are shared through the remote. Census planning, emit Live2D, room, spot, live, story and UI exports use shared object products. Named site builds publish assets, product manifests and indexes in dependency order, preserve unselected regional topics and restore complete layouts from depot records. +Standalone episode, shader and audio exports also reuse depot tasks. Command records retain accessed results and content. Collection protects recent results and their dependencies for a configurable grace period, and remote verification reads remote-only content without populating the local cache. + Configuration and command interface: - Set [paths] depot and, when needed, [depot] remote. - Use depot plan, depot run, depot verify and depot gc to manage stored work. - Use site build --name NAME -o DIR for a named site and bundle pull -o DIR for source bundle copies. - Product playback uses ournotes-player 0.2.x. Story and UI data use the documented nnnotes.product/1 formats. -- The deck extra pins ournotes-sim 0.0.9. +- The deck extra pins ournotes-sim 0.0.12. See the configuration, depot, products and site references for command options and data contracts. diff --git a/src/nnnotes/cli/__init__.py b/src/nnnotes/cli/__init__.py index 756c1dc..57f058d 100644 --- a/src/nnnotes/cli/__init__.py +++ b/src/nnnotes/cli/__init__.py @@ -67,12 +67,21 @@ def main(argv=None): try: cfg = load_config(args) if args.reads_config else None with depot_lease(args, cfg) as args.lease: - args.func(args, cfg) + if args.lease is not None and not getattr(args, "dry_run", False): + from ..pipeline.depot import command_run, open_depot + + name = args.command + "." + args.func.__name__.removeprefix("cmd_") + with command_run(open_depot(cfg), name): + args.func(args, cfg) + else: + args.func(args, cfg) except ConfigError as e: print(f"nnnotes: {e}", file=sys.stderr) sys.exit(2) except GameApiError as e: sys.exit(f"nnnotes: {e}") + except TimeoutError as e: + sys.exit(f"nnnotes: {e}") def depot_lease(args, cfg): @@ -85,7 +94,7 @@ def depot_lease(args, cfg): from ..source.session import Session session = Session(cfg) - return lease(session.depot_root(), session.downloads_budget()) + return lease(session.depot_root(), None if getattr(args, "dry_run", False) else session.downloads_budget()) if __name__ == "__main__": diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index fdb9915..8dc9e0e 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -116,6 +116,9 @@ def matches(self, stage: str) -> bool: StageSource("sprite.crop", "nnnotes.pipeline.objexport:SpriteCropStage", output=True), StageSource("cri.audio", "nnnotes.pipeline.cristages:AudioStage", output=True), StageSource("cri.movie", "nnnotes.pipeline.cristages:MovieStage", output=True), + StageSource("export.episode", "nnnotes.pipeline.exportfiles:EpisodeStage", output=True, product=True), + StageSource("export.audio", "nnnotes.pipeline.exportfiles:AudioFilesStage", output=True, product=True), + StageSource("export.shader", "nnnotes.pipeline.exportfiles:ShaderFilesStage", output=True), StageSource("live2d.model", "nnnotes.pipeline.live2d:ModelStage", output=True, product=True), StageSource("live2d.cubism", "nnnotes.pipeline.live2d:CubismStage", output=True, product=True), StageSource("room.gltf", "nnnotes.pipeline.spot:RoomStage", output=True, product=True), @@ -692,6 +695,11 @@ def pipeline(self) -> list[str]: def params(self, pipeline: list[str]) -> dict: """{stage: parameters} from the flags; a flag no stage of the pipeline takes is a usage error.""" stages, out = self.installed.stages, {} + for name, flag in (("story", "story_languages"), ("live", "live_languages")): + if name in pipeline and hasattr(self.args, flag): + out.setdefault(name, {})["languages"] = getattr(self.args, flag) or [ + self.cfg.get("catalog", "language") + ] level = getattr(self.args, "png_level", None) if level is not None: if not 0 <= level <= 9: @@ -1336,7 +1344,7 @@ def request_orchestrator(ws, cfg): ) -def run_request(args, cfg, ws: Workspace, expand=None, prepared=None): +def run_request(args, cfg, ws: Workspace, expand=None, prepared=None, *, selection_prefix=()): """Execute a product request and optional expansion without materializing an output layout.""" from ..pipeline import builtins from ..pipeline.downloads import BundleDownloads @@ -1364,7 +1372,9 @@ def run_request(args, cfg, ws: Workspace, expand=None, prepared=None): ) aborted = None try: - run = orch.run(pipeline, params=params, facts=facts, context=ws.context(), selection=ws.selection) + run = orch.run( + pipeline, params=params, facts=facts, context=ws.context(), selection=[*selection_prefix, *ws.selection] + ) except ConfigError as e: run = getattr(e, "run", None) if run is None: @@ -1389,7 +1399,7 @@ def run_request(args, cfg, ws: Workspace, expand=None, prepared=None): more = expand(run) if more is not None: ws.request(*more) - return run_request(args, cfg, ws) + return run_request(args, cfg, ws, selection_prefix=selection_prefix) stage_of = {"bundle": "unity.census", "raw": next((n for n in RAW_STAGES if n in pipeline), "cri.audio")} for p in ws.problems(): if p["code"] == "source.error": diff --git a/src/nnnotes/cli/depot.py b/src/nnnotes/cli/depot.py index 1e3a431..4814695 100644 --- a/src/nnnotes/cli/depot.py +++ b/src/nnnotes/cli/depot.py @@ -12,10 +12,10 @@ def cmd_verify(args, cfg): - from ..pipeline.depot import Depot + from ..pipeline.depot import open_depot workers = usable_cpus() if args.workers is None else args.workers - r = Depot(Session(cfg).depot_root()).verify(quick=args.quick, workers=workers) + r = open_depot(cfg).verify(quick=args.quick, workers=workers) report(args, r) if r["problems"]: sys.exit(assets.EXIT_FAILED) @@ -27,6 +27,8 @@ def cmd_gc(args, cfg): if args.keep is not None and args.keep < 1: usage(args, "--keep must be at least 1") + if args.grace < 0: + usage(args, "--grace must be nonnegative") root = Session(cfg).depot_root() with exclusive(root, args.lease) as alone: if not alone: @@ -37,7 +39,7 @@ def cmd_gc(args, cfg): depot = open_depot(cfg) if args.remote and depot.remote is None: usage(args, "--remote: the depot has no remote (setting depot.remote)") - r = depot.gc(keep=args.keep, dry_run=args.dry_run, remote=args.remote) + r = depot.gc(keep=args.keep, dry_run=args.dry_run, remote=args.remote, grace=args.grace) def lines(r: dict, where: str = "") -> list[str]: return [ @@ -84,6 +86,13 @@ def register(sub) -> None: help="keep the N latest runs of each selection (default: every run; the latest run of a selection is kept)", ) c.add_argument("--dry-run", action="store_true", help="count what would be removed, remove nothing") + c.add_argument( + "--grace", + type=float, + default=86400, + metavar="SECONDS", + help="keep recent results and their inputs and contents in both depots (default: 86400 seconds)", + ) c.add_argument( "--remote", action="store_true", diff --git a/src/nnnotes/cli/export.py b/src/nnnotes/cli/export.py index b288d3d..6c68660 100644 --- a/src/nnnotes/cli/export.py +++ b/src/nnnotes/cli/export.py @@ -4,9 +4,9 @@ from __future__ import annotations +import argparse from pathlib import Path -from ..core.jsonio import write_json from ..formats.encoders import DEFAULT_AUDIO_FORMAT, WEB_AUDIO from ..source.config import ConfigError, use from ..source.session import Session @@ -33,19 +33,74 @@ def cmd_episode(args, cfg): - from ..content import adv + from ..pipeline.exportfiles import EpisodeStage md = Session(cfg).master_dir() known_row(args, md, "MasterAdv", args.adv_id, "episode") - cat = Session(cfg).catalog() - doc = adv.to_json(adv.extract(cat, md, args.adv_id)) - out = Path(args.out) - out.parent.mkdir(parents=True, exist_ok=True) - write_json(out, doc) - r = {"out": str(out), "commands": doc["commandCount"], "resources": len(doc["resources"])} + run, result = _file_product(args, cfg, EpisodeStage(), str(args.adv_id)) + doc = _product_document(run, result, "episode.json") + _place_files(run, result, args.out, single="episode.json") + r = {"out": str(args.out), "commands": doc["commandCount"], "resources": len(doc["resources"])} report(args, r, lambda r: [f"{r['out']} ({r['commands']} commands, {r['resources']} resources)"]) +def _checked_run(args, cfg, ws, prefix): + run, _, aborted = assets.run_request(args, cfg, ws, selection_prefix=[prefix]) + run.write() + if aborted or run.exit_code: + report(args, {"run": run.manifest()["id"], "failures": run.failures(), "aborted": aborted}) + raise SystemExit(assets.EXIT_ABORTED if aborted else assets.EXIT_FAILED) + return run + + +def _file_product(args, cfg, stage, subject): + from ..pipeline import contract + from ..pipeline.stages import Env + + use(cfg) + options = argparse.Namespace(**{**vars(args), "views": "none", "audio_format": getattr(args, "format", None)}) + ws = assets.Workspace(options, cfg, layouts=["files"]) + ws.run_index() + env = Env(ws.depot, {"master": ws.master(), "index": ws.index()}) + held = {loc["primaryKey"] for loc in ws.index()["locations"]} + ws.request({stage.name: [subject]}, sorted(f"key:{key}" for key in stage.keys(subject, env) if key in held)) + run = _checked_run(args, cfg, ws, stage.name) + tid = contract.task_id(stage.name, subject) + return run, run.depot.committed(run.tasks[tid]["key"], tid) + + +def _product_document(run, result, path): + from ..pipeline import contract + + artifact = next(a for a in result["artifacts"] if contract.parse_artifact_id(a["id"])[1] == path) + return contract.loads(run.depot.read(artifact["content"]["sha256"])) + + +def _place_files(run, result, directory, *, single=None): + from ..pipeline import contract, layout + from ..pipeline.depot import write_file + + entries = [] + for artifact in result["artifacts"]: + name = contract.parse_artifact_id(artifact["id"])[1] + if name == "objects" or single is not None and name != single: + continue + content = artifact["content"] + entries.append({"path": name, "id": artifact["id"], "sha256": content["sha256"], "size": content["size"]}) + out = Path(directory) + if single is not None: + if len(entries) != 1: + raise ValueError(f"export has no single {single} file") + entry = {**entries[0], "path": out.name} + write_file(out, run.depot.read(entry["sha256"])) + manifest = contract.layout_doc("files", {}, [entry]) + else: + manifest = layout.materialize( + out, "files", {}, entries, layout.from_depot(run.depot), link="copy", fail_fast=True + )["manifest"] + run.write({"files": run.depot.put(contract.encode(manifest))}) + + def prepare_story(args, cfg): from ..pipeline import contract from ..pipeline import story as sp @@ -240,36 +295,94 @@ def cmd_room(args, cfg): def cmd_shader(args, cfg): - from ..formats import shader + from ..pipeline import contract + from ..pipeline.census import CensusStage + from ..pipeline.contract import Input, Task + from ..pipeline.depot import open_depot + from ..pipeline.exportfiles import ShaderFilesStage + from ..pipeline.link import ScriptsStage + from ..pipeline.objexport import ExportStage + from ..pipeline.orchestrate import Orchestrator + from ..pipeline.stages import Stage, execute cat = Session(cfg).catalog() if args.key: known_keys(args, cat, [args.key]) - bundles = cat.fetch_key(args.key) + paths = cat.fetch_key(args.key) else: if cat.apk is None: cfg.require_path("paths", "apk") - bundles = [] + paths = [] for s in args.apk_bundle: try: - bundles.append(cat.apk_bundle(s)) + paths.append(cat.apk_bundle(s)) except KeyError as e: # none or several bundles match usage(args, f"--apk-bundle {e.args[0]}") - r = shader.dump(bundles, Path(args.out)) + use(cfg) + depot, bundles, names = open_depot(cfg), {}, {} + for path in sorted(set(paths)): + subject = contract.stable_bundle_name(path.name) + sha, size = depot.identify(path) + source = Input("bundle", sha, size, path.name, ({"kind": "file", "path": str(path.resolve())},)) + if subject in bundles and bundles[subject].sha256 != sha: + usage(args, f"multiple bundle versions have the stable name {subject}") + bundles[subject], names[subject] = source, path.name + stages: dict[str, Stage] = {s.name: s for s in (CensusStage(), ScriptsStage(), ExportStage())} + run = Orchestrator(depot, stages, workers=0).run( + list(stages), + facts={"bundles": bundles, "catalogs": {}, "selected": sorted(bundles), "classes": ["Shader"]}, + selection=["export.shader", *sorted(bundles)], + ) + run.write() + if run.exit_code: + report(args, {"run": run.manifest()["id"], "failures": run.failures()}) + raise SystemExit(assets.EXIT_FAILED) + records = [] + for tid, result in run.results(): + stage, subject = contract.parse_task_id(tid) + if stage == "unity.export": + bundle = subject.split(":", 1)[0] + records.append( + { + "source": names.get(bundle, bundle), + "items": result["items"], + "artifacts": {a["id"]: a for a in result["artifacts"]}, + } + ) + data, stage = contract.encode(records), ShaderFilesStage() + task = Task( + stage.name, + stage.version, + "selected", + inputs=(Input("records", depot.put(data), len(data), None, ({"kind": "depot"},)),), + ) + executed = execute(task, depot, {stage.name: stage}) + run.tasks[task.id] = {"id": task.id, "key": task.key, "status": executed.status} + run.stages[stage.name] = stage.version + result = depot.committed(task.key, task.id) + run.write() + _place_files(run, result, args.out) + index = _product_document(run, result, "shaders.json") + r = { + "count": len(index), + "names": sorted(row["name"] for row in index), + "variants": sum(len(row["variants"]) for row in index), + } report(args, r, lambda r: [f"{r['count']} shaders, {r['variants']} variants", *(f" {n}" for n in r["names"])]) def cmd_audio(args, cfg): - from ..formats import cri + from ..pipeline.exportfiles import AudioFilesStage - flac = flac_options(args) + flac_options(args) cfg.require_path("paths", "apk") # the HCA keycode is read from the APK cat = Session(cfg).catalog() if not cat.has(f"Cri/Sound/{args.cue_sheet}"): usage(args, f"cue sheet {args.cue_sheet}: no key Cri/Sound/{args.cue_sheet} in the catalog") - out = Path(args.out) - cues = cri.decode(cat, args.cue_sheet, out, fmt=args.format, **flac) - report(args, {"out": str(out), "cues": len(cues)}, lambda r: [f"{r['out']} ({r['cues']} cues)"]) + run, result = _file_product(args, cfg, AudioFilesStage(), args.cue_sheet) + cues = _product_document(run, result, "cues.json") + _place_files(run, result, args.out) + report(args, {"out": str(args.out), "cues": len(cues)}, lambda r: [f"{r['out']} ({r['cues']} cues)"]) def cmd_ui(args, cfg): diff --git a/src/nnnotes/cli/site.py b/src/nnnotes/cli/site.py index 3f75ca3..a49a6a3 100644 --- a/src/nnnotes/cli/site.py +++ b/src/nnnotes/cli/site.py @@ -33,7 +33,7 @@ def product_entries(args, cfg, ws, expand=None): - run, _, aborted = assets.run_request(args, cfg, ws, expand=expand) + run, _, aborted = assets.run_request(args, cfg, ws, expand=expand, selection_prefix=["site-product:" + args.name]) saved = run.write() if aborted or run.exit_code: report(args, {"run": saved["id"], "tasks": saved["summary"], "failures": run.failures(), "aborted": aborted}) @@ -140,8 +140,9 @@ def cmd_build(args, cfg): facts = { **sitefacts.chart(live, music_doc, lang), "audio": not args.no_audio, - "audioFormat": args.format, - "flows": ["normal"], + "audioFormat": None if args.no_audio else args.format, + "flows": ["direct"], + "quality": 1, } snapshot = builder.add( entries, diff --git a/src/nnnotes/content/storyepisode.py b/src/nnnotes/content/storyepisode.py index 60237c6..a2d6a8f 100644 --- a/src/nnnotes/content/storyepisode.py +++ b/src/nnnotes/content/storyepisode.py @@ -108,7 +108,7 @@ def _always(address: str) -> bool: return True -def episode(objects: Objects, record: dict) -> AdvExtract: +def episode(objects: Objects, record: dict, *, has=_always) -> AdvExtract: """The episode of `record` (episode_record) from the object table: pass the result to `adv.to_json`.""" row = record["MasterAdv"][0] base = episode_key(row["_advEpisodeAsset"]) @@ -125,7 +125,7 @@ def chat(chat_id: int) -> dict: row, coll, shards, - _always, + has, chat, lambda: _default_transition(objects), _title(row, record["MasterText"]), diff --git a/src/nnnotes/core/downloads.py b/src/nnnotes/core/downloads.py index f1a79a1..b1f5150 100644 --- a/src/nnnotes/core/downloads.py +++ b/src/nnnotes/core/downloads.py @@ -25,6 +25,7 @@ import re import secrets import socket +import sys import time from collections.abc import Generator, Iterable, Iterator from contextlib import contextmanager @@ -204,9 +205,10 @@ def _record() -> str: return json.dumps({"host": socket.gethostname(), "pid": os.getpid(), "started": time.time()}) -def _wait_alone(leases: Path) -> None: - """Return once no live command holds `.exclusive` (a stale one is removed).""" +def _wait_alone(leases: Path, timeout: float) -> None: + """Wait at most `timeout` seconds for `.exclusive`; leave a live owner's lock intact.""" p, host = leases / EXCLUSIVE, socket.gethostname() + deadline, announced = time.monotonic() + timeout, False while True: try: data, mtime = p.read_bytes(), p.stat().st_mtime @@ -219,6 +221,11 @@ def _wait_alone(leases: Path) -> None: if not live: p.unlink(missing_ok=True) return + if time.monotonic() >= deadline: + raise TimeoutError(f"depot exclusive lease {p} is still held; retry after its owner finishes") + if not announced: + print(f"waiting for depot exclusive lease {p} (at most {timeout:g} seconds)", file=sys.stderr, flush=True) + announced = True time.sleep(0.1) @@ -241,7 +248,7 @@ def exclusive(root: Path, own: Path | None) -> Generator[bool]: @contextmanager -def lease(root: Path, budget: int | None) -> Generator[Path]: +def lease(root: Path, budget: int | None, *, wait_timeout: float = 30) -> Generator[Path]: """Hold a lease on the depot `root` while the block runs; at its end, when no other lease is live, trim the downloads to `budget` (None: keep them all).""" d = root / "leases" @@ -249,7 +256,7 @@ def lease(root: Path, budget: int | None) -> Generator[Path]: own = d / f"{socket.gethostname()}-{os.getpid()}-{secrets.token_hex(4)}.json" own.write_text(_record(), encoding="utf-8") try: - _wait_alone(d) + _wait_alone(d, wait_timeout) yield own finally: try: diff --git a/src/nnnotes/pipeline/catalogdb.py b/src/nnnotes/pipeline/catalogdb.py index 01b37b5..bdc5a26 100644 --- a/src/nnnotes/pipeline/catalogdb.py +++ b/src/nnnotes/pipeline/catalogdb.py @@ -435,6 +435,7 @@ def _load(self) -> list[dict]: if doc.get("schema") != CATALOGS: raise ValueError(f"{self.root / 'index.json'} is not a {CATALOGS} document") versions = {v["id"]: v for v in doc["versions"]} + remote_versions = {} if self.remote is not None: for key, _, _ in self.remote.list("catalogs/versions/"): if not key.endswith(".json"): @@ -443,6 +444,7 @@ def _load(self) -> list[dict]: if data is None: continue v = contract.loads(data) + remote_versions[v["id"]] = data expected = version_id(v["remote"]["sha256"], v["apk"]["sha256"] if v["apk"] else None, v.get("source")) if v["id"] != expected or key != f"catalogs/versions/{expected}.json": raise ValueError(f"catalog version identity differs: {key}") @@ -455,6 +457,16 @@ def _load(self) -> list[dict]: versions[expected] = v merged = sorted(versions.values(), key=lambda v: (v["seq"], v["id"])) if self.remote is not None: + for v in merged: + encoded = contract.encode(v) + if remote_versions.get(v["id"]) != encoded: + for name in CATALOG_NAMES: + record = v[name] + if record is not None: + key = f"catalogs/{record['sha256']}.bin" + if self.remote.size(key) != record["size"]: + self.remote.put(key, self._bytes(record)) + self.remote.put(f"catalogs/versions/{v['id']}.json", encoded) self._save(merged) return merged diff --git a/src/nnnotes/pipeline/depot.py b/src/nnnotes/pipeline/depot.py index 5d7dfc9..2db0974 100644 --- a/src/nnnotes/pipeline/depot.py +++ b/src/nnnotes/pipeline/depot.py @@ -39,6 +39,8 @@ import os import threading import time +from contextlib import contextmanager +from contextvars import ContextVar from pathlib import Path from ..core.cache import temp_path @@ -50,6 +52,58 @@ INPUT_KINDS = ("bundles", "raw", "files") REMOTE_GRACE = 24 * 3600 # `gc(remote=True)` keeps results and contents of the remote younger than this (seconds) DONE = ("ran", "hit") # the statuses of a run's tasks whose results are committed +_COMMAND: ContextVar[dict | None] = ContextVar("depot_command", default=None) + + +@contextmanager +def command_run(depot, name): + """Record a command and retain the content and results it read or stored, including on failure.""" + from .orchestrate import Run + + observed = {"root": depot.root.resolve(), "contents": {}, "tasks": {}, "stages": {}} + token = _COMMAND.set(observed) + outcome = "complete" + failed = False + try: + yield + except BaseException: + outcome, failed = "failed", True + raise + finally: + _COMMAND.reset(token) + try: + run = Run(depot, {"command": name, "outcome": outcome}, ["command:" + name], {}, observed["stages"]) + run.tasks.update(observed["tasks"]) + layouts = {} + if observed["contents"]: + entries = [ + { + "path": sha, + "id": contract.artifact_id(contract.task_id("command", name), sha), + "sha256": sha, + "size": size, + } + for sha, size in sorted(observed["contents"].items()) + ] + layouts["command"] = depot.put(contract.encode(contract.layout_doc("cas", {}, entries))) + run.write(layouts) + except Exception: + if not failed: + raise + + +def _observe(depot, *, sha=None, size=None, result=None): + observed = _COMMAND.get() + if observed is None or depot.root.resolve() != observed["root"]: + return + if sha is not None: + observed["contents"][sha] = size + if result is not None: + parts = result["keyParts"] + stage = parts["stage"] + tid = contract.task_id(stage["name"], parts["subject"]) + observed["tasks"][tid] = {"id": tid, "key": result["key"], "status": "hit"} + observed["stages"][stage["name"]] = stage["version"] class InputMissing(FileNotFoundError): @@ -168,7 +222,6 @@ def __init__(self, root, *, fetch=None, remote: Remote | None = None): self.written = self.reused = 0 self._have: set[str] = set() # content ids this depot object wrote or found, here or on the remote self._sizes: dict[str, int] = {} # remote key -> size, of the objects known to be on the remote - self._listed: set[str] = set() # task keys whose results the remote holds (the latest runs read from it) self._hashed: dict[str, dict] = {} # path -> its record (inputs/paths) self._names: dict[tuple[str, str], tuple[str, int]] = {} # (kind, name) -> (sha256, size) self._docs: dict[str, dict] = {} # the documents document() decoded last @@ -200,6 +253,7 @@ def _on_remote(self, key: str, size: int | None = None) -> bool: def _push(self, sha: str, size: int) -> None: """Copy a content of this depot to the remote unless the remote holds it.""" + _observe(self, sha=sha, size=size) if self.remote is not None: p = self.path(sha) key = self._key(p) @@ -258,6 +312,8 @@ def local(self, sha: str) -> Path: if contract.sha256(data) != sha: raise RuntimeError(f"content {sha} of the remote differs from its id") _write(p, data, lambda q: q.stat().st_size == len(data), read_only=True) + if _COMMAND.get() is not None and p.is_file(): + _observe(self, sha=sha, size=p.stat().st_size) return p def put(self, data) -> str: @@ -367,7 +423,10 @@ def result_path(self, key: str) -> Path: def result(self, key: str) -> dict | None: """The committed result of a task key, or None.""" data = self._fetch_record(self.result_path(key)) - return None if data is None else contract.loads(data) + doc = None if data is None else contract.loads(data) + if doc is not None: + _observe(self, result=doc) + return doc def committed(self, key: str, task: str) -> dict: """The committed result of a task key that has one (a FileNotFoundError naming `task` when it is missing).""" @@ -378,7 +437,7 @@ def committed(self, key: str, task: str) -> dict: def has_result(self, key: str) -> bool: p = self.result_path(key) - return p.is_file() or key in self._listed or self._on_remote(self._key(p)) + return p.is_file() or self._on_remote(self._key(p)) def commit(self, doc: dict) -> bool: """Write a result entry, after checking that the bytes of every artifact it names are stored. Write-once: an @@ -410,7 +469,9 @@ def same(p: Path) -> bool: elif old != data: raise DepotConflict(f"result {doc['key']} differs from the one in the depot's remote") self._sizes[key] = len(data) - return False if here else _write(dst, data, same) + written = False if here else _write(dst, data, same) + _observe(self, result=doc) + return written # ---------------------------------------------------------------- inputs def identify(self, path, kind: str = "files", name: str | None = None) -> tuple[str, int]: @@ -539,6 +600,17 @@ def content(p: Path): sizes[c["sha256"]] = self.path(c["sha256"]).stat().st_size except (OSError, ValueError): sizes[c["sha256"]] = None + sha = c["sha256"] + if self.remote is not None and contract.is_sha256(sha): + remote_key = f"cas/sha256/{sha[:2]}/{sha}" + if quick: + sizes[sha] = self.remote.size(remote_key) + else: + data = self.remote.get(remote_key) + if data is not None: + sizes[sha] = len(data) + if contract.sha256(data) != sha: + problems.append((p, f"artifact {a['id']}: remote content differs from its id")) if sizes[c["sha256"]] != c["size"]: problems.append( ( @@ -561,6 +633,16 @@ def run_path(self, run_id: str) -> Path: def write_run(self, doc: dict, log: list[dict] | None = None) -> Path: """Write a run manifest (and the log of this execution), and make it the latest run of its selection.""" + if self.remote is not None: + # A shared run is published only after its completed results and their stored inputs. + for task in doc.get("tasks") or (): + if task.get("key") and task.get("status") in DONE: + self.commit(self.committed(task["key"], task["id"])) + _, contents = _reach([doc], self.result, lambda sha: contract.loads(self.read(sha))) + for sha in contents: + path = self.path(sha) + if path.is_file(): + self._push(sha, path.stat().st_size) if doc.get("schema") != contract.RUN: raise ValueError("not a run document") p = self.run_path(doc["id"]) @@ -583,19 +665,14 @@ def run(self, run_id: str) -> dict | None: return None if data is None else contract.loads(data) def latest_run(self, selection) -> dict | None: - """The latest run manifest written for `selection` (by any machine of the remote), or None. The results of - the tasks of a run read from the remote are known to be on the remote (gc keeps them).""" + """The latest run manifest written for `selection` (by any machine of the remote), or None.""" pointer = self._selection_path(selection) data = self.remote.get(self._key(pointer)) if self.remote is not None else None - shared = data is not None if data is None: data = _read(pointer) if data is None: return None - doc = self.run(data.decode("ascii").strip()) - if shared and doc is not None: - self._listed.update(t["key"] for t in doc.get("tasks") or () if t.get("key") and t.get("status") in DONE) - return doc + return self.run(data.decode("ascii").strip()) def write_cost(self, key: str, cost: dict) -> None: write_file(self.root / "costs" / f"{key}.json", contract.encode(cost)) @@ -618,14 +695,17 @@ def gc( remote's runs, results and contents too, by the runs of the remote; its results and contents younger than `grace` seconds stay (a run of another machine may be writing them). `dry_run`: count only. -> {"runs", "results", "contents": {"kept", "removed"}, "bytesRemoved"}, and "remote": the same.""" - out = self._gc_local(keep, dry_run) + if grace < 0: + raise ValueError("gc grace must be nonnegative") + out = self._gc_local(keep, dry_run, grace) if remote: if self.remote is None: raise ValueError("the depot has no remote") out["remote"] = self._gc_remote(self.remote, keep, dry_run, grace) return out - def _gc_local(self, keep: int | None, dry_run: bool) -> dict: + def _gc_local(self, keep: int | None, dry_run: bool, grace: float) -> dict: + cutoff = time.time() - grace runs_dir = self.root / "runs" latest = set() for p in (runs_dir / "selections").glob("*"): @@ -649,14 +729,17 @@ def document(sha: str) -> dict | None: p = self.path(sha) return contract.loads(p.read_bytes()) if p.is_file() else None - keys, shas = _reach([doc for _, _, doc in kept], result, document) + results = sorted((self.root / "ac").glob("*/*.json")) + recent = [p.name.removesuffix(".json") for p in results if p.stat().st_mtime > cutoff] + roots = [doc for _, _, doc in kept] + [{"tasks": [{"key": key} for key in recent]}] + keys, shas = _reach(roots, result, document) out: dict = {"runs": {"kept": len(kept), "removed": len(dropped)}} for _, p, _ in dropped: if not dry_run: p.unlink(missing_ok=True) p.with_name(p.name[: -len(".json")] + ".log.jsonl").unlink(missing_ok=True) n_kept = n_removed = 0 - for p in sorted((self.root / "ac").glob("*/*.json")): + for p in results: key = p.name[: -len(".json")] if key in keys: n_kept += 1 @@ -668,7 +751,7 @@ def document(sha: str) -> dict | None: out["results"] = {"kept": n_kept, "removed": n_removed} n_kept = n_removed = removed_bytes = 0 for p in sorted((self.root / "cas" / "sha256").glob("*/*")): - if p.name in shas: + if p.name in shas or p.stat().st_mtime > cutoff: n_kept += 1 continue n_removed += 1 @@ -692,10 +775,17 @@ def _gc_remote(self, remote: Remote, keep: int | None, dry_run: bool, grace: flo elif key.endswith(".json"): runs.append((mtime, key, contract.loads(data))) kept, dropped = _kept_runs(runs, latest, keep) + results = list(remote.list("ac/")) + recent = [key.rsplit("/", 1)[-1].removesuffix(".json") for key, _, mtime in results if now - mtime < grace] + + def document(key): + data = remote.get(key) + return None if data is None else contract.loads(data) + keys, shas = _reach( - [doc for _, _, doc in kept], - self.result, - lambda sha: contract.loads(self.read(sha)) if self.has(sha) else None, + [doc for _, _, doc in kept] + [{"tasks": [{"key": key} for key in recent]}], + lambda key: document(f"ac/{key[:2]}/{key}.json"), + lambda sha: document(f"cas/sha256/{sha[:2]}/{sha}"), ) out: dict = {"runs": {"kept": len(kept), "removed": len(dropped)}} gone = [key for _, key, _ in dropped] @@ -703,7 +793,7 @@ def _gc_remote(self, remote: Remote, keep: int | None, dry_run: bool, grace: flo removed_bytes = 0 for part, prefix in (("results", "ac/"), ("contents", "cas/sha256/")): n_kept = n_removed = 0 - for key, size, mtime in remote.list(prefix): + for key, size, mtime in results if part == "results" else remote.list(prefix): name = key.rsplit("/", 1)[-1] reached = name[: -len(".json")] in keys if part == "results" else name in shas if reached or now - mtime < grace: @@ -719,7 +809,6 @@ def _gc_remote(self, remote: Remote, keep: int | None, dry_run: bool, grace: flo for key in gone: self._sizes.pop(key, None) self._have.clear() - self._listed.clear() out.update(counts) out["bytesRemoved"] = removed_bytes return out diff --git a/src/nnnotes/pipeline/exportfiles.py b/src/nnnotes/pipeline/exportfiles.py new file mode 100644 index 0000000..021970a --- /dev/null +++ b/src/nnnotes/pipeline/exportfiles.py @@ -0,0 +1,141 @@ +"""Standalone file exports assembled from the same object and audio products used by sites.""" + +from __future__ import annotations + +import subprocess +import tempfile +from pathlib import Path + +from ..content import adv, storyepisode +from ..formats.encoders import encode_args +from ..formats.shader import _safe as safe_name +from . import contract +from .assemble import MASTER_INPUT, Assembly +from .cristages import _tool, tool_id +from .stages import Output, Stage +from .story import SoundStage, StoryStage +from .views import master_rows + + +class EpisodeStage(StoryStage): + """The episode document alone, without its playback products.""" + + name, version = "export.episode", 1 + KIND = "Episode" + MASTER = ("MasterAdv",) + + def record(self, subject, env): + tables = master_rows(env, self.MASTER) + row = next(r for r in tables["MasterAdv"] if r["_id"] == int(subject)) + tables["MasterText"] = master_rows(env, ("MasterText",))["MasterText"] if row["_titleTextId"] else [] + tables["MasterAdvChat"] = ( + master_rows(env, ("MasterAdvChat",))["MasterAdvChat"] + if (Path(env.fact("master")) / "MasterAdvChat.json").is_file() + else [] + ) + record = storyepisode.episode_record(tables, int(subject)) + record["catalogKeys"] = sorted( + {row["primaryKey"] for row in (env.facts.get("index") or {}).get("locations", [])} + ) + return record + + def assemble(self, task, objects): + record = contract.loads(objects.depot.input_bytes(task.input(MASTER_INPUT))) + keys = set(record["catalogKeys"]) + episode = adv.to_json(storyepisode.episode(objects, record, has=keys.__contains__)) + return Assembly({"episode.json": (contract.encode(episode), "json", "story.episode")}, set()) + + +class AudioFilesStage(SoundStage): + """Every decoded stream of a cue sheet, with its cue and stream indexes.""" + + name, version = "export.audio", 1 + PARAMS = {"audio": "flac"} + + def normalize(self, params): + params = Stage.normalize(self, params) + if params["audio"] not in {"flac", "ogg", "wav"}: + raise ValueError("export.audio format must be flac, ogg or wav") + return params + + def uses(self, subject, env): + return {} if env.params["audio"] == "flac" else {"audio.encode": tool_id("ffmpeg", _tool("ffmpeg"))} + + def assemble(self, task, objects): + fmt = task.params["audio"] + sheet = objects.sheet(f"Cri/Sound/{task.subject}") + files, renamed = {}, {} + encoder = _tool("ffmpeg") if fmt != "flac" else None + if encoder and task.atoms != {"audio.encode": tool_id("ffmpeg", encoder)}: + raise contract.IncompatibleTask("the audio encoder differs from the task's encoder") + for stream in sheet.streams(): + source = stream["file"] + name = str(Path(source).with_suffix("." + fmt)) + renamed[source] = name + if encoder is None: + body = sheet.file(source) + else: + codec = ["-c:a", "libvorbis", "-q:a", "5"] if fmt == "ogg" else ["-c:a", "pcm_s16le"] + with tempfile.TemporaryDirectory(prefix="nnnotes-audio-") as temp: + src, dst = Path(temp) / "input.flac", Path(temp) / ("output." + fmt) + src.write_bytes(sheet.bytes(source)) + result = subprocess.run([encoder, *encode_args(src, dst, codec)], capture_output=True, text=True) + if result.returncode: + raise RuntimeError(f"audio encoding failed: {result.stderr[-2000:]}") + body = dst.read_bytes() + files[name] = (body, fmt, "audio.stream") + for name in ("cues.json", "streams.json"): + doc = contract.loads(sheet.bytes(name)) + for entry in doc.values() if isinstance(doc, dict) else doc: + entry["file"] = renamed[entry["file"]] + files[name] = (contract.encode(doc), "json", "cri.cues" if name == "cues.json" else "cri.streams") + return Assembly(files, set()) + + +class ShaderFilesStage(Stage): + """A shader directory from converted Shader object artifacts, without reading bundles again.""" + + name, version = "export.shader", 1 + + def subjects(self, env): + return [] + + def run(self, task, depot): + records = contract.loads(depot.input_bytes(task.input("records"))) + files, index, names = {}, [], {} + + def put(path, content): + if path in files and files[path] != content: + raise ValueError(f"shader files have different contents at {path}") + files[path] = content + + for record in records: + artifacts = record["artifacts"] + for item in record["items"]: + if item.get("class") != "Shader" or item["status"] != "exported": + continue + arts = {contract.parse_artifact_id(a)[1]: artifacts[a] for a in item["artifacts"]} + variant = contract.loads(depot.read(arts["index"]["content"]["sha256"])) + name = variant["name"] + signature = contract.digest({role: art["content"] for role, art in arts.items() if role != "typetree"}) + if name in names: + if names[name] != signature: + raise ValueError(f"shader {name}: differing contents share a name") + continue + names[name] = signature + parsed = safe_name(name) + ".json" + put(parsed, arts["json"]["content"]) + variants = [] + for entry in variant["variants"]: + put(entry["file"], arts[entry["role"]]["content"]) + variants.append({k: v for k, v in entry.items() if k != "role"}) + index.append({"name": name, "source": record["source"], "parsed": parsed, "variants": variants}) + files["shaders.json"] = depot.add(contract.encode(index), "json") + return Output( + [ + contract.artifact( + contract.artifact_id(task.id, path), content, contract.provenance(task), {"kind": self.name} + ) + for path, content in sorted(files.items()) + ] + ) diff --git a/src/nnnotes/pipeline/live.py b/src/nnnotes/pipeline/live.py index cfd7e7d..9fabe76 100644 --- a/src/nnnotes/pipeline/live.py +++ b/src/nnnotes/pipeline/live.py @@ -455,10 +455,16 @@ class LiveStage(ProductStage): """live: a music's chart at a difficulty and the products it plays with (module documentation).""" name = LIVE - version = 2 + version = 3 + PARAMS = {"languages": []} KIND = "Live" MASTER = tuple(sorted({*live.LIVE_TABLES, *livenotes.NOTES_TABLES})) + def normalize(self, params): + params = super().normalize(params) + params["languages"] = sorted({languages.check(code) for code in params["languages"]}) + return params + def select(self, keys: list[str], env) -> list[str]: if env.facts.get("master") is None: return [] @@ -502,7 +508,9 @@ def more_inputs(self, subject: str, env) -> list[Input]: parts = self.parts(subject, env) paths = {part: PART_STAGES[name]().manifest(parts[name][0]) for part, name in PARTS.items()} music_id, difficulty, _ = live.parse_subject(subject) - paths["ui"] = {code: UiStage().manifest(f"{music_id}/{difficulty}/{code}") for code in languages.LANGUAGES} + paths["ui"] = { + code: UiStage().manifest(f"{music_id}/{difficulty}/{code}") for code in env.params.get("languages", []) + } return [master_input(env, {"live": self.live_record(subject, env), "parts": paths})] def manifest(self, subject: str) -> str: diff --git a/src/nnnotes/pipeline/siteindex.py b/src/nnnotes/pipeline/siteindex.py index f14e558..8b4180f 100644 --- a/src/nnnotes/pipeline/siteindex.py +++ b/src/nnnotes/pipeline/siteindex.py @@ -13,7 +13,7 @@ class IndexStage(Stage): - name, version = INDEX, 2 + name, version = INDEX, 3 def subjects(self, env): return sorted(env.facts.get(self.name, {})) @@ -36,7 +36,7 @@ def run(self, task, depot): aliases = {} for (kind, tid), versions in sorted(groups.items()): folder, _ = KINDS[kind] - if kind == "model" and len(versions) > 1: + if kind == "model" and len({group["topic"]["manifest"] for group in versions.values()}) > 1: raise ValueError(f"model {tid}: differing contents require separate sites") for index, group in enumerate(versions.values()): topic, rs = group["topic"], group["regions"] diff --git a/src/nnnotes/pipeline/siteproducts.py b/src/nnnotes/pipeline/siteproducts.py index a5f88a9..56515d4 100644 --- a/src/nnnotes/pipeline/siteproducts.py +++ b/src/nnnotes/pipeline/siteproducts.py @@ -96,7 +96,7 @@ def rewrite(value): class ManifestStage(_DocumentStage): """Apply transformed file entries while preserving product and object identity.""" - name, version = MANIFEST, 1 + name, version = MANIFEST, 2 def transform(self, source, spec): if source.get("schema") != contract.PRODUCT: @@ -116,6 +116,10 @@ def transform(self, source, spec): doc["variants"] = spec["variants"] if spec.get("options"): doc["options"] = spec["options"] + metadata = spec.get("metadata", {}) + if set(metadata) - {"musicId", "difficulty", "audio", "audioFormat", "flows", "quality", "regions", "chart"}: + raise ValueError("unsupported site manifest metadata") + doc.update(metadata) return doc diff --git a/src/nnnotes/pipeline/story.py b/src/nnnotes/pipeline/story.py index 858b0fb..1c65123 100644 --- a/src/nnnotes/pipeline/story.py +++ b/src/nnnotes/pipeline/story.py @@ -537,7 +537,9 @@ def _adv_id(subject: str) -> int: return int(subject) -def part_paths(parts: dict, adv_id: int, *, overlay: bool = False, host: dict | None = None) -> dict: +def part_paths( + parts: dict, adv_id: int, *, requested_languages=(), overlay: bool = False, host: dict | None = None +) -> dict: """story.json `parts`: the manifest paths of the products of storyepisode.parts `parts`.""" paths = { "scene": SceneStage().manifest(DEFAULT), @@ -547,11 +549,11 @@ def part_paths(parts: dict, adv_id: int, *, overlay: bool = False, host: dict | "videos": {v: VideoStage().manifest(v) for v in parts["videos"]}, "chat": ChatStage().manifest(DEFAULT) if parts["chat"] else None, "crilips": CriLipsStage().manifest(DEFAULT) if parts["crilips"] else None, - "ui": {lang: UiStage().manifest(lang) for lang in languages.LANGUAGES}, - "fonts": {lang: FontsStage().manifest(f"{adv_id}/{lang}") for lang in languages.LANGUAGES}, + "ui": {lang: UiStage().manifest(lang) for lang in requested_languages}, + "fonts": {lang: FontsStage().manifest(f"{adv_id}/{lang}") for lang in requested_languages}, } if overlay: - paths["simpleui"] = {lang: SimpleUiStage().manifest(lang) for lang in languages.LANGUAGES} + paths["simpleui"] = {lang: SimpleUiStage().manifest(lang) for lang in requested_languages} if host is not None: paths["host"] = {host["subject"]: HostStage().manifest(host["subject"])} if host["kind"] == "home": @@ -604,10 +606,16 @@ class StoryStage(ProductStage): """story: an ADV episode -> episode.json and story.json (module documentation).""" name = STORY - version = 4 + version = 5 + PARAMS = {"languages": []} KIND = "Story" MASTER = tuple(sorted({*storyepisode.TABLES, *storyhostproduct.TABLES})) + def normalize(self, params): + params = super().normalize(params) + params["languages"] = sorted({_language(code) for code in params["languages"]}) + return params + def select(self, keys: list[str], env) -> list[str]: if env.facts.get("master") is None: return [] @@ -641,6 +649,7 @@ def assemble(self, task, objects) -> Assembly: paths = part_paths( storyepisode.parts(episode), _adv_id(task.subject), + requested_languages=task.params["languages"], overlay=record["MasterAdv"][0].get("_playbackMode") == 1, host=record.get("host"), ) diff --git a/src/nnnotes/schemas/product.schema.json b/src/nnnotes/schemas/product.schema.json index 2c510f9..d29d9b2 100644 --- a/src/nnnotes/schemas/product.schema.json +++ b/src/nnnotes/schemas/product.schema.json @@ -101,6 +101,47 @@ }, "audio": { "type": "boolean" + }, + "musicId": { + "type": "integer", + "minimum": 1 + }, + "difficulty": { + "type": "string", + "pattern": "^[a-z]+$" + }, + "chart": { + "$ref": "#/$defs/chart" + }, + "regions": { + "type": "array", + "items": { + "$ref": "#/$defs/regionId" + }, + "minItems": 1 + }, + "audioFormat": { + "enum": [ + "aac", + "flac", + null + ] + }, + "flows": { + "type": "array", + "items": { + "enum": [ + "direct" + ] + }, + "minItems": 1 + }, + "quality": { + "enum": [ + 0, + 1, + 2 + ] } }, "$defs": { @@ -224,6 +265,82 @@ "$ref": "#/$defs/size" } } + }, + "regionId": { + "type": "string", + "pattern": "^[a-z0-9_-]+$" + }, + "chart": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "language": { + "$ref": "#/$defs/language" + }, + "titles": { + "type": "object", + "propertyNames": { + "minLength": 1 + }, + "additionalProperties": { + "type": "string" + } + }, + "bands": { + "type": "array", + "items": { + "type": "string" + } + }, + "bandNames": { + "type": "object", + "propertyNames": { + "minLength": 1 + }, + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "bandIds": { + "type": "array", + "items": { + "type": "integer" + } + }, + "stageBand": { + "type": "integer" + }, + "level": { + "type": "integer" + }, + "displayLevel": { + "type": "number" + }, + "notes": { + "type": "integer", + "minimum": 0 + }, + "fullComboCount": { + "type": "integer", + "minimum": 0 + }, + "durationMs": { + "type": "integer", + "minimum": 0 + }, + "sortOrder": { + "type": "integer" + } + } + }, + "language": { + "type": "string", + "minLength": 1 } }, "$id": "urn:nnnotes:schema:product", diff --git a/src/nnnotes/schemas/story-ui.schema.json b/src/nnnotes/schemas/story-ui.schema.json index 56e3d0c..f25456a 100644 --- a/src/nnnotes/schemas/story-ui.schema.json +++ b/src/nnnotes/schemas/story-ui.schema.json @@ -17,7 +17,8 @@ "texts", "textStyle", "bindings", - "shaders" + "shaders", + "chatStatusTexts" ], "properties": { "format": { @@ -276,6 +277,12 @@ "type": "string", "pattern": "^.+:-?[0-9]+$" } + }, + "chatStatusTexts": { + "type": "object", + "additionalProperties": { + "type": "string" + } } }, "$id": "urn:nnnotes:schema:story-ui", diff --git a/src/nnnotes/site/catalog.py b/src/nnnotes/site/catalog.py index d58fe03..019a5b1 100644 --- a/src/nnnotes/site/catalog.py +++ b/src/nnnotes/site/catalog.py @@ -38,8 +38,11 @@ def merge(old, name, language, regions, topics, replace=()): if (t["region"], t["kind"]) not in replace } known.update({(t["region"], t["kind"], t["id"]): t for t in topics}) - rs = {r["id"]: r for r in (old or {}).get("regions", [])} - rs.update({r["id"]: r for r in regions}) + rs = {r["id"]: r for r in regions} + retained = {t["region"] for t in known.values()} + for region in (old or {}).get("regions", []): + if region["id"] in retained: + rs.setdefault(region["id"], region) return { "format": siteindex.CATALOG, "name": name, @@ -107,8 +110,23 @@ def product(path): for path in aliases.values(): product(path) + charts = json.loads(depot.read(output["charts.json"]["sha256"]))["charts"] + chart_entries = {entry["manifest"]: entry for entry in charts} for path, target in aliases.items(): manifest = json.loads(depot.read(all_entries[target]["sha256"])) + metadata = {} + if path in chart_entries: + entry = chart_entries[path] + metadata = { + key: entry[key] + for key in ("musicId", "difficulty", "audio", "audioFormat", "flows", "quality", "regions") + if key in entry + } + metadata["chart"] = { + key: value + for key, value in entry.items() + if key not in {"id", "manifest", "regions", "audio", "audioFormat", "flows", "quality", "bytes"} + } art = runner._document( siteproducts.ManifestStage(), contract.encode(manifest), @@ -116,6 +134,7 @@ def product(path): "files": manifest["files"], "objects": {oid: obj["files"] for oid, obj in manifest["objects"].items()}, "root": "../" * (len(path.split("/")) - 1), + "metadata": metadata, }, ) c = art["content"] diff --git a/src/nnnotes/site/pages.py b/src/nnnotes/site/pages.py index 05e275c..f703b9e 100644 --- a/src/nnnotes/site/pages.py +++ b/src/nnnotes/site/pages.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import json import re from pathlib import Path @@ -33,7 +34,7 @@ STORIES_INDEX = "stories.json" SITE_DATA = ("assets", "charts", MODELS_DIR, "charts.json", MODELS_INDEX, STORIES_DIR, STORIES_INDEX) PLAYER_VERSION_MARKER = "@PLAYER_VERSION@" # replaced in the page's files by the bundles' short hash -SITE_DATA = (*SITE_DATA, "products", "site.json") +SITE_DATA = (*SITE_DATA, "products", "site.json", "manifest.json", ".nnnotes-layout-journal.json") def site_regions(cfg: Config, regions=None, all_regions: bool = False) -> list[str]: @@ -85,7 +86,7 @@ def region_meta(cfg: Config, regions: list[str]) -> list[dict]: def check_player(player_dir) -> Path: - """The ournotes-player checkout or installed package: its read-set script, built bundle and page must exist.""" + """A compatible ournotes-player 0.2.x package with its read-set script, built bundle and page.""" if player_dir is None: raise ConfigError( "the site needs the player: give --player " @@ -99,6 +100,16 @@ def check_player(player_dir) -> Path: raise ConfigError( f"{d} is not a built ournotes-player (missing {', '.join(missing)}; a checkout needs its build first)" ) + try: + package = json.loads((d / "package.json").read_text(encoding="utf-8")) + except (OSError, ValueError) as e: + raise ConfigError(f"{d}: cannot read the player package.json; ournotes-player 0.2.x is required") from e + if ( + not isinstance(package, dict) + or package.get("name") != "ournotes-player" + or not re.fullmatch(r"0\.2\.(?:0|[1-9][0-9]*)(?:\+[0-9A-Za-z.-]+)?", str(package.get("version", ""))) + ): + raise ConfigError(f"{d}: incompatible player package; a stable ournotes-player 0.2.x is required") return d.resolve() diff --git a/src/nnnotes/site/publish.py b/src/nnnotes/site/publish.py index c95b248..6caa940 100644 --- a/src/nnnotes/site/publish.py +++ b/src/nnnotes/site/publish.py @@ -28,6 +28,8 @@ def materialize(directory, entries, depot, *, link="copy", params=None): seen = {} for entry in entries: path = entry["path"] + if path.casefold() in {layout.MANIFEST.casefold(), layout.JOURNAL.casefold()}: + raise layout.LayoutError(f"site file uses a reserved layout path: {path}") if re.search(r"[\\:?#\x00]", path) or any(p in {"", ".", ".."} for p in path.split("/")): raise layout.LayoutError(f"site file is not a relative path: {path}") old = seen.setdefault(path.casefold(), path) diff --git a/src/nnnotes/site/webmodel.py b/src/nnnotes/site/webmodel.py deleted file mode 100644 index 1e41eef..0000000 --- a/src/nnnotes/site/webmodel.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Catalog queries for Live2D model subjects.""" - -from ..content.live2d import LIVE2D_PREFIX, model_keys, select_models - -__all__ = ["catalog_models"] - - -def catalog_models(cat, specs=None) -> dict[str, str]: - """Selected model ids and keys from a catalog; omitted specs select every model.""" - return select_models(specs, model_keys(cat.keys(LIVE2D_PREFIX))) diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index ab15a5e..c3e97ed 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -552,10 +552,10 @@ def test_plan_emit_tasks_and_run_stage_give_the_exported_depot(tmp_path, capsys, doc = json.loads(out) if not any(n["status"] == "run" for n in doc["nodes"]): break - Depot(tmp_path / "q").gc() + Depot(tmp_path / "q").gc(grace=0) code, out, err = nn(capsys, *flags(d), "depot", "run", tasks, "--depot", tmp_path / "q") assert code in (0, 1), err - Depot(tmp_path / "q").gc() + Depot(tmp_path / "q").gc(grace=0) rounds += 1 assert rounds == 3 # index + census; scripts + addresses; export skip = ["inputs", "runs", "costs", "catalogs", "downloads"] @@ -1231,7 +1231,7 @@ def test_plan_census_takes_the_censuses_first(tmp_path, capsys, fake): depot = Depot(tmp_path / "s") run = depot.latest_run(["depot.plan.census", "all"]) assert run and {t["id"] for t in run["tasks"]} >= {n["id"] for n in census} - depot.gc(keep=1) + depot.gc(keep=1, grace=0) assert all(depot.has_result(n["key"]) for n in census) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index a401ac4..e264391 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -411,13 +411,16 @@ def test_unknown_key_or_id_is_a_usage_error(data, argv, message, capsys): def test_known_id_reaches_the_extractor(data, tmp_path, capsys, monkeypatch): - from nnnotes.content import adv + from nnnotes.cli import export seen = [] - monkeypatch.setattr(adv, "extract", lambda cat, md, adv_id: seen.append(adv_id) or "x") - monkeypatch.setattr(adv, "to_json", lambda x: {"commandCount": 0, "resources": []}) + monkeypatch.setattr( + export, "_file_product", lambda args, cfg, stage, subject: seen.append((stage.name, subject)) or (None, None) + ) + monkeypatch.setattr(export, "_product_document", lambda *a: {"commandCount": 0, "resources": []}) + monkeypatch.setattr(export, "_place_files", lambda *a, **kw: None) code, out, err = run(data + ["export", "episode", "1", "-o", str(tmp_path / "e.json")], capsys) - assert code == 0 and seen == [1], err + assert code == 0 and seen == [("export.episode", "1")], err def test_missing_master_table_names_it(data, tmp_path, capsys): @@ -452,12 +455,16 @@ def test_flac_level_is_checked(data, argv, message, capsys): def test_flac_level_reaches_the_decode(data, tmp_path, capsys, monkeypatch): from types import SimpleNamespace - from nnnotes.formats import cri + from nnnotes.cli import export from nnnotes.source.session import Session seen = [] monkeypatch.setattr(Session, "catalog", lambda self: SimpleNamespace(has=lambda key: True)) - monkeypatch.setattr(cri, "decode", lambda *a, **kw: seen.append(kw.get("flac_level")) or []) + monkeypatch.setattr( + export, "_file_product", lambda args, cfg, stage, subject: seen.append(args.flac_level) or (None, None) + ) + monkeypatch.setattr(export, "_product_document", lambda *a: {}) + monkeypatch.setattr(export, "_place_files", lambda *a, **kw: None) for argv in (["export", "audio", "s"],): for flag, level in (([], None), (["--flac-level", "5"], 5)): seen.clear() diff --git a/tests/content/test_storysimpleui.py b/tests/content/test_storysimpleui.py index 96218ec..6270661 100644 --- a/tests/content/test_storysimpleui.py +++ b/tests/content/test_storysimpleui.py @@ -73,7 +73,7 @@ def test_overlay_font_product_includes_simple_ui_texts(tmp_path, source): def test_overlay_parts_request_simple_ui_for_selected_languages(): parts = {"assets": [], "models": [], "sounds": [], "videos": [], "chat": False, "crilips": False} normal = story.part_paths(parts, 1) - overlay = story.part_paths(parts, 1, overlay=True) + overlay = story.part_paths(parts, 1, requested_languages=["ja"], overlay=True) assert "simpleui" not in normal assert overlay["simpleui"]["ja"] == "storysimpleui/ja.json" products = story.part_products({"advId": 1, "parts": overlay}, ["ja", "ja", "zh-Hant"]) diff --git a/tests/core/test_downloads.py b/tests/core/test_downloads.py index e14751b..4f6a64c 100644 --- a/tests/core/test_downloads.py +++ b/tests/core/test_downloads.py @@ -127,6 +127,18 @@ def test_a_trim_under_way_elsewhere_is_left_to_its_command(tmp_path): assert f.exists() +def test_foreign_exclusive_wait_is_bounded_without_taking_over(tmp_path): + leases = tmp_path / "leases" + leases.mkdir() + lock = leases / downloads.EXCLUSIVE + record = json.dumps({"host": socket.gethostname() + "-other", "pid": 1, "started": time.time()}) + lock.write_text(record) + with pytest.raises(TimeoutError, match="exclusive lease.*still held"): + with downloads.lease(tmp_path, None, wait_timeout=0): + pytest.fail("must not enter while another host holds the exclusive lease") + assert lock.read_text() == record and list(leases.iterdir()) == [lock] + + def test_exclusive_is_granted_to_the_only_command(tmp_path): with downloads.lease(tmp_path, None) as own: with downloads.exclusive(tmp_path, own) as alone: diff --git a/tests/pipeline/test_catalog_remote.py b/tests/pipeline/test_catalog_remote.py index 30f4736..5c6fa2b 100644 --- a/tests/pipeline/test_catalog_remote.py +++ b/tests/pipeline/test_catalog_remote.py @@ -27,6 +27,18 @@ def test_catalog_versions_are_shared_and_content_is_fetched_on_demand(tmp_path): assert CatalogDB(tmp_path / "b").catalog_bytes(version) == (body, apk) +def test_local_catalogs_and_labels_are_shared_when_remote_is_configured(tmp_path): + local = CatalogDB(tmp_path / "a") + body, apk = build(remote_entries()), build(APK) + version = local.add(body, apk, label="stable", region="one", language="en") + remote = open_remote((tmp_path / "remote").as_uri()) + shared = CatalogDB(tmp_path / "a", remote=remote) + assert shared.versions() == [version] + fresh = CatalogDB(tmp_path / "b", remote=remote) + assert fresh.get("stable") == version + assert fresh.catalog_bytes(version) == (body, apk) + + def test_independent_imports_preserve_versions_and_refresh_known_metadata(tmp_path): _, first, second = databases(tmp_path) body = build(remote_entries()) diff --git a/tests/pipeline/test_depot_gc.py b/tests/pipeline/test_depot_gc.py index 7170dd6..2b2b422 100644 --- a/tests/pipeline/test_depot_gc.py +++ b/tests/pipeline/test_depot_gc.py @@ -48,11 +48,11 @@ def test_gc_keeps_what_the_kept_runs_reach(tmp_path): first = write_run(d, ["group:a"], [old], mtime=1_000_000_000) latest = write_run(d, ["group:a"], [new], mtime=2_000_000_000) - counted = d.gc(keep=1, dry_run=True) + counted = d.gc(keep=1, dry_run=True, grace=0) assert counted["runs"] == {"kept": 1, "removed": 1} and counted["contents"] == {"kept": 2, "removed": 2} assert d.run(first["id"]) is not None and d.has(orphan) # a dry run removes nothing - r = d.gc(keep=1) + r = d.gc(keep=1, grace=0) assert r["runs"] == {"kept": 1, "removed": 1} and r["results"] == {"kept": 1, "removed": 1} assert r["contents"] == {"kept": 2, "removed": 2} assert d.run(first["id"]) is None and d.run(latest["id"]) is not None @@ -86,7 +86,7 @@ def test_the_gc_command_runs_only_when_no_other_command_does(tmp_path, capsys): other = tmp_path / "leases" / "other.json" other.parent.mkdir() other.write_text(json.dumps({"host": socket.gethostname() + "-other", "pid": 1, "started": time.time()})) - argv = ["--depot", str(tmp_path), "depot", "gc", "--json"] + argv = ["--depot", str(tmp_path), "depot", "gc", "--grace", "0", "--json"] with pytest.raises(SystemExit, match="another command uses the depot"): cli.main(argv) assert d.path(orphan).exists() @@ -105,3 +105,52 @@ def test_the_gc_command_collects_the_remote_with_remote(tmp_path, capsys, monkey cli.main(argv) r = json.loads(capsys.readouterr().out) assert r["remote"]["runs"] == {"kept": 0, "removed": 0} and r["contents"] == {"kept": 0, "removed": 0} + + +def test_recent_result_keeps_older_inputs_and_artifacts_until_grace_expires(tmp_path): + d = Depot(tmp_path) + stored = d.put(b"old input") + task = Task("t.x", 1, "a", inputs=(Input("doc", stored, 9, None, ({"kind": "depot"},)),)) + doc = commit(d, task, b"old output") + art = doc["artifacts"][0]["content"]["sha256"] + old = time.time() - 172800 + for sha in (stored, art): + os.utime(d.path(sha), (old, old)) + assert d.gc()["results"] == {"kept": 1, "removed": 0} + assert d.path(stored).exists() and d.path(art).exists() + os.utime(d.result_path(task.key), (old, old)) + assert d.gc()["results"] == {"kept": 0, "removed": 1} + assert not d.path(stored).exists() and not d.path(art).exists() + + +@pytest.mark.parametrize("fail", [False, True]) +def test_command_records_keep_completed_results_and_content_even_on_failure(tmp_path, fail): + from nnnotes.pipeline.depot import command_run + + depot = Depot(tmp_path) + task = Task("t.x", 1, "a") + try: + with command_run(depot, "example"): + other = Depot(tmp_path) + content = other.put(b"read input") + commit(other, task, b"completed output") + if fail: + raise RuntimeError("later operation failed") + except RuntimeError: + assert fail + recorded = depot.latest_run(["command:example"]) + assert recorded["context"]["outcome"] == ("failed" if fail else "complete") + depot.gc(keep=1, grace=0) + assert depot.result(task.key) is not None and depot.read(content) == b"read input" + + +def test_cli_dry_run_preserves_downloads_and_writes_no_command_record(tmp_path, capsys): + depot = Depot(tmp_path) + path = depot.downloads / "bundles" / "example.bundle" + path.parent.mkdir(parents=True) + path.write_bytes(b"source bytes") + orphan = depot.put(b"orphan") + cli.main(["--depot", str(tmp_path), "depot", "gc", "--dry-run", "--grace", "0", "--json"]) + assert json.loads(capsys.readouterr().out)["contents"]["removed"] == 1 + assert depot.path(orphan).exists() and path.read_bytes() == b"source bytes" + assert not list((tmp_path / "runs").rglob("*.json")) diff --git a/tests/pipeline/test_depot_remote.py b/tests/pipeline/test_depot_remote.py index fe85851..e4e5cd0 100644 --- a/tests/pipeline/test_depot_remote.py +++ b/tests/pipeline/test_depot_remote.py @@ -74,7 +74,7 @@ def size(self, key): return super().size(key) -def test_the_latest_run_of_the_remote_tells_which_results_it_holds(tmp_path): +def test_remote_results_are_checked_independently_of_the_run_pointer(tmp_path): remote = open_remote((tmp_path / "remote").as_uri()) first = Depot(tmp_path / "a", remote=remote) tasks = [Task("t.x", 1, n) for n in "abc"] @@ -84,8 +84,57 @@ def test_the_latest_run_of_the_remote_tells_which_results_it_holds(tmp_path): counting = Counting(remote.url, remote.root) second = Depot(tmp_path / "b", remote=counting) assert second.latest_run(["group:a"])["id"] == run_doc(["group:a"], tasks)["id"] - assert all(second.has_result(t.key) for t in tasks) and counting.asked == [] - assert not second.has_result(Task("t.x", 1, "z").key) and len(counting.asked) == 1 + assert all(second.has_result(t.key) for t in tasks) and len(counting.asked) == 3 + assert not second.has_result(Task("t.x", 1, "z").key) and len(counting.asked) == 4 + remote.delete([f"ac/{tasks[0].key[:2]}/{tasks[0].key}.json"]) + fresh = Depot(tmp_path / "c", remote=remote) + assert fresh.latest_run(["group:a"]) is not None + assert not fresh.has_result(tasks[0].key) + + +def test_sharing_a_cached_run_uploads_results_layouts_and_stored_inputs(tmp_path): + first = Depot(tmp_path / "a") + inp = first.put(b"input") + task = Task("t.x", 1, "a", inputs=(Input("doc", inp, 5, None, ({"kind": "depot"},)),)) + result = commit(first, task, b"payload") + placed = first.put(b"layout file") + manifest = first.put(contract.encode({"files": [{"sha256": placed, "size": 11}]})) + run = run_doc(["group:a"], [task]) + run["tasks"][0]["status"] = "hit" + run["layouts"] = {"product": manifest} + remote = open_remote((tmp_path / "remote").as_uri()) + Depot(first.root, remote=remote).write_run(run) + second = Depot(tmp_path / "b", remote=remote) + assert second.latest_run(["group:a"]) == run + assert second.committed(task.key, task.id) == result + assert second.open_input(task.inputs[0]).read_bytes() == b"input" + assert second.read(placed) == b"layout file" and second.read(manifest) + + +@pytest.mark.parametrize("quick", [False, True]) +def test_verify_checks_referenced_remote_contents_without_materializing(tmp_path, quick): + remote = open_remote((tmp_path / "remote").as_uri()) + first, second = Depot(tmp_path / "a", remote=remote), Depot(tmp_path / "b", remote=remote) + task = Task("t.x", 1, "a") + doc = commit(first, task, b"payload") + assert second.result(task.key) == doc + assert second.verify(quick=quick)["problemCount"] == 0 + assert not (second.root / "cas").exists() + sha = doc["artifacts"][0]["content"]["sha256"] + remote.put(f"cas/sha256/{sha[:2]}/{sha}", b"invalid") + assert second.verify(quick=False)["problemCount"] == 1 + + +@pytest.mark.parametrize("dry_run", [True, False]) +def test_remote_collection_does_not_fill_the_local_cache(tmp_path, dry_run): + remote = open_remote((tmp_path / "remote").as_uri()) + first = Depot(tmp_path / "a", remote=remote) + task = Task("t.x", 1, "a") + commit(first, task, b"payload") + first.write_run(run_doc(["group:a"], [task])) + second = Depot(tmp_path / "b", remote=remote) + assert second.gc(remote=True, dry_run=dry_run)["remote"]["results"]["kept"] == 1 + assert not second.root.exists() def test_a_depot_crosses_processes_with_its_remote(tmp_path): diff --git a/tests/pipeline/test_exportfiles.py b/tests/pipeline/test_exportfiles.py new file mode 100644 index 0000000..c8d7e78 --- /dev/null +++ b/tests/pipeline/test_exportfiles.py @@ -0,0 +1,179 @@ +"""Standalone file exports consume the common object and cue-sheet results.""" + +import io +import shutil +import subprocess +import wave +from types import SimpleNamespace + +import pytest + +from nnnotes.content import adv, storyepisode +from nnnotes.pipeline import contract +from nnnotes.pipeline.assemble import Objects +from nnnotes.pipeline.contract import Task +from nnnotes.pipeline.depot import Depot +from nnnotes.pipeline.exportfiles import AudioFilesStage, EpisodeStage, ShaderFilesStage +from nnnotes.pipeline.stages import Env, execute +from tests.helpers import story as sh +from tests.pipeline.test_live import master_record, run, stored +from tests.pipeline.test_objexport import exported as exported +from tests.pipeline.test_story import STREAMS, sheet + + +@pytest.mark.parametrize("apk", [False, True]) +def test_shader_command_converts_hashed_files_even_without_catalog_entries(tmp_path, monkeypatch, capsys, apk): + from nnnotes.cli import export as cli_export + from nnnotes.pipeline import census, objexport + from nnnotes.source.config import Config + from nnnotes.source.session import Session + from tests.pipeline.test_objexport import FAKE_READER, bundle_all + + path = tmp_path / ("sample_" + "a" * 32 + ".bundle") + path.write_bytes(b"fake:all") + cat = SimpleNamespace(apk=object(), has=lambda _: True, fetch_key=lambda _: [path], apk_bundle=lambda _: path) + monkeypatch.setattr(Session, "catalog", lambda self: cat) + monkeypatch.setattr(census, "census_file", lambda *a, **kw: census.census_env(bundle_all(), shipped=True)) + original = objexport.ExportStage + monkeypatch.setattr(objexport, "ExportStage", lambda: original(FAKE_READER)) + args = SimpleNamespace( + key=None if apk else "Example/Shader", + apk_bundle=["sample"] if apk else None, + out=str(tmp_path / "out"), + json=True, + ) + cfg = Config({"paths": {"depot": str(tmp_path / "depot")}}, environ={}) + cli_export.cmd_shader(args, cfg) + result = contract.loads(capsys.readouterr().out) + assert result["count"] == 1 and result["names"] == ["Test/Unlit"] + assert (tmp_path / "out/Test_Unlit.json").is_file() + assert Depot(tmp_path / "depot").latest_run(["export.shader", "sample"])["tasks"] + + +@pytest.mark.parametrize("fmt", ["wav", "ogg"]) +def test_audio_formats_encode_a_decoded_flac_stream(tmp_path, monkeypatch, fmt): + from nnnotes.pipeline import exportfiles + from nnnotes.pipeline.cristages import tool_id + + ffmpeg = shutil.which("ffmpeg") + if ffmpeg is None: + pytest.skip("ffmpeg is unavailable") + pcm = b"\x10\x00" * 256 + source = tmp_path / "input.wav" + with wave.open(str(source), "wb") as out: + out.setparams((1, 2, 48000, 256, "NONE", "not compressed")) + out.writeframes(pcm) + flac = tmp_path / "input.flac" + subprocess.run([ffmpeg, "-v", "error", "-i", str(source), str(flac)], check=True, capture_output=True) + monkeypatch.setattr(exportfiles, "_tool", lambda _: ffmpeg) + depot, stage = Depot(tmp_path / "depot"), AudioFilesStage() + streams = [{"stream": 1, "name": "cue", "file": "cue.flac", "sampleRate": 48000, "channels": 1, "samples": 256}] + inputs, sheets = sheet(depot, "sample", streams) + files = sheets["Cri/Sound/sample"]["files"] + files["cue.flac"] = depot.add(flac.read_bytes(), "flac") + files["cues.json"] = depot.add( + contract.encode({"cue": {k: v for k, v in streams[0].items() if k not in {"stream", "name"}}}), "json" + ) + inputs[-1] = stored(depot, contract.encode(sheets), "sheets") + inputs.append( + stored( + depot, + contract.encode( + {"schema": contract.OBJECTS, "roots": [], "keys": {}, "objects": {}, "fenced": {}, "missing": []} + ), + "objects", + ) + ) + _, _, output = run(depot, stage, "sample", inputs, {"audio": fmt}, {"audio.encode": tool_id("ffmpeg", ffmpeg)}) + encoded = depot.read(output["cue." + fmt]["content"]["sha256"]) + assert contract.loads(depot.read(output["cues.json"]["content"]["sha256"]))["cue"]["file"] == "cue." + fmt + if fmt == "wav": + with wave.open(io.BytesIO(encoded)) as audio: + assert audio.getframerate() == 48000 and audio.readframes(256) == pcm + else: + assert encoded.startswith(b"OggS") + file = tmp_path / "encoded.ogg" + file.write_bytes(encoded) + subprocess.run([ffmpeg, "-v", "error", "-i", str(file), "-f", "null", "-"], check=True, capture_output=True) + + +def test_episode_file_uses_the_product_episode_builder(tmp_path): + depot, stage = Depot(tmp_path / "d"), EpisodeStage() + env = Env(depot, {"master": sh.master_dir(tmp_path)}) + table = sh.episode_table(depot) + _, _, artifacts = run( + depot, stage, "1", [stored(depot, contract.encode(table), "objects"), master_record(env, stage, "1")] + ) + record = stage.record("1", env) + expected = adv.to_json( + storyepisode.episode(Objects(table, depot), record, has=set(record["catalogKeys"]).__contains__) + ) + assert contract.loads(depot.read(artifacts["episode.json"]["content"]["sha256"])) == contract.loads( + contract.encode(expected) + ) + assert set(artifacts) == {"objects", "episode.json"} + assert all("present" in resource for resource in expected["resources"]) + + +def test_episode_file_preserves_embedded_resource_fallback_and_presence(tmp_path, monkeypatch): + from nnnotes.content.advcommand import COMMAND + + command = next(code for code, name in COMMAND.items() if name == "TalkWindow") + monkeypatch.setattr(sh, "COLLECTION", [{"Index": 0, "Command": command}]) + depot, stage = Depot(tmp_path / "d"), EpisodeStage() + address = adv.EMBEDDED_PREFIX + adv.TALK_WINDOW_PREFIX + adv.DEFAULT_TALK_WINDOW + env = Env(depot, {"master": sh.master_dir(tmp_path), "index": {"locations": [{"primaryKey": address}]}}) + objects = stored(depot, contract.encode(sh.episode_table(depot)), "objects") + _, _, artifacts = run(depot, stage, "1", [objects, master_record(env, stage, "1")]) + doc = contract.loads(depot.read(artifacts["episode.json"]["content"]["sha256"])) + assert doc["resources"] == [{"kind": "talkwindow", "address": address, "present": True}] + + +def test_flac_files_reuse_decoded_stream_bytes_and_indexes(tmp_path): + depot, stage = Depot(tmp_path), AudioFilesStage() + inputs, sheets = sheet(depot, "sample") + cues = {} + for stream in STREAMS: + cues.setdefault(stream["name"], {k: v for k, v in stream.items() if k not in {"name", "stream"}}) + data = contract.encode(cues) + sheets["Cri/Sound/sample"]["files"]["cues.json"] = {"sha256": depot.put(data), "size": len(data), "ext": "json"} + inputs[-1] = stored(depot, contract.encode(sheets), "sheets") + inputs.append( + stored( + depot, + contract.encode( + {"schema": contract.OBJECTS, "roots": [], "keys": {}, "objects": {}, "fenced": {}, "missing": []} + ), + "objects", + ) + ) + task, result, artifacts = run(depot, stage, "sample", inputs) + for stream in STREAMS: + assert ( + artifacts[stream["file"]]["content"]["sha256"] + == sheets["Cri/Sound/sample"]["files"][stream["file"]]["sha256"] + ) + assert contract.loads(depot.read(artifacts["cues.json"]["content"]["sha256"])) == cues + assert execute(task, depot, {stage.name: stage}).status == "hit" + assert depot.result(task.key) == result + + +def test_shader_files_reuse_converted_programs_and_parsed_forms(exported): + depot, result = exported + record = { + "source": "example.bundle", + "items": result["items"], + "artifacts": {a["id"]: a for a in result["artifacts"]}, + } + stage = ShaderFilesStage() + inp = stored(depot, contract.encode([record]), "records") + task = Task(stage.name, stage.version, "selected", inputs=(inp,)) + execute(task, depot, {stage.name: stage}) + output = {contract.parse_artifact_id(a["id"])[1]: a for a in depot.result(task.key)["artifacts"]} + index = contract.loads(depot.read(output["shaders.json"]["content"]["sha256"])) + assert [row["name"] for row in index] == ["Test/Unlit"] + row = index[0] + assert row["parsed"] == "Test_Unlit.json" and row["source"] == "example.bundle" + sources = {a["content"]["sha256"] for a in result["artifacts"]} + assert output[row["parsed"]]["content"]["sha256"] in sources + assert all(output[v["file"]]["content"]["sha256"] in sources for v in row["variants"]) diff --git a/tests/pipeline/test_live.py b/tests/pipeline/test_live.py index d299fee..0dbb48d 100644 --- a/tests/pipeline/test_live.py +++ b/tests/pipeline/test_live.py @@ -272,6 +272,10 @@ def test_the_live_product_holds_the_chart_and_the_manifests_of_its_parts(tmp_pat env = Env(depot, {"master": live_master_dir(tmp_path)}) stage = LiveStage() record = master_record(env, stage, "7/expert") + env.params = stage.normalize({"languages": ["ja"]}) + localized_record = master_record(env, stage, "7/expert") + assert localized_record.sha256 != record.sha256 + assert contract.loads(depot.input_bytes(localized_record))["parts"]["ui"] == {"ja": "liveui/7_expert_ja.json"} assert contract.loads(depot.input_bytes(record))["parts"] == { "music": "music/7.json", "noteeffect": "noteeffects/fxA.json", @@ -279,7 +283,7 @@ def test_the_live_product_holds_the_chart_and_the_manifests_of_its_parts(tmp_pat "noteskin": "noteskins/skin001.json", "scene": "scenes/default.json", "sounds": "sounds/default.json", - "ui": {code: f"liveui/7_expert_{code}.json" for code in ("ja", "en", "zh-Hant", "zh-Hans", "ko")}, + "ui": {}, } objects = stored(depot, contract.encode(lh.table(depot)), "objects") _, _, arts = run(depot, stage, "7/expert", [objects, record]) diff --git a/tests/pipeline/test_orchestrate.py b/tests/pipeline/test_orchestrate.py index 1b61955..55035bb 100644 --- a/tests/pipeline/test_orchestrate.py +++ b/tests/pipeline/test_orchestrate.py @@ -306,7 +306,7 @@ def interrupt(*args): assert seen == [] and orch._pool == [] saved = depot.latest_run([]) assert saved and any(t["status"] == "ran" for t in saved["tasks"]) - depot.gc(keep=1) + depot.gc(keep=1, grace=0) assert all(depot.has_result(t["key"]) for t in saved["tasks"] if t["key"]) @@ -329,7 +329,7 @@ def run(self, task, depot): saved = orch.depot.latest_run([]) assert saved and any(t["status"] == "ran" for t in saved["tasks"]) - orch.depot.gc(keep=1) + orch.depot.gc(keep=1, grace=0) assert all(orch.depot.has_result(t["key"]) for t in saved["tasks"] if t["status"] == "ran") diff --git a/tests/pipeline/test_story.py b/tests/pipeline/test_story.py index 19aa6ce..1188421 100644 --- a/tests/pipeline/test_story.py +++ b/tests/pipeline/test_story.py @@ -291,7 +291,7 @@ def test_the_story_product_holds_the_episode_and_the_story_document(tmp_path): stage = StoryStage() doc = sh.episode_table(depot) objects = stored(depot, contract.encode(doc), "objects") - _, _, arts = run(depot, stage, "1", [objects, master_record(env, stage, "1")]) + _, _, arts = run(depot, stage, "1", [objects, master_record(env, stage, "1")], {"languages": ["ja", "en"]}) assert {p: a["semantics"]["kind"] for p, a in arts.items()} == { "episode.json": "story.episode", "objects": "product.objects", @@ -325,11 +325,11 @@ def test_the_story_product_holds_the_episode_and_the_story_document(tmp_path): "videos": {"Cri/Video/adv/movie_a/movie_a": "videos/adv~movie_a~movie_a.json"}, "chat": "storychat/default.json", "crilips": None, - "ui": {lang: f"storyui/{lang}.json" for lang in story.languages.LANGUAGES}, - "fonts": {lang: f"storyfonts/1_{lang}.json" for lang in story.languages.LANGUAGES}, + "ui": {lang: f"storyui/{lang}.json" for lang in ("en", "ja")}, + "fonts": {lang: f"storyfonts/1_{lang}.json" for lang in ("en", "ja")}, }, } - assert story_doc["parts"] == story.part_paths(storyepisode.parts(episode), 1) + assert story_doc["parts"] == story.part_paths(storyepisode.parts(episode), 1, requested_languages=["en", "ja"]) # ---------------------------------------------------------------- the chat product @@ -442,7 +442,7 @@ def test_part_paths_give_the_manifest_of_each_part(): "chat": True, "crilips": False, } - assert story.part_paths(parts, 1) == { + assert story.part_paths(parts, 1, requested_languages=["ja"]) == { "scene": "storyscenes/default.json", "assets": { "Adv/Stage/st_a/st_a": "storyassets/Adv~Stage~st_a~st_a.json", @@ -453,8 +453,8 @@ def test_part_paths_give_the_manifest_of_each_part(): "videos": {"Cri/Video/adv/m/m": "videos/adv~m~m.json"}, "chat": "storychat/default.json", "crilips": None, - "ui": {lang: f"storyui/{lang}.json" for lang in story.languages.LANGUAGES}, - "fonts": {lang: f"storyfonts/1_{lang}.json" for lang in story.languages.LANGUAGES}, + "ui": {"ja": "storyui/ja.json"}, + "fonts": {"ja": "storyfonts/1_ja.json"}, } quiet = {**parts, "assets": [], "models": [], "sounds": [], "videos": [], "chat": False, "crilips": True} assert story.part_paths(quiet, 1) == { @@ -465,8 +465,8 @@ def test_part_paths_give_the_manifest_of_each_part(): "videos": {}, "chat": None, "crilips": "crilips/default.json", - "ui": {lang: f"storyui/{lang}.json" for lang in story.languages.LANGUAGES}, - "fonts": {lang: f"storyfonts/1_{lang}.json" for lang in story.languages.LANGUAGES}, + "ui": {}, + "fonts": {}, } diff --git a/tests/site/test_catalog.py b/tests/site/test_catalog.py index 3aedc4f..eaf0863 100644 --- a/tests/site/test_catalog.py +++ b/tests/site/test_catalog.py @@ -91,7 +91,7 @@ def test_saved_complete_layout_survives_collection_and_restores_new_directory(se first.finish(out) second = make() second.finish(out) - depot.gc(keep=1) + depot.gc(keep=1, grace=0) third = make() restored = tmp_path / "restored" third.finish(restored) @@ -148,3 +148,38 @@ def test_fresh_depot_restores_retained_topics_from_remote(tmp_path, monkeypatch) again = Builder(second, *args) again.finish(tmp_path / "out2") assert (tmp_path / "out1/models/sample.json").read_bytes() == (tmp_path / "out2/models/sample.json").read_bytes() + + +def test_model_names_may_differ_across_regions_for_the_same_content(tmp_path): + depot = Depot(tmp_path) + source, rows = model(depot) + from nnnotes.site.build import documents + + snapshot = Snapshot(depot, documents(depot, rows)) + target = snapshot.product(source) + regions = [{"id": r, "name": r} for r in ("one", "two")] + topics = [ + { + "kind": "model", + "id": "sample", + "region": r["id"], + "manifest": target, + "entry": {"key": "sample", "label": r["name"]}, + } + for r in regions + ] + doc = catalog.merge(None, "example", "en", regions, topics) + entries, _ = catalog.assemble(depot, doc, snapshot.entries) + listing = json.loads(depot.read(entries["models.json"]["sha256"]))["models"] + assert [(row["label"], row["regions"]) for row in listing] == [("one", ["one"]), ("two", ["two"])] + assert "models/sample.json" in entries and "models/two/sample.json" in entries + + +def test_current_region_order_takes_precedence_and_unused_regions_are_dropped(): + regions = [{"id": r, "name": r} for r in ("one", "two", "unused")] + topics = [{"region": "one", "kind": "model", "id": "a"}] + old = catalog.merge(None, "example", "en", regions, topics) + updated = catalog.merge(old, "example", "en", [regions[1], regions[0]], []) + assert [r["id"] for r in updated["regions"]] == ["two", "one"] + partial = catalog.merge(old, "example", "en", [regions[1]], []) + assert [r["id"] for r in partial["regions"]] == ["two", "one"] and partial["topics"] == topics diff --git a/tests/site/test_cli.py b/tests/site/test_cli.py index 4a92fbc..dc69958 100644 --- a/tests/site/test_cli.py +++ b/tests/site/test_cli.py @@ -155,6 +155,20 @@ def test_cli_requests_products_and_restores_named_site(command, tmp_path, capsys assert json.loads(capsys.readouterr().out)["write"] == 0 assert (tmp_path / "site/charts/1_expert.json").is_file() assert (tmp_path / "site/models/sample.json").is_file() + chart = json.loads((tmp_path / "site/charts/1_expert.json").read_bytes()) + entry = json.loads((tmp_path / "site/charts.json").read_bytes())["charts"][0] + assert chart["chart"]["title"] == entry["title"] + assert chart["flows"] == ["direct"] and chart["quality"] == 1 + assert chart["regions"] == entry["regions"] and chart["audioFormat"] == "aac" + + +def test_silent_chart_has_no_audio_format_in_either_manifest_or_index(command, tmp_path): + run, _, _, _ = command + run(["--pair", "1:expert", "--no-audio"]) + manifest = json.loads((tmp_path / "site/charts/1_expert.json").read_bytes()) + entry = json.loads((tmp_path / "site/charts.json").read_bytes())["charts"][0] + assert manifest["audio"] is entry["audio"] is False + assert manifest["audioFormat"] is entry["audioFormat"] is None def test_cli_story_passes_fonts_languages_and_audio_policy(command, tmp_path): @@ -245,8 +259,48 @@ def test_failed_product_run_stops_before_layout(tmp_path, monkeypatch, capsys): run = Run(depot, {}, [], {}, {}) run.failures_.append(failure("live:1", "live", "source.error", "missing source")) monkeypatch.setattr(assets, "run_request", lambda *a, **kw: (run, None, None)) - args = SimpleNamespace(json=True) + args = SimpleNamespace(json=True, name="example") with pytest.raises(SystemExit) as error: site.product_entries(args, None, None) assert error.value.code == 1 assert json.loads(capsys.readouterr().out)["failures"][0]["message"] == "missing source" + + +def test_failed_site_product_run_keeps_the_export_selection_pointer(tmp_path): + from nnnotes.pipeline.orchestrate import Orchestrator + from nnnotes.pipeline.stages import Stage + from nnnotes.source.config import ConfigError + + class Stops(Stage): + name, version = "test.stop", 1 + + def subjects(self, env): + return ["sample"] + + def run(self, task, depot): + raise ConfigError("source unavailable") + + depot = Depot(tmp_path) + selection = ["key:source"] + original = Run(depot, {}, selection, {}, {}).write() + stage = Stops() + orch = Orchestrator(depot, {stage.name: stage}, workers=0, log=lambda _: None) + ws = SimpleNamespace( + depot=depot, + selection=selection, + run_index=lambda: None, + facts=lambda *a, **kw: {"raw": {}}, + selected=SimpleNamespace(selected=[], census=[], raw=[]), + log=lambda _: None, + problems=lambda: [], + context=lambda: {}, + wait=lambda: None, + cancel=lambda: None, + ) + run, _, aborted = assets.run_request( + SimpleNamespace(), None, ws, prepared=([stage.name], {}, orch), selection_prefix=["site-product:example"] + ) + assert aborted == "source unavailable" + assert run.selection == ["site-product:example", *selection] + assert depot.latest_run(selection) == original + assert depot.latest_run(run.selection)["tasks"][0]["status"] == "failed" diff --git a/tests/site/test_pages.py b/tests/site/test_pages.py index 0e7436a..50737b5 100644 --- a/tests/site/test_pages.py +++ b/tests/site/test_pages.py @@ -1,6 +1,7 @@ """Player page collection, bundled imports and reserved site paths.""" import hashlib +import json import pytest @@ -10,6 +11,7 @@ def player(root, *, live2d=False, story=False, bundles=True): files = { + "package.json": json.dumps({"name": "ournotes-player", "version": "0.2.0"}), pages.READ_SET_SCRIPT: "// read set\n", pages.PLAYER_BUNDLES[0]: "/* bundle */\n", f"{pages.PLAYER_PAGE_DIR}/index.html": "\n", @@ -40,6 +42,25 @@ def test_check_player(tmp_path): assert pages.check_player(player(tmp_path / "p")) == (tmp_path / "p").resolve() +@pytest.mark.parametrize("version", ["0.1.8", "0.3.0", "1.0.0", "0.2.0-rc.1", None]) +def test_incompatible_player_is_rejected_before_build(tmp_path, version): + root = player(tmp_path) + (root / "package.json").write_text(json.dumps({"name": "ournotes-player", "version": version})) + with pytest.raises(ConfigError, match="stable ournotes-player 0.2.x"): + pages.gather(root) + + +def test_player_package_identity_is_required(tmp_path): + root = player(tmp_path) + package = root / "package.json" + package.write_text('{"name":"another-package","version":"0.2.0"}') + with pytest.raises(ConfigError, match="incompatible player"): + pages.check_player(root) + package.unlink() + with pytest.raises(ConfigError, match="cannot read the player package.json"): + pages.check_player(root) + + def test_pages_bind_imports_and_versions_to_bundles(tmp_path): root = player(tmp_path, live2d=True, story=True) files, stats = pages.gather(root, stories=True) @@ -76,7 +97,9 @@ def test_songs_include_nested_module_dependencies(tmp_path): assert {p.removeprefix("songs/"): b for p, b in files.items() if p.startswith("songs/")} == songs -@pytest.mark.parametrize("name", ["models.json", "site.json", "products/example.json"]) +@pytest.mark.parametrize( + "name", ["models.json", "site.json", "products/example.json", "manifest.json", ".nnnotes-layout-journal.json"] +) def test_reserved_product_and_index_paths(tmp_path, name): root = player(tmp_path) path = root / pages.PLAYER_PAGE_DIR / name diff --git a/tests/site/test_publish.py b/tests/site/test_publish.py index 2853fc7..410a0e9 100644 --- a/tests/site/test_publish.py +++ b/tests/site/test_publish.py @@ -87,3 +87,11 @@ def test_site_paths_are_confined_to_the_output(tmp_path, path): with pytest.raises(layout.LayoutError, match="relative path"): publish.materialize(tmp_path / "site", entries(depot, {path: b"x"}), depot) assert not (tmp_path / "site").exists() + + +@pytest.mark.parametrize("path", [layout.MANIFEST, layout.JOURNAL, "MANIFEST.JSON"]) +def test_reserved_layout_files_are_rejected_before_publication(tmp_path, path): + depot = Depot(tmp_path / "depot") + with pytest.raises(layout.LayoutError, match="reserved layout path"): + publish.materialize(tmp_path / "site", entries(depot, {path: b"page"}), depot) + assert not (tmp_path / "site").exists() diff --git a/tests/source/test_catalog_versions.py b/tests/source/test_catalog_versions.py index c0d56cf..e490b82 100644 --- a/tests/source/test_catalog_versions.py +++ b/tests/source/test_catalog_versions.py @@ -40,7 +40,7 @@ def config(tmp_path, *, api=True, version=None, region="tw"): def test_auto_discovery_ignores_stale_main_and_advances_resource_version(tmp_path, monkeypatch): - from nnnotes.site.webmodel import catalog_models + from nnnotes.content.live2d import LIVE2D_PREFIX, model_keys, select_models cfg = config(tmp_path) downloads = Session(cfg).downloads() @@ -61,7 +61,7 @@ def download(url): monkeypatch.setattr(Session, "master_version", version) monkeypatch.setattr(catalog, "download", download) first = Session(cfg).catalog(bundles=False) - assert first.has(EPISODE) and catalog_models(first) == {"new": MODEL} + assert first.has(EPISODE) and select_models(None, model_keys(first.keys(LIVE2D_PREFIX))) == {"new": MODEL} assert first.resource_version == "1.0.0.201" assert deckdata.catalog_info(first)["resourceVersion"] == "1.0.0.201" assert seen[0] == "version" and len(seen) == 2 @@ -163,8 +163,9 @@ def test_build_workers_inherit_the_version_pin(tmp_path, monkeypatch): sys.executable, "-c", "from nnnotes.source.config import Config; from nnnotes.source.session import Session; " - "from nnnotes.site.webmodel import catalog_models; c=Session(Config.load()).catalog(bundles=False); " - "assert 'new' in catalog_models(c); print(c.resource_version)", + "from nnnotes.content.live2d import LIVE2D_PREFIX, model_keys, select_models; " + "c=Session(Config.load()).catalog(bundles=False); " + "assert 'new' in select_models(None, model_keys(c.keys(LIVE2D_PREFIX))); print(c.resource_version)", ] child = subprocess.run(command, env=os.environ.copy(), text=True, capture_output=True, check=True) assert child.stdout.strip() == "1.0.0.201" diff --git a/tests/test_api.py b/tests/test_api.py index 98905d3..dfdb83b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -13,7 +13,6 @@ "nnnotes.musicdata", "nnnotes.musicdata.deckdata", "nnnotes.schemas", - "nnnotes.site.webmodel", "nnnotes.source.addressables", "nnnotes.source.catalog", "nnnotes.source.config", diff --git a/uv.lock b/uv.lock index 327e072..68a4438 100644 --- a/uv.lock +++ b/uv.lock @@ -1127,7 +1127,7 @@ requires-dist = [ { name = "mkdocs-material", marker = "extra == 'docs'", specifier = ">=9.5" }, { name = "mkdocstrings", extras = ["python"], marker = "extra == 'docs'", specifier = ">=0.29" }, { name = "numpy", specifier = ">=2.0" }, - { name = "ournotes-sim", marker = "extra == 'deck'", specifier = "==0.0.9" }, + { name = "ournotes-sim", marker = "extra == 'deck'", specifier = "==0.0.12" }, { name = "pillow", specifier = ">=10.0" }, { name = "pycryptodome", specifier = ">=3.20" }, { name = "pyright", marker = "extra == 'test'", specifier = ">=1.1.414" }, @@ -1307,14 +1307,14 @@ wheels = [ [[package]] name = "ournotes-sim" -version = "0.0.9" +version = "0.0.12" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/6f/cc7aa9280d3148ddcdf82c17635ee37c7aec445f36cdcc02e2678bfed0be/ournotes_sim-0.0.9-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:f38711f42603ca5dbac1080f0d86653cd2b252568f82a9adca69844349c19f01", size = 1188929, upload-time = "2026-10-14T16:04:17.831Z" }, - { url = "https://files.pythonhosted.org/packages/a4/d8/3dd1fce51ae87dbd5645d4ff17bcdd2744280c8386c3ffeefbb480003bb7/ournotes_sim-0.0.9-cp311-abi3-macosx_11_0_x86_64.whl", hash = "sha256:fc023ae1a396ea6ce395612632379338d4ababac7dbcaf999a18768e79a68ecb", size = 1278009, upload-time = "2026-10-14T16:04:19.963Z" }, - { url = "https://files.pythonhosted.org/packages/c4/91/bf82f3696f1619c0daabe38f8bf0b63fe3f90093343da1f067ce500d2504/ournotes_sim-0.0.9-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c16900b18417e0a23cfd3a31445a604044e75f8ffca3bae406436804140ae1e9", size = 1259438, upload-time = "2026-10-14T16:04:21.609Z" }, - { url = "https://files.pythonhosted.org/packages/24/b8/26a13c40e707fb36b3b673a909a7316b47d663400acf9b292e3e2cb83a2a/ournotes_sim-0.0.9-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f59a1ffa87299a290c2d1b39fde511697794155d5a50bb09b96957bf8093bc2e", size = 1340684, upload-time = "2026-10-14T16:04:23.624Z" }, - { url = "https://files.pythonhosted.org/packages/be/7c/365c01caa5390bae1a1c39036bf2abd9f9d02008e905b78af041db410349/ournotes_sim-0.0.9-cp311-abi3-win_amd64.whl", hash = "sha256:afa4211a89825e766526395fc3d00a9bad332c7d312bbb66eedaff15ae68556a", size = 1234199, upload-time = "2026-10-14T16:04:25.524Z" }, + { url = "https://files.pythonhosted.org/packages/7d/4e/eb46553be18aa30bccc427d0477bf3c03e1730b81e96d09b85fb3510047d/ournotes_sim-0.0.12-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:5873aa17abe6a3d0fcc8565e602be5a1e7274cb331187340cb394eabfa47ecae", size = 1189004, upload-time = "2026-10-18T13:44:37.821Z" }, + { url = "https://files.pythonhosted.org/packages/4e/45/7c5f591f84a96c38b96f5ea95c19d0a7d3b160ba519af563b559665db221/ournotes_sim-0.0.12-cp311-abi3-macosx_11_0_x86_64.whl", hash = "sha256:641c662585c8d0765e1c20459fa63c01cf67366010d32b24f1e8c766d28d21b5", size = 1278163, upload-time = "2026-10-18T13:44:39.92Z" }, + { url = "https://files.pythonhosted.org/packages/4c/3c/315a33d4eaa27632a6639a48a05a53e6bd2a5a38aceb96a107f191827872/ournotes_sim-0.0.12-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c2c41866e383a402e304bee2a0b961e85280b107ad672cd3082f6bac469d62db", size = 1259142, upload-time = "2026-10-18T13:44:41.517Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a3/f1f3ed1c20be95b2772d7d33b19daad235eb05f6013ed583fe3d7239b56d/ournotes_sim-0.0.12-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2ab99e1e572a86b8603cf47441300232f78f1c4e53fc8e9e158e6056f04c5d18", size = 1340277, upload-time = "2026-10-18T13:44:43.355Z" }, + { url = "https://files.pythonhosted.org/packages/fa/52/fbc75f9c5ff98d5c59b7c7d9f3dbec3b0506b4d619115e980dad63a5992b/ournotes_sim-0.0.12-cp311-abi3-win_amd64.whl", hash = "sha256:812d4c72d95b4ab4aa9f31552696570b71a8c905a5a4e114e4f1dbdee0325381", size = 1239208, upload-time = "2026-10-18T13:44:45.222Z" }, ] [[package]] From 1d32ea5a6e75cfc32c2a52ffcb13f04eb22bcfec Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 19 Oct 2026 02:05:40 +0800 Subject: [PATCH 86/91] test: cover output stages with golden records --- .github/goldens-constraints.txt | 3 + .github/workflows/ci.yml | 5 +- CONTRIBUTING.md | 3 +- docs/determinism.md | 4 +- tests/content/test_storyfontproduct.py | 2 +- tests/golden/export.audio.json | 13 + tests/golden/export.episode.json | 17 + tests/golden/export.shader.json | 13 + tests/golden/live.json | 19 + tests/golden/live.music.json | 13 + tests/golden/live.noteeffect.json | 13 + tests/golden/live.notes.json | 13 + tests/golden/live.noteskin.json | 13 + tests/golden/live.scene.json | 13 + tests/golden/live.sounds.json | 13 + tests/golden/live.ui.json | 25 + tests/golden/live2d.cubism.json | 15 + tests/golden/live2d.model.json | 15 + tests/golden/room.gltf.json | 16 + tests/golden/site.asset.json | 15 + tests/golden/site.index.json | 13 + tests/golden/site.manifest.json | 13 + tests/golden/site.readset.json | 19 + tests/golden/site.references.json | 13 + tests/golden/site.selection.json | 13 + tests/golden/spot.files.json | 15 + tests/golden/story.asset.json | 13 + tests/golden/story.chat.json | 13 + tests/golden/story.crilips.json | 13 + tests/golden/story.fonts.json | 25 + tests/golden/story.host.json | 17 + tests/golden/story.json | 17 + tests/golden/story.scene.json | 13 + tests/golden/story.simpleui.json | 13 + tests/golden/story.sound.json | 13 + tests/golden/story.ui.json | 13 + tests/golden/story.video.json | 13 + tests/golden/ui.pack.json | 13 + tests/golden/view.backgrounds.json | 13 + tests/golden/view.band_items.json | 13 + tests/golden/view.bands.json | 13 + tests/golden/view.cards.json | 13 + tests/golden/view.characters.json | 13 + tests/golden/view.comics.json | 13 + tests/golden/view.degrees.json | 13 + tests/golden/view.events.json | 13 + tests/golden/view.exchange_categories.json | 13 + tests/golden/view.exchange_products.json | 13 + tests/golden/view.friendship_episodes.json | 13 + tests/golden/view.friendships.json | 13 + tests/golden/view.gacha.json | 13 + tests/golden/view.home_banners.json | 13 + tests/golden/view.items.json | 13 + tests/golden/view.jackets.json | 13 + tests/golden/view.login_bonuses.json | 13 + tests/golden/view.monthly_passes.json | 13 + tests/golden/view.rewards.json | 13 + tests/golden/view.season_passes.json | 13 + tests/golden/view.shops.json | 13 + tests/golden/view.skill_icons.json | 13 + tests/golden/view.spots.json | 13 + tests/golden/view.stamps.json | 13 + tests/golden/view.story_chapters.json | 13 + tests/golden/view.story_episodes.json | 13 + tests/golden/view.support_cards.json | 13 + tests/golden/view.voices.json | 13 + tests/helpers/fonts.py | 2 + tests/pipeline/product_goldens.py | 558 +++++++++++++++++++++ tests/pipeline/test_goldens.py | 43 +- tests/pipeline/view_goldens.py | 205 ++++++++ 70 files changed, 1670 insertions(+), 7 deletions(-) create mode 100644 tests/golden/export.audio.json create mode 100644 tests/golden/export.episode.json create mode 100644 tests/golden/export.shader.json create mode 100644 tests/golden/live.json create mode 100644 tests/golden/live.music.json create mode 100644 tests/golden/live.noteeffect.json create mode 100644 tests/golden/live.notes.json create mode 100644 tests/golden/live.noteskin.json create mode 100644 tests/golden/live.scene.json create mode 100644 tests/golden/live.sounds.json create mode 100644 tests/golden/live.ui.json create mode 100644 tests/golden/live2d.cubism.json create mode 100644 tests/golden/live2d.model.json create mode 100644 tests/golden/room.gltf.json create mode 100644 tests/golden/site.asset.json create mode 100644 tests/golden/site.index.json create mode 100644 tests/golden/site.manifest.json create mode 100644 tests/golden/site.readset.json create mode 100644 tests/golden/site.references.json create mode 100644 tests/golden/site.selection.json create mode 100644 tests/golden/spot.files.json create mode 100644 tests/golden/story.asset.json create mode 100644 tests/golden/story.chat.json create mode 100644 tests/golden/story.crilips.json create mode 100644 tests/golden/story.fonts.json create mode 100644 tests/golden/story.host.json create mode 100644 tests/golden/story.json create mode 100644 tests/golden/story.scene.json create mode 100644 tests/golden/story.simpleui.json create mode 100644 tests/golden/story.sound.json create mode 100644 tests/golden/story.ui.json create mode 100644 tests/golden/story.video.json create mode 100644 tests/golden/ui.pack.json create mode 100644 tests/golden/view.backgrounds.json create mode 100644 tests/golden/view.band_items.json create mode 100644 tests/golden/view.bands.json create mode 100644 tests/golden/view.cards.json create mode 100644 tests/golden/view.characters.json create mode 100644 tests/golden/view.comics.json create mode 100644 tests/golden/view.degrees.json create mode 100644 tests/golden/view.events.json create mode 100644 tests/golden/view.exchange_categories.json create mode 100644 tests/golden/view.exchange_products.json create mode 100644 tests/golden/view.friendship_episodes.json create mode 100644 tests/golden/view.friendships.json create mode 100644 tests/golden/view.gacha.json create mode 100644 tests/golden/view.home_banners.json create mode 100644 tests/golden/view.items.json create mode 100644 tests/golden/view.jackets.json create mode 100644 tests/golden/view.login_bonuses.json create mode 100644 tests/golden/view.monthly_passes.json create mode 100644 tests/golden/view.rewards.json create mode 100644 tests/golden/view.season_passes.json create mode 100644 tests/golden/view.shops.json create mode 100644 tests/golden/view.skill_icons.json create mode 100644 tests/golden/view.spots.json create mode 100644 tests/golden/view.stamps.json create mode 100644 tests/golden/view.story_chapters.json create mode 100644 tests/golden/view.story_episodes.json create mode 100644 tests/golden/view.support_cards.json create mode 100644 tests/golden/view.voices.json create mode 100644 tests/pipeline/product_goldens.py create mode 100644 tests/pipeline/view_goldens.py diff --git a/.github/goldens-constraints.txt b/.github/goldens-constraints.txt index 9d81f28..c3f8815 100644 --- a/.github/goldens-constraints.txt +++ b/.github/goldens-constraints.txt @@ -3,3 +3,6 @@ Pillow==12.3.0 numpy==2.4.6 texture2ddecoder==1.0.6 astc-encoder-py==0.1.12 +fonttools==4.67.0 +freetype-py==2.5.1 +scipy==1.18.1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6893a7..1245c80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: "24" - uses: actions/setup-python@v7 with: python-version: "3.13" @@ -75,7 +78,7 @@ jobs: - name: Install (pinned) run: | python -m pip install --upgrade pip - python -m pip install -c .github/goldens-constraints.txt -e ".[test]" + python -m pip install -c .github/goldens-constraints.txt -e ".[test,fonts]" - name: Golden records env: GOLDENS_STRICT: "1" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 807a9d9..18f4e7e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,7 +32,8 @@ mkdocs build --strict # the documentation site - **Golden records.** `tests/golden/` records the output of each stage version. When a change alters a stage's output, raise that stage's (or converter's) version and rewrite the records with `GOLDENS_UPDATE=1 pytest tests/pipeline/test_goldens.py`, using the library versions of - `.github/goldens-constraints.txt` (`pip install -c .github/goldens-constraints.txt -e ".[test]"`). + `.github/goldens-constraints.txt` (`pip install -c .github/goldens-constraints.txt -e ".[test,fonts]"`). + Node.js runs the synthetic read-set fixture. CI uses Node.js 24 and Python 3.13 for these records. - **Command reference.** `docs/cli.md` is generated from the parser. After changing a command or an option, run `python -m nnnotes.cli.reference docs/cli.md`; a test fails while the file is out of date. diff --git a/docs/determinism.md b/docs/determinism.md index 5b7113d..1c71e76 100644 --- a/docs/determinism.md +++ b/docs/determinism.md @@ -42,7 +42,9 @@ another installation computes other keys instead of mixing results. bytes and, for every synthetic fixture, the task key and the SHA-256 of the canonical result. A change of a stage's output without a new stage version fails the test. The CI job that checks the records installs the exact library versions of `.github/goldens-constraints.txt` and sets `GOLDENS_STRICT=1`; elsewhere a record made with - other library versions is skipped. + other library versions is skipped. Every registered output stage and site stage has a record; the coverage + check includes the master-data view family. Product fixtures exercise shared objects, localized references, + open and game fonts, host state, media indexes and regional site aliases. - **Two runs, one result.** The tests run exports twice over synthetic inputs and compare the bytes, and compare one worker with several: a depot built with four workers equals one built with one, and parallel audio decoding equals sequential decoding. diff --git a/tests/content/test_storyfontproduct.py b/tests/content/test_storyfontproduct.py index cceb857..e243fef 100644 --- a/tests/content/test_storyfontproduct.py +++ b/tests/content/test_storyfontproduct.py @@ -25,7 +25,7 @@ def fixture(depot): fonts, keys = sui.font_objects() - objs = {**sui.objects(), **fonts} + objs = copy.deepcopy({**sui.objects(), **fonts}) for cls, _, files in objs.values(): if cls == "GameObject": for node in files["json"]["nodes"]: diff --git a/tests/golden/export.audio.json b/tests/golden/export.audio.json new file mode 100644 index 0000000..8a221cf --- /dev/null +++ b/tests/golden/export.audio.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "41112a5fdab532b5721a433f16872efa9a8b89e55d24270410b5e4d92803683e", + "result": "accb5e67cc13afad20c84ba356037f9244e9aa2e51692d6a40c25f14a220e5a6" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:audio_files", + "stage": "export.audio", + "version": 1 +} diff --git a/tests/golden/export.episode.json b/tests/golden/export.episode.json new file mode 100644 index 0000000..fad8f8a --- /dev/null +++ b/tests/golden/export.episode.json @@ -0,0 +1,17 @@ +{ + "atoms": {}, + "fixtures": { + "localized": { + "key": "4ec26e639b22c2874c7c45ae84637bd5bade08cb08d23560df8262245c9f62a3", + "result": "c0da3d411261d619f3abf70062e15bb31c1f734fc258da31dd730310bb0ade21" + }, + "plain": { + "key": "09d2e88eb8cdd8511855d58d3598e62d2aae45bff8aaf4f074f050024488c59d", + "result": "cda1886dc173756a9da11374f4c162112258d1d3e9846d5bd7e02ff936f427ba" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:episode", + "stage": "export.episode", + "version": 1 +} diff --git a/tests/golden/export.shader.json b/tests/golden/export.shader.json new file mode 100644 index 0000000..481eb4c --- /dev/null +++ b/tests/golden/export.shader.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "39193e323146f2805ce341c3cb90c4ede59b939747c6f8bf449de717025a6ead", + "result": "74f460fa67833f249f45c3f46e98f551e034985ee6ce8e6e6481d666a4b106ec" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:shader_files", + "stage": "export.shader", + "version": 1 +} diff --git a/tests/golden/live.json b/tests/golden/live.json new file mode 100644 index 0000000..3751631 --- /dev/null +++ b/tests/golden/live.json @@ -0,0 +1,19 @@ +{ + "atoms": {}, + "fixtures": { + "localized": { + "key": "2a1a030ff8d9c41bc4bea42f8572f3c9527df9edcbf79cdb2f1236467e90fc13", + "result": "d2c5cdf874d2be3fae0d9ee205a00c0566f36e77069e25c73fca5d3657cd6174" + }, + "plain": { + "key": "5f49098180f1ee3dcdd41dff64921268a9d244880b4a3a77953c4903a143e5d4", + "result": "25d86966819f599dcfa455886e7f05d4d6860ebb4cda0d1744d65918191772ea" + } + }, + "libraries": { + "Pillow": "12.3.0" + }, + "provider": "tests.pipeline.product_goldens:live_product", + "stage": "live", + "version": 3 +} diff --git a/tests/golden/live.music.json b/tests/golden/live.music.json new file mode 100644 index 0000000..4d0c44c --- /dev/null +++ b/tests/golden/live.music.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "f722289a5aa1e379e221a5b9ce3bd62fd890fa28fd0b49d7f6a3a08207a42356", + "result": "3bd878866f24b26b4b9ecd0371afb2afaa371f09a3d70693eff536198f38efbd" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:live_music", + "stage": "live.music", + "version": 1 +} diff --git a/tests/golden/live.noteeffect.json b/tests/golden/live.noteeffect.json new file mode 100644 index 0000000..1d0fc3c --- /dev/null +++ b/tests/golden/live.noteeffect.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "2b025288728e21e521677ac4ede0a8cfa2b5acf51824cfec3df89d0d1e307c4b", + "result": "f9664f38f9eac3fe7a152be34da7613e3411a1ee27c7ca39cf53fe657e99184a" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:note_effect", + "stage": "live.noteeffect", + "version": 1 +} diff --git a/tests/golden/live.notes.json b/tests/golden/live.notes.json new file mode 100644 index 0000000..35e9174 --- /dev/null +++ b/tests/golden/live.notes.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "a15dc44191d70eba44810b8b6c92144372b9db71e015b0d9e12634a89e7f1b48", + "result": "365d6f29adf4da2db9f2eb2a5c664b12de4ace3fe0f798f1af96f7793f6dc36d" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:live_notes", + "stage": "live.notes", + "version": 1 +} diff --git a/tests/golden/live.noteskin.json b/tests/golden/live.noteskin.json new file mode 100644 index 0000000..6d52c24 --- /dev/null +++ b/tests/golden/live.noteskin.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "a0d571181dc31d120d04511c7ab3a87fc8079944241502b55cce23d649c2fae1", + "result": "4bf59668b1fe696aec4885f1e6165dd27a63fb35a337034d007e0dcac33f7543" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:note_skin", + "stage": "live.noteskin", + "version": 1 +} diff --git a/tests/golden/live.scene.json b/tests/golden/live.scene.json new file mode 100644 index 0000000..33c1873 --- /dev/null +++ b/tests/golden/live.scene.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "0dc09c47fb308e4631f250f0646d4705f74157b7854998b972e8a5c5ac154606", + "result": "b510730bc5eb43e1e7abf6a887d4012aaadc11b1736e2fe6e2898f4fc31ede7e" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:live_scene", + "stage": "live.scene", + "version": 1 +} diff --git a/tests/golden/live.sounds.json b/tests/golden/live.sounds.json new file mode 100644 index 0000000..7b689ea --- /dev/null +++ b/tests/golden/live.sounds.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "d4eb8729903f30d2d7109543398905354902e528480f1fb7188e708bf61b44ec", + "result": "713fb8026163ef10eaa3dbdae6a23d433fc753ce5c55a0ba4f5ae4d39ad867b6" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:live_sounds", + "stage": "live.sounds", + "version": 1 +} diff --git a/tests/golden/live.ui.json b/tests/golden/live.ui.json new file mode 100644 index 0000000..566f634 --- /dev/null +++ b/tests/golden/live.ui.json @@ -0,0 +1,25 @@ +{ + "atoms": { + "text.glyphs": "fonttools-4.67.0+freetype-py-2.5.1+scipy-1.18.1+numpy-2.4.6+pillow-12.3.0/1" + }, + "fixtures": { + "game": { + "key": "b55d0f5e1dbfa2c760981e5d6e204887ea3406a9ba4eb9dda4935275af0ee65c", + "result": "2512e62b2bf9740848816b3790f9e812728e3482818f819a12252e1b33720215" + }, + "open": { + "key": "b3d25ad7514c04176c3d1b2a824c5cab3459f685491174423cbdd3f05b6eb164", + "result": "14f534ebc34a5c866a1c086660df6c6cacce9a5c42207a5e1cf49d9d7c928475" + } + }, + "libraries": { + "Pillow": "12.3.0", + "fonttools": "4.67.0", + "freetype-py": "2.5.1", + "numpy": "2.4.6", + "scipy": "1.18.1" + }, + "provider": "tests.pipeline.product_goldens:live_ui", + "stage": "live.ui", + "version": 1 +} diff --git a/tests/golden/live2d.cubism.json b/tests/golden/live2d.cubism.json new file mode 100644 index 0000000..0a814ce --- /dev/null +++ b/tests/golden/live2d.cubism.json @@ -0,0 +1,15 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "11b2521f18abe7eace6a3c93a3341e18ea449515404d7dc6022ff79d0dbf8515", + "result": "c2f5e733fb1b1624973ae499d73a93989763e5c212173d8f04f8658ed2b7c890" + } + }, + "libraries": { + "Pillow": "12.3.0" + }, + "provider": "tests.pipeline.product_goldens:cubism", + "stage": "live2d.cubism", + "version": 1 +} diff --git a/tests/golden/live2d.model.json b/tests/golden/live2d.model.json new file mode 100644 index 0000000..89c6af5 --- /dev/null +++ b/tests/golden/live2d.model.json @@ -0,0 +1,15 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "a7a808010e418683d1441c9e322485a193dbf35eae40811f370a205cacde5169", + "result": "4ad4f0b81b595eb7a72b0f913ae1ced0e90721415e6c0996aaf825656799e393" + } + }, + "libraries": { + "Pillow": "12.3.0" + }, + "provider": "tests.pipeline.product_goldens:model", + "stage": "live2d.model", + "version": 2 +} diff --git a/tests/golden/room.gltf.json b/tests/golden/room.gltf.json new file mode 100644 index 0000000..5a4905c --- /dev/null +++ b/tests/golden/room.gltf.json @@ -0,0 +1,16 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "e2cbd5b59e83e9644e3ffffb8878c5f3c20822b258c6fd756b70d71837b2ff2e", + "result": "6e3940b26686ce64451591d838fa246066304494410cff1a0a62bd8c8240de88" + } + }, + "libraries": { + "Pillow": "12.3.0", + "numpy": "2.4.6" + }, + "provider": "tests.pipeline.product_goldens:room", + "stage": "room.gltf", + "version": 3 +} diff --git a/tests/golden/site.asset.json b/tests/golden/site.asset.json new file mode 100644 index 0000000..8a709fe --- /dev/null +++ b/tests/golden/site.asset.json @@ -0,0 +1,15 @@ +{ + "atoms": { + "json.minify": "nnnotes-json-minify/1" + }, + "fixtures": { + "shared-objects": { + "key": "b8054ff0826d71b798a22b2642b0a60a998945c3d50fec39db625a30b09dffb8", + "result": "db2b695801326f1af4f1d0498b032f180f6bb87f7a9dcf7c4fe268d94bb766a0" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:site_asset", + "stage": "site.asset", + "version": 1 +} diff --git a/tests/golden/site.index.json b/tests/golden/site.index.json new file mode 100644 index 0000000..ca0a7be --- /dev/null +++ b/tests/golden/site.index.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "064f1dcdf2e7f6920f0f41469611b386daa9a21bb8a96ccdffbaa7c3a78bb53f", + "result": "7cfe142f0da8737073a51f21cad2709fbed25463bf7140e0236289d2f77ea987" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:site_index", + "stage": "site.index", + "version": 3 +} diff --git a/tests/golden/site.manifest.json b/tests/golden/site.manifest.json new file mode 100644 index 0000000..a882d43 --- /dev/null +++ b/tests/golden/site.manifest.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "126796fb037b4dfd0aaa8c219ead8b70794e13739ff4b0935aadd7f965282491", + "result": "1eaed8e14807c0bfeeebeb186edb4e97b394fa16979a05997f93de645fd2e684" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:site_manifest", + "stage": "site.manifest", + "version": 2 +} diff --git a/tests/golden/site.readset.json b/tests/golden/site.readset.json new file mode 100644 index 0000000..9d149a9 --- /dev/null +++ b/tests/golden/site.readset.json @@ -0,0 +1,19 @@ +{ + "atoms": { + "player.node": "node:synthetic" + }, + "fixtures": { + "full": { + "key": "58df8dde3e7fcf1f4c4d54420fee6c844797616dc9f1ebc984fc60a64e950086", + "result": "c146b94c7a376aec2132f412e23f0377b5b87c7244ea729ab81d9b80179b0494" + }, + "plan": { + "key": "9a24e8efbd4030b10a3c6ad9f200e3710306f771858326d815b9d1e4ca2a0971", + "result": "8f4864f6d755ab1ea2d111beae98658b2ad9a24f6f00c92a4a201cfd4413680a" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:site_readset", + "stage": "site.readset", + "version": 1 +} diff --git a/tests/golden/site.references.json b/tests/golden/site.references.json new file mode 100644 index 0000000..c8a3e3d --- /dev/null +++ b/tests/golden/site.references.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "90de4605df9de8f19d072e5d6a18c6673a102700ce5b434841012cac1b69e450", + "result": "1ae2c8d13482a9b46935aef06e1306336fffdf1db955b4d2e848affa5f4bff56" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:site_references", + "stage": "site.references", + "version": 1 +} diff --git a/tests/golden/site.selection.json b/tests/golden/site.selection.json new file mode 100644 index 0000000..0e093de --- /dev/null +++ b/tests/golden/site.selection.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "c83b495600ff5ed9e33d751d333680fd98d8839e4d7e77c2dff3266759128c77", + "result": "5424bac9bd42c50356c5ae6d219f069097d332ffd2be8be8d1e0f2e5d793c0e4" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:site_selection", + "stage": "site.selection", + "version": 2 +} diff --git a/tests/golden/spot.files.json b/tests/golden/spot.files.json new file mode 100644 index 0000000..40931ba --- /dev/null +++ b/tests/golden/spot.files.json @@ -0,0 +1,15 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "466f3cc629f8a9e32aa61f6f46bc28bed0620ffffdd3388741ea5fb313cc50a0", + "result": "5f54154505303501ee4f500b350ca74aea9180922fef17bbdfdcd4c739da011c" + } + }, + "libraries": { + "Pillow": "12.3.0" + }, + "provider": "tests.pipeline.product_goldens:spot_product", + "stage": "spot.files", + "version": 1 +} diff --git a/tests/golden/story.asset.json b/tests/golden/story.asset.json new file mode 100644 index 0000000..8f7f5eb --- /dev/null +++ b/tests/golden/story.asset.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "3f2e85369f6a16ffaa7de7580b6d25c75921361bf2496cefc11657e84790f707", + "result": "766c92a59db7f027d56129ced3dcfd8b4e60890e56ac575f6880cdbf6d7d1715" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:story_asset", + "stage": "story.asset", + "version": 1 +} diff --git a/tests/golden/story.chat.json b/tests/golden/story.chat.json new file mode 100644 index 0000000..ba1625a --- /dev/null +++ b/tests/golden/story.chat.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "44ffb9d2e46113e7c184510afef8fb2b68c3b4ded898169ae53e8c18eeaff0dc", + "result": "d85782cc5d587f5f739ce0a7472d2274732def5e8c4825dcb5ecf1a1cfd4e107" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:story_chat", + "stage": "story.chat", + "version": 1 +} diff --git a/tests/golden/story.crilips.json b/tests/golden/story.crilips.json new file mode 100644 index 0000000..bb348c0 --- /dev/null +++ b/tests/golden/story.crilips.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "e1fa705792ad005045e97ee4afdbcafcdc901ba29c4192c2d179e3fc6540a8df", + "result": "19f98c075f6762daf7617192b9c2f84388644300dc20629a0ed2d18a3ab15c4c" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:crilips", + "stage": "story.crilips", + "version": 1 +} diff --git a/tests/golden/story.fonts.json b/tests/golden/story.fonts.json new file mode 100644 index 0000000..eb2d5ba --- /dev/null +++ b/tests/golden/story.fonts.json @@ -0,0 +1,25 @@ +{ + "atoms": { + "text.glyphs": "fonttools-4.67.0+freetype-py-2.5.1+scipy-1.18.1+numpy-2.4.6+pillow-12.3.0/1" + }, + "fixtures": { + "game": { + "key": "453b091bd80d784ce02d9c2d2ce7e03b0ca887a51c3b573da9aa7ba2aec8764b", + "result": "3314357a566dd4e9b103a960b3908ed6af9635571dc8b37da76535b1a6ed0e77" + }, + "open": { + "key": "7c0c1007f05e47d204bb0d573770ff923dafbd8421fcb843d8bc008745aef821", + "result": "849badd509b7529a2f184f95cb4ea93e8afc6768f0602d1a27a83dcc0b4c32a5" + } + }, + "libraries": { + "Pillow": "12.3.0", + "fonttools": "4.67.0", + "freetype-py": "2.5.1", + "numpy": "2.4.6", + "scipy": "1.18.1" + }, + "provider": "tests.pipeline.product_goldens:story_fonts", + "stage": "story.fonts", + "version": 3 +} diff --git a/tests/golden/story.host.json b/tests/golden/story.host.json new file mode 100644 index 0000000..8dd30fa --- /dev/null +++ b/tests/golden/story.host.json @@ -0,0 +1,17 @@ +{ + "atoms": {}, + "fixtures": { + "afterlive": { + "key": "faedfe601ee4a8a89a8ba68baab442f2d048d862535e5d141e647b9d19135f39", + "result": "193b31ded6b4bfac281a9f3c468af703549abafc0b1e662258f07f9f6d324b54" + }, + "home": { + "key": "9ef7c1a1282b788f5c7cdde68bfccef20252ce6bc1e06288d99064ccac244e22", + "result": "cfb64bb54cddcc909d543b17527a2bf47cba85232d301b4a31e2385c3255377e" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:host", + "stage": "story.host", + "version": 1 +} diff --git a/tests/golden/story.json b/tests/golden/story.json new file mode 100644 index 0000000..4465741 --- /dev/null +++ b/tests/golden/story.json @@ -0,0 +1,17 @@ +{ + "atoms": {}, + "fixtures": { + "localized": { + "key": "2f2bbf606c58457fa9f5225c3e58c525a4c015dc668141531f613a2918981633", + "result": "3eb28b45fbc3200aa009c08f436aa5caaf2732e1fead5d7445b808ec96ad15fd" + }, + "plain": { + "key": "c198a2ae7b598c86a5ba0fd557580f6ca5969350acd60b274c6de9bbf3b00eaa", + "result": "358a660979cf5e51e28151920f6e2b38c836c4ea7b2346a0a52b8c0b8255dc58" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:story_product", + "stage": "story", + "version": 5 +} diff --git a/tests/golden/story.scene.json b/tests/golden/story.scene.json new file mode 100644 index 0000000..35a1948 --- /dev/null +++ b/tests/golden/story.scene.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "e8e13d794eafcd74d4d69923449cb3e8abf479b363149b7113becb4aa07a53a4", + "result": "ce348bf420e92cbb62fbdf303faf97b8f6729b716dd2f55e5aaca5ef26d94612" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:story_scene", + "stage": "story.scene", + "version": 1 +} diff --git a/tests/golden/story.simpleui.json b/tests/golden/story.simpleui.json new file mode 100644 index 0000000..7db2ff5 --- /dev/null +++ b/tests/golden/story.simpleui.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "7a4feb04fbd83f2dca4e69ecf43d9b1261c83aabde7c58a50188030029e20c1b", + "result": "400189fcad2abcb283d9ee56e212cb4d0f60cddf5bd01bc88c4dffd5eb268266" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:simple_ui", + "stage": "story.simpleui", + "version": 1 +} diff --git a/tests/golden/story.sound.json b/tests/golden/story.sound.json new file mode 100644 index 0000000..ec76e0f --- /dev/null +++ b/tests/golden/story.sound.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "22bf1839c37bc5c4c6ff8844f0cc97dea89477b77f0395d2da1fbd6874b0af93", + "result": "53556645ce87446e995ea6ed81595cd98543328c4e8c4aa45b87935f1c877f0f" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:story_sound", + "stage": "story.sound", + "version": 2 +} diff --git a/tests/golden/story.ui.json b/tests/golden/story.ui.json new file mode 100644 index 0000000..e53f003 --- /dev/null +++ b/tests/golden/story.ui.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "f3f6116fb51339721fd02de646c4ac45d47ba544071b487d1022b9c56c2152f0", + "result": "5b21052ba82a14b5ad4cb886b72849ce2e76a0e9204f562163afe8608a5dd82e" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:story_ui", + "stage": "story.ui", + "version": 1 +} diff --git a/tests/golden/story.video.json b/tests/golden/story.video.json new file mode 100644 index 0000000..076c997 --- /dev/null +++ b/tests/golden/story.video.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "5baae3e8d9b5bee09af0bdaaa27f655293d99d841fc209ff07f9fd6559405296", + "result": "9e7a03082bdef3b44ead86e1b076f6a8e4b53c0dbe370b92c6a316f9f975192e" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:story_video", + "stage": "story.video", + "version": 1 +} diff --git a/tests/golden/ui.pack.json b/tests/golden/ui.pack.json new file mode 100644 index 0000000..4ed11fd --- /dev/null +++ b/tests/golden/ui.pack.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "a08f47640556a6263e28a9b4b1245eff94d7b1788b587c9993629e9b7c8208b0", + "result": "96ae51c62b243fa334cdfe11cc965d4ac70f4db4ac5fc39ebcdfab0a1b9d41df" + } + }, + "libraries": {}, + "provider": "tests.pipeline.product_goldens:ui_pack", + "stage": "ui.pack", + "version": 1 +} diff --git a/tests/golden/view.backgrounds.json b/tests/golden/view.backgrounds.json new file mode 100644 index 0000000..5c7931c --- /dev/null +++ b/tests/golden/view.backgrounds.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "f6c9b23f289ed6ab71924429ec5139efd292283cee7745fd93524f4128fda245", + "result": "d7cc3ba8322b6d2ff7bc0126688a39a170661c1dd690ad984279c5fc77ac1f3c" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_backgrounds", + "stage": "view.backgrounds", + "version": 1 +} diff --git a/tests/golden/view.band_items.json b/tests/golden/view.band_items.json new file mode 100644 index 0000000..eb2675a --- /dev/null +++ b/tests/golden/view.band_items.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "ad917eeb1c6570c6a2d2a1a53244d72dd95cb69a8001a058b3ce70768e58c033", + "result": "0164bd9d021b0102734a9c11a41ace779170baa32da67644f6cefadb2111c8da" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_band_items", + "stage": "view.band_items", + "version": 1 +} diff --git a/tests/golden/view.bands.json b/tests/golden/view.bands.json new file mode 100644 index 0000000..d3974c2 --- /dev/null +++ b/tests/golden/view.bands.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "7cfe9fdd0c755508477d54cf4a6df1929c4039b9391daa590780e6c09e4b1662", + "result": "4734f4d3f56c350ef339256bcfa10b23f05631c5c4dea10f49ca107dab1d3b99" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_bands", + "stage": "view.bands", + "version": 1 +} diff --git a/tests/golden/view.cards.json b/tests/golden/view.cards.json new file mode 100644 index 0000000..271f51f --- /dev/null +++ b/tests/golden/view.cards.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "fe6485be8c2aa2b79c66febeb528fde1c0679dfc19656d5e1ef375f582d2c28e", + "result": "5ec47ae939095de3ecdccec1c065acd1df87f818158ecd2efda7807494bf3272" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_cards", + "stage": "view.cards", + "version": 1 +} diff --git a/tests/golden/view.characters.json b/tests/golden/view.characters.json new file mode 100644 index 0000000..699b9c4 --- /dev/null +++ b/tests/golden/view.characters.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "58c1456b03c375131bd713f2050d2ca8f0dfd1dbf8cadd8fc569fa45037b6f0e", + "result": "292264e77326c28db3d2d801a5275d5a345b1fe092ae85f71813744bc9ca8f61" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_characters", + "stage": "view.characters", + "version": 1 +} diff --git a/tests/golden/view.comics.json b/tests/golden/view.comics.json new file mode 100644 index 0000000..8c3dfb4 --- /dev/null +++ b/tests/golden/view.comics.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "fa0c8942c3cc20550e0bbe4630235177bac1b492570db81bb8c87d4cb01c3129", + "result": "928bca5b15d7935621afd5bd950bb5013eebce78f9b7373a61688a39a57133cf" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_comics", + "stage": "view.comics", + "version": 1 +} diff --git a/tests/golden/view.degrees.json b/tests/golden/view.degrees.json new file mode 100644 index 0000000..1c12f2e --- /dev/null +++ b/tests/golden/view.degrees.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "fc847afb0b278a21094df63d0d9457343d729f7792b0b161637d9c5790c5e51f", + "result": "21606e5e1d8fbda4ac672d61174885de65644da6d63e96c9ff1f2dc430fe6fc9" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_degrees", + "stage": "view.degrees", + "version": 1 +} diff --git a/tests/golden/view.events.json b/tests/golden/view.events.json new file mode 100644 index 0000000..d2fbdcb --- /dev/null +++ b/tests/golden/view.events.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "58170e5c90d35a2c3e74654c2e183a9d9ac6e50cd87a1f02d6f8261ca20b4c33", + "result": "e6201d588d993313888eaab78ad8cbb28a5b9952cae7e6a7addbf790173c82ae" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_events", + "stage": "view.events", + "version": 1 +} diff --git a/tests/golden/view.exchange_categories.json b/tests/golden/view.exchange_categories.json new file mode 100644 index 0000000..8d9524d --- /dev/null +++ b/tests/golden/view.exchange_categories.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "2dfcdd2d6cb26de71c832502752bdce4310887d7dcd225b61918b599732538bb", + "result": "1c4887c2ebad414e722c6a2d4f43cf11e5eb38788180ad4d1db6dce25bb5f9aa" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_exchange_categories", + "stage": "view.exchange_categories", + "version": 1 +} diff --git a/tests/golden/view.exchange_products.json b/tests/golden/view.exchange_products.json new file mode 100644 index 0000000..c960758 --- /dev/null +++ b/tests/golden/view.exchange_products.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "efad238d0f91fccd88abfe51d0f71b106838ce47ba84e13a92b6c2603b183413", + "result": "fcf7e9b74b88e14562a904ca1d9de98547d1db90f2d3cfa38dd3d5d52536521d" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_exchange_products", + "stage": "view.exchange_products", + "version": 1 +} diff --git a/tests/golden/view.friendship_episodes.json b/tests/golden/view.friendship_episodes.json new file mode 100644 index 0000000..350cd4e --- /dev/null +++ b/tests/golden/view.friendship_episodes.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "d7522bb8db568af3e3f34113aae778d3bb94e3d146087a820473bbb255b0b603", + "result": "339158925684117dd994ee49ce27583464dd719538a5295cacaed50319734cf6" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_friendship_episodes", + "stage": "view.friendship_episodes", + "version": 1 +} diff --git a/tests/golden/view.friendships.json b/tests/golden/view.friendships.json new file mode 100644 index 0000000..035ba8b --- /dev/null +++ b/tests/golden/view.friendships.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "0bfc1475ab5bca824f460f18ca2b8a731007d2c77d2dd464b39d7122e432e5dc", + "result": "ea836b69716e19871eb5c81db80676bb90813ba61345fa2ad28ae3dcfa3cd192" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_friendships", + "stage": "view.friendships", + "version": 1 +} diff --git a/tests/golden/view.gacha.json b/tests/golden/view.gacha.json new file mode 100644 index 0000000..cec45e3 --- /dev/null +++ b/tests/golden/view.gacha.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "81ee1322c864df3c3d547a393fad01508590d718bc9695b12f8e9ccb5ed45131", + "result": "2160c0afcf2b32fc8e4b1fbadf3175d18c292619845fabfbe28fea1338fd7eb2" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_gacha", + "stage": "view.gacha", + "version": 1 +} diff --git a/tests/golden/view.home_banners.json b/tests/golden/view.home_banners.json new file mode 100644 index 0000000..3e3f91f --- /dev/null +++ b/tests/golden/view.home_banners.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "f7aed1008633113de170bca61480f1479b1fd0d42f6cf5a7850317769b08307a", + "result": "21882c5f127b7ca313f2ac697ebe8c134bf9d0de5283f112a66672c40ce4c68d" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_home_banners", + "stage": "view.home_banners", + "version": 1 +} diff --git a/tests/golden/view.items.json b/tests/golden/view.items.json new file mode 100644 index 0000000..b999f9f --- /dev/null +++ b/tests/golden/view.items.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "872586d6f74aaf17cf558639d86c21e4f130b75dcc157241989f6a559924373c", + "result": "b0b16978fd32f0dc8d45647d4df643d78c89677fca6197791b4dcd0484444506" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_items", + "stage": "view.items", + "version": 1 +} diff --git a/tests/golden/view.jackets.json b/tests/golden/view.jackets.json new file mode 100644 index 0000000..eac33e9 --- /dev/null +++ b/tests/golden/view.jackets.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "375c2ce66e5468b01c690d3ced074f88afe469c04d695b73bed78b36d98306a5", + "result": "901122ff2828fe85a1913e81811981a7c8edb5e3e820a38c95ce4b328d408788" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_jackets", + "stage": "view.jackets", + "version": 1 +} diff --git a/tests/golden/view.login_bonuses.json b/tests/golden/view.login_bonuses.json new file mode 100644 index 0000000..abe40ff --- /dev/null +++ b/tests/golden/view.login_bonuses.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "25d92188efa6e7c4cb00425043adb85ba5be4561baf72e2b84485fbfa0dcaf78", + "result": "6071b205c7491bceb6a5fa5d7dc7beaf31d1a608a96044afdcd6f9b146f1be7f" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_login_bonuses", + "stage": "view.login_bonuses", + "version": 1 +} diff --git a/tests/golden/view.monthly_passes.json b/tests/golden/view.monthly_passes.json new file mode 100644 index 0000000..43ea8ef --- /dev/null +++ b/tests/golden/view.monthly_passes.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "079a4ed0323270e237f8b319299ffaa6ec51dedba552f6760b630ce1e7282b33", + "result": "59a2fac563cb7742f0a0196f813c2377f17ce3610c599ca596dcfe7a32f3abaf" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_monthly_passes", + "stage": "view.monthly_passes", + "version": 1 +} diff --git a/tests/golden/view.rewards.json b/tests/golden/view.rewards.json new file mode 100644 index 0000000..d2a67ed --- /dev/null +++ b/tests/golden/view.rewards.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "6851cea55e29ef62edaaf8bd01708dd3a8ee050ca43607e3f79408c5df4fc902", + "result": "7b6dd70c05c19456197cc3bef6507cadf3e6c83eb9fba5a08db4406b359b1a80" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_rewards", + "stage": "view.rewards", + "version": 1 +} diff --git a/tests/golden/view.season_passes.json b/tests/golden/view.season_passes.json new file mode 100644 index 0000000..7540ddc --- /dev/null +++ b/tests/golden/view.season_passes.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "3fb41cba0d46d6fa6cde0566dd412388dcfb967f8f3cc647d3e73e606db4c34d", + "result": "a86ead1fa4fee5c060811209117ffd7606da227b442c8fc5159ae748c8ccea6c" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_season_passes", + "stage": "view.season_passes", + "version": 1 +} diff --git a/tests/golden/view.shops.json b/tests/golden/view.shops.json new file mode 100644 index 0000000..300d259 --- /dev/null +++ b/tests/golden/view.shops.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "68eeb9135617ec42b7ab304693ace0fa2740adaac7de0ec67c1b7d5c1715bf54", + "result": "a0ac1eb948b1bef61613c4c5e1f8324e5b2e6e8d47cd8c529b3cfc68d4502171" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_shops", + "stage": "view.shops", + "version": 1 +} diff --git a/tests/golden/view.skill_icons.json b/tests/golden/view.skill_icons.json new file mode 100644 index 0000000..2987d76 --- /dev/null +++ b/tests/golden/view.skill_icons.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "4136a7ae83122732a92d00b18ad929e722e8e4f15f55d7d19338d6513fd5d037", + "result": "793dcf09837ae17ec99d1bc4e0b1c791bdb12a3cbb434456503cf829386e85c6" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_skill_icons", + "stage": "view.skill_icons", + "version": 1 +} diff --git a/tests/golden/view.spots.json b/tests/golden/view.spots.json new file mode 100644 index 0000000..52332b6 --- /dev/null +++ b/tests/golden/view.spots.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "ba66e2c80fb68ae5fae9385500dcb423efd0273c0db2c8ce1a6c80de62c9fa5d", + "result": "5dcbbc29e5800cc7a124a4dcd5d269800ce773b781f882aaaf8ad2e48f89912b" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_spots", + "stage": "view.spots", + "version": 1 +} diff --git a/tests/golden/view.stamps.json b/tests/golden/view.stamps.json new file mode 100644 index 0000000..4d87d08 --- /dev/null +++ b/tests/golden/view.stamps.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "a4830c8f81286576d359b2d8e65376b59d49c791f76096dfc8098959be952892", + "result": "c871bacd85c219c910851957e591b295880449707f21a7cbdaec4e3f8abfd793" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_stamps", + "stage": "view.stamps", + "version": 1 +} diff --git a/tests/golden/view.story_chapters.json b/tests/golden/view.story_chapters.json new file mode 100644 index 0000000..e73d03b --- /dev/null +++ b/tests/golden/view.story_chapters.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "e6384d5e61c6820923e36c0ab5e63829232b39c12d14f803c6ce7cf75555232d", + "result": "36ef14f3ea366477131d38036e0df978ccfd10fa8f6aa44a674dffc85319b69d" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_story_chapters", + "stage": "view.story_chapters", + "version": 1 +} diff --git a/tests/golden/view.story_episodes.json b/tests/golden/view.story_episodes.json new file mode 100644 index 0000000..316df68 --- /dev/null +++ b/tests/golden/view.story_episodes.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "8ae87e3d0b3a20e94e6d092753cf43054f43b17ca4b4575600a8e7593f5c9b27", + "result": "7e6ffda8a16ffae0ddbf4e9582f117109ec2c8fc7875c74e46f56745a1c1d421" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_story_episodes", + "stage": "view.story_episodes", + "version": 1 +} diff --git a/tests/golden/view.support_cards.json b/tests/golden/view.support_cards.json new file mode 100644 index 0000000..34d0bbb --- /dev/null +++ b/tests/golden/view.support_cards.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "c6b79da86cc7081424d5d3b45db295b49389162b8eb27c40ada82eb560c53811", + "result": "5c9e316cbedc230807a739d08b01299a7ee95f867f33c999a56159c8414371a3" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:view_support_cards", + "stage": "view.support_cards", + "version": 1 +} diff --git a/tests/golden/view.voices.json b/tests/golden/view.voices.json new file mode 100644 index 0000000..4761467 --- /dev/null +++ b/tests/golden/view.voices.json @@ -0,0 +1,13 @@ +{ + "atoms": {}, + "fixtures": { + "shared-objects": { + "key": "561e5b256f71c0dc357f4eac0f355e0c55a8f7ebefe704594aac92f9a086f0e8", + "result": "8beb275a0504ec24f84206e0a0baa52318da99b300aa6f7133f22cc734035688" + } + }, + "libraries": {}, + "provider": "tests.pipeline.view_goldens:voice_index", + "stage": "view.voices", + "version": 2 +} diff --git a/tests/helpers/fonts.py b/tests/helpers/fonts.py index d01b1cb..ba31068 100644 --- a/tests/helpers/fonts.py +++ b/tests/helpers/fonts.py @@ -73,6 +73,8 @@ def glyph(contours): "licenseInfoURL": "https://example.invalid/license", } ) + fb.font["head"].created = fb.font["head"].modified = 2082844800 + fb.font.recalcTimestamp = False buf = io.BytesIO() fb.save(buf) path.write_bytes(buf.getvalue()) diff --git a/tests/pipeline/product_goldens.py b/tests/pipeline/product_goldens.py new file mode 100644 index 0000000..8b7a35d --- /dev/null +++ b/tests/pipeline/product_goldens.py @@ -0,0 +1,558 @@ +"""Synthetic task inputs for product-stage golden records.""" + +from nnnotes.pipeline import contract, live, live2d, readsets, siteassets, siteindex, siteproducts, spot, story +from nnnotes.pipeline.contract import Task +from nnnotes.pipeline.exportfiles import EpisodeStage +from nnnotes.pipeline.stages import Env +from tests.helpers import cri, live2dmodel, livenotes, livescene, spotobjects, storyui, synth +from tests.helpers import live as live_objects +from tests.helpers import story as story_objects +from tests.pipeline import test_live, test_spot, test_story +from tests.pipeline.test_live import master_record, stored + + +def task(stage, subject, inputs, params=None, atoms=None): + return Task(stage.name, stage.version, subject, stage.normalize(params), atoms or {}, tuple(inputs)) + + +def objects(depot, table): + return stored(depot, contract.encode(table), "objects") + + +def environment(depot, name, builder): + directory = depot.root / "fixtures" / name + directory.mkdir(parents=True) + return Env(depot, {"master": builder(directory)}) + + +def provider(stage, build, libraries=()): + return {"stage": stage, "fixtures": {"shared-objects": build}, "libraries": libraries} + + +def model(): + stage = live2d.ModelStage() + return provider(stage, lambda d: task(stage, live2dmodel.KEY, [objects(d, live2dmodel.table(d))]), ["Pillow"]) + + +def cubism(): + stage = live2d.CubismStage() + return provider(stage, lambda d: task(stage, live2dmodel.KEY, [objects(d, live2dmodel.table(d))]), ["Pillow"]) + + +def room(): + stage = spot.RoomStage() + + def build(depot): + table = spotobjects.table(depot, spotobjects.room_objects(), [spotobjects.ROOM_ROOT]) + return task(stage, spotobjects.BACKGROUND, [objects(depot, table)]) + + return provider(stage, build, ["Pillow", "numpy"]) + + +def spot_product(): + stage = spot.SpotStage() + + def build(depot): + env = environment(depot, "spot", test_spot.master_dir) + table = spotobjects.table(depot, spotobjects.situation_objects(), [spotobjects.SIT_ROOT]) + return task( + stage, spotobjects.SITUATION, [objects(depot, table), master_record(env, stage, spotobjects.SITUATION)] + ) + + return provider(stage, build, ["Pillow"]) + + +def live_scene(): + stage = live.SceneStage() + + def build(depot): + env = environment(depot, "scene", test_live.scene_master_dir) + record = master_record(env, stage, "default") + table = livescene.table(depot, contract.loads(depot.input_bytes(record))) + return task(stage, "default", [objects(depot, table), record]) + + return provider(stage, build) + + +def live_notes(): + stage = live.NotesStage() + + def build(depot): + env = environment(depot, "notes", test_live.master_dir) + table = livenotes.table(depot, livenotes.NOTES_KEYS) + return task(stage, "default", [objects(depot, table), master_record(env, stage, "default")]) + + return provider(stage, build) + + +def live_music(): + stage = live.MusicStage() + + def build(depot): + env = environment(depot, "music", test_live.master_dir) + sheets, _ = test_live.sheet_inputs(depot, ["bgm_7"]) + return task(stage, "7", [test_live.table(depot), *sheets, master_record(env, stage, "7")], {"audio": "flac"}) + + return provider(stage, build) + + +def live_sounds(): + stage = live.SoundsStage() + + def build(depot): + env = environment(depot, "sounds", test_live.master_dir) + subject = "NoteSePatternId=1,2" + sheets, _ = test_live.sheet_inputs(depot, ["LiveSe", "se1", "se2"]) + acf = cri.acf({"LiveNotesSe": (3, 0), "LiveSe": (4, 0), "Bgm": (0, 0)}) + return task( + stage, + subject, + [ + test_live.table(depot, [(94, test_live.SVS), (97, test_live.ACD)]), + *sheets, + stored(depot, acf, f"apk:{test_live.ACF}"), + master_record(env, stage, subject), + ], + ) + + return provider(stage, build) + + +def live_product(): + stage = live.LiveStage() + + def fixture(name, languages): + def build(depot): + env = environment(depot, name, test_live.live_master_dir) + env.params = stage.normalize({"languages": languages}) + return task( + stage, + "7/expert", + [objects(depot, live_objects.table(depot, gz=False)), master_record(env, stage, "7/expert")], + env.params, + ) + + return build + + return { + "stage": stage, + "fixtures": {"plain": fixture("plain", []), "localized": fixture("localized", ["ja", "en"])}, + "libraries": ["Pillow"], + } + + +def episode_provider(stage): + def fixture(name, languages): + def build(depot): + env = environment(depot, name, story_objects.master_dir) + env.params = stage.normalize({"languages": languages}) + return task( + stage, + "1", + [objects(depot, story_objects.episode_table(depot)), master_record(env, stage, "1")], + env.params, + ) + + return build + + return { + "stage": stage, + "fixtures": {"plain": fixture("plain", []), "localized": fixture("localized", ["ja", "en"])}, + } + + +def story_product(): + return episode_provider(story.StoryStage()) + + +def episode(): + return episode_provider(EpisodeStage()) + + +def story_chat(): + stage = story.ChatStage() + return provider(stage, lambda d: task(stage, "default", [objects(d, story_objects.chat_table(d))])) + + +def story_sound(): + stage = story.SoundStage() + + def build(depot): + inputs, _ = test_story.sheet(depot, "adv_voice_1") + return task(stage, "adv_voice_1", [test_live.table(depot), *inputs], {"audio": "flac"}) + + return provider(stage, build) + + +def story_ui(): + stage = story.UiStage() + + def build(depot): + fonts, keys = storyui.font_objects() + chat_key, chat_oid, chat = storyui.chat_window() + table = storyui.table_doc( + depot, + objs={**storyui.objects(), **fonts, chat_oid: chat}, + keys={**storyui.table_keys(), **keys, chat_key: [chat_oid]}, + ) + return task( + stage, + "zh-Hant", + [ + objects(depot, table), + stored(depot, contract.encode(storyui.table_doc(depot)), "ui.objects"), + stored(depot, contract.encode({"CAB-font": "font.bundle"}), "font-bundles"), + stored(depot, synth.axml(["versionName", "manifest", "1.2.3"], 2), "apk:AndroidManifest.xml"), + stored( + depot, + contract.encode({"texts": {str(i): f"zh {i}" for i in (*range(101, 111), 301, 302)}}), + "master", + ), + ], + ) + + return provider(stage, build) + + +def story_video(): + stage = story.VideoStage() + return provider( + stage, lambda d: task(stage, test_story.VIDEO_KEY, [test_story.video_table(d), *test_story.movie_inputs(d)]) + ) + + +def note_skin(): + stage = live.NoteSkinStage() + return provider(stage, lambda d: task(stage, "skin001", [objects(d, livenotes.table(d, livenotes.SKIN_KEYS))])) + + +def note_effect(): + stage = live.NoteEffectStage() + return provider( + stage, lambda d: task(stage, "effect001Light", [objects(d, livenotes.table(d, livenotes.EFFECT_KEYS))]) + ) + + +def story_scene(): + from tests.content import test_storyscene as scene + + stage = story.SceneStage() + return provider( + stage, lambda d: task(stage, "default", [objects(d, livenotes.table(d, scene.keys(), scene.objects()))]) + ) + + +def story_asset(): + from tests.content import test_storyasset as asset + + stage = story.AssetStage() + return provider( + stage, + lambda d: task(stage, asset.KEY, [objects(d, livenotes.table(d, {asset.KEY: [asset.PREFAB]}, asset.SMALL))]), + ) + + +def ui_pack(): + from nnnotes.pipeline.ui import UiStage + from tests.content import test_ui + + stage = UiStage() + + def build(depot): + directory = depot.root / "fixture" + directory.mkdir(parents=True) + other, graph = test_ui.graph(directory) + for entry in graph.doc["objects"].values(): + for file in entry["files"].values(): + depot.put(other.read(file["sha256"])) + return task(stage, test_ui.KEY, [objects(depot, graph.doc)]) + + return provider(stage, build) + + +def host(): + from nnnotes.pipeline.storyhost import HostStage + from tests.content.test_storyhostproduct import host_table + + stage = HostStage() + + def fixture(kind): + def build(depot): + table, record, _ = host_table(depot, kind) + master = {"host": record, "parts": {"room": "rooms/room.json", "spot": "spots/day.json"}} + return task( + stage, + "home/7" if kind == "home" else "afterlive", + [objects(depot, table), stored(depot, contract.encode(master), "master")], + ) + + return build + + return {"stage": stage, "fixtures": {kind: fixture(kind) for kind in ("home", "afterlive")}} + + +def simple_ui(): + from tests.content.test_storysimpleui import simple_keys + + stage = story.SimpleUiStage() + + def build(depot): + base = story_ui()["fixtures"]["shared-objects"](depot) + inputs = [i for i in base.inputs if i.role != "ui.objects"] + inputs.append(stored(depot, contract.encode(storyui.table_doc(depot, keys=simple_keys())), "ui.objects")) + return task(stage, "zh-Hant", inputs) + + return provider(stage, build) + + +FONT_LIBRARIES = ("fonttools", "freetype-py", "scipy", "numpy", "Pillow") + + +def font_provider(stage, fixture, subject, *, extra=None): + from tests.helpers.fonts import make_font + + def build(source): + def inputs(depot): + values = fixture(depot)[0] + if source == "open": + data = make_font(depot.root / "font.ttf", underscore=True, extra=extra).read_bytes() + values += [stored(depot, data, f"font:{code}") for code in ("ja", "zh-Hant")] + return task(stage, subject, values, {"fonts": source}, story.font_atoms()) + + return inputs + + return { + "stage": stage, + "fixtures": {source: build(source) for source in ("game", "open")}, + "libraries": FONT_LIBRARIES, + } + + +def story_fonts(): + from tests.content.test_storyfontproduct import fixture + + return font_provider(story.FontsStage(), fixture, "1/zh-Hant") + + +def live_ui(): + from tests.content.test_liveuiproduct import fixture + + return font_provider(live.UiStage(), fixture, "7/expert/zh-Hant", extra={ord(c): "A" for c in "0257"}) + + +class SyntheticLips(story.CriLipsStage): + def run(self, task, depot): + from unittest.mock import patch + + from nnnotes.formats import crilips + + version, _, _ = test_story._synthetic_version() + with patch.object(crilips, "SUPPORTED", (version,)): + return super().run(task, depot) + + +def crilips(): + stage = SyntheticLips() + + def build(depot): + _, rodata, _ = test_story._synthetic_version() + data = test_story._elf_with_rodata(rodata) + return task(stage, "default", [test_live.table(depot), stored(depot, data, f"apk:{story.LIBRARY}")]) + + return provider(stage, build) + + +def audio_files(): + from nnnotes.pipeline.exportfiles import AudioFilesStage + + stage = AudioFilesStage() + + def build(depot): + inputs, sheets = test_story.sheet(depot, "sample") + cues = {} + for stream in test_story.STREAMS: + cues.setdefault(stream["name"], {k: v for k, v in stream.items() if k not in {"name", "stream"}}) + sheets["Cri/Sound/sample"]["files"]["cues.json"] = depot.add(contract.encode(cues), "json") + inputs[-1] = stored(depot, contract.encode(sheets), "sheets") + return task(stage, "sample", [test_live.table(depot), *inputs]) + + return provider(stage, build) + + +def shader_files(): + from nnnotes.pipeline.exportfiles import ShaderFilesStage + + stage = ShaderFilesStage() + + def build(depot): + variant = { + "name": "Example/Unlit", + "variants": [ + { + "platform": "gles3", + "stage": "vertex", + "role": "program:gles3/vertex", + "file": "Example_Unlit/vertex.glsl", + } + ], + } + files = { + "json": (contract.encode({"name": variant["name"], "properties": []}), "json"), + "index": (contract.encode(variant), "json"), + "program:gles3/vertex": (b"void main() {}", "glsl"), + } + artifacts = { + contract.artifact_id("unity.export:shader", role): {"content": depot.add(body, ext)} + for role, (body, ext) in files.items() + } + records = [ + { + "source": "example.bundle", + "items": [{"class": "Shader", "status": "exported", "artifacts": list(artifacts)}], + "artifacts": artifacts, + } + ] + return task(stage, "selected", [stored(depot, contract.encode(records), "records")]) + + return provider(stage, build) + + +def document_provider(stage, source, spec): + def build(depot): + return task( + stage, + "example", + [stored(depot, contract.encode(source), "source"), stored(depot, contract.encode(spec), "spec")], + ) + + return provider(stage, build) + + +def site_references(): + source = {"parts": {"ui": {"ja": "ui/a.json", "en": "ui/b.json"}, "models": ["models/a.json"]}} + spec = { + "languagePaths": [["parts", "ui"]], + "languages": ["en"], + "paths": [["parts", "ui"], ["parts", "models"]], + "routes": {"ui/b.json": "products/ui.json", "models/a.json": "products/model.json"}, + } + return document_provider(siteproducts.ReferencesStage(), source, spec) + + +def site_product(): + return { + "schema": contract.PRODUCT, + "product": "live", + "subject": "7/expert", + "files": { + "live.json": {"asset": "assets/source.json", "size": 2}, + "music.flac": {"asset": "assets/music.flac", "size": 4}, + }, + "objects": { + "model:1": { + "class": "TextAsset", + "name": "Chart", + "files": { + "json": {"asset": "assets/chart.json", "size": 2}, + "data": {"asset": "assets/chart.bin", "size": 1}, + }, + } + }, + } + + +def site_manifest(): + source = site_product() + spec = { + "files": {name: {**entry, "asset": "assets/rewritten-" + name} for name, entry in source["files"].items()}, + "objects": {"model:1": source["objects"]["model:1"]["files"]}, + "root": "../../", + "language": "en", + "variants": ["notes/other.json"], + "metadata": { + "musicId": 7, + "difficulty": "expert", + "audio": False, + "audioFormat": None, + "quality": 1, + "flows": ["direct"], + "regions": ["one"], + "chart": {"title": "Song", "notes": 3}, + }, + } + return document_provider(siteproducts.ManifestStage(), source, spec) + + +def site_selection(): + return document_provider( + siteproducts.SelectionStage(), site_product(), {"path": "lives/7_expert.json", "reads": [], "audio": False} + ) + + +def site_asset(): + from tests.pipeline.test_siteassets import task_for + + stage = siteassets.AssetStage() + return provider( + stage, + lambda d: task_for( + d, + b'{"a":[1,2,3],"b":[1,2,3],"c":{"x":true,"label":"text"}}', + params={"encoding": "none", "split": 10, "omit": [["c", "label"]]}, + ), + ) + + +def site_index(): + stage = siteindex.IndexStage() + catalog = { + "format": siteindex.CATALOG, + "name": "example", + "language": "en", + "regions": [{"id": "one", "name": "One"}, {"id": "two", "name": "Two"}], + "topics": [ + { + "kind": "model", + "id": "sample", + "region": region, + "manifest": "products/model.json", + "entry": {"key": "Character/Model", "names": {"en": label}}, + } + for region, label in [("one", "Model"), ("two", "Other label")] + ], + } + return provider(stage, lambda d: task(stage, "example", [stored(d, contract.encode(catalog), "catalog")])) + + +class SyntheticReader(readsets.ReadSetStage): + def run(self, task, depot): + from unittest.mock import patch + + with patch.object(readsets, "node_identity", lambda _: "node:synthetic"): + return super().run(task, depot) + + +def site_readset(): + from tests.pipeline.test_readsets import SCRIPT + + stage = SyntheticReader() + + def fixture(mode): + def build(depot): + inputs = [ + stored(depot, b'{"type":"module"}', "player:package.json"), + stored(depot, SCRIPT.encode(), "player:scripts/read-set.mjs"), + stored(depot, b'{"files":["model:1#json","scene.json"]}', "data:lives/one.json"), + stored( + depot, + contract.encode( + readsets.request("lives/one.json", mode=mode, settings={"Alternate": True}, keys=True) + ), + "request", + ), + ] + return task(stage, "one", inputs, atoms={"player.node": "node:synthetic"}) + + return build + + return {"stage": stage, "fixtures": {mode: fixture(mode) for mode in ("plan", "full")}} diff --git a/tests/pipeline/test_goldens.py b/tests/pipeline/test_goldens.py index bc0dc7c..918abe1 100644 --- a/tests/pipeline/test_goldens.py +++ b/tests/pipeline/test_goldens.py @@ -38,10 +38,17 @@ def record(spec: str, depot: Depot) -> dict: return dict(doc, provider=spec) -@pytest.mark.parametrize("path", sorted(GOLDEN_DIR.glob("*.json")), ids=lambda p: p.stem) -def test_golden_record(path, tmp_path): +def record_provider(path): expected = contract.loads(path.read_bytes()) p = provided(expected["provider"]) + assert path.stem == expected["stage"] == p["stage"].name, f"{path.name}: stage identity differs" + assert expected["fixtures"] and p["fixtures"], f"{path.name}: fixtures must be nonempty" + return expected, p + + +@pytest.mark.parametrize("path", sorted(GOLDEN_DIR.glob("*.json")), ids=lambda p: p.stem) +def test_golden_record(path, tmp_path): + expected, p = record_provider(path) if os.environ.get("GOLDENS_UPDATE") == "1": path.write_bytes(contract.encode(record(expected["provider"], Depot(tmp_path / "depot")))) return @@ -137,4 +144,34 @@ def test_records_are_canonical_documents(tmp_path): def test_the_export_stages_have_golden_records(): - assert {"unity.export", "sprite.crop"} <= {p.stem for p in GOLDEN_DIR.glob("*.json")} + from nnnotes.cli.assets import STAGE_SOURCES, load_source + + required = { + stage.name + for source in STAGE_SOURCES + if source.output or source.name.startswith("site.") + for stage in load_source(source) + } + assert required <= {record_provider(path)[0]["stage"] for path in GOLDEN_DIR.glob("*.json")} + + +def test_a_record_cannot_cover_another_stages_filename(tmp_path): + path = tmp_path / "site.selection.json" + path.write_bytes((GOLDEN_DIR / "site.references.json").read_bytes()) + with pytest.raises(AssertionError, match="stage identity differs"): + record_provider(path) + + +@pytest.mark.parametrize("empty", ["record", "provider"]) +def test_record_and_provider_fixtures_must_be_nonempty(tmp_path, monkeypatch, empty): + p = toy_provider() + doc = record("tests.pipeline.test_goldens:toy_provider", Depot(tmp_path / "depot")) + if empty == "record": + doc["fixtures"] = {} + else: + p["fixtures"] = {} + monkeypatch.setattr("tests.pipeline.test_goldens.provided", lambda spec: p) + path = tmp_path / "toy.src.json" + path.write_bytes(contract.encode(doc)) + with pytest.raises(AssertionError, match="fixtures must be nonempty"): + record_provider(path) diff --git a/tests/pipeline/view_goldens.py b/tests/pipeline/view_goldens.py new file mode 100644 index 0000000..ce55dca --- /dev/null +++ b/tests/pipeline/view_goldens.py @@ -0,0 +1,205 @@ +"""Master-view golden inputs with localized rows and partial address coverage.""" + +from nnnotes.pipeline import contract, views, voices +from nnnotes.pipeline.stages import describe +from tests.pipeline import test_views, test_voices +from tests.pipeline.product_goldens import provider, stored, task + +ROWS = { + "MasterSupportCard": {"_assetID": 1}, + "MasterBandItem": {"_bandId": 1}, + "MasterStamp": {"_stampAsset": "Stamp/s1"}, + "MasterItem": {"_imagePath": "Item/one"}, + "MasterDegree": {"_imagePath": "Image/Degree/one"}, + "MasterBackground": {"_assetPath": "Image/Background/one", "_thumbnailAssetPath": "thumbnail/Background/one"}, + "MasterHomeSpot": { + "_thumbnailAssetPath": "Image/Spot/one", + "_backgroundAssetPath": "Spot/one/background", + "_situationAssetPath": "Spot/one/day", + }, + "MasterLiveMusic": {"_jacketAssetName": "one", "_titleTextID": 1}, + "MasterGacha": {"_bannerAssetName": "Gacha/Banner/one", "_logoAssetName": "Gacha/Logo/one"}, + "MasterHomeBanner": {"_imageAsset": "Home/Banner/one"}, + "MasterLoginBonus": { + "_sheetImageAsset": "LoginBonus/one/sheet", + "_backgroundImageAsset": "LoginBonus/one/background", + }, + "MasterStoryChapter": {"_banner": "one", "_image": "one", "_icon": "one"}, + "MasterStoryEpisode": {"_banner": "one", "_image": "one"}, + "MasterStoryFriendshipEpisode": {"_banner": "one"}, + "MasterCharacterFriendship": {"_storyBanner": "one"}, + "MasterEvent": { + "_imageAsset": "image", + "_logoAsset": "logo", + "_backgroundAsset": "background", + "_bannerAsset": "banner", + }, + "MasterShop": {"_thumbnailAsset": "one", "_dialogAsset": "one", "_tabAsset": "one"}, + "MasterMonthlyPass": {}, + "MasterSeasonPass": {"_bannerAsset": "one"}, + "MasterExchangeCategory": {"_bannerAsset": "one"}, + "MasterExchangeProduct": {"_thumbnailAsset": "one"}, + "MasterLoadingComics": {"_imageAsset": "one"}, + "MasterSkillIcon": {"_normalIconAssetName": "one"}, + "MasterArenaDailyReward": {"_resourceType": 1, "_resourceId": 1}, +} + + +def view_provider(name): + stage = views.ViewStage(name, views.load_rules()) + + def build(depot): + tables = {table: [] for table in stage.tables} + tables.update(test_views.CARD_TABLES) + tables.update( + { + table: [{"_id": 1, "_nameTextId": 1, "_nameTextID": 1, "_descriptionTextId": 1, **row}] + for table, row in ROWS.items() + } + ) + tables["MasterText"] = [*tables["MasterText"], test_views.text(1, "Example")] + entries = { + **test_views.card_entries(), + "Stamp/s1": [test_views.o(95, "Sprite", "s1")], + "Item/one": [test_views.o(96, "Sprite", "item")], + "SupportCard/1/snap_full": [test_views.o(97, "Sprite", "snap_full")], + } + recorder = views.Recorder(views.AddressIndex(entries)) + views.build(stage.rules, name, tables, recorder) + inputs = [ + stored(depot, contract.encode({"_allData": tables[table]}), "master:" + table) for table in stage.tables + ] + inputs += [ + stored(depot, contract.encode(recorder.subset()), "addresses"), + stored(depot, stage.rules_bytes, "rules"), + ] + return task(stage, "main", inputs) + + return provider(stage, build) + + +def voice_index(): + stage = voices.VoiceStage() + + def build(depot): + env = test_voices.planning(depot, depot.root / "fixtures") + described = describe(stage, "main", None, env) + inputs = [] + for inp in described.inputs: + data = depot.input_bytes(inp) + if inp.role.startswith("master:"): + data = contract.encode(contract.loads(data)) + inputs.append(stored(depot, data, inp.role)) + return contract.Task( + stage.name, stage.version, "main", described.params, described.atoms, tuple(inputs), described.context + ) + + return provider(stage, build) + + +def view_backgrounds(): + return view_provider("backgrounds") + + +def view_band_items(): + return view_provider("band_items") + + +def view_bands(): + return view_provider("bands") + + +def view_cards(): + return view_provider("cards") + + +def view_characters(): + return view_provider("characters") + + +def view_comics(): + return view_provider("comics") + + +def view_degrees(): + return view_provider("degrees") + + +def view_events(): + return view_provider("events") + + +def view_exchange_categories(): + return view_provider("exchange_categories") + + +def view_exchange_products(): + return view_provider("exchange_products") + + +def view_friendship_episodes(): + return view_provider("friendship_episodes") + + +def view_friendships(): + return view_provider("friendships") + + +def view_gacha(): + return view_provider("gacha") + + +def view_home_banners(): + return view_provider("home_banners") + + +def view_items(): + return view_provider("items") + + +def view_jackets(): + return view_provider("jackets") + + +def view_login_bonuses(): + return view_provider("login_bonuses") + + +def view_monthly_passes(): + return view_provider("monthly_passes") + + +def view_rewards(): + return view_provider("rewards") + + +def view_season_passes(): + return view_provider("season_passes") + + +def view_shops(): + return view_provider("shops") + + +def view_skill_icons(): + return view_provider("skill_icons") + + +def view_spots(): + return view_provider("spots") + + +def view_stamps(): + return view_provider("stamps") + + +def view_story_chapters(): + return view_provider("story_chapters") + + +def view_story_episodes(): + return view_provider("story_episodes") + + +def view_support_cards(): + return view_provider("support_cards") From 64f56ecbc745ffe2443b5564003fdccb2762eda2 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 19 Oct 2026 02:29:55 +0800 Subject: [PATCH 87/91] fix: include native PNG codecs in task identities --- CHANGELOG.md | 1 + CONTRIBUTING.md | 1 + docs/determinism.md | 4 +++ docs/stages.md | 4 +-- release-notes/v0.3.0.txt | 2 ++ src/nnnotes/core/versions.py | 31 ++++++++++++++++++ src/nnnotes/pipeline/stages.py | 6 +++- src/nnnotes/pipeline/textfonts.py | 3 +- src/nnnotes/unity/atoms/__init__.py | 8 +++-- tests/core/test_versions.py | 51 +++++++++++++++++++++++++++++ tests/golden/live.json | 3 +- tests/golden/live.ui.json | 11 ++++--- tests/golden/live2d.cubism.json | 3 +- tests/golden/live2d.model.json | 3 +- tests/golden/room.gltf.json | 5 +-- tests/golden/spot.files.json | 7 ++-- tests/golden/sprite.crop.json | 7 ++-- tests/golden/story.fonts.json | 11 ++++--- tests/golden/unity.export.json | 11 ++++--- tests/pipeline/product_goldens.py | 6 ++-- tests/pipeline/test_objexport.py | 2 +- 21 files changed, 145 insertions(+), 35 deletions(-) create mode 100644 src/nnnotes/core/versions.py create mode 100644 tests/core/test_versions.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4aeb4a6..4e7d00d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor v ### Features +- Keep PNG and font task results distinct across native compression library implementations. - Export episode documents, shader files and cue-sheet audio through depot tasks and retained file layouts. - Share cached results when adding remote storage, verify remote-only content, and retain recent results with their dependencies during collection. - Validate player package compatibility and publish chart metadata and requested language references consistently. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 18f4e7e..7bda8be 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,6 +34,7 @@ mkdocs build --strict # the documentation site `GOLDENS_UPDATE=1 pytest tests/pipeline/test_goldens.py`, using the library versions of `.github/goldens-constraints.txt` (`pip install -c .github/goldens-constraints.txt -e ".[test,fonts]"`). Node.js runs the synthetic read-set fixture. CI uses Node.js 24 and Python 3.13 for these records. + PNG-dependent records also use the native `Pillow.zlib` identity listed in their library versions. - **Command reference.** `docs/cli.md` is generated from the parser. After changing a command or an option, run `python -m nnnotes.cli.reference docs/cli.md`; a test fails while the file is out of date. diff --git a/docs/determinism.md b/docs/determinism.md index 1c71e76..1effd04 100644 --- a/docs/determinism.md +++ b/docs/determinism.md @@ -38,6 +38,10 @@ another installation computes other keys instead of mixing results. ## How it is tested +PNG and glyph task identities include `Pillow.zlib`, the version of Pillow's linked zlib codec and its zlib-ng +backend when present. This distinguishes native encoders even when the Pillow package version is the same. +Golden records that use those codecs also record this native identity with their library versions. + - **Golden records.** `tests/golden/.json` records, for each stage version, the libraries that shape its bytes and, for every synthetic fixture, the task key and the SHA-256 of the canonical result. A change of a stage's output without a new stage version fails the test. The CI job that checks the records installs the exact diff --git a/docs/stages.md b/docs/stages.md index 144db7e..591786c 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -240,7 +240,7 @@ the ids of the atoms a task uses, so upgrading a library or changing an atom re- |---|---|---|---| | `reader` | `nnnotes.unity.atoms.reader.open_bundle` (UnityPy) | `unitypy-/1` | one bundle loaded alone from its bytes: serialized files, objects, raw bytes, typetrees, container, externals, and the inputs of `texture.decode` and `mesh.arrays` (resource files are read from inside the bundle only) | | `texture.decode` | UnityPy `Texture2DConverter.parse_image_data` | `unitypy-+texture2ddecoder-+astc-encoder-py-+pillow-/1` | the first mip level, top row first, as UnityPy's `Texture2D.image` | -| `png.encode` | Pillow | `pillow-/1` | PNG bytes; level 6 (the default) gives Pillow's default bytes, the ones the other commands write | +| `png.encode` | Pillow and its native zlib codec | `pillow-+pillow.zlib-/1` | PNG bytes; level 6 (the default) gives Pillow's default bytes, the ones the other commands write | | `astc.container` | nnnotes | `nnnotes/1` | the standard `.astc` file (16-byte header, first mip level's blocks) | | `sprite.crop` | nnnotes (a port of UnityPy's `SpriteHelper`) | `pillow-+numpy-/1` | a sprite's image from its texture's image: texture rect, packing rotation / flip, alpha texture, tight mask or mesh drawing; optionally on the sprite's full rect | | `mesh.arrays` | nnnotes | `numpy-/1` | every vertex channel with its stored type and component count, primitives per submesh (base vertex applied), skin | @@ -272,7 +272,7 @@ The current implementations: |---|---| | `reader` | UnityPy | | `texture.decode` | UnityPy (`Texture2DConverter`) | -| `png.encode` | Pillow, zlib level `png.level` (6) | +| `png.encode` | Pillow, native zlib codec identity, zlib level `png.level` (6) | | `astc.container` | nnnotes | | `sprite.crop` | nnnotes | | `mesh.arrays` | nnnotes | diff --git a/release-notes/v0.3.0.txt b/release-notes/v0.3.0.txt index 955dfec..1219212 100644 --- a/release-notes/v0.3.0.txt +++ b/release-notes/v0.3.0.txt @@ -8,6 +8,8 @@ Live2D, room, spot, live, story and UI exports use shared object products. Named Standalone episode, shader and audio exports also reuse depot tasks. Command records retain accessed results and content. Collection protects recent results and their dependencies for a configurable grace period, and remote verification reads remote-only content without populating the local cache. +PNG and font task keys include the native compression library identity, so stored results are shared only between matching encoder implementations. + Configuration and command interface: - Set [paths] depot and, when needed, [depot] remote. - Use depot plan, depot run, depot verify and depot gc to manage stored work. diff --git a/src/nnnotes/core/versions.py b/src/nnnotes/core/versions.py new file mode 100644 index 0000000..386a43e --- /dev/null +++ b/src/nnnotes/core/versions.py @@ -0,0 +1,31 @@ +"""Native codec identities used by content-producing tasks.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from functools import cache + +PILLOW_ZLIB = "Pillow.zlib" +_PROBE = ( + "import json,sys; sys.path=json.loads(sys.argv[1]); from PIL import features; " + "print(json.dumps([features.version('zlib'), " + "features.version_feature('zlib_ng') if 'zlib_ng' in features.get_supported_features() else None]))" +) + + +@cache +def pillow_zlib_version() -> str: + """Pillow's zlib backend and version, probed without loading image codecs into the planning process.""" + result = subprocess.run( + [sys.executable, "-I", "-c", _PROBE, json.dumps([p for p in sys.path if isinstance(p, str)])], + capture_output=True, + text=True, + check=True, + timeout=15, + ) + zlib, zlib_ng = json.loads(result.stdout) + if not isinstance(zlib, str) or not zlib: + raise RuntimeError("Pillow has no zlib codec") + return zlib if zlib_ng is None else f"{zlib}-{zlib_ng}" diff --git a/src/nnnotes/pipeline/stages.py b/src/nnnotes/pipeline/stages.py index 1435c28..4025838 100644 --- a/src/nnnotes/pipeline/stages.py +++ b/src/nnnotes/pipeline/stages.py @@ -29,6 +29,7 @@ from dataclasses import dataclass, field from importlib import metadata +from ..core.versions import PILLOW_ZLIB, pillow_zlib_version from . import contract from .contract import Cost, IncompatibleTask, Input, Task @@ -271,9 +272,12 @@ def children_cpu() -> float | None: # ---------------------------------------------------------------- golden tests def library_versions(dists) -> dict[str, str]: - """{distribution: installed version} ("-" when not installed).""" + """Installed distribution versions and `Pillow.zlib`'s native codec identity ("-" for absent distributions).""" out = {} for d in sorted(dists): + if d == PILLOW_ZLIB: + out[d] = pillow_zlib_version() + continue try: out[d] = metadata.version(d) except metadata.PackageNotFoundError: diff --git a/src/nnnotes/pipeline/textfonts.py b/src/nnnotes/pipeline/textfonts.py index 42cca80..910ce3b 100644 --- a/src/nnnotes/pipeline/textfonts.py +++ b/src/nnnotes/pipeline/textfonts.py @@ -3,6 +3,7 @@ from __future__ import annotations from ..content.prefabs import script_class +from ..core.versions import PILLOW_ZLIB from ..source.apkset import MANIFEST_IN_APK, manifest_version_name from ..unity import fontsettings from ..unity.ids import parse_object_id @@ -40,7 +41,7 @@ def font_settings(task, objects, lang): def font_atoms() -> dict: from ..unity.atoms import impl_id - return {"text.glyphs": impl_id(("fonttools", "freetype-py", "scipy", "numpy", "Pillow"), 1)} + return {"text.glyphs": impl_id(("fonttools", "freetype-py", "scipy", "numpy", "Pillow", PILLOW_ZLIB), 1)} class FontInputsStage(ProductStage): diff --git a/src/nnnotes/unity/atoms/__init__.py b/src/nnnotes/unity/atoms/__init__.py index 2786e13..5f205f0 100644 --- a/src/nnnotes/unity/atoms/__init__.py +++ b/src/nnnotes/unity/atoms/__init__.py @@ -32,6 +32,8 @@ from functools import cache from importlib import metadata +from ...core.versions import PILLOW_ZLIB, pillow_zlib_version + __all__ = ["ATOMS", "Impl", "Unsupported", "atom_id", "ids", "impl_id", "resolve"] @@ -54,6 +56,8 @@ class Impl: @cache def _version(dist: str) -> str: + if dist == PILLOW_ZLIB: + return pillow_zlib_version() try: return metadata.version(dist) except metadata.PackageNotFoundError: @@ -61,7 +65,7 @@ def _version(dist: str) -> str: def impl_id(libraries: tuple[str, ...], revision: int) -> str: - """`lib-version+lib-version/revision` for the installed versions of `libraries` (distribution names), or + """`lib-version+lib-version/revision` for distribution names and the native `Pillow.zlib` codec, or `nnnotes/revision` for an implementation that depends on no library's behaviour.""" if not libraries: return f"nnnotes/{revision}" @@ -72,7 +76,7 @@ def impl_id(libraries: tuple[str, ...], revision: int) -> str: _REFERENCE: dict[str, tuple[str, str, str, tuple[str, ...], int]] = { "reader": ("reader", "open_bundle", "cost", ("UnityPy",), 1), "texture.decode": ("texture", "decode", "cost", ("UnityPy", "texture2ddecoder", "astc-encoder-py", "Pillow"), 1), - "png.encode": ("png", "encode", "cost", ("Pillow",), 1), + "png.encode": ("png", "encode", "cost", ("Pillow", PILLOW_ZLIB), 1), "astc.container": ("astc", "container", "cost", (), 1), "sprite.crop": ("sprite", "crop", "cost", ("Pillow", "numpy"), 1), "mesh.arrays": ("mesh", "arrays", "cost", ("numpy",), 1), diff --git a/tests/core/test_versions.py b/tests/core/test_versions.py new file mode 100644 index 0000000..22309ff --- /dev/null +++ b/tests/core/test_versions.py @@ -0,0 +1,51 @@ +"""Native PNG codec identity participates in image-producing task keys.""" + +import subprocess +import sys + +from PIL import features + +from nnnotes.core.versions import PILLOW_ZLIB, pillow_zlib_version +from nnnotes.pipeline import contract, textfonts +from nnnotes.pipeline.stages import library_versions +from nnnotes.unity import atoms + + +def test_pillow_codec_identity_names_its_backend(): + expected = features.version("zlib") + if "zlib_ng" in features.get_supported_features(): + expected += "-" + features.version_feature("zlib_ng") + assert pillow_zlib_version() == expected + assert library_versions([PILLOW_ZLIB]) == {PILLOW_ZLIB: expected} + + +def test_png_and_font_task_keys_change_with_native_codec(monkeypatch): + version = atoms._version + monkeypatch.setattr(atoms, "_reference", atoms._reference.__wrapped__) + + def keys(native): + monkeypatch.setattr(atoms, "_version", lambda name: native if name == PILLOW_ZLIB else version(name)) + png = atoms.ids("png.encode") + glyphs = textfonts.font_atoms() + return ( + contract.Task("unity.export", 1, "texture", atoms=png).key, + contract.Task("story.fonts", 1, "1/en", atoms=glyphs).key, + contract.Task("live.ui", 1, "7/expert/en", atoms=glyphs).key, + ) + + assert all(a != b for a, b in zip(keys("1.2.3"), keys("1.2.3.zlib-ng-2.0"), strict=True)) + + +def test_codec_probe_uses_the_running_process_import_paths(tmp_path): + package = tmp_path / "PIL" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "features.py").write_text( + "def version(name): return '7.8.9'\ndef get_supported_features(): return []\n", encoding="utf-8" + ) + code = ( + "import sys; sys.path.insert(0, sys.argv[1]); from PIL import features; " + "from nnnotes.core.versions import pillow_zlib_version; " + "assert features.version('zlib') == pillow_zlib_version() == '7.8.9'" + ) + subprocess.run([sys.executable, "-c", code, str(tmp_path)], check=True, capture_output=True, text=True) diff --git a/tests/golden/live.json b/tests/golden/live.json index 3751631..0d11522 100644 --- a/tests/golden/live.json +++ b/tests/golden/live.json @@ -11,7 +11,8 @@ } }, "libraries": { - "Pillow": "12.3.0" + "Pillow": "12.3.0", + "Pillow.zlib": "1.3-2.3.3" }, "provider": "tests.pipeline.product_goldens:live_product", "stage": "live", diff --git a/tests/golden/live.ui.json b/tests/golden/live.ui.json index 566f634..1e28032 100644 --- a/tests/golden/live.ui.json +++ b/tests/golden/live.ui.json @@ -1,19 +1,20 @@ { "atoms": { - "text.glyphs": "fonttools-4.67.0+freetype-py-2.5.1+scipy-1.18.1+numpy-2.4.6+pillow-12.3.0/1" + "text.glyphs": "fonttools-4.67.0+freetype-py-2.5.1+scipy-1.18.1+numpy-2.4.6+pillow-12.3.0+pillow.zlib-1.3-2.3.3/1" }, "fixtures": { "game": { - "key": "b55d0f5e1dbfa2c760981e5d6e204887ea3406a9ba4eb9dda4935275af0ee65c", - "result": "2512e62b2bf9740848816b3790f9e812728e3482818f819a12252e1b33720215" + "key": "1dbcb102341e2b99aabea4aaefb412c6848d5ee900140b554f9b985736b333c8", + "result": "2beeb0e31090a3ef2f8b0ec588e1c89ca5c34900b96f7f5a3d1c2ce08a8b2f17" }, "open": { - "key": "b3d25ad7514c04176c3d1b2a824c5cab3459f685491174423cbdd3f05b6eb164", - "result": "14f534ebc34a5c866a1c086660df6c6cacce9a5c42207a5e1cf49d9d7c928475" + "key": "a4d8f57af62d310e4684703c77f6b86f4e47f44b0251c449d32c344bc1441bc1", + "result": "4c9db2ca61fa1699f85a9af25db23c5deb4b726a08cb2d30bc1e7880973e10f3" } }, "libraries": { "Pillow": "12.3.0", + "Pillow.zlib": "1.3-2.3.3", "fonttools": "4.67.0", "freetype-py": "2.5.1", "numpy": "2.4.6", diff --git a/tests/golden/live2d.cubism.json b/tests/golden/live2d.cubism.json index 0a814ce..9fe299f 100644 --- a/tests/golden/live2d.cubism.json +++ b/tests/golden/live2d.cubism.json @@ -7,7 +7,8 @@ } }, "libraries": { - "Pillow": "12.3.0" + "Pillow": "12.3.0", + "Pillow.zlib": "1.3-2.3.3" }, "provider": "tests.pipeline.product_goldens:cubism", "stage": "live2d.cubism", diff --git a/tests/golden/live2d.model.json b/tests/golden/live2d.model.json index 89c6af5..4e26bcf 100644 --- a/tests/golden/live2d.model.json +++ b/tests/golden/live2d.model.json @@ -7,7 +7,8 @@ } }, "libraries": { - "Pillow": "12.3.0" + "Pillow": "12.3.0", + "Pillow.zlib": "1.3-2.3.3" }, "provider": "tests.pipeline.product_goldens:model", "stage": "live2d.model", diff --git a/tests/golden/room.gltf.json b/tests/golden/room.gltf.json index 5a4905c..29de53a 100644 --- a/tests/golden/room.gltf.json +++ b/tests/golden/room.gltf.json @@ -2,12 +2,13 @@ "atoms": {}, "fixtures": { "shared-objects": { - "key": "e2cbd5b59e83e9644e3ffffb8878c5f3c20822b258c6fd756b70d71837b2ff2e", - "result": "6e3940b26686ce64451591d838fa246066304494410cff1a0a62bd8c8240de88" + "key": "1caf7baaba836050777f813e668c41bc7b575953e843353c13f7788d4bdf41d4", + "result": "511a28aee143417d1cbe9627c0a635d9e9f6b089569cd94d513e5bf1661c4279" } }, "libraries": { "Pillow": "12.3.0", + "Pillow.zlib": "1.3-2.3.3", "numpy": "2.4.6" }, "provider": "tests.pipeline.product_goldens:room", diff --git a/tests/golden/spot.files.json b/tests/golden/spot.files.json index 40931ba..1f60418 100644 --- a/tests/golden/spot.files.json +++ b/tests/golden/spot.files.json @@ -2,12 +2,13 @@ "atoms": {}, "fixtures": { "shared-objects": { - "key": "466f3cc629f8a9e32aa61f6f46bc28bed0620ffffdd3388741ea5fb313cc50a0", - "result": "5f54154505303501ee4f500b350ca74aea9180922fef17bbdfdcd4c739da011c" + "key": "61b0ed7b1a399e2adf68fbe7ad3e6207f0f7e7b16d4b2aea015e8c078153fc47", + "result": "fa43a00881448e52b0dc7c54b268394e6e5059176052505be9bc515fab4e0e83" } }, "libraries": { - "Pillow": "12.3.0" + "Pillow": "12.3.0", + "Pillow.zlib": "1.3-2.3.3" }, "provider": "tests.pipeline.product_goldens:spot_product", "stage": "spot.files", diff --git a/tests/golden/sprite.crop.json b/tests/golden/sprite.crop.json index d15f112..a91f051 100644 --- a/tests/golden/sprite.crop.json +++ b/tests/golden/sprite.crop.json @@ -1,16 +1,17 @@ { "atoms": { - "png.encode": "pillow-12.3.0/1", + "png.encode": "pillow-12.3.0+pillow.zlib-1.3-2.3.3/1", "sprite.crop": "pillow-12.3.0+numpy-2.4.6/1" }, "fixtures": { "atlas": { - "key": "8bff1abc736f8e76c8ff92e0f87f28317e25bcd2a59dfbe4b8753144b862566b", - "result": "746170244ebd43a73004fc8c369d817d50216f51062ab73ebb9b57fbb7f6ad2f" + "key": "52636fa1d5806134ddbadff9c8dc4cc6fdf8e3f7710779d18a05b076e6cfd452", + "result": "82717f1b80bbc21be6436bdc9c44abed91165cef249a2da5ea1f79821939b6fc" } }, "libraries": { "Pillow": "12.3.0", + "Pillow.zlib": "1.3-2.3.3", "UnityPy": "1.25.3", "astc-encoder-py": "0.1.12", "numpy": "2.4.6", diff --git a/tests/golden/story.fonts.json b/tests/golden/story.fonts.json index eb2d5ba..62d90e2 100644 --- a/tests/golden/story.fonts.json +++ b/tests/golden/story.fonts.json @@ -1,19 +1,20 @@ { "atoms": { - "text.glyphs": "fonttools-4.67.0+freetype-py-2.5.1+scipy-1.18.1+numpy-2.4.6+pillow-12.3.0/1" + "text.glyphs": "fonttools-4.67.0+freetype-py-2.5.1+scipy-1.18.1+numpy-2.4.6+pillow-12.3.0+pillow.zlib-1.3-2.3.3/1" }, "fixtures": { "game": { - "key": "453b091bd80d784ce02d9c2d2ce7e03b0ca887a51c3b573da9aa7ba2aec8764b", - "result": "3314357a566dd4e9b103a960b3908ed6af9635571dc8b37da76535b1a6ed0e77" + "key": "1d26c9ba7fffd1f08479db839c852e9f1a555d823b7e367885d4a6c04fdf0bb6", + "result": "1229ae7f018f34522bdde026ffaf519b256527612f6d3b94a6afb1203d677342" }, "open": { - "key": "7c0c1007f05e47d204bb0d573770ff923dafbd8421fcb843d8bc008745aef821", - "result": "849badd509b7529a2f184f95cb4ea93e8afc6768f0602d1a27a83dcc0b4c32a5" + "key": "a6b02cc1006788a375a8666f83f6f35d3f3c3209ebd338c69429ffbcfe41e31b", + "result": "f628120a030fee7142a1ece12a3ba37f2b1066a23cae7fbc3ca4c04cbc4e2f09" } }, "libraries": { "Pillow": "12.3.0", + "Pillow.zlib": "1.3-2.3.3", "fonttools": "4.67.0", "freetype-py": "2.5.1", "numpy": "2.4.6", diff --git a/tests/golden/unity.export.json b/tests/golden/unity.export.json index 9df4cc3..969a479 100644 --- a/tests/golden/unity.export.json +++ b/tests/golden/unity.export.json @@ -12,7 +12,7 @@ "mesh.arrays": "numpy-2.4.6/1", "mesh.glb": "mesh.glb/3", "mono.json": "mono.json/3", - "png.encode": "pillow-12.3.0/1", + "png.encode": "pillow-12.3.0+pillow.zlib-1.3-2.3.3/1", "prefab.json": "prefab.json/3", "reader": "fakeunity/1", "shader.json": "shader.json/2", @@ -68,16 +68,17 @@ "result": "aab00b71441e5978a98431d780fa1c4facb394389ca66901414541a238583197" }, "sprite.png": { - "key": "f8fcd13c515ca0d0acd08cd983b23e917fbec42e7de0c351980b69cab4f4af09", - "result": "d053cdaf856f61c785987e324a847e04c4cdc0e9556059eba66dc737f7809531" + "key": "a46e8cf4adb1416db2e51d9ea1135cf5f530d24e6d212a585819879724369a4c", + "result": "c486b3e2274ad24933af5390ed83cc1a9988d0132d992b87ff750137d606b0dd" }, "tex.png": { - "key": "5659b2e28ffb247c956a6273a41b1f6c0196e64737060a7228fde0ff1d946c3e", - "result": "84ac758d275634016b105b57698e5cac22ac57379ea8783486b833f1650c27f5" + "key": "4358a8342567c9f5a59c835d3000573e5ecd7b113052eff86d7e734075303a66", + "result": "851cfd63b09f3f51c1a368e232b49a6980ba1b0db09b8a578c3d362711da9901" } }, "libraries": { "Pillow": "12.3.0", + "Pillow.zlib": "1.3-2.3.3", "UnityPy": "1.25.3", "astc-encoder-py": "0.1.12", "numpy": "2.4.6", diff --git a/tests/pipeline/product_goldens.py b/tests/pipeline/product_goldens.py index 8b7a35d..77082e4 100644 --- a/tests/pipeline/product_goldens.py +++ b/tests/pipeline/product_goldens.py @@ -26,6 +26,8 @@ def environment(depot, name, builder): def provider(stage, build, libraries=()): + if "Pillow" in libraries: + libraries = (*libraries, "Pillow.zlib") return {"stage": stage, "fixtures": {"shared-objects": build}, "libraries": libraries} @@ -137,7 +139,7 @@ def build(depot): return { "stage": stage, "fixtures": {"plain": fixture("plain", []), "localized": fixture("localized", ["ja", "en"])}, - "libraries": ["Pillow"], + "libraries": ["Pillow", "Pillow.zlib"], } @@ -305,7 +307,7 @@ def build(depot): return provider(stage, build) -FONT_LIBRARIES = ("fonttools", "freetype-py", "scipy", "numpy", "Pillow") +FONT_LIBRARIES = ("fonttools", "freetype-py", "scipy", "numpy", "Pillow", "Pillow.zlib") def font_provider(stage, fixture, subject, *, extra=None): diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index 633f302..30ba6d0 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -1644,7 +1644,7 @@ def test_every_reason_code_and_converter_is_documented(): "shader.json": "all:51", "generic.json": "all:95", } -LIBRARIES = ["Pillow", "UnityPy", "astc-encoder-py", "numpy", "texture2ddecoder"] +LIBRARIES = ["Pillow", "Pillow.zlib", "UnityPy", "astc-encoder-py", "numpy", "texture2ddecoder"] def export_golden(): From f9f977eff42a73a0acc3c27d5f771934fad7f50c Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 19 Oct 2026 02:32:19 +0800 Subject: [PATCH 88/91] fix: annotate codec probe search paths --- src/nnnotes/core/versions.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/nnnotes/core/versions.py b/src/nnnotes/core/versions.py index 386a43e..26eeb86 100644 --- a/src/nnnotes/core/versions.py +++ b/src/nnnotes/core/versions.py @@ -6,6 +6,7 @@ import subprocess import sys from functools import cache +from typing import cast PILLOW_ZLIB = "Pillow.zlib" _PROBE = ( @@ -19,7 +20,13 @@ def pillow_zlib_version() -> str: """Pillow's zlib backend and version, probed without loading image codecs into the planning process.""" result = subprocess.run( - [sys.executable, "-I", "-c", _PROBE, json.dumps([p for p in sys.path if isinstance(p, str)])], + [ + sys.executable, + "-I", + "-c", + _PROBE, + json.dumps([p for p in cast(list[object], sys.path) if isinstance(p, str)]), + ], capture_output=True, text=True, check=True, From d904d8e395beca6e566812a8dc01dfe525c857df Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 19 Oct 2026 12:55:31 +0800 Subject: [PATCH 89/91] fix: report S3 object deletion failures --- CHANGELOG.md | 3 +- release-notes/v0.3.0.txt | 2 + src/nnnotes/pipeline/remote.py | 6 ++- tests/pipeline/test_depot_remote.py | 71 +++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e7d00d..43b5f68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project are listed here. Versions follow [Semantic V over the command line (commands, options, exit statuses and `--json` reports), the configuration keys, the JSON schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor version can change any of them. -## [0.3.0](https://github.com/MetaSekaiLab/nnnotes/compare/v0.2.5...v0.3.0) - 2026-10-18 +## [0.3.0](https://github.com/MetaSekaiLab/nnnotes/compare/v0.2.5...v0.3.0) - 2026-10-19 ### Compatibility @@ -15,6 +15,7 @@ schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor v ### Features +- Report per-object S3 deletion failures during depot collection. - Keep PNG and font task results distinct across native compression library implementations. - Export episode documents, shader files and cue-sheet audio through depot tasks and retained file layouts. - Share cached results when adding remote storage, verify remote-only content, and retain recent results with their dependencies during collection. diff --git a/release-notes/v0.3.0.txt b/release-notes/v0.3.0.txt index 1219212..e4f52c6 100644 --- a/release-notes/v0.3.0.txt +++ b/release-notes/v0.3.0.txt @@ -10,6 +10,8 @@ Standalone episode, shader and audio exports also reuse depot tasks. Command rec PNG and font task keys include the native compression library identity, so stored results are shared only between matching encoder implementations. +S3 collection follows paginated listings, batches deletion requests and reports per-object deletion failures. + Configuration and command interface: - Set [paths] depot and, when needed, [depot] remote. - Use depot plan, depot run, depot verify and depot gc to manage stored work. diff --git a/src/nnnotes/pipeline/remote.py b/src/nnnotes/pipeline/remote.py index 9e89c9a..503667f 100644 --- a/src/nnnotes/pipeline/remote.py +++ b/src/nnnotes/pipeline/remote.py @@ -203,7 +203,11 @@ def delete(self, keys: list[str]) -> None: for i in range(0, len(keys), 1000): batch = [{"Key": self.prefix + k} for k in keys[i : i + 1000]] if batch: - self._s3.delete_objects(Bucket=self.bucket, Delete={"Objects": batch, "Quiet": True}) + response = self._s3.delete_objects(Bucket=self.bucket, Delete={"Objects": batch, "Quiet": True}) + errors = response.get("Errors", []) + if errors: + codes = ", ".join(sorted({error.get("Code", "Unknown") for error in errors})) + raise RuntimeError(f"S3 deletion failed for {len(errors)} object(s): {codes}") def list(self, prefix: str) -> Iterator[tuple[str, int, float]]: pages = self._s3.get_paginator("list_objects_v2").paginate(Bucket=self.bucket, Prefix=self.prefix + prefix) diff --git a/tests/pipeline/test_depot_remote.py b/tests/pipeline/test_depot_remote.py index e4e5cd0..7c82421 100644 --- a/tests/pipeline/test_depot_remote.py +++ b/tests/pipeline/test_depot_remote.py @@ -207,6 +207,77 @@ def test_a_result_differing_from_the_remote_one_is_a_conflict(tmp_path): commit(Depot(tmp_path / "b", remote=remote), task, b"two") +def test_s3_listing_follows_continuation_tokens(): + pytest.importorskip("boto3") + from botocore.stub import Stubber + + remote = open_remote("s3://bucket/depot", key_id="k", secret="s") + written = datetime(2026, 1, 2, tzinfo=UTC) + with Stubber(remote._s3) as stub: + stub.add_response( + "list_objects_v2", + { + "Contents": [{"Key": "depot/cas/a", "Size": 3, "LastModified": written}], + "IsTruncated": True, + "NextContinuationToken": "next", + }, + {"Bucket": "bucket", "Prefix": "depot/cas/"}, + ) + stub.add_response( + "list_objects_v2", + {"Contents": [{"Key": "depot/cas/b", "Size": 5, "LastModified": written}], "IsTruncated": False}, + {"Bucket": "bucket", "Prefix": "depot/cas/", "ContinuationToken": "next"}, + ) + assert list(remote.list("cas/")) == [("cas/a", 3, written.timestamp()), ("cas/b", 5, written.timestamp())] + stub.assert_no_pending_responses() + + +def test_s3_deletion_batches_at_the_service_limit(): + pytest.importorskip("boto3") + from botocore.stub import Stubber + + remote = open_remote("s3://bucket/depot", key_id="k", secret="s") + keys = [f"cas/{i}" for i in range(1001)] + with Stubber(remote._s3) as stub: + for batch in (keys[:1000], keys[1000:]): + stub.add_response( + "delete_objects", + {}, + { + "Bucket": "bucket", + "Delete": { + "Objects": [{"Key": f"depot/{key}"} for key in batch], + "Quiet": True, + }, + }, + ) + remote.delete(keys) + stub.assert_no_pending_responses() + + +def test_s3_deletion_reports_per_object_errors_and_stops(): + pytest.importorskip("boto3") + from botocore.stub import Stubber + + remote = open_remote("s3://bucket/depot", key_id="k", secret="s") + keys = [f"cas/{i}" for i in range(1001)] + with Stubber(remote._s3) as stub: + stub.add_response( + "delete_objects", + {"Errors": [{"Key": "depot/cas/0", "Code": "AccessDenied", "Message": "Denied"}]}, + { + "Bucket": "bucket", + "Delete": { + "Objects": [{"Key": f"depot/{key}"} for key in keys[:1000]], + "Quiet": True, + }, + }, + ) + with pytest.raises(RuntimeError, match="S3 deletion failed for 1 object\\(s\\): AccessDenied"): + remote.delete(keys) + stub.assert_no_pending_responses() + + def test_gc_of_the_remote_keeps_what_its_kept_runs_reach(tmp_path): remote = open_remote((tmp_path / "remote").as_uri()) d = Depot(tmp_path / "a", remote=remote) From 4d1b1d57d6039ea449335d2d2eb62d03b63f4b62 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 19 Oct 2026 12:55:32 +0800 Subject: [PATCH 90/91] docs: explain depot exports and player site setup --- README.md | 163 ++++++++++++++++++++++----------------------- README.zh-CN.md | 155 +++++++++++++++++++++--------------------- docs/depot.md | 2 + docs/index.md | 74 ++++++++------------ docs/quickstart.md | 104 ++++++++++++++++------------- docs/site.md | 19 +++++- docs/validation.md | 95 +++++++++++++++----------- 7 files changed, 319 insertions(+), 293 deletions(-) diff --git a/README.md b/README.md index 15c1a27..990186d 100644 --- a/README.md +++ b/README.md @@ -1,100 +1,97 @@ -# nnnotes？！ +# nnnotes -[English](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.md) | [简体中文](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.zh-CN.md) +[English](README.md) | [简体中文](README.zh-CN.md) -nnnotes is an offline data toolkit for the game files of BanG Dream! Our Notes. It reads Addressables catalogs, -downloads and decrypts asset bundles, downloads and decodes master data, and exports stories, Live2D models, spots, -shaders, CRI audio and live charts as structured JSON and common file formats. `nnnotes site build` writes the static -site that [ournotes-player](https://github.com/empty-sekai/ournotes-player) plays, and `nnnotes music-data` one JSON -file with every song and chart. The name was inspired by sssekai. +nnnotes turns your BanG Dream! Our Notes game files into structured data, media and static player sites. +It reads Addressables catalogs, bundles, master data and APK resources; exports stories, Live2D models, live +charts, rooms, UI, shaders and audio; and assembles sites for +[ournotes-player](https://github.com/empty-sekai/ournotes-player). -This is an unofficial fan project, not affiliated with the game's developer or operator. The repository contains no -game assets, keys or server addresses: you supply the game files, the keys and the server addresses in your own -configuration, and the exports stay in the local directories you choose. +A persistent **depot** stores converted objects, task results and catalog versions. Exports share these results, +and named sites retain their selected products across builds. A depot can also share work through a directory +or S3-compatible object storage. -**Documentation: ** +**[Documentation](https://metasekailab.github.io/nnnotes/)** · +[Quick start](https://metasekailab.github.io/nnnotes/quickstart/) · +[Commands](https://metasekailab.github.io/nnnotes/cli/) ## Install +Python 3.11 or newer: + ```sh -pip install nnnotes # Python 3.11+ -pip install 'nnnotes[fonts]' # the game's fonts (--fonts game) and the story text of site build -pip install 'nnnotes[deck]' # the deck model's chart statistics of music-data +pip install nnnotes==0.3.0 +pip install 'nnnotes[fonts]==0.3.0' # story and UI fonts +pip install 'nnnotes[remote]==0.3.0' # S3-compatible depot storage +pip install 'nnnotes[deck]==0.3.0' # music-data chart statistics ``` -nnnotes also needs the game's `base.apk`, decoded master data for the commands that read it, and the external tools -[vgmstream](https://vgmstream.org/) and [FFmpeg](https://ffmpeg.org/) for audio and video (Node.js 22+ and a built -ournotes-player for `site build`). +Choose the extras your commands need; they can be combined as `nnnotes[fonts,remote,deck]==0.3.0`. +Audio/video conversion uses [vgmstream](https://vgmstream.org/) and [FFmpeg](https://ffmpeg.org/). +Site builds additionally require Node.js 22+ and a built **ournotes-player 0.2.x** package. +The deck extra installs `ournotes-sim==0.0.12`. + +## First export -## Quick start +Supply your own client files, keys and server addresses through the configuration wizard: ```sh -nnnotes config init --user # asks for each setting (keys without echo) and writes the per-user config file -nnnotes config check # where each setting comes from and whether it is valid, never its value +nnnotes config init --user +nnnotes config check nnnotes master download --latest -o work/master-bin nnnotes master decode work/master-bin -o work/master -nnnotes --master work/master export story 10462 -o out/story_10462 -nnnotes site build -o out/site --all --player path/to/ournotes-player +nnnotes config set paths.master work/master +nnnotes export episode 10462 -o out/episode.json +nnnotes export story 10462 --fonts game -o out/story +``` + +Set `paths.depot` to a persistent directory and `paths.apk` to your game APK. The wizard reads keys without echo; +`config check` reports setting names, origins and validity. Story export requires the `fonts` extra and audio +tools. See the [quick start](https://metasekailab.github.io/nnnotes/quickstart/) for prerequisites and output layout. + +## Build a player site + +```sh +npm install ournotes-player@0.2.0 +nnnotes site build --name stories -o out/site --story 10462 --fonts game --player node_modules/ournotes-player +node node_modules/ournotes-player/scripts/validate-data.mjs out/site +python -m http.server 8000 --directory out/site ``` -Every command prints a short summary, or with `--json` its report as JSON. Scripts set the config file without -questions: `nnnotes config init --user --no-input --set SECTION.KEY=VALUE ...`. - -## Commands - -| Command | What it does | -| --- | --- | -| `config init\|set\|unset\|check\|path` | write, edit and check the config file | -| `servers` | the game API's server list: regions with their CDN and API roots | -| `catalog keys\|browse` | the addressable keys of a region's catalog; a local web page to browse catalogs and bundles | -| `catalog versions\|import\|fetch\|diff` | catalog versions kept in the depot, and what changed between two | -| `bundle pull` | fetch the bundle closure of keys and copy the decrypted bundles to a directory | -| `master version\|download\|decode` | the master data version a region serves, its download and its decoding to JSON | -| `export assets` | every object of the selected bundles in common formats, through the content-addressed depot | -| `export episode\|story\|live\|live2d\|spot\|room\|shader\|audio\|ui` | one subject: an ADV episode, a story, a live, a Live2D model, a spot, a room, shaders, a cue sheet, the UI library | -| `apk settings\|hca-key` | the player's graphics settings and the CRI HCA keycode of `base.apk` | -| `depot plan\|run\|verify\|gc` | what `export assets` would run and why, single tasks, the depot's check and collection | -| `voices list\|search\|summary\|get` | the character and story voices of the master data | -| `site build` | the static site of ournotes-player: charts, Live2D models and stories, in several regions and languages | -| `music-data` | every song and chart in one JSON file, with the deck model's chart statistics | - -The [command reference](https://metasekailab.github.io/nnnotes/cli/) lists every option. - -## Documentation - -- [Quick start](https://metasekailab.github.io/nnnotes/quickstart/): from an empty config to an exported story. -- How-to guides: [configure nnnotes](https://metasekailab.github.io/nnnotes/configuration/), - [export stories and lives](https://metasekailab.github.io/nnnotes/export/), - [build an ournotes-player site](https://metasekailab.github.io/nnnotes/site/), - [export every asset](https://metasekailab.github.io/nnnotes/assets/), - [the Japanese release](https://metasekailab.github.io/nnnotes/jp/), - [music data](https://metasekailab.github.io/nnnotes/music-data/). -- Reference: [commands](https://metasekailab.github.io/nnnotes/cli/), - [command outputs](https://metasekailab.github.io/nnnotes/commands/), - [JSON schemas](https://metasekailab.github.io/nnnotes/schemas/), - [Python API](https://metasekailab.github.io/nnnotes/api/), - [contracts](https://metasekailab.github.io/nnnotes/contracts/), [stages](https://metasekailab.github.io/nnnotes/stages/), - [views](https://metasekailab.github.io/nnnotes/views/). -- Explanation: [architecture](https://metasekailab.github.io/nnnotes/architecture/), - [determinism](https://metasekailab.github.io/nnnotes/determinism/), - [validation](https://metasekailab.github.io/nnnotes/validation/). - -Exports are deterministic for a given installation: the same input with the same versions of nnnotes, its Python -dependencies (UnityPy, Pillow, numpy, ...) and the external tools gives byte-identical files; JSON is UTF-8 with LF -line endings and non-finite numbers written as `1e999`. Other versions can encode the same content into other bytes -(PNG files written by another Pillow version can differ in their bytes while their pixels are equal). - -## Changelog and versions - -[CHANGELOG.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/CHANGELOG.md) lists the changes of every version. -Versions follow [Semantic Versioning](https://semver.org/) over the command line, the configuration keys, the JSON -schemas and the documented Python API. - -## Contributing - -Setup, checks and commit conventions are in -[CONTRIBUTING.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/CONTRIBUTING.md). - -## License - -MIT, see [LICENSE](https://github.com/MetaSekaiLab/nnnotes/blob/main/LICENSE). +Open `http://localhost:8000/story/`. Live2D, MotionSync and Spine features require the corresponding browser +runtimes supplied by the embedding page. See the [site guide](https://metasekailab.github.io/nnnotes/site/). +Use `--pair MUSIC_ID:DIFFICULTY` for a chart, `--live2d KEY` for a model, and `--story-languages ja,en` for selected +story languages. Reusing a site name updates selected topics and retains the others. + +## Choose a workflow + +| Task | Commands | Guide | +|---|---|---| +| Configure inputs | `config init`, `config check`, `servers` | [Configuration](https://metasekailab.github.io/nnnotes/configuration/) | +| Inspect or retain catalogs | `catalog keys`, `browse`, `versions`, `import`, `fetch`, `diff` | [Commands](https://metasekailab.github.io/nnnotes/cli/) | +| Copy source bundles | `bundle pull` | [Asset export](https://metasekailab.github.io/nnnotes/assets/) | +| Export a subject | `export episode`, `story`, `live`, `live2d`, `spot`, `room`, `shader`, `audio`, `ui` | [Exports](https://metasekailab.github.io/nnnotes/export/) | +| Convert selected assets | `export assets`, `depot plan`, `depot run` | [Asset export](https://metasekailab.github.io/nnnotes/assets/) | +| Check and retain stored work | `depot verify`, `depot gc` | [Depot](https://metasekailab.github.io/nnnotes/depot/) | +| Export song and chart statistics | `music-data` | [Music data](https://metasekailab.github.io/nnnotes/music-data/) | +| Find character voices | `voices list`, `search`, `summary`, `get` | [Commands](https://metasekailab.github.io/nnnotes/cli/) | + +Commands print summaries; `--json` selects a structured report. The reference describes exit statuses and options. + +## Data and compatibility + +Product exports use versioned manifests and shared object records. Read the +[format reference](https://metasekailab.github.io/nnnotes/formats/), +[product reference](https://metasekailab.github.io/nnnotes/products/) and +[schemas](https://metasekailab.github.io/nnnotes/schemas/) to consume them. +Task identities include input contents, options and conversion implementations, including native encoders. +[Determinism](https://metasekailab.github.io/nnnotes/determinism/) explains reproducibility across installations. + +Versions cover the CLI, configuration, schemas and documented Python API. Before 1.0, a minor version can change +these interfaces. See [CHANGELOG.md](CHANGELOG.md) and the [architecture](https://metasekailab.github.io/nnnotes/architecture/). + +## Contributing and license + +[CONTRIBUTING.md](CONTRIBUTING.md) describes development and synthetic tests. nnnotes is licensed under +[MIT](LICENSE). This is an unofficial fan project, unaffiliated with the game's developer or operator. +Game resources, keys and server configuration are supplied by each user and are subject to their owners' rights. diff --git a/README.zh-CN.md b/README.zh-CN.md index 332cfc3..b34546a 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -1,96 +1,93 @@ -# nnnotes？！ +# nnnotes -[English](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.md) | [简体中文](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.zh-CN.md) +[English](README.md) | [简体中文](README.zh-CN.md) -nnnotes 是 BanG Dream! Our Notes 游戏文件的离线数据工具包：读取 Addressables catalog，下载并解密资源包，下载并解码 -masterdata，再把剧情、Live2D 模型、据点、着色器、CRI 音频和谱面导出成结构化 JSON 与通用格式文件。 -`nnnotes site build` 写出 [ournotes-player](https://github.com/empty-sekai/ournotes-player) 播放的静态站点， -`nnnotes music-data` 写出包含全部歌曲与谱面的单个 JSON 文件。命名灵感来自 sssekai。 +nnnotes 将你自己的 BanG Dream! Our Notes 游戏文件导出为结构化数据、媒体文件和静态播放器站点。 +它读取 Addressables 目录、资源包、主数据和 APK 资源，导出剧情、Live2D、谱面、房间、UI、着色器与音频， +并为 [ournotes-player](https://github.com/empty-sekai/ournotes-player) 组装站点。 -本项目为非官方爱好者项目，与游戏的开发和运营方无关。仓库不包含任何游戏资源、密钥或服务器地址：游戏文件、密钥和服务器地址 -都由使用者在自己的配置中提供，导出结果只保存在使用者指定的本地目录。 +持久化的 **depot** 保存转换后的对象、任务结果和目录版本。各次导出共享这些结果；命名站点在构建之间保留 +所选内容。depot 也可以通过共享目录或兼容 S3 的对象存储在机器之间复用工作。 -**文档（英文）：** +**[文档](https://metasekailab.github.io/nnnotes/)** · +[快速开始](https://metasekailab.github.io/nnnotes/quickstart/) · +[命令参考](https://metasekailab.github.io/nnnotes/cli/) ## 安装 +需要 Python 3.11 或更新版本： + ```sh -pip install nnnotes # Python 3.11+ -pip install 'nnnotes[fonts]' # 游戏字体（--fonts game）与 site build 的剧情文字 -pip install 'nnnotes[deck]' # music-data 的组卡模型谱面统计 +pip install nnnotes==0.3.0 +pip install 'nnnotes[fonts]==0.3.0' # 剧情与 UI 字体 +pip install 'nnnotes[remote]==0.3.0' # 兼容 S3 的 depot 远端 +pip install 'nnnotes[deck]==0.3.0' # music-data 谱面统计 ``` -此外还需要游戏的 `base.apk`；读取 masterdata 的命令需要解码后的 masterdata；音频与视频需要外部工具 -[vgmstream](https://vgmstream.org/) 和 [FFmpeg](https://ffmpeg.org/)（`site build` 另需 Node.js 20+ 与构建好的 -ournotes-player）。 +按需选择附加依赖，也可以组合为 `nnnotes[fonts,remote,deck]==0.3.0`。 +音视频转换需要 [vgmstream](https://vgmstream.org/) 与 [FFmpeg](https://ffmpeg.org/)。 +站点构建另需 Node.js 22+ 和已构建的 **ournotes-player 0.2.x** 包。 +`deck` 附加依赖安装 `ournotes-sim==0.0.12`。 + +## 第一次导出 -## 快速上手 +通过配置向导提供自己的客户端文件、密钥与服务地址： ```sh -nnnotes config init --user # 逐项询问设置（密钥不回显），写入用户配置文件 -nnnotes config check # 每项设置的来源与是否有效，不显示设置值 +nnnotes config init --user +nnnotes config check nnnotes master download --latest -o work/master-bin nnnotes master decode work/master-bin -o work/master -nnnotes --master work/master export story 10462 -o out/story_10462 -nnnotes site build -o out/site --all --player path/to/ournotes-player +nnnotes config set paths.master work/master +nnnotes export episode 10462 -o out/episode.json +nnnotes export story 10462 --fonts game -o out/story +``` + +将 `paths.depot` 设为持久目录，`paths.apk` 设为游戏 APK。向导输入密钥时不回显；`config check` 显示配置项名称、 +来源及有效性。剧情导出需要 `fonts` 附加依赖及音频工具。完整前置条件和输出结构见 +[快速开始](https://metasekailab.github.io/nnnotes/quickstart/)。 + +## 构建播放器站点 + +```sh +npm install ournotes-player@0.2.0 +nnnotes site build --name stories -o out/site --story 10462 --fonts game --player node_modules/ournotes-player +node node_modules/ournotes-player/scripts/validate-data.mjs out/site +python -m http.server 8000 --directory out/site ``` -每个命令输出简短摘要，加 `--json` 时输出 JSON 报告。脚本中可以不经提问写好配置文件： -`nnnotes config init --user --no-input --set 段.键=值 ...`。 - -## 命令 - -| 命令 | 作用 | -| --- | --- | -| `config init\|set\|unset\|check\|path` | 写入、修改和检查配置文件 | -| `servers` | 游戏 API 的服务器列表：各区服及其 CDN 与 API 根地址 | -| `catalog keys\|browse` | 区服 catalog 的资源键；在本地网页中浏览 catalog 与资源包 | -| `catalog versions\|import\|fetch\|diff` | depot 中保存的 catalog 版本，以及两个版本之间的变化 | -| `bundle pull` | 下载资源键的依赖闭包，把解密后的 bundle 复制到指定目录 | -| `master version\|download\|decode` | 区服当前的 masterdata 版本、下载，以及解码为 JSON | -| `export assets` | 经内容寻址的 depot，把所选资源包中的每个对象导出为通用格式 | -| `export episode\|story\|live\|live2d\|spot\|room\|shader\|audio\|ui` | 单个对象：ADV 剧情集、完整剧情、谱面、Live2D 模型、据点、房间、着色器、cue sheet、UI 库 | -| `apk settings\|hca-key` | `base.apk` 中的画面设置与 CRI HCA 密钥 | -| `depot plan\|run\|verify\|gc` | `export assets` 将运行什么及原因、单独运行任务、检查和回收 depot | -| `voices list\|search\|summary\|get` | masterdata 中的角色语音与剧情语音 | -| `site build` | ournotes-player 静态站点：谱面、Live2D 模型与剧情，支持多个区服与语言 | -| `music-data` | 全部歌曲与谱面的单个 JSON 文件，附组卡模型的谱面统计 | - -[命令参考](https://metasekailab.github.io/nnnotes/cli/)列出了所有参数。 - -## 文档 - -- [快速上手](https://metasekailab.github.io/nnnotes/quickstart/)：从空配置到导出一集剧情。 -- 操作指南：[配置](https://metasekailab.github.io/nnnotes/configuration/)、 - [导出剧情与谱面](https://metasekailab.github.io/nnnotes/export/)、 - [构建 ournotes-player 站点](https://metasekailab.github.io/nnnotes/site/)、 - [导出全部资源](https://metasekailab.github.io/nnnotes/assets/)、 - [日服](https://metasekailab.github.io/nnnotes/jp/)、 - [music-data](https://metasekailab.github.io/nnnotes/music-data/)。 -- 参考：[命令](https://metasekailab.github.io/nnnotes/cli/)、 - [命令输出](https://metasekailab.github.io/nnnotes/commands/)、 - [JSON schema](https://metasekailab.github.io/nnnotes/schemas/)、 - [Python API](https://metasekailab.github.io/nnnotes/api/)、 - [契约](https://metasekailab.github.io/nnnotes/contracts/)、[stage](https://metasekailab.github.io/nnnotes/stages/)、 - [视图](https://metasekailab.github.io/nnnotes/views/)。 -- 原理：[架构](https://metasekailab.github.io/nnnotes/architecture/)、 - [确定性](https://metasekailab.github.io/nnnotes/determinism/)、 - [验证](https://metasekailab.github.io/nnnotes/validation/)。 - -在同一套环境下导出是确定的：同样的输入，在 nnnotes、Python 依赖库（UnityPy、Pillow、numpy 等）和外部工具版本都相同时， -导出文件逐字节相同；JSON 一律 UTF-8、LF 换行，非有限数写作 `1e999`。版本不同时，相同内容可能写成不同的字节（不同版本的 -Pillow 写出的 PNG 像素相同，字节可能不同）。 - -## 版本记录与版本号 - -每个版本的改动见 [CHANGELOG.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/CHANGELOG.md)。版本号遵循 -[语义化版本](https://semver.org/lang/zh-CN/)，约束的接口是命令行、配置键、JSON schema 和文档中的 Python API。 - -## 参与贡献 - -开发环境、检查与提交规范见 -[CONTRIBUTING.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/CONTRIBUTING.md)。 - -## License - -MIT，见 [LICENSE](https://github.com/MetaSekaiLab/nnnotes/blob/main/LICENSE)。 +打开 `http://localhost:8000/story/`。Live2D、MotionSync 和 Spine 功能需要嵌入页面提供对应的浏览器运行时， +具体见[站点指南](https://metasekailab.github.io/nnnotes/site/)。 +用 `--pair MUSIC_ID:DIFFICULTY` 选择谱面，`--live2d KEY` 选择模型，`--story-languages ja,en` 选择剧情语言。 +复用同一站点名会更新本次选中的内容，并保留其他内容。 + +## 选择工作流 + +| 任务 | 命令 | 指南 | +|---|---|---| +| 配置输入 | `config init`、`config check`、`servers` | [配置](https://metasekailab.github.io/nnnotes/configuration/) | +| 查看与保存目录 | `catalog keys`、`browse`、`versions`、`import`、`fetch`、`diff` | [命令](https://metasekailab.github.io/nnnotes/cli/) | +| 复制源资源包 | `bundle pull` | [资源导出](https://metasekailab.github.io/nnnotes/assets/) | +| 导出主题 | `export episode`、`story`、`live`、`live2d`、`spot`、`room`、`shader`、`audio`、`ui` | [导出](https://metasekailab.github.io/nnnotes/export/) | +| 转换指定资源 | `export assets`、`depot plan`、`depot run` | [资源导出](https://metasekailab.github.io/nnnotes/assets/) | +| 检查与保留存储结果 | `depot verify`、`depot gc` | [Depot](https://metasekailab.github.io/nnnotes/depot/) | +| 导出歌曲与谱面统计 | `music-data` | [音乐数据](https://metasekailab.github.io/nnnotes/music-data/) | +| 查找角色语音 | `voices list`、`search`、`summary`、`get` | [命令](https://metasekailab.github.io/nnnotes/cli/) | + +命令默认打印摘要；`--json` 输出结构化报告。退出状态和各项参数见命令参考。 + +## 数据与兼容性 + +产物通过版本化清单和共享对象记录描述。消费这些数据时，请查阅 +[格式](https://metasekailab.github.io/nnnotes/formats/)、[产物](https://metasekailab.github.io/nnnotes/products/)和 +[Schema](https://metasekailab.github.io/nnnotes/schemas/)参考。 +任务标识包括输入内容、选项和转换实现，其中也包括原生编码器。 +[确定性说明](https://metasekailab.github.io/nnnotes/determinism/)解释不同安装环境中的可复现性。 + +版本覆盖 CLI、配置、Schema 和文档中的 Python API。1.0 之前，次版本可以改变这些接口。 +改动见 [CHANGELOG.md](CHANGELOG.md)，实现分层见[架构](https://metasekailab.github.io/nnnotes/architecture/)。 + +## 贡献与许可 + +[CONTRIBUTING.md](CONTRIBUTING.md) 说明开发和合成测试流程。nnnotes 采用 [MIT](LICENSE) 许可。 +这是非官方爱好者项目，与游戏开发或运营方无关。游戏资源、密钥和服务配置由各用户自行提供，并受其权利人的权利约束。 diff --git a/docs/depot.md b/docs/depot.md index b34cf36..40f67aa 100644 --- a/docs/depot.md +++ b/docs/depot.md @@ -22,6 +22,8 @@ the `remote` extra. See [configuration](configuration.md) for endpoint and crede Results and content are fetched when needed. Publishing a run first uploads its completed results, available stored inputs and layout contents. This also shares results computed before remote storage was configured. A run pointer alone is not evidence that a result exists: planning checks the result itself. +S3 listings follow continuation tokens and deletion requests use batches of at most 1,000 objects. +A per-object deletion error stops collection and reports failure, even when the service returns HTTP success. Catalog discovery merges version metadata and shares locally imported versions with their catalog bytes. `depot verify` checks local content and results. When a local result refers to content held only by the remote, diff --git a/docs/index.md b/docs/index.md index 1fbbb30..73cdad5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,57 +1,41 @@ # nnnotes -nnnotes is an offline data toolkit for the game files of BanG Dream! Our Notes. It reads Addressables catalogs, -downloads and decrypts asset bundles, downloads and decodes master data, and exports stories, Live2D models, spots, -shaders, CRI audio and live charts as structured JSON and common file formats. `nnnotes site build` writes the static -site that [ournotes-player](https://github.com/empty-sekai/ournotes-player) plays, and `nnnotes music-data` one JSON -file with every song and chart. +Export your BanG Dream! Our Notes game files as structured products, media or a static player site. +nnnotes reads catalogs, bundles, APK resources and master data, then stores reusable conversion results in a +persistent depot. You supply the client files, keys and server configuration. -This is an unofficial fan project, not affiliated with the game's developer or operator. nnnotes contains no game -assets, keys or server addresses: you supply the game files, the keys and the server addresses in your own -configuration, and the exports stay in the local directories you choose. +## Start here -## Where to start +1. [Install and export a story](quickstart.md). +2. [Build and serve a player site](site.md) with ournotes-player 0.2.x. +3. [Share and retain depot results](depot.md), locally or through S3-compatible storage. -
+## Guides -- **[Quick start](quickstart.md)** +| Goal | Guide | +|---|---| +| Set input paths, regions and tools | [Configuration](configuration.md) | +| Export a story, live, model or other subject | [Exports](export.md) | +| Convert selected bundles and objects | [Asset export](assets.md) | +| Use Japanese client files | [Japanese release](jp.md) | +| Produce song and chart statistics | [Music data](music-data.md) | +| Check your own outputs | [Validation](validation.md) | - From an empty config file to an exported story, step by step. +## Reference -- **How-to guides** +[CLI](cli.md) · [Command reports](commands.md) · [Python API](api.md) · [Schemas](schemas.md) · +[Formats](formats.md) · [Products](products.md) · [Depot](depot.md) · [Contracts](contracts.md) · +[Stages](stages.md) · [Views](views.md) · [Replay data](replay.md) · [UI library](ui.md) - [Configure nnnotes](configuration.md), [export stories and lives](export.md), - [build an ournotes-player site](site.md), [export every asset](assets.md), - [the Japanese release](jp.md), [music data](music-data.md). +[Architecture](architecture.md) explains the conversion and assembly layers. +[Determinism](determinism.md) explains task identities and output reproducibility. -- **Reference** +## Requirements and versions - [Commands](cli.md), [command outputs](commands.md), [JSON schemas](schemas.md), [Python API](api.md), - [contracts](contracts.md), [stages](stages.md), [views](views.md), [replay data](replay.md), - [UI library](ui.md). +nnnotes 0.3.0 requires Python 3.11+. Optional extras provide fonts (`fonts`), S3 clients (`remote`) and +ournotes-sim 0.0.12 chart statistics (`deck`). Site builds require Node.js 22+ and a built ournotes-player 0.2.x +package. Audio and video conversion require vgmstream and FFmpeg. -- **Explanation** - - [Architecture](architecture.md), [determinism](determinism.md), [validation](validation.md). - -
- -## Install - -```sh -pip install nnnotes # Python 3.11+ -pip install 'nnnotes[fonts]' # the game's fonts (--fonts game) and the story text of site build -pip install 'nnnotes[deck]' # the deck model's chart statistics of music-data -``` - -Besides the package, nnnotes reads the game's `base.apk` and, for the commands that need it, decoded master data; -audio and video need [vgmstream](https://vgmstream.org/) and [FFmpeg](https://ffmpeg.org/), and `site build` also -Node.js 22+ and a built ournotes-player. [Configuration](configuration.md#what-each-command-needs) lists what each -command needs. - -## Versions - -Versions follow [Semantic Versioning](https://semver.org/). The interface a version describes is the command line -(commands, options, exit statuses and the `--json` reports), the configuration keys, the JSON schemas shipped in -`nnnotes.schemas` and the documented Python API. Before 1.0 a minor version can change any of them; the -[changelog](https://github.com/MetaSekaiLab/nnnotes/blob/main/CHANGELOG.md) names every change. +Versions cover the CLI, configuration, schemas and documented Python API. Before 1.0, minor versions may change +these contracts; see the [changelog](https://github.com/MetaSekaiLab/nnnotes/blob/main/CHANGELOG.md). +This unofficial fan project is MIT-licensed. Game resources remain subject to their owners' rights. diff --git a/docs/quickstart.md b/docs/quickstart.md index 9be670c..2c80669 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,82 +1,94 @@ # Quick start -This walk-through takes you from an empty configuration to an exported ADV episode and a story export. It needs -your own installation of the game: its `base.apk`, and the keys and server addresses of your client -([where the values come from](configuration.md#where-the-values-come-from)). +This guide exports one episode, a playable story product and a static story site. Use your own matching client +files, keys and server addresses; [configuration](configuration.md#where-the-values-come-from) explains the inputs. -## 1. Install +## 1. Install the tools ```sh -pip install 'nnnotes[fonts]' +pip install 'nnnotes[fonts]==0.3.0' ``` -Python 3.11 or later. Install [vgmstream](https://vgmstream.org/) (`vgmstream-cli`) and -[FFmpeg](https://ffmpeg.org/) as well, on `PATH` or named in the config file; stories decode their audio and -videos with them. +Use Python 3.11+. Install [vgmstream](https://vgmstream.org/) (`vgmstream-cli`) and +[FFmpeg](https://ffmpeg.org/) on PATH, or configure their executable paths. They decode story audio and video. +The final site step also needs Node.js 22+. -## 2. Write the config file +## 2. Configure your inputs ```sh nnnotes config init --user +nnnotes config check ``` -In a terminal this asks for each setting in turn, among them the bundle and master data keys, the names of your -regions with each region's CDN base and API root, the catalog language, and the paths of the depot, the APK and the -tools. Enter leaves a setting empty; the keys are read without echo. The file is written to the per-user location, -which every command reads from any directory (`nnnotes config path` shows it). +The wizard asks for region names, keys, server addresses and paths. Set `paths.depot` to a persistent directory, +`paths.apk` to your game APK and `catalog.language` to the desired language. Keys are entered without echo. +`config check` lists setting names, their source and validity. A command names any required setting it lacks. +The per-user file is available from any directory; `nnnotes config path` shows the configuration search order. -Check it: +## 3. Inspect the catalog and prepare master data ```sh -nnnotes config check +nnnotes catalog keys --prefix Adv/ --limit 5 +nnnotes master version +nnnotes master download --latest -o work/master-bin +nnnotes master decode work/master-bin -o work/master +nnnotes config set paths.master work/master ``` -Every setting is listed with where it comes from and whether it is valid, never with its value. A command that -needs an unset setting stops with exit status 2 and names the TOML key, the environment variable and the flag that -set it. +The catalog is retained in the depot. Master downloads are checked against their manifest; decoding writes a +JSON file for each table. Use a story ID available in your selected region in the examples below. -## 3. Look at the catalog +## 4. Export the episode and story ```sh -nnnotes catalog keys --prefix Adv/ --limit 5 +nnnotes export episode 10462 -o out/episode.json +nnnotes export story 10462 --fonts game -o out/story ``` -The first command downloads the region's catalog into the depot; the keys it lists are the addressable keys of the -game's assets. `nnnotes catalog browse` serves a local web page to browse the catalogs and bundles of every -configured region. +The episode file contains commands and localized lines. The story directory contains a product manifest at +`stories/10462.json`, its referenced products and content-addressed files under `assets/`. It includes the +scene, models, text resources and any media the story uses. Audio defaults to AAC; `--format` selects another +supported encoding. UI and font products use the catalog language by default. -## 4. Get the master data +For two languages, add `--story-languages ja,en`. Open fonts are also supported: +`--fonts open --font ja=fonts/NotoSansJP.otf --font en=fonts/NotoSans.ttf`. +Supply a font for every selected language, either as a flag or in `[paths.fonts]`. + +## 5. Build, validate and serve a site ```sh -nnnotes master version -nnnotes master download --latest -o work/master-bin -nnnotes master decode work/master-bin -o work/master -nnnotes config set paths.master work/master +npm install ournotes-player@0.2.0 +nnnotes site build --name stories -o out/site --story 10462 --fonts game --player node_modules/ournotes-player +node node_modules/ournotes-player/scripts/validate-data.mjs out/site +python -m http.server 8000 --directory out/site ``` -`master version` asks the region's game API which master data version it serves now; `master download` fetches -that version and checks every file's SHA-256; `master decode` writes one JSON file per table. The last line makes -the decoded directory the default of every command that reads master data. +Open `http://localhost:8000/story/`. Configure the page's Cubism Core and, for the features in your story, +MotionSync Core or Spine runtime; see the player's +[getting started guide](https://github.com/empty-sekai/ournotes-player/blob/main/docs/getting-started.md). +These runtimes have their own licenses and are provided by the page. + +The site builder accepts a built ournotes-player 0.2.x package, validates its package identity and required files, +and reuses the story's depot tasks. The validator checks manifests, references and data contracts before playback. + +## 6. Update and retain your work -## 5. Export an episode and a story +Reuse `--name stories` to update selected topics while retaining the rest. Use a different name for a separate +site. To materialize its saved products with the current player, configure `paths.player` and run: ```sh -nnnotes export episode 10462 -o out/episode_10462.json -nnnotes export story 10462 --fonts game -o out/story_10462 +nnnotes site build --name stories -o out/restored-site --player-only --player node_modules/ournotes-player +nnnotes depot verify +nnnotes depot gc --dry-run ``` -`export episode` writes the episode's command list with its lines in five languages and the index of its voices, -sounds and videos. `export story` writes the story product of the episode and the products it plays with below -`out/story_10462`: the story index, the stage scene, the assets, the Live2D models, the cue sheets decoded to audio -(AAC by default, `--format` chooses another web format), the videos as WebM, and the shared chat rows and CRI Lips -data when the episode uses them. It also exports the UI and glyph pages in `[catalog] language`; select several -with `--story-languages ja,en`. The example uses the client's TMP fonts. For open fonts, use -`--fonts open --font ja=/path/to/font.otf` (or configure `[paths.fonts]`). Both modes require the `fonts` extra. -The files are under `assets/`. Add `--json` to any command to get its report as JSON. +The depot and the published output are separate directories. Read [retention](depot.md#retention) before collecting +stored results. Add `--json` to a command for its structured report. ## Next -- [Export stories and lives](export.md): the other single-subject exports and their options. -- [Build an ournotes-player site](site.md): charts, models and stories as one static site. -- [Export every asset](assets.md): every object of the selected bundles through the content-addressed depot. -- [Command reference](cli.md): every command and option. +- [Export stories and lives](export.md) for individual products. +- [Build a site](site.md) for charts, regional selections, language choices and hosting. +- [Depot](depot.md) for shared storage, verification and collection. +- [Japanese release](jp.md) for its catalog and APK requirements. +- [Commands](cli.md) for flags and exit statuses. diff --git a/docs/site.md b/docs/site.md index 6987475..029ebae 100644 --- a/docs/site.md +++ b/docs/site.md @@ -4,7 +4,12 @@ ## Requirements -Configure `[paths] depot`, the catalog and APK, and the decoded master data for the selected charts and stories. Supply a built [ournotes-player](https://github.com/empty-sekai/ournotes-player) checkout or package through `--player` or `[paths] player`. Node.js 22 or newer runs chart read sets. Audio conversion uses vgmstream and FFmpeg. Text rendering needs the `fonts` extra and either `--fonts game` or a font file for every requested language. +Configure `[paths] depot`, the catalog and APK, and the decoded master data for the selected charts and stories. Supply a built [ournotes-player](https://github.com/empty-sekai/ournotes-player) 0.2.x checkout or npm package through `--player` or `[paths] player`. Node.js 22 or newer runs chart read sets. Audio conversion uses vgmstream and FFmpeg. Text rendering needs the `fonts` extra and either `--fonts game` or a font file for every requested language. + +Install the complete player package with `npm install ournotes-player@0.2.0`, then pass +`--player node_modules/ournotes-player`. The browser-only ZIP supplies bundles for embedding; the site builder +also needs the package metadata, scripts and example pages. It checks the package name, stable 0.2.x version +and required files before starting. ## Select products @@ -44,3 +49,15 @@ A chart read set is a depot task whose inputs include the player sources, produc Publication writes assets, immutable products, aliases/pages, and then indexes. It collects retired owned files after successful replacements and retains unowned files. A failed publication keeps its journal and successful-run pointer for retry. Serve the directory with a static server and open `index.html`, `live2d/`, or `story/`. `--workers`, `--memory`, and `--fetch-workers` control product task execution. `--stored-catalog LABEL|SHA` selects a catalog already registered in the depot. Reports include product task counts, site task counts, read-set checks, and file placement counts. See the [command reference](cli.md#nnnotes-site-build) for all flags and [site snapshots](products.md#site-snapshots) for product references. + +## Validate and open + +```sh +node node_modules/ournotes-player/scripts/validate-data.mjs out/site +python -m http.server 8000 --directory out/site +``` + +Open the chart page at `/`, the model page at `/live2d/` or the story page at `/story/`. +Supply Cubism Core and the MotionSync/Spine runtimes needed by the selected content through the player's page +configuration. See its [runtime guide](https://github.com/empty-sekai/ournotes-player/blob/main/docs/getting-started.md#browser-runtimes) +and [hosting guide](https://github.com/empty-sekai/ournotes-player/blob/main/docs/embedding.md#hosting-the-data). diff --git a/docs/validation.md b/docs/validation.md index 63e7ea7..dcd1008 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -1,51 +1,68 @@ # Validation -The test suite runs on synthetic inputs, so it needs no game data. The experiments on this page check nnnotes -against a whole game version, which you supply yourself: each one names what it checks, how to run it with your own -configuration, and the result for the version named. Only commands and checks are listed. The outputs stay with -whoever runs them. +Validation has three distinct scopes: synthetic contract tests, checks over generated files, and browser playback. +Use the checks that match the output and consumer you are working on. -The results below are for the Traditional Chinese release, client 1.0.1, catalog language `zh-Hant`, with its master -data. The [Japanese release](jp.md#validation-and-current-limits) has its own section. +## Synthetic tests -## Setup +```sh +pip install -e '.[test,fonts,remote,deck]' +python -m pytest +python -m ruff check . +python -m pyright +``` + +The tests exercise object conversion, product assembly, language selection, regional aliases, reference closure, +task identities and retained layouts. Directory remotes exercise sharing between depots. S3 protocol tests use +botocore stubs to check request parameters, pagination, deletion batches and errors without an account. +Publication tests inject failures between file replacements and check recovery and index ordering. + +Every registered output and site stage has a golden record. With the dependency versions in +`.github/goldens-constraints.txt`, run: ```sh -nnnotes config check # every setting the commands below need is valid -nnnotes master download --latest -o work/master-bin --json > work/download.json -nnnotes master decode work/master-bin -o work/master --json > work/decode.json -nnnotes config set paths.master work/master +python -m pytest tests/pipeline/test_goldens.py tests/pipeline/test_objexport.py ``` -The lists of subjects come from the master data and the catalog: +Set `GOLDENS_STRICT=1` for a matching pinned environment so a dependency mismatch fails the check. +The records include native codec identities; see [determinism](determinism.md). + +## Check stored content ```sh -python -c "from nnnotes.source.master import table; print(*sorted(r['_id'] for r in table('work/master', 'MasterAdv')), sep='\n')" > work/episodes.txt -nnnotes catalog keys --prefix Character/Live2D/ --limit 0 > work/live2d-keys.txt +nnnotes depot verify +nnnotes depot verify --quick +nnnotes depot gc --dry-run ``` -## Experiments - -| Experiment | How to run it | Result (client 1.0.1, zh-Hant) | -|---|---|---| -| Master data | `master download` checks every file against the SHA-256 of `MasterManifest.json`; `master decode` reports each table it could not decode under `failed` | every file verified, every table decoded | -| ADV episodes | `export episode ID` for every id of `episodes.txt` | 946 / 946 exported | -| Stories | `export story ID -o out/stories/ID` for every id, all with the same models directory; then read every JSON file and resolve every path `story.json` and the files it indexes name | 946 / 946 exported; every JSON file parses, every named file, model and cue exists | -| Story reproducibility | export the stories that use frames, effects, post effects, stills, chats, talk windows or videos twice into two directories and compare them byte by byte | 235 such episodes; the two exports are equal except the models directory path in `story.json` | -| Live2D models | `export live2d KEY` for every model key of `live2d-keys.txt` (the keys ending in `/model/`) | 239 / 239 exported, 185 of them used by stories | -| CRI audio | `export audio CUE_SHEET` for every cue sheet under `Cri/Sound/` | 681 / 681 decoded | -| Lives | `export live MUSIC_ID --difficulty D` for every music and difficulty with a `MasterLiveMusicScore` row | 336 / 336 exported | -| Spots and rooms | `export spot ID` for every `MasterHomeSpot` row and `export room KEY` for every room background prefab; validate every `.glb` with the [Khronos glTF-Validator](https://github.com/KhronosGroup/glTF-Validator) | 39 / 39 spots, 20 / 20 rooms; every reference resolves; every glb has 0 validator errors | -| Site, one region | `site build --all --all-live2d --all-stories --story-languages en --no-audio` twice into two empty directories, then compare them | 336 charts, 239 models, 946 stories; no failures file; the two sites are byte-identical | -| Site, every region | `site build --all --all-regions --format aac` twice into two empty directories, then compare them | 340 charts of 85 songs from the regions' current master data, five languages; no failures file; the two sites are byte-identical | -| Regions | for each region and language, `catalog keys --limit 0` and the catalog's bundles; compare the master tables a chart build reads across regions | the regions serve the same catalog and bundles for a language with the same keys; the chart tables are equal across the three regions; the text tables have all five languages | - -## Checks of every run - -These hold for every command and are checked by the test suite on synthetic data: - -- A key, id or model that the catalog or the master data does not have stops the command with exit status 2 before - any bundle is fetched or file written. -- The documents of an asset export run (plan, tasks, results, artifacts, run manifest, layout, coverage, failures, - cost) and the music data file validate against their [JSON schemas](schemas.md). -- The same inputs give byte-identical outputs ([determinism](determinism.md)). +Full verification checks hashes; quick verification checks sizes. Referenced content held only by the remote +is checked there. A collection dry run reports eligible removals under the configured retention policy. +See [depot](depot.md) for run roots, grace periods and remote collection. + +## Check exported products + +Build a selected site with your configured inputs, then use the compatible player's validator: + +```sh +nnnotes site build --name example -o out/site --story 10462 --fonts game --player node_modules/ournotes-player +node node_modules/ournotes-player/scripts/validate-data.mjs out/site +``` + +The validator checks product schemas, shared objects, file references, chart metadata, unique asset byte counts, +language-specific resources and model dependencies. By default it checks every declared language. +Chart site builds also compare sampled read-set plans with full simulation and fall back to full simulation +when a plan fails or disagrees. See [site](site.md#player-read-sets-and-variants). + +For your own corpus, enumerate subjects from matching master data and catalogs, preserve command reports, +and record the configuration and tool versions used. Compare normalized JSON with references resolved; +compare decoded image pixels and audio samples when encoder differences affect stored bytes. + +## Check playback and hosting + +Serve the output over HTTP, load required runtimes and exercise a chart, a model and the story features in your +selection. Check language changes, seek, completion and browser network errors. Structural validation and rendered +appearance answer different questions. Hosting instructions cover CORS, compression and cache headers in the +[player documentation](https://github.com/empty-sekai/ournotes-player/blob/main/docs/embedding.md). + +An S3 endpoint also has its own permissions, consistency and gateway behavior. Check reads and writes under an +isolated test prefix when configuring a new endpoint, and scope collection to the depot prefix. From 13751daae3e3b2b585974fc17d7e6c23aed29e26 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 19 Oct 2026 12:58:29 +0800 Subject: [PATCH 91/91] docs: keep package page links and data guarantees explicit --- README.md | 10 ++++++---- README.zh-CN.md | 8 +++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 990186d..84751bd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # nnnotes -[English](README.md) | [简体中文](README.zh-CN.md) +[English](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.md) | [简体中文](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.zh-CN.md) nnnotes turns your BanG Dream! Our Notes game files into structured data, media and static player sites. It reads Addressables catalogs, bundles, master data and APK resources; exports stories, Live2D models, live @@ -85,13 +85,15 @@ Product exports use versioned manifests and shared object records. Read the [product reference](https://metasekailab.github.io/nnnotes/products/) and [schemas](https://metasekailab.github.io/nnnotes/schemas/) to consume them. Task identities include input contents, options and conversion implementations, including native encoders. +Byte-identical output requires matching inputs and library/tool implementations, including Pillow and its native +compression backend. JSON uses UTF-8 and LF; infinite values use `1e999` or `-1e999`. [Determinism](https://metasekailab.github.io/nnnotes/determinism/) explains reproducibility across installations. Versions cover the CLI, configuration, schemas and documented Python API. Before 1.0, a minor version can change -these interfaces. See [CHANGELOG.md](CHANGELOG.md) and the [architecture](https://metasekailab.github.io/nnnotes/architecture/). +these interfaces. See [CHANGELOG.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/CHANGELOG.md) and the [architecture](https://metasekailab.github.io/nnnotes/architecture/). ## Contributing and license -[CONTRIBUTING.md](CONTRIBUTING.md) describes development and synthetic tests. nnnotes is licensed under -[MIT](LICENSE). This is an unofficial fan project, unaffiliated with the game's developer or operator. +[CONTRIBUTING.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/CONTRIBUTING.md) describes development and synthetic tests. nnnotes is licensed under +[MIT](https://github.com/MetaSekaiLab/nnnotes/blob/main/LICENSE). This is an unofficial fan project, unaffiliated with the game's developer or operator. Game resources, keys and server configuration are supplied by each user and are subject to their owners' rights. diff --git a/README.zh-CN.md b/README.zh-CN.md index b34546a..413a3ea 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -1,6 +1,6 @@ # nnnotes -[English](README.md) | [简体中文](README.zh-CN.md) +[English](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.md) | [简体中文](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.zh-CN.md) nnnotes 将你自己的 BanG Dream! Our Notes 游戏文件导出为结构化数据、媒体文件和静态播放器站点。 它读取 Addressables 目录、资源包、主数据和 APK 资源，导出剧情、Live2D、谱面、房间、UI、着色器与音频， @@ -82,12 +82,14 @@ python -m http.server 8000 --directory out/site [格式](https://metasekailab.github.io/nnnotes/formats/)、[产物](https://metasekailab.github.io/nnnotes/products/)和 [Schema](https://metasekailab.github.io/nnnotes/schemas/)参考。 任务标识包括输入内容、选项和转换实现，其中也包括原生编码器。 +字节一致的输出需要相同输入及库和工具实现，其中包括 Pillow 及其原生压缩后端。 +JSON 使用 UTF-8 和 LF，无穷值写为 `1e999` 或 `-1e999`。 [确定性说明](https://metasekailab.github.io/nnnotes/determinism/)解释不同安装环境中的可复现性。 版本覆盖 CLI、配置、Schema 和文档中的 Python API。1.0 之前，次版本可以改变这些接口。 -改动见 [CHANGELOG.md](CHANGELOG.md)，实现分层见[架构](https://metasekailab.github.io/nnnotes/architecture/)。 +改动见 [CHANGELOG.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/CHANGELOG.md)，实现分层见[架构](https://metasekailab.github.io/nnnotes/architecture/)。 ## 贡献与许可 -[CONTRIBUTING.md](CONTRIBUTING.md) 说明开发和合成测试流程。nnnotes 采用 [MIT](LICENSE) 许可。 +[CONTRIBUTING.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/CONTRIBUTING.md) 说明开发和合成测试流程。nnnotes 采用 [MIT](https://github.com/MetaSekaiLab/nnnotes/blob/main/LICENSE) 许可。 这是非官方爱好者项目，与游戏开发或运营方无关。游戏资源、密钥和服务配置由各用户自行提供，并受其权利人的权利约束。