From dc198b0c977b2b3ae1bffaca0e3ce3c2653133c6 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Fri, 23 Oct 2026 00:25:15 +0800 Subject: [PATCH 1/2] feat(packing): build song packages and HCA cue sheets --- .github/scripts/smoke_wheel.py | 12 +- README.md | 11 +- README.zh-CN.md | 11 +- docs/api.md | 10 + docs/architecture.md | 2 +- docs/cli.md | 101 ++ docs/commands.md | 70 ++ docs/configuration.md | 2 + docs/contracts.md | 11 + docs/index.md | 2 +- docs/packing.md | 154 ++++ docs/quickstart.md | 2 +- docs/schemas.md | 6 + docs/song-packing.md | 190 ++++ mkdocs.yml | 2 + pyproject.toml | 6 +- src/nnnotes/__init__.py | 2 +- src/nnnotes/cli/__init__.py | 9 +- src/nnnotes/cli/catalog.py | 161 ++++ src/nnnotes/cli/common.py | 4 + src/nnnotes/cli/pack.py | 453 +++++++++ src/nnnotes/formats/VGAudio.LICENSE.txt | 22 + src/nnnotes/formats/acbwrite.py | 267 ++++++ src/nnnotes/formats/hca.py | 150 +++ src/nnnotes/packing/__init__.py | 20 + src/nnnotes/packing/assets.py | 241 +++++ src/nnnotes/packing/audio.py | 302 ++++++ src/nnnotes/packing/build.py | 326 +++++++ src/nnnotes/packing/plan.py | 213 +++++ src/nnnotes/packing/project.py | 157 ++++ src/nnnotes/packing/song.py | 154 ++++ src/nnnotes/schemas/audio-build.schema.json | 52 ++ src/nnnotes/schemas/pack-target.schema.json | 75 ++ src/nnnotes/schemas/package-plan.schema.json | 56 ++ .../schemas/package-validation.schema.json | 40 + src/nnnotes/schemas/song-build.schema.json | 60 ++ src/nnnotes/schemas/song-package.schema.json | 633 +++++++++++++ src/nnnotes/source/addressables.py | 28 +- src/nnnotes/source/catalogbuild.py | 571 ++++++++++++ src/nnnotes/source/configfile.py | 2 +- src/nnnotes/source/master.py | 60 +- src/nnnotes/source/nnnotes.example.toml | 4 + src/nnnotes/unity/bundlefs.py | 178 ++++ src/nnnotes/unity/pack.py | 868 ++++++++++++++++++ tests/cli/test_catalog.py | 294 ++++++ tests/cli/test_pack.py | 464 ++++++++++ tests/formats/test_acbwrite.py | 43 + tests/formats/test_hca.py | 51 + tests/helpers/packassets.py | 133 +++ tests/helpers/packing.py | 252 +++++ tests/helpers/synth.py | 68 ++ tests/packing/test_audio.py | 50 + tests/packing/test_package.py | 59 ++ tests/packing/test_song.py | 39 + tests/source/test_catalogbuild.py | 492 ++++++++++ tests/source/test_pack.py | 76 ++ tests/unity/test_bundlefs.py | 99 ++ tests/unity/test_pack.py | 672 ++++++++++++++ 58 files changed, 8463 insertions(+), 29 deletions(-) create mode 100644 docs/packing.md create mode 100644 docs/song-packing.md create mode 100644 src/nnnotes/cli/pack.py create mode 100644 src/nnnotes/formats/VGAudio.LICENSE.txt create mode 100644 src/nnnotes/formats/acbwrite.py create mode 100644 src/nnnotes/formats/hca.py create mode 100644 src/nnnotes/packing/__init__.py create mode 100644 src/nnnotes/packing/assets.py create mode 100644 src/nnnotes/packing/audio.py create mode 100644 src/nnnotes/packing/build.py create mode 100644 src/nnnotes/packing/plan.py create mode 100644 src/nnnotes/packing/project.py create mode 100644 src/nnnotes/packing/song.py create mode 100644 src/nnnotes/schemas/audio-build.schema.json create mode 100644 src/nnnotes/schemas/pack-target.schema.json create mode 100644 src/nnnotes/schemas/package-plan.schema.json create mode 100644 src/nnnotes/schemas/package-validation.schema.json create mode 100644 src/nnnotes/schemas/song-build.schema.json create mode 100644 src/nnnotes/schemas/song-package.schema.json create mode 100644 src/nnnotes/source/catalogbuild.py create mode 100644 src/nnnotes/unity/bundlefs.py create mode 100644 src/nnnotes/unity/pack.py create mode 100644 tests/cli/test_catalog.py create mode 100644 tests/cli/test_pack.py create mode 100644 tests/formats/test_acbwrite.py create mode 100644 tests/formats/test_hca.py create mode 100644 tests/helpers/packassets.py create mode 100644 tests/helpers/packing.py create mode 100644 tests/packing/test_audio.py create mode 100644 tests/packing/test_package.py create mode 100644 tests/packing/test_song.py create mode 100644 tests/source/test_catalogbuild.py create mode 100644 tests/source/test_pack.py create mode 100644 tests/unity/test_bundlefs.py create mode 100644 tests/unity/test_pack.py diff --git a/.github/scripts/smoke_wheel.py b/.github/scripts/smoke_wheel.py index 354258c..cece55d 100644 --- a/.github/scripts/smoke_wheel.py +++ b/.github/scripts/smoke_wheel.py @@ -14,8 +14,11 @@ import nnnotes import ournotes_sim -from nnnotes import musicdata +from nnnotes import musicdata, schemas +from nnnotes.formats.hca import key_table from nnnotes.musicdata import deckdata +from nnnotes.packing import AudioOptions +from nnnotes.source.master import MasterKey, decode, encode assert Path(nnnotes.__file__).resolve().is_relative_to(Path(sys.prefix).resolve()) deck = musicdata.Deck(workers=1, aptitude=False) @@ -28,6 +31,12 @@ stats = json.loads(ournotes_sim.chart_stats(json.dumps(data), workers=1, aptitude=False)) assert stats["format"] == info["format"] assert stats["charts"] == [] +assert AudioOptions().quality == "high" +assert schemas.load("song-package")["title"] == "nnnotes.song-package/1" +assert schemas.load("pack-target")["title"] == "nnnotes.pack-target/1" +assert bytes(key_table(123)).translate(key_table(123, decrypt=True)) == bytes(range(256)) +key = MasterKey(bytes(range(32)), bytes(reversed(range(32)))) +assert decode(encode(b'{"_allData": []}', key), key) == b'{"_allData": []}' print(f"nnnotes {nnnotes.__version__}: {info['name']} {info['version']} ({info['commit']})") """ @@ -43,6 +52,7 @@ def main(directory: str) -> None: subprocess.run([python, "-I", "-m", "pip", "install", f"{wheel}[deck]"], cwd=root, check=True) subprocess.run([python, "-I", "-c", SMOKE], cwd=root, check=True) subprocess.run([scripts / ("nnnotes.exe" if os.name == "nt" else "nnnotes"), "--help"], cwd=root, check=True) + subprocess.run([python, "-I", "-m", "nnnotes", "pack", "audio", "--help"], cwd=root, check=True) if __name__ == "__main__": diff --git a/README.md b/README.md index 84751bd..00009e0 100644 --- a/README.md +++ b/README.md @@ -20,13 +20,13 @@ or S3-compatible object storage. Python 3.11 or newer: ```sh -pip install nnnotes==0.3.0 -pip install 'nnnotes[fonts]==0.3.0' # story and UI fonts -pip install 'nnnotes[remote]==0.3.0' # S3-compatible depot storage -pip install 'nnnotes[deck]==0.3.0' # music-data chart statistics +pip install nnnotes==0.3.1 +pip install 'nnnotes[fonts]==0.3.1' # story and UI fonts +pip install 'nnnotes[remote]==0.3.1' # S3-compatible depot storage +pip install 'nnnotes[deck]==0.3.1' # music-data chart statistics ``` -Choose the extras your commands need; they can be combined as `nnnotes[fonts,remote,deck]==0.3.0`. +Choose the extras your commands need; they can be combined as `nnnotes[fonts,remote,deck]==0.3.1`. Audio/video conversion uses [vgmstream](https://vgmstream.org/) and [FFmpeg](https://ffmpeg.org/). Site builds additionally require Node.js 22+ and a built **ournotes-player 0.2.x** package. The deck extra installs `ournotes-sim==0.0.12`. @@ -70,6 +70,7 @@ story languages. Reusing a site name updates selected topics and retains the oth | Configure inputs | `config init`, `config check`, `servers` | [Configuration](https://metasekailab.github.io/nnnotes/configuration/) | | Inspect or retain catalogs | `catalog keys`, `browse`, `versions`, `import`, `fetch`, `diff` | [Commands](https://metasekailab.github.io/nnnotes/cli/) | | Copy source bundles | `bundle pull` | [Asset export](https://metasekailab.github.io/nnnotes/assets/) | +| Package songs and audio | `pack plan`, `pack build`, `pack verify`, `pack audio` | [Song packing](https://metasekailab.github.io/nnnotes/song-packing/) | | Export a subject | `export episode`, `story`, `live`, `live2d`, `spot`, `room`, `shader`, `audio`, `ui` | [Exports](https://metasekailab.github.io/nnnotes/export/) | | Convert selected assets | `export assets`, `depot plan`, `depot run` | [Asset export](https://metasekailab.github.io/nnnotes/assets/) | | Check and retain stored work | `depot verify`, `depot gc` | [Depot](https://metasekailab.github.io/nnnotes/depot/) | diff --git a/README.zh-CN.md b/README.zh-CN.md index 413a3ea..e5522ac 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -18,13 +18,13 @@ nnnotes 将你自己的 BanG Dream! Our Notes 游戏文件导出为结构化数 需要 Python 3.11 或更新版本： ```sh -pip install nnnotes==0.3.0 -pip install 'nnnotes[fonts]==0.3.0' # 剧情与 UI 字体 -pip install 'nnnotes[remote]==0.3.0' # 兼容 S3 的 depot 远端 -pip install 'nnnotes[deck]==0.3.0' # music-data 谱面统计 +pip install nnnotes==0.3.1 +pip install 'nnnotes[fonts]==0.3.1' # 剧情与 UI 字体 +pip install 'nnnotes[remote]==0.3.1' # 兼容 S3 的 depot 远端 +pip install 'nnnotes[deck]==0.3.1' # music-data 谱面统计 ``` -按需选择附加依赖，也可以组合为 `nnnotes[fonts,remote,deck]==0.3.0`。 +按需选择附加依赖，也可以组合为 `nnnotes[fonts,remote,deck]==0.3.1`。 音视频转换需要 [vgmstream](https://vgmstream.org/) 与 [FFmpeg](https://ffmpeg.org/)。 站点构建另需 Node.js 22+ 和已构建的 **ournotes-player 0.2.x** 包。 `deck` 附加依赖安装 `ournotes-sim==0.0.12`。 @@ -68,6 +68,7 @@ python -m http.server 8000 --directory out/site | 配置输入 | `config init`、`config check`、`servers` | [配置](https://metasekailab.github.io/nnnotes/configuration/) | | 查看与保存目录 | `catalog keys`、`browse`、`versions`、`import`、`fetch`、`diff` | [命令](https://metasekailab.github.io/nnnotes/cli/) | | 复制源资源包 | `bundle pull` | [资源导出](https://metasekailab.github.io/nnnotes/assets/) | +| 打包歌曲与音频 | `pack plan`、`pack build`、`pack verify`、`pack audio` | [歌曲打包](https://metasekailab.github.io/nnnotes/song-packing/) | | 导出主题 | `export episode`、`story`、`live`、`live2d`、`spot`、`room`、`shader`、`audio`、`ui` | [导出](https://metasekailab.github.io/nnnotes/export/) | | 转换指定资源 | `export assets`、`depot plan`、`depot run` | [资源导出](https://metasekailab.github.io/nnnotes/assets/) | | 检查与保留存储结果 | `depot verify`、`depot gc` | [Depot](https://metasekailab.github.io/nnnotes/depot/) | diff --git a/docs/api.md b/docs/api.md index ecb2d1a..4011219 100644 --- a/docs/api.md +++ b/docs/api.md @@ -30,6 +30,16 @@ raises `GameApiError`. Other names are internal and can change in any version. ::: nnnotes.source.jp +## Asset packing + +::: nnnotes.packing + +::: nnnotes.unity.pack + +::: nnnotes.unity.bundlefs + +::: nnnotes.source.catalogbuild + ## Music data ::: nnnotes.musicdata diff --git a/docs/architecture.md b/docs/architecture.md index a257b19..8adec39 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,7 +9,7 @@ subpackage of `nnnotes` and imports only the layers below it. | Layer | Subpackage | What it holds | |---|---|---| | Command line | `nnnotes.cli` | one module per command group; parses options, reads the configuration, prints summaries and `--json` reports | -| Products | `nnnotes.site`, `nnnotes.musicdata` | the ournotes-player site (charts, models, stories, indexes); the music data file and its deck statistics | +| Products | `nnnotes.site`, `nnnotes.musicdata`, `nnnotes.packing` | the ournotes-player site; music data and deck statistics; local song and audio packages | | Pipeline | `nnnotes.pipeline` | stages, task descriptions and results, the content-addressed depot, plans, layouts, the catalog versions, the views of the master data | | Content | `nnnotes.content` | the game's subjects as exports: ADV episodes and stories, lives and scores, spots, rooms, the UI library | | Unity objects | `nnnotes.unity` | bundle reading, object ids, atomic texture/sprite/mesh/glTF/PNG conversions, class converters (`nnnotes.unity.convert`), clip bindings (`nnnotes.unity.bindings`), script type trees (`nnnotes.unity.scripttypes`), player data, text styles, font protocols and texel packing | diff --git a/docs/cli.md b/docs/cli.md index e7de8bd..2582f59 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -17,6 +17,8 @@ to standard error. Exit status 2 is a usage or settings error. | `--master` `DIR` | decoded master data directory ([paths] master) | | `--apk` `FILE` | base.apk ([paths] apk) | | `--ffmpeg` `EXE` | ffmpeg executable ([paths] ffmpeg; else on PATH) | +| `--ffprobe` `EXE` | ffprobe executable ([paths] ffprobe; else on PATH) | +| `--vgaudio` `FILE` | VGAudioCli executable or .NET DLL ([paths] vgaudio; else on PATH) | | `--vgmstream` `EXE` | vgmstream-cli executable ([paths] vgmstream; else on PATH) | | `--node` `EXE` | Node.js executable ([paths] node; else on PATH) | | `--json` | print the command's report as JSON on standard output (default: a short summary) | @@ -126,6 +128,19 @@ keys, bundles, raw files (and objects) that differ between two catalog versions | `old` | label, version id or sha256 prefix | | `new` | label, version id or sha256 prefix | +## nnnotes catalog rebuild + +write a catalog with added locations (`--add`), removed primary keys (`--drop`) and updated bundle request options (`--bundle`) + +| Argument | Description | +| --- | --- | +| `CATALOG` | the catalog to read (catalog__.bin) | +| `-o`, `--out` `OUT` | the catalog to write (required) | +| `--add` `FILE` | JSON location entries or bundle descriptions with file, internal_id, bundle_name and assets; bundle descriptions generate crc, bundleSize and a cache hash from the bundle (repeatable) | +| `--drop` `KEY` | drop every location of this addressable key (repeatable) | +| `--bundle` `FILE` | write this bundle's decompressed content crc, stored bundleSize and deterministic cache hash (MD5 of the UnityFS bytes) into matching existing or added locations (repeatable) | +| `--hash` `HASH` | override the generated cache hash for one --bundle with 32 hexadecimal digits | + ## nnnotes bundle pull fetch the bundle closure of keys and copy the decrypted bundles to a directory (prints the bundle files) @@ -162,6 +177,92 @@ a master data version from the region's CDN (SHA-256 checked) Exactly one of `--version`, `--latest`. +## nnnotes pack plan + +check a song project and its local resource dependencies + +| Argument | Description | +| --- | --- | +| `file` | nnnotes.song-package/1 JSON project | + +## nnnotes pack verify + +verify a generated song package against its local baseline + +| Argument | Description | +| --- | --- | +| `directory` | directory containing package.json | +| `--target` `TARGET` | nnnotes.pack-target/1 JSON target (required) | + +## nnnotes pack audio + +MP3/WAV/FLAC/AAC -> a validated single-cue HCA ACB + +| Argument | Description | +| --- | --- | +| `source` | local source audio file | +| `-o`, `--out` `OUT` | output ACB file (required) | +| `--target` `TARGET` | JSON target with local ACB templates and cipher settings (required) | +| `--template` `TEMPLATE` | target ACB profile (default full) (one of `full`, `preview`) | +| `--cue-name` `CUE_NAME` | name of the output cue and sheet (required) | +| `--report` `REPORT` | validation JSON path (default .json) | +| `--sample-rate` `SAMPLE_RATE` | output sample rate (default preserve input) (one of `44100`, `48000`) | +| `--channels` `CHANNELS` | output channels (default preserve input) (one of `1`, `2`) | +| `--stream` `STREAM` | source audio stream index when the file contains several tracks | +| `--gain-db` `GAIN_DB` | explicit PCM gain in decibels (default 0) | +| `--start-ms` `START_MS` | crop start on the decoded timeline (default 0) | +| `--duration-ms` `DURATION_MS` | crop duration; omitted keeps the remainder | +| `--fade-in-ms` `FADE_IN_MS` | linear fade-in duration (default 0) | +| `--fade-out-ms` `FADE_OUT_MS` | linear fade-out duration (default 0) | +| `--quality` `QUALITY` | HCA encoder quality preset (default high) (one of `highest`, `high`, `middle`, `low`, `lowest`) | + +## nnnotes pack master + +master data tables (.json) -> .bin (Rijndael-256-CBC over gzip JSON) + +| Argument | Description | +| --- | --- | +| `inputs` ... | directories (their *.json files) or files | +| `-o`, `--out` `OUT` | output directory (required) | +| `--prefix-from` `BIN` | copy the 64-byte prefix of this master data file (the loader ignores it; a replacement should keep it) | +| `--manifest` `VERSION` | also write MasterManifest.json (the file `master download` reads) for this master data version | + +## nnnotes pack bundle + +UnityFS bundles -> the form the CDN serves (AES-128-CTR, first 16 KiB) + +| Argument | Description | +| --- | --- | +| `inputs` ... | directories (their bundles) or files | +| `-o`, `--out` `OUT` | output directory (required) | +| `--suffix` `SUFFIX` | the file name suffix bundles have among the inputs (default .bundle) | +| `--name` `NAME` | the file name the bundle is served under (it seeds the nonce; default: the input's name) | + +## nnnotes pack asset + +replace a binary TextAsset in a bundle; --new-name changes its object name + +| Argument | Description | +| --- | --- | +| `bundle` | the bundle to copy | +| `-o`, `--out` `OUT` | output bundle (required) | +| `--name` `NAME` | the TextAsset's name inside the bundle (required) | +| `--data` `DATA` | the file whose bytes become the asset's payload (required) | +| `--new-name` `NEW_NAME` | also rename the object; its container path is preserved | + +## nnnotes pack build + +build a song package or a UnityFS bundle from a JSON manifest + +| Argument | Description | +| --- | --- | +| `file` | song project or asset manifest with template, mode and asset operations | +| `-o`, `--out` `OUT` | new song package directory, or output UnityFS bundle (required) | +| `--encrypt` `FILE` | also write the encrypted bundle to this file | +| `--name` `NAME` | served file name used for encryption and catalog validation | +| `--catalog-add` `JSON` | also write a manifest accepted by catalog rebuild --add | +| `--catalog-out` `FILE` | rebuild the --catalog source with the bundle's locations | + ## nnnotes export assets every object of the selected bundles -> common formats (the depot and its layouts) diff --git a/docs/commands.md b/docs/commands.md index 6969eef..ff49cf6 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -106,6 +106,20 @@ Serves a local web page on `HOST:PORT` (default `127.0.0.1:8000`) that lists eve languages, and per catalog the asset paths as directories. A link under an asset path downloads the bundle holding it, decrypted; `bundles/` lists every bundle directly. +## catalog rebuild + +``` +nnnotes catalog rebuild CATALOG -o OUT [--add FILE ...] [--drop KEY ...] + [--bundle FILE ...] [--hash HEX] +``` + +Rebuilds an Addressables binary catalog, preserving headers, typed keys and location dependencies. `--add` accepts +location objects or bundle manifests (`file`, `internal_id`, `bundle_name`, `assets`), computing content CRC, size +and a deterministic cache hash for each plain UnityFS file. Asset locations depend on bundles by key. `--drop` +removes locations and supports replacements under the same key. `--bundle` refreshes an existing bundle entry +with its decompressed-content CRC, size and MD5 cache version; `--hash` overrides one bundle cache hash. +See [packing.md](packing.md) for the input contract. Reports `{in, out, added, dropped, updated, addedBundles, bytes}`. + ## bundle pull ``` @@ -163,6 +177,57 @@ against the manifest's SHA-256. Files already present with the right hash are ke the exit status is 1 when a file failed or, with `--latest`, the version could not be fetched. +## pack master + +``` +nnnotes pack master INPUT [INPUT ...] -o OUT [--prefix-from BIN] [--manifest VERSION] +``` + +The inverse of `master decode`: each `.json` file, or the `*.json` files of a directory, is written as +`OUT/.bin` — gzip-compressed, then Rijndael-256 CBC with PKCS7 padding, behind a 64-byte prefix. The key +and IV are `[master] key` and `[master] iv`, so a client configured with the same pair reads the result like any +other master data file. `--prefix-from` copies the prefix of an existing master data file: the loader ignores it, so a +file that replaces another should carry that file's prefix; without it the prefix is 64 zero bytes. `--manifest` +also writes `OUT/MasterManifest.json` for that master data version — the file `master download` reads, listing each +packed file's name, SHA-256 and size — which is what a CDN tree of our own needs. The report is +`{out, packed, files, manifest}`; the exit status is 1 when a file could not be packed. + +## pack bundle + +``` +nnnotes pack bundle INPUT [INPUT ...] -o OUT [--suffix .bundle] [--name NAME] +``` + +The inverse of the decryption `bundle pull` applies: each UnityFS bundle is written as the CDN serves it — AES-128 in +CTR mode over the first 16 KiB, the nonce being the first 8 bytes of SHA-256(`[bundle] nonce_seed` + the file name). +`--name` is the file name the bundle is served under, which seeds the nonce; it applies to a single input (default: +the input's own name). `--suffix` (default `.bundle`) is the suffix bundles have among the inputs of a directory. The +report is `{out, packed, files}`; the exit status is 1 when a file is not a UnityFS bundle or could not be packed. + +## pack asset + +``` +nnnotes pack asset BUNDLE -o OUT --name NAME --data FILE [--new-name NAME] +``` + +Replaces a binary TextAsset while preserving its path ID and container path, then validates and saves the +UnityFS bundle with its original flags. `--new-name` changes the object name. Reports +`{in, out, asset, payloadBytes, bundleBytes, crc}`. + +## pack build + +``` +nnnotes pack build MANIFEST -o PLAIN.bundle [--encrypt ENCRYPTED.bundle] [--name NAME] + [--catalog SOURCE --catalog-out CATALOG] [--catalog-add JSON] +``` + +Builds or edits a UnityFS bundle from a JSON manifest. New bundles use template schemas and flags with supplied +TextAsset, PNG texture, PCM WAV audio, serialized or raw object inputs. Operations add, replace and rename objects; +container paths, unique path IDs and preload references are managed by the builder. Optional outputs include an +encrypted bundle, a rebuilt catalog and a reusable catalog-add manifest. All outputs are prepared before writing. +See [packing.md](packing.md) for examples, schema selection, input limits and encryption. Reports +`{out, bundle_name, bundle_provider, assets, bundleBytes, crc, hash, encrypted, catalog, catalog_add}`. + ## export episode ``` @@ -542,3 +607,8 @@ unplayable, full, bgm, jackets, bytes, fileBytes, sha256}`. A missing or unreada column, a text id, a chart asset, a cue sheet or cue, a jacket), a chart the deck model cannot measure, or deck statistics that disagree with the chart facts stop the command with exit status 1 before the file is written. The format is described in [music-data.md](music-data.md). + +## Song and audio packages + +`pack plan`, `pack build`, `pack verify` and `pack audio` work with local song projects and audio. +See [Package a song](song-packing.md) for input manifests, tool requirements, validation and output layouts. diff --git a/docs/configuration.md b/docs/configuration.md index a5419a3..811d12b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -105,6 +105,8 @@ exist when they are given. | `[paths] master` | `NNNOTES_PATHS_MASTER` | `--master` | decoded master data directory, one `
.json` per table; the flag overrides `[servers.] master` | | `[paths] apk` | `NNNOTES_PATHS_APK` | `--apk` | the game's `base.apk` (the story export reads the CRI Lips library from it, or from `split_config.arm64_v8a.apk` next to it; the asset export its built-in Unity files) | | `[paths] ffmpeg` | `NNNOTES_PATHS_FFMPEG` | `--ffmpeg` | `ffmpeg` executable; unset: `ffmpeg` on `PATH` | +| `[paths] ffprobe` | `NNNOTES_PATHS_FFPROBE` | `--ffprobe` | `ffprobe` executable for audio input inspection | +| `[paths] vgaudio` | `NNNOTES_PATHS_VGAUDIO` | `--vgaudio` | VGAudioCli executable or .NET DLL for HCA encoding; unset: `VGAudioCli` on `PATH` | | `[paths] vgmstream` | `NNNOTES_PATHS_VGMSTREAM` | `--vgmstream` | `vgmstream-cli` executable; unset: `vgmstream-cli` on `PATH` | | `[paths] node` | `NNNOTES_PATHS_NODE` | `--node` | Node.js executable; unset: `node` on `PATH` | | `[paths] player` | `NNNOTES_PATHS_PLAYER` | `--player` (of `site build`) | ournotes-player checkout (built) or installed package | diff --git a/docs/contracts.md b/docs/contracts.md index 9c47051..f5382bb 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -357,3 +357,14 @@ The fixed product documents have the following schemas, also available through ` - [story-video](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/story-video.schema.json) - [story-sound](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/story-sound.schema.json) - [crilips](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/crilips.schema.json) + +## Packing document schemas + +Song projects, local targets and package reports use versioned schemas: + +- [song-package](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/song-package.schema.json) +- [pack-target](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/pack-target.schema.json) +- [package-plan](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/package-plan.schema.json) +- [audio-build](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/audio-build.schema.json) +- [song-build](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/song-build.schema.json) +- [package-validation](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/package-validation.schema.json) diff --git a/docs/index.md b/docs/index.md index 73cdad5..1ca2023 100644 --- a/docs/index.md +++ b/docs/index.md @@ -32,7 +32,7 @@ persistent depot. You supply the client files, keys and server configuration. ## Requirements and versions -nnnotes 0.3.0 requires Python 3.11+. Optional extras provide fonts (`fonts`), S3 clients (`remote`) and +nnnotes 0.3.1 requires Python 3.11+. Optional extras provide fonts (`fonts`), S3 clients (`remote`) and ournotes-sim 0.0.12 chart statistics (`deck`). Site builds require Node.js 22+ and a built ournotes-player 0.2.x package. Audio and video conversion require vgmstream and FFmpeg. diff --git a/docs/packing.md b/docs/packing.md new file mode 100644 index 0000000..7e8f0f6 --- /dev/null +++ b/docs/packing.md @@ -0,0 +1,154 @@ +# Build and edit asset bundles + +For complete song projects and common audio to ACB conversion, see [Package a song](song-packing.md). + +`nnnotes pack build` constructs UnityFS bundles from assets you supply. A template provides the Unity version, +serialization schemas and container flags. The default `new` mode starts with an empty object and resource table; +`edit` mode keeps existing objects and applies selected changes. Both save with `packer="original"` and check the +saved header, object table and container references. + +## Build a bundle and catalog + +Save this as `build.json`, with paths relative to that file: + +```json +{ + "template": "templates/text.bundle", + "mode": "new", + "bundle_name": "example_assets_v1", + "internal_id": "https://example.com/asset/Android/example_assets_v1.bundle", + "assets": [ + { + "type": "TextAsset", + "name": "example", + "file": "assets/example.bytes", + "container": "Assets/Examples/example.bytes", + "primary_key": "Examples/Text" + } + ] +} +``` + +```sh +nnnotes pack build build.json -o build/example_assets_v1.bundle \ + --encrypt served/example_assets_v1.bundle \ + --catalog catalog.bin --catalog-out build/catalog.bin \ + --catalog-add build/catalog-add.json +``` + +The command builds and validates the outputs before writing them. It writes the plain UnityFS file, an encrypted +file when `--encrypt` is supplied, and an updated catalog when `--catalog-out` is supplied. The plain file is kept +for inspection and further catalog generation. The optional sidecar can be consumed independently: + +```sh +nnnotes catalog rebuild catalog.bin --add build/catalog-add.json -o build/catalog.bin +``` + +Give each independently loaded new bundle a distinct `bundle_name`. New serialized file identities are derived +deterministically from that name. `cab_name` can specify the serialized file identity explicitly. An asset's +`name` is its object name, `container` is the path used by Unity to load it, and `primary_key` is the Addressables +lookup key. The catalog's asset internal ID must match the container path. A bundle's internal ID ends with the +served file name; that same name seeds encryption. `--name` specifies it explicitly when output names differ. + +In edit mode, changing `bundle_name` also gives a single serialized file an independent identity while preserving +objects and managed-reference schemas. Internal archive resource paths are remapped. An explicit `cab_name` +also applies in edit mode. + +## Asset inputs + +| Type | Input | Result | +| --- | --- | --- | +| `TextAsset` | Any file | Binary payload, preserved without text decoding | +| `Texture2D` | PNG | RGBA32 pixels, one image and one mip level | +| `AudioClip` | Mono or stereo 16-bit integer PCM WAV | Single-sample PCM FSB5 bank and AudioClip resource reference | +| `serialized` | Complete `fields` object or JSON file | Object serialized with a selected or explicit type tree | +| `raw` | Serialized object bytes | Bytes supplied by the caller, with `managed_references: true` | + +Built-in types need a compatible schema in the bundle template. An asset can use another template with +`"template": {"file": "templates/texture.bundle", "type": "Texture2D", "name": "texture"}`. Its Unity version, +serialized format, endianness and target platform must match the bundle. Select a unique object by `type`, `name` +or `path_id`. A bundle containing multiple serialized files requires `serialized_file` in the build manifest. + +For custom serialized objects, supply a template selector or `schema` with `class_id`, `type_hash`, `nodes` and, +when needed, `script_id`, `script_type_index` and type dependencies. `typetree` overrides the selected object's +type tree. Complete fields must include the object's required pointers and metadata. Local pointer targets are +checked and included in preload ranges. Explicit external references use a top-level `externals` array of +`{path, guid?, type?}` objects, with a 32-digit zero GUID and type `0` as defaults. Their order gives one-based +PPtr file IDs. Top-level `dependencies` lists internal bundle names for Unity; `dependency_keys` lists catalog +keys for Addressables. Supply the corresponding external bundles separately. The client must support the +serialized class and script schema. New managed-reference registry schemas are currently unsupported. + +An operation that fails invalidates the builder; start a new builder before retrying. Output files are staged +before replacement. If a filesystem error also prevents restoration, recoverable backups are retained and their +paths are included in the error. + +Raw inputs require explicit reference ownership because arbitrary serialized bytes do not describe their +dependency graph. `AudioClip` output is a Unity audio resource; a client that requests a separate cue-sheet or +sound-bank class needs an asset of that class and its dependencies. + +When a source catalog is supplied, generated bundle locations use its unique bundle provider. This preserves a +custom provider's download and decoding behavior. A catalog with multiple bundle providers requires either +`"bundle_provider": "ExampleEncryptedBundleProvider"` or `"catalog_template": "source.bundle"` in the manifest. +The template selects an existing bundle by primary key or full internal ID and also copies its timeout, redirect +limit, retry count and flags. Explicit `bundle_provider` and `request_options` fields take precedence. The new +bundle's CRC, size, cache hash and internal name are computed separately. Without a source catalog, the default is +Unity's standard AssetBundleProvider; use an explicit provider when the runtime loads encrypted bundles through +a custom provider. The runtime must already register that provider. + +These fields also apply to `catalog rebuild --add` bundle descriptions. A sidecar written by `pack build` records +the chosen provider and request options, so importing it retains that selection. Asset-level `provider` controls +the bundled asset loader independently. + +## Edit existing objects + +Use `"mode": "edit"` and describe operations in order: + +```json +{ + "template": "input.bundle", + "mode": "edit", + "assets": [ + {"op": "replace", "type": "TextAsset", "name": "example", "file": "changed.bytes"}, + {"op": "rename", "name": "example", "new_name": "updated", "new_container": "Assets/Examples/updated.bytes"} + ] +} +``` + +`op` defaults to `add`. New objects receive unique positive path IDs, or an explicit `path_id`. Replace and rename +operations select an existing object with `target`, or their `name`/`path_id` fields. Object renaming preserves the +container path unless `new_container` is supplied. Use `primary_key`, `aliases`, `provider`, `resource_type`, +`resource_type_assembly` and `dependency_keys` to describe catalog locations when needed. + +With a source catalog, `edit` updates the matching bundle location and only the asset locations affected by the +manifest. Asset locations match the source container path and the bundle dependency; existing keys, aliases, +types, providers and dependencies are retained. A container rename updates those locations' internal IDs. +Explicit catalog fields override the retained values, including empty `aliases` or `dependency_keys` arrays. +Other locations sharing a lookup key remain available. Added assets receive new locations. The edited bundle's +sidecar records these changes and applies the same incremental update through `catalog rebuild --add`. + +The compact TextAsset replacement command is also available: + +```sh +nnnotes pack asset input.bundle -o build/input.bundle --name example --data changed.bytes +nnnotes catalog rebuild catalog.bin -o updated-catalog.bin --bundle build/input.bundle +``` + +The replacement bundle's file name must identify its existing catalog location. The second command updates its +content CRC, size and cache version. `--hash HEX` supplies a particular 32-digit cache hash. + +## Catalog and encryption fields + +Catalog request options use the CRC32 of concatenated decompressed UnityFS blocks, excluding the header and +block/directory information. `bundleSize` is the served byte length, which encryption preserves. The default +cache hash is MD5 of the plain UnityFS bytes: a deterministic Hash128 cache version. It can be overridden; it is +independent of Unity's build pipeline hash. `bundleName` identifies the bundle in the cache. + +`catalog rebuild --add` also accepts bundle descriptions with `file`, `internal_id`, `bundle_name` and `assets`. +It computes CRC, size and cache hash from the plain file. Low-level location descriptions remain available for +custom providers, with dependencies named by key. Existing catalog headers, typed keys and dependencies survive +rebuilding. Missing dependencies, invalid fields and unsupported extra-data types stop the command. + +Encryption uses `[bundle] key` and `[bundle] nonce_seed` from the local configuration. AES-128-CTR transforms the +first 16 KiB; the remaining bytes are preserved. The nonce is the first eight bytes of SHA-256(seed followed by +the UTF-8 served file name), followed by a 64-bit big-endian counter starting at zero. To encrypt existing plain +bundles independently, use `nnnotes pack bundle INPUT -o DIR [--name NAME]`. diff --git a/docs/quickstart.md b/docs/quickstart.md index 2c80669..c77608d 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -6,7 +6,7 @@ files, keys and server addresses; [configuration](configuration.md#where-the-val ## 1. Install the tools ```sh -pip install 'nnnotes[fonts]==0.3.0' +pip install 'nnnotes[fonts]==0.3.1' ``` Use Python 3.11+. Install [vgmstream](https://vgmstream.org/) (`vgmstream-cli`) and diff --git a/docs/schemas.md b/docs/schemas.md index 003c67e..9edc12c 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -16,6 +16,7 @@ schemas.load("plan") # the schema as a dict |---|---|---| | `addresses` | `nnnotes.addresses/3` | the address table of a catalog version and the censuses of its bundles | | `artifact` | `nnnotes.artifact/1` | the content, provenance and semantics of one output file | +| `audio-build` | `nnnotes.audio-build/1` | audio conversion report | | `catalog-diff` | `nnnotes.catalog-diff/1` | the difference between two catalog versions (`catalog diff`) | | `catalog-index` | `nnnotes.catalog-index/1` | the index of a catalog pair: remote and APK catalogs, every location in catalog order | | `catalogs` | `nnnotes.catalogs/1` | the catalog versions of a depot (`catalog versions`) | @@ -38,6 +39,9 @@ schemas.load("plan") # the schema as a dict | `model-json` | `3` | Live2D model objects and animation bindings | | `music-data` | `nnnotes.music-data/2` | every song and chart of one master data version ([music data](music-data.md)) | | `objects` | `nnnotes.objects/1` | the objects a product reaches: their class, item status and files ([products](products.md)) | +| `pack-target` | `nnnotes.pack-target/1` | local packing target input | +| `package-plan` | `nnnotes.package-plan/1` | song package plan | +| `package-validation` | `nnnotes.package-validation/1` | automatic package verification | | `plan` | `nnnotes.plan/1` | what a run would do and why (`depot plan`) | | `prefab` | product document | prefab nodes, transforms and components | | `product` | `nnnotes.product/1` | a product's manifest: its files and the files of the objects it uses, as content-addressed assets ([products](products.md)) | @@ -46,6 +50,8 @@ schemas.load("plan") # the schema as a dict | `run` | `nnnotes.run/1` | a run manifest: context, selection, tasks with keys and statuses, layouts | | `scripts` | `nnnotes.scripts/1` | every MonoScript of the censuses, by object id | | `site` | `nnnotes.site/1` | a named site's complete region-topic catalog and immutable product references | +| `song-build` | `nnnotes.song-build/1` | incremental song package manifest | +| `song-package` | `nnnotes.song-package/1` | song project input | | `spot` | product document | spot settings, characters and Spine data | | `story` | `1` | story entry and shared parts | | `story-asset` | `1` | story media prefab objects | diff --git a/docs/song-packing.md b/docs/song-packing.md new file mode 100644 index 0000000..f5cb227 --- /dev/null +++ b/docs/song-packing.md @@ -0,0 +1,190 @@ +# Package a song + +`nnnotes pack` builds a song from local chart JSON, a PNG jacket and common audio files. It creates independent +Unity bundles, an updated Addressables catalog and encrypted master tables. The output is an incremental package +for a specific local resource baseline. + +## Audio tools + +Install [FFmpeg](https://ffmpeg.org/download.html), [VGAudioCli](https://github.com/Thealexbarney/VGAudio) and +[vgmstream](https://vgmstream.org/). FFmpeg decodes audio, VGAudioCli encodes HCA and vgmstream independently +decodes the completed ACB. Configure their locations: + +```sh +nnnotes config set paths.ffmpeg /tools/ffmpeg +nnnotes config set paths.ffprobe /tools/ffprobe +nnnotes config set paths.vgaudio /tools/VGAudioCli +nnnotes config set paths.vgmstream /tools/vgmstream-cli +``` + +Programs on `PATH` are used when these settings are empty. `paths.vgaudio` can also name a VGAudioCli `.dll`, +which requires `dotnet` on `PATH`. Tool binaries are external to the Python distribution. Builds use their content +fingerprints and finish without downloading tools or assets. + +## Local target + +Create `target.json`; paths are relative to this file: + +```json +{ + "format": "nnnotes.pack-target/1", + "catalog": "baseline/catalog.bin", + "master": "baseline/master", + "bundles": "baseline/bundles", + "bundle_base_url": "https://example.com/asset/Android", + "audio": { + "cipher_type": 56, + "templates": {"full": "templates/full.acb", "preview": "templates/preview.acb"} + } +} +``` + +The master directory contains encoded `Master*.bin` files and complete `MasterManifest.json`. The bundle +directory contains the local files named by the catalog. The template song supplies chart, jacket and sound +schemas. Full sound supports split ACB data; preview supports embedded ACB managed references or split data. +Jacket templates contain a Sprite and its local Texture2D; both normal and small jacket keys are required. +Single serialized-file bundles and finite, single-cue, single-waveform ACB templates are supported. + +`cipher_type` is `0` for unencrypted HCA or `56` for encryption with the configured APK's HCA key. Cipher 56 uses +`paths.apk`; key material stays outside project files and reports. ACB templates specify the compatible cue graph +and routing. Select templates whose playback settings match the target application. + +Bundle transport follows its catalog provider. Unity's standard AssetBundleProvider uses plain UnityFS. +Other providers need an explicit mapping in `target.json`, for example: + +```json +{"providers": {"ExampleEncryptedBundleProvider": "encrypted"}} +``` + +Values are `plain` or `encrypted`. The runtime must implement the provider. HCA encryption and Bundle transport +are independent settings. + +## Convert audio to ACB + +An audio-only target needs `format` and `audio`; it can omit catalog, master and bundle paths. + +```sh +nnnotes pack audio media/song.mp3 --target target.json --template full \ + --cue-name example_full -o audio/full.acb +nnnotes pack audio media/song.flac --target target.json --template preview \ + --cue-name example_short --start-ms 45000 --duration-ms 20000 \ + --fade-in-ms 20 --fade-out-ms 150 -o audio/preview.acb +``` + +MP3, WAV, FLAC and AAC, including `.aac` and `.m4a`, use the same decoded PCM processing path. Mono/stereo at +44,100 or 48,000 Hz is supported. Other inputs require explicit `--sample-rate` and/or `--channels` conversion. +`--stream` selects an audio stream by its absolute index when several exist. + +Gain defaults to 0 dB; `--gain-db` applies a fixed gain once. Start/end silence is preserved. Cropping and fades +operate on decoded samples before HCA encoding. Milliseconds map to samples by rounding half upward. Invalid +crop/fade ranges are rejected. Sample conversion rounds to PCM16 without dither; values outside the PCM range +stop conversion. Encoding quality defaults to `high`. The output is finite and contains its AWB in the ACB. + +The command writes `.json` with input properties, processing settings, tool fingerprints and validation. +Checks cover HCA CRCs, encryption round trips, ACB references, sample counts and independently decoded PCM. +Active-channel normalized mean-square error must be at most 0.05 and power change at most 1 dB; silent-channel +error is checked separately. HCA is lossy; decoded PCM is compared to the processed source. + +## Song project + +Create `song.json`: + +```json +{ + "format": "nnnotes.song-package/1", + "target": "target.json", + "template_song": 1001, + "song": { + "id": 900001, + "title": "Example Song", + "performer": "Example Artist", + "credits": {"composer": "Example Composer", "lyricist": "Example Writer"}, + "tags": ["Example"], + "bands": [] + }, + "assets": { + "jacket": "media/jacket.png", + "audio": { + "full": {"file": "media/song.flac"}, + "processing": {"gain_db": 0, "sample_rate": 48000}, + "preview": {"from": "full", "start_ms": 45000, "duration_ms": 20000, + "fade_in_ms": 20, "fade_out_ms": 150} + } + }, + "charts": { + "easy": {"file": "charts/easy.json", "score_id": 90000100, "level": 5}, + "normal": {"file": "charts/normal.json", "score_id": 90000101, "level": 10}, + "hard": {"file": "charts/hard.json", "score_id": 90000102, "level": 15}, + "expert": {"file": "charts/expert.json", "score_id": 90000103, "level": 20} + } +} +``` + +Paths resolve against `song.json`. Charts use client JSON or gzip JSON. Notes and events are preserved, binary +TextAssets are generated, and combo counts come from the existing score converter. Four difficulties are +required; they may explicitly share a file. `display_level` sets a displayed numeric level. `input_format` +can require `json` or `gzip-json`. + +The jacket must be a square PNG. Both sizes use the template's dimensions; each Sprite references its own RGBA +texture. `preview.file` accepts a separate common audio file instead of `preview.from`. Audio fields use +`AudioOptions` names: `sample_rate`, `channels`, `stream`, `gain_db`, `start_ms`, `duration_ms`, `fade_in_ms`, +`fade_out_ms`, `quality`. Shared `processing` values apply to both inputs; input values override shared ones. +Both outputs are encoded directly from decoded source PCM. + +Text can be a string for all languages or a mapping with `ja`, `en`, `zh-Hant`, `zh-Hans`, `ko`. Missing +translations are empty unless `song.fallback_language` names a provided translation. Optional `phonetic`, +`ruby_title`, `performer` and credits use the same text form. Unspecified credits and performer are empty. +Tags receive independent rows. Band IDs reference baseline bands; omitted bands and tags are empty. + +Song/score IDs must be unused. Derived tag, rank, sound and cue-sheet IDs are checked for collisions. Songs are +unlocked by default. Presentation, reward and stage behavior use template defaults. `sort_order` and `start_at` +override their defaults. `song.fields` can override existing non-identity behavior fields. Resource IDs and +display-text references are assigned by the packager. Vocal, video and alternate-vocal references start empty. +The `song-package` and `pack-target` JSON Schemas ship in `nnnotes.schemas`. + +## Plan, build and verify + +```sh +nnnotes pack plan song.json --json +nnnotes pack build song.json -o build/example +nnnotes pack verify build/example --target target.json --json +``` + +Planning reports templates/providers, new resource keys, master row IDs and diagnostics. It reads local inputs +and probes tools without writing resource outputs. Missing inputs, conflicting IDs/keys and unsupported schemas +stop building. A Python `PackagePlan` also rejects changed inputs or tool binaries. + +Build outputs use a new directory. Required validations finish in staging before publication. Existing output +directories are rejected. The result contains: + +| Path | Contents | +| --- | --- | +| `package.json` | Package format, file sizes/SHA-256, Bundle metadata and baseline catalog identity | +| `bundles/` | Eight independent chart, jacket and sound bundles | +| `audio/` | Full/preview ACBs and conversion reports | +| `catalog/catalog.bin`, `catalog/catalog.hash` | Updated catalog and its matching marker | +| `master/` | Changed encoded tables and complete `MasterManifest.json` | +| `validation.json` | Automatic structure and audio verification | + +Unchanged master files and existing Bundle dependencies come from the baseline. The output is applied as an +overlay. New Bundle URLs, catalog and master version must be available through the application's resource update +mechanism. Copying a single ACB or Bundle does not register a song. + +Build does not upload resources or operate an application. Client playback compatibility is checked for supported +target profiles separately from automatic per-package validation. Repeated builds with the same inputs, target +and tool binaries produce identical files; HCA encoding across tool versions can produce different bytes. + +## Python API + +```python +from nnnotes.packing import SongProject, build_package, plan_package, verify_package + +project = SongProject.load("song.json") +plan = plan_package(project, config=config) +plan.raise_for_errors() +result = build_package(plan, out="build/example") +verification = verify_package(result.root, target="target.json", config=config) +``` + +`build_audio` is the standalone audio service. `AudioOptions` controls processing and `AudioTools` accepts explicit +tool paths. Keys are separate arguments with no representation in reports. diff --git a/mkdocs.yml b/mkdocs.yml index a388376..664935d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -35,6 +35,8 @@ nav: - Export stories and lives: export.md - Build an ournotes-player site: site.md - Export every asset: assets.md + - Build asset bundles: packing.md + - Package a song: song-packing.md - Japanese release: jp.md - Music data: music-data.md - Reference: diff --git a/pyproject.toml b/pyproject.toml index 8f9e997..61eea85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,8 @@ dependencies = [ "pycryptodome>=3.20", "grpcio>=1.49.1", "brotli>=1.1", + "lz4>=4.0", + "jsonschema>=4.18", ] classifiers = [ "Programming Language :: Python :: 3", @@ -26,7 +28,7 @@ classifiers = [ dynamic = ["version"] [project.optional-dependencies] -test = ["pytest>=8", "ruff>=0.15", "jsonschema>=4.18", "import-linter>=2.1", "pyright>=1.1.414"] +test = ["pytest>=8", "ruff>=0.15", "import-linter>=2.1", "pyright>=1.1.414"] fonts = ["fonttools>=4.50", "freetype-py>=2.5", "scipy>=1.13"] docs = ["mkdocs>=1.6,<2", "mkdocs-material>=9.5", "mkdocstrings[python]>=0.29"] # a depot remote in S3-compatible object storage ([depot] remote = "s3://...") @@ -87,7 +89,7 @@ name = "Layers: each subpackage imports only the layers below it" type = "layers" layers = [ "cli", - "site | musicdata", + "site | musicdata | packing", "pipeline", "content", "unity", diff --git a/src/nnnotes/__init__.py b/src/nnnotes/__init__.py index c111a90..d1ac20d 100644 --- a/src/nnnotes/__init__.py +++ b/src/nnnotes/__init__.py @@ -19,7 +19,7 @@ import importlib from typing import TYPE_CHECKING -__version__ = "0.3.0" +__version__ = "0.3.1" _LAZY = { "Catalog": "nnnotes.source.catalog", diff --git a/src/nnnotes/cli/__init__.py b/src/nnnotes/cli/__init__.py index 57f058d..b5b63d8 100644 --- a/src/nnnotes/cli/__init__.py +++ b/src/nnnotes/cli/__init__.py @@ -8,9 +8,10 @@ nnnotes config init|set|unset|check|path nnnotes servers [--show-hosts] - nnnotes catalog keys|browse|versions|import|fetch|diff + nnnotes catalog keys|browse|versions|import|fetch|diff|rebuild nnnotes bundle pull KEY... nnnotes master version|download|decode + nnnotes pack plan|build|verify|audio|master|bundle|asset ... nnnotes export assets|episode|story|live|live2d|spot|room|shader|audio|ui ... -o OUT nnnotes apk settings -o FILE | hca-key [--write DIR] nnnotes depot plan|run|verify|gc @@ -37,11 +38,11 @@ from .. import __version__ from ..source.config import ConfigError from ..source.gameapi import GameApiError -from . import apk, bundle, catalog, config, depot, export, master, musicdata, servers, site, voices +from . import apk, bundle, catalog, config, depot, export, master, musicdata, pack, servers, site, voices from .common import GLOBALS, Formatter, fill_globals, load_config -GROUPS = (config, servers, catalog, bundle, master, export, apk, depot, voices, site, musicdata) -NO_DEPOT = {"config", "servers", "master", "apk"} # the groups whose commands never read the depot +GROUPS = (config, servers, catalog, bundle, master, pack, export, apk, depot, voices, site, musicdata) +NO_DEPOT = {"config", "servers", "master", "pack", "apk"} # the groups whose commands never read the depot def build_parser() -> argparse.ArgumentParser: diff --git a/src/nnnotes/cli/catalog.py b/src/nnnotes/cli/catalog.py index d49fd60..b3de727 100644 --- a/src/nnnotes/cli/catalog.py +++ b/src/nnnotes/cli/catalog.py @@ -211,6 +211,139 @@ def cmd_diff(args, cfg): write(catalogdb.format_diff(d)) +def _catalog_additions(path: Path, catalog: bytes) -> tuple[list[dict], dict[str, dict], dict[int, dict]]: + """Read location entries or bundle descriptions; bundle file paths are relative to the JSON file.""" + import hashlib + import json + + from ..source import catalogbuild + from ..unity import bundlefs + + doc = json.loads(path.read_text(encoding="utf-8")) + items = doc.get("bundles") if isinstance(doc, dict) and "bundles" in doc else doc + if not isinstance(items, list): + items = [items] + add: list[dict] = [] + bundles: dict[str, dict] = {} + replacements: dict[int, dict] = {} + for item in items: + if not isinstance(item, dict): + raise ValueError("catalog additions must be JSON objects") + if "file" not in item: + add.append(item) + continue + if not all(item.get(field) for field in ("file", "internal_id", "bundle_name")): + raise ValueError("bundle descriptions require file, internal_id and bundle_name") + bundle = Path(item["file"]) + if not bundle.is_absolute(): + bundle = path.parent / bundle + packed = bundle.read_bytes() + fields = { + "crc": bundlefs.content_crc(packed), + "bundleSize": len(packed), + "hash": item.get("hash") or hashlib.md5(packed).hexdigest(), + "bundleName": item["bundle_name"], + } + options: dict = dict( + hash=fields["hash"], + crc=fields["crc"], + bundle_size=fields["bundleSize"], + primary_key=item.get("primary_key"), + request_options=item.get("request_options"), + dependency_keys=item.get("dependency_keys"), + provider=item.get("bundle_provider"), + template=item.get("catalog_template"), + ) + if item.get("mode") == "edit": + entries, changed = catalogbuild.edit_entries( + catalog, + item["internal_id"], + item["bundle_name"], + item.get("assets") or [], + item.get("catalog_edits") or [], + **options, + ) + add.extend(entries) + for offset, changes in changed.items(): + replacements[offset] = {**replacements.get(offset, {}), **changes} + else: + options["dependency_keys"] = options["dependency_keys"] or [] + add.extend( + catalogbuild.bundle_entries( + item["internal_id"], + item["bundle_name"], + item.get("assets") or [], + catalog=catalog, + **options, + ) + ) + bundles[item["internal_id"]] = fields + return add, bundles, replacements + + +def cmd_rebuild(args, cfg): + """Write a catalog with added locations, removed primary keys and updated bundle request options.""" + import hashlib + + from ..source import addressables, catalogbuild + from ..unity import bundlefs + + try: + data = Path(args.source).read_bytes() + add: list[dict] = [] + added_bundles: dict[str, dict] = {} + replacements: dict[int, dict] = {} + for path in args.add or []: + entries, bundles, changed = _catalog_additions(Path(path), data) + add.extend(entries) + added_bundles.update(bundles) + for offset, changes in changed.items(): + replacements[offset] = {**replacements.get(offset, {}), **changes} + except (OSError, ValueError, TypeError) as e: + usage(args, str(e)) + update: dict[str, dict] = {} + for path in args.bundle or []: + try: + packed = Path(path).read_bytes() + crc = bundlefs.content_crc(packed) + except (OSError, ValueError) as e: + usage(args, f"{path}: {e}") + fields = {"crc": crc, "bundleSize": len(packed), "hash": args.hash or hashlib.md5(packed).hexdigest()} + update[Path(path).name] = fields + if args.hash and len(update) != 1: + usage(args, "--hash sets the cache hash of one bundle, so it needs exactly one --bundle") + try: + out = catalogbuild.rebuild(data, add=add, drop=set(args.drop or []), update=update, replace=replacements) + except (ValueError, TypeError) as e: + usage(args, str(e)) + for name in update: + written = [ + loc["extra_data"] + for loc in addressables.parse_locations(out) + if loc["internal_id"].rsplit("/", 1)[-1] == name and loc["extra_data"] + ] + if not written: + usage(args, f"no location of this catalog names the bundle {name}") + Path(args.out).parent.mkdir(parents=True, exist_ok=True) + Path(args.out).write_bytes(out) + report( + args, + { + "in": args.source, + "out": args.out, + "added": len(add), + "dropped": len(args.drop or []), + "updated": update, + "addedBundles": added_bundles, + "bytes": len(out), + }, + lambda r: [ + f"{r['out']} ({r['bytes']} bytes, {r['added']} added, {r['dropped']} dropped keys, " + f"{len(r['updated'])} bundles updated)" + ], + ) + + def register(sub) -> None: cs = group(sub, "catalog", "the region's catalog: its keys, a local browser, and the catalog versions of the depot") c = command(cs, "keys", cmd_keys, "the addressable keys of the catalog (the total on standard error)") @@ -240,3 +373,31 @@ def register(sub) -> None: ) c.add_argument("old", help="label, version id or sha256 prefix") c.add_argument("new", help="label, version id or sha256 prefix") + c = command( + cs, + "rebuild", + cmd_rebuild, + "write a catalog with added locations (`--add`), removed primary keys (`--drop`) and updated bundle " + "request options (`--bundle`)", + ) + c.add_argument("source", metavar="CATALOG", help="the catalog to read (catalog__.bin)") + c.add_argument("-o", "--out", required=True, help="the catalog to write") + c.add_argument( + "--add", + action="append", + metavar="FILE", + help="JSON location entries or bundle descriptions with file, internal_id, bundle_name and assets; " + "bundle descriptions generate crc, bundleSize and a cache hash from the bundle", + ) + c.add_argument("--drop", action="append", metavar="KEY", help="drop every location of this addressable key") + c.add_argument( + "--bundle", + action="append", + metavar="FILE", + help="write this bundle's decompressed content crc, stored bundleSize and deterministic cache hash " + "(MD5 of the UnityFS bytes) into matching existing or added locations", + ) + c.add_argument( + "--hash", + help="override the generated cache hash for one --bundle with 32 hexadecimal digits", + ) diff --git a/src/nnnotes/cli/common.py b/src/nnnotes/cli/common.py index fe14e73..aa46643 100644 --- a/src/nnnotes/cli/common.py +++ b/src/nnnotes/cli/common.py @@ -26,6 +26,8 @@ ("paths", "master"): ("master", "--master"), ("paths", "apk"): ("apk", "--apk"), ("paths", "ffmpeg"): ("ffmpeg", "--ffmpeg"), + ("paths", "ffprobe"): ("ffprobe", "--ffprobe"), + ("paths", "vgaudio"): ("vgaudio", "--vgaudio"), ("paths", "vgmstream"): ("vgmstream", "--vgmstream"), ("paths", "node"): ("node", "--node"), ("paths", "depot"): ("depot", "--depot"), @@ -51,6 +53,8 @@ "master": ("--master", "DIR", "decoded master data directory ([paths] master)"), "apk": ("--apk", "FILE", "base.apk ([paths] apk)"), "ffmpeg": ("--ffmpeg", "EXE", "ffmpeg executable ([paths] ffmpeg; else on PATH)"), + "ffprobe": ("--ffprobe", "EXE", "ffprobe executable ([paths] ffprobe; else on PATH)"), + "vgaudio": ("--vgaudio", "FILE", "VGAudioCli executable or .NET DLL ([paths] vgaudio; else on PATH)"), "vgmstream": ("--vgmstream", "EXE", "vgmstream-cli executable ([paths] vgmstream; else on PATH)"), "node": ("--node", "EXE", "Node.js executable ([paths] node; else on PATH)"), } diff --git a/src/nnnotes/cli/pack.py b/src/nnnotes/cli/pack.py new file mode 100644 index 0000000..a8359d4 --- /dev/null +++ b/src/nnnotes/cli/pack.py @@ -0,0 +1,453 @@ +"""`nnnotes pack`: JSON tables and UnityFS bundles -> the encrypted form the game stores. + +The inverse of `master decode` and of the bundle decryption: what a build has to write into a CDN tree of its own. +Both keys come from the configuration (`[master] key`/`[master] iv`, `[bundle] key`/`[bundle] nonce_seed`), so a +packed file is only readable by a client configured with the same pair. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path +from typing import NoReturn + +from ..source.session import Session +from .common import command, group, out_arg, report + + +def _fail(message: str) -> NoReturn: + """An error of this command: `nnnotes: …` on standard error, exit status 1.""" + print(f"nnnotes: {message}", file=sys.stderr) + raise SystemExit(1) + + +def _sha256(path: Path) -> str: + with path.open("rb") as f: + return hashlib.file_digest(f, "sha256").hexdigest() + + +def _inputs(inputs: list[Path], suffix: str) -> list[Path]: + """Files given, and the `*` files of directories given, sorted by name per directory.""" + out: list[Path] = [] + for p in map(Path, inputs): + if p.is_dir(): + out += sorted(f for f in p.iterdir() if f.is_file() and f.suffix.lower() == suffix) + elif p.is_file(): + out.append(p) + else: + raise FileNotFoundError(p) + return out + + +def _write(path: Path, data: bytes) -> Path: + from ..unity.pack import write_outputs + + write_outputs({path: data}) + return path + + +def _filename(name: str) -> str: + if not name or name in (".", "..") or any(c in name for c in ("/", "\\", "\0", ":")): + raise ValueError("served names must be file names without directory components") + return name + + +def _batch(outputs: dict[Path, bytes], path: Path, data: bytes) -> None: + if path.name.casefold() in {p.name.casefold() for p in outputs}: + raise ValueError(f"duplicate output file name {path.name!r}") + outputs[path] = data + + +def cmd_master(args, cfg): + from ..source import master + from ..unity.pack import write_outputs + + try: + files = _inputs(args.inputs, ".json") + except OSError as e: + _fail(str(e)) + if not files: + _fail("no master data tables (*.json) among the inputs") + key = Session(cfg).master_key() + try: + prefix = Path(args.prefix_from).read_bytes()[: master.PREFIX] if args.prefix_from else b"" + except OSError as e: + _fail(str(e)) + if args.prefix_from and len(prefix) != master.PREFIX: + _fail(f"{args.prefix_from} is shorter than the {master.PREFIX}-byte prefix") + out = Path(args.out) + outputs: dict[Path, bytes] = {} + rk = master.round_keys(key.key) + for src in files: + try: + data = master.encode(src.read_bytes(), key, prefix, rk) + _batch(outputs, out / _filename(f"{src.stem}.bin"), data) + except (OSError, ValueError) as e: + _fail(f"{src.name}: {e}") + packed = [str(path) for path in outputs] + manifest = "" + if args.manifest: + listed = [ + {"name": p.name, "hash": hashlib.sha256(data).hexdigest(), "size": len(data)} for p, data in outputs.items() + ] + doc = {"version": args.manifest, "files": listed} + manifest = str(out / "MasterManifest.json") + outputs[Path(manifest)] = json.dumps(doc, indent=1).encode("utf-8") + try: + write_outputs(outputs) + except (OSError, ValueError) as e: + _fail(str(e)) + report( + args, + {"out": str(out), "packed": len(packed), "files": packed, "manifest": manifest or None}, + lambda r: [ + f"{r['out']} ({r['packed']} tables packed)", + *r["files"], + *([r["manifest"]] if r["manifest"] else []), + ], + ) + + +def cmd_bundle(args, cfg): + from ..source import addressables + from ..unity.pack import write_outputs + + try: + files = _inputs(args.inputs, args.suffix) + except OSError as e: + _fail(str(e)) + if not files: + _fail(f"no bundle files (*{args.suffix}) among the inputs") + if args.name and len(files) != 1: + _fail("--name applies to a single input") + key = Session(cfg).bundle_key() + out = Path(args.out) + outputs: dict[Path, bytes] = {} + for src in files: + try: + name = _filename(args.name or src.name) + data = addressables.pack(src.read_bytes(), name, key) + _batch(outputs, out / name, data) + except (OSError, ValueError) as e: + _fail(f"{src.name}: {e}") + try: + write_outputs(outputs) + except (OSError, ValueError) as e: + _fail(str(e)) + packed = [str(path) for path in outputs] + report( + args, + {"out": str(out), "packed": len(packed), "files": packed}, + lambda r: [f"{r['out']} ({r['packed']} bundles packed)", *r["files"]], + ) + + +def cmd_asset(args, cfg): + from ..unity import bundlefs + from ..unity import pack as assetpack + + out = Path(args.out) + try: + data = Path(args.data).read_bytes() + assetpack.replace_text_asset(Path(args.bundle), out, args.name, data, new_name=args.new_name) + except (OSError, KeyError, ValueError) as e: + _fail(str(e).strip("'")) + content = out.read_bytes() + report( + args, + { + "in": args.bundle, + "out": str(out), + "asset": args.new_name or args.name, + "payloadBytes": len(data), + "bundleBytes": len(content), + "crc": bundlefs.content_crc(content), + }, + lambda r: [ + f"{r['out']} (asset {r['asset']}: {r['payloadBytes']} B payload, {r['bundleBytes']} B bundle, " + f"catalog crc {r['crc']})" + ], + ) + + +def cmd_build(args, cfg): + from ..source import addressables, catalogbuild + from ..unity import pack as assetpack + + try: + manifest = Path(args.file) + doc = json.loads(manifest.read_text(encoding="utf-8")) + if doc.get("format") == "nnnotes.song-package/1": + from ..packing import build_package + + if args.encrypt or args.catalog_add or args.catalog_out or args.name: + raise ValueError("song builds define bundle and catalog outputs through their target manifest") + package = build_package(manifest, out=args.out, config=cfg) + report( + args, + {"out": str(package.root), **package.manifest}, + lambda r: [f"{r['out']} (song {r['song']}, {len(r['bundles'])} bundles)"], + ) + return + result = assetpack.build_manifest(doc, base=manifest.resolve().parent) + catalog_source = args.catalog or (cfg.path("paths", "catalog") if args.catalog_out and cfg else None) + existing = Path(catalog_source).read_bytes() if catalog_source else None + if args.catalog_out and existing is None: + raise ValueError("--catalog-out requires --catalog SOURCE") + catalog_template = doc.get("catalog_template") + if existing is not None and doc.get("mode") == "edit" and catalog_template is None and doc.get("internal_id"): + matches = [ + location + for location in addressables.parse_locations(existing) + if location["resource_type"] == catalogbuild.BUNDLE_TYPE + and ( + location["internal_id"] == doc["internal_id"] + or location["internal_id"].rsplit("/", 1)[-1] == doc["internal_id"].rsplit("/", 1)[-1] + ) + ] + if len(matches) == 1: + catalog_template = matches[0]["internal_id"] + provider, request_options = catalogbuild.bundle_settings( + existing, + provider=doc.get("bundle_provider"), + template=catalog_template, + request_options=doc.get("request_options"), + ) + out = Path(args.out) + outputs = {out: result.data} + served_name = _filename(args.name or (Path(args.encrypt).name if args.encrypt else out.name)) + if args.encrypt: + outputs[Path(args.encrypt)] = addressables.pack(result.data, served_name, Session(cfg).bundle_key()) + internal_id = doc.get("internal_id") + if (args.catalog_add or args.catalog_out) and not internal_id: + raise ValueError("catalog output requires a bundle internal_id in the build manifest") + if internal_id and internal_id.rsplit("/", 1)[-1] != served_name: + raise ValueError("bundle internal_id file name must match the served file name") + description = { + "file": str(out.resolve()), + "internal_id": internal_id, + "bundle_name": result.bundle_name, + "bundle_provider": provider, + "assets": result.assets, + "hash": doc.get("hash") or hashlib.md5(result.data).hexdigest(), + "request_options": request_options, + } + if doc.get("mode") == "edit": + description.update(mode="edit", catalog_edits=result.catalog_edits) + if catalog_template: + description["catalog_template"] = catalog_template + for field in ("primary_key", "dependency_keys"): + if field in doc: + description[field] = doc[field] + if args.catalog_add: + outputs[Path(args.catalog_add)] = ( + json.dumps({"bundles": [description]}, ensure_ascii=False, indent=2).encode("utf-8") + b"\n" + ) + if args.catalog_out: + assert existing is not None + if doc.get("mode") == "edit": + additions, replacements = catalogbuild.edit_entries( + existing, + internal_id, + result.bundle_name, + result.assets, + result.catalog_edits, + hash=description["hash"], + crc=result.crc, + bundle_size=len(result.data), + primary_key=doc.get("primary_key"), + request_options=request_options, + dependency_keys=doc.get("dependency_keys"), + provider=provider, + template=catalog_template, + ) + outputs[Path(args.catalog_out)] = catalogbuild.rebuild(existing, add=additions, replace=replacements) + else: + additions = catalogbuild.bundle_entries( + internal_id, + result.bundle_name, + result.assets, + hash=description["hash"], + crc=result.crc, + bundle_size=len(result.data), + primary_key=doc.get("primary_key"), + request_options=request_options, + dependency_keys=doc.get("dependency_keys", ()), + provider=provider, + ) + drop = {entry["primary_key"] for entry in additions} + outputs[Path(args.catalog_out)] = catalogbuild.rebuild(existing, add=additions, drop=drop) + expected = 1 + bool(args.encrypt) + bool(args.catalog_add) + bool(args.catalog_out) + if len(outputs) != expected: + raise ValueError("output paths must be distinct") + assetpack.write_outputs(outputs) + except (OSError, ValueError, KeyError, TypeError) as e: + _fail(str(e).strip("'")) + report( + args, + { + "out": str(out), + "bundle_name": result.bundle_name, + "bundle_provider": provider, + "assets": result.assets, + "bundleBytes": len(result.data), + "crc": result.crc, + "hash": description["hash"], + "encrypted": args.encrypt, + "catalog": args.catalog_out, + "catalog_add": args.catalog_add, + }, + lambda r: [ + f"{r['out']} ({len(r['assets'])} assets, {r['bundleBytes']} B, crc {r['crc']})", + *[str(path) for path in outputs if path != out], + ], + ) + + +def cmd_plan(args, cfg): + from ..packing import plan_package + + try: + plan = plan_package(args.file, config=cfg) + except (OSError, ValueError, KeyError, TypeError, RuntimeError) as error: + _fail(str(error)) + report( + args, + plan.report, + lambda r: [ + f"{len(r['assets'])} assets; {len(r['diagnostics'])} errors", + *[f"{item['path']}: {item['message']}" for item in r["diagnostics"]], + ], + ) + if plan.report["diagnostics"]: + raise SystemExit(1) + + +def cmd_verify(args, cfg): + from ..packing import verify_package + + try: + result = verify_package(args.directory, target=args.target, config=cfg) + except (OSError, ValueError, KeyError, TypeError, RuntimeError) as error: + _fail(str(error)) + report(args, result, lambda r: [f"{r['status']}: {r['files']} files, {r['bundles']} bundles"]) + + +def cmd_audio(args, cfg): + from ..core.jsonio import dumps + from ..packing import AudioOptions, AudioTools, Target, build_audio + from ..unity.pack import write_outputs + + try: + target = Target.load(args.target, cfg) + template, cipher, key = target.audio(args.template) + options = AudioOptions( + sample_rate=args.sample_rate, + channels=args.channels, + stream=args.stream, + gain_db=args.gain_db, + start_ms=args.start_ms, + duration_ms=args.duration_ms, + fade_in_ms=args.fade_in_ms, + fade_out_ms=args.fade_out_ms, + quality=args.quality, + ) + result = build_audio( + args.source, + template, + cue_name=args.cue_name, + options=options, + tools=AudioTools.configured(cfg), + cipher_type=cipher, + key=key, + ) + out = Path(args.out) + report_path = Path(args.report) if args.report else out.with_suffix(out.suffix + ".json") + if out.resolve() == report_path.resolve() or args.source.resolve() in (out.resolve(), report_path.resolve()): + raise ValueError("audio input, ACB output and report paths must be distinct") + write_outputs({out: result.data, report_path: (dumps(result.report, indent=2) + "\n").encode("utf-8")}) + except (OSError, ValueError, KeyError, TypeError, RuntimeError) as error: + _fail(str(error)) + report( + args, + {"out": str(out), "report": str(report_path), **result.report}, + lambda r: [ + f"{r['out']} ({r['processed']['samples']} samples, {r['processed']['sampleRate']} Hz)", + r["report"], + ], + ) + + +def register(sub) -> None: + ps = group(sub, "pack", "build audio, song packages, master tables and UnityFS bundles from local inputs") + p = command(ps, "plan", cmd_plan, "check a song project and its local resource dependencies") + p.add_argument("file", type=Path, help="nnnotes.song-package/1 JSON project") + p = command(ps, "verify", cmd_verify, "verify a generated song package against its local baseline") + p.add_argument("directory", type=Path, help="directory containing package.json") + p.add_argument("--target", type=Path, required=True, help="nnnotes.pack-target/1 JSON target") + p = command(ps, "audio", cmd_audio, "MP3/WAV/FLAC/AAC -> a validated single-cue HCA ACB") + p.add_argument("source", type=Path, help="local source audio file") + out_arg(p, "output ACB file") + p.add_argument( + "--target", type=Path, required=True, help="JSON target with local ACB templates and cipher settings" + ) + p.add_argument("--template", choices=("full", "preview"), default="full", help="target ACB profile (default full)") + p.add_argument("--cue-name", required=True, help="name of the output cue and sheet") + p.add_argument("--report", type=Path, help="validation JSON path (default .json)") + p.add_argument( + "--sample-rate", type=int, choices=(44100, 48000), help="output sample rate (default preserve input)" + ) + p.add_argument("--channels", type=int, choices=(1, 2), help="output channels (default preserve input)") + p.add_argument("--stream", type=int, help="source audio stream index when the file contains several tracks") + p.add_argument("--gain-db", type=float, default=0, help="explicit PCM gain in decibels (default 0)") + p.add_argument("--start-ms", type=int, default=0, help="crop start on the decoded timeline (default 0)") + p.add_argument("--duration-ms", type=int, help="crop duration; omitted keeps the remainder") + p.add_argument("--fade-in-ms", type=int, default=0, help="linear fade-in duration (default 0)") + p.add_argument("--fade-out-ms", type=int, default=0, help="linear fade-out duration (default 0)") + p.add_argument( + "--quality", + choices=("highest", "high", "middle", "low", "lowest"), + default="high", + help="HCA encoder quality preset (default high)", + ) + p = command(ps, "master", cmd_master, "master data tables (.json) ->
.bin (Rijndael-256-CBC over gzip JSON)") + p.add_argument("inputs", nargs="+", type=Path, help="directories (their *.json files) or files") + out_arg(p, "output directory") + p.add_argument( + "--prefix-from", + metavar="BIN", + help="copy the 64-byte prefix of this master data file (the loader ignores it; a replacement should keep it)", + ) + p.add_argument( + "--manifest", + metavar="VERSION", + help="also write MasterManifest.json (the file `master download` reads) for this master data version", + ) + p = command(ps, "bundle", cmd_bundle, "UnityFS bundles -> the form the CDN serves (AES-128-CTR, first 16 KiB)") + p.add_argument("inputs", nargs="+", type=Path, help="directories (their bundles) or files") + out_arg(p, "output directory") + p.add_argument( + "--suffix", + default=".bundle", + help="the file name suffix bundles have among the inputs (default .bundle)", + ) + p.add_argument( + "--name", + metavar="NAME", + help="the file name the bundle is served under (it seeds the nonce; default: the input's name)", + ) + p = command(ps, "asset", cmd_asset, "replace a binary TextAsset in a bundle; --new-name changes its object name") + p.add_argument("bundle", type=Path, help="the bundle to copy") + out_arg(p, "output bundle") + p.add_argument("--name", required=True, help="the TextAsset's name inside the bundle") + p.add_argument("--data", required=True, help="the file whose bytes become the asset's payload") + p.add_argument("--new-name", help="also rename the object; its container path is preserved") + p = command(ps, "build", cmd_build, "build a song package or a UnityFS bundle from a JSON manifest") + p.add_argument("file", type=Path, help="song project or asset manifest with template, mode and asset operations") + out_arg(p, "new song package directory, or output UnityFS bundle") + p.add_argument("--encrypt", metavar="FILE", help="also write the encrypted bundle to this file") + p.add_argument("--name", metavar="NAME", help="served file name used for encryption and catalog validation") + p.add_argument("--catalog-add", metavar="JSON", help="also write a manifest accepted by catalog rebuild --add") + p.add_argument("--catalog-out", metavar="FILE", help="rebuild the --catalog source with the bundle's locations") diff --git a/src/nnnotes/formats/VGAudio.LICENSE.txt b/src/nnnotes/formats/VGAudio.LICENSE.txt new file mode 100644 index 0000000..97328ad --- /dev/null +++ b/src/nnnotes/formats/VGAudio.LICENSE.txt @@ -0,0 +1,22 @@ +VGAudio HCA cipher permutation +https://github.com/Thealexbarney/VGAudio + +The MIT License (MIT) + +Copyright (c) 2016 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/nnnotes/formats/acbwrite.py b/src/nnnotes/formats/acbwrite.py new file mode 100644 index 0000000..45c62fa --- /dev/null +++ b/src/nnnotes/formats/acbwrite.py @@ -0,0 +1,267 @@ +"""Build single-cue ACB files and aligned memory AWB banks from local templates and encoded audio.""" + +from __future__ import annotations + +import hashlib +import struct +from dataclasses import dataclass + +from .acb import awb_ids, cue_streams, tables, utf_table +from .hca import inspect as inspect_hca + +__all__ = ["Column", "Utf", "author", "memory_awb"] + +FORMATS = {0: ">B", 1: ">b", 2: ">H", 3: ">h", 4: ">I", 5: ">i", 6: ">Q", 7: ">q", 8: ">f", 9: ">d"} +WIDTHS = {key: struct.calcsize(value) for key, value in FORMATS.items()} | {10: 4, 11: 8} + + +@dataclass(frozen=True) +class Column: + name: str + kind: int + storage: int + + +@dataclass +class Utf: + name: str + version: int + columns: list[Column] + rows: list[dict] + + @classmethod + def read(cls, data: bytes) -> Utf: + rows = utf_table(data) + size = struct.unpack_from(">I", data, 4)[0] + body = memoryview(data)[8 : size + 8] + version, _, strings_at, data_at, name_at, count, _, _ = struct.unpack_from(">HHIIIHHI", body) + strings = bytes(body[strings_at:data_at]) + + def text(offset: int) -> str: + return strings[offset : strings.index(b"\0", offset)].decode("utf8") + + columns, position = [], 24 + for _ in range(count): + flag = body[position] + position += 1 + if not flag & 0x10 or flag & 0xF0 not in (0x10, 0x30, 0x50): + raise ValueError(f"unsupported UTF column flag {flag:#x}") + name_offset = struct.unpack_from(">I", body, position)[0] + position += 4 + kind, storage = flag & 15, flag & 0xF0 + if kind not in WIDTHS: + raise ValueError(f"unsupported UTF value type {kind}") + columns.append(Column(text(name_offset), kind, storage)) + if storage == 0x30: + position += WIDTHS[kind] + return cls(text(name_at), version, columns, rows) + + def write(self) -> bytes: + """Write the original column types/storage and equivalent @UTF rows.""" + strings, string_ids = bytearray(), {} + blobs = bytearray() + + def string(value: str) -> int: + if value not in string_ids: + if "\0" in value: + raise ValueError("UTF strings cannot contain null characters") + string_ids[value] = len(strings) + strings.extend(value.encode("utf8") + b"\0") + return string_ids[value] + + table_name = string(self.name) + + def value(kind: int, data) -> bytes: + if kind in FORMATS: + return struct.pack(FORMATS[kind], data) + if kind == 10: + return struct.pack(">I", string(data)) + if kind == 11: + raw = bytes(data) + if not raw: + return bytes(8) + # Every blob begins at a 32-byte relative offset, including AWB. + blobs.extend(bytes(-len(blobs) % 32)) + offset = len(blobs) + blobs.extend(raw) + return struct.pack(">II", offset, len(raw)) + raise ValueError(f"unsupported UTF value type {kind}") + + definitions = bytearray() + row_columns = [] + for column in self.columns: + storage = column.storage + values = [row[column.name] for row in self.rows] + if storage == 0x30 and values and any(item != values[0] for item in values[1:]): + storage = 0x50 + definitions.extend(bytes([storage | column.kind]) + struct.pack(">I", string(column.name))) + if storage == 0x30: + if not values: + raise ValueError("constant UTF columns require one row") + definitions.extend(value(column.kind, values[0])) + elif storage == 0x50: + row_columns.append(column) + elif any(item is not None for item in values): + raise ValueError(f"name-only UTF column {column.name} cannot hold a value") + row_width = sum(WIDTHS[column.kind] for column in row_columns) + encoded_rows = bytearray() + for row in self.rows: + for column in row_columns: + encoded_rows.extend(value(column.kind, row[column.name])) + rows_at = 24 + len(definitions) + strings_at = rows_at + len(encoded_rows) + # UTF offsets are relative to byte 8; blob addresses must align in the file. + data_at = ((8 + strings_at + len(strings) + 31) & ~31) - 8 + if rows_at > 65535 or row_width > 65535 or len(self.columns) > 65535: + raise ValueError("UTF row/header limits exceeded") + header = struct.pack( + ">HHIIIHHI", + self.version, + rows_at, + strings_at, + data_at, + table_name, + len(self.columns), + row_width, + len(self.rows), + ) + body = header + definitions + encoded_rows + strings + body += bytes(data_at - len(body)) + blobs + result = b"@UTF" + struct.pack(">I", len(body)) + body + if utf_table(result) != self.rows: + raise ValueError("written UTF did not preserve row values") + return result + + +def adx_metadata(data: bytes) -> dict: + if len(data) < 20 or data[:2] != b"\x80\0": + raise ValueError("input must be an ADX stream") + data_start = struct.unpack_from(">H", data, 2)[0] + 4 + encoding, block_size, sample_bits, channels = struct.unpack_from("4B", data, 4) + rate, samples = struct.unpack_from(">II", data, 8) + if data_start > len(data) or data[data_start - 6 : data_start] != b"(c)CRI": + raise ValueError("invalid ADX data offset/copyright marker") + if encoding not in (2, 3, 4) or block_size < 3 or sample_bits != 4 or channels not in (1, 2): + raise ValueError("expected mono/stereo four-bit ADX") + if not rate or not samples: + raise ValueError("ADX sample rate/count must be positive") + if data[19] != 0: + raise ValueError("input ADX must be unencrypted") + samples_per_block = (block_size - 2) * 8 // sample_bits + required_blocks = (samples + samples_per_block - 1) // samples_per_block + if len(data) - data_start < required_blocks * block_size * channels: + raise ValueError("ADX audio blocks are truncated") + # ADX v3/v4 finite loop metadata, when present before the CRI marker. + loop_offset = {3: 20, 4: 24}.get(data[18]) + if loop_offset is not None and loop_offset + 24 <= data_start - 6: + if struct.unpack_from(">I", data, loop_offset + 4)[0]: + raise ValueError("input ADX must not loop") + return { + "encoding": encoding, + "blockSize": block_size, + "channels": channels, + "sampleRate": rate, + "samples": samples, + "headerVersion": data[18], + "lengthMs": (samples * 1000 + rate - 1) // rate, + } + + +def memory_awb(stream: bytes, *, alignment: int = 32) -> bytes: + """One-entry AFS2 memory bank: entry ID 0, no encryption subkey.""" + if alignment < 1 or alignment > 65535: + raise ValueError("AFS2 alignment must be 1..65535") + header = struct.pack("<4sBBBBIHH", b"AFS2", 2, 4, 4, 0, 1, alignment, 0) + table_end = len(header) + 4 + 8 + stream_start = (table_end + alignment - 1) // alignment * alignment + result = header + struct.pack(" dict: + metadata = inspect_hca(data) + if not allow_encrypted and metadata["cipherType"] != 0: + raise ValueError("input HCA must explicitly use cipher type zero") + metadata["headerVersion"] = metadata.pop("version") + metadata.pop("tagOffsets") + return metadata + + +def author( + template: bytes, adx: bytes, *, cue_name: str, sheet_name: str | None = None, allow_encrypted_hca: bool = False +) -> tuple[bytes, dict]: + """Preserve a one-cue template graph while replacing its finite audio.""" + if not cue_name or "\0" in cue_name: + raise ValueError("cue_name must be nonempty without null characters") + sheet_name = sheet_name or cue_name + if not sheet_name or "\0" in sheet_name: + raise ValueError("sheet_name must be nonempty without null characters") + is_hca = bytes(x & 127 for x in adx[:4]) == b"HCA\0" + meta = hca_metadata(adx, allow_encrypted=allow_encrypted_hca) if is_hca else adx_metadata(adx) + top = Utf.read(template) + if len(top.rows) != 1: + raise ValueError("template must have exactly one top-level row") + original_head, nested = tables(template) + if any(len(nested.get(name, [])) != 1 for name in ("CueTable", "CueNameTable", "WaveformTable")): + raise ValueError("template must have exactly one cue, cue name and waveform") + if len(cue_streams(template)) != 1: + raise ValueError("template must resolve to exactly one cue") + top.rows[0]["Name"] = sheet_name + modified = {} + for name in ("CueTable", "CueNameTable", "WaveformTable"): + table = Utf.read(top.rows[0][name]) + row = table.rows[0] + if name == "CueTable": + row["Length"] = meta["lengthMs"] + elif name == "CueNameTable": + row["CueName"] = cue_name + else: + row.update( + EncodeType=2 if is_hca else 0, + Streaming=0, + NumChannels=meta["channels"], + LoopFlag=1, + SamplingRate=meta["sampleRate"], + NumSamples=meta["samples"], + ) + row["MemoryAwbId" if "MemoryAwbId" in row else "Id"] = 0 + if "StreamAwbId" in row: + row["StreamAwbId"] = 65535 + if "StreamAwbPortNo" in row: + row["StreamAwbPortNo"] = 65535 + top.rows[0][name] = table.write() + modified[name] = table.rows + top.rows[0]["AwbFile"] = memory_awb(adx) + output = top.write() + out_head, out_tables = tables(output) + expected_preserved = set(nested) - set(modified) + for name in expected_preserved: + if out_head[name] != original_head[name] or out_tables[name] != nested[name]: + raise ValueError(f"unmodified table {name} was changed") + cues = cue_streams(output) + cue = cues.get(cue_name) + if cue is None or cue["lengthMs"] != meta["lengthMs"] or len(cue["streams"]) != 1: + raise ValueError("authored cue identity or timing is invalid") + stream = cue["streams"][0] + if any( + stream[field] != meta[source] + for field, source in (("samples", "samples"), ("sampleRate", "sampleRate"), ("channels", "channels")) + ): + raise ValueError("authored waveform metadata does not match ADX") + report = { + "schema": "nnnotes.acb-build/1", + "cueName": cue_name, + "sheetName": sheet_name, + "hca" if is_hca else "adx": meta, + "cues": cues, + "preservedTables": sorted(expected_preserved), + "templateSha256": hashlib.sha256(template).hexdigest(), + "hcaSha256" if is_hca else "adxSha256": hashlib.sha256(adx).hexdigest(), + "acbSha256": hashlib.sha256(output).hexdigest(), + "bytes": len(output), + "memoryAwb": {"count": 1, "id": 0, "subkey": 0, "alignment": 32}, + } + return output, report diff --git a/src/nnnotes/formats/hca.py b/src/nnnotes/formats/hca.py new file mode 100644 index 0000000..b927249 --- /dev/null +++ b/src/nnnotes/formats/hca.py @@ -0,0 +1,150 @@ +"""Finite HCA framing, CRC validation and cipher-56 substitution with caller-supplied keys. + +The key permutation follows the public HCA cipher described by VGAudio's CriHcaKey (MIT). +https://github.com/Thealexbarney/VGAudio/blob/master/src/VGAudio/Codecs/CriHca/CriHcaKey.cs +""" + +from __future__ import annotations + +import struct + +import numpy as np + +__all__ = ["crc16", "crypt", "inspect", "key_table"] + + +def _crc_table() -> np.ndarray: + table = np.arange(256, dtype=np.uint16) << 8 + for _ in range(8): + table = (table << 1) ^ np.where(table & 0x8000, 0x8005, 0).astype(np.uint16) + return table + + +_CRC = _crc_table() + + +def crc16(data: bytes | bytearray) -> int: + """CRC16 with polynomial 0x8005 and initial value zero.""" + value = 0 + for byte in data: + value = ((value << 8) ^ int(_CRC[(value >> 8) ^ byte])) & 65535 + return value + + +def _tag(data: bytes | bytearray) -> bytes: + return bytes(value & 127 for value in data) + + +def inspect(data: bytes, *, check_crc: bool = True) -> dict: + """Read a finite mono/stereo HCA, rejecting truncated frames, loops and invalid checksums.""" + if len(data) < 48 or _tag(data[:4]) != b"HCA\0" or _tag(data[8:12]) != b"fmt\0": + raise ValueError("invalid HCA header") + version, header_size = struct.unpack_from(">HH", data, 4) + channels, rate = data[12], int.from_bytes(data[13:16], "big") + frames, inserted, appended = struct.unpack_from(">IHH", data, 16) + at, frame_size, cipher, volume, tags = 24, 0, 0, 1.0, [0, 8] + if not 48 <= header_size <= len(data): + raise ValueError("invalid HCA header size") + while at < header_size - 2: + tag = _tag(data[at : at + 4]) + tags.append(at) + if tag in (b"pad\0", b"comm"): + break + if tag == b"loop": + raise ValueError("HCA input must have finite playback") + sizes = {b"comp": 16, b"dec\0": 12, b"vbr\0": 8, b"ath\0": 6, b"ciph": 6, b"rva\0": 8} + if tag not in sizes or at + sizes[tag] > header_size - 2: + raise ValueError(f"unsupported or truncated HCA chunk at {at}") + if tag in (b"comp", b"dec\0"): + frame_size = struct.unpack_from(">H", data, at + 4)[0] + elif tag == b"ciph": + cipher = struct.unpack_from(">H", data, at + 4)[0] + elif tag == b"rva\0": + volume = struct.unpack_from(">f", data, at + 4)[0] + at += sizes[tag] + samples = frames * 1024 - inserted - appended + if channels not in (1, 2) or rate <= 0 or samples <= 0 or frame_size < 8 or cipher not in (0, 56): + raise ValueError("unsupported HCA dimensions or cipher") + if len(data) != header_size + frames * frame_size: + raise ValueError("HCA frames are truncated or contain trailing data") + if volume != 1.0: + raise ValueError("HCA relative volume must be 1") + if check_crc: + if crc16(data[: header_size - 2]) != int.from_bytes(data[header_size - 2 : header_size], "big"): + raise ValueError("HCA header CRC mismatch") + blocks = np.frombuffer(data, dtype=np.uint8, offset=header_size).reshape(frames, frame_size) + sums = np.zeros(frames, dtype=np.uint16) + for index in range(frame_size - 2): + sums = (sums << 8) ^ _CRC[(sums >> 8) ^ blocks[:, index]] + expected = (blocks[:, -2].astype(np.uint16) << 8) | blocks[:, -1] + if np.any(sums != expected): + raise ValueError("HCA frame CRC mismatch") + return { + "codec": "hca", + "version": version, + "channels": channels, + "sampleRate": rate, + "samples": samples, + "frames": frames, + "frameSize": frame_size, + "headerSize": header_size, + "inserted": inserted, + "appended": appended, + "cipherType": cipher, + "lengthMs": (samples * 1000 + rate - 1) // rate, + "tagOffsets": tags, + } + + +def key_table(key: int, *, decrypt: bool = False) -> bytes: + """Build the cipher-56 substitution permutation for a nonzero unsigned 64-bit key.""" + if isinstance(key, bool) or not isinstance(key, int) or not 0 < key < 1 << 64: + raise ValueError("cipher 56 requires a nonzero unsigned 64-bit key") + values = (key - 1).to_bytes(8, "little") + a, b, c, d, e, f, g = values[:7] + seeds = (b, g ^ b, c ^ d, c, b ^ c, d ^ e, d, c ^ d, e ^ f, e, d ^ e, f ^ g, f, e ^ f, g ^ b, g) + + def row(seed): + value, result = seed >> 4, [] + for _ in range(16): + value = (value * (((seed & 1) << 3) | 5) + ((seed & 14) | 1)) & 15 + result.append(value) + return result + + grid = [(high << 4) | low for high, seed in zip(row(a), seeds) for low in row(seed)] + table = bytes([0, *[grid[(i * 17) & 255] for i in range(1, 257) if grid[(i * 17) & 255] not in (0, 255)], 255]) + if decrypt: + return table + inverse = bytearray(256) + for i, value in enumerate(table): + inverse[value] = i + return bytes(inverse) + + +def crypt(data: bytes, key: int, *, decrypt: bool = False) -> bytes: + """Transform a finite HCA between cipher 0 and 56, rebuilding chunk flags and CRCs.""" + info = inspect(data) + if info["cipherType"] != (56 if decrypt else 0): + raise ValueError("HCA cipher does not match the requested transform") + head = bytearray(data[: info["headerSize"]]) + cipher = None + for at in info["tagOffsets"]: + if _tag(head[at : at + 4]) == b"ciph": + cipher = at + for index in range(at, at + 4): + value = head[index] & 127 + head[index] = value if decrypt or value == 0 else value | 128 + if cipher is None: + raise ValueError("HCA cipher chunk is required") + struct.pack_into(">H", head, cipher + 4, 0 if decrypt else 56) + head[-2:] = struct.pack(">H", crc16(head[:-2])) + translated = np.frombuffer(data[info["headerSize"] :], np.uint8).copy().reshape(info["frames"], info["frameSize"]) + table = np.frombuffer(key_table(key, decrypt=decrypt), np.uint8) + translated[:, :-2] = table[translated[:, :-2]] + sums = np.zeros(info["frames"], dtype=np.uint16) + for index in range(info["frameSize"] - 2): + sums = (sums << 8) ^ _CRC[(sums >> 8) ^ translated[:, index]] + translated[:, -2], translated[:, -1] = sums >> 8, sums & 255 + output = bytes(head) + translated.tobytes() + inspect(output) + return output diff --git a/src/nnnotes/packing/__init__.py b/src/nnnotes/packing/__init__.py new file mode 100644 index 0000000..b7f6649 --- /dev/null +++ b/src/nnnotes/packing/__init__.py @@ -0,0 +1,20 @@ +"""Build audio and song resource packages from user-supplied local inputs.""" + +from .audio import AudioOptions, AudioResult, AudioTools, build_audio +from .build import PackageResult, build_package, verify_package +from .plan import PackagePlan, plan_package +from .project import SongProject, Target + +__all__ = [ + "AudioOptions", + "AudioResult", + "AudioTools", + "PackagePlan", + "PackageResult", + "SongProject", + "Target", + "build_audio", + "build_package", + "plan_package", + "verify_package", +] diff --git a/src/nnnotes/packing/assets.py b/src/nnnotes/packing/assets.py new file mode 100644 index 0000000..febfd17 --- /dev/null +++ b/src/nnnotes/packing/assets.py @@ -0,0 +1,241 @@ +"""Song asset adapters over the template-driven Unity bundle builder.""" + +from __future__ import annotations + +import copy +import hashlib +import io +from dataclasses import dataclass +from typing import Any, cast + +import UnityPy +from PIL import Image + +from ..formats import cri +from ..source import addressables, catalogbuild +from ..unity.pack import BuildResult, BundleBuilder, parse_text_asset +from .project import Target + + +@dataclass +class Template: + data: bytes + asset: dict + bundle: dict + dependencies: list[str] + + +def containers(env) -> dict: + """Local container roots without resolving preload references or searching the host filesystem.""" + output = {} + for file in env.assets: + for key, pointer in file.container.items(): + if pointer.m_FileID != 0: + raise ValueError("asset container roots must be local objects") + output[key] = file.objects[pointer.path_id] + return output + + +def select_template(target: Target, key: str, resource_type: str | None = None) -> Template: + locations = addressables.parse_locations(target.catalog()) + by_offset = {item["offset"]: item for item in locations} + assets = [ + item + for item in locations + if item["primary_key"] == key and (resource_type is None or item["resource_type"] == resource_type) + ] + if len(assets) != 1: + raise ValueError(f"template key must select one asset with the requested type: {key}") + asset = assets[0] + pending = list(asset["dependencies"]) + candidates, seen = [], set() + while pending: + offset = pending.pop(0) + if offset in seen: + continue + seen.add(offset) + location = by_offset[offset] + if location["resource_type"] == catalogbuild.BUNDLE_TYPE: + candidates.append(location) + pending.extend(location["dependencies"]) + for location in candidates: + data = target.bundle(location) + env = UnityPy.load(data) + if asset["internal_id"] in containers(env): + dependencies = [ + by_offset[offset]["primary_key"] for offset in asset["dependencies"] if offset != location["offset"] + ] + return Template(data, asset, location, dependencies) + raise ValueError(f"template container is missing from local bundle dependencies: {key}") + + +def _asset_spec(template: Template, key: str, internal: str) -> dict: + return { + "primary_key": key, + "internal_id": internal, + "resource_type": template.asset["resource_type"], + "resource_type_assembly": template.asset.get("resource_type_assembly"), + "provider": template.asset["provider"], + "dependency_keys": template.dependencies, + } + + +def chart_bundle(template: Template, payload: bytes, name: str, bundle_name: str) -> BuildResult: + builder = BundleBuilder(template.data, bundle_name=bundle_name) + internal = f"Assets/AddressableResources/Live/MusicScore/{name}.bytes" + builder.apply( + { + "type": "TextAsset", + "name": name.rsplit("/", 1)[-1], + "data": payload, + "template": {"type": "TextAsset"}, + "container": internal, + } + ) + result = builder.build() + asset = containers(UnityPy.load(result.data))[internal] + if parse_text_asset(asset.get_raw_data(), endian=asset.assets_file.header.endian).payload != payload: + raise ValueError("chart bundle did not preserve its payload") + return BuildResult(result.data, result.bundle_name, [_asset_spec(template, "Live/MusicScore/" + name, internal)]) + + +def jacket_bundle(template: Template, png: bytes, name: str, bundle_name: str, *, small: bool) -> BuildResult: + builder = BundleBuilder(template.data, bundle_name=bundle_name) + sprite = builder.select({"type": "Sprite"}, templates=True) + fields = copy.deepcopy(sprite.read_typetree()) + source = builder.select({"path_id": fields["m_RD"]["texture"]["m_PathID"]}, templates=True) + texture = source.read_typetree() + width, height = texture["m_Width"], texture["m_Height"] + with Image.open(io.BytesIO(png)) as image: + if image.format != "PNG" or image.width != image.height: + raise ValueError("jacket must be a square PNG") + image = image.convert("RGBA").resize((width, height), Image.Resampling.LANCZOS) + output = io.BytesIO() + image.save(output, format="PNG") + resized = output.getvalue() + prefix = "Assets/AddressableResources/Image/Jacket/" + ("small/" if small else "") + key = "Image/Jacket/" + ("small/" if small else "") + name + # The texture encoder accepts PNG paths; its temporary input stays local to this adapter. + import tempfile + from pathlib import Path + + with tempfile.TemporaryDirectory(prefix="nnnotes-jacket-") as directory: + path = Path(directory) / "jacket.png" + path.write_bytes(resized) + builder.apply( + { + "type": "Texture2D", + "name": name, + "path_id": 2, + "file": str(path), + "template": {"type": "Texture2D", "path_id": source.path_id}, + "container": prefix + name + ".texture", + } + ) + fields["m_Name"] = name + fields["m_RD"]["texture"] = {"m_FileID": 0, "m_PathID": 2} + if fields["m_RD"].get("alphaTexture", {}).get("m_PathID"): + fields["m_RD"]["alphaTexture"] = {"m_FileID": 0, "m_PathID": 0} + if "m_SpriteAtlas" in fields: + fields["m_SpriteAtlas"] = {"m_FileID": 0, "m_PathID": 0} + if "m_RenderDataKey" in fields: + guid = hashlib.md5(key.encode()).digest() + fields["m_RenderDataKey"] = ( + {f"data[{i}]": int.from_bytes(guid[4 * i : 4 * i + 4], "little") for i in range(4)}, + fields["m_RenderDataKey"][1], + ) + internal = prefix + name + ".png" + builder.apply( + { + "type": "serialized", + "template": {"type": "Sprite", "path_id": sprite.path_id}, + "name": name, + "path_id": 3, + "fields": fields, + "container": internal, + } + ) + result = builder.build() + reread = cast(Any, containers(UnityPy.load(result.data))[internal].read()) + if reread.m_RD.texture.deref().read().image.size != (width, height): + raise ValueError("jacket Sprite does not resolve to the expected texture") + return BuildResult(result.data, result.bundle_name, [_asset_spec(template, key, internal)]) + + +def extract_acb(data: bytes, internal: str) -> bytes: + env = UnityPy.load(data) + root = containers(env)[internal] + fields = cast(dict, root.read_typetree()) + if "_chunks" in fields: + parts = [] + for pointer in fields["_chunks"]: + if pointer["m_FileID"] != 0: + raise ValueError("split ACB chunks must be local TextAssets") + obj = root.assets_file.objects[pointer["m_PathID"]] + parts.append(parse_text_asset(obj.get_raw_data(), endian=obj.assets_file.header.endian).payload) + return cri.split_acb(parts) + reference = fields["implementation"]["rid"] + impl = next(item for item in fields["references"]["RefIds"] if item["rid"] == reference) + return cri.embedded_acb(fields, impl) + + +def audio_bundle(template: Template, acb: bytes, sheet: str, bundle_name: str) -> BuildResult: + source = containers(UnityPy.load(template.data))[template.asset["internal_id"]] + fields = cast(dict, source.read_typetree()) + if "_chunks" in fields: + builder = BundleBuilder(template.data, bundle_name=bundle_name) + for external in source.assets_file.externals: + builder.add_external(external.path, guid=external.guid.hex(), type=external.type) + masked = bytes(byte ^ 0x5A for byte in acb) + count = len(fields["_chunks"]) + if not count: + raise ValueError("split ACB template has no chunks") + pointers = [] + for index, source_chunk in enumerate(fields["_chunks"]): + identity = index + 2 + part = masked[len(masked) * index // count : len(masked) * (index + 1) // count] + builder.apply( + { + "type": "TextAsset", + "template": {"path_id": source_chunk["m_PathID"]}, + "path_id": identity, + "name": f"{sheet}-{index:03}", + "data": part, + "container": f"Assets/AddressableResources/Cri/Sound/MusicScore/{sheet}/{index:03}", + } + ) + pointers.append({"m_FileID": 0, "m_PathID": identity}) + fields.update(m_Name=sheet, _cueSheetName=sheet, _chunks=pointers) + internal = f"Assets/AddressableResources/Cri/Sound/MusicScore/{sheet}.asset" + builder.apply( + { + "type": "serialized", + "template": {"path_id": source.path_id}, + "name": sheet, + "path_id": count + 2, + "fields": fields, + "container": internal, + } + ) + else: + builder = BundleBuilder(template.data, new=False, bundle_name=bundle_name) + reference = fields["implementation"]["rid"] + impl = next(item for item in fields["references"]["RefIds"] if item["rid"] == reference) + if impl["type"]["class"] != "CriSerializedBytesAssetImpl": + raise ValueError("unsupported embedded ACB implementation") + impl["data"]["data"] = list(acb) + internal = f"Assets/AddressableResources/Cri/Sound/MusicScoreShort/{sheet}.acb" + builder.apply( + { + "op": "replace", + "type": "serialized", + "target": {"path_id": source.path_id}, + "new_name": sheet, + "fields": fields, + "container": internal, + } + ) + result = builder.build() + if extract_acb(result.data, internal) != acb: + raise ValueError("sound bundle did not preserve its ACB payload") + return BuildResult(result.data, result.bundle_name, [_asset_spec(template, "Cri/Sound/" + sheet, internal)]) diff --git a/src/nnnotes/packing/audio.py b/src/nnnotes/packing/audio.py new file mode 100644 index 0000000..8bc508b --- /dev/null +++ b/src/nnnotes/packing/audio.py @@ -0,0 +1,302 @@ +"""Common audio inputs to validated, finite, single-cue ACB files.""" + +from __future__ import annotations + +import hashlib +import json +import math +import shutil +import subprocess +import tempfile +import wave +from dataclasses import asdict, dataclass +from pathlib import Path + +import numpy as np + +from ..formats import acbwrite, crikey, hca +from ..source.config import Config, tool + +__all__ = ["AudioOptions", "AudioResult", "AudioTools", "build_audio", "probe_audio"] + + +@dataclass(frozen=True) +class AudioTools: + """External tools; VGAudioCli can be an executable or a .NET DLL.""" + + ffmpeg: str + ffprobe: str + vgaudio: str + vgmstream: str + + @classmethod + def configured(cls, config: Config | None = None) -> AudioTools: + def select(name, executable): + path = config.path("paths", name) if config else None + return str(path) if path else tool(name, executable) + + return cls( + select("ffmpeg", "ffmpeg"), + select("ffprobe", "ffprobe"), + select("vgaudio", "VGAudioCli"), + select("vgmstream", "vgmstream-cli"), + ) + + def fingerprints(self) -> dict: + output = {} + for name, value in asdict(self).items(): + path = Path(shutil.which(value) or value) + if not path.is_file(): + raise ValueError(f"audio tool not found: {name}") + output[name] = hashlib.sha256(path.read_bytes()).hexdigest() + return output + + +@dataclass(frozen=True) +class AudioOptions: + """PCM processing. All time values refer to the decoded source timeline.""" + + sample_rate: int | None = None + channels: int | None = None + stream: int | None = None + gain_db: float = 0 + start_ms: int = 0 + duration_ms: int | None = None + fade_in_ms: int = 0 + fade_out_ms: int = 0 + quality: str = "high" + + def __post_init__(self): + for name in ("start_ms", "duration_ms", "fade_in_ms", "fade_out_ms"): + value = getattr(self, name) + if value is not None and (isinstance(value, bool) or not isinstance(value, int) or value < 0): + raise ValueError(f"{name} must be a nonnegative integer") + if self.duration_ms == 0 or not math.isfinite(self.gain_db) or not -120 <= self.gain_db <= 24: + raise ValueError("audio duration must be positive and gain_db must be in -120..24") + if ( + isinstance(self.channels, bool) + or self.channels not in (None, 1, 2) + or self.sample_rate not in (None, 44100, 48000) + ): + raise ValueError("audio output supports mono/stereo at 44100 or 48000 Hz") + if self.stream is not None and ( + isinstance(self.stream, bool) or not isinstance(self.stream, int) or self.stream < 0 + ): + raise ValueError("stream must be a nonnegative integer") + if self.quality not in ("highest", "high", "middle", "low", "lowest"): + raise ValueError("unsupported HCA quality") + + +@dataclass(frozen=True) +class AudioResult: + """ACB bytes and a portable verification report.""" + + data: bytes + report: dict + + +def _run(command: list[str]) -> bytes: + try: + process = subprocess.run(command, capture_output=True, timeout=600) + except (OSError, subprocess.TimeoutExpired) as error: + raise ValueError(f"audio tool failed: {Path(command[0]).name}") from error + if process.returncode: + message = process.stderr.decode("utf-8", "replace")[-2000:] + raise ValueError(f"{Path(command[0]).name} failed: {message}") + return process.stdout + + +def probe_audio(source: Path, tools: AudioTools, *, stream: int | None = None) -> dict: + """Probe actual audio streams, requiring explicit selection when there are several.""" + raw = _run([tools.ffprobe, "-v", "error", "-show_streams", "-of", "json", str(source)]) + streams = [item for item in json.loads(raw)["streams"] if item.get("codec_type") == "audio"] + selected = [item for item in streams if item["index"] == stream] if stream is not None else streams + if len(selected) != 1: + raise ValueError("input must have exactly one selected audio stream; set stream to its index") + audio = selected[0] + return { + "stream": audio["index"], + "codec": audio["codec_name"], + "sampleRate": int(audio["sample_rate"]), + "channels": int(audio["channels"]), + } + + +def _sample_time(milliseconds: int, rate: int) -> int: + return (milliseconds * rate + 500) // 1000 + + +def _pcm(source: Path, info: dict, options: AudioOptions, tools: AudioTools) -> tuple[np.ndarray, int]: + rate = options.sample_rate or info["sampleRate"] + channels = options.channels or info["channels"] + if rate not in (44100, 48000) or channels not in (1, 2): + raise ValueError("set sample_rate/channels to convert this input to mono/stereo at 44100 or 48000 Hz") + raw = _run( + [ + tools.ffmpeg, + "-nostdin", + "-v", + "error", + "-xerror", + "-err_detect", + "explode", + "-i", + str(source), + "-map", + f"0:{info['stream']}", + "-vn", + "-sn", + "-dn", + "-ar", + str(rate), + "-ac", + str(channels), + "-c:a", + "pcm_f32le", + "-f", + "f32le", + "pipe:1", + ] + ) + if not raw or len(raw) % (4 * channels): + raise ValueError("audio decoder produced empty or incomplete PCM") + samples = np.frombuffer(raw, dtype=" len(samples): + raise ValueError("audio fades exceed the selected sample range") + if fade_in: + samples[:fade_in] *= np.linspace(0, 1, fade_in, dtype=np.float32)[:, None] + if fade_out: + samples[-fade_out:] *= np.linspace(1, 0, fade_out, dtype=np.float32)[:, None] + if np.max(samples) > 1 or np.min(samples) < -1: + raise ValueError("PCM would clip; lower gain_db") + return samples, rate + + +def _statistics(pcm: np.ndarray) -> dict: + return { + "peak": float(np.max(np.abs(pcm))), + "rms": [float(value) for value in np.sqrt(np.mean(np.square(pcm, dtype=np.float64), axis=0))], + } + + +def _compare(reference: np.ndarray, decoded: np.ndarray) -> dict: + if decoded.shape != reference.shape: + raise ValueError("ACB decoded sample count/channels do not match the processed PCM") + energy = np.mean(np.square(reference, dtype=np.float64), axis=0) + errors = np.mean(np.square(decoded.astype(np.float64) - reference), axis=0) + active = energy > 1e-8 + ratios = np.zeros_like(energy) + ratios[active] = errors[active] / energy[active] + decoded_energy = np.mean(np.square(decoded, dtype=np.float64), axis=0) + gain = np.ones_like(energy) + gain[active] = decoded_energy[active] / energy[active] + if ( + np.any(ratios[active] > 0.05) + or np.any(errors[~active] > 1e-6) + or np.any(gain[active] > 10**0.1) + or np.any(gain[active] < 10**-0.1) + ): + raise ValueError("ACB decoded waveform differs substantially from the processed PCM") + return { + "status": "passed", + "relativeMeanSquareError": ratios.tolist(), + "powerRatio": gain.tolist(), + "decoded": _statistics(decoded), + "samples": len(decoded), + } + + +def build_audio( + source: Path, + template: bytes, + *, + cue_name: str, + sheet_name: str | None = None, + options: AudioOptions | None = None, + tools: AudioTools | None = None, + cipher_type: int = 0, + key: int | None = None, +) -> AudioResult: + """Decode common audio, encode HCA, build ACB and validate using an independent decoder. + + The key is used only in memory and an isolated temporary decoder key file. Reports contain no key material. + FFmpeg decodes the source; VGAudioCli encodes HCA; vgmstream verifies the resulting ACB. + """ + source = Path(source) + options, tools = options or AudioOptions(), tools or AudioTools.configured() + if cipher_type not in (0, 56) or (cipher_type == 56 and not key): + raise ValueError("audio cipher must be 0 or 56 with a configured nonzero key") + identities = tools.fingerprints() + source_data = source.read_bytes() + info = probe_audio(source, tools, stream=options.stream) + pcm, rate = _pcm(source, info, options, tools) + quantized = np.clip(np.rint(pcm.astype(np.float64) * 32768), -32768, 32767).astype(" bytes: + return (dumps(value, indent=2) + "\n").encode("utf-8") + + +@dataclass(frozen=True) +class PackageResult: + root: Path + manifest: dict + + +def _verify(root: Path, target: Target | None) -> dict: + manifest = json.loads((root / "package.json").read_bytes()) + if manifest.get("format") != "nnnotes.song-build/1": + raise ValueError("unsupported song package format") + schema_errors = list(Draft202012Validator(schemas.load("song-build")).iter_errors(manifest)) + if schema_errors: + raise ValueError(f"invalid song package manifest: {schema_errors[0].message}") + declared = set() + for item in manifest["files"]: + relative = Path(item["path"]) + path = (root / relative).resolve() + if relative.is_absolute() or not path.is_relative_to(root.resolve()) or item["path"] in declared: + raise ValueError("package contains an invalid or duplicate file path") + declared.add(item["path"]) + data = path.read_bytes() + if len(data) != item["size"] or fingerprint(data) != item["sha256"]: + raise ValueError(f"package file integrity mismatch: {item['path']}") + actual = {path.relative_to(root).as_posix() for path in root.rglob("*") if path.is_file()} + if actual != declared | {"package.json"}: + raise ValueError("package file set does not match its manifest") + + def content(relative: str) -> bytes: + if relative not in declared: + raise ValueError("package references a file absent from its file manifest") + return (root / relative).read_bytes() + + catalog = content("catalog/catalog.bin") + if (root / "catalog/catalog.hash").read_text(encoding="ascii") != fingerprint(catalog)[:32]: + raise ValueError("catalog hash does not match catalog.bin") + locations = addressables.parse_locations(catalog) + offsets = {item["offset"] for item in locations} + if any(dependency not in offsets for item in locations for dependency in item["dependencies"]): + raise ValueError("catalog contains an unresolved dependency") + cabs = set() + built_assets = {} + for bundle in manifest["bundles"]: + data = content(bundle["path"]) + if bundle["transport"] not in ("plain", "encrypted"): + raise ValueError("unsupported bundle transport") + if bundle["transport"] == "encrypted": + if target is None: + raise ValueError("encrypted bundle verification requires the target configuration") + data = addressables.decrypt(data, Path(bundle["path"]).name, target.session.bundle_key()) + if bundlefs.content_crc(data) != bundle["crc"] or hashlib.md5(data).hexdigest() != bundle["hash"]: + raise ValueError("bundle content does not match its catalog identity") + env = UnityPy.load(data) + for file in env.assets: + if file.name in cabs: + raise ValueError("new bundles share a serialized file identity") + cabs.add(file.name) + matches = [item for item in locations if item["internal_id"] == bundle["internal_id"]] + if len(matches) != 1 or matches[0]["provider"] != bundle["provider"]: + raise ValueError("bundle catalog location does not match its provider") + options = matches[0]["extra_data"] + if (options["crc"], options["bundleSize"], options["hash"]) != (bundle["crc"], len(data), bundle["hash"]): + raise ValueError("catalog bundle request options do not match the file") + for asset in bundle["assets"]: + if asset["internal_id"] not in containers(env): + raise ValueError("catalog container does not resolve to its bundled asset") + matches = [ + item + for item in locations + if item["primary_key"] == asset["primary_key"] and item["resource_type"] == asset["resource_type"] + ] + if len(matches) != 1 or matches[0]["internal_id"] != asset["internal_id"]: + raise ValueError("asset catalog key/type does not match its container") + built_assets[asset["primary_key"]] = (containers(env)[asset["internal_id"]], bundle) + if bundle.get("audio"): + acb = content(bundle["audio"]) + if extract_acb(data, asset["internal_id"]) != acb: + raise ValueError("bundled ACB does not match the audio artifact") + master_manifest = json.loads(content("master/MasterManifest.json")) + if master_manifest["version"] != manifest["version"]: + raise ValueError("package and master manifest versions differ") + baseline = target.master_files() if target else {} + if target and fingerprint(target.catalog()) != manifest["baseline"]["catalogSha256"]: + raise ValueError("target catalog does not match the package baseline") + for item in master_manifest["files"]: + if Path(item["name"]).name != item["name"]: + raise ValueError("master manifest contains an invalid table path") + path = root / "master" / item["name"] + data = path.read_bytes() if path.is_file() else baseline.get(item["name"]) + if data is None: + raise ValueError("package verification requires its local master baseline") + if len(data) != item["size"] or fingerprint(data) != item["hash"]: + raise ValueError("master manifest does not match a generated or baseline table") + if target: + + def table(name): + relative = f"master/{name}.bin" + raw = content(relative) if relative in declared else baseline[name + ".bin"] + return json.loads(master.decode(raw, target.session.master_key())) + + names = ( + "MasterLiveMusic", + "MasterLiveMusicScore", + "MasterText", + "MasterSound", + "MasterSoundCueSheet", + "MasterTag", + "MasterBand", + "MasterLiveScoreRank", + ) + tables = {name: table(name) for name in names} + song = find(tables, "MasterLiveMusic", manifest["song"]) + for field, identity in song.items(): + if field.endswith("TextID"): + find(tables, "MasterText", identity) + for band in song["_bandIDs"]: + find(tables, "MasterBand", band) + for tag in song["_bestMusicTagIDs"]: + record = find(tables, "MasterTag", tag) + find(tables, "MasterText", record["_nameTextID"]) + if not any(row["_group"] == song["_liveScoreRankGroup"] for row in tables["MasterLiveScoreRank"]["_allData"]): + raise ValueError("song score rank group is missing") + for field in ("_musicSoundID", "_jingleSoundID"): + sound = find(tables, "MasterSound", song[field]) + sheet = find(tables, "MasterSoundCueSheet", sound["_soundCueSheetID"]) + _, bundle = built_assets["Cri/Sound/" + sheet["_cueSheetName"]] + cues = cue_streams(content(bundle["audio"])) + if sound["_cueName"] not in cues: + raise ValueError("song cue name is absent from its ACB") + for difficulty in DIFFICULTIES: + chart = find(tables, "MasterLiveMusicScore", song[f"_{difficulty}ID"]) + obj, _ = built_assets["Live/MusicScore/" + chart["_musicScoreTextFileName"]] + from ..unity.pack import parse_text_asset + + payload = parse_text_asset(obj.get_raw_data(), endian=obj.assets_file.header.endian).payload + if score.convert(score.load_bytes(payload))["judgementNoteCount"] != chart["_fullComboCount"]: + raise ValueError("chart combo count does not match its master row") + for prefix in ("Image/Jacket/", "Image/Jacket/small/"): + if prefix + song["_jacketAssetName"] not in built_assets: + raise ValueError("song jacket reference is missing") + return { + "format": "nnnotes.package-validation/1", + "status": "passed", + "files": len(declared), + "bundles": len(manifest["bundles"]), + "masterTables": len(master_manifest["files"]), + } + + +def verify_package(root: str | Path, *, target: Target | str | Path | None = None, config=None) -> dict: + """Validate files, bundle payloads, catalog references and master files without accessing the network.""" + selected = Target.load(target, config) if isinstance(target, (str, Path)) else target + return _verify(Path(root).resolve(), selected) + + +def build_package( + plan: PackagePlan | SongProject | str | Path, *, out: str | Path, config=None, tools: AudioTools | None = None +) -> PackageResult: + """Build into a new directory; all required validations finish before the directory becomes visible.""" + plan = plan if isinstance(plan, PackagePlan) else plan_package(plan, config=config, tools=tools) + plan.check_inputs() + target = plan.target + if target is None: + raise ValueError("package plan has no target") + tools = tools or AudioTools.configured(target.session.cfg) + if tools.fingerprints() != plan.report.get("tools"): + raise ValueError("audio tools changed after planning") + out = Path(out).resolve() + if out.exists(): + raise ValueError("package output must be a new directory") + project, names = plan.project, plan.names + outputs: dict[str, bytes] = {} + records, additions, audio_reports = [], [], {} + catalog = target.catalog() + templates = plan.templates + url = target.document["bundle_base_url"].rstrip("/") + if not url or "\0" in url: + raise ValueError("target bundle_base_url must be a nonempty runtime URL/path") + + def publish(result, template, audio=None): + digest = hashlib.md5(result.data).hexdigest() + name = f"{result.bundle_name}_{digest}.bundle" + internal = url + "/" + name + provider, options = catalogbuild.bundle_settings(catalog, template=template.bundle["internal_id"]) + transport = target.transport(provider) + data = ( + addressables.pack(result.data, name, target.session.bundle_key()) + if transport == "encrypted" + else result.data + ) + path = "bundles/" + name + if path in outputs: + raise ValueError("bundle output name collision") + outputs[path] = data + additions.extend( + catalogbuild.bundle_entries( + internal, + result.bundle_name, + result.assets, + hash=digest, + crc=result.crc, + bundle_size=len(data), + provider=provider, + request_options=options, + ) + ) + records.append( + { + "path": path, + "internal_id": internal, + "provider": provider, + "transport": transport, + "hash": digest, + "crc": result.crc, + "assets": result.assets, + **({"audio": audio} if audio else {}), + } + ) + + for difficulty in DIFFICULTIES: + name = names["charts"][difficulty] + publish( + chart_bundle(templates["chart"], plan.charts[difficulty], name, name.replace("/", "_")), templates["chart"] + ) + png = project.file(project.document["assets"]["jacket"]).read_bytes() + for kind, small in (("jacket", False), ("jacket_small", True)): + publish( + jacket_bundle(templates[kind], png, names["jacket"], f"{names['jacket']}_{kind}", small=small), + templates[kind], + ) + for kind in ("full", "preview"): + template, cipher, key = target.audio(kind) + source, options = plan.audio_options[kind] + audio = build_audio( + source, template, cue_name=names[kind], tools=tools, options=options, cipher_type=cipher, key=key + ) + if kind == "full": + duration = audio.report["processed"]["samples"] * 1000 / audio.report["processed"]["sampleRate"] + if any(chart["lastNoteMs"] > duration for chart in plan.report["charts"].values()): + raise ValueError("chart has notes after the full audio ends") + audio_path = f"audio/{kind}.acb" + outputs[audio_path] = audio.data + outputs[f"audio/{kind}.json"] = _json(audio.report) + audio_reports[kind] = audio.report["validation"] + publish( + audio_bundle(templates[kind], audio.data, names[kind], f"{names[kind]}_audio"), templates[kind], audio_path + ) + outputs["catalog/catalog.bin"] = catalogbuild.rebuild(catalog, add=additions) + outputs["catalog/catalog.hash"] = fingerprint(outputs["catalog/catalog.bin"])[:32].encode("ascii") + originals = target.master_files() + key = target.session.master_key() + round_keys = master.round_keys(key.key) + for name in plan.report["master"]: + original = originals[name + ".bin"] + encoded = master.encode(_json(plan.tables[name]), key, original[: master.PREFIX], round_keys) + if json.loads(master.decode(encoded, key, round_keys)) != plan.tables[name]: + raise ValueError("master table did not preserve its assembled rows") + outputs[f"master/{name}.bin"] = encoded + version = project.document.get("version") or fingerprint(b"".join(outputs[name] for name in sorted(outputs)))[:32] + manifest = {"version": version, "files": []} + for name, original in sorted(originals.items()): + data = outputs.get("master/" + name, original) + manifest["files"].append({"name": name, "hash": fingerprint(data), "size": len(data)}) + outputs["master/MasterManifest.json"] = _json(manifest) + outputs["validation.json"] = _json( + { + "format": "nnnotes.package-validation/1", + "structure": "passed", + "audio": audio_reports, + "inputFingerprint": plan.report["inputFingerprint"], + } + ) + package = { + "format": "nnnotes.song-build/1", + "song": project.document["song"]["id"], + "version": version, + "baseline": {"catalogSha256": fingerprint(catalog)}, + "bundles": records, + "files": [ + {"path": name, "size": len(data), "sha256": fingerprint(data)} for name, data in sorted(outputs.items()) + ], + } + outputs["package.json"] = _json(package) + plan.check_inputs() + out.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=out.parent, prefix=".nnnotes-package-") as directory: + staging = Path(directory) / "package" + for name, data in outputs.items(): + path = staging / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + _verify(staging, target) + os.replace(staging, out) + return PackageResult(out, package) diff --git a/src/nnnotes/packing/plan.py b/src/nnnotes/packing/plan.py new file mode 100644 index 0000000..43247f6 --- /dev/null +++ b/src/nnnotes/packing/plan.py @@ -0,0 +1,213 @@ +"""Offline package planning, input fingerprints and structured diagnostics.""" + +from __future__ import annotations + +import gzip +from dataclasses import dataclass, field +from pathlib import Path + +from PIL import Image + +from ..content import score +from ..core.jsonio import dumps +from ..source import addressables +from ..source.config import Config, ConfigError +from . import assets +from .audio import AudioOptions, AudioTools, probe_audio +from .project import SongProject, Target, fingerprint +from .song import DIFFICULTIES, assemble, find + +__all__ = ["PackagePlan", "plan_package"] + + +@dataclass +class PackagePlan: + """Resolved local inputs and a public report. Errors prevent building.""" + + project: SongProject + report: dict + target: Target | None = field(default=None, repr=False) + tables: dict = field(default_factory=dict, repr=False) + names: dict = field(default_factory=dict, repr=False) + charts: dict = field(default_factory=dict, repr=False) + templates: dict = field(default_factory=dict, repr=False) + audio_options: dict = field(default_factory=dict, repr=False) + inputs: dict[Path, str] = field(default_factory=dict, repr=False) + + def raise_for_errors(self) -> None: + diagnostics = self.report["diagnostics"] + if diagnostics: + raise ValueError("; ".join(f"{item['path']}: {item['message']}" for item in diagnostics)) + + def read(self, path: Path) -> bytes: + data = path.read_bytes() + self.inputs[path] = fingerprint(data) + return data + + def check_inputs(self) -> None: + self.raise_for_errors() + for path, expected in self.inputs.items(): + if fingerprint(path.read_bytes()) != expected: + raise ValueError(f"build input changed: {path.name}") + if self.target: + self.target.check_inputs() + + +def plan_package( + project: SongProject | str | Path, *, config: Config | None = None, tools: AudioTools | None = None +) -> PackagePlan: + """Resolve a local song package without downloading or writing resource outputs.""" + project = project if isinstance(project, SongProject) else SongProject.load(project) + report = {"format": "nnnotes.package-plan/1", "diagnostics": [], "charts": {}, "assets": [], "master": []} + plan = PackagePlan(project, report) + doc = project.document + + def attempt(path, operation): + try: + return operation() + except (OSError, ValueError, KeyError, TypeError, RuntimeError, ConfigError) as error: + report["diagnostics"].append({"code": "invalid_input", "path": path, "message": str(error)}) + return None + + plan.read(project.path) + target = attempt("/target", lambda: Target.load(project.file(doc["target"]), config)) + if target is None: + return plan + plan.target = target + for difficulty in DIFFICULTIES: + + def load_chart(): + spec = doc["charts"][difficulty] + data = plan.read(project.file(spec["file"])) + if spec.get("input_format") == "gzip-json" and not data.startswith(b"\x1f\x8b"): + raise ValueError("chart input_format gzip-json requires a gzip payload") + if spec.get("input_format") == "json" and data.startswith(b"\x1f\x8b"): + raise ValueError("chart input_format json requires an uncompressed JSON payload") + root = score.load_bytes(data) + runtime = score.convert(root) + if runtime["log"]: + raise ValueError("chart conversion reported unresolved notes: " + "; ".join(runtime["log"])) + if not runtime["judgementNoteCount"]: + raise ValueError("chart must contain playable notes") + if not isinstance(spec["level"], int) or isinstance(spec["level"], bool) or spec["level"] <= 0: + raise ValueError("chart level must be a positive integer") + payload = data if data.startswith(b"\x1f\x8b") else gzip.compress(data, mtime=0) + plan.charts[difficulty] = payload + report["charts"][difficulty] = { + "notes": runtime["judgementNoteCount"], + "lastNoteMs": runtime["lastNoteTimeMs"], + "sha256": fingerprint(payload), + } + + attempt("/charts/" + difficulty, load_chart) + table_names = ( + "MasterLiveMusic", + "MasterLiveMusicScore", + "MasterText", + "MasterSound", + "MasterSoundCueSheet", + "MasterLiveScoreRank", + "MasterTag", + "MasterBand", + ) + baseline = attempt("/target/master", lambda: target.tables(table_names)) + if baseline is None or len(plan.charts) != len(DIFFICULTIES): + return plan + assembled = attempt("/song", lambda: assemble(baseline, doc, report["charts"])) + if assembled is None: + return plan + plan.tables, plan.names = assembled + report["song"] = doc["song"]["id"] + report["master"] = [name for name in plan.tables if plan.tables[name] != baseline[name]] + report["masterChanges"] = {} + for name in report["master"]: + original_ids = {row["_id"] for row in baseline[name]["_allData"]} + report["masterChanges"][name] = { + "addedIds": [row["_id"] for row in plan.tables[name]["_allData"] if row["_id"] not in original_ids] + } + report["names"] = plan.names + music = find(baseline, "MasterLiveMusic", doc["template_song"]) + chart_row = find(baseline, "MasterLiveMusicScore", music["_easyID"]) + selectors: dict[str, tuple[str, str | None]] = { + "chart": ("Live/MusicScore/" + chart_row["_musicScoreTextFileName"], "UnityEngine.TextAsset"), + "jacket": ("Image/Jacket/" + music["_jacketAssetName"], "UnityEngine.Sprite"), + "jacket_small": ("Image/Jacket/small/" + music["_jacketAssetName"], "UnityEngine.Sprite"), + } + for kind, sound_field in (("full", "_musicSoundID"), ("preview", "_jingleSoundID")): + sound = find(baseline, "MasterSound", music[sound_field]) + sheet = find(baseline, "MasterSoundCueSheet", sound["_soundCueSheetID"]) + selectors[kind] = ("Cri/Sound/" + sheet["_cueSheetName"], None) + for kind, (key, resource_type) in selectors.items(): + template = attempt("/target/templates/" + kind, lambda: assets.select_template(target, key, resource_type)) + if template: + plan.templates[kind] = template + transport = attempt( + "/target/providers/" + kind, lambda selected=template: target.transport(selected.bundle["provider"]) + ) + report.setdefault("templates", {})[kind] = { + "asset": key, + "provider": template.bundle["provider"], + "transport": transport, + "dependencies": template.dependencies, + } + + def jacket(): + path = project.file(doc["assets"]["jacket"]) + plan.read(path) + with Image.open(path) as image: + if image.format != "PNG" or image.width != image.height: + raise ValueError("jacket must be a square PNG") + + attempt("/assets/jacket", jacket) + audio = doc["assets"]["audio"] + for kind in ("full", "preview"): + + def audio_input(): + spec = audio[kind] + if kind == "preview" and spec.get("from") == "full": + if "file" in spec or "start_ms" not in spec or "duration_ms" not in spec: + raise ValueError("preview from full requires start_ms/duration_ms and no file") + source = audio["full"]["file"] + else: + if "from" in spec: + raise ValueError("audio from is supported only for preview from full") + source = spec["file"] + path = project.file(source) + plan.read(path) + options = AudioOptions( + **{ + **audio.get("processing", {}), + **{key: value for key, value in spec.items() if key not in ("file", "from")}, + } + ) + selected_tools = tools or AudioTools.configured(target.session.cfg) + report["tools"] = selected_tools.fingerprints() + info = probe_audio(path, selected_tools, stream=options.stream) + target.audio(kind) + plan.audio_options[kind] = (path, options) + report.setdefault("audio", {})[kind] = {"source": info, "options": options.__dict__} + + attempt("/assets/audio/" + kind, audio_input) + existing = {item["primary_key"] for item in addressables.parse_locations(target.catalog())} + keys = [ + "Image/Jacket/" + plan.names["jacket"], + "Image/Jacket/small/" + plan.names["jacket"], + "Cri/Sound/" + plan.names["full"], + "Cri/Sound/" + plan.names["preview"], + *["Live/MusicScore/" + name for name in plan.names["charts"].values()], + ] + for key in keys: + if key in existing: + report["diagnostics"].append( + {"code": "key_conflict", "path": "/song/id", "message": f"resource key exists: {key}"} + ) + report["assets"] = keys + report["inputFingerprint"] = fingerprint( + dumps( + { + "project": fingerprint(project.path.read_bytes()), + "inputs": sorted([*plan.inputs.values(), *target.inputs.values()]), + } + ).encode() + ) + return plan diff --git a/src/nnnotes/packing/project.py b/src/nnnotes/packing/project.py new file mode 100644 index 0000000..9c5e43c --- /dev/null +++ b/src/nnnotes/packing/project.py @@ -0,0 +1,157 @@ +"""Versioned local song projects and target resource sets.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from jsonschema import Draft202012Validator + +from .. import schemas +from ..source import addressables, master +from ..source.config import Config +from ..source.session import Session + +__all__ = ["SongProject", "Target", "fingerprint"] + + +def fingerprint(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _document(path: Path, format: str) -> dict: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict) or value.get("format") != format: + raise ValueError(f"expected {format} document") + schema = "song-package" if format == "nnnotes.song-package/1" else "pack-target" + errors = sorted(Draft202012Validator(schemas.load(schema)).iter_errors(value), key=lambda error: str(error.path)) + if errors: + error = errors[0] + pointer = "/" + "/".join(str(part) for part in error.path) + raise ValueError(f"{pointer}: {error.message}") + return value + + +@dataclass +class SongProject: + """A song manifest. All relative paths resolve against the manifest's directory.""" + + path: Path + document: dict[str, Any] + + @classmethod + def load(cls, path: str | Path) -> SongProject: + path = Path(path).resolve() + return cls(path, _document(path, "nnnotes.song-package/1")) + + def file(self, value: str) -> Path: + return (self.path.parent / value).resolve() + + +@dataclass +class Target: + """An offline catalog, master set, local bundle directory and audio compatibility profile.""" + + path: Path + document: dict[str, Any] + session: Session = field(repr=False) + inputs: dict[Path, str] = field(default_factory=dict, repr=False) + _audio_key: int | None = field(default=None, repr=False) + + @classmethod + def load(cls, path: str | Path, config: Config | None = None) -> Target: + path = Path(path).resolve() + result = cls(path, _document(path, "nnnotes.pack-target/1"), Session(config or Config.load())) + result.read(path) + return result + + def file(self, value: str) -> Path: + return (self.path.parent / value).resolve() + + def read(self, path: Path) -> bytes: + path = path.resolve() + data = path.read_bytes() + digest = fingerprint(data) + if path in self.inputs and self.inputs[path] != digest: + raise ValueError(f"build input changed: {path.name}") + self.inputs[path] = digest + return data + + def catalog(self) -> bytes: + return self.read(self.file(self.document["catalog"])) + + def master_files(self) -> dict[str, bytes]: + root = self.file(self.document["master"]) + files = {} + for path in sorted(root.glob("Master*.bin")): + files[path.name] = self.read(path) + if not files: + raise ValueError("target master directory must contain encoded Master*.bin files") + manifest = root / "MasterManifest.json" + if not manifest.is_file(): + raise ValueError("target master directory requires MasterManifest.json") + doc = json.loads(self.read(manifest)) + for item in doc["files"]: + name = item["name"] + if name != Path(name).name or name not in files: + raise ValueError("master manifest names a missing or invalid table file") + if len(files[name]) != item["size"] or fingerprint(files[name]) != item["hash"]: + raise ValueError(f"target master integrity mismatch: {name}") + if {item["name"] for item in doc["files"]} != set(files): + raise ValueError("target master manifest must cover every table") + return files + + def tables(self, names: tuple[str, ...] | None = None) -> dict[str, dict]: + key = self.session.master_key() + return { + name[:-4]: json.loads(master.decode(data, key)) + for name, data in self.master_files().items() + if names is None or name[:-4] in names + } + + def bundle(self, location: dict) -> bytes: + name = location["internal_id"].rsplit("/", 1)[-1] + if name != Path(name).name or name in ("", ".", ".."): + raise ValueError("bundle internal ID must end with a file name") + root = self.file(self.document["bundles"]) + data = self.read(root / name) + if not data.startswith(b"UnityFS\0"): + data = addressables.decrypt(data, name, self.session.bundle_key()) + if not data.startswith(b"UnityFS\0"): + raise ValueError(f"local template is not a readable UnityFS bundle: {name}") + return data + + def transport(self, provider: str) -> str: + from ..source.catalogbuild import BUNDLE_PROVIDER + + value = self.document.get("providers", {}).get(provider, "plain" if provider == BUNDLE_PROVIDER else None) + if value not in ("plain", "encrypted"): + raise ValueError("target providers must explicitly map this bundle provider to plain or encrypted") + return value + + def audio(self, kind: str) -> tuple[bytes, int, int | None]: + from ..formats import crikey + + profile = self.document["audio"] + cipher = profile["cipher_type"] + if cipher not in (0, 56): + raise ValueError("audio cipher_type must be 0 or 56") + key = None + if cipher == 56: + apk = self.session.required_path("paths", "apk") + if self._audio_key is None: + if apk.is_file(): + self.read(apk) + self._audio_key = crikey.find_key(apk) + key = self._audio_key + if not key: + raise ValueError("target APK has no enabled HCA key") + return self.read(self.file(profile["templates"][kind])), cipher, key + + def check_inputs(self) -> None: + for path, expected in self.inputs.items(): + if fingerprint(path.read_bytes()) != expected: + raise ValueError(f"build input changed: {path.name}") diff --git a/src/nnnotes/packing/song.py b/src/nnnotes/packing/song.py new file mode 100644 index 0000000..97fa144 --- /dev/null +++ b/src/nnnotes/packing/song.py @@ -0,0 +1,154 @@ +"""Assemble one song's master rows and localized metadata with explicit resource identities.""" + +from __future__ import annotations + +import copy + +from ..source.languages import LANGUAGES + +DIFFICULTIES = ("easy", "normal", "hard", "expert") + + +def find(tables: dict, name: str, identity): + matches = [row for row in tables[name]["_allData"] if row["_id"] == identity] + if len(matches) != 1: + raise ValueError(f"{name}: expected one row for ID {identity}") + return matches[0] + + +def append(tables: dict, name: str, row: dict) -> None: + rows = tables[name]["_allData"] + if any(item["_id"] == row["_id"] for item in rows): + raise ValueError(f"{name}: new ID conflicts with an existing row") + rows.append(row) + + +def assemble(tables: dict, project: dict, chart_stats: dict) -> tuple[dict, dict]: + tables = copy.deepcopy(tables) + song = project["song"] + song_id = song["id"] + if isinstance(song_id, bool) or not isinstance(song_id, int) or not 0 < song_id <= (2**31 - 101) // 100: + raise ValueError("song.id must allow its derived IDs to fit in a signed 32-bit integer") + source = find(tables, "MasterLiveMusic", project["template_song"]) + row = copy.deepcopy(source) + prefix = f"custom_{song_id}" + full, preview = f"M_{prefix}", f"M_{prefix}_short" + fallback = song.get("fallback_language") + if fallback is not None and fallback not in LANGUAGES: + raise ValueError("song.fallback_language is not a supported language") + + def text(field: str, value) -> str: + if isinstance(value, str): + value = {language: value for language in LANGUAGES} + if not isinstance(value, dict) or set(value) - set(LANGUAGES): + raise ValueError(f"song.{field}: expected text or a mapping of supported languages") + identity = f"Custom_{song_id}_{field}" + record = {"_id": identity} + for language, (_, column) in LANGUAGES.items(): + record[column] = value.get(language, value.get(fallback, "") if fallback else "") + if not isinstance(record[column], str): + raise ValueError(f"song.{field}: localized text must be a string") + append(tables, "MasterText", record) + return identity + + credits = song.get("credits", {}) + display = { + "title": song["title"], + "phonetic": song.get("phonetic", song["title"]), + "rubyTitle": song.get("ruby_title", song["title"]), + "bandName": song.get("performer", ""), + "lyricist": credits.get("lyricist", ""), + "composer": credits.get("composer", ""), + "arranger": credits.get("arranger", ""), + } + fields = { + "_titleTextID": "title", + "_phoneticTextID": "phonetic", + "_rubyTitleTextID": "rubyTitle", + "_bandNameTextID": "bandName", + "_lyricistTextID": "lyricist", + "_composerTextID": "composer", + "_arrangerTextID": "arranger", + } + for field, name in fields.items(): + if field in row: + row[field] = text(name, display[name]) + for field in row: + if field.endswith("TextID") and field not in fields: + raise ValueError(f"unsupported song text field: {field}") + bands = song.get("bands", []) + if not isinstance(bands, list) or any(isinstance(value, bool) or not isinstance(value, int) for value in bands): + raise ValueError("song.bands must be an array of band IDs") + for band in bands: + find(tables, "MasterBand", band) + tags = [] + if not isinstance(song.get("tags", []), list) or len(song.get("tags", [])) > 100: + raise ValueError("song.tags must be an array with at most 100 entries") + for index, tag in enumerate(song.get("tags", [])): + identity = song_id * 100 + index + append(tables, "MasterTag", {"_id": identity, "_nameTextID": text(f"tag{index}", tag)}) + tags.append(identity) + row.update( + _id=song_id, + _sortOrder=song.get("sort_order", song_id), + _bandIDs=bands, + _bestMusicTagIDs=tags, + _vocalCharacterIDs=[], + _musicVideoIDs=[], + _anotherVocalIDs=[], + _jacketAssetName=prefix, + _defaultUnlock=True, + _unlockConditionID=0, + _startAt=song.get("start_at", "2000/01/01 0:00:00"), + ) + names = {"jacket": prefix, "full": full, "preview": preview, "charts": {}} + for i, difficulty in enumerate(DIFFICULTIES): + spec = project["charts"][difficulty] + identity = spec["score_id"] + if isinstance(identity, bool) or not isinstance(identity, int) or not 0 < identity < 2**31: + raise ValueError("chart score_id must be a positive signed 32-bit integer") + score = copy.deepcopy(find(tables, "MasterLiveMusicScore", source[f"_{difficulty}ID"])) + name = f"{prefix}/{prefix}_{i:02}" + names["charts"][difficulty] = name + score.update( + _id=identity, + _musicScoreTextFileName=name, + _musicScoreLevel=spec["level"], + _musicScoreDisplayLevel=spec.get("display_level", float(spec["level"])), + _fullComboCount=chart_stats[difficulty]["notes"], + ) + append(tables, "MasterLiveMusicScore", score) + row[f"_{difficulty}ID"] = identity + group = song_id + if any(item["_group"] == group for item in tables["MasterLiveScoreRank"]["_allData"]): + raise ValueError("new score rank group conflicts with an existing group") + ranks = [ + item for item in tables["MasterLiveScoreRank"]["_allData"] if item["_group"] == source["_liveScoreRankGroup"] + ] + if not ranks: + raise ValueError("template song has no score rank group") + for index, item in enumerate(ranks): + append(tables, "MasterLiveScoreRank", {**item, "_id": song_id * 100 + index, "_group": group}) + row["_liveScoreRankGroup"] = group + for index, (field, sheet) in enumerate((("_musicSoundID", full), ("_jingleSoundID", preview))): + sound = copy.deepcopy(find(tables, "MasterSound", source[field])) + identity = song_id * 100 + 90 + index + sound.update(_id=identity, _soundCueSheetID=identity, _cueName=sheet) + for pitch_field, value in (("_isRandomPitch", False), ("_minPitch", 0.0), ("_maxPitch", 0.0)): + if pitch_field in sound: + sound[pitch_field] = value + append(tables, "MasterSound", sound) + append(tables, "MasterSoundCueSheet", {"_id": identity, "_cueSheetName": sheet}) + row[field] = identity + for field, value in song.get("fields", {}).items(): + if ( + field not in source + or field in fields + or field in ("_id", "_jacketAssetName", "_liveScoreRankGroup") + or field in row + and (field.endswith("ID") or field.endswith("IDs")) + ): + raise ValueError("song.fields may only override existing presentation/behavior fields") + row[field] = value + append(tables, "MasterLiveMusic", row) + return tables, names diff --git a/src/nnnotes/schemas/audio-build.schema.json b/src/nnnotes/schemas/audio-build.schema.json new file mode 100644 index 0000000..a5735ea --- /dev/null +++ b/src/nnnotes/schemas/audio-build.schema.json @@ -0,0 +1,52 @@ +{ + "title": "nnnotes.audio-build/1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:nnnotes:schema:audio-build", + "type": "object", + "required": [ + "format", + "source", + "sourceSha256", + "options", + "tools", + "processed", + "hca", + "bank", + "validation", + "acbSha256" + ], + "properties": { + "format": { + "const": "nnnotes.audio-build/1" + }, + "source": { + "type": "object" + }, + "sourceSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "options": { + "type": "object" + }, + "tools": { + "type": "object" + }, + "processed": { + "type": "object" + }, + "hca": { + "type": "object" + }, + "bank": { + "type": "object" + }, + "validation": { + "type": "object" + }, + "acbSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } +} diff --git a/src/nnnotes/schemas/pack-target.schema.json b/src/nnnotes/schemas/pack-target.schema.json new file mode 100644 index 0000000..f7692e7 --- /dev/null +++ b/src/nnnotes/schemas/pack-target.schema.json @@ -0,0 +1,75 @@ +{ + "title": "nnnotes.pack-target/1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:nnnotes:schema:pack-target", + "type": "object", + "properties": { + "format": { + "const": "nnnotes.pack-target/1" + }, + "catalog": { + "type": "string", + "minLength": 1 + }, + "master": { + "type": "string", + "minLength": 1 + }, + "bundles": { + "type": "string", + "minLength": 1 + }, + "bundle_base_url": { + "type": "string", + "minLength": 1 + }, + "providers": { + "type": "object", + "additionalProperties": { + "enum": [ + "plain", + "encrypted" + ] + } + }, + "audio": { + "type": "object", + "properties": { + "cipher_type": { + "enum": [ + 0, + 56 + ] + }, + "templates": { + "type": "object", + "properties": { + "full": { + "type": "string", + "minLength": 1 + }, + "preview": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "full", + "preview" + ], + "additionalProperties": false + } + }, + "required": [ + "cipher_type", + "templates" + ], + "additionalProperties": false + } + }, + "required": [ + "format", + "audio" + ], + "additionalProperties": false +} diff --git a/src/nnnotes/schemas/package-plan.schema.json b/src/nnnotes/schemas/package-plan.schema.json new file mode 100644 index 0000000..7cfe02c --- /dev/null +++ b/src/nnnotes/schemas/package-plan.schema.json @@ -0,0 +1,56 @@ +{ + "title": "nnnotes.package-plan/1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:nnnotes:schema:package-plan", + "type": "object", + "required": [ + "format", + "diagnostics", + "charts", + "assets", + "master" + ], + "properties": { + "format": { + "const": "nnnotes.package-plan/1" + }, + "diagnostics": { + "type": "array", + "items": { + "type": "object", + "required": [ + "code", + "path", + "message" + ], + "properties": { + "code": { + "type": "string" + }, + "path": { + "type": "string" + }, + "message": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "charts": { + "type": "object" + }, + "assets": { + "type": "array", + "items": { + "type": "string" + } + }, + "master": { + "type": "array", + "items": { + "type": "string" + } + } + } +} diff --git a/src/nnnotes/schemas/package-validation.schema.json b/src/nnnotes/schemas/package-validation.schema.json new file mode 100644 index 0000000..ebebe50 --- /dev/null +++ b/src/nnnotes/schemas/package-validation.schema.json @@ -0,0 +1,40 @@ +{ + "title": "nnnotes.package-validation/1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:nnnotes:schema:package-validation", + "type": "object", + "required": [ + "format" + ], + "properties": { + "format": { + "const": "nnnotes.package-validation/1" + } + }, + "anyOf": [ + { + "required": [ + "status" + ], + "properties": { + "status": { + "const": "passed" + } + } + }, + { + "required": [ + "structure", + "audio" + ], + "properties": { + "structure": { + "const": "passed" + }, + "audio": { + "type": "object" + } + } + } + ] +} diff --git a/src/nnnotes/schemas/song-build.schema.json b/src/nnnotes/schemas/song-build.schema.json new file mode 100644 index 0000000..24bebd3 --- /dev/null +++ b/src/nnnotes/schemas/song-build.schema.json @@ -0,0 +1,60 @@ +{ + "title": "nnnotes.song-build/1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:nnnotes:schema:song-build", + "type": "object", + "required": [ + "format", + "song", + "version", + "baseline", + "bundles", + "files" + ], + "properties": { + "format": { + "const": "nnnotes.song-build/1" + }, + "song": { + "type": "integer", + "minimum": 1 + }, + "version": { + "type": "string" + }, + "baseline": { + "type": "object" + }, + "bundles": { + "type": "array", + "items": { + "type": "object" + } + }, + "files": { + "type": "array", + "items": { + "type": "object", + "required": [ + "path", + "size", + "sha256" + ], + "properties": { + "path": { + "type": "string" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + }, + "additionalProperties": false + } + } + } +} diff --git a/src/nnnotes/schemas/song-package.schema.json b/src/nnnotes/schemas/song-package.schema.json new file mode 100644 index 0000000..3cdb1bf --- /dev/null +++ b/src/nnnotes/schemas/song-package.schema.json @@ -0,0 +1,633 @@ +{ + "title": "nnnotes.song-package/1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:nnnotes:schema:song-package", + "type": "object", + "properties": { + "format": { + "const": "nnnotes.song-package/1" + }, + "target": { + "type": "string", + "minLength": 1 + }, + "template_song": { + "type": "integer", + "minimum": 1, + "maximum": 2147483647 + }, + "version": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{1,128}$" + }, + "song": { + "type": "object", + "properties": { + "id": { + "type": "integer", + "minimum": 1, + "maximum": 2147483647 + }, + "title": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "propertyNames": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] + }, + "additionalProperties": { + "type": "string" + } + } + ] + }, + "phonetic": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "propertyNames": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] + }, + "additionalProperties": { + "type": "string" + } + } + ] + }, + "ruby_title": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "propertyNames": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] + }, + "additionalProperties": { + "type": "string" + } + } + ] + }, + "performer": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "propertyNames": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] + }, + "additionalProperties": { + "type": "string" + } + } + ] + }, + "fallback_language": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] + }, + "credits": { + "type": "object", + "properties": { + "lyricist": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "propertyNames": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] + }, + "additionalProperties": { + "type": "string" + } + } + ] + }, + "composer": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "propertyNames": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] + }, + "additionalProperties": { + "type": "string" + } + } + ] + }, + "arranger": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "propertyNames": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] + }, + "additionalProperties": { + "type": "string" + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "bands": { + "type": "array", + "items": { + "type": "integer", + "minimum": 1, + "maximum": 2147483647 + }, + "uniqueItems": true + }, + "tags": { + "type": "array", + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "propertyNames": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] + }, + "additionalProperties": { + "type": "string" + } + } + ] + }, + "maxItems": 100 + }, + "sort_order": { + "type": "integer", + "minimum": 1, + "maximum": 2147483647 + }, + "start_at": { + "type": "string", + "minLength": 1 + }, + "fields": { + "type": "object" + } + }, + "required": [ + "id", + "title" + ], + "additionalProperties": false + }, + "assets": { + "type": "object", + "properties": { + "jacket": { + "type": "string", + "minLength": 1 + }, + "audio": { + "type": "object", + "properties": { + "full": { + "type": "object", + "properties": { + "file": { + "type": "string", + "minLength": 1 + }, + "sample_rate": { + "enum": [ + 44100, + 48000 + ] + }, + "channels": { + "enum": [ + 1, + 2 + ] + }, + "stream": { + "type": "integer", + "minimum": 0 + }, + "gain_db": { + "type": "number", + "minimum": -120, + "maximum": 24 + }, + "start_ms": { + "type": "integer", + "minimum": 0 + }, + "duration_ms": { + "type": "integer", + "minimum": 1 + }, + "fade_in_ms": { + "type": "integer", + "minimum": 0 + }, + "fade_out_ms": { + "type": "integer", + "minimum": 0 + }, + "quality": { + "enum": [ + "highest", + "high", + "middle", + "low", + "lowest" + ] + } + }, + "required": [ + "file" + ], + "additionalProperties": false + }, + "preview": { + "type": "object", + "properties": { + "file": { + "type": "string", + "minLength": 1 + }, + "from": { + "const": "full" + }, + "sample_rate": { + "enum": [ + 44100, + 48000 + ] + }, + "channels": { + "enum": [ + 1, + 2 + ] + }, + "stream": { + "type": "integer", + "minimum": 0 + }, + "gain_db": { + "type": "number", + "minimum": -120, + "maximum": 24 + }, + "start_ms": { + "type": "integer", + "minimum": 0 + }, + "duration_ms": { + "type": "integer", + "minimum": 1 + }, + "fade_in_ms": { + "type": "integer", + "minimum": 0 + }, + "fade_out_ms": { + "type": "integer", + "minimum": 0 + }, + "quality": { + "enum": [ + "highest", + "high", + "middle", + "low", + "lowest" + ] + } + }, + "required": [], + "additionalProperties": false, + "oneOf": [ + { + "required": [ + "file" + ], + "not": { + "required": [ + "from" + ] + } + }, + { + "required": [ + "from", + "start_ms", + "duration_ms" + ], + "not": { + "required": [ + "file" + ] + } + } + ] + }, + "processing": { + "type": "object", + "properties": { + "sample_rate": { + "enum": [ + 44100, + 48000 + ] + }, + "channels": { + "enum": [ + 1, + 2 + ] + }, + "stream": { + "type": "integer", + "minimum": 0 + }, + "gain_db": { + "type": "number", + "minimum": -120, + "maximum": 24 + }, + "start_ms": { + "type": "integer", + "minimum": 0 + }, + "duration_ms": { + "type": "integer", + "minimum": 1 + }, + "fade_in_ms": { + "type": "integer", + "minimum": 0 + }, + "fade_out_ms": { + "type": "integer", + "minimum": 0 + }, + "quality": { + "enum": [ + "highest", + "high", + "middle", + "low", + "lowest" + ] + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [ + "full", + "preview" + ], + "additionalProperties": false + } + }, + "required": [ + "jacket", + "audio" + ], + "additionalProperties": false + }, + "charts": { + "type": "object", + "properties": { + "easy": { + "type": "object", + "properties": { + "file": { + "type": "string", + "minLength": 1 + }, + "score_id": { + "type": "integer", + "minimum": 1, + "maximum": 2147483647 + }, + "level": { + "type": "integer", + "minimum": 1 + }, + "display_level": { + "type": "number", + "exclusiveMinimum": 0 + }, + "input_format": { + "enum": [ + "json", + "gzip-json" + ] + } + }, + "required": [ + "file", + "score_id", + "level" + ], + "additionalProperties": false + }, + "normal": { + "type": "object", + "properties": { + "file": { + "type": "string", + "minLength": 1 + }, + "score_id": { + "type": "integer", + "minimum": 1, + "maximum": 2147483647 + }, + "level": { + "type": "integer", + "minimum": 1 + }, + "display_level": { + "type": "number", + "exclusiveMinimum": 0 + }, + "input_format": { + "enum": [ + "json", + "gzip-json" + ] + } + }, + "required": [ + "file", + "score_id", + "level" + ], + "additionalProperties": false + }, + "hard": { + "type": "object", + "properties": { + "file": { + "type": "string", + "minLength": 1 + }, + "score_id": { + "type": "integer", + "minimum": 1, + "maximum": 2147483647 + }, + "level": { + "type": "integer", + "minimum": 1 + }, + "display_level": { + "type": "number", + "exclusiveMinimum": 0 + }, + "input_format": { + "enum": [ + "json", + "gzip-json" + ] + } + }, + "required": [ + "file", + "score_id", + "level" + ], + "additionalProperties": false + }, + "expert": { + "type": "object", + "properties": { + "file": { + "type": "string", + "minLength": 1 + }, + "score_id": { + "type": "integer", + "minimum": 1, + "maximum": 2147483647 + }, + "level": { + "type": "integer", + "minimum": 1 + }, + "display_level": { + "type": "number", + "exclusiveMinimum": 0 + }, + "input_format": { + "enum": [ + "json", + "gzip-json" + ] + } + }, + "required": [ + "file", + "score_id", + "level" + ], + "additionalProperties": false + } + }, + "required": [ + "easy", + "normal", + "hard", + "expert" + ], + "additionalProperties": false + } + }, + "required": [ + "format", + "target", + "template_song", + "song", + "assets", + "charts" + ], + "additionalProperties": false +} diff --git a/src/nnnotes/source/addressables.py b/src/nnnotes/source/addressables.py index bac959b..1fdee97 100644 --- a/src/nnnotes/source/addressables.py +++ b/src/nnnotes/source/addressables.py @@ -10,7 +10,8 @@ Asset bundles on the CDN and inside the APK are encrypted in their first 16 KiB with AES-128 in CTR mode: the nonce is the first 8 bytes of SHA-256(nonce seed + UTF-8 bundle file name), the counter block is nonce + 64-bit big-endian counter starting at 0. A file that already starts with the `UnityFS` signature is not encrypted. The key and the -nonce seed come from the configuration (`[bundle] key`, `[bundle] nonce_seed`). +nonce seed come from the configuration (`[bundle] key`, `[bundle] nonce_seed`). `pack` applies the same transform in +the other direction, for a bundle of our own that has to look like the CDN's. """ from __future__ import annotations @@ -26,7 +27,7 @@ from Crypto.Cipher import AES -__all__ = ["BundleKey", "decrypt", "parse", "parse_header", "parse_keys", "parse_locations"] +__all__ = ["BundleKey", "decrypt", "pack", "parse", "parse_header", "parse_keys", "parse_locations"] # a location of a catalog: {offset, primary_key, internal_id, dependencies} (parse), with the fields parse_locations # decodes @@ -68,13 +69,29 @@ def __post_init__(self) -> None: raise ValueError("bundle key must be 16 bytes") +def _ctr(data: bytes, filename: str, key: BundleKey) -> bytes: + """AES-128-CTR over the first `ENCRYPTED_BYTES` of a bundle; the transform is its own inverse.""" + nonce = hashlib.sha256(key.seed + filename.encode("utf-8")).digest()[:8] + aes = AES.new(key.key, AES.MODE_CTR, nonce=nonce, initial_value=0) # pyright: ignore[reportUnknownMemberType] (pycryptodome stubs: untyped overloads) + return aes.decrypt(data[:ENCRYPTED_BYTES]) + data[ENCRYPTED_BYTES:] + + def decrypt(data: bytes, filename: str, key: BundleKey) -> bytes: """A bundle file as stored -> UnityFS bytes. `filename`: the bundle's bare file name (it seeds the nonce).""" if data.startswith(UNITYFS): return data - nonce = hashlib.sha256(key.seed + filename.encode("utf-8")).digest()[:8] - aes = AES.new(key.key, AES.MODE_CTR, nonce=nonce, initial_value=0) # pyright: ignore[reportUnknownMemberType] (pycryptodome stubs: untyped overloads) - return aes.decrypt(data[:ENCRYPTED_BYTES]) + data[ENCRYPTED_BYTES:] + return _ctr(data, filename, key) + + +def pack(plain: bytes, filename: str, key: BundleKey) -> bytes: + """UnityFS bytes -> a bundle file as stored: the inverse of `decrypt`. + + `filename` must be the name the bundle is served under — it seeds the nonce, so a file packed as `a.bundle` and + served as `b.bundle` does not decrypt to the same bytes. + """ + if not plain.startswith(UNITYFS): + raise ValueError("not a UnityFS bundle") + return _ctr(plain, filename, key) def relative_path(value: str) -> str: @@ -353,6 +370,7 @@ def parse_locations(data: bytes) -> list[Location]: "provider": buf.string(provider, "."), "dependency_hash": dep_hash, "resource_type": rtype[1] if rtype else None, + "resource_type_assembly": rtype[0] if rtype else None, "extra_data": buf.extra(extra), } ) diff --git a/src/nnnotes/source/catalogbuild.py b/src/nnnotes/source/catalogbuild.py new file mode 100644 index 0000000..f890dd6 --- /dev/null +++ b/src/nnnotes/source/catalogbuild.py @@ -0,0 +1,571 @@ +"""Write binary Addressables catalogs and generate locations for bundles and their assets. + +`rebuild` preserves the header, typed key table and location dependencies while adding, removing or updating +locations. `bundle_entries` generates a bundle location with request options and asset locations that depend on it. +Request option CRCs describe the decompressed UnityFS content; bundle sizes describe the stored file. +""" + +from __future__ import annotations + +import struct +from collections.abc import Iterable, Sequence +from typing import Any + +from . import addressables + +__all__ = ["bundle_entries", "bundle_settings", "edit_entries", "rebuild"] + +RM = "UnityEngine.ResourceManagement.ResourceProviders." +RM_ASSEMBLY = "Unity.ResourceManager, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null" +CORE_ASSEMBLY = "UnityEngine.CoreModule, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null" +LIB_ASSEMBLY = "mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" +BUNDLE_PROVIDER = RM + "AssetBundleProvider" +ASSET_PROVIDER = RM + "BundledAssetProvider" +BUNDLE_TYPE = RM + "IAssetBundleResource" +NONE = addressables.NONE +HEADER = addressables.HEADER +LOCATION = addressables.LOCATION +REQUEST_OPTIONS = addressables.REQUEST_OPTIONS +FLAG_BITS = addressables.FLAG_BITS + + +class _Writer: + """The catalog buffer: 32-byte header, then every value at an offset (arrays and strings carry their length).""" + + def __init__(self) -> None: + self.buf = bytearray(HEADER.size) + self._types: dict[tuple[str, str], int] = {} + self._commons: dict[tuple[int, ...], int] = {} + # the same id, provider or type name repeats in thousands of locations: write each value once + self._texts: dict[str, int] = {} + self._parts: dict[tuple[str, str], int] = {} + self._arrays: dict[tuple[int, ...], int] = {} + + def blob(self, data: bytes) -> int: + self.buf += struct.pack(" int: + off = len(self.buf) + self.buf += data + return off + + def reserve(self, size: int) -> int: + return self.value(bytes(size)) + + def text(self, s: str) -> int: + if s in self._texts: + return self._texts[s] + try: + off = self.blob(s.encode("ascii")) + except UnicodeEncodeError: + off = self.blob(s.encode("utf-16-le")) | 0x80000000 + self._texts[s] = off + return off + + def parts(self, s: str, sep: str) -> int: + key = (s, sep) + if key in self._parts: + return self._parts[key] + head = NONE + for part in s.split(sep): + head = self.value(struct.pack(" int: + if s is None: + return NONE + return self.parts(s, sep) if sep and sep in s else self.text(s) + + def array(self, values: Iterable[int]) -> int: + values = tuple(values) + if not values: + return NONE + if values in self._arrays: + return self._arrays[values] + off = self.blob(struct.pack(f"<{len(values)}I", *values)) + self._arrays[values] = off + return off + + def type_data(self, cls: str, assembly: str = RM_ASSEMBLY) -> int: + key = (cls, assembly) + if key not in self._types: + self._types[key] = self.value(struct.pack(" int: + """A TypeSerializer.Data for `cls`; `assembly` is what the source catalog said, else a guess.""" + if not cls: + return NONE + if not assembly: + assembly = CORE_ASSEMBLY if cls.startswith("UnityEngine.") and not cls.startswith(RM) else RM_ASSEMBLY + return self.type_data(cls, assembly) + + def object_init(self, init: dict[str, Any] | None) -> int: + if not init: + return NONE + return self.value( + struct.pack( + " int: + cls = cls or "System.String" + if cls == "System.Int32": + obj = self.value(struct.pack(" int: + common = ( + extra.get("timeout"), + extra.get("redirectLimit"), + extra.get("retryCount"), + extra.get("flags"), + ) + common_id = NONE + if any(v is not None for v in common): + values = ( + _integer(common[0] or 0, "bundle timeout", -(2**15), 2**15 - 1), + _integer(common[1] or 0, "bundle redirectLimit", 0, 255), + _integer(common[2] or 0, "bundle retryCount", 0, 255), + _integer(common[3] or 0, "bundle flags", -(2**31), 2**31 - 1), + ) + if values not in self._commons: + self._commons[values] = self.value(struct.pack(" int: + if not isinstance(value, int) or isinstance(value, bool) or not low <= value <= high: + raise ValueError(f"{name} must be an integer from {low} to {high}") + return value + + +def _hash_bytes(value: str) -> bytes: + if len(value) != 32: + raise ValueError("bundle hash must contain 32 hexadecimal digits") + try: + raw = bytes.fromhex(value) + except ValueError: + raise ValueError("bundle hash must contain 32 hexadecimal digits") from None + if len(raw) != 16: + raise ValueError("bundle hash must contain 32 hexadecimal digits") + return raw + + +def _entry(extra: dict[str, Any] | None) -> dict[str, Any]: + """The AssetBundleRequestOptions fields a parsed location carries, in the writer's terms.""" + if not extra: + return {} + if extra.get("type") not in (None, REQUEST_OPTIONS): + raise ValueError(f"unsupported catalog extra data type: {extra['type']}") + return { + "hash": extra.get("hash"), + "bundleName": extra.get("bundleName"), + "crc": extra.get("crc"), + "bundleSize": extra.get("bundleSize"), + "timeout": extra.get("timeout"), + "redirectLimit": extra.get("redirectLimit"), + "retryCount": extra.get("retryCount"), + "flags": extra.get("flags"), + } + + +def _provider(value: Any) -> str: + if not isinstance(value, str) or not value or "\0" in value: + raise ValueError("bundle_provider must be a nonempty string without null characters") + return value + + +def bundle_settings( + data: bytes | None = None, + *, + provider: str | None = None, + template: str | None = None, + request_options: dict[str, Any] | None = None, +) -> tuple[str, dict[str, Any]]: + """Select a bundle provider and common request options from a source catalog. + + An explicit provider takes precedence. Otherwise, a selected bundle location supplies its provider, or the + source catalog's unique bundle provider is used. Empty catalogs use the standard AssetBundleProvider. + `template` selects one bundle by primary key or full internal ID and also supplies its common request options; + explicit request options override those defaults. A catalog with multiple bundle providers needs a selection. + """ + if provider is not None: + provider = _provider(provider) + locations = ( + [loc for loc in addressables.parse_locations(data) if loc.get("resource_type") == BUNDLE_TYPE] + if data is not None + else [] + ) + defaults: dict[str, Any] = {} + if template is not None: + matches = [loc for loc in locations if template in (loc["primary_key"], loc["internal_id"])] + if len(matches) != 1: + raise ValueError("catalog_template must select exactly one bundle location by key or internal ID") + selected = matches[0] + provider = provider or selected.get("provider") + selected_options: dict[str, Any] = selected.get("extra_data") or {} + defaults = {field: selected_options.get(field) for field in ("timeout", "redirectLimit", "retryCount", "flags")} + if provider is None: + providers = {loc["provider"] for loc in locations if loc.get("provider")} + if len(providers) > 1: + raise ValueError("catalog has multiple bundle providers; set bundle_provider or catalog_template") + provider = next(iter(providers), BUNDLE_PROVIDER) + return _provider(provider), {**defaults, **(request_options or {})} + + +def bundle_entries( + internal_id: str, + bundle_name: str, + assets: Sequence[dict[str, Any]], + *, + hash: str, + crc: int, + bundle_size: int, + primary_key: str | None = None, + request_options: dict[str, Any] | None = None, + dependency_keys: Sequence[str] = (), + provider: str | None = None, + catalog: bytes | None = None, + template: str | None = None, +) -> list[dict[str, Any]]: + """Locations for a bundle and the assets stored in its AssetBundle container. + + `internal_id` is the bundle's download URL or runtime path. `bundle_name` is the AssetBundle's internal name; + `hash` is a 128-bit cache version encoded as 32 hexadecimal digits. Each asset specifies `primary_key`, + `internal_id` (its container path) and `resource_type`, with optional `aliases`, `provider` and + `resource_type_assembly`. Bundle dependencies and asset dependencies may name existing or added primary keys. + `catalog` supplies provider defaults; `template` selects an existing bundle's provider and common options. + An explicit `provider` or `request_options` takes precedence. + """ + if not internal_id or not bundle_name: + raise ValueError("bundle internal_id and bundle_name are required") + _hash_bytes(hash) + _integer(crc, "bundle crc", 0, 0xFFFFFFFF) + _integer(bundle_size, "bundle_size", 0, 0xFFFFFFFF) + provider, request_options = bundle_settings( + catalog, provider=provider, template=template, request_options=request_options + ) + key = primary_key or internal_id.rsplit("/", 1)[-1] + extra = { + "timeout": 0, + "redirectLimit": 32, + "retryCount": 0, + "flags": 0, + **(request_options or {}), + "hash": hash.lower(), + "crc": crc, + "bundleSize": bundle_size, + "bundleName": bundle_name, + } + out = [ + { + "primary_key": key, + "internal_id": internal_id, + "provider": provider, + "resource_type": BUNDLE_TYPE, + "extra_data": extra, + "dependency_keys": list(dependency_keys), + } + ] + for asset in assets: + if not all(asset.get(field) for field in ("primary_key", "internal_id", "resource_type")): + raise ValueError("each bundle asset requires primary_key, internal_id and resource_type") + location = dict(asset) + location.setdefault("provider", ASSET_PROVIDER) + location["dependency_keys"] = list(dict.fromkeys([key, *(asset.get("dependency_keys") or [])])) + out.append(location) + return out + + +def edit_entries( + data: bytes, + internal_id: str, + bundle_name: str, + assets: Sequence[dict[str, Any]], + edits: Sequence[dict[str, Any]], + *, + hash: str, + crc: int, + bundle_size: int, + primary_key: str | None = None, + provider: str | None = None, + request_options: dict[str, Any] | None = None, + dependency_keys: Sequence[str] | None = None, + template: str | None = None, +) -> tuple[list[dict[str, Any]], dict[int, dict[str, Any]]]: + """Incremental catalog locations for an edited bundle, preserving unrelated locations and lookup keys. + + `edits` identifies each final container path, its source container paths and explicit catalog field overrides. + Existing asset locations match both source container and bundle dependency. Their types, providers, aliases + and dependency graph remain intact unless overridden. New container paths produce added locations. + """ + locations = addressables.parse_locations(data) + bundles = [loc for loc in locations if loc.get("resource_type") == BUNDLE_TYPE] + selector = template or internal_id + matches = [loc for loc in bundles if selector in (loc["primary_key"], loc["internal_id"])] + if not matches and template is None: + matches = [loc for loc in bundles if loc["internal_id"].rsplit("/", 1)[-1] == internal_id.rsplit("/", 1)[-1]] + if not matches: + return bundle_entries( + internal_id, + bundle_name, + assets, + hash=hash, + crc=crc, + bundle_size=bundle_size, + primary_key=primary_key, + request_options=request_options, + dependency_keys=dependency_keys or (), + provider=provider, + catalog=data, + template=template, + ), {} + if len(matches) != 1: + raise ValueError("edited bundle must match exactly one catalog location") + bundle = matches[0] + selected_provider, options = bundle_settings( + data, provider=provider, template=bundle["internal_id"], request_options=request_options + ) + generated = bundle_entries( + internal_id, + bundle_name, + [], + hash=hash, + crc=crc, + bundle_size=bundle_size, + primary_key=primary_key or bundle["primary_key"], + provider=selected_provider, + request_options=options, + )[0] + if dependency_keys is None: + generated.pop("dependency_keys") + else: + generated["dependencies"] = [] + generated["dependency_keys"] = list(dependency_keys) + replacements: dict[int, dict[str, Any]] = {bundle["offset"]: generated} + by_offset = {loc["offset"]: loc for loc in locations} + + def depends_on_bundle(location: dict[str, Any]) -> bool: + pending = list(location.get("dependencies") or []) + seen: set[int] = set() + while pending: + offset = pending.pop() + if offset == bundle["offset"]: + return True + if offset not in seen: + seen.add(offset) + pending.extend((by_offset.get(offset) or {}).get("dependencies") or []) + return False + + added: list[dict[str, Any]] = [] + asset_defaults = {asset["internal_id"]: asset for asset in assets} + for edit in edits: + container = edit["internal_id"] + overrides = dict(edit.get("overrides") or {}) + matched = [ + loc + for loc in locations + if loc["internal_id"] in edit.get("source_internal_ids", ()) and depends_on_bundle(loc) + ] + for location in matched: + fields = {"internal_id": container, **overrides} + if "dependency_keys" in overrides: + fields["dependencies"] = [bundle["offset"]] + replacements[location["offset"]] = {**replacements.get(location["offset"], {}), **fields} + if not matched: + asset = {**asset_defaults[container], **overrides} + asset.setdefault("provider", ASSET_PROVIDER) + asset["dependencies"] = [bundle["offset"]] + added.append(asset) + return added, replacements + + +def rebuild( + data: bytes, + add: list[dict[str, Any]] | None = None, + drop: set[str] | None = None, + update: dict[str, dict[str, Any]] | None = None, + replace: dict[int, dict[str, Any]] | None = None, +) -> bytes: + """Write `data` with added locations, removed primary keys and updated bundle request options. + + Added locations specify `primary_key` and `internal_id`. Optional `aliases` add string or integer lookup keys. + `dependencies` refers to old-catalog offsets, `dependency_on_added` to zero-based indices in `add`, and + `dependency_keys` to existing or added primary keys. All three forms can be combined. + + `update` maps full internal ids or bundle file names to `crc`, `bundleSize`, `hash` and `bundleName` fields. + It applies to existing and added request options; full internal ids take precedence over file names. + `replace` updates selected old location offsets in place. Existing aliases and dependency edges survive; + explicit `primary_key` and `aliases` fields update only that location's lookup memberships. + """ + add = add or [] + drop = drop or set() + update = update or {} + replace = replace or {} + for name, fields in update.items(): + if unknown := set(fields) - set(UPDATE_FIELDS): + raise ValueError(f"bundle update {name}: unsupported fields {', '.join(sorted(unknown))}") + header = addressables.parse_header(data) + locations = addressables.parse_locations(data) + keys = addressables.parse_keys(data) + + keep = [loc for loc in locations if loc["primary_key"] not in drop] + original_by_offset = {loc["offset"]: loc for loc in locations} + old_offsets = [loc["offset"] for loc in keep] + index_of = {off: i for i, off in enumerate(old_offsets)} + + by_primary: dict[str, list[int]] = {} + if any(offset not in index_of for offset in replace): + raise ValueError("replacement offset is unavailable in the retained catalog") + sources = [{**loc, **replace.get(loc["offset"], {})} for loc in keep] + add + for i, src in enumerate(sources): + if not isinstance(src.get("primary_key"), str) or not src["primary_key"]: + raise ValueError("catalog locations require a nonempty string primary_key") + if not isinstance(src.get("internal_id"), str) or not src["internal_id"]: + raise ValueError(f"catalog location {src['primary_key']}: internal_id is required") + by_primary.setdefault(src["primary_key"], []).append(i) + + key_locations: dict[tuple[str, str | int], list[int]] = {} + + def add_key(value: str | int, typ: str, indices: Sequence[int]) -> None: + if typ not in ("System.String", "System.Int32"): + raise ValueError(f"unsupported catalog key type: {typ}") + if ( + typ == "System.String" + and not isinstance(value, str) + or typ == "System.Int32" + and (not isinstance(value, int) or isinstance(value, bool)) + ): + raise ValueError(f"catalog key does not match its type: {typ}") + locs = key_locations.setdefault((typ, value), []) + locs.extend(i for i in indices if i not in locs) + + for entry in keys: + offsets: list[int] = entry.get("locations") or [] + indices: list[int] = [] + for offset in offsets: + if offset not in index_of: + continue + fields = replace.get(offset, {}) + old_primary = original_by_offset[offset]["primary_key"] + new_primary = fields.get("primary_key", old_primary) + if entry["key"] == old_primary and new_primary != old_primary: + continue + if "aliases" in fields and entry["key"] != new_primary and entry["key"] not in fields["aliases"]: + continue + indices.append(index_of[offset]) + if indices: + add_key(entry["key"], entry["type"], indices) + for offset, fields in replace.items(): + slot = index_of[offset] + add_key(sources[slot]["primary_key"], "System.String", [slot]) + aliases: list[str | int] = fields.get("aliases") or [] + for alias in aliases: + add_key(alias, "System.Int32" if isinstance(alias, int) else "System.String", [slot]) + for j, src in enumerate(add): + slot = len(keep) + j + add_key(src["primary_key"], "System.String", [slot]) + aliases: list[str | int] = src.get("aliases") or [] + for alias in aliases: + add_key(alias, "System.Int32" if isinstance(alias, int) else "System.String", [slot]) + + w = _Writer() + table_len = 8 * len(key_locations) + w.buf += struct.pack(" None: + internal = src.get("internal_id") or "" + extra = _entry(src.get("extra_data")) + provider = src.get("provider") or (bundle_settings(data)[0] if extra else ASSET_PROVIDER) + fields = update.get(internal, update.get(internal.rsplit("/", 1)[-1])) if extra else None + if fields: + extra.update(fields) + dep_indices: list[int] = [] + dependencies: list[int] = src.get("dependencies") or [] + for offset in dependencies: + if offset in index_of: + dep_indices.append(index_of[offset]) + elif (original := original_by_offset.get(offset)) and original["primary_key"] in by_primary: + dep_indices.extend(by_primary[original["primary_key"]]) + else: + raise ValueError(f"catalog location {src['primary_key']}: dependency offset {offset} is unavailable") + added_dependencies: list[Any] = src.get("dependency_on_added") or [] + for idx in added_dependencies: + if not isinstance(idx, int) or isinstance(idx, bool) or not 0 <= idx < len(add): + raise ValueError(f"catalog location {src['primary_key']}: added dependency index {idx} is unavailable") + dep_indices.append(len(keep) + idx) + dependency_keys: list[str] = src.get("dependency_keys") or [] + for key in dependency_keys: + if key not in by_primary: + raise ValueError(f"catalog location {src['primary_key']}: dependency key {key} is unavailable") + dep_indices.extend(by_primary[key]) + dep_ids = [records[idx] for idx in dict.fromkeys(dep_indices)] + extra_id = NONE + if extra: + extra_id = w.request_options(extra) + rtype = src.get("resource_type") + if not rtype and extra: + rtype = BUNDLE_TYPE + struct.pack_into( + " list[int]: SBOX = _sbox() +SBOX_NP = np.array(SBOX, dtype=np.uint8) INV_SBOX = np.zeros(256, dtype=np.uint8) INV_SBOX[np.array(SBOX)] = np.arange(256, dtype=np.uint8) -MUL = {k: np.array([_gmul(i, k) for i in range(256)], dtype=np.uint8) for k in (9, 11, 13, 14)} +MUL = {k: np.array([_gmul(i, k) for i in range(256)], dtype=np.uint8) for k in (2, 3, 9, 11, 13, 14)} INV_SHIFT = (0, 1, 3, 4) # InvShiftRows: row r moves right by INV_SHIFT[r] columns (Nb = 8) @@ -167,6 +168,61 @@ def decode(raw: bytes, key: MasterKey, rk: np.ndarray | None = None) -> bytes: return gzip.decompress(out[:-pad]) +SHIFT = (0, 1, 3, 4) # ShiftRows: row r moves left by SHIFT[r] columns (Nb = 8) + + +def encrypt_blocks(pt: np.ndarray, rk: np.ndarray) -> np.ndarray: + """(n, 32) plaintext blocks -> (n, 32) block encryptions (no CBC chaining).""" + s = pt.reshape(-1, NB, 4).transpose(0, 2, 1).copy() # (n, 4, NB): s[r][c] + s ^= rk[0] + cols = np.arange(NB) + for rnd in range(1, NR + 1): + s = SBOX_NP[s] # SubBytes + for r in range(4): # ShiftRows + s[:, r, :] = s[:, r, (cols + SHIFT[r]) % NB] + if rnd != NR: # MixColumns + a0, a1, a2, a3 = s[:, 0, :], s[:, 1, :], s[:, 2, :], s[:, 3, :] + s = np.stack( # pyright: ignore[reportUnknownMemberType] (numpy stubs: partly untyped overloads) + [ + MUL[2][a0] ^ MUL[3][a1] ^ a2 ^ a3, + a0 ^ MUL[2][a1] ^ MUL[3][a2] ^ a3, + a0 ^ a1 ^ MUL[2][a2] ^ MUL[3][a3], + MUL[3][a0] ^ a1 ^ a2 ^ MUL[2][a3], + ], + axis=1, + ) + s ^= rk[rnd] # AddRoundKey + return s.transpose(0, 2, 1).reshape(-1, BLOCK) + + +def encrypt_cbc(data: bytes, key: bytes, iv: bytes, rk: np.ndarray | None = None) -> bytes: + """Rijndael-256 CBC encryption of `data`, PKCS7 padding added.""" + if len(iv) != BLOCK: + raise ValueError("iv must be 32 bytes") + pad = BLOCK - len(data) % BLOCK + padded = data + bytes([pad]) * pad + pt = np.frombuffer(padded, dtype=np.uint8).reshape(-1, BLOCK) # pyright: ignore[reportUnknownMemberType] + out = np.empty_like(pt) + prev = np.frombuffer(iv, dtype=np.uint8) # pyright: ignore[reportUnknownMemberType] + rk = round_keys(key) if rk is None else rk + for i in range(len(pt)): # CBC chains block by block; the cipher itself runs over all blocks at once + out[i : i + 1] = encrypt_blocks((pt[i : i + 1] ^ prev)[None, :], rk) + prev = out[i] + return out.tobytes() + + +def encode(text: bytes, key: MasterKey, prefix: bytes = b"", rk: np.ndarray | None = None) -> bytes: + """JSON text (UTF-8 bytes) -> a master data file as stored: `prefix` + Rijndael-256-CBC(gzip(JSON)). + + The prefix is the 64 bytes the game keeps in front of the ciphertext; its content is not part of the cipher (the + loader starts at `PREFIX`), so a file that replaces another one should carry that file's prefix unchanged. Left + out, it is 64 zero bytes. + """ + if prefix and len(prefix) != PREFIX: + raise ValueError(f"prefix must be {PREFIX} bytes") + return (prefix or bytes(PREFIX)) + encrypt_cbc(gzip.compress(text, mtime=0), key.key, key.iv, rk) + + def table_json(text: bytes) -> str: """The stored form of a decoded table: parsed and written again by jsonio.dumps (UTF-8, one-space indent, non-finite numbers as 1e999 / -1e999; a NaN raises ValueError naming where it is).""" diff --git a/src/nnnotes/source/nnnotes.example.toml b/src/nnnotes/source/nnnotes.example.toml index 350e687..0b871bb 100644 --- a/src/nnnotes/source/nnnotes.example.toml +++ b/src/nnnotes/source/nnnotes.example.toml @@ -70,6 +70,10 @@ master = "" apk = "" # ffmpeg executable; empty: ffmpeg on PATH (flag --ffmpeg) ffmpeg = "" +# ffprobe executable; empty: ffprobe on PATH +ffprobe = "" +# VGAudioCli executable or .NET DLL for HCA encoding; empty: VGAudioCli on PATH +vgaudio = "" # vgmstream-cli executable; empty: vgmstream-cli on PATH (flag --vgmstream) vgmstream = "" # Node.js executable for the web build's read-set script; empty: node on PATH (flag --node) diff --git a/src/nnnotes/unity/bundlefs.py b/src/nnnotes/unity/bundlefs.py new file mode 100644 index 0000000..25d6294 --- /dev/null +++ b/src/nnnotes/unity/bundlefs.py @@ -0,0 +1,178 @@ +"""The UnityFS container: the bytes a bundle stores, and the checksum a client compares against the catalog. + +A bundle has a header, compressed block and directory information, and file data divided into blocks. The content +CRC describes the concatenated decompressed blocks. The header and block/directory information are excluded. +A catalog entry's `AssetBundleRequestOptions.crc` is a value of this kind: + + from nnnotes.unity import bundlefs + crc = bundlefs.content_crc(Path("assets.bundle").read_bytes()) + +The header flags decide the layout: the low six bits are the compression of the block information; each block has +its own compression flags +(0 none, 1 LZMA, 2/3 LZ4), `0x80` puts the block table at the end of the file, and `0x200` pads the block data to a +16-byte boundary. A version 7 or later header is itself padded to 16 bytes after its strings. +""" + +from __future__ import annotations + +import lzma +import struct +import zlib +from dataclasses import dataclass + +import lz4.block + +__all__ = ["Header", "content", "content_crc", "parse_header"] + +SIGNATURE = b"UnityFS\0" +ALIGNMENT = 16 +BLOCK_TABLE_AT_END = 0x80 +BLOCK_INFO_PADDING = 0x200 +ENCRYPTED = 0x400 # a key the container carries itself, not one a caller can supply +COMPRESSION_MASK = 0x3F +COMPRESSION_NONE = 0 +COMPRESSION_LZMA = 1 +COMPRESSION_LZ4 = 2 +COMPRESSION_LZ4HC = 3 +HASH_SIZE = 16 +BLOCK_ENTRY = 10 # uncompressed size (u32), compressed size (u32), flags (u16) + + +@dataclass(frozen=True) +class Header: + """The fields of a UnityFS header that decide where the block table and the block data are.""" + + version: int + engine: str + revision: str + size: int + info_compressed: int + info_uncompressed: int + flags: int + offset: int # where the block table or the block data starts, whichever comes first + + @property + def compression(self) -> int: + return self.flags & COMPRESSION_MASK + + @property + def table_at_end(self) -> bool: + return bool(self.flags & BLOCK_TABLE_AT_END) + + @property + def encrypted(self) -> bool: + return bool(self.flags & ENCRYPTED) + + +def _string(data: bytes, at: int) -> tuple[str, int]: + end = data.index(b"\0", at) + return data[at:end].decode("utf-8", "replace"), end + 1 + + +def _aligned(offset: int) -> int: + return (offset + ALIGNMENT - 1) // ALIGNMENT * ALIGNMENT + + +def _decompress(data: bytes, compression: int, uncompressed: int) -> bytes: + """One compressed region -> its bytes; `uncompressed` is what the header says it expands to.""" + if compression == COMPRESSION_NONE: + result = data + elif compression in (COMPRESSION_LZ4, COMPRESSION_LZ4HC): + try: + result = lz4.block.decompress(data, uncompressed_size=uncompressed) + except lz4.block.LZ4BlockError as e: + raise ValueError(f"an LZ4 region of this bundle does not decompress: {e}") from None + elif compression == COMPRESSION_LZMA: + # LZMA1 raw: one byte of lc/lp/pb, four of dictionary size, then the stream + try: + props, dictionary = struct.unpack(" Header: + """The header of a UnityFS bundle; ValueError when the file is not one.""" + if not data.startswith(SIGNATURE): + raise ValueError("not a UnityFS bundle") + try: + version = struct.unpack_from(">I", data, len(SIGNATURE))[0] + engine, at = _string(data, len(SIGNATURE) + 4) + revision, at = _string(data, at) + size, info_compressed, info_uncompressed, flags = struct.unpack_from(">qIII", data, at) + except (struct.error, ValueError) as e: + raise ValueError(f"truncated or corrupt UnityFS header: {e}") from None + at += 20 + if version >= 7: + at = _aligned(at) + header = Header(version, engine, revision, size, info_compressed, info_uncompressed, flags, at) + if size != len(data) or at > len(data): + raise ValueError("UnityFS file size does not match its header") + if header.encrypted: + raise ValueError("the bundle carries its own encryption key, which its header does not expose") + return header + + +def _block_table(data: bytes, header: Header) -> tuple[list[tuple[int, int, int]], int]: + """The block table and where the block data starts.""" + if header.table_at_end: + at = len(data) - header.info_compressed + body = header.offset + else: + at = header.offset + body = at + header.info_compressed + if at < header.offset or at + header.info_compressed > len(data): + raise ValueError("UnityFS block information lies outside the file") + table = _decompress(data[at : at + header.info_compressed], header.compression, header.info_uncompressed) + try: + count = struct.unpack_from(">I", table, HASH_SIZE)[0] + at = HASH_SIZE + 4 + if count > (len(table) - at) // BLOCK_ENTRY: + raise ValueError("UnityFS block count exceeds its block information") + blocks = [] + for _ in range(count): + uncompressed, compressed, flags = struct.unpack_from(">IIH", table, at) + at += BLOCK_ENTRY + blocks.append((uncompressed, compressed, flags & COMPRESSION_MASK)) + except struct.error: + raise ValueError("truncated UnityFS block information") from None + if header.flags & BLOCK_INFO_PADDING: + body = _aligned(body) + end = len(data) - header.info_compressed if header.table_at_end else len(data) + if body + sum(block[1] for block in blocks) > end: + raise ValueError("UnityFS block data lies outside the content region") + return blocks, body + + +def content(data: bytes) -> bytes: + """Concatenated decompressed file data, excluding the header and block/directory information.""" + header = parse_header(data) + blocks, at = _block_table(data, header) + out = [] + for uncompressed, compressed, compression in blocks: + out.append(_decompress(data[at : at + compressed], compression, uncompressed)) + at += compressed + return b"".join(out) + + +def content_crc(data: bytes) -> int: + """The CRC32 a catalog entry has to carry for this bundle: the checksum of `content`, as unsigned.""" + return zlib.crc32(content(data)) & 0xFFFFFFFF diff --git a/src/nnnotes/unity/pack.py b/src/nnnotes/unity/pack.py new file mode 100644 index 0000000..3370b4e --- /dev/null +++ b/src/nnnotes/unity/pack.py @@ -0,0 +1,868 @@ +"""Build and edit UnityFS assets using the template's serialization schema and container flags. + +New bundles contain the supplied assets and a rebuilt AssetBundle index. TextAsset payloads are binary, +PNG textures use RGBA32, and PCM WAV audio uses a single-sample FSB5 resource. Serialized objects accept +complete type-tree fields; raw objects accept serialized bytes with references managed by the caller. +""" + +from __future__ import annotations + +import copy +import gzip +import hashlib +import io +import json +import os +import struct +import wave +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, cast + +import UnityPy +from attrs import evolve +from UnityPy.files import BundleFile, ObjectReader, SerializedFile +from UnityPy.files.SerializedFile import FileIdentifier, LocalSerializedObjectIdentifier, SerializedType +from UnityPy.helpers import TypeTreeHelper +from UnityPy.helpers.TypeTreeNode import TypeTreeNode +from UnityPy.streams import EndianBinaryReader, EndianBinaryWriter + +from ..core.cache import temp_path +from . import bundlefs + +__all__ = [ + "BundleBuilder", + "BuildResult", + "TextAsset", + "build_manifest", + "builder_from_manifest", + "build_text_asset", + "parse_text_asset", + "replace_text_asset", + "wav_to_fsb5", + "write_outputs", +] + + +@dataclass(frozen=True) +class TextAsset: + """A TextAsset's name and binary payload.""" + + name: str + payload: bytes + + +def parse_text_asset(raw: bytes, *, endian: str = "<") -> TextAsset: + """Decode the two-field TextAsset layout, rejecting truncated data.""" + try: + name_size = struct.unpack_from(endian + "I", raw)[0] + at = (4 + name_size + 3) & ~3 + size = struct.unpack_from(endian + "I", raw, at)[0] + end = (at + 4 + size + 3) & ~3 + if end != len(raw): + raise ValueError("TextAsset size does not match its serialized fields") + return TextAsset(raw[4 : 4 + name_size].decode("utf-8"), raw[at + 4 : at + 4 + size]) + except (struct.error, UnicodeDecodeError) as e: + raise ValueError(f"invalid TextAsset: {e}") from None + + +def build_text_asset(name: str, payload: bytes, *, endian: str = "<") -> bytes: + """Encode a name and binary payload with Unity string alignment.""" + encoded = name.encode("utf-8") + out = struct.pack(endian + "I", len(encoded)) + encoded + out += bytes(-len(out) % 4) + out += struct.pack(endian + "I", len(payload)) + payload + return out + bytes(-len(out) % 4) + + +def wav_to_fsb5(data: bytes, name: str) -> tuple[bytes, dict[str, Any]]: + """Wrap uncompressed mono/stereo 16-bit PCM WAV frames in a single-sample FSB5 bank.""" + try: + with wave.open(io.BytesIO(data)) as source: + channels, width, rate, count, compression, _ = source.getparams() + frames = source.readframes(count) + except (wave.Error, EOFError) as e: + raise ValueError(f"invalid PCM WAV: {e}") from None + if compression != "NONE" or channels not in (1, 2) or width != 2: + raise ValueError("WAV must contain mono or stereo 16-bit integer PCM") + if not count or count >= 1 << 30 or len(frames) != count * channels * width: + raise ValueError("WAV sample count is empty, truncated or too large") + sample = struct.pack(" None: + """Stage all files before replacement. Failed recovery preserves backups and reports their paths.""" + resolved = [path.resolve() for path in outputs] + if len(set(resolved)) != len(resolved): + raise ValueError("output paths must be distinct") + if any(path.exists() and not path.is_file() for path in outputs): + raise ValueError("output paths must be files") + staged: dict[Path, Path] = {} + backups: dict[Path, Path] = {} + published: list[Path] = [] + try: + for path, data in outputs.items(): + path.parent.mkdir(parents=True, exist_ok=True) + temporary = staged[path] = temp_path(path) + temporary.write_bytes(data) + for path in outputs: + if path.exists(): + backup = backups[path] = temp_path(path) + os.replace(path, backup) + os.replace(staged[path], path) + published.append(path) + except BaseException as failure: + recovery_errors = [] + for path in reversed(published): + try: + path.unlink(missing_ok=True) + except OSError as e: + recovery_errors.append(str(e)) + for path, backup in list(backups.items()): + try: + os.replace(backup, path) + del backups[path] + except OSError as e: + recovery_errors.append(f"{path}: {e}; recoverable backup: {backup}") + if recovery_errors: + raise OSError( + f"output replacement failed: {failure}; recovery incomplete: " + "; ".join(recovery_errors) + ) from failure + raise + else: + for backup in backups.values(): + backup.unlink(missing_ok=True) + backups.clear() + finally: + for temporary in staged.values(): + temporary.unlink(missing_ok=True) + + +def _raw(obj) -> bytes: + return obj.data if obj.data is not None else obj.get_raw_data() + + +def _set(obj, raw: bytes) -> None: + obj.set_raw_data(raw) + obj.reader = EndianBinaryReader(raw, endian=obj.assets_file.header.endian) + obj.byte_start, obj.byte_size = 0, len(raw) + + +def _tree(obj, node=None) -> dict: + reader = EndianBinaryReader(_raw(obj), endian=obj.assets_file.header.endian) + return cast( + dict, + TypeTreeHelper.read_typetree( + node or obj._get_typetree_node(), + reader, + as_dict=True, + assetsfile=obj.assets_file, + byte_size=len(_raw(obj)), + check_read=True, + ), + ) + + +def _save(obj, fields: dict, node=None) -> None: + writer = EndianBinaryWriter(endian=obj.assets_file.header.endian) + TypeTreeHelper.write_typetree(fields, node or obj._get_typetree_node(), writer, obj.assets_file) + _set(obj, writer.bytes) + + +def _name(obj) -> str | None: + if obj.type.name == "TextAsset": + return parse_text_asset(_raw(obj), endian=obj.assets_file.header.endian).name + return _tree(obj).get("m_Name") + + +def _node(value): + if isinstance(value, TypeTreeNode): + return value + if isinstance(value, list) and value and isinstance(value[0], (list, tuple)): + node = TypeTreeNode.from_list([TypeTreeNode(lv, ty, nm, 0, 1, m_MetaFlag=mf) for lv, ty, nm, mf in value]) + else: + rows = [ + {key: child for key, child in row.items() if key != "m_Children"} if isinstance(row, dict) else row + for row in value + ] + node = TypeTreeNode.from_list(rows) + for child in node.traverse(): + for key in ("m_TypeFlags", "m_RefTypeHash", "m_Index", "m_MetaFlag"): + if getattr(child, key) is None: + setattr(child, key, 0) + return node + + +def _merge(fields: dict, overrides: dict) -> dict: + for key, value in overrides.items(): + if key not in fields: + raise ValueError(f"field {key!r} is absent from the selected schema") + if isinstance(value, dict) and isinstance(fields[key], dict): + _merge(fields[key], value) + else: + fields[key] = value + return fields + + +def _references(value): + if isinstance(value, dict): + if set(value) == {"m_FileID", "m_PathID"}: + yield value + else: + for child in value.values(): + yield from _references(child) + elif isinstance(value, (list, tuple)): + for child in value: + yield from _references(child) + + +def _payload(spec: dict, base: Path) -> bytes: + if "data" in spec and "file" in spec: + raise ValueError("supply either data or file for a binary payload") + data = bytes(spec["data"]) if "data" in spec else (base / spec["file"]).read_bytes() + encoding = spec.get("encoding", "binary") + if encoding == "gzip-json": + json.loads(data) + return gzip.compress(data, mtime=0) + if encoding != "binary": + raise ValueError("payload encoding must be binary or gzip-json") + return data + + +def _binary_fields(fields: dict, specs: list, base: Path) -> None: + for spec in specs: + path = spec.get("path") + if not isinstance(path, list) or not path: + raise ValueError("binary field path must be a nonempty array of field names and indices") + target = fields + try: + for key in path[:-1]: + target = target[key] + previous = target[path[-1]] + data = _payload(spec, base) + target[path[-1]] = list(data) if isinstance(previous, list) else data + except (KeyError, IndexError, TypeError) as e: + raise ValueError(f"binary field path does not resolve: {path}") from e + + +@dataclass(frozen=True) +class BuildResult: + """Validated bundle bytes, container locations and explicit catalog edits. + + Each catalog edit names its original container paths, final internal ID and fields supplied by asset + operations. Added assets have no original paths; unchanged assets have no edit record. + """ + + data: bytes + bundle_name: str + assets: list[dict] + catalog_edits: list[dict] = field(default_factory=list) + + @property + def crc(self) -> int: + return bundlefs.content_crc(self.data) + + +class BundleBuilder: + """A template edited in memory; ``new=True`` starts with an empty object and resource table. + + Asset schemas come from the template, compatible additional templates, or explicit serialized type metadata. + Path IDs are unique within the selected serialized file. Raw references remain the caller's responsibility; + serialized field references are validated before the bundle is saved. + """ + + def __init__( + self, + template, + *, + new: bool = True, + bundle_name: str | None = None, + serialized_file: str | None = None, + cab_name: str | None = None, + ): + self.env = UnityPy.load(str(template) if isinstance(template, (str, Path)) else template) + if not isinstance(self.env.file, BundleFile) or self.env.file.signature != "UnityFS": + raise ValueError("template must be a UnityFS bundle") + self.bundle = self.env.file + original = Path(template).read_bytes() if isinstance(template, (str, Path)) else template + self.header = bundlefs.parse_header(original) + files = [f for f in self.env.assets if not serialized_file or f.name == serialized_file] + if len(files) != 1: + raise ValueError("select exactly one serialized_file from the template") + self.file = cast(SerializedFile, files[0]) + self.templates = list(self.file.objects.values()) + indices = [o for o in self.templates if o.type.name == "AssetBundle"] + if len(indices) != 1: + raise ValueError("template must contain exactly one AssetBundle index in the selected file") + self.index = indices[0] + self.index_fields = _tree(self.index) + if new and not bundle_name: + raise ValueError("new bundles require a nonempty bundle_name") + self.bundle_name = bundle_name or self.index_fields["m_Name"] + independent = new or (bundle_name is not None and bundle_name != self.index_fields["m_Name"]) + self.entries = list(self.index_fields["m_Container"]) + self.locations: dict[str, dict] = {} + self.original_containers: dict[int, list[str]] = {} + if not new: + for key, entry in self.entries: + if entry["asset"]["m_FileID"] == 0: + self.original_containers.setdefault(entry["asset"]["m_PathID"], []).append(key) + self.catalog_overrides: dict[int, dict] = {} + self.checked: set[int] = set() + self.raw_objects: set[int] = set() + self.failed = False + self.nodes: dict[int, Any] = {} + self.new = new + if new: + self.file.objects = {} + self.file.externals = [] + self.file.script_types = [] + self.file.ref_types = [] + self.file.userInformation = "" + self.file._cache.clear() + self.bundle.files = {self.file.name: self.file} + self.entries = [] + self.index = self._clone(self.index, 1) + self.index_fields["m_PreloadTable"] = [] + self.index_fields["m_MainAsset"] = { + "preloadIndex": 0, + "preloadSize": 0, + "asset": {"m_FileID": 0, "m_PathID": 0}, + } + for key in ("m_Dependencies", "m_SceneHashes"): + if key in self.index_fields: + self.index_fields[key] = [] + if "m_IsStreamedSceneAssetBundle" in self.index_fields: + self.index_fields["m_IsStreamedSceneAssetBundle"] = False + if independent or cab_name is not None: + self._rename_cab(cab_name or f"CAB-{hashlib.md5(self.bundle_name.encode('utf-8')).hexdigest()}") + self.index_fields["m_Name"] = self.bundle_name + if "m_AssetBundleName" in self.index_fields: + self.index_fields["m_AssetBundleName"] = self.bundle_name + + def _rename_cab(self, name: str) -> None: + if not isinstance(name, str) or name in ("", ".", "..") or any(c in name for c in ("/", "\\", "\0", ":")): + raise ValueError("cab_name must be a nonempty file name") + old = self.file.name + if name == old: + return + if len(self.env.assets) != 1: + raise ValueError("independent bundle identity requires a single serialized file") + mapping = { + key: name + key[len(old) :] for key in self.bundle.files if isinstance(key, str) and key.startswith(old) + } + mapping[old] = name + + def remap(value: Any) -> Any: + if isinstance(value, str): + for source, target in sorted(mapping.items(), key=lambda item: -len(item[0])): + value = value.replace(f"archive:/{old}/{source}", f"archive:/{name}/{target}") + return value + if isinstance(value, dict): + return {key: remap(child) for key, child in value.items()} + if isinstance(value, (tuple, list)): + return [remap(child) for child in value] + return value + + for obj in self.file.objects.values(): + if obj.type.name in ("AssetBundle", "TextAsset"): + continue + fields = _tree(obj) + changed = remap(fields) + if changed != fields: + _save(obj, changed) + self.bundle.files = { + mapping.get(key, key) if isinstance(key, str) else key: value for key, value in self.bundle.files.items() + } + self.file.name = name + + def _clone(self, source, path_id: int): + if isinstance(path_id, bool) or not isinstance(path_id, int) or not 0 < path_id < 1 << 63: + raise ValueError("path_id must be a positive signed 64-bit integer") + if path_id in self.file.objects: + raise ValueError(f"path_id {path_id} already exists") + serialized_type = source.serialized_type + type_id = next((i for i, typ in enumerate(self.file.types) if typ is serialized_type), None) + if type_id is None: + serialized_type = copy.copy(serialized_type) + if serialized_type is None: + raise ValueError("template object has no serialized type metadata") + if self.file._enable_type_tree and serialized_type.node is None: + original_node = source._get_typetree_node() + serialized_type.node = _node( + [{k: v for k, v in row.items() if k != "m_Children"} for row in original_node.to_dict_list()] + ) + serialized_type.type_dependencies = serialized_type.type_dependencies or () + self.file.types.append(serialized_type) + type_id = len(self.file.types) - 1 + obj = evolve(source, assets_file=self.file, path_id=path_id, type_id=type_id, serialized_type=serialized_type) + _set(obj, _raw(source)) + self.file.objects[path_id] = obj + return obj + + def select(self, selector: dict, *, templates: bool = False): + objects = self.templates if templates else list(self.file.objects.values()) + matches = [ + o + for o in objects + if ("path_id" not in selector or o.path_id == selector["path_id"]) + and (not selector.get("type") or o.type.name == selector["type"]) + and (not selector.get("name") or _name(o) == selector["name"]) + ] + if len(matches) != 1: + raise ValueError(f"asset selector must match exactly one object; matched {len(matches)}") + return matches[0] + + def add_external(self, path: str, *, guid: str = "0" * 32, type: int = 0) -> int: + """Register a serialized-file reference and return its one-based PPtr file ID.""" + if self.failed: + raise ValueError("builder contains a failed operation; create a new builder") + if not isinstance(path, str) or not path or "\0" in path: + raise ValueError("external path must be a nonempty string without null characters") + encoded_guid = bytes.fromhex(guid) + if len(encoded_guid) != 16 or not isinstance(type, int) or isinstance(type, bool) or type not in range(4): + raise ValueError("external guid must have 32 hexadecimal digits and type must be in 0..3") + identifier = FileIdentifier.__new__(FileIdentifier) + identifier.path, identifier.temp_empty, identifier.guid, identifier.type = path, "", encoded_guid, type + self.file.externals.append(identifier) + return len(self.file.externals) + + def _source(self, spec: dict, base: Path): + selector = spec.get("template") or {"type": spec.get("type")} + if isinstance(selector, str): + selector = {"name": selector, "type": spec.get("type")} + if template_file := selector.get("file"): + env = UnityPy.load(str(base / template_file)) + candidates = [ + o + for o in env.objects + if ("path_id" not in selector or o.path_id == selector["path_id"]) + and (not selector.get("name") or _name(o) == selector["name"]) + and (not selector.get("type") or o.type.name == selector["type"]) + ] + if len(candidates) != 1: + raise ValueError("external template selector must match exactly one object") + source = candidates[0] + if ( + source.assets_file.unity_version, + source.assets_file.header.version, + source.assets_file.header.endian, + source.assets_file.target_platform, + ) != ( + self.file.unity_version, + self.file.header.version, + self.file.header.endian, + self.file.target_platform, + ): + raise ValueError("asset template Unity version, format, endian and platform must match the bundle") + return source + if schema := spec.get("schema"): + from UnityPy.enums import ClassIDType + + typ = SerializedType.__new__(SerializedType) + typ.__attrs_init__(schema["class_id"]) # pyright: ignore[reportAttributeAccessIssue] + typ.is_stripped_type = False + typ.node = _node(schema["nodes"]) + typ.old_type_hash = bytes.fromhex(schema["type_hash"]) + if len(typ.old_type_hash) != 16: + raise ValueError("schema type_hash must contain 32 hexadecimal digits") + typ.script_type_index = schema.get("script_type_index", -1) + typ.script_id = bytes.fromhex(schema["script_id"]) if schema.get("script_id") else None + if typ.class_id == 114 and (typ.script_id is None or len(typ.script_id) != 16): + raise ValueError("MonoBehaviour schema requires a 32-digit script_id") + typ.type_dependencies = tuple(schema.get("dependencies", ())) + typ.type_tree_content_hash = bytes(16) + return ObjectReader( + self.file, EndianBinaryReader(b""), 0, 0, typ, typ.class_id, ClassIDType(typ.class_id), 0, 0, 0, 0, b"" + ) + return self.select(selector, templates=True) + + def apply(self, spec: dict, *, base: Path = Path(".")) -> None: + """Apply an add, replace or rename operation. File paths are relative to ``base``.""" + if self.failed: + raise ValueError("builder contains a failed operation; create a new builder") + try: + self._apply(spec, base=base) + except BaseException: + self.failed = True + raise + + def _apply(self, spec: dict, *, base: Path) -> None: + operation = spec.get("op", "add") + if operation not in ("add", "replace", "rename"): + raise ValueError(f"unknown asset operation {operation!r}") + kind = spec.get("type") + if operation == "add": + if not spec.get("name"): + raise ValueError("new assets require a nonempty name") + source = self._source(spec, base) + obj = self._clone(source, spec.get("path_id", max(self.file.objects, default=0) + 1)) + else: + obj = self.select(spec.get("target") or {k: spec[k] for k in ("name", "path_id") if k in spec}) + source = obj + if obj.type.name == "AssetBundle": + raise ValueError("the AssetBundle index is managed by the builder") + name = spec.get("new_name") or (spec.get("name") if operation == "add" else _name(obj)) + if not isinstance(name, str) or not name or "\0" in name: + raise ValueError("asset name must be a nonempty string without null characters") + nodes = _node(spec["typetree"]) if spec.get("typetree") else None + if nodes: + self.nodes[obj.path_id] = nodes + if obj.serialized_type is None: + raise ValueError("object has no serialized type metadata") + typ = copy.copy(obj.serialized_type) + typ.node = nodes + self.file.types.append(typ) + obj.serialized_type, obj.type_id = typ, len(self.file.types) - 1 + schema_node = nodes or obj._get_typetree_node() + if ( + self.new + and kind != "raw" + and any( + child.m_Type in ("ManagedReferencesRegistry", "ReferencedObjectData") + for child in schema_node.traverse() + ) + ): + raise ValueError("managed-reference objects require raw serialized data with caller-managed metadata") + if operation == "rename": + if obj.type.name == "TextAsset": + payload = parse_text_asset(_raw(obj), endian=self.file.header.endian).payload + _set(obj, build_text_asset(name, payload, endian=self.file.header.endian)) + else: + fields = _tree(obj, nodes) + fields["m_Name"] = name + _save(obj, fields, nodes) + elif kind == "raw": + if not spec.get("managed_references"): + raise ValueError("raw objects require managed_references=true") + _set(obj, (base / spec["file"]).read_bytes()) + self.raw_objects.add(obj.path_id) + elif kind == "serialized": + fields = spec.get("fields") + if spec.get("file"): + fields = json.loads((base / spec["file"]).read_text(encoding="utf-8")) + if "patch" in spec: + if fields is not None: + raise ValueError("serialized fields and patch are mutually exclusive") + fields = _merge(copy.deepcopy(_tree(source, nodes)), spec["patch"]) + if not isinstance(fields, dict): + raise ValueError("serialized objects require complete fields or a JSON file") + fields = copy.deepcopy(fields) + _binary_fields(fields, spec.get("binary_fields", []), base) + if "m_Name" in fields: + fields["m_Name"] = name + _save(obj, fields, nodes) + self.checked.add(obj.path_id) + elif obj.type.name == "TextAsset": + node = nodes or obj._get_typetree_node() + if [child.m_Name for child in node.m_Children] != ["m_Name", "m_Script"]: + raise ValueError("binary TextAsset requires the m_Name/m_Script schema") + _set(obj, build_text_asset(name, _payload(spec, base), endian=self.file.header.endian)) + elif obj.type.name in ("Texture2D", "AudioClip"): + fields = _merge(copy.deepcopy(_tree(source, nodes)), spec.get("fields", {})) + fields["m_Name"] = name + if obj.type.name == "Texture2D": + from PIL import Image + from UnityPy.export.Texture2DConverter import image_to_texture2d + + with Image.open(base / spec["file"]) as image: + if image.format != "PNG": + raise ValueError("Texture2D input must be a PNG image") + pixels, format_id = image_to_texture2d( + image.convert("RGBA"), 4, obj.platform, fields.get("m_PlatformBlob") + ) + _merge( + fields, + { + "m_Width": image.width, + "m_Height": image.height, + "m_TextureFormat": int(format_id), + "m_CompleteImageSize": len(pixels), + "image data": pixels, + }, + ) + for key, value in ( + ("m_MipCount", 1), + ("m_MipsStripped", 0), + ("m_MipMap", False), + ("m_StreamingMipmaps", False), + ("m_ImageCount", 1), + ("m_TextureDimension", 2), + ): + if key in fields: + fields[key] = value + if "m_StreamData" in fields: + fields["m_StreamData"] = {"offset": 0, "size": 0, "path": ""} + else: + bank, audio_fields = wav_to_fsb5((base / spec["file"]).read_bytes(), name) + _merge(fields, {key: value for key, value in audio_fields.items() if key in fields}) + if "m_Resource" in fields: + resource = f"{self.file.name}.resource" + resources = cast(dict[str, Any], self.bundle.files) + suffix = 0 + while resource in resources and not isinstance(resources[resource], EndianBinaryWriter): + suffix += 1 + resource = f"{self.file.name}_{suffix}.resource" + writer = self.bundle.get_writeable_cab(resource) + if writer is None: + raise ValueError("bundle cannot create an audio resource") + offset = writer.Position + writer.write(bank) + fields["m_Resource"] = { + "m_Source": f"archive:/{self.file.name}/{resource}", + "m_Offset": offset, + "m_Size": len(bank), + } + elif "m_AudioData" in fields: + fields["m_AudioData"] = bank + else: + raise ValueError("AudioClip schema has no m_Resource or m_AudioData field") + for key, value in ( + ("m_LoadType", 0), + ("m_PreloadAudioData", True), + ("m_LoadInBackground", False), + ("m_IsTrackerFormat", False), + ): + if key in fields: + fields[key] = value + _save(obj, fields, nodes) + self.checked.add(obj.path_id) + else: + raise ValueError("use type='serialized' or type='raw' for this object class") + if kind not in ("serialized", "raw") and kind and kind != obj.type.name: + raise ValueError("asset type does not match the selected template") + prior = [key for key, entry in self.entries if entry["asset"] == {"m_FileID": 0, "m_PathID": obj.path_id}] + previous_locations = {key: self.locations[key] for key in prior if key in self.locations} + container = spec.get("new_container") or spec.get("container") + if operation == "add": + container = container or name + if container: + if container in {key for key, _ in self.entries} - set(prior): + raise ValueError(f"container path {container!r} already exists") + self.entries = [(key, entry) for key, entry in self.entries if key not in prior] + self.entries.append( + (container, {"preloadIndex": 0, "preloadSize": 0, "asset": {"m_FileID": 0, "m_PathID": obj.path_id}}) + ) + for key in prior: + self.locations.pop(key, None) + if len(previous_locations) == 1: + self.locations[container] = next(iter(previous_locations.values())) + prior = [container] + catalog_fields = ( + "primary_key", + "resource_type", + "aliases", + "dependency_keys", + "resource_type_assembly", + "provider", + ) + overrides = self.catalog_overrides.setdefault(obj.path_id, {}) + overrides.update({name: copy.deepcopy(spec[name]) for name in catalog_fields if name in spec}) + for key in prior: + location = dict(self.locations.get(key, {})) + location.setdefault("primary_key", key) + location["internal_id"] = key + location.setdefault("resource_type", f"UnityEngine.{obj.type.name}") + for name in catalog_fields: + if name in spec: + location[name] = spec[name] + self.locations[key] = location + + def build(self) -> BuildResult: + """Validate generated field references, rebuild preload ranges and save with ``packer='original'``.""" + if self.failed: + raise ValueError("builder contains a failed operation; create a new builder") + for path_id in self.checked: + for ref in _references(_tree(self.file.objects[path_id], self.nodes.get(path_id))): + if ref["m_PathID"] and ( + (ref["m_FileID"] == 0 and ref["m_PathID"] not in self.file.objects) + or not 0 <= ref["m_FileID"] <= len(self.file.externals) + ): + raise ValueError(f"object {path_id} contains an unresolved reference {ref}") + + # Serialized MonoBehaviour types index the file's script-reference table. + # Derive that index from each object's final script pointer after edits. + for obj in list(self.file.objects.values()): + if obj.type.name != "MonoBehaviour" or obj.path_id in self.raw_objects: + continue + if obj.serialized_type is None: + raise ValueError("MonoBehaviour has no serialized type metadata") + script = _tree(obj, self.nodes.get(obj.path_id)).get("m_Script") + if not isinstance(script, dict) or not script.get("m_PathID"): + continue + reference = (script["m_FileID"], script["m_PathID"]) + script_index = next( + ( + i + for i, item in enumerate(self.file.script_types) + if (item.local_serialized_file_index, item.local_identifier_in_file) == reference + ), + len(self.file.script_types), + ) + if script_index == len(self.file.script_types): + item = LocalSerializedObjectIdentifier.__new__(LocalSerializedObjectIdentifier) + item.local_serialized_file_index, item.local_identifier_in_file = reference + self.file.script_types.append(item) + if obj.serialized_type.script_type_index != script_index: + typ = copy.copy(obj.serialized_type) + typ.script_type_index = script_index + self.file.types.append(typ) + obj.serialized_type, obj.type_id = typ, len(self.file.types) - 1 + for obj in self.file.objects.values(): + if obj.serialized_type is None or obj.serialized_type.script_type_index >= len(self.file.script_types): + raise ValueError(f"object {obj.path_id} contains an unresolved script type index") + + def closure(pointer: dict) -> list[dict]: + pending, seen, pointers = [pointer], set(), [] + while pending: + ref = pending.pop(0) + key = ref["m_FileID"], ref["m_PathID"] + if key in seen or not ref["m_PathID"]: + continue + seen.add(key) + pointers.append(ref) + if ref["m_FileID"] == 0 and ref["m_PathID"] not in self.raw_objects: + if ref["m_PathID"] not in self.file.objects: + raise ValueError(f"container contains an unresolved local reference {ref}") + obj = self.file.objects[ref["m_PathID"]] + if obj.type.name != "TextAsset": + pending.extend(_references(_tree(obj, self.nodes.get(obj.path_id)))) + return pointers + + preload = [] + for _, entry in self.entries: + root = entry["asset"] + if root["m_FileID"] == 0 and root["m_PathID"] in self.raw_objects and not self.new: + previous = self.index_fields["m_PreloadTable"] + start, size = entry["preloadIndex"], entry["preloadSize"] + if start < 0 or start + size > len(previous): + raise ValueError("container preload range is outside the preload table") + entry["preloadIndex"] = len(preload) + preload.extend(previous[start : start + size]) + else: + start = len(preload) + preload.extend(closure(root)) + entry["preloadIndex"], entry["preloadSize"] = start, len(preload) - start + self.index_fields["m_PreloadTable"] = preload + main = self.index_fields.get("m_MainAsset") + if main and main["asset"]["m_PathID"]: + matching = next((entry for _, entry in self.entries if entry["asset"] == main["asset"]), None) + if matching: + main["preloadIndex"], main["preloadSize"] = matching["preloadIndex"], matching["preloadSize"] + else: + main["preloadIndex"] = len(preload) + refs = closure(main["asset"]) + preload.extend(refs) + main["preloadSize"] = len(refs) + self.index_fields["m_Container"] = self.entries + _save(self.index, self.index_fields) + if self.new: + used = {id(o.serialized_type) for o in self.file.objects.values()} + self.file.types = [typ for typ in self.file.types if id(typ) in used] + for obj in self.file.objects.values(): + obj.type_id = next(i for i, typ in enumerate(self.file.types) if typ is obj.serialized_type) + data = self.bundle.save(packer="original") + header = bundlefs.parse_header(data) + if (header.version, header.engine, header.revision, header.flags) != ( + self.header.version, + self.header.engine, + self.header.revision, + self.header.flags, + ): + raise ValueError("saved UnityFS header does not match the template") + reread = UnityPy.load(data) + file = cast(SerializedFile, next(f for f in reread.assets if f.name == self.file.name)) + if file.unity_version != self.file.unity_version or set(file.objects) != set(self.file.objects): + raise ValueError("saved serialized file does not match the object table") + for path_id, obj in self.file.objects.items(): + if file.objects[path_id].get_raw_data() != _raw(obj): + raise ValueError(f"saved object {path_id} does not match its serialized payload") + for key, entry in self.entries: + pointer = file.container[key] + if pointer.path_id != entry["asset"]["m_PathID"]: + raise ValueError(f"saved container entry {key!r} does not resolve to its asset") + assets = [ + self.locations.get(key, {"primary_key": key, "internal_id": key, "resource_type": "UnityEngine.Object"}) + for key, _ in self.entries + ] + catalog_edits = [] + for key, entry in self.entries: + path_id = entry["asset"]["m_PathID"] + if entry["asset"]["m_FileID"] != 0 or path_id not in self.catalog_overrides: + continue + source_paths = self.original_containers.get(path_id, []) + catalog_edits.append( + { + "source_internal_ids": [key] if key in source_paths else list(source_paths), + "internal_id": key, + "overrides": copy.deepcopy(self.catalog_overrides[path_id]), + } + ) + return BuildResult(data, self.bundle_name, assets, catalog_edits) + + +def build_manifest(doc: dict, *, base: Path = Path(".")) -> BuildResult: + """Build a JSON manifest; relative template and payload paths resolve against ``base``.""" + return builder_from_manifest(doc, base=base).build() + + +def builder_from_manifest(doc: dict, *, base: Path = Path(".")) -> BundleBuilder: + """Apply manifest operations and return an in-memory builder for further asset operations.""" + if not isinstance(doc, dict) or not isinstance(doc.get("assets"), list): + raise ValueError("build manifest requires a template and an assets array") + mode = doc.get("mode", "new") + if mode not in ("new", "edit"): + raise ValueError("build mode must be 'new' or 'edit'") + if not isinstance(doc.get("template"), str) or not doc["template"]: + raise ValueError("build manifest requires a template file") + builder = BundleBuilder( + base / doc["template"], + new=mode == "new", + bundle_name=doc.get("bundle_name"), + serialized_file=doc.get("serialized_file"), + cab_name=doc.get("cab_name"), + ) + for external in doc.get("externals", []): + builder.add_external(**external) + if "dependencies" in doc: + if "m_Dependencies" not in builder.index_fields: + raise ValueError("AssetBundle schema has no dependency names field") + if not isinstance(doc["dependencies"], list) or any( + not isinstance(name, str) or not name for name in doc["dependencies"] + ): + raise ValueError("bundle dependencies must be nonempty strings") + builder.index_fields["m_Dependencies"] = doc["dependencies"] + for spec in doc["assets"]: + if not isinstance(spec, dict): + raise ValueError("asset operations must be objects") + builder.apply(spec, base=base) + return builder + + +def replace_text_asset( + bundle_in, bundle_out, name: str, payload: bytes, *, new_name: str | None = None, packer: str = "original" +) -> None: + """Replace binary TextAsset bytes while preserving the bundle layout and existing container paths.""" + if packer != "original": + raise ValueError("asset editing uses packer='original'") + builder = BundleBuilder(bundle_in, new=False) + try: + obj = builder.select({"type": "TextAsset", "name": name}) + except ValueError: + raise KeyError(f"no unique TextAsset {name!r} in the bundle") from None + _set(obj, build_text_asset(new_name or name, payload, endian=builder.file.header.endian)) + write_outputs({Path(bundle_out): builder.build().data}) diff --git a/tests/cli/test_catalog.py b/tests/cli/test_catalog.py new file mode 100644 index 0000000..ecbd327 --- /dev/null +++ b/tests/cli/test_catalog.py @@ -0,0 +1,294 @@ +import hashlib +import json +import zlib + +from nnnotes import cli +from nnnotes.source import addressables +from tests.helpers import synth + +PAYLOADS = [b"file-table", b"data" * 400] +CONTENT = b"".join(PAYLOADS) + + +def run(argv, capsys): + """(exit code, stdout, stderr) of the command line.""" + code = 0 + try: + cli.main(argv) + except SystemExit as e: + code = e.code if isinstance(e.code, int) else 1 + out, err = capsys.readouterr() + return code, out, err + + +def catalog(tmp_path): + path = tmp_path / "catalog_main.bin" + path.write_bytes(synth.CatalogWriter().build([("x_01.bundle", synth.remote("x_01.bundle"), [])])) + return path + + +def entries(path): + return { + loc["primary_key"]: (loc.get("extra_data") or {}) for loc in addressables.parse_locations(path.read_bytes()) + } + + +def test_rebuild_writes_the_checksum_the_client_validates(tmp_path, capsys): + cat = catalog(tmp_path) + bundle = tmp_path / "x_01.bundle" + bundle.write_bytes(synth.unityfs(PAYLOADS, compression="lz4")) + out = tmp_path / "out.bin" + code, stdout, err = run(["catalog", "rebuild", str(cat), "-o", str(out), "--bundle", str(bundle), "--json"], capsys) + assert code == 0, err + fields = entries(out)["x_01.bundle"] + assert fields["crc"] == zlib.crc32(CONTENT) + assert fields["bundleSize"] == len(synth.unityfs(PAYLOADS, compression="lz4")) + assert json.loads(stdout)["updated"]["x_01.bundle"]["crc"] == zlib.crc32(CONTENT) + + +def test_rebuild_keeps_everything_else(tmp_path, capsys): + cat = catalog(tmp_path) + bundle = tmp_path / "x_01.bundle" + bundle.write_bytes(synth.unityfs(PAYLOADS)) + out = tmp_path / "out.bin" + before = entries(cat)["x_01.bundle"] + code, _, err = run(["catalog", "rebuild", str(cat), "-o", str(out), "--bundle", str(bundle)], capsys) + assert code == 0, err + after = entries(out)["x_01.bundle"] + assert after["hash"] == hashlib.md5(bundle.read_bytes()).hexdigest() + assert after["bundleName"] == before["bundleName"] + assert len(addressables.parse_locations(out.read_bytes())) == len(addressables.parse_locations(cat.read_bytes())) + assert len(addressables.parse_keys(out.read_bytes())) == len(addressables.parse_keys(cat.read_bytes())) + + +def test_rebuild_can_give_the_bundle_a_new_cache_hash(tmp_path, capsys): + cat = catalog(tmp_path) + bundle = tmp_path / "x_01.bundle" + bundle.write_bytes(synth.unityfs(PAYLOADS)) + out = tmp_path / "out.bin" + code, _, err = run( + ["catalog", "rebuild", str(cat), "-o", str(out), "--bundle", str(bundle), "--hash", "ab" * 16], capsys + ) + assert code == 0, err + assert entries(out)["x_01.bundle"]["hash"] == "ab" * 16 + + +def test_a_bundle_the_catalog_does_not_name_is_an_error(tmp_path, capsys): + cat = catalog(tmp_path) + bundle = tmp_path / "z_99.bundle" + bundle.write_bytes(synth.unityfs(PAYLOADS)) + code, _, err = run( + ["catalog", "rebuild", str(cat), "-o", str(tmp_path / "out.bin"), "--bundle", str(bundle)], capsys + ) + assert code != 0 and "z_99.bundle" in err + assert not (tmp_path / "out.bin").exists() + + +def test_a_file_that_is_not_a_bundle_is_an_error(tmp_path, capsys): + cat = catalog(tmp_path) + bundle = tmp_path / "x_01.bundle" + bundle.write_bytes(b"not a bundle") + code, _, err = run( + ["catalog", "rebuild", str(cat), "-o", str(tmp_path / "out.bin"), "--bundle", str(bundle)], capsys + ) + assert code != 0 and "UnityFS" in err + + +def test_a_cache_hash_needs_exactly_one_bundle(tmp_path, capsys): + cat = catalog(tmp_path) + code, _, err = run(["catalog", "rebuild", str(cat), "-o", str(tmp_path / "out.bin"), "--hash", "ab" * 16], capsys) + assert code != 0 and "--bundle" in err + + +def test_add_bundle_description_generates_metadata_and_asset_dependency(tmp_path, capsys): + cat = catalog(tmp_path) + package = tmp_path / "package" + package.mkdir() + packed = synth.unityfs(PAYLOADS, compression="lz4") + (package / "custom.bundle").write_bytes(packed) + manifest = package / "catalog.json" + manifest.write_text( + json.dumps( + { + "file": "custom.bundle", + "internal_id": "https://example.invalid/custom.bundle", + "bundle_name": "custom_bundle", + "assets": [ + { + "primary_key": "Chart/Custom", + "internal_id": "Assets/Custom.bytes", + "resource_type": "UnityEngine.TextAsset", + "aliases": ["chart"], + }, + ], + } + ) + ) + out = tmp_path / "out.bin" + code, stdout, err = run(["catalog", "rebuild", str(cat), "--add", str(manifest), "-o", str(out), "--json"], capsys) + assert code == 0, err + locations = {loc["primary_key"]: loc for loc in addressables.parse_locations(out.read_bytes())} + bundle = locations["custom.bundle"] + assert bundle["extra_data"]["crc"] == zlib.crc32(CONTENT) + assert bundle["extra_data"]["bundleSize"] == len(packed) + assert bundle["extra_data"]["hash"] == hashlib.md5(packed).hexdigest() + assert bundle["extra_data"]["bundleName"] == "custom_bundle" + assert locations["Chart/Custom"]["dependencies"] == [bundle["offset"]] + assert json.loads(stdout)["addedBundles"]["https://example.invalid/custom.bundle"]["crc"] == zlib.crc32(CONTENT) + + +def test_add_entry_list_and_bundle_updates_apply_together(tmp_path, capsys): + cat = catalog(tmp_path) + bundle = tmp_path / "custom.bundle" + packed = synth.unityfs(PAYLOADS) + bundle.write_bytes(packed) + manifest = tmp_path / "entries.json" + manifest.write_text( + json.dumps( + [ + { + "primary_key": "custom.bundle", + "internal_id": "custom.bundle", + "extra_data": {"bundleName": "custom", "hash": "11" * 16, "crc": 0, "bundleSize": 0}, + }, + { + "primary_key": "Chart/Custom", + "internal_id": "Assets/Custom.bytes", + "resource_type": "UnityEngine.TextAsset", + "dependency_keys": ["custom.bundle"], + }, + ] + ) + ) + out = tmp_path / "out.bin" + code, _, err = run( + [ + "catalog", + "rebuild", + str(cat), + "--add", + str(manifest), + "--bundle", + str(bundle), + "--hash", + "ab" * 16, + "-o", + str(out), + ], + capsys, + ) + assert code == 0, err + fields = entries(out)["custom.bundle"] + assert (fields["crc"], fields["bundleSize"], fields["hash"]) == (zlib.crc32(CONTENT), len(packed), "ab" * 16) + + +def test_bundle_description_selects_a_catalog_template_and_overrides_provider(tmp_path, capsys): + cat = tmp_path / "catalog.bin" + cat.write_bytes( + synth.CatalogWriter().build( + [ + ( + "source.bundle", + synth.remote("source.bundle"), + [], + { + "provider": "ExampleEncryptedBundleProvider", + "extra": {"flags": 8, "timeout": 60}, + }, + ), + ("other.bundle", synth.remote("other.bundle"), [], {"provider": synth.BUNDLE_PROVIDER}), + ] + ) + ) + packed = synth.unityfs(PAYLOADS) + (tmp_path / "custom.bundle").write_bytes(packed) + manifest, out = tmp_path / "add.json", tmp_path / "out.bin" + description = { + "file": "custom.bundle", + "internal_id": synth.remote("custom.bundle"), + "bundle_name": "custom", + "catalog_template": "source.bundle", + "request_options": {"timeout": 120}, + } + manifest.write_text(json.dumps(description)) + code, _, err = run(["catalog", "rebuild", str(cat), "--add", str(manifest), "-o", str(out)], capsys) + assert code == 0, err + bundle = next( + loc for loc in addressables.parse_locations(out.read_bytes()) if loc["primary_key"] == "custom.bundle" + ) + assert bundle["provider"] == "ExampleEncryptedBundleProvider" + assert (bundle["extra_data"]["flags"], bundle["extra_data"]["timeout"]) == (8, 120) + assert bundle["extra_data"]["crc"] == zlib.crc32(CONTENT) + description["bundle_provider"] = "ExplicitBundleProvider" + manifest.write_text(json.dumps(description)) + code, _, err = run(["catalog", "rebuild", str(cat), "--add", str(manifest), "-o", str(out)], capsys) + assert code == 0, err + bundle = next( + loc for loc in addressables.parse_locations(out.read_bytes()) if loc["primary_key"] == "custom.bundle" + ) + assert bundle["provider"] == "ExplicitBundleProvider" + + +def test_multiple_bundle_descriptions_resolve_dependencies_by_key(tmp_path, capsys): + cat = catalog(tmp_path) + packed = synth.unityfs(PAYLOADS) + for name in ("a.bundle", "b.bundle"): + (tmp_path / name).write_bytes(packed) + manifest = tmp_path / "bundles.json" + manifest.write_text( + json.dumps( + { + "bundles": [ + { + "file": "a.bundle", + "internal_id": "https://example.invalid/a.bundle", + "bundle_name": "a", + "dependency_keys": ["b.bundle"], + "hash": "12" * 16, + "assets": [ + { + "primary_key": "Texture/A", + "internal_id": "Assets/A.png", + "resource_type": "UnityEngine.Texture2D", + } + ], + }, + {"file": "b.bundle", "internal_id": "https://example.invalid/b.bundle", "bundle_name": "b"}, + ] + } + ) + ) + out = tmp_path / "out.bin" + code, _, err = run(["catalog", "rebuild", str(cat), "--add", str(manifest), "-o", str(out)], capsys) + assert code == 0, err + locations = {loc["primary_key"]: loc for loc in addressables.parse_locations(out.read_bytes())} + assert locations["a.bundle"]["dependencies"] == [locations["b.bundle"]["offset"]] + assert locations["Texture/A"]["dependencies"] == [locations["a.bundle"]["offset"]] + assert locations["a.bundle"]["extra_data"]["hash"] == "12" * 16 + + +def test_invalid_addition_is_reported_before_writing(tmp_path, capsys): + cat = catalog(tmp_path) + manifest = tmp_path / "invalid.json" + manifest.write_text( + json.dumps( + {"primary_key": "custom", "internal_id": "Assets/Custom.bytes", "dependency_keys": ["missing.bundle"]} + ) + ) + out = tmp_path / "out.bin" + code, _, err = run(["catalog", "rebuild", str(cat), "--add", str(manifest), "-o", str(out)], capsys) + assert code != 0 and "missing.bundle" in err + assert not out.exists() + + +def test_invalid_cache_hash_is_a_usage_error(tmp_path, capsys): + cat = catalog(tmp_path) + bundle = tmp_path / "x_01.bundle" + bundle.write_bytes(synth.unityfs(PAYLOADS)) + out = tmp_path / "out.bin" + code, _, err = run( + ["catalog", "rebuild", str(cat), "--bundle", str(bundle), "--hash", "bad", "-o", str(out)], + capsys, + ) + assert code != 0 and "32 hexadecimal digits" in err + assert not out.exists() diff --git a/tests/cli/test_pack.py b/tests/cli/test_pack.py new file mode 100644 index 0000000..15ff843 --- /dev/null +++ b/tests/cli/test_pack.py @@ -0,0 +1,464 @@ +import hashlib +import json + +import pytest + +from nnnotes import cli +from nnnotes.source import addressables, catalogbuild, master +from nnnotes.source.addressables import BundleKey +from nnnotes.source.catalog import Catalog +from nnnotes.source.master import MasterKey +from tests.helpers import synth +from tests.helpers.packassets import template + +MASTER_KEY = MasterKey(synth.MASTER_KEY, synth.MASTER_IV) +BUNDLE_KEY = BundleKey(synth.BUNDLE_KEY, synth.BUNDLE_SEED) + + +def run(argv, capsys): + """(exit code, stdout, stderr) of the command line.""" + code = 0 + try: + cli.main(argv) + except SystemExit as e: + code = e.code if isinstance(e.code, int) else 1 + out, err = capsys.readouterr() + return code, out, err + + +@pytest.fixture +def keys(monkeypatch): + monkeypatch.setenv("NNNOTES_MASTER_KEY", synth.MASTER_KEY.hex()) + monkeypatch.setenv("NNNOTES_MASTER_IV", synth.MASTER_IV.hex()) + monkeypatch.setenv("NNNOTES_BUNDLE_KEY", synth.BUNDLE_KEY.hex()) + monkeypatch.setenv("NNNOTES_BUNDLE_NONCE_SEED", synth.BUNDLE_SEED.hex()) + + +def test_pack_master_decodes_back_to_the_same_json(tmp_path, capsys, keys): + src = tmp_path / "json" / "MasterX.json" + src.parent.mkdir() + src.write_text(json.dumps(synth.MASTER_TABLE, ensure_ascii=False), encoding="utf-8") + code, out, err = run(["pack", "master", str(src.parent), "-o", str(tmp_path / "bin"), "--json"], capsys) + assert code == 0, err + assert json.loads(out)["packed"] == 1 + raw = (tmp_path / "bin" / "MasterX.bin").read_bytes() + assert raw[: master.PREFIX] == bytes(master.PREFIX) + assert master.decode(raw, MASTER_KEY) == src.read_bytes() + + +def test_pack_master_keeps_the_prefix_of_another_file(tmp_path, capsys, keys): + src = tmp_path / "MasterX.json" + src.write_text("{}", encoding="utf-8") + original = tmp_path / "original.bin" + original.write_bytes(synth.master_file(synth.MASTER_TABLE, prefix=bytes(range(64)))) + code, _, err = run( + ["pack", "master", str(src), "-o", str(tmp_path / "bin"), "--prefix-from", str(original)], capsys + ) + assert code == 0, err + assert (tmp_path / "bin" / "MasterX.bin").read_bytes()[:64] == bytes(range(64)) + + +def test_pack_master_rejects_a_short_prefix(tmp_path, capsys, keys): + src = tmp_path / "MasterX.json" + src.write_text("{}", encoding="utf-8") + short = tmp_path / "short.bin" + short.write_bytes(bytes(10)) + code, _, err = run(["pack", "master", str(src), "-o", str(tmp_path / "bin"), "--prefix-from", str(short)], capsys) + assert code != 0 and "prefix" in err + + +def test_pack_master_writes_a_manifest(tmp_path, capsys, keys): + src = tmp_path / "MasterX.json" + src.write_text("{}", encoding="utf-8") + code, out, err = run( + ["pack", "master", str(src), "-o", str(tmp_path / "bin"), "--manifest", "1.2.3", "--json"], capsys + ) + assert code == 0, err + data = (tmp_path / "bin" / "MasterX.bin").read_bytes() + manifest = json.loads((tmp_path / "bin" / "MasterManifest.json").read_text(encoding="utf-8")) + assert manifest["version"] == "1.2.3" + assert manifest["files"] == [{"name": "MasterX.bin", "hash": hashlib.sha256(data).hexdigest(), "size": len(data)}] + assert json.loads(out)["manifest"].endswith("MasterManifest.json") + + +def test_pack_master_needs_no_manifest_by_default(tmp_path, capsys, keys): + src = tmp_path / "MasterX.json" + src.write_text("{}", encoding="utf-8") + code, out, err = run(["pack", "master", str(src), "-o", str(tmp_path / "bin"), "--json"], capsys) + assert code == 0, err + assert not (tmp_path / "bin" / "MasterManifest.json").exists() + assert json.loads(out)["manifest"] is None + + +def test_pack_bundle_is_what_the_client_decrypts(tmp_path, capsys, keys): + src = tmp_path / "b_02.bundle" + src.write_bytes(synth.fake_bundle("b_02.bundle")) + code, out, err = run(["pack", "bundle", str(src), "-o", str(tmp_path / "cdn"), "--json"], capsys) + assert code == 0, err + assert json.loads(out)["packed"] == 1 + stored = (tmp_path / "cdn" / "b_02.bundle").read_bytes() + assert not stored.startswith(b"UnityFS") + assert addressables.decrypt(stored, "b_02.bundle", BUNDLE_KEY) == src.read_bytes() + + +def test_pack_bundle_names_the_file_it_is_served_as(tmp_path, capsys, keys): + src = tmp_path / "draft.bundle" + src.write_bytes(synth.fake_bundle("served.bundle")) + code, _, err = run(["pack", "bundle", str(src), "-o", str(tmp_path / "cdn"), "--name", "served.bundle"], capsys) + assert code == 0, err + stored = (tmp_path / "cdn" / "served.bundle").read_bytes() + assert addressables.decrypt(stored, "served.bundle", BUNDLE_KEY) == src.read_bytes() + + +def test_pack_bundle_rejects_a_file_that_is_not_a_bundle(tmp_path, capsys, keys): + src = tmp_path / "x.bundle" + src.write_bytes(b"not a bundle") + code, _, err = run(["pack", "bundle", str(src), "-o", str(tmp_path / "cdn")], capsys) + assert code != 0 and "UnityFS" in err + + +def test_pack_needs_the_keys(tmp_path, capsys, monkeypatch): + for name in ("NNNOTES_MASTER_KEY", "NNNOTES_MASTER_IV", "NNNOTES_BUNDLE_KEY", "NNNOTES_BUNDLE_NONCE_SEED"): + monkeypatch.delenv(name, raising=False) + src = tmp_path / "MasterX.json" + src.write_text("{}", encoding="utf-8") + code, _, err = run(["pack", "master", str(src), "-o", str(tmp_path / "bin")], capsys) + assert code == 2 and "master.key" in err + + +def build_input(tmp_path): + (tmp_path / "template.bundle").write_bytes(template()) + (tmp_path / "data.bin").write_bytes(bytes(range(256))) + doc = { + "template": "template.bundle", + "bundle_name": "example", + "internal_id": synth.remote("example.bundle"), + "assets": [ + { + "type": "TextAsset", + "name": "binary", + "file": "data.bin", + "container": "Assets/Example/binary.bytes", + "primary_key": "Example/Binary", + } + ], + } + path = tmp_path / "build.json" + path.write_text(json.dumps(doc), encoding="utf-8") + return path + + +def test_pack_build_emits_plain_encrypted_and_resolvable_catalog(tmp_path, capsys, keys): + source = build_input(tmp_path) + existing = tmp_path / "catalog.bin" + existing.write_bytes(synth.CatalogWriter().build([])) + plain, encrypted = tmp_path / "plain" / "example.bundle", tmp_path / "cdn" / "example.bundle" + catalog, additions = tmp_path / "catalog-built.bin", tmp_path / "add.json" + code, out, err = run( + [ + "pack", + "build", + str(source), + "-o", + str(plain), + "--encrypt", + str(encrypted), + "--catalog", + str(existing), + "--catalog-out", + str(catalog), + "--catalog-add", + str(additions), + "--json", + ], + capsys, + ) + assert code == 0, err + assert json.loads(out)["bundle_name"] == "example" + assert addressables.decrypt(encrypted.read_bytes(), encrypted.name, BUNDLE_KEY) == plain.read_bytes() + resolved = Catalog(catalog.read_bytes(), tmp_path / "cache").resolve("Example/Binary") + assert [bundle.name for bundle in resolved] == ["example.bundle"] + from nnnotes.unity.bundlefs import content_crc + + locations = addressables.parse_locations(catalog.read_bytes()) + assert locations[0]["extra_data"]["crc"] == content_crc(plain.read_bytes()) + assert locations[0]["extra_data"]["bundleSize"] == plain.stat().st_size + second = tmp_path / "catalog-sidecar.bin" + code, _, err = run(["catalog", "rebuild", str(existing), "-o", str(second), "--add", str(additions)], capsys) + assert code == 0, err + assert Catalog(second.read_bytes(), tmp_path / "cache-two").resolve("Example/Binary")[0].name == "example.bundle" + + +def test_pack_build_failure_leaves_all_existing_outputs_unchanged(tmp_path, capsys): + source = build_input(tmp_path) + doc = json.loads(source.read_text()) + doc["assets"].append({"type": "TextAsset", "name": "missing", "file": "absent"}) + source.write_text(json.dumps(doc)) + plain, additions = tmp_path / "example.bundle", tmp_path / "add.json" + plain.write_bytes(b"original") + additions.write_bytes(b"original additions") + code, _, err = run(["pack", "build", str(source), "-o", str(plain), "--catalog-add", str(additions)], capsys) + assert code != 0 and "absent" in err + assert plain.read_bytes() == b"original" and additions.read_bytes() == b"original additions" + + +def test_pack_build_inherits_custom_provider_and_freezes_it_in_the_sidecar(tmp_path, capsys, keys): + source = build_input(tmp_path) + existing = tmp_path / "catalog.bin" + existing.write_bytes( + synth.CatalogWriter().build( + [("source.bundle", synth.remote("source.bundle"), [], {"provider": "ExampleEncryptedBundleProvider"})] + ) + ) + plain, encrypted = tmp_path / "example.bundle", tmp_path / "encrypted" / "example.bundle" + catalog, additions = tmp_path / "built.bin", tmp_path / "add.json" + code, stdout, err = run( + [ + "pack", + "build", + str(source), + "-o", + str(plain), + "--encrypt", + str(encrypted), + "--catalog", + str(existing), + "--catalog-out", + str(catalog), + "--catalog-add", + str(additions), + "--json", + ], + capsys, + ) + assert code == 0, err + assert json.loads(stdout)["bundle_provider"] == "ExampleEncryptedBundleProvider" + description = json.loads(additions.read_text())["bundles"][0] + assert description["bundle_provider"] == "ExampleEncryptedBundleProvider" + locations = {loc["primary_key"]: loc for loc in addressables.parse_locations(catalog.read_bytes())} + assert locations["example.bundle"]["provider"] == "ExampleEncryptedBundleProvider" + assert locations["Example/Binary"]["provider"] == synth.ASSET_PROVIDER + assert addressables.decrypt(encrypted.read_bytes(), encrypted.name, BUNDLE_KEY) == plain.read_bytes() + empty = tmp_path / "empty.bin" + empty.write_bytes(synth.CatalogWriter().build([])) + second = tmp_path / "sidecar.bin" + code, _, err = run(["catalog", "rebuild", str(empty), "--add", str(additions), "-o", str(second)], capsys) + assert code == 0, err + assert addressables.parse_locations(second.read_bytes())[0]["provider"] == "ExampleEncryptedBundleProvider" + + +def test_pack_build_ambiguous_catalog_providers_leave_outputs_unchanged(tmp_path, capsys): + source = build_input(tmp_path) + existing = tmp_path / "catalog.bin" + existing.write_bytes( + synth.CatalogWriter().build( + [ + ("a.bundle", synth.remote("a.bundle"), [], {"provider": "ExampleEncryptedBundleProvider"}), + ("b.bundle", synth.remote("b.bundle"), [], {"provider": synth.BUNDLE_PROVIDER}), + ] + ) + ) + plain, catalog = tmp_path / "example.bundle", tmp_path / "built.bin" + plain.write_bytes(b"previous") + code, _, err = run( + ["pack", "build", str(source), "-o", str(plain), "--catalog", str(existing), "--catalog-out", str(catalog)], + capsys, + ) + assert code != 0 and "multiple bundle providers" in err + assert plain.read_bytes() == b"previous" and not catalog.exists() + + +def test_pack_build_catalog_requires_source_before_writing(tmp_path, capsys): + source = build_input(tmp_path) + plain = tmp_path / "example.bundle" + code, _, err = run( + ["pack", "build", str(source), "-o", str(plain), "--catalog-out", str(tmp_path / "out.bin")], capsys + ) + assert code != 0 and "--catalog SOURCE" in err + assert not plain.exists() + + +def edit_catalog_input(tmp_path, operations): + (tmp_path / "template.bundle").write_bytes(template()) + (tmp_path / "changed.bin").write_bytes(b"changed payload") + manifest = tmp_path / "edit.json" + manifest.write_text( + json.dumps( + { + "mode": "edit", + "template": "template.bundle", + "internal_id": synth.remote("source.bundle"), + "assets": operations, + } + ) + ) + catalog = tmp_path / "catalog.bin" + catalog.write_bytes( + catalogbuild.rebuild( + synth.CatalogWriter().build( + [ + ( + "source.bundle", + synth.remote("source.bundle"), + [], + { + "provider": "ExampleEncryptedBundleProvider", + "extra": {"flags": 8, "timeout": 60}, + }, + ), + ("dep.bundle", synth.remote("dep.bundle"), []), + ] + ), + add=[ + { + "primary_key": "Example/Text", + "internal_id": "assets/text.bytes", + "provider": "Example.TextProvider", + "resource_type": "UnityEngine.TextAsset", + "aliases": ["Example/Alias", 42], + "dependency_keys": ["source.bundle", "dep.bundle"], + }, + { + "primary_key": "Example/Text", + "internal_id": "assets/other.bytes", + "provider": "Example.OtherProvider", + "resource_type": "UnityEngine.TextAsset", + "aliases": ["Example/Alias", 42], + "dependency_keys": ["dep.bundle"], + }, + ], + ) + ) + return manifest, catalog + + +def catalog_semantics(data): + locations = addressables.parse_locations(data) + by_offset = {loc["offset"]: loc for loc in locations} + + def identity(offset): + loc = by_offset[offset] + return loc["primary_key"], loc["internal_id"], loc["provider"] + + records = [ + { + **{key: value for key, value in loc.items() if key not in ("offset", "dependencies")}, + "dependencies": [identity(offset) for offset in loc["dependencies"]], + } + for loc in locations + ] + keys = [ + (key["key"], key["type"], [identity(offset) for offset in key["locations"]]) + for key in addressables.parse_keys(data) + ] + return records, keys + + +@pytest.mark.parametrize("operations", [[], [{"op": "replace", "name": "text", "file": "changed.bin"}]]) +def test_edit_bundle_catalog_preserves_types_providers_aliases_and_other_locations(tmp_path, capsys, operations): + source, existing = edit_catalog_input(tmp_path, operations) + plain, built, sidecar = tmp_path / "source.bundle", tmp_path / "built.bin", tmp_path / "add.json" + before, before_keys = catalog_semantics(existing.read_bytes()) + code, _, err = run( + [ + "pack", + "build", + str(source), + "-o", + str(plain), + "--catalog", + str(existing), + "--catalog-out", + str(built), + "--catalog-add", + str(sidecar), + ], + capsys, + ) + assert code == 0, err + after, after_keys = catalog_semantics(built.read_bytes()) + assert after[1:] == before[1:] and after_keys == before_keys + assert after[0]["provider"] == before[0]["provider"] == "ExampleEncryptedBundleProvider" + assert (after[0]["extra_data"]["flags"], after[0]["extra_data"]["timeout"]) == (8, 60) + second = tmp_path / "sidecar.bin" + code, _, err = run(["catalog", "rebuild", str(existing), "--add", str(sidecar), "-o", str(second)], capsys) + assert code == 0, err + assert catalog_semantics(second.read_bytes()) == catalog_semantics(built.read_bytes()) + + +def test_edit_catalog_rename_and_overrides_change_only_matching_location(tmp_path, capsys): + source, existing = edit_catalog_input( + tmp_path, + [ + { + "op": "rename", + "name": "text", + "new_name": "renamed", + "new_container": "assets/new.bytes", + "primary_key": "Example/Updated", + "provider": "Example.UpdatedProvider", + "aliases": ["updated", 43], + "dependency_keys": [], + }, + {"type": "TextAsset", "name": "added", "file": "changed.bin", "primary_key": "Example/Added"}, + ], + ) + plain, built = tmp_path / "source.bundle", tmp_path / "built.bin" + code, _, err = run( + [ + "pack", + "build", + str(source), + "-o", + str(plain), + "--catalog", + str(existing), + "--catalog-out", + str(built), + ], + capsys, + ) + assert code == 0, err + locations = {loc["internal_id"]: loc for loc in addressables.parse_locations(built.read_bytes())} + changed, other, added = locations["assets/new.bytes"], locations["assets/other.bytes"], locations["added"] + assert changed["primary_key"] == "Example/Updated" and changed["provider"] == "Example.UpdatedProvider" + assert changed["resource_type"] == "UnityEngine.TextAsset" + assert changed["dependencies"] == [locations[synth.remote("source.bundle")]["offset"]] + assert other["primary_key"] == "Example/Text" and other["provider"] == "Example.OtherProvider" + keys = {key["key"]: key["locations"] for key in addressables.parse_keys(built.read_bytes())} + assert keys["Example/Text"] == keys["Example/Alias"] == keys[42] == [other["offset"]] + assert keys["Example/Updated"] == keys["updated"] == keys[43] == [changed["offset"]] + assert keys["Example/Added"] == [added["offset"]] + + +@pytest.mark.parametrize("name", ["../escape.bundle", "a/b.bundle", "..\\escape.bundle", "C:escape.bundle"]) +def test_pack_bundle_rejects_served_paths(tmp_path, capsys, keys, name): + src = tmp_path / "source.bundle" + src.write_bytes(synth.fake_bundle("source.bundle")) + target = tmp_path / "output" + code, _, err = run(["pack", "bundle", str(src), "-o", str(target), "--name", name], capsys) + assert code != 0 and "file names" in err + assert not target.exists() + + +@pytest.mark.parametrize("command,suffix", [("master", ".json"), ("bundle", ".bundle")]) +def test_pack_batch_duplicate_names_leave_output_absent(tmp_path, capsys, keys, command, suffix): + sources = [] + for dirname in ("a", "b"): + path = tmp_path / dirname / f"same{suffix}" + path.parent.mkdir() + path.write_bytes(b"{}" if command == "master" else synth.fake_bundle("same.bundle")) + sources.append(str(path)) + target = tmp_path / "output" + code, _, err = run(["pack", command, *sources, "-o", str(target)], capsys) + assert code != 0 and "duplicate" in err + assert not target.exists() + + +def test_pack_bundle_batch_invalid_input_does_not_publish_valid_input(tmp_path, capsys, keys): + first, last = tmp_path / "a.bundle", tmp_path / "z.bundle" + first.write_bytes(synth.fake_bundle("a.bundle")) + last.write_bytes(b"invalid") + target = tmp_path / "output" + code, _, _ = run(["pack", "bundle", str(first), str(last), "-o", str(target)], capsys) + assert code != 0 and not target.exists() diff --git a/tests/formats/test_acbwrite.py b/tests/formats/test_acbwrite.py new file mode 100644 index 0000000..0840f87 --- /dev/null +++ b/tests/formats/test_acbwrite.py @@ -0,0 +1,43 @@ +import struct + +import pytest + +from nnnotes.formats.acb import cue_streams, tables, utf_table +from nnnotes.formats.acbwrite import Column, Utf, author +from tests.helpers.packing import acb_template, hca_stream + + +def test_acb_preserves_encoded_stream_and_updates_cue_metadata(): + source = hca_stream(1500, channels=1, rate=44100) + data, report = author(acb_template(), source, cue_name="example") + head, nested = tables(data) + assert head["Name"] == "example" + assert source in head["AwbFile"] + assert data.index(b"AFS2") % 32 == 0 + assert nested["WaveformTable"][0]["NumSamples"] == 1500 + assert cue_streams(data)["example"]["streams"][0]["channels"] == 1 + assert report["hca"]["lengthMs"] == (1500 * 1000 + 44100 - 1) // 44100 + + +def test_utf_constant_promotion_and_absolute_blob_alignment(): + table = Utf( + "Example", + 0, + [Column("name", 10, 0x30), Column("data", 11, 0x50)], + [{"name": "one", "data": b"first"}, {"name": "two", "data": b"second"}], + ) + data = table.write() + assert utf_table(data) == table.rows + assert data.index(b"first") % 32 == data.index(b"second") % 32 == 0 + assert (struct.unpack_from(">I", data, 16)[0] + 8) % 32 == 0 + + +def test_encrypted_stream_requires_explicit_compatibility(): + from nnnotes.formats.hca import crypt + + source = crypt(hca_stream(), 321) + with pytest.raises(ValueError, match="cipher"): + author(acb_template(), source, cue_name="example") + data, report = author(acb_template(), source, cue_name="example", allow_encrypted_hca=True) + assert source in tables(data)[0]["AwbFile"] + assert report["hca"]["cipherType"] == 56 diff --git a/tests/formats/test_hca.py b/tests/formats/test_hca.py new file mode 100644 index 0000000..1a4d6c7 --- /dev/null +++ b/tests/formats/test_hca.py @@ -0,0 +1,51 @@ +import struct + +import pytest + +from nnnotes.formats.hca import crc16, crypt, inspect, key_table +from tests.helpers.packing import hca_stream + + +def test_crc_known_polynomial_vector(): + assert crc16(b"123456789") == 0xFEE8 + + +def test_cipher_preserves_framing_and_is_reversible(): + source = hca_stream(4097) + encrypted = crypt(source, 123456789) + assert encrypted != source + assert inspect(encrypted)["cipherType"] == 56 + assert inspect(encrypted)["samples"] == 4097 + assert crypt(encrypted, 123456789, decrypt=True) == source + assert crypt(encrypted, 987654321, decrypt=True) != source + + +def test_key_tables_are_inverse_permutations(): + encrypt, decrypt = key_table(123), key_table(123, decrypt=True) + assert len(set(encrypt)) == 256 + assert bytes(encrypt).translate(decrypt) == bytes(range(256)) + assert encrypt[0] == 0 and encrypt[255] == 255 + + +@pytest.mark.parametrize("key", [0, -1, 2**64, True]) +def test_invalid_keys_are_rejected(key): + with pytest.raises(ValueError, match="key"): + key_table(key) + + +@pytest.mark.parametrize("where", [40, 70, -1]) +def test_checksum_damage_is_rejected(where): + data = bytearray(hca_stream()) + data[where] ^= 1 + with pytest.raises(ValueError): + inspect(bytes(data)) + + +def test_truncated_and_looping_audio_are_rejected(): + with pytest.raises(ValueError, match="truncated"): + inspect(hca_stream()[:-1]) + data = bytearray(hca_stream()) + data[46:50] = b"loop" + struct.pack_into(">H", data, 62, crc16(data[:62])) + with pytest.raises(ValueError, match="finite"): + inspect(bytes(data)) diff --git a/tests/helpers/packassets.py b/tests/helpers/packassets.py new file mode 100644 index 0000000..d920f66 --- /dev/null +++ b/tests/helpers/packassets.py @@ -0,0 +1,133 @@ +"""Self-contained serialized Unity objects and source media for bundle builder tests.""" + +from __future__ import annotations + +import io +import struct +import wave + +from UnityPy.helpers.Tpk import get_typetree_node +from UnityPy.helpers.TypeTreeHelper import write_typetree +from UnityPy.helpers.TypeTreeNode import TypeTreeNode +from UnityPy.helpers.UnityVersion import UnityVersion +from UnityPy.streams import EndianBinaryWriter + +from nnnotes.unity.pack import build_text_asset +from tests.helpers.synth import _compress + +VERSION = "2022.3.21f1" + + +def defaults(node): + if node.m_Type == "string": + return "" + if node.m_Type == "TypelessData": + return b"" + if node.m_Children and node.m_Children[0].m_Type == "Array": + return [] + if node.m_Children: + return {child.m_Name: defaults(child) for child in node.m_Children} + if node.m_Type in ("float", "double"): + return 0.0 + if node.m_Type == "bool": + return False + return 0 + + +def template(*, flags=0x242, stripped=False): + """A format-22 bundle with a container, binary TextAsset, texture and audio schemas.""" + version = UnityVersion.from_str(VERSION) + nodes = [ + TypeTreeNode.from_list( + [ + {k: v for k, v in row.items() if k != "m_Children"} + for row in get_typetree_node(class_id, version).to_dict_list() + ] + ) + for class_id in (142, 49, 28, 83) + ] + for node in nodes: + for child in node.traverse(): + for key in ("m_TypeFlags", "m_RefTypeHash", "m_Index", "m_MetaFlag"): + if getattr(child, key) is None: + setattr(child, key, 0) + fields = [defaults(node) for node in nodes] + for value, name in zip(fields, ("source", "text", "picture", "sound")): + value["m_Name"] = name + index = fields[0] + index["m_AssetBundleName"] = "source" + index["m_PreloadTable"] = [{"m_FileID": 0, "m_PathID": 20}] + index["m_Container"] = [ + ("assets/text.bytes", {"preloadIndex": 0, "preloadSize": 1, "asset": {"m_FileID": 0, "m_PathID": 20}}) + ] + texture = fields[2] + texture.update(m_Width=1, m_Height=1, m_TextureFormat=4, m_CompleteImageSize=4, m_MipCount=1) + texture["image data"] = b"\x11\x22\x33\xff" + meta, objects = EndianBinaryWriter(endian="<"), EndianBinaryWriter(endian="<") + meta.write_string_to_null(VERSION) + meta.write_int(13) + meta.write_boolean(not stripped) + meta.write_int(4) + for class_id, node in zip((142, 49, 28, 83), nodes): + meta.write_int(class_id) + meta.write_boolean(False) + meta.write_short(-1) + meta.write_bytes(bytes(16)) + if not stripped: + node.dump_blob(meta, 22) + meta.write_int_array([], True) + meta.write_int(4) + for i, (node, value) in enumerate(zip(nodes, fields)): + data = EndianBinaryWriter(endian="<") + if i == 1: + data.write(build_text_asset("text", b"template payload")) + else: + write_typetree(value, node, data) + meta.align_stream() + meta.write_long((i + 1) * 10) + meta.write_long(objects.Position) + meta.write_u_int(data.Length) + meta.write_int(i) + objects.write(data.bytes) + objects.align_stream(8) + meta.write_int(0) + meta.write_int(0) + meta.write_int(0) + meta.write_string_to_null("") + offset = (48 + meta.Length + 15) & ~15 + header = struct.pack(">4IB3sI3q", 0, 0, 22, 0, 0, bytes(3), meta.Length, offset + objects.Length, offset, 0) + serialized = header + meta.bytes + bytes(offset - 48 - meta.Length) + objects.bytes + files = {"CAB-template": serialized, "CAB-template.resS": b"unused template resource"} + payload = b"".join(files.values()) + compression = {0: "none", 1: "lzma", 2: "lz4", 3: "lz4"}[flags & 0x3F] + packed = _compress(payload, compression) + table = bytes(16) + struct.pack(">IIIH", 1, len(payload), len(packed), flags & 0x3F) + table += struct.pack(">I", len(files)) + at = 0 + for name, data in files.items(): + table += struct.pack(">qqI", at, len(data), 4 if name == "CAB-template" else 0) + name.encode() + b"\0" + at += len(data) + info = _compress(table, compression) + head = bytearray(b"UnityFS\0" + struct.pack(">I", 8) + VERSION.encode() + b"\0" + VERSION.encode() + b"\0") + size_at = len(head) + head += struct.pack(">qIII", 0, len(info), len(table), flags) + head += bytes(-len(head) % 16) + out = bytearray(head) + if flags & 0x80: + out += packed + info + else: + out += info + if flags & 0x200: + out += bytes(-len(out) % 16) + out += packed + struct.pack_into(">q", out, size_at, len(out)) + return bytes(out) + + +def wav(*, channels=1, width=2, rate=16000): + stream = io.BytesIO() + frames = bytes(range(1, 101)) * (channels * width) + with wave.open(stream, "wb") as out: + out.setparams((channels, width, rate, 0, "NONE", "not compressed")) + out.writeframes(frames) + return stream.getvalue(), frames diff --git a/tests/helpers/packing.py b/tests/helpers/packing.py new file mode 100644 index 0000000..101390e --- /dev/null +++ b/tests/helpers/packing.py @@ -0,0 +1,252 @@ +"""Synthetic single-cue banks and song package inputs.""" + +import json +import struct + +from nnnotes.formats.hca import crc16 +from tests.helpers.cri import utf + + +def hca_stream(samples=2048, channels=2, rate=48000): + frames = (samples + 1023) // 1024 + head = bytearray(b"HCA\0" + struct.pack(">HH", 0x200, 64)) + head += b"fmt\0" + bytes([channels]) + rate.to_bytes(3, "big") + head += struct.pack(">IHH", frames, 0, frames * 1024 - samples) + head += b"comp" + struct.pack(">H", 16) + bytes(10) + head += b"ciph" + struct.pack(">H", 0) + b"pad\0" + head += bytes(62 - len(head)) + head += struct.pack(">H", crc16(head)) + output = bytes(head) + for index in range(frames): + frame = b"\xff\xff" + bytes((i * 19 + index) & 255 for i in range(12)) + output += frame + struct.pack(">H", crc16(frame)) + return output + + +def acb_template(): + nested = { + "CueNameTable": utf("CueName", [("CueName", 10), ("CueIndex", 2)], [{"CueName": "tone", "CueIndex": 0}]), + "CueTable": utf( + "Cue", + [("CueId", 4), ("ReferenceType", 0), ("ReferenceIndex", 2), ("Length", 4)], + [{"CueId": 0, "ReferenceType": 1, "ReferenceIndex": 0, "Length": 100}], + ), + "WaveformTable": utf( + "Waveform", + [ + ("MemoryAwbId", 2), + ("Streaming", 0), + ("NumSamples", 4), + ("LoopFlag", 0), + ("EncodeType", 0), + ("NumChannels", 0), + ("SamplingRate", 4), + ], + [ + { + "MemoryAwbId": 0, + "Streaming": 0, + "NumSamples": 2048, + "LoopFlag": 1, + "EncodeType": 2, + "NumChannels": 2, + "SamplingRate": 48000, + } + ], + ), + } + from nnnotes.formats.acbwrite import memory_awb + + return utf( + "Header", + [(key, 11) for key in nested] + [("AwbFile", 11), ("Name", 10)], + [{**nested, "AwbFile": memory_awb(hca_stream()), "Name": "template"}], + ) + + +def song_tables(): + song = { + "_id": 1, + "_titleTextID": "title", + "_bandNameTextID": "band", + "_bandIDs": [1], + "_bestMusicTagIDs": [1], + "_musicSoundID": 10, + "_jingleSoundID": 11, + "_jacketAssetName": "jacket", + "_liveScoreRankGroup": 1, + **{f"_{name}ID": 10 + index for index, name in enumerate(("easy", "normal", "hard", "expert"))}, + } + return { + "MasterLiveMusic": {"_allData": [song]}, + "MasterLiveMusicScore": { + "_allData": [ + {"_id": 10 + i, "_musicScoreTextFileName": "chart", "_musicScoreLevel": i + 1, "_fullComboCount": 1} + for i in range(4) + ] + }, + "MasterText": {"_allData": [{"_id": "title", "_english": "Template"}, {"_id": "band", "_english": "Band"}]}, + "MasterSound": {"_allData": [{"_id": i, "_soundCueSheetID": i, "_cueName": "tone"} for i in (10, 11)]}, + "MasterSoundCueSheet": {"_allData": [{"_id": i, "_cueSheetName": "tone"} for i in (10, 11)]}, + "MasterLiveScoreRank": {"_allData": [{"_id": 1, "_group": 1, "_requiredScore": 10}]}, + "MasterBand": {"_allData": [{"_id": 1}]}, + "MasterTag": {"_allData": [{"_id": 1}]}, + } + + +def project_document(): + return { + "format": "nnnotes.song-package/1", + "target": "target.json", + "template_song": 1, + "song": {"id": 7, "title": "Example", "performer": "Artist", "bands": [], "tags": ["Example"]}, + "assets": { + "jacket": "jacket.png", + "audio": {"full": {"file": "tone.wav"}, "preview": {"from": "full", "start_ms": 0, "duration_ms": 10}}, + }, + "charts": { + name: {"file": "chart.json", "score_id": 700 + i, "level": i + 1} + for i, name in enumerate(("easy", "normal", "hard", "expert")) + }, + } + + +def write_json(path, value): + path.write_text(json.dumps(value), encoding="utf-8") + + +def resource_templates(): + from UnityPy.helpers.Tpk import get_typetree_node + from UnityPy.helpers.UnityVersion import UnityVersion + + from nnnotes.unity.pack import BundleBuilder + from tests.helpers.packassets import VERSION, defaults, template + + node = get_typetree_node(213, UnityVersion.from_str(VERSION)) + fields = defaults(node) + pair = fields["m_RenderDataKey"] + fields["m_RenderDataKey"] = (pair["first"], pair["second"]) + fields["m_Name"] = "jacket" + fields["m_RD"]["texture"] = {"m_FileID": 0, "m_PathID": 30} + builder = BundleBuilder(template(), new=False) + builder.apply( + { + "type": "serialized", + "name": "jacket", + "path_id": 50, + "schema": {"class_id": 213, "type_hash": "12" * 16, "nodes": node.to_dict_list()}, + "fields": fields, + "container": "jacket", + } + ) + jacket = builder.build().data + builder = BundleBuilder(template(), new=False) + builder.apply( + {"type": "TextAsset", "name": "text", "op": "replace", "data": bytes(byte ^ 0x5A for byte in acb_template())} + ) + nodes = [ + [0, "MonoBehaviour", "Base", 0], + [1, "string", "m_Name", 0x4000], + [1, "string", "_cueSheetName", 0x4000], + [1, "vector", "_chunks", 0], + [2, "Array", "Array", 0x4000], + [3, "int", "size", 0], + [3, "PPtr", "data", 0], + [4, "int", "m_FileID", 0], + [4, "SInt64", "m_PathID", 0], + ] + builder.apply( + { + "type": "serialized", + "name": "tone", + "path_id": 50, + "schema": {"class_id": 114, "type_hash": "12" * 16, "script_id": "34" * 16, "nodes": nodes}, + "fields": {"m_Name": "tone", "_cueSheetName": "tone", "_chunks": [{"m_FileID": 0, "m_PathID": 20}]}, + "container": "sound", + } + ) + return {"chart": template(), "jacket": jacket, "full": builder.build().data} + + +def package_inputs(root, monkeypatch): + import hashlib + + from PIL import Image + + from nnnotes.source import catalogbuild, master + from tests.helpers import synth + from tests.helpers.packassets import wav + + monkeypatch.setenv("NNNOTES_MASTER_KEY", synth.MASTER_KEY.hex()) + monkeypatch.setenv("NNNOTES_MASTER_IV", synth.MASTER_IV.hex()) + tables = song_tables() + directory = root / "master" + directory.mkdir() + listed = [] + key = master.MasterKey(synth.MASTER_KEY, synth.MASTER_IV) + for name, table in tables.items(): + data = master.encode(json.dumps(table).encode(), key) + (directory / (name + ".bin")).write_bytes(data) + listed.append({"name": name + ".bin", "size": len(data), "hash": hashlib.sha256(data).hexdigest()}) + write_json(directory / "MasterManifest.json", {"version": "example", "files": listed}) + inputs = resource_templates() + bundles = root / "bundles" + bundles.mkdir() + for name, data in inputs.items(): + (bundles / (name + ".bundle")).write_bytes(data) + entries = [] + for key, name, internal, resource in [ + ("Live/MusicScore/chart", "chart", "assets/text.bytes", "UnityEngine.TextAsset"), + ("Image/Jacket/jacket", "jacket", "jacket", "UnityEngine.Sprite"), + ("Image/Jacket/small/jacket", "jacket", "jacket", "UnityEngine.Sprite"), + ("Cri/Sound/tone", "full", "sound", "Example.SplitAcbData"), + ]: + entries.append((key, internal, name, resource)) + # Several asset locations can depend on the same bundle location. + locations = [] + for name in inputs: + locations.extend( + catalogbuild.bundle_entries( + f"https://example.com/{name}.bundle", + name, + [], + hash=hashlib.md5(inputs[name]).hexdigest(), + crc=0, + bundle_size=len(inputs[name]), + ) + ) + for key, internal, name, resource in entries: + locations.append( + { + "primary_key": key, + "internal_id": internal, + "resource_type": resource, + "dependency_keys": [name + ".bundle"], + } + ) + (root / "catalog.bin").write_bytes(catalogbuild.rebuild(synth.CatalogWriter().build([]), add=locations)) + (root / "template.acb").write_bytes(acb_template()) + Image.new("RGBA", (2, 2), "red").save(root / "jacket.png") + (root / "tone.wav").write_bytes(wav(rate=48000)[0]) + write_json( + root / "chart.json", + { + "score": { + "events": {"bpm": [{"t": 0, "bpm": 120}], "sig": [{"t": 0, "sig": [4, 4]}]}, + "notes": [{"t": 1, "pos": 8, "size": 3}], + } + }, + ) + write_json( + root / "target.json", + { + "format": "nnnotes.pack-target/1", + "catalog": "catalog.bin", + "master": "master", + "bundles": "bundles", + "bundle_base_url": "https://example.com/custom", + "audio": {"cipher_type": 0, "templates": {"full": "template.acb", "preview": "template.acb"}}, + }, + ) + write_json(root / "song.json", project_document()) + return root / "song.json" diff --git a/tests/helpers/synth.py b/tests/helpers/synth.py index 7adb86b..1353c12 100644 --- a/tests/helpers/synth.py +++ b/tests/helpers/synth.py @@ -7,9 +7,11 @@ import gzip import hashlib import json +import lzma import re import struct +import lz4.block from Crypto.Cipher import AES # obviously fake keys @@ -147,6 +149,72 @@ def encrypt_bundle(data: bytes, filename: str, key: bytes = BUNDLE_KEY, seed: by return bytes(a ^ b for a, b in zip(head, stream)) + data[16384:] +# ---------------------------------------------------------------- UnityFS container +def _lzma_raw(data: bytes) -> bytes: + """LZMA1 raw with the 5-byte properties Unity's container puts in front of the stream.""" + dictionary = 1 << 20 + compressor = lzma.LZMACompressor( + format=lzma.FORMAT_RAW, filters=[{"id": lzma.FILTER_LZMA1, "dict_size": dictionary}] + ) + return struct.pack(" bytes: + if compression == "none": + return data + if compression == "lz4": + return lz4.block.compress(data, store_size=False) + if compression == "lzma": + return _lzma_raw(data) + raise ValueError(compression) + + +def unityfs( + payloads: list[bytes], + *, + compression: str = "none", + table_at_end: bool = False, + padding: bool = False, + version: int = 8, + bits: int | None = None, + extra_flags: int = 0, +) -> bytes: + """A UnityFS container holding `payloads` as its blocks, written independently of `nnnotes.unity.bundlefs`. + + `compression` ("none", "lz4", "lzma") applies to the block table and to every block; `table_at_end` moves the + table behind the data and `padding` aligns the data to 16 bytes — the `0x80` and `0x200` header flags. `bits` + overrides the compression field of the flags with any value, and `extra_flags` sets further flag bits. + """ + kind = {"none": 0, "lzma": 1, "lz4": 2}[compression] + flags = (kind if bits is None else bits) | (0x80 if table_at_end else 0) | (0x200 if padding else 0) + flags |= extra_flags + blocks = b"".join(_compress(payload, compression) for payload in payloads) + parts = [(payload, len(_compress(payload, compression)), kind) for payload in payloads] + table = b"\0" * 16 + struct.pack(">I", len(parts)) + for payload, compressed, block_bits in parts: + table += struct.pack(">IIH", len(payload), compressed, block_bits) + table += struct.pack(">I", 0) # no directory entries + packed = _compress(table, compression) + + head = bytearray(b"UnityFS\0" + struct.pack(">I", version)) + head += b"5.x.x\0" + b"6000.3.12f1\0" + size_at = len(head) + head += struct.pack(">qIII", 0, len(packed), len(table), flags) + if version >= 7: + head += bytes((-len(head)) % 16) + + out = bytearray(head) + if table_at_end: + out += bytes((-len(out)) % 16) if padding else b"" + out += blocks + packed + else: + out += packed + out += bytes((-len(out)) % 16) if padding else b"" + out += blocks + struct.pack_into(">q", out, size_at, len(out)) + return bytes(out) + + # ---------------------------------------------------------------- binary catalog (format version 2) RM = "UnityEngine.ResourceManagement.ResourceProviders." RM_ASSEMBLY = "Unity.ResourceManager, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null" diff --git a/tests/packing/test_audio.py b/tests/packing/test_audio.py new file mode 100644 index 0000000..d157a40 --- /dev/null +++ b/tests/packing/test_audio.py @@ -0,0 +1,50 @@ +import numpy as np +import pytest + +from nnnotes.packing.audio import AudioOptions, AudioTools, _compare, _pcm, probe_audio + +TOOLS = AudioTools("ffmpeg", "ffprobe", "encoder", "decoder") + + +def test_pcm_crop_fades_and_gain_preserve_the_decoded_timeline(tmp_path, monkeypatch): + source = np.full((4800, 2), 0.5, dtype=" bytes: + return synth.CatalogWriter().build( + [ + ("x_01.bundle", BUNDLE, []), + ("Char/A", "Assets/Game/Char/A.prefab", [0]), + ] + ) + + +def provider_catalog(*providers: str) -> bytes: + return synth.CatalogWriter().build( + [ + ( + f"source_{i}.bundle", + synth.remote(f"source_{i}.bundle"), + [], + {"provider": provider, "extra": {"flags": 8, "timeout": 60, "retryCount": 2}}, + ) + for i, provider in enumerate(providers) + ] + ) + + +def test_bundle_settings_inherit_a_unique_provider_and_require_ambiguous_selection(): + assert catalogbuild.bundle_settings(provider_catalog(CUSTOM_PROVIDER)) == (CUSTOM_PROVIDER, {}) + assert catalogbuild.bundle_settings(synth.CatalogWriter().build([])) == (catalogbuild.BUNDLE_PROVIDER, {}) + mixed = provider_catalog(CUSTOM_PROVIDER, catalogbuild.BUNDLE_PROVIDER) + with pytest.raises(ValueError, match="multiple bundle providers"): + catalogbuild.bundle_settings(mixed) + assert catalogbuild.bundle_settings(mixed, provider="ExplicitBundleProvider") == ("ExplicitBundleProvider", {}) + + +def test_bundle_template_inherits_common_options_and_explicit_fields_take_precedence(): + catalog = provider_catalog(CUSTOM_PROVIDER, catalogbuild.BUNDLE_PROVIDER) + provider, options = catalogbuild.bundle_settings( + catalog, template="source_0.bundle", request_options={"timeout": 120} + ) + assert provider == CUSTOM_PROVIDER + assert options == {"flags": 8, "timeout": 120, "redirectLimit": 32, "retryCount": 2} + assert ( + catalogbuild.bundle_settings( + catalog, template=synth.remote("source_0.bundle"), provider="ExplicitBundleProvider" + )[0] + == "ExplicitBundleProvider" + ) + with pytest.raises(ValueError, match="select exactly one"): + catalogbuild.bundle_settings(catalog, template="absent.bundle") + + +def test_generated_and_low_level_bundle_locations_use_the_source_provider(): + catalog = provider_catalog(CUSTOM_PROVIDER) + generated = catalogbuild.bundle_entries( + synth.remote("generated.bundle"), "generated", [], hash="11" * 16, crc=0, bundle_size=32, catalog=catalog + ) + assert generated[0]["provider"] == CUSTOM_PROVIDER + low_level = {"primary_key": "low.bundle", "internal_id": synth.remote("low.bundle"), "extra_data": {"crc": 0}} + rebuilt = catalogbuild.rebuild(catalog, add=[*generated, low_level]) + assert all(loc["provider"] == CUSTOM_PROVIDER for loc in addressables.parse_locations(rebuilt)) + + +@pytest.mark.parametrize("provider", ["", 4, "bad\0provider"]) +def test_bundle_provider_rejects_invalid_identifiers(provider): + with pytest.raises(ValueError, match="nonempty string"): + catalogbuild.bundle_settings(provider=provider) + + +def test_location_replacement_preserves_shared_keys_and_dependency_targets(): + data = catalogbuild.rebuild( + small_catalog(), + add=[ + {"primary_key": "Example/Text", "internal_id": "assets/text.bytes", "aliases": ["shared", 42]}, + {"primary_key": "Example/Text", "internal_id": "assets/other.bytes", "aliases": ["shared", 42]}, + ], + ) + original = addressables.parse_locations(data) + target = next(loc for loc in original if loc["internal_id"] == "assets/text.bytes") + output = catalogbuild.rebuild( + data, replace={target["offset"]: {"primary_key": "Example/Updated", "aliases": ["new", 43]}} + ) + locations = {loc["internal_id"]: loc for loc in addressables.parse_locations(output)} + memberships = {key["key"]: key["locations"] for key in addressables.parse_keys(output)} + assert ( + memberships["Example/Text"] + == memberships["shared"] + == memberships[42] + == [locations["assets/other.bytes"]["offset"]] + ) + assert ( + memberships["Example/Updated"] + == memberships["new"] + == memberships[43] + == [locations["assets/text.bytes"]["offset"]] + ) + assert locations["Assets/Game/Char/A.prefab"]["dependencies"] == [locations[BUNDLE]["offset"]] + + +def test_edit_entries_preserve_multiple_asset_locations_and_ignore_other_bundles(): + data = synth.CatalogWriter().build( + [ + ("source.bundle", synth.remote("source.bundle"), [], {"provider": CUSTOM_PROVIDER}), + ("other.bundle", synth.remote("other.bundle"), []), + ( + "Example/Text", + "assets/text.bytes", + [0, 1], + {"provider": "Example.TextProvider", "type": "UnityEngine.TextAsset"}, + ), + ( + "Example/Second", + "assets/text.bytes", + [0], + {"provider": "Example.SecondProvider", "type": "UnityEngine.TextAsset"}, + ), + ( + "Example/Text", + "assets/text.bytes", + [1], + {"provider": "Example.OtherProvider", "type": "UnityEngine.TextAsset"}, + ), + ] + ) + added, replacements = catalogbuild.edit_entries( + data, + synth.remote("source.bundle"), + "source", + [], + [{"source_internal_ids": ["assets/text.bytes"], "internal_id": "assets/new.bytes", "overrides": {}}], + hash="a1" * 16, + crc=123, + bundle_size=456, + ) + assert added == [] + output = catalogbuild.rebuild(data, replace=replacements) + locations = addressables.parse_locations(output) + changed = [loc for loc in locations if loc["internal_id"] == "assets/new.bytes"] + assert {loc["primary_key"] for loc in changed} == {"Example/Text", "Example/Second"} + assert {loc["provider"] for loc in changed} == {"Example.TextProvider", "Example.SecondProvider"} + assert all(loc["resource_type"] == "UnityEngine.TextAsset" for loc in changed) + unchanged = next(loc for loc in locations if loc["provider"] == "Example.OtherProvider") + assert unchanged["internal_id"] == "assets/text.bytes" + bundle = next(loc for loc in locations if loc["primary_key"] == "source.bundle") + assert (bundle["extra_data"]["crc"], bundle["extra_data"]["bundleSize"]) == (123, 456) + + +def signatures(data: bytes): + return sorted( + ( + loc["primary_key"] or "", + loc["internal_id"] or "", + loc["resource_type"] or "", + (loc.get("extra_data") or {}).get("hash"), + (loc.get("extra_data") or {}).get("bundleSize"), + ) + for loc in addressables.parse_locations(data) + ) + + +def test_rebuild_without_changes_keeps_every_entry(): + data = small_catalog() + out = catalogbuild.rebuild(data) + assert signatures(out) == signatures(data) + assert len(addressables.parse_keys(out)) == len(addressables.parse_keys(data)) + + +def test_rebuild_adds_a_bundle_and_its_asset(): + data = small_catalog() + bundle = { + "primary_key": "y_02.bundle", + "internal_id": "https://dummy.net/asset/Android/y_02.bundle", + "extra_data": { + "hash": "11" * 16, + "bundleName": "2222", + "crc": 0, + "bundleSize": 1234, + "flags": 0, + "timeout": 0, + "redirectLimit": 0, + "retryCount": 0, + }, + } + asset = { + "primary_key": "Live/MusicScore/9001/9001_00", + "internal_id": "Assets/AddressableResources/Live/MusicScore/9001/9001_00.bytes", + "resource_type": "UnityEngine.TextAsset", + "provider": "UnityEngine.ResourceManagement.ResourceProviders.BundledAssetProvider", + "dependency_on_added": [0], + } + out = catalogbuild.rebuild(data, add=[bundle, asset]) + locations = {loc["primary_key"]: loc for loc in addressables.parse_locations(out)} + assert "y_02.bundle" in locations and locations["y_02.bundle"]["resource_type"].endswith("IAssetBundleResource") + music = locations["Live/MusicScore/9001/9001_00"] + assert music["resource_type"] == "UnityEngine.TextAsset" + assert music["dependencies"] == [locations["y_02.bundle"]["offset"]] + keys = {k["key"]: k for k in addressables.parse_keys(out)} + assert keys["Live/MusicScore/9001/9001_00"]["locations"] == [music["offset"]] + assert (locations["y_02.bundle"]["extra_data"] or {})["bundleSize"] == 1234 + + +def test_rebuild_drops_a_keyed_location(): + data = small_catalog() + out = catalogbuild.rebuild(data, drop={"Char/A"}) + assert "Char/A" not in {loc["primary_key"] for loc in addressables.parse_locations(out)} + assert "x_01.bundle" in {loc["primary_key"] for loc in addressables.parse_locations(out)} + + +def test_rebuild_updates_a_bundles_size_and_checksum(): + data = small_catalog() + before = {loc["primary_key"]: (loc.get("extra_data") or {}) for loc in addressables.parse_locations(data)} + out = catalogbuild.rebuild(data, update={BUNDLE_NAME: {"crc": 0x1234, "bundleSize": 999}}) + after = {loc["primary_key"]: (loc.get("extra_data") or {}) for loc in addressables.parse_locations(out)} + assert (after["x_01.bundle"]["crc"], after["x_01.bundle"]["bundleSize"]) == (0x1234, 999) + assert after["x_01.bundle"]["hash"] == before["x_01.bundle"]["hash"] # a field we did not name is kept + assert len(addressables.parse_locations(out)) == len(addressables.parse_locations(data)) + kept = [s for s in signatures(out) if s[0] != "x_01.bundle"] + assert kept == [s for s in signatures(data) if s[0] != "x_01.bundle"] + + +def test_rebuild_does_not_invent_request_options(): + data = small_catalog() + out = catalogbuild.rebuild(data, update={"A.prefab": {"crc": 1, "bundleSize": 2}}) + asset = {loc["primary_key"]: loc for loc in addressables.parse_locations(out)}["Char/A"] + assert asset["extra_data"] is None # not a bundle location: nothing to correct + + +def test_add_to_existing_key_preserves_all_locations_and_dependencies(): + data = small_catalog() + original = {loc["primary_key"]: loc for loc in addressables.parse_locations(data)} + out = catalogbuild.rebuild( + data, + add=[ + { + "primary_key": "Char/A", + "internal_id": "Assets/Game/Char/Alternative.prefab", + "resource_type": "UnityEngine.GameObject", + "dependency_keys": ["x_01.bundle"], + } + ], + ) + locations = addressables.parse_locations(out) + keys = addressables.parse_keys(out) + assert len(locations) == 3 + assert len(keys) == 2 + bundle, first, second = locations + assert first["internal_id"] == original["Char/A"]["internal_id"] + assert second["internal_id"] == "Assets/Game/Char/Alternative.prefab" + assert first["dependencies"] == second["dependencies"] == [bundle["offset"]] + assert next(key for key in keys if key["key"] == "Char/A")["locations"] == [first["offset"], second["offset"]] + + +def test_added_assets_can_combine_existing_offsets_added_indices_and_keys(): + data = small_catalog() + old_bundle = addressables.parse_locations(data)[0]["offset"] + out = catalogbuild.rebuild( + data, + add=[ + { + "primary_key": "shared.bundle", + "internal_id": "https://example.invalid/shared.bundle", + "extra_data": {"hash": "ab" * 16, "bundleName": "shared", "crc": 7, "bundleSize": 50}, + }, + { + "primary_key": "Texture/Checker", + "internal_id": "Assets/Checker.png", + "resource_type": "UnityEngine.Texture2D", + "dependencies": [old_bundle], + "dependency_on_added": [0], + "dependency_keys": ["shared.bundle", "x_01.bundle"], + }, + ], + ) + locations = {loc["primary_key"]: loc for loc in addressables.parse_locations(out)} + assert locations["Texture/Checker"]["dependencies"] == [ + locations["x_01.bundle"]["offset"], + locations["shared.bundle"]["offset"], + ] + + +def test_bundle_entries_generate_options_aliases_and_resource_types(): + additions = catalogbuild.bundle_entries( + "{Runtime.RemoteRoot}/custom.bundle", + "custom_bundle", + [ + { + "primary_key": "Chart/Custom", + "internal_id": "Assets/Custom.bytes", + "resource_type": "UnityEngine.TextAsset", + "aliases": ["chart", 42], + }, + { + "primary_key": "Audio/Custom", + "internal_id": "Assets/Custom.wav", + "resource_type": "UnityEngine.AudioClip", + }, + ], + hash="AB" * 16, + crc=0xF2345678, + bundle_size=4096, + dependency_keys=["x_01.bundle"], + request_options={"flags": 4, "timeout": 120, "retryCount": 2}, + ) + out = catalogbuild.rebuild(small_catalog(), add=additions) + locations = {loc["primary_key"]: loc for loc in addressables.parse_locations(out)} + bundle = locations["custom.bundle"] + assert bundle["resource_type"] == catalogbuild.BUNDLE_TYPE + assert bundle["resource_type_assembly"] == catalogbuild.RM_ASSEMBLY + assert bundle["dependencies"] == [locations["x_01.bundle"]["offset"]] + opts = bundle["extra_data"] + assert (opts["bundleName"], opts["hash"], opts["crc"], opts["bundleSize"]) == ( + "custom_bundle", + "ab" * 16, + 0xF2345678, + 4096, + ) + assert (opts["timeout"], opts["redirectLimit"], opts["retryCount"], opts["flags"]) == (120, 32, 2, 4) + for key in ("Chart/Custom", "Audio/Custom"): + assert locations[key]["provider"] == catalogbuild.ASSET_PROVIDER + assert locations[key]["resource_type_assembly"] == catalogbuild.CORE_ASSEMBLY + assert locations[key]["dependencies"] == [bundle["offset"]] + keys = {key["key"]: key for key in addressables.parse_keys(out)} + assert keys[42]["type"] == "System.Int32" + assert keys[42]["locations"] == keys["chart"]["locations"] == [locations["Chart/Custom"]["offset"]] + rebuilt = catalogbuild.rebuild(out) + assert signatures(rebuilt) == signatures(out) + assert [(key["key"], key["type"]) for key in addressables.parse_keys(rebuilt)] == [ + (key["key"], key["type"]) for key in addressables.parse_keys(out) + ] + + +def test_full_internal_id_update_takes_precedence_and_updates_added_entries(): + added = catalogbuild.bundle_entries( + "https://example.invalid/new/x_01.bundle", + "new_bundle", + [], + hash="00" * 16, + crc=0, + bundle_size=0, + primary_key="new.bundle", + ) + out = catalogbuild.rebuild( + small_catalog(), + add=added, + update={ + "x_01.bundle": {"crc": 1, "bundleSize": 2}, + BUNDLE: {"crc": 3, "bundleSize": 4}, + }, + ) + locations = {loc["primary_key"]: loc for loc in addressables.parse_locations(out)} + assert locations["x_01.bundle"]["extra_data"]["crc"] == 3 + assert locations["new.bundle"]["extra_data"]["crc"] == 1 + + +@pytest.mark.parametrize( + "dependency", + [ + {"dependencies": [123456]}, + {"dependency_on_added": [-1]}, + {"dependency_on_added": [1]}, + {"dependency_keys": ["unknown.bundle"]}, + ], +) +def test_unavailable_added_dependencies_are_rejected(dependency): + with pytest.raises(ValueError, match="dependency.*unavailable"): + catalogbuild.rebuild( + small_catalog(), + add=[ + {"primary_key": "Chart/Custom", "internal_id": "Assets/Custom.bytes", **dependency}, + ], + ) + + +@pytest.mark.parametrize("hash", ["a" * 30, "gg" * 16, "ab " * 10 + "ab"]) +def test_bundle_hash_has_exactly_sixteen_bytes(hash): + with pytest.raises(ValueError, match="32 hexadecimal digits"): + catalogbuild.bundle_entries( + "https://example.invalid/custom.bundle", "custom", [], hash=hash, crc=0, bundle_size=0 + ) + + +def test_drop_removes_empty_lookup_keys(): + out = catalogbuild.rebuild(small_catalog(), drop={"Char/A"}) + assert [key["key"] for key in addressables.parse_keys(out)] == ["x_01.bundle"] + + +def semantic_catalog(data): + header = addressables.parse_header(data) + header.pop("keysOffset") + locations = addressables.parse_locations(data) + index = {loc["offset"]: i for i, loc in enumerate(locations)} + entries = [] + for loc in locations: + entry = dict(loc) + entry.pop("offset") + entry["dependencies"] = [index[off] for off in loc["dependencies"]] + entries.append(entry) + keys = [dict(key, locations=[index[off] for off in key["locations"]]) for key in addressables.parse_keys(data)] + return header, entries, keys + + +def test_roundtrip_preserves_header_unicode_options_types_and_typed_aliases(): + additions = catalogbuild.bundle_entries( + "https://example.invalid/custom.bundle", + "custom_bundle", + [ + { + "primary_key": "Chart/自作", + "internal_id": "Assets/自作.bytes", + "resource_type": "UnityEngine.TextAsset", + "aliases": ["42", 42], + "dependency_hash": -123456, + }, + ], + hash="fa" * 16, + crc=0xFFFFFFFF, + bundle_size=0xFFFFFFFF, + request_options={"timeout": 32767, "redirectLimit": 255, "retryCount": 255, "flags": 31}, + ) + original = catalogbuild.rebuild(small_catalog(), add=additions) + out = catalogbuild.rebuild(original) + assert semantic_catalog(out) == semantic_catalog(original) + typed_keys = {(key["type"], key["key"]) for key in addressables.parse_keys(out)} + assert ("System.String", "42") in typed_keys + assert ("System.Int32", 42) in typed_keys + + +def test_roundtrip_preserves_null_request_common_and_bundle_name(): + data = synth.CatalogWriter().build( + [ + ("empty.bundle", synth.remote("empty.bundle"), [], {"extra": {"hash": None, "bundleName": None}}), + ] + ) + parsed = addressables.parse_locations(data)[0] + extra = parsed["extra_data"] + common = None + for field in ("timeout", "redirectLimit", "retryCount", "flags"): + extra[field] = common + out = catalogbuild.rebuild(small_catalog(), add=[parsed]) + location = addressables.parse_locations(out)[-1] + assert location["extra_data"]["bundleName"] is None + assert all(location["extra_data"][field] is None for field in ("timeout", "redirectLimit", "retryCount", "flags")) + assert semantic_catalog(catalogbuild.rebuild(out)) == semantic_catalog(out) + + +def test_unsupported_extra_data_is_reported(): + data = synth.CatalogWriter().build( + [ + ("custom", "Assets/Custom.asset", [], {"extra_object": ("UnityEngine.CustomOptions", bytes(8))}), + ] + ) + with pytest.raises(ValueError, match="unsupported catalog extra data type"): + catalogbuild.rebuild(data) + + +def test_bundle_update_fields_are_validated(): + with pytest.raises(ValueError, match="unsupported fields"): + catalogbuild.rebuild(small_catalog(), update={BUNDLE_NAME: {"crcc": 7}}) + + +def test_replacing_a_dropped_bundle_redirects_existing_dependencies(): + replacements = catalogbuild.bundle_entries( + "https://example.invalid/custom.bundle", + "custom", + [], + primary_key="x_01.bundle", + hash="ab" * 16, + crc=3, + bundle_size=100, + ) + out = catalogbuild.rebuild(small_catalog(), add=replacements, drop={"x_01.bundle"}) + locations = {loc["primary_key"]: loc for loc in addressables.parse_locations(out)} + assert locations["Char/A"]["dependencies"] == [locations["x_01.bundle"]["offset"]] + assert locations["x_01.bundle"]["internal_id"] == "https://example.invalid/custom.bundle" + + +def test_dropping_a_required_dependency_is_reported(): + with pytest.raises(ValueError, match="dependency offset.*unavailable"): + catalogbuild.rebuild(small_catalog(), drop={"x_01.bundle"}) + + +@pytest.mark.parametrize("fields", [{"crc": -1}, {"bundleSize": 2**32}, {"crc": 1.5}]) +def test_request_options_integer_fields_are_validated(fields): + with pytest.raises(ValueError, match="must be an integer"): + catalogbuild.rebuild(small_catalog(), update={BUNDLE_NAME: fields}) diff --git a/tests/source/test_pack.py b/tests/source/test_pack.py new file mode 100644 index 0000000..3b0ae9c --- /dev/null +++ b/tests/source/test_pack.py @@ -0,0 +1,76 @@ +import json + +import numpy as np +import pytest + +from nnnotes.source import addressables, master +from nnnotes.source.addressables import BundleKey +from nnnotes.source.master import MasterKey +from tests.helpers import synth + +KEY = MasterKey(synth.MASTER_KEY, synth.MASTER_IV) +BUNDLE_KEY = BundleKey(synth.BUNDLE_KEY, synth.BUNDLE_SEED) +TABLE = synth.MASTER_TABLE +JSON = json.dumps(TABLE, ensure_ascii=False).encode("utf-8") + +# the same known answer tests/source/test_master.py decrypts: the all-zero block under the all-zero key +KAT_C1 = bytes.fromhex("C6227E7740B7E53B5CB77865278EAB0726F62366D9AABAD908936123A1FC8AF3") + + +def test_encrypt_known_answer(): + rk = master.round_keys(bytes(32)) + out = master.encrypt_blocks(np.zeros((1, 32), dtype=np.uint8), rk) + assert out.tobytes() == KAT_C1 + + +def test_encode_matches_the_test_encryptor(): + # the independent pure-Python encryptor of the tests builds the same file, byte for byte + assert master.encode(JSON, KEY, prefix=bytes(range(64))) == synth.master_file(TABLE, prefix=bytes(range(64))) + + +def test_encode_decode_round_trip(): + for prefix in (b"", bytes(range(64))): + raw = master.encode(JSON, KEY, prefix=prefix) + assert raw[:64] == (prefix or bytes(64)) + assert master.decode(raw, KEY) == JSON + + +def test_encode_pads_to_a_block(): + raw = master.encode(b"{}", KEY) + assert (len(raw) - master.PREFIX) % master.BLOCK == 0 + + +def test_encode_rejects_a_short_prefix(): + with pytest.raises(ValueError, match="64 bytes"): + master.encode(JSON, KEY, prefix=bytes(10)) + + +def test_pack_bundle_round_trip(): + plain = synth.fake_bundle("a_01.bundle") + stored = addressables.pack(plain, "a_01.bundle", BUNDLE_KEY) + assert not stored.startswith(b"UnityFS") and stored[addressables.ENCRYPTED_BYTES :] == plain[16384:] + assert addressables.decrypt(stored, "a_01.bundle", BUNDLE_KEY) == plain + + +def test_pack_bundle_matches_the_test_encryptor(): + plain = synth.fake_bundle("a_01.bundle") + assert addressables.pack(plain, "a_01.bundle", BUNDLE_KEY) == synth.encrypt_bundle(plain, "a_01.bundle") + + +def test_pack_bundle_requires_unityfs(): + with pytest.raises(ValueError, match="UnityFS"): + addressables.pack(b"not a bundle", "a.bundle", BUNDLE_KEY) + + +def test_bundle_name_seeds_the_nonce(): + plain = synth.fake_bundle("a_01.bundle") + stored = addressables.pack(plain, "a_01.bundle", BUNDLE_KEY) + assert addressables.decrypt(stored, "b_02.bundle", BUNDLE_KEY) != plain + + +def test_pack_is_the_inverse_of_a_decrypted_file(tmp_path): + # what `bundle pull` writes (UnityFS) goes back to what the CDN serves + p = tmp_path / "b_02.bundle" + p.write_bytes(synth.fake_bundle("b_02.bundle")) + stored = addressables.pack(p.read_bytes(), p.name, BUNDLE_KEY) + assert addressables.decrypt(stored, p.name, BUNDLE_KEY) == p.read_bytes() diff --git a/tests/unity/test_bundlefs.py b/tests/unity/test_bundlefs.py new file mode 100644 index 0000000..40f586b --- /dev/null +++ b/tests/unity/test_bundlefs.py @@ -0,0 +1,99 @@ +import struct +import zlib + +import pytest + +from nnnotes.unity import bundlefs +from tests.helpers import synth + +PAYLOADS = [b"file-table", b"x" * 5000, b""] + + +def test_content_is_the_blocks_in_order(): + data = synth.unityfs(PAYLOADS) + assert bundlefs.content(data) == b"".join(PAYLOADS) + + +@pytest.mark.parametrize("compression", ["none", "lz4", "lzma"]) +def test_every_compression_reads_back(compression): + data = synth.unityfs(PAYLOADS, compression=compression) + assert bundlefs.content(data) == b"".join(PAYLOADS) + + +@pytest.mark.parametrize("compression", ["none", "lz4", "lzma"]) +def test_a_block_table_at_the_end(compression): + data = synth.unityfs(PAYLOADS, compression=compression, table_at_end=True) + assert bundlefs.content(data) == b"".join(PAYLOADS) + + +def test_padded_block_data(): + data = synth.unityfs(PAYLOADS, compression="lz4", padding=True, table_at_end=True) + assert bundlefs.parse_header(data).flags & bundlefs.BLOCK_INFO_PADDING + assert bundlefs.content(data) == b"".join(PAYLOADS) + + +def test_content_crc_is_the_crc_of_the_content_not_of_the_file(): + data = synth.unityfs(PAYLOADS, compression="lz4") + content = b"".join(PAYLOADS) + assert bundlefs.content_crc(data) == zlib.crc32(content) + assert bundlefs.content_crc(data) != zlib.crc32(data) + + +def test_the_header_reports_the_layout(): + plain = bundlefs.parse_header(synth.unityfs(PAYLOADS)) + assert (plain.version, plain.compression, plain.table_at_end) == (8, bundlefs.COMPRESSION_NONE, False) + lz4 = bundlefs.parse_header(synth.unityfs(PAYLOADS, compression="lz4", table_at_end=True)) + assert (lz4.compression, lz4.table_at_end) == (bundlefs.COMPRESSION_LZ4, True) + + +def test_a_file_that_is_not_a_bundle_is_rejected(): + with pytest.raises(ValueError): + bundlefs.parse_header(b"PK\x03\x04 not a bundle") + with pytest.raises(ValueError): + bundlefs.content_crc(b"UnityFS\0 short") + + +def test_an_unknown_compression_is_rejected(): + data = synth.unityfs(PAYLOADS, bits=5) # neither none, LZMA nor LZ4 + assert bundlefs.parse_header(data).compression == 5 + with pytest.raises(ValueError): + bundlefs.content(data) + + +def test_a_container_that_carries_its_own_key_is_rejected(): + data = synth.unityfs(PAYLOADS, extra_flags=0x400) + with pytest.raises(ValueError): + bundlefs.parse_header(data) + + +def test_file_size_mismatch_is_rejected(): + data = synth.unityfs(PAYLOADS) + with pytest.raises(ValueError, match="file size"): + bundlefs.content(data[:-1]) + with pytest.raises(ValueError, match="file size"): + bundlefs.content(data + b"extra") + + +def test_block_size_mismatch_is_rejected(): + data = bytearray(synth.unityfs([b"payload"])) + at = bundlefs.parse_header(data).offset + bundlefs.HASH_SIZE + 4 + struct.pack_into(">I", data, at, 2) + with pytest.raises(ValueError, match="region size mismatch"): + bundlefs.content(data) + + +def test_block_count_is_checked_before_reading_records(): + data = bytearray(synth.unityfs([b"payload"])) + at = bundlefs.parse_header(data).offset + bundlefs.HASH_SIZE + struct.pack_into(">I", data, at, 2**32 - 1) + with pytest.raises(ValueError, match="block count"): + bundlefs.content(data) + + +def test_compressed_block_cannot_overlap_block_information(): + data = bytearray(synth.unityfs([b"payload"], table_at_end=True)) + header = bundlefs.parse_header(data) + at = len(data) - header.info_compressed + bundlefs.HASH_SIZE + 4 + 4 + struct.pack_into(">I", data, at, len(data)) + with pytest.raises(ValueError, match="content region"): + bundlefs.content(data) diff --git a/tests/unity/test_pack.py b/tests/unity/test_pack.py new file mode 100644 index 0000000..bf2f094 --- /dev/null +++ b/tests/unity/test_pack.py @@ -0,0 +1,672 @@ +import hashlib +import io +import wave + +import pytest +import UnityPy +from PIL import Image + +from nnnotes.unity import bundlefs +from nnnotes.unity.pack import ( + BundleBuilder, + TextAsset, + build_manifest, + build_text_asset, + parse_text_asset, + replace_text_asset, + wav_to_fsb5, + write_outputs, +) +from tests.helpers.packassets import template, wav + + +def test_text_asset_round_trip_keeps_binary_payload(): + payload = bytes(range(256)) * 3 # not valid UTF-8, like gzip-compressed JSON + raw = build_text_asset("0001_00", payload) + asset = parse_text_asset(raw) + assert asset == TextAsset("0001_00", payload) + assert len(raw) % 4 == 0 + + +def test_text_asset_pads_the_name_before_the_payload(): + raw = build_text_asset("abcd", b"xy") + assert raw[:4] == (4).to_bytes(4, "little") and raw[4:8] == b"abcd" + assert raw[8:12] == (2).to_bytes(4, "little") and raw[12:14] == b"xy" + assert len(raw) == 16 # 2 payload bytes padded to 4 + odd = build_text_asset("abc", b"") + assert odd[4:7] == b"abc" and odd[7] == 0 # the name is padded to 4 as well + + +def test_a_payload_that_looks_like_a_name_is_not_mistaken_for_one(): + first = build_text_asset("a", b"\x01\x00\x00\x00abcd") + raw = build_text_asset("a", parse_text_asset(first).payload) + assert parse_text_asset(raw).payload == b"\x01\x00\x00\x00abcd" + + +@pytest.mark.parametrize("raw", [b"", bytes(3), b"\x01\0\0\0a\0\0\0\x09\0\0\0short"]) +def test_text_asset_rejects_truncated_fields(raw): + with pytest.raises(ValueError): + parse_text_asset(raw) + + +@pytest.mark.parametrize("flags", [0x40, 0x41, 0x42, 0xC2, 0x242, 0x2C2]) +@pytest.mark.parametrize("stripped", [False, True]) +def test_new_bundle_preserves_schema_and_flags_and_contains_only_supplied_assets(tmp_path, flags, stripped): + payload = bytes(range(256)) + (tmp_path / "payload.bin").write_bytes(payload) + builder = BundleBuilder(template(flags=flags, stripped=stripped), bundle_name="example") + builder.apply( + { + "type": "TextAsset", + "name": "binary", + "file": "payload.bin", + "path_id": 7, + "container": "assets/binary.bytes", + "primary_key": "Example/Binary", + }, + base=tmp_path, + ) + result = builder.build() + env = UnityPy.load(result.data) + assert set(env.assets[0].objects) == {1, 7} + assert len(env.assets[0].types) == 2 + assert env.assets[0]._enable_type_tree is not stripped + assert env.file.dataflags == flags + assert set(env.file.files) == {"CAB-" + hashlib.md5(b"example").hexdigest()} + assert env.assets[0].unity_version == "2022.3.21f1" + assert parse_text_asset(env.assets[0].objects[7].get_raw_data()) == TextAsset("binary", payload) + assert env.container["assets/binary.bytes"].path_id == 7 + index = env.assets[0].objects[1].read_typetree() + assert index["m_Name"] == "example" and index["m_AssetBundleName"] == "example" + assert index["m_PreloadTable"] == [{"m_FileID": 0, "m_PathID": 7}] + assert result.crc == bundlefs.content_crc(result.data) + assert result.assets == [ + { + "primary_key": "Example/Binary", + "internal_id": "assets/binary.bytes", + "resource_type": "UnityEngine.TextAsset", + } + ] + + +def test_batch_add_replace_and_rename_keep_payload_and_update_container(tmp_path): + (tmp_path / "first").write_bytes(b"first") + (tmp_path / "second").write_bytes(b"second") + (tmp_path / "template.bundle").write_bytes(template()) + result = build_manifest( + { + "template": "template.bundle", + "bundle_name": "example", + "assets": [ + {"type": "TextAsset", "name": "first", "file": "first", "path_id": 9}, + {"op": "replace", "type": "TextAsset", "name": "first", "file": "second"}, + {"op": "rename", "name": "first", "new_name": "final", "new_container": "assets/final.bytes"}, + ], + }, + base=tmp_path, + ) + env = UnityPy.load(result.data) + assert set(env.container) == {"assets/final.bytes"} + assert parse_text_asset(env.assets[0].objects[9].get_raw_data()) == TextAsset("final", b"second") + + +@pytest.mark.parametrize("operation", ["rename", "replace"]) +def test_asset_operations_preserve_catalog_fields_and_allow_explicit_overrides(tmp_path, operation): + (tmp_path / "payload").write_bytes(b"payload") + builder = BundleBuilder(template(), bundle_name="example") + metadata = { + "primary_key": "Example/Text", + "aliases": ["Example/Alias"], + "dependency_keys": ["Example/Dependency"], + "provider": "Example.AssetProvider", + "resource_type": "Example.Text", + "resource_type_assembly": "Example.Assembly", + } + builder.apply( + {"type": "TextAsset", "name": "text", "file": "payload", "container": "assets/text.bytes", **metadata}, + base=tmp_path, + ) + spec = {"op": operation, "name": "text", "new_name": "updated"} + if operation == "replace": + spec.update(type="TextAsset", file="payload") + builder.apply(spec, base=tmp_path) + assert builder.build().assets == [{**metadata, "internal_id": "assets/text.bytes"}] + builder.apply( + { + "op": "rename", + "name": "updated", + "new_name": "final", + "primary_key": "Example/Final", + "aliases": [], + "provider": "Example.FinalProvider", + } + ) + assert builder.build().assets == [ + { + **metadata, + "internal_id": "assets/text.bytes", + "primary_key": "Example/Final", + "aliases": [], + "provider": "Example.FinalProvider", + } + ] + + +def test_container_rename_preserves_asset_keys_and_metadata(tmp_path): + (tmp_path / "payload").write_bytes(b"payload") + builder = BundleBuilder(template(), bundle_name="example") + builder.apply( + { + "type": "TextAsset", + "name": "text", + "file": "payload", + "container": "assets/text.bytes", + "aliases": ["Example/Alias"], + "provider": "Example.AssetProvider", + }, + base=tmp_path, + ) + builder.apply({"op": "rename", "name": "text", "new_name": "updated", "new_container": "assets/updated.bytes"}) + result = builder.build() + assert result.assets == [ + { + "primary_key": "assets/text.bytes", + "internal_id": "assets/updated.bytes", + "resource_type": "UnityEngine.TextAsset", + "aliases": ["Example/Alias"], + "provider": "Example.AssetProvider", + } + ] + assert set(UnityPy.load(result.data).container) == {"assets/updated.bytes"} + + +def test_catalog_edits_track_original_paths_and_explicit_fields_across_operations(tmp_path): + (tmp_path / "payload").write_bytes(b"payload") + builder = BundleBuilder(template(), new=False) + assert builder.build().catalog_edits == [] + builder.apply({"op": "rename", "name": "text", "new_name": "updated", "new_container": "assets/updated.bytes"}) + builder.apply( + { + "op": "replace", + "type": "TextAsset", + "name": "updated", + "file": "payload", + "aliases": ["Example/Alias"], + "provider": "Example.Provider", + }, + base=tmp_path, + ) + builder.apply( + { + "op": "rename", + "name": "updated", + "new_name": "final", + "new_container": "assets/final.bytes", + "aliases": [], + } + ) + assert builder.build().catalog_edits == [ + { + "source_internal_ids": ["assets/text.bytes"], + "internal_id": "assets/final.bytes", + "overrides": {"aliases": [], "provider": "Example.Provider"}, + } + ] + + +def test_catalog_edits_identify_added_assets_without_source_paths(tmp_path): + (tmp_path / "payload").write_bytes(b"payload") + builder = BundleBuilder(template(), new=False) + builder.apply( + {"type": "TextAsset", "name": "added", "file": "payload", "primary_key": "Example/Added"}, base=tmp_path + ) + builder.apply({"op": "rename", "name": "added", "new_name": "final", "new_container": "assets/final.bytes"}) + assert builder.build().catalog_edits == [ + { + "source_internal_ids": [], + "internal_id": "assets/final.bytes", + "overrides": {"primary_key": "Example/Added"}, + } + ] + + +def test_catalog_edits_preserve_each_original_path_and_record_merged_paths(tmp_path): + source = UnityPy.load(template()) + index = source.assets[0].objects[10] + fields = index.read_typetree() + fields["m_Container"].append(("assets/alias.bytes", dict(fields["m_Container"][0][1]))) + index.save_typetree(fields) + builder = BundleBuilder(source.file.save(packer="original"), new=False) + builder.apply({"op": "rename", "name": "text", "new_name": "updated"}) + assert builder.build().catalog_edits == [ + {"source_internal_ids": [key], "internal_id": key, "overrides": {}} + for key in ("assets/text.bytes", "assets/alias.bytes") + ] + builder.apply({"op": "rename", "name": "updated", "new_name": "final", "new_container": "assets/final.bytes"}) + assert builder.build().catalog_edits == [ + { + "source_internal_ids": ["assets/text.bytes", "assets/alias.bytes"], + "internal_id": "assets/final.bytes", + "overrides": {}, + } + ] + + +def test_replace_compatibility_keeps_existing_path_id_and_container(tmp_path): + src, out = tmp_path / "source.bundle", tmp_path / "new" / "output.bundle" + src.write_bytes(template()) + replace_text_asset(src, out, "text", b"\xff\xfe", new_name="renamed") + env = UnityPy.load(out.read_bytes()) + assert set(env.assets[0].objects) == {10, 20, 30, 40} + assert env.container["assets/text.bytes"].path_id == 20 + assert parse_text_asset(env.assets[0].objects[20].get_raw_data()) == TextAsset("renamed", b"\xff\xfe") + + +def test_png_texture_round_trip_preserves_pixels_and_dimensions(tmp_path): + image = Image.new("RGBA", (3, 2)) + image.putdata([(r, 30, 50, a) for r, a in ((10, 0), (20, 255), (30, 50), (40, 80), (50, 90), (60, 100))]) + image.save(tmp_path / "picture.png") + builder = BundleBuilder(template(), bundle_name="example") + builder.apply( + {"type": "Texture2D", "name": "picture", "file": "picture.png", "fields": {"m_TextureDimension": 3}}, + base=tmp_path, + ) + env = UnityPy.load(builder.build().data) + texture = env.assets[0].objects[2].read() + assert texture.image.size == image.size + assert texture.image.tobytes() == image.tobytes() + assert texture.m_TextureFormat == 4 and texture.m_MipCount == 1 + assert texture.m_TextureDimension == 2 and texture.m_ImageCount == 1 + assert texture.m_StreamData.size == 0 + + +def test_png_texture_clears_stripped_mip_metadata_from_the_template(tmp_path): + image = Image.new("RGBA", (8, 4), (10, 20, 30, 255)) + image.save(tmp_path / "picture.png") + source = UnityPy.load(template()) + texture = source.assets[0].objects[30] + fields = texture.read_typetree() + fields["m_MipsStripped"] = 3 + texture.save_typetree(fields) + builder = BundleBuilder(source.file.save(packer="original"), bundle_name="example") + builder.apply({"type": "Texture2D", "name": "picture", "file": "picture.png"}, base=tmp_path) + texture = UnityPy.load(builder.build().data).assets[0].objects[2].read() + assert texture.m_MipsStripped == 0 and texture.m_MipCount == 1 + assert texture.image.size == image.size and texture.image.tobytes() == image.tobytes() + + +@pytest.mark.parametrize("channels,width,rate", [(1, 2, 16000), (2, 2, 22050), (1, 2, 12345), (2, 2, 48000)]) +def test_audio_clip_wav_round_trip_preserves_pcm(tmp_path, channels, width, rate): + source, frames = wav(channels=channels, width=width, rate=rate) + (tmp_path / "tone.wav").write_bytes(source) + builder = BundleBuilder(template(), bundle_name="example") + builder.apply({"type": "AudioClip", "name": "tone", "file": "tone.wav"}, base=tmp_path) + env = UnityPy.load(builder.build().data) + audio = env.assets[0].objects[2].read() + with wave.open(io.BytesIO(next(iter(audio.samples.values())))) as decoded: + assert decoded.getnchannels() == channels + assert decoded.getsampwidth() == width + assert decoded.getframerate() == rate + assert decoded.readframes(decoded.getnframes()) == frames + assert audio.m_CompressionFormat == 0 and audio.m_SubsoundIndex == 0 + assert audio.m_Resource.m_Size > len(frames) + + +def test_audio_replacements_preserve_other_clips_across_saved_bundle_edits(tmp_path): + first, first_frames = wav() + second, second_frames = wav(channels=2, rate=22050) + replacement, replacement_frames = wav(channels=2, rate=48000) + final, final_frames = wav(rate=12345) + for name, data in (("first", first), ("second", second), ("replacement", replacement), ("final", final)): + (tmp_path / f"{name}.wav").write_bytes(data) + builder = BundleBuilder(template(), bundle_name="example") + for name in ("first", "second"): + builder.apply({"type": "AudioClip", "name": name, "file": f"{name}.wav"}, base=tmp_path) + bundle = tmp_path / "example.bundle" + bundle.write_bytes(builder.build().data) + original = UnityPy.load(bundle.read_bytes()) + original_resource = next(name for name in original.file.files if name.endswith(".resource")) + original_bytes = original.file.files[original_resource].bytes + + def decoded_clips(data): + result = {} + for obj in UnityPy.load(data).assets[0].objects.values(): + if obj.type.name != "AudioClip": + continue + clip = obj.read() + with wave.open(io.BytesIO(next(iter(clip.samples.values())))) as decoded: + result[clip.m_Name] = ( + decoded.getnchannels(), + decoded.getframerate(), + decoded.readframes(decoded.getnframes()), + ) + return result + + assert decoded_clips(bundle.read_bytes()) == { + "first": (1, 16000, first_frames), + "second": (2, 22050, second_frames), + } + for name, payload in (("first", "replacement"), ("second", "final")): + edited = BundleBuilder(bundle, new=False) + edited.apply({"op": "replace", "type": "AudioClip", "name": name, "file": f"{payload}.wav"}, base=tmp_path) + bundle.write_bytes(edited.build().data) + saved = UnityPy.load(bundle.read_bytes()) + assert saved.file.files[original_resource].bytes == original_bytes + expected = {"first": (2, 48000, replacement_frames)} + expected["second"] = (2, 22050, second_frames) if name == "first" else (1, 12345, final_frames) + assert decoded_clips(bundle.read_bytes()) == expected + resources = [name for name in saved.file.files if name.endswith(".resource")] + assert len(resources) == 3 + + +def test_fsb5_rejects_non_wav_and_empty_pcm(): + with pytest.raises(ValueError, match="WAV"): + wav_to_fsb5(b"not audio", "tone") + data = io.BytesIO() + with wave.open(data, "wb") as writer: + writer.setparams((1, 2, 16000, 0, "NONE", "not compressed")) + with pytest.raises(ValueError, match="empty"): + wav_to_fsb5(data.getvalue(), "tone") + with pytest.raises(ValueError, match="16-bit"): + wav_to_fsb5(wav(width=1)[0], "tone") + + +def test_serialized_fields_use_explicit_schema_and_validate_references(tmp_path): + schema = { + "class_id": 114, + "type_hash": "12" * 16, + "script_id": "34" * 16, + "nodes": [ + [0, "MonoBehaviour", "Base", 0], + [1, "string", "m_Name", 0x4000], + [1, "PPtr", "target", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + ], + } + (tmp_path / "data").write_bytes(b"binary") + builder = BundleBuilder(template(), bundle_name="example") + builder.apply( + { + "type": "serialized", + "name": "link", + "path_id": 2, + "schema": schema, + "fields": {"m_Name": "", "target": {"m_FileID": 0, "m_PathID": 3}}, + } + ) + with pytest.raises(ValueError, match="unresolved reference"): + builder.build() + builder.apply({"type": "TextAsset", "name": "binary", "path_id": 3, "file": "data"}, base=tmp_path) + env = UnityPy.load(builder.build().data) + assert env.assets[0].objects[2].read_typetree()["target"]["m_PathID"] == 3 + index = env.assets[0].objects[1].read_typetree() + assert index["m_Container"][0][1]["preloadSize"] == 2 + + +def test_cloned_monobehaviours_keep_resolvable_script_type_indices(): + schema = { + "class_id": 114, + "type_hash": "12" * 16, + "script_id": "34" * 16, + "nodes": [ + [0, "MonoBehaviour", "Base", 0], + [1, "string", "m_Name", 0x4000], + [1, "PPtr", "m_Script", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + ], + } + source = BundleBuilder(template(), bundle_name="source") + source.add_external("archive:/scripts/scripts") + source.apply( + { + "type": "serialized", + "schema": schema, + "path_id": 2, + "name": "component", + "fields": {"m_Name": "component", "m_Script": {"m_FileID": 1, "m_PathID": 42}}, + } + ) + builder = BundleBuilder(source.build().data, bundle_name="example") + builder.add_external("archive:/scripts/scripts") + for path_id, script_id in [(2, 42), (3, 43)]: + builder.apply( + { + "type": "serialized", + "template": {"type": "MonoBehaviour"}, + "path_id": path_id, + "name": f"component{path_id}", + "fields": {"m_Name": "", "m_Script": {"m_FileID": 1, "m_PathID": script_id}}, + } + ) + result = UnityPy.load(builder.build().data).assets[0] + for path_id, script_id in [(2, 42), (3, 43)]: + obj = result.objects[path_id] + ref = result.script_types[obj.serialized_type.script_type_index] + assert (ref.local_serialized_file_index, ref.local_identifier_in_file) == (1, script_id) + + +def test_raw_assets_require_explicit_reference_ownership(tmp_path): + (tmp_path / "raw").write_bytes(build_text_asset("raw", b"opaque")) + builder = BundleBuilder(template(), bundle_name="example") + spec = {"type": "raw", "template": {"type": "TextAsset"}, "name": "raw", "file": "raw"} + with pytest.raises(ValueError, match="managed_references"): + builder.apply(spec, base=tmp_path) + builder = BundleBuilder(template(), bundle_name="example") + builder.apply({**spec, "managed_references": True}, base=tmp_path) + env = UnityPy.load(builder.build().data) + assert parse_text_asset(env.assets[0].objects[2].get_raw_data()) == TextAsset("raw", b"opaque") + + +def test_duplicate_path_ids_and_container_paths_are_rejected(tmp_path): + (tmp_path / "data").write_bytes(b"data") + builder = BundleBuilder(template(), bundle_name="example") + builder.apply({"type": "TextAsset", "name": "one", "file": "data", "container": "same"}, base=tmp_path) + with pytest.raises(ValueError, match="path_id"): + builder.apply({"type": "TextAsset", "name": "two", "file": "data", "path_id": 2}, base=tmp_path) + builder = BundleBuilder(template(), bundle_name="example") + builder.apply({"type": "TextAsset", "name": "one", "file": "data", "container": "same"}, base=tmp_path) + with pytest.raises(ValueError, match="container"): + builder.apply({"type": "TextAsset", "name": "two", "file": "data", "container": "same"}, base=tmp_path) + with pytest.raises(ValueError, match="failed operation"): + builder.build() + + +def test_atomic_outputs_restore_prior_files_when_publication_fails(tmp_path, monkeypatch): + import nnnotes.unity.pack as pack + + one, two = tmp_path / "one", tmp_path / "two" + one.write_bytes(b"original one") + two.write_bytes(b"original two") + original = pack.os.replace + count = 0 + + def failing(source, target): + nonlocal count + count += 1 + if count == 4: + raise OSError("replacement failed") + return original(source, target) + + monkeypatch.setattr(pack.os, "replace", failing) + with pytest.raises(OSError, match="replacement failed"): + write_outputs({one: b"new one", two: b"new two"}) + assert one.read_bytes() == b"original one" and two.read_bytes() == b"original two" + assert set(tmp_path.iterdir()) == {one, two} + + +def test_new_bundle_requires_explicit_bundle_identity(): + with pytest.raises(ValueError, match="bundle_name"): + BundleBuilder(template()) + + +def test_edit_serialized_reference_rebuilds_preload_closure(tmp_path): + schema = { + "class_id": 114, + "type_hash": "12" * 16, + "script_id": "34" * 16, + "nodes": [ + [0, "MonoBehaviour", "Base", 0], + [1, "string", "m_Name", 0x4000], + [1, "PPtr", "target", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + ], + } + (tmp_path / "data").write_bytes(b"data") + builder = BundleBuilder(template(), bundle_name="example") + for path_id in (3, 4): + builder.apply({"type": "TextAsset", "name": str(path_id), "path_id": path_id, "file": "data"}, base=tmp_path) + builder.apply( + { + "type": "serialized", + "name": "link", + "path_id": 2, + "schema": schema, + "fields": {"m_Name": "link", "target": {"m_FileID": 0, "m_PathID": 3}}, + } + ) + edited = BundleBuilder(builder.build().data, new=False) + edited.apply( + { + "op": "replace", + "type": "serialized", + "name": "link", + "fields": {"m_Name": "link", "target": {"m_FileID": 0, "m_PathID": 4}}, + } + ) + env = UnityPy.load(edited.build().data) + index = env.assets[0].objects[1].read_typetree() + entry = dict(index["m_Container"])["link"] + refs = index["m_PreloadTable"][entry["preloadIndex"] : entry["preloadIndex"] + entry["preloadSize"]] + assert refs == [{"m_FileID": 0, "m_PathID": 2}, {"m_FileID": 0, "m_PathID": 4}] + + +def test_failed_recovery_keeps_original_output_backups(tmp_path, monkeypatch): + import nnnotes.unity.pack as pack + + one, two = tmp_path / "one", tmp_path / "two" + one.write_bytes(b"original one") + two.write_bytes(b"original two") + original = pack.os.replace + count = 0 + + def failing(source, target): + nonlocal count + count += 1 + if count >= 4: + raise OSError("filesystem unavailable") + return original(source, target) + + monkeypatch.setattr(pack.os, "replace", failing) + with pytest.raises(OSError, match="recoverable backup") as failure: + write_outputs({one: b"new one", two: b"new two"}) + backups = list(tmp_path.iterdir()) + assert {path.read_bytes() for path in backups} == {b"original one", b"original two"} + assert all(str(path) in str(failure.value) for path in backups) + + +def test_external_stripped_template_recovers_builtin_schema(tmp_path): + (tmp_path / "stripped.bundle").write_bytes(template(stripped=True)) + (tmp_path / "data").write_bytes(b"binary") + builder = BundleBuilder(template(), bundle_name="example") + builder.apply( + { + "type": "TextAsset", + "name": "binary", + "file": "data", + "template": {"file": "stripped.bundle", "type": "TextAsset"}, + }, + base=tmp_path, + ) + result = builder.build() + env = UnityPy.load(result.data) + assert env.assets[0].objects[2].serialized_type.node is not None + assert parse_text_asset(env.assets[0].objects[2].get_raw_data()).payload == b"binary" + + +def test_external_reference_is_included_in_preload_without_local_dereference(tmp_path): + schema = { + "class_id": 114, + "type_hash": "12" * 16, + "script_id": "34" * 16, + "nodes": [ + [0, "MonoBehaviour", "Base", 0], + [1, "string", "m_Name", 0x4000], + [1, "PPtr", "target", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + ], + } + builder = BundleBuilder(template(), bundle_name="example") + assert builder.add_external("CAB-external") == 1 + builder.apply( + { + "type": "serialized", + "name": "link", + "path_id": 2, + "schema": schema, + "fields": {"m_Name": "link", "target": {"m_FileID": 1, "m_PathID": 99}}, + } + ) + env = UnityPy.load(builder.build().data) + index = env.assets[0].objects[1].read_typetree() + assert index["m_PreloadTable"] == [{"m_FileID": 0, "m_PathID": 2}, {"m_FileID": 1, "m_PathID": 99}] + + +def test_edit_main_asset_preload_tracks_reordered_container(tmp_path): + source = UnityPy.load(template()) + index = source.assets[0].objects[10] + fields = index.read_typetree() + fields["m_MainAsset"] = {"preloadIndex": 0, "preloadSize": 1, "asset": {"m_FileID": 0, "m_PathID": 20}} + index.save_typetree(fields) + builder = BundleBuilder(source.file.save(packer="original"), new=False) + (tmp_path / "data").write_bytes(b"second") + builder.apply( + {"type": "TextAsset", "name": "second", "file": "data", "template": {"type": "TextAsset"}}, base=tmp_path + ) + builder.apply({"op": "rename", "name": "text", "new_name": "text", "new_container": "Assets/renamed.bytes"}) + env = UnityPy.load(builder.build().data) + fields = env.assets[0].objects[10].read_typetree() + main = fields["m_MainAsset"] + assert fields["m_PreloadTable"][main["preloadIndex"] : main["preloadIndex"] + main["preloadSize"]] == [ + {"m_FileID": 0, "m_PathID": 20} + ] + + +def test_manifest_registers_explicit_externals_and_bundle_dependencies(tmp_path): + (tmp_path / "template.bundle").write_bytes(template()) + schema = { + "class_id": 114, + "type_hash": "12" * 16, + "script_id": "34" * 16, + "nodes": [ + [0, "MonoBehaviour", "Base", 0], + [1, "string", "m_Name", 0x4000], + [1, "PPtr", "target", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + ], + } + result = build_manifest( + { + "template": "template.bundle", + "bundle_name": "example", + "externals": [{"path": "CAB-dependency", "guid": "56" * 16, "type": 2}], + "dependencies": ["dependency"], + "assets": [ + { + "type": "serialized", + "name": "link", + "schema": schema, + "fields": {"m_Name": "link", "target": {"m_FileID": 1, "m_PathID": 99}}, + }, + ], + }, + base=tmp_path, + ) + env = UnityPy.load(result.data) + external = env.assets[0].externals[0] + assert (external.path, external.guid, external.type) == ("CAB-dependency", bytes.fromhex("56" * 16), 2) + assert env.assets[0].objects[1].read_typetree()["m_Dependencies"] == ["dependency"] From 1804580d75de379e92755a4125b1835da67e02db Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Fri, 23 Oct 2026 00:25:48 +0800 Subject: [PATCH 2/2] chore(release): prepare version 0.3.1 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43b5f68..c258882 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project are listed here. Versions follow [Semantic V over the command line (commands, options, exit statuses and `--json` reports), the configuration keys, the JSON schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor version can change any of them. +## [0.3.1](https://github.com/MetaSekaiLab/nnnotes/compare/v0.3.0...v0.3.1) - 2026-10-22 + +### Features + +- **packing:** Build song packages and HCA cue sheets ([dc198b0](https://github.com/MetaSekaiLab/nnnotes/commit/dc198b0c977b2b3ae1bffaca0e3ce3c2653133c6)) + ## [0.3.0](https://github.com/MetaSekaiLab/nnnotes/compare/v0.2.5...v0.3.0) - 2026-10-19 ### Compatibility