feat: add synthetic test data sets

.gitattributes

@@ -0,0 +1,2 @@
+# The test data are compared byte for byte and checksummed: keep LF endings
+data/** -text

.github/workflows/ci.yml

@@ -0,0 +1,93 @@
+# Everything that must pass on pull requests and pushes to `main`. Needs no
+# secrets. No `paths:` filters, so that the gate always reports. The gate
+# `passed` ("CI passed") is the only check to require; add every new job to
+# its `needs`. The retired demo `2023-ecp-f2f` is not checked.
+name: CI
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+# Only a new head commit of a pull request cancels its runs in progress
+# (action `synchronize`, first attempt); runs for other actions and reruns
+# wait, so that a run on the same commit never leaves a cancelled, failed
+# `CI passed` (see "Cancelled runs" in `actions/gate/README.md` of
+# `elixir-cloud-aai/.github`). Runs for pushes get their own group, so that
+# none is dropped
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.ref }}
+  cancel-in-progress: >-
+    ${{ github.event_name == 'pull_request'
+    && github.event.action == 'synchronize'
+    && github.run_attempt == 1 }}
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
+        with:
+          python-version: "3.13"
+      - name: Set up uv
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
+        with:
+          version: "0.13.6"
+      - name: Install dependencies
+        run: |
+          uv sync --locked --python "$(which python)"
+          echo "${PWD}/.venv/bin" >> "$GITHUB_PATH"
+      - name: Run linters and type checks
+        run: |
+          ruff check .
+          ruff format --check .
+          mypy
+
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
+        with:
+          python-version: "3.13"
+      - name: Set up uv
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
+        with:
+          version: "0.13.6"
+      - name: Install dependencies
+        run: |
+          uv sync --locked --python "$(which python)"
+          echo "${PWD}/.venv/bin" >> "$GITHUB_PATH"
+      # Fails below 100% coverage and if the test data differ from what
+      # `scripts/generate_test_data.py` writes (`pyproject.toml`)
+      - name: Run tests
+        run: pytest
+
+  passed:
+    name: CI passed
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    if: always()
+    needs: [lint, test]
+    permissions: {}
+    steps:
+      - name: Check results of all jobs
+        uses: elixir-cloud-aai/.github/actions/gate@743e82d290a53768454999e413756a924e383259 # v1.0.0
+        with:
+          needs: ${{ toJSON(needs) }}

.gitignore

@@ -180,3 +180,4 @@ poetry.toml
 .netrc
 results/
 .snakemake/
+.uv-cache/

README.md

@@ -26,6 +26,13 @@ Follow the shortcuts in the following table to the individual demos.
 | --- | --- | --- |
 | [2023-ecp-f2f](demos/2023-ecp-f2f/README.md) | ELIXIR Compute Face-to-Face, Helsinki, Feb 7-8, 2023 | Task execution via the [Task Execution Service (TES) API](https://github.com/ga4gh/task-execution-schemas/) on Kubernetes and HPC; TES-based task distribution with random and distance-based task distribution logic; execution of a CWL workflow via the `cwl-tes` TES backend |
 
+## Test data
+
+The folder [`data/`](data/README.md) holds small, open and synthetic test
+data sets for the demos, e.g. one partition per Node for federated analytics.
+Its [`manifest.json`](data/manifest.json) lists the URL, size and checksum of
+every file. The data are made up; never add real or sensitive data.
+
 ## Requirements
 
 All demos require the following software to be installed:
@@ -46,6 +53,16 @@ reports, pull requests or discussions. Please read our [contributor
 guidelines][docs-contributing] if you want to contribute. And please mind our
 [Code of Conduct][docs-coc] for all interactions with the community.
 
+To run the checks of the CI on your computer, install [uv][req-uv] and run,
+from the repository root:
+
+```sh
+uv run ruff check .
+uv run ruff format --check .
+uv run mypy
+uv run pytest
+```
+
 ## Versioning
 
 Outside of version control, individual demos are not explicitly versioned. When
@@ -78,6 +95,7 @@ thread in our [Q&A forum][contact-qa], or send us an [email][contact-email].
 [req-conda]: <https://conda.io/>
 [req-git]: <https://git-scm.com/>
 [req-mamba]: <https://mamba.readthedocs.io/>
+[req-uv]: <https://docs.astral.sh/uv/>
 [res-elixir]: <https://elixir-europe.org/>
 [res-elixir-cloud]: <https://elixir-cloud.dcc.sib.swiss/>
 [res-ga4gh]: <https://ga4gh.org/>

data/README.md

@@ -0,0 +1,55 @@
+# Test data
+
+Small, open test data sets for the demos of this repository, and for anyone
+who wants to try a federated analysis on ELIXIR On-Cloud services without
+real data.
+
+> [!IMPORTANT]
+> All data sets here are **synthetic**: a seeded random number generator made
+> them up. They describe no person, organism or experiment. Don't add real
+> data, and never personal or other sensitive data, to this folder.
+
+## Data sets
+
+| Data set | Files | Columns | License |
+| --- | --- | --- | --- |
+| `synthetic-growth` | One CSV file per Node: [`node-a.csv`](synthetic-growth/node-a.csv), [`node-b.csv`](synthetic-growth/node-b.csv), 50 rows each | `sample_id`, `condition` (`control` or `treated`), `od600` (optical density at 600 nm), `expression` (reporter expression level) | [CC0 1.0](https://creativecommons.org/publicdomain/zero/1.0/) |
+
+The partitions differ slightly on purpose (`node-b` has a higher mean
+`od600`), like data of the same kind from two sites.
+
+## Locations
+
+[`manifest.json`](manifest.json) lists every file with its Node, its URL, its
+size and its SHA-256 checksum. The URLs point to the `main` branch of this
+repository on GitHub, e.g.:
+
+```text
+https://raw.githubusercontent.com/elixir-cloud-aai/elixir-cloud-demos/main/data/synthetic-growth/node-a.csv
+```
+
+The URLs follow `main`: a change to the data changes what they serve and
+the checksums in the manifest. For a fixed copy, replace `main` by a commit
+SHA in the URLs.
+
+A TES task can use such a URL directly as an input (`inputs[].url`), as long
+as the TES service may download from GitHub. To place the data at a Node
+instead, e.g. in its object store, copy the files there and check them
+against the checksums of the manifest:
+
+```sh
+sha256sum synthetic-growth/*.csv
+```
+
+## Regenerate
+
+The files are made by
+[`scripts/generate_test_data.py`](../scripts/generate_test_data.py). To change
+them, change the script and run, from the repository root:
+
+```sh
+uv run python scripts/generate_test_data.py
+```
+
+The tests (`uv run pytest`) fail if the committed files and the
+manifest differ from what the script writes.

data/manifest.json

@@ -0,0 +1,31 @@
+{
+  "datasets": [
+    {
+      "name": "synthetic-growth",
+      "description": "Synthetic growth measurements (optical density at 600 nm and a reporter expression level) under two conditions, split into one partition per Node. Made up by a seeded random number generator; describes no real sample.",
+      "license": "CC0-1.0",
+      "columns": [
+        "sample_id",
+        "condition",
+        "od600",
+        "expression"
+      ],
+      "files": [
+        {
+          "node": "node-a",
+          "path": "synthetic-growth/node-a.csv",
+          "url": "https://raw.githubusercontent.com/elixir-cloud-aai/elixir-cloud-demos/main/data/synthetic-growth/node-a.csv",
+          "sha256": "425161d3670348f82b5fa1144dc9528b236c5a02fab4e609d5bbcd02fcff0f0f",
+          "size": 1626
+        },
+        {
+          "node": "node-b",
+          "path": "synthetic-growth/node-b.csv",
+          "url": "https://raw.githubusercontent.com/elixir-cloud-aai/elixir-cloud-demos/main/data/synthetic-growth/node-b.csv",
+          "sha256": "70b57e5c779eb9f1d14c9206d396e80a2e066ca031167ac1e71ff32089648ad6",
+          "size": 1624
+        }
+      ]
+    }
+  ]
+}

data/synthetic-growth/node-a.csv

@@ -0,0 +1,51 @@
+sample_id,condition,od600,expression
+node-a-001,control,0.864,10.819
+node-a-002,treated,1.096,12.936
+node-a-003,control,1.016,8.690
+node-a-004,treated,1.144,12.350
+node-a-005,control,0.930,10.701
+node-a-006,treated,1.027,12.872
+node-a-007,control,0.880,6.775
+node-a-008,treated,1.067,12.276
+node-a-009,control,0.659,11.929
+node-a-010,treated,1.193,13.309
+node-a-011,control,0.884,9.614
+node-a-012,treated,1.092,14.759
+node-a-013,control,0.770,7.679
+node-a-014,treated,0.834,11.691
+node-a-015,control,0.951,10.412
+node-a-016,treated,1.139,11.847
+node-a-017,control,0.845,11.500
+node-a-018,treated,1.067,14.695
+node-a-019,control,0.847,9.262
+node-a-020,treated,0.966,10.246
+node-a-021,control,0.885,8.949
+node-a-022,treated,1.149,11.580
+node-a-023,control,0.623,9.943
+node-a-024,treated,1.202,14.908
+node-a-025,control,0.831,11.627
+node-a-026,treated,1.066,11.924
+node-a-027,control,0.983,11.107
+node-a-028,treated,1.239,12.276
+node-a-029,control,0.801,6.771
+node-a-030,treated,1.158,11.784
+node-a-031,control,0.853,11.522
+node-a-032,treated,1.054,11.541
+node-a-033,control,0.904,9.268
+node-a-034,treated,0.890,11.165
+node-a-035,control,0.663,12.035
+node-a-036,treated,1.046,12.340
+node-a-037,control,0.782,10.524
+node-a-038,treated,1.283,12.174
+node-a-039,control,0.840,10.768
+node-a-040,treated,1.136,12.226
+node-a-041,control,0.702,9.111
+node-a-042,treated,0.942,12.595
+node-a-043,control,0.778,8.067
+node-a-044,treated,1.164,11.861
+node-a-045,control,0.774,10.597
+node-a-046,treated,1.193,12.906
+node-a-047,control,0.802,10.976
+node-a-048,treated,1.108,14.895
+node-a-049,control,0.794,11.480
+node-a-050,treated,1.116,12.720

data/synthetic-growth/node-b.csv

@@ -0,0 +1,51 @@
+sample_id,condition,od600,expression
+node-b-001,control,0.822,8.013
+node-b-002,treated,1.229,10.905
+node-b-003,control,0.919,9.204
+node-b-004,treated,1.070,11.367
+node-b-005,control,0.732,12.331
+node-b-006,treated,1.127,10.545
+node-b-007,control,0.953,10.564
+node-b-008,treated,1.092,12.271
+node-b-009,control,1.031,6.892
+node-b-010,treated,1.182,13.765
+node-b-011,control,0.903,8.882
+node-b-012,treated,1.261,12.358
+node-b-013,control,0.885,11.261
+node-b-014,treated,1.326,15.815
+node-b-015,control,0.816,11.073
+node-b-016,treated,1.001,11.708
+node-b-017,control,1.008,9.024
+node-b-018,treated,0.964,15.086
+node-b-019,control,0.859,11.567
+node-b-020,treated,1.195,13.818
+node-b-021,control,0.712,10.524
+node-b-022,treated,1.146,15.652
+node-b-023,control,0.750,12.510
+node-b-024,treated,1.210,12.185
+node-b-025,control,0.848,12.203
+node-b-026,treated,1.305,10.544
+node-b-027,control,0.937,10.820
+node-b-028,treated,1.156,12.424
+node-b-029,control,0.843,11.200
+node-b-030,treated,1.154,13.368
+node-b-031,control,0.914,9.690
+node-b-032,treated,1.088,12.533
+node-b-033,control,0.935,9.914
+node-b-034,treated,1.096,13.135
+node-b-035,control,0.828,13.415
+node-b-036,treated,1.216,11.050
+node-b-037,control,0.949,9.063
+node-b-038,treated,1.238,13.244
+node-b-039,control,0.819,11.919
+node-b-040,treated,1.164,10.574
+node-b-041,control,0.944,9.256
+node-b-042,treated,1.180,12.525
+node-b-043,control,0.972,9.316
+node-b-044,treated,0.970,12.671
+node-b-045,control,0.841,9.745
+node-b-046,treated,1.114,12.840
+node-b-047,control,0.846,9.972
+node-b-048,treated,1.143,11.635
+node-b-049,control,0.845,7.865
+node-b-050,treated,1.044,12.991

pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "elixir-cloud-demos"
+version = "0.0.0"
+description = "Demonstrators of ELIXIR On-Cloud"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.12"
+dependencies = []
+
+[dependency-groups]
+dev = [
+  "mypy>=1.13",
+  "pytest>=8.3",
+  "pytest-cov>=6.0",
+  "ruff>=0.8",
+]
+
+[tool.uv]
+package = false
+
+[tool.pytest.ini_options]
+pythonpath = ["scripts"]
+testpaths = ["tests"]
+addopts = "--cov=scripts --cov-report=term-missing --cov-fail-under=100"
+
+[tool.ruff]
+extend-exclude = ["demos/2023-ecp-f2f"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP", "D", "ANN"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["generate_test_data"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.mypy]
+strict = true
+files = ["scripts", "tests"]
+mypy_path = "scripts"