From b85fc9492a42111a23c83bcba5e0d2deaa42ea35 Mon Sep 17 00:00:00 2001 From: Mistium <92952823+Mistium@users.noreply.github.com> Date: Sun, 1 Nov 2026 06:55:42 +0000 Subject: [PATCH 1/8] Cover uploads and project updates in the smoke test Exercise plain, sparse, workspace, delta, restore, branch merge, pull merge and history replacement uploads, their failure paths, and the project update fields and refusals. A fake Rotur answers profile lookups so team changes and transfers run offline, and the run ends with each stored project record and a blob inventory. --- scripts/e2e_smoke.py | 464 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 453 insertions(+), 11 deletions(-) diff --git a/scripts/e2e_smoke.py b/scripts/e2e_smoke.py index c376c7a..580f28b 100644 --- a/scripts/e2e_smoke.py +++ b/scripts/e2e_smoke.py @@ -11,11 +11,14 @@ import subprocess import sys import tempfile +import threading import time import urllib.error import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import uuid import zipfile +import zlib PROJECT_JSON = { "targets": [{ @@ -26,9 +29,10 @@ "monitors": [], "extensions": [], "meta": {"semver": "3.0.0", "vm": "0.2.0", "agent": "smoke"}, } -USERS = {"sam": "u_sam", "alice": "u_alice"} +USERS = {"sam": "u_sam", "alice": "u_alice", "bob": "u_bob"} +SCOPED_SESSIONS = {"sam_editor": ("sam", "editor")} VOLATILE_KEYS = re.compile(r"(^|_)(id|_id|at|time|timestamp|created|updated|modified|expires|sha|commit|head|token|etag|key|url)$|At$|Id$|Ms$|Url$|Key$|Sha$", re.I) -APPROX_KEYS = {"bytes", "mistwarp", "sizeBytes"} +APPROX_KEYS = {"bytes", "mistwarp", "sizeBytes", "historyBytes", "storedJsonBytes", "used", "usedBytes", "storageUsed", "received", "size"} HEX = re.compile(r"^[0-9a-f]{32,64}$") ISO = re.compile(r"^\d{4}-\d{2}-\d{2}T") @@ -46,6 +50,26 @@ def default_sb3(): return buffer.getvalue() +class FakeRotur(BaseHTTPRequestHandler): + def do_GET(self): + name = self.path.split("?", 1)[0][len("/profile/"):] if self.path.startswith("/profile/") else "" + if name not in USERS: + self.close_connection = True + return + body = json.dumps({"id": USERS[name], "username": name}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_POST(self): + self.close_connection = True + + def log_message(self, *args): + pass + + class Server: def __init__(self, binary, keep): self.binary = os.path.abspath(binary) @@ -53,10 +77,12 @@ def __init__(self, binary, keep): self.dir = tempfile.mkdtemp(prefix="mw-smoke-") self.port = free_port() self.base = "http://127.0.0.1:%d" % self.port + self.rotur = ThreadingHTTPServer(("127.0.0.1", 0), FakeRotur) + threading.Thread(target=self.rotur.serve_forever, daemon=True).start() data = os.path.join(self.dir, "data") os.makedirs(os.path.join(data, "sessions")) with open(os.path.join(self.dir, ".env"), "w") as env: - env.write("PORT=%d\nAPP_URL=http://127.0.0.1:%d\nROTUR_BASE=http://127.0.0.1:9\n" % (self.port, self.port)) + env.write("PORT=%d\nAPP_URL=http://127.0.0.1:%d\nROTUR_BASE=http://127.0.0.1:%d\n" % (self.port, self.port, self.rotur.server_address[1])) with open(os.path.join(data, "agreement.json"), "w") as handle: json.dump({"version": 1, "text": "", "updatedAt": 1700000000000}, handle) with open(os.path.join(data, "agreement.md"), "w") as handle: @@ -66,6 +92,9 @@ def __init__(self, binary, keep): for name, uid in USERS.items(): with open(os.path.join(data, "sessions", "tok_%s.json" % name), "w") as handle: json.dump({"username": name, "userId": uid, "createdAt": int(time.time() * 1000)}, handle) + for token, (name, scope) in SCOPED_SESSIONS.items(): + with open(os.path.join(data, "sessions", "tok_%s.json" % token), "w") as handle: + json.dump({"username": name, "userId": USERS[name], "scope": scope, "parent": "tok_" + name, "createdAt": int(time.time() * 1000), "expiresAt": int(time.time() * 1000) + 86400000}, handle) self.log = open(os.path.join(self.dir, "server.log"), "w") self.process = subprocess.Popen([self.binary], cwd=self.dir, stdout=self.log, stderr=subprocess.STDOUT, start_new_session=True) for _ in range(300): @@ -87,6 +116,8 @@ def stop(self): except subprocess.TimeoutExpired: self.process.kill() self.log.close() + self.rotur.shutdown() + self.rotur.server_close() if not self.keep: shutil.rmtree(self.dir) @@ -111,14 +142,416 @@ def call(self, method, path, who=None, body=None, raw=None, content_type="applic return status, parsed, headers def upload(self, project_id, who, sb3): + return self.form("/v1/projects/%s/upload" % project_id, who, {}, {"project": sb3}) + + def form(self, path, who, fields, files): boundary = uuid.uuid4().hex - body = ( - ("--%s\r\nContent-Disposition: form-data; name=\"project\"; filename=\"p.sb3\"\r\n" - "Content-Type: application/octet-stream\r\n\r\n" % boundary).encode() - + sb3 + ("\r\n--%s--\r\n" % boundary).encode() - ) - return self.call("POST", "/v1/projects/%s/upload" % project_id, who, raw=body, - content_type="multipart/form-data; boundary=" + boundary) + body = b"" + for name, value in fields.items(): + body += ("--%s\r\nContent-Disposition: form-data; name=\"%s\"\r\n\r\n" % (boundary, name)).encode() + value.encode() + b"\r\n" + for name, value in files.items(): + body += ("--%s\r\nContent-Disposition: form-data; name=\"%s\"; filename=\"%s.bin\"\r\n" + "Content-Type: application/octet-stream\r\n\r\n" % (boundary, name, name)).encode() + value + b"\r\n" + body += ("--%s--\r\n" % boundary).encode() + return self.call("POST", path, who, raw=body, content_type="multipart/form-data; boundary=" + boundary) + + def read_project(self, project_id): + with open(os.path.join(self.dir, "data", "projects", project_id + ".json")) as handle: + return json.load(handle) + + def blob_inventory(self): + counts = {} + root = os.path.join(self.dir, "data", "blobs") + for folder, _, names in os.walk(root): + for name in names: + prefix = os.path.relpath(folder, root).split(os.sep)[0] + counts[prefix] = counts.get(prefix, 0) + 1 + return counts + + +SMOKE_AUTHOR = b"Smoke 1700000000 +0000" +ZERO_HEAD = "0" * 40 +EMBEDDED_HEX = re.compile(r"[0-9a-f]{32,64}") + + +def sb3_archive(project_json, files=None): + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + if project_json is not None: + archive.writestr("project.json", json.dumps(project_json)) + for name, content in (files or {}).items(): + archive.writestr(name, content) + return buffer.getvalue() + + +def zip_files(files): + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + for name, content in sorted(files.items()): + archive.writestr(name, content) + return buffer.getvalue() + + +def svg_asset(label): + content = ("%s" % label).encode() + return hashlib.md5(content).hexdigest() + ".svg", content + + +def project_with_costumes(names, extension_urls=None, targets=True): + project = json.loads(json.dumps(PROJECT_JSON)) + if not targets: + project.pop("targets") + return project + project["targets"][0]["costumes"] = [ + {"name": "c%d" % index, "assetId": name.split(".")[0], "md5ext": name, "dataFormat": "svg", "rotationCenterX": 0, "rotationCenterY": 0} + for index, name in enumerate(names) + ] + if extension_urls: + project["extensionURLs"] = extension_urls + project["extensions"] = list(extension_urls) + return project + + +def git_object(kind, body): + raw = b"%s %d\x00" % (kind.encode(), len(body)) + body + return hashlib.sha1(raw).hexdigest(), zlib.compress(raw) + + +class History: + def __init__(self, data): + with zipfile.ZipFile(io.BytesIO(data)) as archive: + self.entries = {name: archive.read(name) for name in archive.namelist() if not name.endswith("/")} + self.branch = self.entries[".git/HEAD"].decode().strip()[len("ref: refs/heads/"):] + self.refs = {name[len(".git/refs/heads/"):]: value.decode().strip() for name, value in self.entries.items() if name.startswith(".git/refs/heads/")} + self.objects = {name[len(".git/objects/"):].replace("/", ""): value for name, value in self.entries.items() if name.startswith(".git/objects/")} + self.added = {} + + def head(self): + return self.refs[self.branch] + + def read(self, oid): + raw = zlib.decompress(self.objects.get(oid) or self.added[oid]) + header, body = raw.split(b"\x00", 1) + return header.split(b" ")[0].decode(), body + + def tree_of(self, commit): + return self.read(commit)[1].split(b"\n", 1)[0].decode()[len("tree "):] + + def entries_of(self, tree): + body = self.read(tree)[1] + entries, position = [], 0 + while position < len(body): + space = body.index(b" ", position) + nul = body.index(b"\x00", space) + entries.append((body[position:space], body[space + 1:nul], body[nul + 1:nul + 21])) + position = nul + 21 + return entries + + def files(self, tree, prefix=""): + found = {} + for mode, name, raw_oid in self.entries_of(tree): + path = prefix + name.decode() + if mode in (b"40000", b"040000"): + found.update(self.files(raw_oid.hex(), path + "/")) + else: + found[path] = self.read(raw_oid.hex())[1] + return found + + def add(self, kind, body): + oid, compressed = git_object(kind, body) + if oid not in self.objects: + self.added[oid] = compressed + return oid + + def tree_with(self, tree, name, content): + blob = self.add("blob", content) + entries = [entry for entry in self.entries_of(tree) if entry[1] != name.encode()] + entries.append((b"100644", name.encode(), bytes.fromhex(blob))) + entries.sort(key=lambda entry: entry[1] + (b"/" if entry[0] in (b"40000", b"040000") else b"")) + return self.add("tree", b"".join(mode + b" " + entry_name + b"\x00" + raw_oid for mode, entry_name, raw_oid in entries)) + + def commit(self, tree, parents, message): + lines = [b"tree " + tree.encode()] + [b"parent " + parent.encode() for parent in parents] + lines += [b"author " + SMOKE_AUTHOR, b"committer " + SMOKE_AUTHOR] + return self.add("commit", b"\n".join(lines) + b"\n\n" + message.encode() + b"\n") + + def archive(self, head, delta): + refs = dict(self.refs) if not delta else {} + refs[self.branch] = head + manifest = self.metadata(head, delta) + manifest.update({"format": "mistwarp-project", "version": 1, "commits": [], "graph": {}}) + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("mwp.json", json.dumps(manifest)) + archive.writestr(".git/HEAD", "ref: refs/heads/%s\n" % self.branch) + for branch, oid in sorted(refs.items()): + archive.writestr(".git/refs/heads/" + branch, oid + "\n") + objects = dict(self.added) if delta else dict(self.objects, **self.added) + for oid, compressed in sorted(objects.items()): + archive.writestr(".git/objects/%s/%s" % (oid[:2], oid[2:]), compressed) + return buffer.getvalue() + + def metadata(self, head, delta): + return {"head": head, "branch": self.branch, "delta": delta, "baseHead": self.head() if delta else None, "worktree": False} + + +def raw_get(server, path, who): + request = urllib.request.Request(server.base + path, method="GET") + request.add_header("Authorization", "Bearer tok_" + who) + with urllib.request.urlopen(request, timeout=30) as response: + data = response.read() + return gzip.decompress(data) if data[:2] == b"\x1f\x8b" else data + + +def real_id(ids, alias): + return next(real for real, known in ids.items() if known == alias) + + +def stored_project(project): + def scrub(value): + if isinstance(value, dict): + return {key: scrub(item) for key, item in value.items()} + if isinstance(value, list): + return [scrub(item) for item in value] + if isinstance(value, str): + return EMBEDDED_HEX.sub("", value) + return value + return scrub(project) + + +def committed_workspace(server, ids, alias, who, message, delta=False, same_tree=False): + history = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % real_id(ids, alias), who)) + tree = history.tree_of(history.head()) + if not same_tree: + tree = history.tree_with(tree, "smoke-notes.txt", message.encode()) + head = history.commit(tree, [history.head()], message) + return history.archive(head, delta), json.dumps(history.metadata(head, delta)) + + +def staged_part(server, step, path, who, kind, data, name): + status, started = step(name + " start", "POST", path + "/upload-parts", who, {"kind": kind, "name": "", "size": len(data), "chunks": 1}) + token = started.get("token", "") if isinstance(started, dict) else "" + step(name + " chunk", "POST", path + "/upload-parts/%s/chunk" % token, who, form=({"index": "0"}, {"chunk": data})) + return token + + +def upload_flow(server, step, ids): + status, created = step("history create", "POST", "/v1/projects", "sam", {"title": "History smoke"}) + ids[str(created.get("id", ""))] = "" + path = "/v1/projects/" + asset_name, asset_content = svg_asset("first") + other_name, other_content = svg_asset("second") + plain = sb3_archive(project_with_costumes([])) + + def upload(name, fields=None, files=None, who="sam"): + return step(name, "POST", path + "/upload", who, form=(fields or {}, files if files is not None else {"project": plain})) + + def head_and_edited(): + status, project = server.call("GET", "/v1/projects/" + real_id(ids, ""), "sam")[:2] + return str(project.get("gitHead") or (project.get("project") or {}).get("gitHead") or ""), str(project.get("edited") or (project.get("project") or {}).get("edited") or "") + + upload("upload refused for stranger", who="alice") + upload("upload missing project file", files={}) + upload("upload invalid sb3", files={"project": b"not a zip"}) + upload("upload without targets", files={"project": sb3_archive(project_with_costumes([], targets=False))}) + upload("upload missing asset", files={"project": sb3_archive(project_with_costumes([asset_name]))}) + upload("upload asset hash mismatch", files={"project": sb3_archive(project_with_costumes([asset_name]), {asset_name: other_content})}) + upload("upload invalid extension sources", {"extensions": "not json"}) + upload("upload unsavable extension url", files={"project": sb3_archive(project_with_costumes([], {"bad": "ftp://example.com/x.js"}))}) + upload("upload unread extension source", files={"project": sb3_archive(project_with_costumes([], {"ext": "https://example.com/ext.js"}))}) + with_asset = sb3_archive(project_with_costumes([asset_name], {"ext": "https://example.com/ext.js"}), {asset_name: asset_content}) + upload("upload with asset and extension", {"extensions": json.dumps({"https://example.com/ext.js": "class SmokeExtension {}"})}, + {"project": with_asset, "thumbnail": b"\x89PNG smoke thumbnail"}) + step("check assets", "POST", path + "/assets/check", "sam", {"assets": [asset_name, other_name]}) + step("check invalid asset name", "POST", path + "/assets/check", "sam", {"assets": ["nope"]}) + sparse = sb3_archive(project_with_costumes([asset_name], {"ext": "https://example.com/ext.js"})) + upload("upload sparse", {"extensions": json.dumps({"https://example.com/ext.js": "class SmokeExtension {}"})}, {"project": sparse}) + upload("upload second asset", files={"project": sb3_archive(project_with_costumes([asset_name, other_name]), {other_name: other_content})}) + + head, edited = head_and_edited() + upload("upload stale edited", {"expectedEdited": "1"}) + upload("upload stale head", {"expectedHead": ZERO_HEAD}) + upload("upload replace without head", {"replaceHistory": "true"}) + upload("upload replace without workspace", {"replaceHistory": "true", "expectedHead": head}) + upload("upload invalid git metadata", {"git": "not json"}) + upload("upload stale delta base", {"git": json.dumps({"delta": True, "baseHead": ZERO_HEAD})}) + upload("upload invalid restore commit", {"restoreCommit": "xyz"}) + upload("upload invalid workspace token", {"workspaceToken": "!!"}) + upload("upload expired workspace token", {"workspaceToken": "abc123"}) + upload("upload invalid merge tree token", {"mergeTreeToken": "!!"}) + upload("upload unsafe workspace", files={"project": plain, "workspace": b"not a zip"}) + upload("upload replace as editor session", {"replaceHistory": "true", "expectedHead": head}, who="sam_editor") + + unchanged = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % real_id(ids, ""), "sam")) + upload("upload unchanged workspace", {"git": json.dumps(unchanged.metadata(unchanged.head(), False))}, + {"project": plain, "workspace": unchanged.archive(unchanged.head(), False)}) + archive, metadata = committed_workspace(server, ids, "", "sam", "Same tree", same_tree=True) + upload("upload same tree commit", {"git": metadata}, {"project": plain, "workspace": archive}) + archive, metadata = committed_workspace(server, ids, "", "sam", "Full save") + upload("upload full workspace", {"git": metadata}, {"project": plain, "workspace": archive}) + archive, metadata = committed_workspace(server, ids, "", "sam", "Delta save", delta=True) + upload("upload delta workspace", {"git": metadata}, {"project": plain, "workspace": archive}) + archive, metadata = committed_workspace(server, ids, "", "sam", "Staged save", delta=True) + token = staged_part(server, step, path, "sam", "workspace", archive, "stage workspace") + upload("upload staged workspace", {"git": metadata, "workspaceToken": token}) + upload("upload reused workspace token", {"git": metadata, "workspaceToken": token}) + + first = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % real_id(ids, ""), "sam")) + oldest = first.head() + while True: + parents = [line[7:].decode() for line in first.read(oldest)[1].split(b"\n\n", 1)[0].split(b"\n") if line.startswith(b"parent ")] + if not parents: + break + oldest = parents[0] + upload("upload restore", {"restoreCommit": oldest, "restoreMessage": "Restore the first version"}) + upload("upload restore unknown commit", {"restoreCommit": "1" * 40}) + + step("create branch", "POST", path + "/branches", "sam", {"name": "feature"}) + status, listed = step("list branches", "GET", path + "/branches", "sam") + heads = {branch.get("name"): branch.get("head") for branch in (listed.get("branches") or [])} if isinstance(listed, dict) else {} + merged = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % real_id(ids, ""), "sam")) + tree_files = merged.files(merged.tree_of(merged.head())) + tree_files["smoke-merge.txt"] = b"merged" + merge_tree = zip_files(tree_files) + branch_fields = {"mergeSourceBranch": "feature", "mergeTargetBranch": "main", "mergeSourceHead": heads.get("feature", ""), "mergeTargetHead": heads.get("main", "")} + upload("upload branch merge stale", dict(branch_fields, mergeSourceHead=ZERO_HEAD), {"project": plain, "mergeTree": merge_tree}) + upload("upload branch merge same branch", dict(branch_fields, mergeSourceBranch="main", mergeSourceHead=heads.get("main", "")), {"project": plain, "mergeTree": merge_tree}) + upload("upload branch merge invalid tree", branch_fields, {"project": plain, "mergeTree": b"not a zip"}) + upload("upload branch merge", branch_fields, {"project": plain, "mergeTree": merge_tree}) + upload("upload pull not open", {"pullId": "99"}, {"project": plain, "mergeTree": merge_tree}) + upload("upload merge tree without target", {}, {"project": plain, "mergeTree": merge_tree}) + + step("share team", "PUT", path, "sam", {"collaborators": [{"username": "alice", "role": "editor"}]}) + upload("upload live editor without edited", who="alice") + upload("upload live editor history refused", {"expectedEdited": "1", "restoreCommit": oldest}, who="alice") + head, edited = head_and_edited() + upload("upload live editor", {"expectedEdited": edited}, who="alice") + + step("share history", "POST", path + "/visibility", "sam", {"visibility": "public"}) + status, remixed = step("remix history", "POST", path + "/remix", "alice", {}) + remix_id = str(remixed.get("id", "") or (remixed.get("project") or {}).get("id", "")) if isinstance(remixed, dict) else "" + ids[remix_id] = "" + archive, metadata = committed_workspace(server, ids, "", "alice", "Remix change") + step("upload remix change", "POST", "/v1/projects//upload", "alice", form=({"git": metadata}, {"project": plain, "workspace": archive})) + status, pull = step("open pull", "POST", path + "/pulls", "alice", {"sourceProjectId": remix_id, "title": "Smoke pull", "body": "From the remix"}) + pull_index = str(((pull.get("pull") or {}).get("index", "")) if isinstance(pull, dict) else "") + remix = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % remix_id, "alice")) + pull_tree = zip_files(remix.files(remix.tree_of(remix.head()))) + head, edited = head_and_edited() + upload("upload pull merge stale head", {"pullId": pull_index, "expectedHead": ZERO_HEAD}, {"project": plain, "mergeTree": pull_tree}) + tree_token = staged_part(server, step, path, "sam", "mergeTree", pull_tree, "stage merge tree") + upload("upload pull merge with unfinished tree", {"pullId": pull_index, "expectedHead": head, "mergeTreeToken": tree_token}) + upload("upload pull merge", {"pullId": pull_index, "expectedHead": head}, {"project": plain, "mergeTree": pull_tree}) + step("merged pull", "GET", path + "/pulls/" + pull_index, "sam") + upload("upload merged pull again", {"pullId": pull_index}, {"project": plain, "mergeTree": pull_tree}) + + head, edited = head_and_edited() + fresh = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % real_id(ids, ""), "sam")) + root_tree = fresh.add("tree", b"100644 replaced.txt\x00" + bytes.fromhex(fresh.add("blob", b"replaced"))) + root = fresh.commit(root_tree, [], "Replaced history") + fresh.objects = {} + fresh.refs = {fresh.branch: root} + upload("upload replace history", {"replaceHistory": "true", "expectedHead": head, "git": json.dumps(fresh.metadata(root, False))}, + {"project": plain, "workspace": fresh.archive(root, False)}) + step("history after replace", "GET", path + "/branches", "sam") + step("history quota", "GET", "/v1/me/quota", "sam") + step("get history project", "GET", path, "sam") + + +def update_flow(server, step, ids): + status, created = step("update create", "POST", "/v1/projects", "sam", {"title": "Update smoke"}) + ids[str(created.get("id", ""))] = "" + path = "/v1/projects/" + + def patch(name, body, who="sam"): + return step(name, "PUT", path, who, body) + + patch("patch anonymous", {"title": "x"}, who=None) + patch("patch by stranger", {"title": "x"}, who="alice") + patch("patch text", {"title": " Updated smoke ", "description": "About", "instructions": "Press space", "notes": "Thanks"}) + patch("patch empty title", {"title": " "}) + patch("patch long title", {"title": "t" * 101}) + patch("patch long description", {"description": "d" * 5001}) + patch("patch long instructions", {"instructions": "i" * 5001}) + patch("patch long notes", {"notes": "n" * 5001}) + patch("patch credits", {"credits": [ + {"who": "Ada", "role": "Art", "url": "https://example.com/ada"}, + {"who": "Bo", "role": "Music", "url": None}, + {"who": "", "role": "skipped"}, + {"who": "w" * 61, "role": "skipped"}, + ]}) + patch("patch credit bad url", {"credits": [{"who": "Ada", "role": "Art", "url": "ftp://example.com"}]}) + patch("patch credit long url", {"credits": [{"who": "Ada", "role": "Art", "url": "https://example.com/" + "u" * 500}]}) + patch("patch too many credits", {"credits": [{"who": "A", "role": "B"}] * 51}) + patch("patch flags", {"commentsOff": True, "remixable": False, "seeInside": True}) + patch("patch price", {"price": 5}) + patch("patch zero price", {"price": 0, "remixable": True}) + patch("patch remix share", {"remixShare": 20}) + patch("patch remix share too high", {"remixShare": 80}) + patch("patch tags", {"tags": ["Games", "games", "Puzzle", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]}) + items = [ + {"id": "Sword", "name": "Sword", "visual": {"type": "image", "url": "https://example.com/s.png"}}, + {"id": "gem", "name": "Gem", "image": "https://example.com/g.png"}, + {"id": "hat", "name": "Hat", "visual": {"type": "costume", "assetId": "abcdefgh12", "dataFormat": "PNG"}, "gameAwardable": False}, + {"id": "pixel", "name": "Pixel", "visual": {"type": "image", "url": "data:image/png;base64,AAAA"}}, + ] + product = {"id": "pack", "name": "Pack", "description": "Starter", "price": 10, "grantsItem": "sword", "icon": "https://example.com/i.png"} + patch("patch game items and products", {"gameItems": items, "gameProducts": [product]}) + patch("patch product bad id", {"gameProducts": [dict(product, id="Bad Id")]}) + patch("patch product duplicate id", {"gameProducts": [product, product]}) + patch("patch product bad name", {"gameProducts": [dict(product, name="")]}) + patch("patch product bad price", {"gameProducts": [dict(product, price=2000)]}) + patch("patch product bad icon", {"gameProducts": [dict(product, icon="ftp://example.com/i.png")]}) + patch("patch product big icon", {"gameProducts": [dict(product, icon="data:image/png;base64," + "A" * 100001)]}) + patch("patch product bad grant", {"gameProducts": [dict(product, grantsItem="BAD!")]}) + patch("patch product undefined grant", {"gameProducts": [dict(product, grantsItem="ghost")]}) + patch("patch too many products", {"gameProducts": [product] * 51}) + patch("patch item bad id", {"gameItems": [dict(items[0], id="Bad Id")]}) + patch("patch item bad name", {"gameItems": [dict(items[0], name="")]}) + patch("patch item http image", {"gameItems": [dict(items[0], visual={"type": "image", "url": "http://example.com/s.png"})]}) + patch("patch item big data image", {"gameItems": [dict(items[0], visual={"type": "image", "url": "data:image/png;base64," + "A" * 100001})]}) + patch("patch item bad costume", {"gameItems": [dict(items[0], visual={"type": "costume", "assetId": "short", "dataFormat": "png"})]}) + patch("patch item without visual", {"gameItems": [{"id": "plain", "name": "Plain"}]}) + patch("patch too many items", {"gameItems": [items[1]] * 101}) + patch("patch items dropping a granted item", {"gameItems": [items[1]]}) + patch("patch inventory bad mode", {"inventoryPolicy": {"mode": "some"}}) + patch("patch inventory too large", {"inventoryPolicy": {"mode": "all", "allowedItems": ["p:x"] * 101}}) + patch("patch inventory bad item", {"inventoryPolicy": {"mode": "allowlist", "allowedItems": ["nope"]}}) + patch("patch inventory bad project", {"inventoryPolicy": {"mode": "allowlist", "allowedProjects": ["../x"]}}) + patch("patch inventory", {"inventoryPolicy": {"mode": "Allowlist", "allowedItems": ["p1:sword", "p1:sword"], "allowedProjects": ["p1", "p1"]}}) + patch("patch compatibility", {"compatibility": {"mobile": True, "keyboard": False, "controller": "yes"}}) + patch("patch branding bad accent", {"branding": {"accentColor": "red"}}) + patch("patch branding long tagline", {"branding": {"tagline": "t" * 121}}) + patch("patch branding without plan", {"branding": {"accentColor": "#123456", "tagline": "Hi"}}) + patch("patch branding cleared", {"branding": {}}) + patch("patch vanity without plan", {"vanitySlug": "smoke-game"}) + patch("patch vanity cleared", {"vanitySlug": ""}) + patch("patch group tag invalid", {"groupTag": "!!!"}) + patch("patch group tag not managed", {"groupTag": "somegroup"}) + patch("patch group tag cleared", {"groupTag": " "}) + patch("patch team share too high", {"collaborators": [{"username": "alice", "role": "editor", "share": 60}]}) + patch("patch team unknown user", {"collaborators": [{"username": "ghost_user", "role": "editor"}]}) + patch("patch team duplicate", {"collaborators": [{"username": "alice", "role": "editor"}, {"username": "Alice", "role": "tester"}]}) + patch("patch team total share", {"collaborators": [{"username": "alice", "role": "editor", "share": 40}, {"username": "bob", "role": "tester", "share": 40}]}) + patch("patch team", {"collaborators": [ + {"username": "alice", "role": "Maintainer", "share": 10}, + {"username": "bob", "role": "tester"}, + {"username": "sam", "role": "editor"}, + {"username": "carol", "role": "owner"}, + ]}) + patch("patch team same roles", {"collaborators": [{"username": "alice", "role": "maintainer", "share": 10}, {"username": "bob", "role": "editor"}]}) + step("alice notifications", "GET", "/v1/notifications", "alice") + step("bob notifications", "GET", "/v1/notifications", "bob") + patch("patch remix share as maintainer", {"remixShare": 5}, who="alice") + patch("patch team as maintainer", {"collaborators": []}, who="alice") + patch("patch title as maintainer", {"title": "Maintained"}, who="alice") + patch("patch editor session allowed", {"title": "From the editor"}, who="sam_editor") + patch("patch editor session refused", {"collaborators": []}, who="sam_editor") + patch("patch transfer as maintainer", {"owner": "bob"}, who="alice") + patch("patch transfer unknown", {"owner": "ghost_user"}) + patch("patch transfer", {"owner": "bob", "keepGroup": True, "title": "Given away"}) + patch("patch after transfer", {"title": "Mine again"}) + step("get updated project", "GET", path, "bob") def normalize(value, ids, key=""): @@ -148,12 +581,14 @@ def run(binary, sb3, keep): ids = {} steps = [] - def step(name, method, path, who=None, body=None, upload=None, headers=()): + def step(name, method, path, who=None, body=None, upload=None, headers=(), form=None): real_path = path for alias, real in ((a, r) for r, a in ids.items()): real_path = real_path.replace(alias, real) if upload: status, parsed, response_headers = server.upload(upload, who, sb3) + elif form is not None: + status, parsed, response_headers = server.form(real_path, who, form[0], form[1]) else: status, parsed, response_headers = server.call(method, real_path, who, body) record = {"step": name, "status": status, "body": parsed} @@ -199,6 +634,13 @@ def step(name, method, path, who=None, body=None, upload=None, headers=()): step("wrong method", "PATCH", "/v1/projects/", "sam", {}, headers=("Allow",)) step("legacy prefix", "GET", "/api/agreement") step("health", "GET", "/v1/health") + upload_flow(server, step, ids) + update_flow(server, step, ids) + time.sleep(1.5) + for real, alias in list(ids.items()): + if alias in ("", "", ""): + steps.append({"step": "stored " + alias, "status": 0, "body": stored_project(server.read_project(real))}) + steps.append({"step": "blob inventory", "status": 0, "body": server.blob_inventory()}) finally: server.stop() for record in steps: From eccc0e28e5724198eef110860bf261b0a2243923 Mon Sep 17 00:00:00 2001 From: Mistium <92952823+Mistium@users.noreply.github.com> Date: Sun, 1 Nov 2026 07:11:59 +0000 Subject: [PATCH 2/8] Split the upload and update handlers into named steps handleUploadProjectLocked and handleUpdateProjectLocked now read as a sequence of small steps that each return a Refusal record, so every early return keeps its status, body and order. Upload state moves through UploadRequest, UploadHistory, UploadArchive and UploadRollback records; update state through OwnerTransfer. Pure decisions live in policy files with tests, and the request, refusal and editor scope helpers are shared by both handlers. --- src/projects/project-refusal.osl | 28 + src/projects/project-refusal.test.osl | 13 + src/projects/project-request.osl | 32 + src/projects/project-team.osl | 61 ++ src/projects/project-team.test.osl | 44 + src/projects/project-update-policy.osl | 317 +++++++ src/projects/project-update-policy.test.osl | 107 +++ src/projects/project-update.osl | 616 ++++---------- src/projects/project-upload-files.osl | 190 +++++ src/projects/project-upload-history.osl | 295 +++++++ src/projects/project-upload-policy.osl | 229 +++++ src/projects/project-upload-policy.test.osl | 157 ++++ src/projects/project-upload-publish.osl | 142 ++++ src/projects/project-upload-receive.osl | 111 +++ src/projects/project-upload.osl | 871 +------------------- 15 files changed, 1937 insertions(+), 1276 deletions(-) create mode 100644 src/projects/project-refusal.osl create mode 100644 src/projects/project-refusal.test.osl create mode 100644 src/projects/project-request.osl create mode 100644 src/projects/project-team.osl create mode 100644 src/projects/project-team.test.osl create mode 100644 src/projects/project-update-policy.osl create mode 100644 src/projects/project-update-policy.test.osl create mode 100644 src/projects/project-upload-files.osl create mode 100644 src/projects/project-upload-history.osl create mode 100644 src/projects/project-upload-policy.osl create mode 100644 src/projects/project-upload-policy.test.osl create mode 100644 src/projects/project-upload-publish.osl create mode 100644 src/projects/project-upload-receive.osl diff --git a/src/projects/project-refusal.osl b/src/projects/project-refusal.osl new file mode 100644 index 0000000..fca8097 --- /dev/null +++ b/src/projects/project-refusal.osl @@ -0,0 +1,28 @@ +export {} + +type Refusal ( + number status = 0 + string message = "" + boolean hasBody = false + object body = {} +) + +def noRefusal() Refusal ( + return Refusal() +) + +def refuse(number status, string message) Refusal ( + return Refusal{status: status, message: message} +) + +def refuseWithBody(number status, object body) Refusal ( + return Refusal{status: status, hasBody: true, body: body} +) + +def editorScopeRefusal(string message) Refusal ( + return refuseWithBody(403, {ok: false, code: "editor_scope", error: message}) +) + +def isRefused(Refusal refusal) boolean ( + return refusal.status != 0 +) diff --git a/src/projects/project-refusal.test.osl b/src/projects/project-refusal.test.osl new file mode 100644 index 0000000..2426355 --- /dev/null +++ b/src/projects/project-refusal.test.osl @@ -0,0 +1,13 @@ +import "std:testing" +import "./project-refusal.osl" + +testing.assert(!isRefused(noRefusal()), "no refusal lets the request continue") +Refusal plain = refuse(400, "invalid title") +testing.assert(isRefused(plain) and plain.status == 400 and plain.message == "invalid title" and !plain.hasBody, "plain refusals carry a status and message") +Refusal detailed = refuseWithBody(409, {ok: false, code: "head_changed"}) +testing.assert(detailed.hasBody and detailed.body.code == "head_changed" and detailed.status == 409, "refusals with a body keep it whole") +Refusal scoped = editorScopeRefusal("Open MistWarp to do this.") +testing.assert(scoped.status == 403 and scoped.body.code == "editor_scope" and scoped.body.ok == false and scoped.body.error == "Open MistWarp to do this.", "editor scope refusals share one shape") +Refusal other = refuse(404, "x") +Refusal fresh = noRefusal() +testing.assert(fresh.body.len == 0 and other.body.len == 0, "each refusal gets its own body") diff --git a/src/projects/project-request.osl b/src/projects/project-request.osl new file mode 100644 index 0000000..03a4b9c --- /dev/null +++ b/src/projects/project-request.osl @@ -0,0 +1,32 @@ +export {} + +def loadRequestedProject(*serve.Context c) object? ( + auto res = loadProject(c.param("id")) + if !res.ok ( + c.notFound("project not found") + return null + ) + return res.project.assert(object) +) + +def isEditorSession(*serve.Context c) boolean ( + return contextString(c, "sessionScope") == "editor" +) + +def refused(*serve.Context c, Refusal refusal) boolean ( + if !isRefused(refusal) ( + return false + ) + if refusal.hasBody ( + c.json(refusal.status, refusal.body) + ) else if refusal.status == 400 ( + c.badRequest(refusal.message) + ) else if refusal.status == 403 ( + c.forbidden(refusal.message) + ) else if refusal.status == 404 ( + c.notFound(refusal.message) + ) else ( + c.internalError(refusal.message) + ) + return true +) diff --git a/src/projects/project-team.osl b/src/projects/project-team.osl new file mode 100644 index 0000000..a8853e8 --- /dev/null +++ b/src/projects/project-team.osl @@ -0,0 +1,61 @@ +export {} + +def applyCollaborators(object project, array incoming) Refusal ( + array collaborators = [] + number totalShare = 0 + for i incoming.len ( + if collaborators.len < 25 ( + object entry = incoming[i].assert(object) + string username = normalizeUsername(entry.username.toStr()) + string role = entry.role.toStr().toLower().strip() + boolean validRole = role == "editor" or role == "maintainer" or role == "contributor" or role == "tester" + number share = 0 + if entry.contains("share") and entry.share != null ( + share = entry.share.toNum() + ) + if share < 0 or share > 50 ( + return refuse(400, "each teammate share must be between 0 and 50 percent") + ) + if username != "" and validRole and normalizeUsername(username) != normalizeUsername(project.owner.toStr()) ( + string teammateId = discoverUserIdentity(username) + if teammateId == "" ( + return refuse(400, "Rotur user not found: " ++ username) + ) + for memberIndex collaborators.len ( + if collaborators[memberIndex].username.toStr() == username ( + return refuse(400, "a person can only appear once in the project team") + ) + ) + totalShare += share + collaborators.append({username: username, userId: teammateId, role: role, share: share}) + ) + ) + ) + if totalShare > 70 ( + return refuse(400, "teammate revenue shares cannot exceed 70 percent") + ) + project.collaborators = collaborators + return noRefusal() +) + +def isNewTeammate(object member, array previousTeam) boolean ( + for j previousTeam.len ( + object previousMember = previousTeam[j].assert(object) + if normalizeUsername(previousMember.username.toStr()) == normalizeUsername(member.username.toStr()) and previousMember.role == member.role ( + return false + ) + ) + return true +) + +def notifyNewTeammates(object project, array previousTeam, string actor) ( + array nextTeam = project.collaborators.assert(array) + for i nextTeam.len ( + object member = nextTeam[i].assert(object) + if isNewTeammate(member, previousTeam) ( + addNotification(member.username.toStr(), actor, "project_shared", { + projectId: project.id, projectTitle: project.title, role: member.role + }) + ) + ) +) diff --git a/src/projects/project-team.test.osl b/src/projects/project-team.test.osl new file mode 100644 index 0000000..4087620 --- /dev/null +++ b/src/projects/project-team.test.osl @@ -0,0 +1,44 @@ +import "std:testing" +import "std:crypto" +import "std:regex" +import "std:fs" +import "go:mistwarp.local/storagefs" +import "../core/utils.osl" +import "./project-refusal.osl" +import "./project-team.osl" + +array lookups = [] +def discoverUserIdentity(string username) string ( + lookups.append(username) + if username == "alice" or username == "bob" ( + return "u_" ++ username + ) + return "" +) + +array sent = [] +def addNotification(string target, string actor, string notifType, object payload) ( + sent.append({target: target, actor: actor, type: notifType, role: payload.role}) +) + +object project = {id: "p1", title: "Team", owner: "Sam"} +Refusal saved = applyCollaborators(project, [ + {username: " Alice ", role: "Maintainer", share: 10}, + {username: "bob", role: "tester"}, + {username: "sam", role: "editor"}, + {username: "carol", role: "owner"} +]) +testing.assert(!isRefused(saved), "a valid team is saved") +testing.assert(project.collaborators.len == 2 and project.collaborators[1].username == "alice" and project.collaborators[1].userId == "u_alice" and project.collaborators[1].role == "maintainer", "teammates are normalized and resolved") +testing.assert(project.collaborators[2].share == 0, "missing shares are zero") +testing.assert(!lookups.contains("sam") and !lookups.contains("carol"), "the owner and invalid roles are skipped before any lookup") +testing.assert(applyCollaborators(project, [{username: "alice", role: "editor", share: 51}]).message == "each teammate share must be between 0 and 50 percent", "shares are capped per teammate") +testing.assert(applyCollaborators(project, [{username: "ghost", role: "editor"}]).message == "Rotur user not found: ghost", "unknown users are refused") +testing.assert(applyCollaborators(project, [{username: "alice", role: "editor"}, {username: "ALICE", role: "tester"}]).message == "a person can only appear once in the project team", "teammates appear once") +testing.assert(applyCollaborators(project, [{username: "alice", role: "editor", share: 40}, {username: "bob", role: "editor", share: 40}]).message == "teammate revenue shares cannot exceed 70 percent", "total shares are capped") +testing.assert(project.collaborators.len == 2, "a refused team leaves the saved team alone") + +array previousTeam = [{username: "Alice", role: "maintainer"}, {username: "bob", role: "editor"}] +notifyNewTeammates(project, previousTeam, "sam") +testing.assert(sent.len == 1 and sent[1].target == "bob" and sent[1].role == "tester" and sent[1].type == "project_shared" and sent[1].actor == "sam", "only new teammates and changed roles are notified") +testing.assert(isNewTeammate({username: "dan", role: "editor"}, previousTeam) and !isNewTeammate({username: "ALICE", role: "maintainer"}, previousTeam), "teammates match by normalized name and role") diff --git a/src/projects/project-update-policy.osl b/src/projects/project-update-policy.osl new file mode 100644 index 0000000..dc27a53 --- /dev/null +++ b/src/projects/project-update-policy.osl @@ -0,0 +1,317 @@ +export {} + +def applyBranding(object project, object body, object creatorBenefits) Refusal ( + if !body.contains("branding") ( + return noRefusal() + ) + object incomingBranding = body.branding.assertElse(object, {}) + boolean clearingBranding = incomingBranding.getKeys().len == 0 + string accentColor = (incomingBranding.accentColor ?? "").toStr().strip() + string tagline = (incomingBranding.tagline ?? "").toStr().strip() + if accentColor != "" and !regex.match("^#[0-9a-fA-F]{6}$", accentColor) ( + return refuse(400, "branding accent must be a six-digit hex colour") + ) + if tagline.len > 120 ( + return refuse(400, "branding tagline is too long") + ) + object nextBranding = if clearingBranding then {} else {accentColor: accentColor, tagline: tagline} + object currentBranding = if project.contains("branding") then project.branding.assertElse(object, {}) else {} + if creatorBenefits.customProjectBranding != true and json.stringify(nextBranding) != json.stringify(currentBranding) ( + return refuse(403, "custom project branding requires Rotur Plus or Pro") + ) + project.branding = nextBranding + return noRefusal() +) + +def vanitySlugRefusal(string slug, string currentSlug, object creatorBenefits) Refusal ( + if slug != "" and slug != currentSlug and creatorBenefits.vanityProjectUrls != true ( + return refuse(403, "vanity project URLs require Rotur Pro") + ) + if slug != "" and (slug.len < 3 or slug.len > 40 or !regex.match("^[a-z0-9][a-z0-9-]*[a-z0-9]$", slug)) ( + return refuse(400, "vanity URLs must be 3 to 40 letters, numbers, or hyphens") + ) + return noRefusal() +) + +def applyProjectText(object project, object body) Refusal ( + if body.contains("title") ( + string title = body.title.toStr().strip() + if title == "" or title.len > 100 ( + return refuse(400, "invalid title") + ) + project.title = title + ) + if body.contains("description") ( + string description = body.description.toStr() + if description.len > 5000 ( + return refuse(400, "description too long") + ) + project.description = description + ) + if body.contains("instructions") ( + string instructions = body.instructions.toStr() + if instructions.len > 5000 ( + return refuse(400, "instructions too long") + ) + project.instructions = instructions + ) + if body.contains("notes") ( + string notes = body.notes.toStr() + if notes.len > 5000 ( + return refuse(400, "notes too long") + ) + project.notes = notes + ) + return noRefusal() +) + +def applyCredits(object project, object body) Refusal ( + if !body.contains("credits") ( + return noRefusal() + ) + array incoming = body.credits.assert(array) + if incoming.len > 50 ( + return refuse(400, "too many credits") + ) + array credits = [] + for i incoming.len ( + object entry = incoming[i] + string who = entry.who.toStr().strip() + string role = entry.role.toStr().strip() + string creditUrl = "" + if entry.contains("url") and entry.url != null ( + creditUrl = entry.url.toStr().strip() + ) + if creditUrl != "" and (!creditUrl.startsWith("https://") and !creditUrl.startsWith("http://")) ( + return refuse(400, "credit URL must use http or https") + ) + if creditUrl.len > 500 ( + return refuse(400, "credit URL too long") + ) + if who != "" and who.len <= 60 and role.len <= 120 ( + object credit = {who: who, role: role} + if creditUrl != "" ( + credit.url = creditUrl + ) + credits.append(credit) + ) + ) + project.credits = credits + return noRefusal() +) + +def applyProjectFlags(object project, object body) ( + if body.contains("commentsOff") ( + project.commentsOff = body.commentsOff == true + ) + if body.contains("remixable") ( + project.remixable = body.remixable != false + ) + if body.contains("seeInside") ( + project.seeInside = body.seeInside != false + ) +) + +def projectSaleRefusal(object body) Refusal ( + if body.contains("price") and (body.price ?? 0).toNum() != 0 ( + return refuseWithBody(400, {ok: false, code: "project_sales_ended", error: "Projects can no longer be sold on MistWarp."}) + ) + return noRefusal() +) + +def applyRemixShare(object project, number remixShare) Refusal ( + if remixShare < 0 or remixShare > 50 ( + return refuse(400, "remix share must be between 0 and 50 percent") + ) + project.remixShare = remixShare + return noRefusal() +) + +def applyTags(object project, object body) ( + if !body.contains("tags") ( + return + ) + array incoming = body.tags.assert(array) + array tags = [] + for i incoming.len ( + if tags.len < 10 ( + string t = normalizeTag(incoming[i].toStr()) + if t != "" and !tags.contains(t) ( + tags.append(t) + ) + ) + ) + project.tags = tags +) + +def applyGameProducts(object project, object body) Refusal ( + if !body.contains("gameProducts") ( + return noRefusal() + ) + array incomingProducts = body.gameProducts.assertElse(array, []) + if incomingProducts.len > 50 ( + return refuse(400, "projects can have at most 50 game products") + ) + array products = [] + array productIds = [] + for i incomingProducts.len ( + object incomingProduct = incomingProducts[i].assertElse(object, {}) + string productId = incomingProduct.id.toStr().toLower().strip() + string productName = incomingProduct.name.toStr().strip() + string productDescription = incomingProduct.description.toStr().strip() + number productPrice = incomingProduct.price.toNum() + string grantsItem = incomingProduct.grantsItem.toStr().toLower().strip() + string productIcon = incomingProduct.icon.toStr().strip() + if !regex.match("^[a-z0-9_-]{1,40}$", productId) or productIds.contains(productId) ( + return refuse(400, "game product ids must be unique lowercase identifiers") + ) + if productName.len < 1 or productName.len > 80 or productDescription.len > 300 ( + return refuse(400, "invalid game product name or description") + ) + if productPrice < 0 or productPrice > 1000 ( + return refuse(400, "game product prices must be between 0 and 1000 credits") + ) + if productIcon != "" and productIcon.toBytes().len > 100000 ( + return refuse(400, "game product icons must be smaller than 100KB") + ) + if productIcon != "" and !productIcon.startsWith("data:image/") and !productIcon.startsWith("https://") and !productIcon.startsWith("http://") ( + return refuse(400, "game product icons must be image data URLs or image URLs") + ) + if grantsItem != "" and !regex.match("^[a-z0-9_-]{1,40}$", grantsItem) ( + return refuse(400, "invalid portable item id on game product") + ) + productIds.append(productId) + products.append({id: productId, name: productName, description: productDescription, price: productPrice, grantsItem: grantsItem, icon: productIcon}) + ) + project.gameProducts = products + return noRefusal() +) + +def portableItemVisual(object visual) object ( + string visualType = visual.type.toStr().toLower().strip() + if visualType == "image" ( + string itemImage = visual.url.toStr().strip() + if itemImage.startsWith("data:image/") ( + if itemImage.toBytes().len > 100000 ( + return {error: "portable item images must be smaller than 100KB"} + ) + return {visual: {type: "image", url: itemImage}} + ) + if !itemImage.startsWith("https://") or itemImage.len > 500 ( + return {error: "portable item images must use a short HTTPS URL"} + ) + return {visual: {type: "image", url: itemImage}} + ) + if visualType == "costume" ( + string assetId = visual.assetId.toStr().strip() + string dataFormat = visual.dataFormat.toStr().toLower().strip() + if !regex.match("^[A-Za-z0-9_-]{8,128}$", assetId) or (dataFormat != "png" and dataFormat != "jpg" and dataFormat != "jpeg" and dataFormat != "svg") ( + return {error: "invalid portable item costume asset"} + ) + return {visual: {type: "costume", assetId: assetId, dataFormat: dataFormat}} + ) + return {error: "portable items need an image or costume"} +) + +def applyGameItems(object project, object body) Refusal ( + if !body.contains("gameItems") ( + return noRefusal() + ) + array incomingItems = body.gameItems.assertElse(array, []) + if incomingItems.len > 100 ( + return refuse(400, "projects can define at most 100 portable items") + ) + array items = [] + array itemIds = [] + for i incomingItems.len ( + object incomingItem = incomingItems[i].assertElse(object, {}) + string itemId = incomingItem.id.toStr().toLower().strip() + string itemName = incomingItem.name.toStr().strip() + object visual = incomingItem.visual.assertElse(object, {}) + if visual.getKeys().len == 0 and incomingItem.image.toStr().strip() != "" ( + visual = {type: "image", url: incomingItem.image.toStr().strip()} + ) + if !regex.match("^[a-z0-9_-]{1,40}$", itemId) or itemIds.contains(itemId) ( + return refuse(400, "portable item ids must be unique lowercase identifiers") + ) + if itemName.len < 1 or itemName.len > 80 ( + return refuse(400, "invalid portable item definition") + ) + object cleaned = portableItemVisual(visual) + if cleaned.contains("error") ( + return refuse(400, cleaned.error.toStr()) + ) + itemIds.append(itemId) + items.append({id: itemId, name: itemName, visual: cleaned.visual, gameAwardable: incomingItem.gameAwardable != false}) + ) + project.gameItems = items + return noRefusal() +) + +def applyInventoryPolicy(object project, object body) Refusal ( + if !body.contains("inventoryPolicy") ( + return noRefusal() + ) + object incomingPolicy = body.inventoryPolicy.assertElse(object, {}) + string mode = incomingPolicy.mode.toStr().toLower().strip() + if mode != "none" and mode != "allowlist" and mode != "all" ( + return refuse(400, "inventory policy must be none, allowlist, or all") + ) + array allowedItems = incomingPolicy.allowedItems.assertElse(array, []) + array allowedProjects = incomingPolicy.allowedProjects.assertElse(array, []) + if allowedItems.len > 100 or allowedProjects.len > 50 ( + return refuse(400, "portable item allowlist is too large") + ) + array cleanItems = [] + for i allowedItems.len ( + string globalId = allowedItems[i].toStr().strip() + if !regex.match("^[A-Za-z0-9_-]{1,80}:[a-z0-9_-]{1,40}$", globalId) ( + return refuse(400, "invalid global portable item id") + ) + if !cleanItems.contains(globalId) ( + cleanItems.append(globalId) + ) + ) + array cleanProjects = [] + for i allowedProjects.len ( + string allowedProject = allowedProjects[i].toStr().strip() + if !isSafePathPart(allowedProject) ( + return refuse(400, "invalid portable item source project") + ) + if !cleanProjects.contains(allowedProject) ( + cleanProjects.append(allowedProject) + ) + ) + project.inventoryPolicy = {mode: mode, allowedItems: cleanItems, allowedProjects: cleanProjects} + return noRefusal() +) + +def gameGrantRefusal(object project, object body) Refusal ( + if !body.contains("gameProducts") and !body.contains("gameItems") ( + return noRefusal() + ) + array configuredProducts = project.gameProducts.assertElse(array, []) + array configuredItems = project.gameItems.assertElse(array, []) + array configuredItemIds = [] + for i configuredItems.len ( + configuredItemIds.append(configuredItems[i].assert(object).id.toStr()) + ) + for i configuredProducts.len ( + string grantedItem = configuredProducts[i].assert(object).grantsItem.toStr() + if grantedItem != "" and !configuredItemIds.contains(grantedItem) ( + return refuse(400, "a game product refers to an undefined portable item") + ) + ) + return noRefusal() +) + +def applyCompatibility(object project, object body) ( + if body.contains("compatibility") and body.compatibility != null ( + object compatibilityInfo = body.compatibility.assert(object) + project.compatibility = { + mobile: compatibilityInfo.contains("mobile") and compatibilityInfo.mobile == true, + keyboard: compatibilityInfo.contains("keyboard") and compatibilityInfo.keyboard == true, + controller: compatibilityInfo.contains("controller") and compatibilityInfo.controller == true + } + ) +) diff --git a/src/projects/project-update-policy.test.osl b/src/projects/project-update-policy.test.osl new file mode 100644 index 0000000..e6f9635 --- /dev/null +++ b/src/projects/project-update-policy.test.osl @@ -0,0 +1,107 @@ +import "std:testing" +import "std:crypto" +import "std:regex" +import "std:fs" +import "std:json" +import "go:mistwarp.local/storagefs" +import "../core/utils.osl" +import "./project-refusal.osl" +import "./project-update-policy.osl" + +object free = {customProjectBranding: false, vanityProjectUrls: false} +object pro = {customProjectBranding: true, vanityProjectUrls: true} + +object project = {id: "p1", title: "Old"} +testing.assert(!isRefused(applyBranding(project, {}, free)) and !project.contains("branding"), "branding is left alone when not sent") +testing.assert(applyBranding(project, {branding: {accentColor: "red"}}, pro).message == "branding accent must be a six-digit hex colour", "accents must be hex colours") +testing.assert(applyBranding(project, {branding: {tagline: "t" * 121}}, pro).message == "branding tagline is too long", "taglines are capped") +testing.assert(applyBranding(project, {branding: {accentColor: "#123456"}}, free).status == 403, "custom branding needs a plan") +testing.assert(!isRefused(applyBranding(project, {branding: {}}, free)) and project.branding.len == 0, "anyone can clear branding") +testing.assert(!isRefused(applyBranding(project, {branding: {accentColor: " #ABCDEF ", tagline: " Hi "}}, pro)) and project.branding.accentColor == "#ABCDEF" and project.branding.tagline == "Hi", "plans can set trimmed branding") +testing.assert(!isRefused(applyBranding(project, {branding: {accentColor: "#ABCDEF", tagline: "Hi"}}, free)), "keeping existing branding needs no plan") + +testing.assert(vanitySlugRefusal("game", "", free).status == 403, "vanity URLs need Pro") +testing.assert(!isRefused(vanitySlugRefusal("game", "game", free)), "a lapsed plan can keep its slug") +testing.assert(!isRefused(vanitySlugRefusal("", "game", free)), "anyone can clear a slug") +testing.assert(vanitySlugRefusal("ab", "", pro).status == 400 and isRefused(vanitySlugRefusal("-ab", "", pro)) and isRefused(vanitySlugRefusal("a" * 41, "", pro)), "slugs are 3 to 40 safe characters") +testing.assert(!isRefused(vanitySlugRefusal("my-game", "", pro)), "valid slugs are accepted") + +object text = {id: "p1", title: "Old"} +testing.assert(applyProjectText(text, {title: " "}).message == "invalid title", "titles cannot be blank") +testing.assert(isRefused(applyProjectText(text, {title: "t" * 101})), "titles are capped") +testing.assert(!isRefused(applyProjectText(text, {title: " New ", description: "d", instructions: "i", notes: "n"})) and text.title == "New" and text.notes == "n", "text fields are saved") +testing.assert(applyProjectText(text, {description: "d" * 5001}).message == "description too long" and text.description == "d", "long descriptions are refused") +testing.assert(applyProjectText(text, {title: "Kept", notes: "n" * 5001}).message == "notes too long" and text.title == "Kept", "fields before a refused one are already applied") + +object credited = {id: "p1"} +testing.assert(!isRefused(applyCredits(credited, {credits: [{who: " Ada ", role: "Art", url: "https://a.example"}, {who: "", role: "x"}, {who: "Bo", role: "Music", url: null}]})), "credits are accepted") +testing.assert(credited.credits.len == 2 and credited.credits[1].url == "https://a.example" and !credited.credits[2].assert(object).contains("url"), "blank credits are dropped and empty URLs omitted") +testing.assert(applyCredits(credited, {credits: [{who: "A", role: "B", url: "ftp://x"}]}).message == "credit URL must use http or https", "credit URLs must be web links") +testing.assert(applyCredits(credited, {credits: [{who: "A", role: "B", url: "https://" ++ ("u" * 500)}]}).message == "credit URL too long", "credit URLs are capped") +array manyCredits = [] +for i 51 ( + manyCredits.append({who: "A", role: "B"}) +) +testing.assert(applyCredits(credited, {credits: manyCredits}).message == "too many credits", "credits are capped at 50") + +object flagged = {id: "p1"} +applyProjectFlags(flagged, {commentsOff: "yes", remixable: 0, seeInside: false}) +testing.assert(flagged.commentsOff == false and flagged.remixable == true and flagged.seeInside == false, "flags only switch on an exact boolean") +testing.assert(projectSaleRefusal({price: 5}).body.code == "project_sales_ended" and !isRefused(projectSaleRefusal({price: 0})) and !isRefused(projectSaleRefusal({})), "only a non-zero price is refused") +testing.assert(applyRemixShare(flagged, 51).status == 400 and isRefused(applyRemixShare(flagged, -1)), "remix shares are 0 to 50") +testing.assert(!isRefused(applyRemixShare(flagged, 50)) and flagged.remixShare == 50, "remix shares are saved") + +object tagged = {id: "p1"} +applyTags(tagged, {tags: ["Games", "games", "#Puzzle", "bad tag!", "a", "b", "c", "d", "e", "f", "g", "h", "i"]}) +testing.assert(tagged.tags == ["games", "puzzle", "a", "b", "c", "d", "e", "f", "g", "h"], "tags are normalized, unique and capped at 10") + +object product = {id: "Pack", name: "Pack", description: "d", price: 10, grantsItem: "sword", icon: "https://i.example/i.png"} +object shop = {id: "p1"} +def productWith(object base, string id, string name, number price, string icon, string grantsItem) object ( + object copy = base.clone() + copy.id = id + copy.name = name + copy.price = price + copy.icon = icon + copy.grantsItem = grantsItem + return copy +) +testing.assert(!isRefused(applyGameProducts(shop, {gameProducts: [product]})) and shop.gameProducts[1].id == "pack", "product ids are lowercased") +testing.assert(applyGameProducts(shop, {gameProducts: [product, product]}).message == "game product ids must be unique lowercase identifiers", "product ids are unique") +testing.assert(applyGameProducts(shop, {gameProducts: [productWith(product, "x", "", 1, "", "")]}).message == "invalid game product name or description", "products need a name") +testing.assert(applyGameProducts(shop, {gameProducts: [productWith(product, "x", "X", 1001, "", "")]}).message == "game product prices must be between 0 and 1000 credits", "product prices are capped") +testing.assert(applyGameProducts(shop, {gameProducts: [productWith(product, "x", "X", 1, "ftp://i", "")]}).message == "game product icons must be image data URLs or image URLs", "product icons are images") +testing.assert(applyGameProducts(shop, {gameProducts: [productWith(product, "x", "X", 1, "data:image/png," ++ ("a" * 100000), "")]}).message == "game product icons must be smaller than 100KB", "product icons are capped in bytes") +testing.assert(applyGameProducts(shop, {gameProducts: [productWith(product, "x", "X", 1, "", "Bad!")]}).message == "invalid portable item id on game product", "granted item ids are checked") + +testing.assert(portableItemVisual({type: "image", url: "https://i.example/a.png"}).visual.url == "https://i.example/a.png", "HTTPS images are kept") +testing.assert(portableItemVisual({type: "image", url: "http://i.example/a.png"}).error == "portable item images must use a short HTTPS URL", "plain HTTP images are refused") +testing.assert(portableItemVisual({type: "image", url: "data:image/png," ++ ("a" * 100000)}).error == "portable item images must be smaller than 100KB", "data images are capped") +testing.assert(portableItemVisual({type: "Costume", assetId: "abcdefgh", dataFormat: "SVG"}).visual.dataFormat == "svg", "costumes are normalized") +testing.assert(portableItemVisual({type: "costume", assetId: "short", dataFormat: "png"}).error == "invalid portable item costume asset", "costume ids are checked") +testing.assert(portableItemVisual({}).error == "portable items need an image or costume", "items need a visual") + +object items = {id: "p1"} +testing.assert(!isRefused(applyGameItems(items, {gameItems: [{id: "Sword", name: "Sword", image: "https://i.example/s.png"}, {id: "hat", name: "Hat", visual: {type: "costume", assetId: "abcdefgh", dataFormat: "png"}, gameAwardable: false}]})), "items are accepted") +testing.assert(items.gameItems[1].id == "sword" and items.gameItems[1].visual.type == "image" and items.gameItems[1].gameAwardable == true and items.gameItems[2].gameAwardable == false, "item images fall back and awardability defaults on") +testing.assert(applyGameItems(items, {gameItems: [{id: "a", name: "A", image: "https://i"}, {id: "A", name: "A", image: "https://i"}]}).message == "portable item ids must be unique lowercase identifiers", "item ids are unique") +testing.assert(applyGameItems(items, {gameItems: [{id: "a", name: "", image: ""}]}).message == "invalid portable item definition", "items need a name") +testing.assert(applyGameItems(items, {gameItems: [{id: "a", name: "A", image: ""}]}).message == "portable items need an image or costume", "item visual errors reach the caller") + +object granted = {id: "p1", gameProducts: [{id: "pack", grantsItem: "sword"}], gameItems: [{id: "sword"}]} +testing.assert(!isRefused(gameGrantRefusal(granted, {gameItems: []})), "products may grant defined items") +testing.assert(!isRefused(gameGrantRefusal({id: "p1", gameProducts: [{grantsItem: "ghost"}]}, {title: "x"})), "the grant check only runs when products or items change") +testing.assert(gameGrantRefusal({id: "p1", gameProducts: [{grantsItem: "ghost"}], gameItems: []}, {gameProducts: []}).message == "a game product refers to an undefined portable item", "products cannot grant undefined items") + +object inventory = {id: "p1"} +testing.assert(applyInventoryPolicy(inventory, {inventoryPolicy: {mode: "some"}}).message == "inventory policy must be none, allowlist, or all", "inventory modes are fixed") +testing.assert(applyInventoryPolicy(inventory, {inventoryPolicy: {mode: "all", allowedProjects: ["../x"]}}).message == "invalid portable item source project", "source projects must be safe ids") +testing.assert(applyInventoryPolicy(inventory, {inventoryPolicy: {mode: "all", allowedItems: ["nope"]}}).message == "invalid global portable item id", "allowed items are global ids") +testing.assert(!isRefused(applyInventoryPolicy(inventory, {inventoryPolicy: {mode: " Allowlist ", allowedItems: ["p1:sword", "p1:sword"], allowedProjects: ["p1", "p1"]}})), "a valid allowlist is accepted") +testing.assert(inventory.inventoryPolicy.mode == "allowlist" and inventory.inventoryPolicy.allowedItems == ["p1:sword"] and inventory.inventoryPolicy.allowedProjects == ["p1"], "allowlists are deduplicated") + +object compatible = {id: "p1"} +applyCompatibility(compatible, {compatibility: null}) +testing.assert(!compatible.contains("compatibility"), "null compatibility is ignored") +applyCompatibility(compatible, {compatibility: {mobile: true, controller: "yes"}}) +testing.assert(compatible.compatibility.mobile == true and compatible.compatibility.keyboard == false and compatible.compatibility.controller == false, "compatibility only accepts true") diff --git a/src/projects/project-update.osl b/src/projects/project-update.osl index 44f038f..aba371d 100644 --- a/src/projects/project-update.osl +++ b/src/projects/project-update.osl @@ -1,3 +1,9 @@ +type OwnerTransfer ( + string fromId = "" + string toId = "" + object hold = {} +) + def handleUpdateProject(*serve.Context c) ( string lockName = "project-upload:" ++ c.param("id") sync.lock(lockName) @@ -6,487 +12,197 @@ def handleUpdateProject(*serve.Context c) ( ) def handleUpdateProjectLocked(*serve.Context c) ( - auto res = loadProject(c.param("id")) - if !res.ok ( - c.notFound("project not found") - return - ) - object project = res.project.assert(object) + object? loaded = loadRequestedProject(c) + if loaded == null return + object project = loaded array previousTeam = project.collaborators.assertElse(array, []) if !canEditProject(project, c) ( c.forbidden("not your project") return ) - object body = requestJSON(c) - if contextString(c, "sessionScope") == "editor" and !editorProjectUpdateAllowed(body.getKeys()) ( - c.json(403, {ok: false, code: "editor_scope", error: "Open MistWarp to do this."}) + if refused(c, updateScopeRefusal(c, project, body)) return + + OwnerTransfer transfer = OwnerTransfer() + if refused(c, applyOwnerTransfer(c, project, body, transfer)) return + if refused(c, applyGroupTag(c, project, body)) return + object creatorBenefits = mistwarpBenefitsForProject(project) + if refused(c, applyBranding(project, body, creatorBenefits)) return + if refused(c, applyVanitySlug(project, body, creatorBenefits)) return + if refused(c, applyProjectText(project, body)) return + if refused(c, applyCredits(project, body)) return + applyProjectFlags(project, body) + if refused(c, projectSaleRefusal(body)) return + if refused(c, applyRemixShareUpdate(c, project, body)) return + applyTags(project, body) + if refused(c, applyGameProducts(project, body)) return + if refused(c, applyGameItems(project, body)) return + if refused(c, applyInventoryPolicy(project, body)) return + if refused(c, gameGrantRefusal(project, body)) return + applyCompatibility(project, body) + if refused(c, applyTeamUpdate(c, project, body)) return + + if refused(c, holdTransferredStorage(c, project, transfer)) return + project.edited = timestamp + if !saveProject(project) ( + storageHoldRelease(transfer.toId, transfer.hold) + c.internalError("could not save project access") return ) - if isStudentUserId((project.ownerId ?? "").toStr()) and !contextBool(c, "isAdmin") and !studentProjectUpdateAllowed(body.getKeys()) ( - c.json(403, {ok: false, code: "student_restricted", error: "Class work can only change its title, description, instructions, notes and credits."}) - return + upsertIndexEntry(project) + settleTransferredStorage(transfer) + if body.contains("collaborators") ( + notifyNewTeammates(project, previousTeam, contextString(c, "username")) ) - string transferredFromId = "" - if body.contains("owner") ( - boolean canTransfer = contextBool(c, "isAdmin") or projectRole(project, contextString(c, "username")) == "owner" or canManageGroupContent(project, c) - if !canTransfer ( - c.forbidden("you cannot transfer this project") - return - ) - string requestedOwner = body.owner.toStr().strip() - string nextOwnerId = discoverUserIdentity(requestedOwner) - if nextOwnerId == "" ( - c.badRequest("Rotur user not found") - return - ) - string nextOwner = usernameForId(nextOwnerId) - if nextOwner == "" ( - c.badRequest("Rotur user not found") - return - ) - number nextOwnerLimit = mistwarpBenefitsForUserId(nextOwnerId).maxProjects.toNum() - if !identityMatches((project.ownerId ?? "").toStr(), project.owner.toStr(), nextOwnerId, nextOwner) and ownedProjectSlotCount(nextOwnerId, nextOwner) >= nextOwnerLimit ( - c.badRequest("the new owner has reached their project limit") - return - ) - if !identityMatches((project.ownerId ?? "").toStr(), project.owner.toStr(), nextOwnerId, nextOwner) ( - transferredFromId = projectOwnerId(project) - ) - project.owner = nextOwner - project.ownerId = nextOwnerId - if !body.contains("keepGroup") or body.keepGroup != true ( - project.groupTag = "" - ) + c.json(200, {ok: true, project: projectResponse(project, c)}) +) + +def updateScopeRefusal(*serve.Context c, object project, object body) Refusal ( + if isEditorSession(c) and !editorProjectUpdateAllowed(body.getKeys()) ( + return editorScopeRefusal("Open MistWarp to do this.") ) - if body.contains("groupTag") ( - string nextGroupTag = normalizeGroupTag(body.groupTag.toStr()) - if body.groupTag.toStr().strip() == "" ( - nextGroupTag = "" - ) else if nextGroupTag == "" ( - c.badRequest("invalid group tag") - return - ) else if !canManageGroupTag(nextGroupTag, c) ( - c.forbidden("you cannot move this project into that group") - return - ) - project.groupTag = nextGroupTag + if isStudentUserId((project.ownerId ?? "").toStr()) and !contextBool(c, "isAdmin") and !studentProjectUpdateAllowed(body.getKeys()) ( + return refuseWithBody(403, {ok: false, code: "student_restricted", error: "Class work can only change its title, description, instructions, notes and credits."}) ) - object creatorBenefits = mistwarpBenefitsForProject(project) - if body.contains("branding") ( - object incomingBranding = body.branding.assertElse(object, {}) - boolean clearingBranding = incomingBranding.getKeys().len == 0 - string accentColor = (incomingBranding.accentColor ?? "").toStr().strip() - string tagline = (incomingBranding.tagline ?? "").toStr().strip() - if accentColor != "" and !regex.match("^#[0-9a-fA-F]{6}$", accentColor) ( - c.badRequest("branding accent must be a six-digit hex colour") - return - ) - if tagline.len > 120 ( - c.badRequest("branding tagline is too long") - return - ) - object nextBranding = if clearingBranding then {} else {accentColor: accentColor, tagline: tagline} - object currentBranding = if project.contains("branding") then project.branding.assertElse(object, {}) else {} - if creatorBenefits.customProjectBranding != true and json.stringify(nextBranding) != json.stringify(currentBranding) ( - c.forbidden("custom project branding requires Rotur Plus or Pro") - return - ) - project.branding = nextBranding + return noRefusal() +) + +def isOwnerOrAdmin(*serve.Context c, object project) boolean ( + return projectRole(project, contextString(c, "username")) == "owner" or contextBool(c, "isAdmin") +) + +def applyOwnerTransfer(*serve.Context c, object project, object body, OwnerTransfer transfer) Refusal ( + if !body.contains("owner") ( + return noRefusal() ) - if body.contains("vanitySlug") ( - string slug = body.vanitySlug.toStr().toLower().strip() - string currentSlug = if project.contains("vanitySlug") then project.vanitySlug.toStr() else "" - if slug != "" and slug != currentSlug and creatorBenefits.vanityProjectUrls != true ( - c.forbidden("vanity project URLs require Rotur Pro") - return - ) - if slug != "" and (slug.len < 3 or slug.len > 40 or !regex.match("^[a-z0-9][a-z0-9-]*[a-z0-9]$", slug)) ( - c.badRequest("vanity URLs must be 3 to 40 letters, numbers, or hyphens") - return - ) - if slug != "" and slug != currentSlug ( - array matches = vanityClaimEntries(slug) - for i matches.len ( - object holder = matches[i].assert(object) - string holderId = holder.id.toStr() - if holderId != project.id.toStr() ( - // Free if it has been lapsed for 30 days. Never wait on the other - // project's lock while holding this one's. - string holderEligibility = vanityEligibility(holder) - string holderLock = "project-upload:" ++ holderId - boolean held = true - if sync.tryLock(holderLock) ( - held = applyVanityLapseLocked(holderId, slug, holderEligibility) == slug - sync.unlock(holderLock) - ) - if held ( - c.json(409, {ok: false, error: "that vanity URL is already taken"}) - return - ) - ) - ) - ) - if slug != currentSlug ( - project.delete("vanityLapsedAt") - ) - project.vanitySlug = slug + boolean canTransfer = contextBool(c, "isAdmin") or projectRole(project, contextString(c, "username")) == "owner" or canManageGroupContent(project, c) + if !canTransfer ( + return refuse(403, "you cannot transfer this project") ) - if body.contains("title") ( - string title = body.title.toStr().strip() - if title == "" or title.len > 100 ( - c.badRequest("invalid title") - return - ) - project.title = title + string requestedOwner = body.owner.toStr().strip() + string nextOwnerId = discoverUserIdentity(requestedOwner) + if nextOwnerId == "" ( + return refuse(400, "Rotur user not found") ) - if body.contains("description") ( - string description = body.description.toStr() - if description.len > 5000 ( - c.badRequest("description too long") - return - ) - project.description = description + string nextOwner = usernameForId(nextOwnerId) + if nextOwner == "" ( + return refuse(400, "Rotur user not found") ) - if body.contains("instructions") ( - string instructions = body.instructions.toStr() - if instructions.len > 5000 ( - c.badRequest("instructions too long") - return - ) - project.instructions = instructions + number nextOwnerLimit = mistwarpBenefitsForUserId(nextOwnerId).maxProjects.toNum() + boolean sameOwner = identityMatches((project.ownerId ?? "").toStr(), project.owner.toStr(), nextOwnerId, nextOwner) + if !sameOwner and ownedProjectSlotCount(nextOwnerId, nextOwner) >= nextOwnerLimit ( + return refuse(400, "the new owner has reached their project limit") ) - if body.contains("notes") ( - string notes = body.notes.toStr() - if notes.len > 5000 ( - c.badRequest("notes too long") - return - ) - project.notes = notes + if !sameOwner ( + transfer.fromId = projectOwnerId(project) ) - if body.contains("credits") ( - array incoming = body.credits.assert(array) - if incoming.len > 50 ( - c.badRequest("too many credits") - return - ) - array credits = [] - for i incoming.len ( - object entry = incoming[i] - string who = entry.who.toStr().strip() - string role = entry.role.toStr().strip() - string creditUrl = "" - if entry.contains("url") and entry.url != null ( - creditUrl = entry.url.toStr().strip() - ) - if creditUrl != "" and (!creditUrl.startsWith("https://") and !creditUrl.startsWith("http://")) ( - c.badRequest("credit URL must use http or https") - return - ) - if creditUrl.len > 500 ( - c.badRequest("credit URL too long") - return + project.owner = nextOwner + project.ownerId = nextOwnerId + if !body.contains("keepGroup") or body.keepGroup != true ( + project.groupTag = "" + ) + return noRefusal() +) + +def applyGroupTag(*serve.Context c, object project, object body) Refusal ( + if !body.contains("groupTag") ( + return noRefusal() + ) + string nextGroupTag = normalizeGroupTag(body.groupTag.toStr()) + if body.groupTag.toStr().strip() == "" ( + nextGroupTag = "" + ) else if nextGroupTag == "" ( + return refuse(400, "invalid group tag") + ) else if !canManageGroupTag(nextGroupTag, c) ( + return refuse(403, "you cannot move this project into that group") + ) + project.groupTag = nextGroupTag + return noRefusal() +) + +def vanitySlugHeldElsewhere(object project, string slug) boolean ( + array matches = vanityClaimEntries(slug) + for i matches.len ( + object holder = matches[i].assert(object) + string holderId = holder.id.toStr() + if holderId != project.id.toStr() ( + string holderEligibility = vanityEligibility(holder) + string holderLock = "project-upload:" ++ holderId + boolean held = true + if sync.tryLock(holderLock) ( + held = applyVanityLapseLocked(holderId, slug, holderEligibility) == slug + sync.unlock(holderLock) ) - if who != "" and who.len <= 60 and role.len <= 120 ( - object credit = {who: who, role: role} - if creditUrl != "" ( - credit.url = creditUrl - ) - credits.append(credit) + if held ( + return true ) ) - project.credits = credits - ) - if body.contains("commentsOff") ( - project.commentsOff = body.commentsOff == true - ) - if body.contains("remixable") ( - project.remixable = body.remixable != false - ) - if body.contains("seeInside") ( - project.seeInside = body.seeInside != false - ) - if body.contains("price") and (body.price ?? 0).toNum() != 0 ( - c.json(400, {ok: false, code: "project_sales_ended", error: "Projects can no longer be sold on MistWarp."}) - return ) - if body.contains("remixShare") ( - if projectRole(project, contextString(c, "username")) != "owner" and !contextBool(c, "isAdmin") ( - c.forbidden("only the owner can change the remix share") - return - ) - number remixShare = body.remixShare.toNum() - if remixShare < 0 or remixShare > 50 ( - c.badRequest("remix share must be between 0 and 50 percent") - return - ) - project.remixShare = remixShare + return false +) + +def applyVanitySlug(object project, object body, object creatorBenefits) Refusal ( + if !body.contains("vanitySlug") ( + return noRefusal() ) - if body.contains("tags") ( - array incoming = body.tags.assert(array) - array tags = [] - for i incoming.len ( - if tags.len < 10 ( - string t = normalizeTag(incoming[i].toStr()) - if t != "" and !tags.contains(t) ( - tags.append(t) - ) - ) - ) - project.tags = tags + string slug = body.vanitySlug.toStr().toLower().strip() + string currentSlug = if project.contains("vanitySlug") then project.vanitySlug.toStr() else "" + Refusal invalidSlug = vanitySlugRefusal(slug, currentSlug, creatorBenefits) + if isRefused(invalidSlug) ( + return invalidSlug ) - if body.contains("gameProducts") ( - array incomingProducts = body.gameProducts.assertElse(array, []) - if incomingProducts.len > 50 ( - c.badRequest("projects can have at most 50 game products") - return - ) - array products = [] - array productIds = [] - for i incomingProducts.len ( - object incomingProduct = incomingProducts[i].assertElse(object, {}) - string productId = incomingProduct.id.toStr().toLower().strip() - string productName = incomingProduct.name.toStr().strip() - string productDescription = incomingProduct.description.toStr().strip() - number productPrice = incomingProduct.price.toNum() - string grantsItem = incomingProduct.grantsItem.toStr().toLower().strip() - string productIcon = incomingProduct.icon.toStr().strip() - if !regex.match("^[a-z0-9_-]{1,40}$", productId) or productIds.contains(productId) ( - c.badRequest("game product ids must be unique lowercase identifiers") - return - ) - if productName.len < 1 or productName.len > 80 or productDescription.len > 300 ( - c.badRequest("invalid game product name or description") - return - ) - if productPrice < 0 or productPrice > 1000 ( - c.badRequest("game product prices must be between 0 and 1000 credits") - return - ) - if productIcon != "" and productIcon.toBytes().len > 100000 ( - c.badRequest("game product icons must be smaller than 100KB") - return - ) - if productIcon != "" and !productIcon.startsWith("data:image/") and !productIcon.startsWith("https://") and !productIcon.startsWith("http://") ( - c.badRequest("game product icons must be image data URLs or image URLs") - return - ) - if grantsItem != "" and !regex.match("^[a-z0-9_-]{1,40}$", grantsItem) ( - c.badRequest("invalid portable item id on game product") - return - ) - productIds.append(productId) - products.append({id: productId, name: productName, description: productDescription, price: productPrice, grantsItem: grantsItem, icon: productIcon}) - ) - project.gameProducts = products + if slug != "" and slug != currentSlug and vanitySlugHeldElsewhere(project, slug) ( + return refuseWithBody(409, {ok: false, error: "that vanity URL is already taken"}) ) - if body.contains("gameItems") ( - array incomingItems = body.gameItems.assertElse(array, []) - if incomingItems.len > 100 ( - c.badRequest("projects can define at most 100 portable items") - return - ) - array items = [] - array itemIds = [] - for i incomingItems.len ( - object incomingItem = incomingItems[i].assertElse(object, {}) - string itemId = incomingItem.id.toStr().toLower().strip() - string itemName = incomingItem.name.toStr().strip() - object visual = incomingItem.visual.assertElse(object, {}) - if visual.getKeys().len == 0 and incomingItem.image.toStr().strip() != "" ( - visual = {type: "image", url: incomingItem.image.toStr().strip()} - ) - string visualType = visual.type.toStr().toLower().strip() - if !regex.match("^[a-z0-9_-]{1,40}$", itemId) or itemIds.contains(itemId) ( - c.badRequest("portable item ids must be unique lowercase identifiers") - return - ) - if itemName.len < 1 or itemName.len > 80 ( - c.badRequest("invalid portable item definition") - return - ) - object cleanVisual = {} - if visualType == "image" ( - string itemImage = visual.url.toStr().strip() - if itemImage.startsWith("data:image/") ( - if itemImage.toBytes().len > 100000 ( - c.badRequest("portable item images must be smaller than 100KB") - return - ) - cleanVisual = {type: "image", url: itemImage} - ) else ( - if !itemImage.startsWith("https://") or itemImage.len > 500 ( - c.badRequest("portable item images must use a short HTTPS URL") - return - ) - cleanVisual = {type: "image", url: itemImage} - ) - ) else if visualType == "costume" ( - string assetId = visual.assetId.toStr().strip() - string dataFormat = visual.dataFormat.toStr().toLower().strip() - if !regex.match("^[A-Za-z0-9_-]{8,128}$", assetId) or (dataFormat != "png" and dataFormat != "jpg" and dataFormat != "jpeg" and dataFormat != "svg") ( - c.badRequest("invalid portable item costume asset") - return - ) - cleanVisual = {type: "costume", assetId: assetId, dataFormat: dataFormat} - ) else ( - c.badRequest("portable items need an image or costume") - return - ) - itemIds.append(itemId) - items.append({id: itemId, name: itemName, visual: cleanVisual, gameAwardable: incomingItem.gameAwardable != false}) - ) - project.gameItems = items + if slug != currentSlug ( + project.delete("vanityLapsedAt") ) - if body.contains("inventoryPolicy") ( - object incomingPolicy = body.inventoryPolicy.assertElse(object, {}) - string mode = incomingPolicy.mode.toStr().toLower().strip() - if mode != "none" and mode != "allowlist" and mode != "all" ( - c.badRequest("inventory policy must be none, allowlist, or all") - return - ) - array allowedItems = incomingPolicy.allowedItems.assertElse(array, []) - array allowedProjects = incomingPolicy.allowedProjects.assertElse(array, []) - if allowedItems.len > 100 or allowedProjects.len > 50 ( - c.badRequest("portable item allowlist is too large") - return - ) - array cleanItems = [] - for i allowedItems.len ( - string globalId = allowedItems[i].toStr().strip() - if !regex.match("^[A-Za-z0-9_-]{1,80}:[a-z0-9_-]{1,40}$", globalId) ( - c.badRequest("invalid global portable item id") - return - ) - if !cleanItems.contains(globalId) ( - cleanItems.append(globalId) - ) - ) - array cleanProjects = [] - for i allowedProjects.len ( - string allowedProject = allowedProjects[i].toStr().strip() - if !isSafePathPart(allowedProject) ( - c.badRequest("invalid portable item source project") - return - ) - if !cleanProjects.contains(allowedProject) ( - cleanProjects.append(allowedProject) - ) - ) - project.inventoryPolicy = {mode: mode, allowedItems: cleanItems, allowedProjects: cleanProjects} + project.vanitySlug = slug + return noRefusal() +) + +def applyRemixShareUpdate(*serve.Context c, object project, object body) Refusal ( + if !body.contains("remixShare") ( + return noRefusal() ) - if body.contains("gameProducts") or body.contains("gameItems") ( - array configuredProducts = project.gameProducts.assertElse(array, []) - array configuredItems = project.gameItems.assertElse(array, []) - array configuredItemIds = [] - for i configuredItems.len ( - configuredItemIds.append(configuredItems[i].assert(object).id.toStr()) - ) - for i configuredProducts.len ( - string grantedItem = configuredProducts[i].assert(object).grantsItem.toStr() - if grantedItem != "" and !configuredItemIds.contains(grantedItem) ( - c.badRequest("a game product refers to an undefined portable item") - return - ) - ) + if !isOwnerOrAdmin(c, project) ( + return refuse(403, "only the owner can change the remix share") ) - if body.contains("compatibility") and body.compatibility != null ( - object compatibilityInfo = body.compatibility.assert(object) - project.compatibility = { - mobile: compatibilityInfo.contains("mobile") and compatibilityInfo.mobile == true, - keyboard: compatibilityInfo.contains("keyboard") and compatibilityInfo.keyboard == true, - controller: compatibilityInfo.contains("controller") and compatibilityInfo.controller == true - } + return applyRemixShare(project, body.remixShare.toNum()) +) + +def applyTeamUpdate(*serve.Context c, object project, object body) Refusal ( + if !body.contains("collaborators") ( + return noRefusal() ) - if body.contains("collaborators") ( - if projectRole(project, contextString(c, "username")) != "owner" and !contextBool(c, "isAdmin") ( - c.forbidden("only the owner can manage the project team") - return - ) - array incoming = body.collaborators.assert(array) - array collaborators = [] - number totalShare = 0 - for i incoming.len ( - if collaborators.len < 25 ( - object entry = incoming[i].assert(object) - string username = normalizeUsername(entry.username.toStr()) - string role = entry.role.toStr().toLower().strip() - boolean validRole = role == "editor" or role == "maintainer" or role == "contributor" or role == "tester" - number share = 0 - if entry.contains("share") and entry.share != null ( - share = entry.share.toNum() - ) - if share < 0 or share > 50 ( - c.badRequest("each teammate share must be between 0 and 50 percent") - return - ) - if username != "" and validRole and normalizeUsername(username) != normalizeUsername(project.owner.toStr()) ( - string teammateId = discoverUserIdentity(username) - if teammateId == "" ( - c.badRequest("Rotur user not found: " ++ username) - return - ) - for memberIndex collaborators.len ( - if collaborators[memberIndex].username.toStr() == username ( - c.badRequest("a person can only appear once in the project team") - return - ) - ) - totalShare += share - collaborators.append({username: username, userId: teammateId, role: role, share: share}) - ) - ) - ) - if totalShare > 70 ( - c.badRequest("teammate revenue shares cannot exceed 70 percent") - return - ) - project.collaborators = collaborators + if !isOwnerOrAdmin(c, project) ( + return refuse(403, "only the owner can manage the project team") ) + return applyCollaborators(project, body.collaborators.assert(array)) +) - // A project given to someone else counts against their storage from now. - object transferHold = noStorageHold() - string transferredToId = projectOwnerId(project) - if transferredFromId != "" and usesRoturStorage(transferredToId) ( - object moving = {} - moving[project.id.toStr()] = projectStorageSummary(project, loadAssetsIndex().assets.assertElse(object, {})) - transferHold = storageHold(transferredToId, {projects: moving}, !contextBool(c, "isAdmin")) - if transferHold.ok != true ( - object refusal = transferHold.refusal.assertElse(object, {}) - c.json(413, {ok: false, code: "storage_full", error: "The new owner's Rotur storage doesn't have room for this project.", needed: (refusal.needed ?? 0).toNum()}) - return - ) - ) - project.edited = timestamp - if !saveProject(project) ( - storageHoldRelease(transferredToId, transferHold) - c.internalError("could not save project access") +def holdTransferredStorage(*serve.Context c, object project, OwnerTransfer transfer) Refusal ( + transfer.hold = noStorageHold() + transfer.toId = projectOwnerId(project) + if transfer.fromId == "" or !usesRoturStorage(transfer.toId) ( + return noRefusal() + ) + object moving = {} + moving[project.id.toStr()] = projectStorageSummary(project, loadAssetsIndex().assets.assertElse(object, {})) + transfer.hold = storageHold(transfer.toId, {projects: moving}, !contextBool(c, "isAdmin")) + if transfer.hold.ok != true ( + object storageRefusal = transfer.hold.refusal.assertElse(object, {}) + return refuseWithBody(413, {ok: false, code: "storage_full", error: "The new owner's Rotur storage doesn't have room for this project.", needed: (storageRefusal.needed ?? 0).toNum()}) + ) + return noRefusal() +) + +def settleTransferredStorage(OwnerTransfer transfer) ( + if transfer.fromId == "" ( return ) - upsertIndexEntry(project) - if transferredFromId != "" ( - storageSettleSoon(transferredFromId) - if usesRoturStorage(transferredToId) ( - thread.new(storageSettle, transferredToId, transferHold) - ) - ) - if body.contains("collaborators") ( - array nextTeam = project.collaborators.assert(array) - for i nextTeam.len ( - object member = nextTeam[i].assert(object) - boolean newlyShared = true - for j previousTeam.len ( - object previousMember = previousTeam[j].assert(object) - if normalizeUsername(previousMember.username.toStr()) == normalizeUsername(member.username.toStr()) and previousMember.role == member.role ( - newlyShared = false - ) - ) - if newlyShared ( - addNotification(member.username.toStr(), contextString(c, "username"), "project_shared", { - projectId: project.id, projectTitle: project.title, role: member.role - }) - ) - ) + storageSettleSoon(transfer.fromId) + if usesRoturStorage(transfer.toId) ( + thread.new(storageSettle, transfer.toId, transfer.hold) ) - c.json(200, {ok: true, project: projectResponse(project, c)}) ) diff --git a/src/projects/project-upload-files.osl b/src/projects/project-upload-files.osl new file mode 100644 index 0000000..e93b9f6 --- /dev/null +++ b/src/projects/project-upload-files.osl @@ -0,0 +1,190 @@ +export {} + +def scanProjectJsonEvents(*json.Stream stream) object ( + boolean started = false + boolean readingMd5ext = false + number targets = 0 + array md5ext = [] + for event in stream.events() ( + if !started ( + started = true + if event.type != "object-start" ( + return {ok: false} + ) + ) else if readingMd5ext ( + readingMd5ext = false + if event.type == "string" ( + md5ext.append(event.value) + if md5ext.len > maxAssetCount * 50 ( + return {ok: false} + ) + ) + ) else if event.type == "key" ( + string key = event.value.toStr() + if key == "targets" ( + targets += 1 + ) + readingMd5ext = key == "md5ext" + ) + ) + if !started ( + return {ok: false} + ) + return {ok: true, targets: targets, md5ext: md5ext} +) + +def inspectProjectJson(string path) object ( + *json.Stream stream = json.open(path, maxProjectJsonBytes) + object inspected = scanProjectJsonEvents(stream) catch (err) ( + stream.close() + return {ok: false} + ) + stream.close() + return inspected +) + +// storageAllowanceBytes is a student's local limit, or -1. A Rotur owner's +// storage is held in the ledger instead, unless enforceStorage is off +// (admins); the result's hold is committed or released by the caller. +def uploadProjectFiles(object project, string extractDir, object benefits, number storageAllowanceBytes, boolean enforceStorage) object ( + array names = fs.readDir(extractDir).unwrapOr([]) + string jsonPath = extractDir ++ "/project.json" + if !fs.exists(jsonPath) ( + return {ok: false, error: "sb3 has no project.json", status: 400} + ) + number projectJsonBytes = fs.size(jsonPath).unwrapOr(0) + if projectJsonBytes > maxProjectJsonBytes ( + return {ok: false, error: "project.json exceeds the 256 MB expanded limit", code: "project_too_large", status: 413} + ) + object inspected = inspectProjectJson(jsonPath) + if !inspected.ok ( + return {ok: false, error: "invalid project.json", status: 400} + ) + if inspected.targets.toNum() == 0 ( + return {ok: false, error: "invalid project.json", status: 400} + ) + string gzipPath = jsonPath ++ ".gz" + if !zip.gzip(jsonPath, gzipPath, {maxInputBytes: maxProjectJsonBytes, maxBytes: maxStoredProjectJsonBytes}) ( + return {ok: false, error: "compressed project data exceeds the 20 MB limit", code: "project_too_large", status: 413} + ) + + object assetsIndex = loadAssetsIndex() + object knownAssets = assetsIndex.assets + array projectAssets = [] + array pendingUploads = [] + object pendingNames = {} + number assetCount = 0 + + for i names.len ( + string name = names[i].toStr() + if name != "project.json" ( + if isValidMd5Ext(name) ( + assetCount += 1 + if assetCount > maxAssetCount ( + return {ok: false, error: "too many assets", status: 400} + ) + string assetPath = extractDir ++ "/" ++ name + object tooLarge = assetSizeFailure(name, fs.size(assetPath).unwrapOr(0), benefits.maxProjectAssetBytes.toNum()) + if tooLarge.len > 0 ( + return tooLarge + ) + string content = fs.readFile(assetPath).unwrapOr("") + string actualHash = content.hashMD5() + if actualHash != md5PartOf(name) ( + return {ok: false, error: "asset filename does not match its content: " ++ name, status: 400} + ) + projectAssets.append(name) + string localAssetPath = localBlobPath(assetObjectKey(name)) + boolean needsUpload = assetNeedsUpload(knownAssets.contains(name), fs.exists(localAssetPath),!r2Local) + if needsUpload and !pendingNames.contains(name) ( + pendingUploads.append({path: assetPath, name: name}) + pendingNames[name] = true + knownAssets[name] = content.toBytes().len + ) + ) + ) + ) + + object providedSet = {} + for i projectAssets.len ( + providedSet[projectAssets[i].toStr()] = true + ) + array referencedList = inspected.md5ext.assert(array) + object referencedNames = {} + for i referencedList.len ( + string name = referencedList[i].toStr() + if isValidMd5Ext(name) ( + referencedNames[name] = true + ) + ) + array referencedUnique = referencedNames.getKeys() + for i referencedUnique.len ( + string name = referencedUnique[i].toStr() + if !providedSet.contains(name) ( + if !knownAssets.contains(name) ( + return {ok: false, error: "missing asset: " ++ name, status: 400} + ) + projectAssets.append(name) + string localAssetPath = localBlobPath(assetObjectKey(name)) + if !r2Local and fs.exists(localAssetPath) and !pendingNames.contains(name) ( + pendingUploads.append({path: localAssetPath, name: name}) + pendingNames[name] = true + ) + ) + ) + if projectAssets.len > maxAssetCount ( + return {ok: false, error: "too many assets", status: 400} + ) + + number assetBytes = 0 + for i projectAssets.len ( + string name = projectAssets[i].toStr() + if knownAssets.contains(name) ( + assetBytes += knownAssets[name].toNum() + ) + ) + object assetsTooLarge = projectAssetsFailure(assetBytes, benefits.maxProjectAssetsBytes.toNum()) + if assetsTooLarge.len > 0 ( + return assetsTooLarge + ) + object storageFull = storageAllowanceFailure(assetBytes + fs.size(gzipPath).unwrapOr(0) + projectHistoryBytes(project), storageAllowanceBytes) + if storageFull.len > 0 ( + return storageFull + ) + object hold = noStorageHold() + string ownerId = projectOwnerId(project) + if usesRoturStorage(ownerId) ( + // Assets uploaded ahead are counted by this save now, not twice. + releasePendingAssetHolds(ownerId, project.owner.toStr(), projectAssets) + object incoming = projectStorageSummary(project, knownAssets) + object incomingJson = {} + incomingJson["incoming:" ++ project.id.toStr()] = fs.size(gzipPath).unwrapOr(0) + incoming.json = incomingJson + incoming.assets = projectAssets + object saving = {} + saving[project.id.toStr()] = incoming + hold = storageHold(ownerId, {projects: saving}, enforceStorage) + if hold.ok != true ( + return {ok: false, error: "storage full", code: "storage_full", status: 413, refusal: hold.refusal} + ) + ) + + number uploaded = 0 + for i pendingUploads.len ( + object upload = pendingUploads[i].assert(object) + string path = upload.path.toStr() + string name = upload.name.toStr() + string content = fs.readFile(path).unwrapOr("") + if !stageR2Blob(assetObjectKey(name), content, contentTypeForExt(extPartOf(name)), "") ( + storageHoldRelease(ownerId, hold) + return {ok: false, error: "asset upload failed", status: 502} + ) + knownAssets[name] = content.toBytes().len + uploaded += 1 + ) + if uploaded > 0 ( + saveAssetsIndex(assetsIndex) + ) + + return {ok: true, assets: projectAssets, uploaded: uploaded, skipped: projectAssets.len - uploaded, gzipPath: gzipPath, jsonBytes: projectJsonBytes, storedJsonBytes: fs.size(gzipPath).unwrapOr(0), assetBytes: assetBytes, hold: hold} +) diff --git a/src/projects/project-upload-history.osl b/src/projects/project-upload-history.osl new file mode 100644 index 0000000..197e156 --- /dev/null +++ b/src/projects/project-upload-history.osl @@ -0,0 +1,295 @@ +export { + validateUploadedHistory +} + +def invalidHistoryArchiveRefusal() Refusal ( + return refuseWithBody(400, {ok: false, code: "invalid_workspace", error: "invalid or unsafe MistWarp history archive"}) +) + +def unsafeProjectHistoryRefusal() Refusal ( + return refuseWithBody(400, {ok: false, error: "invalid or unsafe MistWarp project history", code: "invalid_workspace"}) +) + +def isSafeHistoryArchive(byte[] body, UploadArchive archive, boolean readContents) boolean ( + return validateZipArchive(body, maxWorkspaceBytes, archive.expandedLimit, maxWorkspaceFiles, readContents) +) + +def checkUploadedWorkspace(object project, UploadRequest request, UploadHistory history, UploadArchive archive) Refusal ( + if !history.hasWorkspace ( + return noRefusal() + ) + if history.workspaceSize > maxWorkspaceBytes or !isSafeHistoryArchive(history.workspace, archive, true) ( + return invalidHistoryArchiveRefusal() + ) + object materialized = materializeWorktreeAssets(project, history.workspace, archive.extractDir, request.replaceHistory) + if !materialized.ok ( + return refuseWithBody(409, {ok: false, code: "worktree_assets_unavailable", error: "some project files could not be found on the server; save again to upload them"}) + ) + if materialized.refs.toNum() > 0 ( + if materialized.size.toNum() > maxWorkspaceBytes or !isSafeHistoryArchive(materialized.data.assertElse(byte[], []), archive, false) ( + return invalidHistoryArchiveRefusal() + ) + history.workspace = materialized.data.assertElse(byte[], []) + history.workspaceSize = materialized.size.toNum() + ) + object checked = validateUploadedHistoryMode(project, history.workspace, history.metadata, history.currentHead, request.replaceHistory) + if !checked.ok ( + return refuseWithBody(409, {ok: false, code: "invalid_workspace", error: checked.error}) + ) + history.workspace = checked.data.assertElse(byte[], []) + history.workspaceSize = checked.size.toNum() + history.info = checked.manifest.assert(object) + history.metadata = json.stringify(history.info) + return noRefusal() +) + +def unchangedVersionCheck(object project, UploadRequest request, UploadHistory history, UploadArchive archive) Refusal ( + if !isOrdinaryVersionSave(request, history) ( + return noRefusal() + ) + string incomingHead = canonicalHistoryHead(history.info) + if incomingHead == history.currentHead ( + return unchangedVersionRefusal() + ) + if incomingHead == "" ( + return noRefusal() + ) + if history.workspaceSize > maxWorkspaceBytes or !isSafeHistoryArchive(history.workspace, archive, false) ( + return unsafeProjectHistoryRefusal() + ) + string compareBasePath = cachedProjectWorkspacePath(project) + string compareIncomingPath = tmpDir ++ archive.workId ++ ".compare-incoming.mwp" + boolean compareReady = compareBasePath != "" and fs.writeFileBytes(compareIncomingPath, history.workspace) + string comparisonJson = "" + if compareReady ( + comparisonJson = gitinspection.CommitTreesDiffer(compareBasePath, history.currentHead, compareIncomingPath, incomingHead) + ) + void fs.remove(compareIncomingPath) + auto parsedComparison = json.parse(comparisonJson, {type: "object"}) + if !compareReady or parsedComparison.isErr() or parsedComparison.unwrap().assert(object).ok != true ( + return refuseWithBody(400, {ok: false, error: "could not validate the MistWarp project history", code: "invalid_workspace"}) + ) + if parsedComparison.unwrap().assert(object).changed != true ( + return unchangedVersionRefusal() + ) + return noRefusal() +) + +def createMergeCommit(object project, UploadRequest request, UploadHistory history, UploadArchive archive, string uploader) Refusal ( + if !needsMergeCommit(request, history) ( + return noRefusal() + ) + boolean branchMerge = isBranchMerge(request) + object mergeSource = project + string mergeSourceHead = request.mergeSourceHead + string mergeTargetHead = request.mergeTargetHead + if branchMerge ( + Refusal branchChanged = branchMergeRefusal(request, projectBranchHead(project, request.mergeSourceBranch), projectBranchHead(project, request.mergeTargetBranch)) + if isRefused(branchChanged) ( + return branchChanged + ) + history.baseHead = mergeTargetHead + ) else ( + object mergePull = pullDocument(project.id.toStr(), request.pullId) + Refusal pullClosed = pullMergeRefusal(request, mergePull, project.id.toStr()) + if isRefused(pullClosed) ( + return pullClosed + ) + auto mergeSourceResult = loadProject(mergePull.sourceProjectId.toStr()) + if !mergeSourceResult.ok ( + return refuse(404, "merge source project not found") + ) + mergeSource = mergeSourceResult.project.assert(object) + mergeSourceHead = mergePull.headCommit.toStr() + mergeTargetHead = history.currentHead + ) + if !isSafeHistoryArchive(history.mergeTree, archive, false) ( + return refuse(400, "invalid merged project tree") + ) + string mergeTargetPath = tmpDir ++ archive.workId ++ ".merge-target.mwp" + string mergeSourcePath = tmpDir ++ archive.workId ++ ".merge-source.mwp" + string mergeTreePath = tmpDir ++ archive.workId ++ ".merge-tree.zip" + string mergeDeltaPath = tmpDir ++ archive.workId ++ ".merge-delta.mwp" + boolean mergeReady = copyProjectWorkspaceToPath(project, mergeTargetPath) + mergeReady = mergeReady and copyProjectWorkspaceToPath(mergeSource, mergeSourcePath) + mergeReady = mergeReady and fs.writeFileBytes(mergeTreePath, history.mergeTree) + HistoryIdentity identity = historyIdentity(project) + string mergeBranch = if branchMerge then request.mergeTargetBranch else identity.branch + number mergeTimestamp = timestamp + string mergeJson = "" + if mergeReady ( + mergeJson = gitinspection.CreateMergeArchiveJSON(mergeTargetPath, mergeTargetHead, mergeSourcePath, mergeSourceHead, mergeTreePath, mergeDeltaPath, identity.projectId, identity.remixParent, identity.baseCommit, mergeBranch, uploader, mergeCommitMessage(request, branchMerge), mergeTimestamp / 1000) + ) + object merged = parsedGeneratedHistory(mergeJson) + if !mergeReady or merged.len == 0 ( + void fs.remove(mergeTargetPath) + void fs.remove(mergeSourcePath) + void fs.remove(mergeTreePath) + void fs.remove(mergeDeltaPath) + return refuseWithBody(409, {ok: false, error: "could not create the merge commit", code: "merge_unavailable"}) + ) + adoptGeneratedHistory(history, merged, fs.readFileBytes(mergeDeltaPath).unwrapOr("".toBytes())) + void fs.remove(mergeTargetPath) + void fs.remove(mergeSourcePath) + void fs.remove(mergeTreePath) + void fs.remove(mergeDeltaPath) + return noRefusal() +) + +def createRestoreCommit(object project, UploadRequest request, UploadHistory history, UploadArchive archive, string uploader) Refusal ( + if !needsRestoreCommit(request, history) ( + return noRefusal() + ) + string restoreBasePath = tmpDir ++ archive.workId ++ ".restore-base.mwp" + string restoreDeltaPath = tmpDir ++ archive.workId ++ ".restore-delta.mwp" + HistoryIdentity identity = historyIdentity(project) + number restoreTimestamp = timestamp + boolean restoreReady = copyProjectWorkspaceToPath(project, restoreBasePath) + string restoreJson = "" + if restoreReady ( + restoreJson = gitinspection.RestoreArchiveJSON(restoreBasePath, history.currentHead, request.restoreCommit, restoreDeltaPath, identity.projectId, identity.remixParent, identity.baseCommit, identity.branch, uploader, request.restoreMessage, restoreTimestamp / 1000) + ) + object restored = parsedGeneratedHistory(restoreJson) + if !restoreReady or restored.len == 0 ( + void fs.remove(restoreBasePath) + void fs.remove(restoreDeltaPath) + return refuseWithBody(409, {ok: false, error: "could not restore that project version", code: "restore_unavailable"}) + ) + adoptGeneratedHistory(history, restored, fs.readFileBytes(restoreDeltaPath).unwrapOr("".toBytes())) + void fs.remove(restoreBasePath) + void fs.remove(restoreDeltaPath) + return noRefusal() +) + +def checkGeneratedHistory(object project, UploadHistory history) Refusal ( + if !history.generated ( + return noRefusal() + ) + object checked = validateUploadedHistory(project, history.generatedWorkspace, history.metadata, history.baseHead) + if !checked.ok ( + return refuseWithBody(409, {ok: false, code: "invalid_workspace", error: checked.error}) + ) + history.generatedWorkspace = checked.data.assert(byte[]) + history.info = checked.manifest.assert(object) + history.metadata = json.stringify(history.info) + return noRefusal() +) + +def storeUploadedHistory(object project, UploadRequest request, UploadHistory history, UploadArchive archive) Refusal ( + if !history.hasWorkspace and !history.generated ( + return noRefusal() + ) + byte[] incomingWorkspaceBody = history.generatedWorkspace + number incomingWorkspaceSize = history.generatedWorkspace.len + if history.hasWorkspace ( + incomingWorkspaceBody = history.workspace + incomingWorkspaceSize = history.workspaceSize + ) + if incomingWorkspaceSize > maxWorkspaceBytes ( + return refuseWithBody(413, {ok: false, error: "one history update exceeds the 512 MB safety limit", code: "workspace_update_too_large"}) + ) + if !isSafeHistoryArchive(incomingWorkspaceBody, archive, false) ( + return unsafeProjectHistoryRefusal() + ) + string workspaceKey = projectWorkspaceSnapshotKey(contentHash(incomingWorkspaceBody)) + if !stageR2Blob(workspaceKey, incomingWorkspaceBody, "application/x-mistwarp-project", "") ( + return refuse(500, "failed to store MistWarp project history") + ) + project.workspaceKey = workspaceKey + project.workspaceLayers = [workspaceKey] + project.historyBytes = incomingWorkspaceSize + project.gitHead = history.info.head + project.gitBranch = history.info.branch + project.gitCommits = history.info.commits + project.gitGraph = history.info.graph + if request.replaceHistory ( + project.remixBaseCommit = "" + ) + return noRefusal() +) + +def materializeWorktreeAssets(object project, byte[] body, string extractDir, boolean replaceHistory) object ( + string workId = tmpDir ++ project.id.toStr() ++ "." ++ randomString(8) + string incomingPath = workId ++ ".refs.mwp" + string outputPath = workId ++ ".materialized.mwp" + string fetchDir = workId ++ ".assets" + if !fs.writeFileBytes(incomingPath, body) ( + return {ok: false} + ) + string basePath = "" + if !replaceHistory and projectHasWorkspace(project) ( + basePath = cachedProjectWorkspacePath(project) + ) + string assetDirs = extractDir ++ "\n" ++ localBlobsDir ++ "assets" + auto parsed = json.parse(gitinspection.MaterializeWorktreeAssetsJSON(incomingPath, outputPath, basePath, assetDirs), {type: "object"}) + object res = if parsed.isErr() then {ok: false} else parsed.unwrap().assert(object) + array missing = (res.missing ?? []).assertElse(array, []) + if !res.ok and missing.len > 0 and missing.len <= maxAssetCount and !r2Local ( + void fs.mkdir(fetchDir, {recursive: true}) + for i missing.len ( + string name = missing[i].toStr() + if isValidMd5Ext(name) ( + void copyStorageKeyToPath(assetObjectKey(name), fetchDir ++ "/" ++ name) + ) + ) + parsed = json.parse(gitinspection.MaterializeWorktreeAssetsJSON(incomingPath, outputPath, basePath, assetDirs ++ "\n" ++ fetchDir), {type: "object"}) + res = if parsed.isErr() then {ok: false} else parsed.unwrap().assert(object) + ) + void fs.remove(incomingPath) + void fs.remove(fetchDir) + if res.ok != true ( + void fs.remove(outputPath) + return {ok: false} + ) + number refs = (res.refs ?? 0).toNum() + if refs == 0 ( + return {ok: true, refs: 0} + ) + byte[] data = fs.readFileBytes(outputPath).unwrapOr("".toBytes()) + void fs.remove(outputPath) + return {ok: data.len > 0, refs: refs, data: data, size: data.len} +) + +def validateUploadedHistory(object project, byte[] body, string metadata, string currentHead) object ( + return validateUploadedHistoryMode(project, body, metadata, currentHead, false) +) + +// Only the explicit maintainer upload path may discard the stored graph. +// With no base archive, validation also rejects deltas and incomplete history. +def validateUploadedHistoryMode(object project, byte[] body, string metadata, string currentHead, boolean replaceHistory) object ( + string workId = tmpDir ++ project.id.toStr() ++ "." ++ randomString(8) + string incomingPath = workId ++ ".incoming.mwp" + string outputPath = workId ++ ".validated.mwp" + string basePath = "" + if !replaceHistory and projectHasWorkspace(project) ( + basePath = cachedProjectWorkspacePath(project) + if basePath == "" ( + return {ok: false, error: "stored history unavailable; retry later"} + ) + ) + if !fs.writeFileBytes(incomingPath, body) ( + return {ok: false, error: "could not stage history"} + ) + string forkBase = (project.remixBaseCommit ?? "").toStr() + if replaceHistory ( + currentHead = "" + forkBase = "" + ) + string projectId = project.id.toStr() + string remixParent = (project.remixParent ?? "").toStr() + string resultJson = gitinspection.ValidateHistoryArchiveJSON(basePath, incomingPath, outputPath, currentHead, forkBase, metadata, projectId, remixParent) + void fs.remove(incomingPath) + auto parsed = json.parse(resultJson, {type: "object"}) + if parsed.isErr() ( + void fs.remove(outputPath) + return {ok: false, error: "could not validate history"} + ) + object res = parsed.unwrap().assert(object) + if res.ok ( + byte[] data = fs.readFileBytes(outputPath).unwrapOr("".toBytes()) + res.data = data + res.size = data.len + ) + void fs.remove(outputPath) + return res +) diff --git a/src/projects/project-upload-policy.osl b/src/projects/project-upload-policy.osl new file mode 100644 index 0000000..0333bcb --- /dev/null +++ b/src/projects/project-upload-policy.osl @@ -0,0 +1,229 @@ +export {} + +type UploadRequest ( + boolean replaceHistory = false + string expectedHead = "" + string expectedEdited = "" + string restoreCommit = "" + string restoreMessage = "" + string pullId = "" + string mergeSourceBranch = "" + string mergeTargetBranch = "" + string mergeSourceHead = "" + string mergeTargetHead = "" + string git = "" + string extensions = "" + string workspaceToken = "" + string mergeTreeToken = "" +) + +type UploadHistory ( + string currentHead = "" + string baseHead = "" + object info = {} + string metadata = "" + boolean hasWorkspace = false + byte[] workspace = [] + number workspaceSize = 0 + boolean hasMergeTree = false + byte[] mergeTree = [] + boolean generated = false + byte[] generatedWorkspace = [] +) + +type HistoryIdentity ( + string projectId = "" + string remixParent = "" + string baseCommit = "" + string branch = "main" +) + +def newUploadHistory(UploadRequest request, string currentHead) UploadHistory ( + return UploadHistory{currentHead: currentHead, baseHead: currentHead, metadata: request.git} +) + +def liveContributorRefusal(UploadRequest request, boolean canEdit) Refusal ( + if canEdit ( + return noRefusal() + ) + if request.expectedEdited == "" or request.restoreCommit != "" or request.pullId != "" or request.mergeSourceBranch != "" or request.replaceHistory ( + return refuse(403, "live contributors can save the current project; history and publishing require a maintainer") + ) + return noRefusal() +) + +def staleUploadRefusal(UploadRequest request, object project, string currentHead) Refusal ( + if request.expectedEdited != "" and request.expectedEdited.toNum() != project.edited.toNum() ( + return refuseWithBody(409, {ok: false, error: "someone saved newer changes; refresh project history before saving again", code: "head_changed"}) + ) + if request.expectedHead != "" and request.expectedHead != currentHead ( + return refuseWithBody(409, {ok: false, error: "the project changed while you were merging", code: "head_changed", currentHead: currentHead}) + ) + if request.replaceHistory and (request.expectedHead != currentHead or request.restoreCommit != "" or request.pullId != "" or request.mergeSourceBranch != "") ( + return refuseWithBody(409, {ok: false, error: "history replacement requires the current saved head and a full workspace", code: "head_changed"}) + ) + return noRefusal() +) + +def readUploadGitMetadata(UploadHistory history, boolean storedWorkspace) Refusal ( + if history.metadata == "" ( + return noRefusal() + ) + auto parsedGit = json.parse(history.metadata, {type: "object"}) + if parsedGit.isErr() ( + return refuse(400, "invalid MistWarp history metadata") + ) + history.info = parsedGit.unwrap().assert(object) + if history.info.contains("delta") and history.info.delta == true ( + string baseHead = (history.info.baseHead ?? "").toStr() + if baseHead == "" or baseHead != history.currentHead or !storedWorkspace ( + return refuseWithBody(409, {ok: false, error: "the saved project history changed", code: "workspace_base_changed", currentHead: history.currentHead}) + ) + ) + return noRefusal() +) + +def restoreCommitRefusal(UploadRequest request) Refusal ( + if request.restoreCommit != "" and !validGitObjectId(request.restoreCommit) ( + return refuse(400, "invalid restore commit") + ) + return noRefusal() +) + +def replaceWithoutWorkspaceRefusal(UploadRequest request, UploadHistory history) Refusal ( + if request.replaceHistory and !history.hasWorkspace ( + return refuse(400, "history replacement requires a full workspace") + ) + return noRefusal() +) + +def isOrdinaryVersionSave(UploadRequest request, UploadHistory history) boolean ( + return !request.replaceHistory and (history.info.worktree ?? false) != true and history.hasWorkspace and request.restoreCommit == "" and !history.hasMergeTree and history.metadata != "" and history.currentHead != "" +) + +def unchangedVersionRefusal() Refusal ( + return refuseWithBody(409, {ok: false, error: "No files changed in this commit.", code: "no_changes"}) +) + +def extensionRefusal(object extensionResult) Refusal ( + if extensionResult.ok ( + return noRefusal() + ) + object extensionError = {ok: false, error: extensionResult.error} + if extensionResult.contains("code") ( + extensionError.code = extensionResult.code + ) + return refuseWithBody(400, extensionError) +) + +def needsMergeCommit(UploadRequest request, UploadHistory history) boolean ( + return history.hasMergeTree and !history.hasWorkspace and request.restoreCommit == "" +) + +def needsRestoreCommit(UploadRequest request, UploadHistory history) boolean ( + return request.restoreCommit != "" and !history.hasWorkspace +) + +def isBranchMerge(UploadRequest request) boolean ( + return request.pullId == "" and request.mergeSourceBranch != "" and request.mergeTargetBranch != "" and request.mergeSourceHead != "" and request.mergeTargetHead != "" +) + +def branchMergeRefusal(UploadRequest request, string latestSourceHead, string latestTargetHead) Refusal ( + if request.mergeSourceBranch == request.mergeTargetBranch or latestSourceHead == "" or latestTargetHead == "" or latestSourceHead != request.mergeSourceHead or latestTargetHead != request.mergeTargetHead ( + return refuseWithBody(409, {ok: false, error: "a branch changed while this merge was processing", code: "branch_head_changed"}) + ) + return noRefusal() +) + +def pullMergeRefusal(UploadRequest request, object pull, string projectId) Refusal ( + if request.pullId == "" or pull.len == 0 or pull.state.toStr() != "open" or pull.targetProjectId.toStr() != projectId ( + return refuseWithBody(409, {ok: false, error: "the pull request is no longer open", code: "pull_not_open"}) + ) + return noRefusal() +) + +def mergeCommitMessage(UploadRequest request, boolean branchMerge) string ( + return if branchMerge then "Merge branch '" ++ request.mergeSourceBranch ++ "' into '" ++ request.mergeTargetBranch ++ "'" else "Merge pull request #" ++ request.pullId +) + +def historyIdentity(object project) HistoryIdentity ( + return HistoryIdentity{ + projectId: project.id.toStr(), + remixParent: if project.contains("remixParent") and project.remixParent != null then project.remixParent.toStr() else "", + baseCommit: if project.contains("remixBaseCommit") and project.remixBaseCommit != null then project.remixBaseCommit.toStr() else "", + branch: if project.contains("gitBranch") and project.gitBranch != null then project.gitBranch.toStr() else "main" + } +) + +def parsedGeneratedHistory(string resultJson) object ( + auto parsed = json.parse(resultJson, {type: "object"}) + if parsed.isErr() or parsed.unwrap().assert(object).ok != true ( + return {} + ) + return parsed.unwrap().assert(object) +) + +def adoptGeneratedHistory(UploadHistory history, object generated, byte[] generatedWorkspace) ( + history.info = generated.manifest.assert(object) + history.metadata = json.stringify(history.info) + history.generatedWorkspace = generatedWorkspace + history.generated = true +) + +def storedWorkspaceLayers(object project) array ( + if project.contains("workspaceLayers") and project.workspaceLayers != null ( + return project.workspaceLayers.assertElse(array, []) + ) + return [] +) + +def keysNotIn(array keys, array kept) array ( + array missing = [] + for i keys.len ( + string key = keys[i].toStr() + if !kept.contains(key) ( + missing.append(key) + ) + ) + return missing +) + +def replacedWorkspaceKeys(object project, array previousWorkspaceKeys) array ( + if project.contains("workspaceLayers") and project.workspaceLayers != null ( + return keysNotIn(previousWorkspaceKeys, project.workspaceLayers.assertElse(array, [])) + ) + return [] +) + +def projectFilesErrorBody(object uploadResult) object ( + object errorBody = {ok: false, error: uploadResult.error} + if uploadResult.contains("code") ( + errorBody.code = uploadResult.code + ) + return errorBody +) + +def megabytes(number bytes) number ( + return (bytes / 1048576).toInt() +) + +def assetSizeFailure(string name, number size, number limit) object ( + if size > limit ( + return {ok: false, error: "an asset exceeds your " ++ megabytes(limit).toStr() ++ " MB limit: " ++ name, code: "project_too_large", status: 413} + ) + return {} +) + +def projectAssetsFailure(number assetBytes, number limit) object ( + if assetBytes > limit ( + return {ok: false, error: "project assets exceed your " ++ megabytes(limit).toStr() ++ " MB limit", code: "project_too_large", status: 413} + ) + return {} +) + +def storageAllowanceFailure(number projectBytes, number allowanceBytes) object ( + if allowanceBytes >= 0 and projectBytes > allowanceBytes ( + return {ok: false, error: "storage full", code: "storage_full", status: 413, overBytes: projectBytes - allowanceBytes} + ) + return {} +) diff --git a/src/projects/project-upload-policy.test.osl b/src/projects/project-upload-policy.test.osl new file mode 100644 index 0000000..7669d9c --- /dev/null +++ b/src/projects/project-upload-policy.test.osl @@ -0,0 +1,157 @@ +import "std:testing" +import "std:json" +import "std:regex" +import "./project-refusal.osl" +import "./project-upload-policy.osl" + +def validGitObjectId(string oid) boolean ( + return regex.match("^[a-f0-9]{40}$", oid) +) + +string headA = "a" * 40 +string headB = "b" * 40 + +def request() UploadRequest ( + return UploadRequest() +) + +UploadRequest plain = request() +testing.assert(!isRefused(liveContributorRefusal(plain, true)), "maintainers may save without an edited stamp") +testing.assert(liveContributorRefusal(plain, false).status == 403, "live contributors must send the edited stamp they saw") +UploadRequest contributor = request() +contributor.expectedEdited = "5" +testing.assert(!isRefused(liveContributorRefusal(contributor, false)), "live contributors may save the current project") +contributor.restoreCommit = headA +testing.assert(liveContributorRefusal(contributor, false).message == "live contributors can save the current project; history and publishing require a maintainer", "live contributors cannot restore") +UploadRequest replacing = request() +replacing.expectedEdited = "5" +replacing.replaceHistory = true +testing.assert(isRefused(liveContributorRefusal(replacing, false)), "live contributors cannot replace history") + +object project = {id: "p1", edited: 5} +UploadRequest stale = request() +stale.expectedEdited = "4" +Refusal staleEdit = staleUploadRefusal(stale, project, headA) +testing.assert(staleEdit.status == 409 and staleEdit.body.code == "head_changed" and !staleEdit.body.contains("currentHead"), "an older edited stamp is a conflict") +stale.expectedEdited = "5" +testing.assert(!isRefused(staleUploadRefusal(stale, project, headA)), "the current edited stamp is accepted") +stale.expectedHead = headB +Refusal staleHead = staleUploadRefusal(stale, project, headA) +testing.assert(staleHead.status == 409 and staleHead.body.currentHead == headA and staleHead.body.error == "the project changed while you were merging", "a moved head reports the current head") +UploadRequest replace = request() +replace.replaceHistory = true +Refusal replaceWithoutHead = staleUploadRefusal(replace, project, headA) +testing.assert(replaceWithoutHead.status == 409 and replaceWithoutHead.body.error == "history replacement requires the current saved head and a full workspace", "replacing history needs the saved head") +replace.expectedHead = headA +testing.assert(!isRefused(staleUploadRefusal(replace, project, headA)), "replacing from the saved head is accepted") +replace.pullId = "3" +testing.assert(isRefused(staleUploadRefusal(replace, project, headA)), "replacing history cannot merge a pull request") + +UploadHistory noMetadata = newUploadHistory(request(), headA) +testing.assert(!isRefused(readUploadGitMetadata(noMetadata, true)) and noMetadata.info.len == 0, "uploads without metadata keep an empty manifest") +testing.assert(noMetadata.baseHead == headA and noMetadata.currentHead == headA, "history starts from the saved head") +UploadRequest withGit = request() +withGit.git = "not json" +testing.assert(readUploadGitMetadata(newUploadHistory(withGit, headA), true).message == "invalid MistWarp history metadata", "unreadable metadata is a bad request") +withGit.git = json.stringify({delta: true, baseHead: headB}) +Refusal staleDelta = readUploadGitMetadata(newUploadHistory(withGit, headA), true) +testing.assert(staleDelta.status == 409 and staleDelta.body.code == "workspace_base_changed" and staleDelta.body.currentHead == headA, "a delta from another base is a conflict") +withGit.git = json.stringify({delta: true, baseHead: headA}) +testing.assert(isRefused(readUploadGitMetadata(newUploadHistory(withGit, headA), false)), "a delta needs stored history") +UploadHistory delta = newUploadHistory(withGit, headA) +testing.assert(!isRefused(readUploadGitMetadata(delta, true)) and delta.info.delta == true, "a delta from the saved head is read") + +UploadRequest restore = request() +restore.restoreCommit = "xyz" +testing.assert(restoreCommitRefusal(restore).message == "invalid restore commit", "restore targets must be commit ids") +restore.restoreCommit = headB +testing.assert(!isRefused(restoreCommitRefusal(restore)), "a commit id can be restored") + +UploadHistory bare = newUploadHistory(replace, headA) +testing.assert(replaceWithoutWorkspaceRefusal(replace, bare).message == "history replacement requires a full workspace", "replacing history needs a workspace") +bare.hasWorkspace = true +testing.assert(!isRefused(replaceWithoutWorkspaceRefusal(replace, bare)), "a workspace can replace history") + +UploadRequest save = request() +save.git = "{}" +UploadHistory version = newUploadHistory(save, headA) +version.hasWorkspace = true +testing.assert(isOrdinaryVersionSave(save, version), "a workspace with metadata on a saved head is a version save") +version.info = {worktree: true} +testing.assert(!isOrdinaryVersionSave(save, version), "worktree saves skip the unchanged check") +version.info = {} +version.hasMergeTree = true +testing.assert(!isOrdinaryVersionSave(save, version), "merges skip the unchanged check") +version.hasMergeTree = false +testing.assert(!isOrdinaryVersionSave(save, newUploadHistory(save, "")), "first saves skip the unchanged check") +testing.assert(unchangedVersionRefusal().body.code == "no_changes", "unchanged saves report no_changes") + +testing.assert(!isRefused(extensionRefusal({ok: true})), "collected extensions pass") +Refusal blocked = extensionRefusal({ok: false, error: "blocked", code: "blocked_extension"}) +testing.assert(blocked.status == 400 and blocked.body.code == "blocked_extension" and blocked.body.ok == false, "extension errors keep their code") +testing.assert(!extensionRefusal({ok: false, error: "bad"}).body.contains("code"), "extension errors without a code stay without one") + +UploadHistory merging = newUploadHistory(request(), headA) +merging.hasMergeTree = true +testing.assert(needsMergeCommit(request(), merging), "a merge tree without a workspace makes a merge commit") +UploadRequest restoring = request() +restoring.restoreCommit = headB +testing.assert(!needsMergeCommit(restoring, merging) and needsRestoreCommit(restoring, merging), "a restore wins over a merge tree") +merging.hasWorkspace = true +testing.assert(!needsMergeCommit(request(), merging) and !needsRestoreCommit(restoring, merging), "a full workspace needs no generated commit") + +UploadRequest branch = request() +branch.mergeSourceBranch = "feature" +branch.mergeTargetBranch = "main" +branch.mergeSourceHead = headB +branch.mergeTargetHead = headA +testing.assert(isBranchMerge(branch), "two branches and their heads make a branch merge") +testing.assert(!isRefused(branchMergeRefusal(branch, headB, headA)), "unchanged branch heads can merge") +testing.assert(branchMergeRefusal(branch, headA, headA).body.code == "branch_head_changed", "a moved source head stops the merge") +testing.assert(isRefused(branchMergeRefusal(branch, headB, "")), "a missing target branch stops the merge") +testing.assert(mergeCommitMessage(branch, true) == "Merge branch 'feature' into 'main'", "branch merges name both branches") +UploadRequest sameBranch = request() +sameBranch.mergeSourceBranch = "main" +sameBranch.mergeTargetBranch = "main" +sameBranch.mergeSourceHead = headA +sameBranch.mergeTargetHead = headA +testing.assert(isRefused(branchMergeRefusal(sameBranch, headA, headA)), "a branch cannot merge into itself") +branch.pullId = "7" +testing.assert(!isBranchMerge(branch) and mergeCommitMessage(branch, false) == "Merge pull request #7", "a pull id makes a pull request merge") +object openPull = {state: "open", targetProjectId: "p1"} +testing.assert(!isRefused(pullMergeRefusal(branch, openPull, "p1")), "an open pull into this project can merge") +testing.assert(pullMergeRefusal(branch, {state: "closed", targetProjectId: "p1"}, "p1").body.code == "pull_not_open", "a closed pull cannot merge") +testing.assert(isRefused(pullMergeRefusal(branch, openPull, "p2")), "a pull into another project cannot merge") +testing.assert(isRefused(pullMergeRefusal(branch, {}, "p1")), "a missing pull cannot merge") +testing.assert(isRefused(pullMergeRefusal(request(), openPull, "p1")), "a merge without a pull id or branches is refused") + +HistoryIdentity fresh = historyIdentity({id: "p1", remixParent: null}) +testing.assert(fresh.projectId == "p1" and fresh.remixParent == "" and fresh.baseCommit == "" and fresh.branch == "main", "missing identity fields fall back") +HistoryIdentity forked = historyIdentity({id: "p2", remixParent: "p1", remixBaseCommit: headA, gitBranch: "dev"}) +testing.assert(forked.remixParent == "p1" and forked.baseCommit == headA and forked.branch == "dev", "fork identity is read from the project") + +testing.assert(parsedGeneratedHistory("").len == 0 and parsedGeneratedHistory("{\"ok\":false}").len == 0, "failed generation yields nothing") +object generated = parsedGeneratedHistory(json.stringify({ok: true, manifest: {head: headB, branch: "main"}})) +UploadHistory adopted = newUploadHistory(request(), headA) +adoptGeneratedHistory(adopted, generated, "delta".toBytes()) +testing.assert(adopted.generated and adopted.info.head == headB and adopted.generatedWorkspace.len == 5, "generated history replaces the manifest and body") +testing.assert(json.parse(adopted.metadata, {type: "object"}).unwrap().head == headB, "generated metadata matches the manifest") + +testing.assert(storedWorkspaceLayers({}).len == 0 and storedWorkspaceLayers({workspaceLayers: null}).len == 0, "projects without layers have none") +testing.assert(keysNotIn(["a", "b", "c"], ["b"]) == ["a", "c"], "keysNotIn keeps order") +testing.assert(replacedWorkspaceKeys({workspaceLayers: ["n"]}, ["o", "n"]) == ["o"], "replaced layers are the old ones not kept") +testing.assert(replacedWorkspaceKeys({}, ["o"]).len == 0, "nothing is replaced without stored layers") + +object filesError = projectFilesErrorBody({ok: false, error: "too many assets", status: 400}) +testing.assert(filesError.ok == false and filesError.error == "too many assets" and !filesError.contains("status") and !filesError.contains("code"), "file errors expose only ok, error and code") +testing.assert(projectFilesErrorBody({ok: false, error: "big", code: "project_too_large"}).code == "project_too_large", "file errors keep their code") + +testing.assert(megabytes(10485760) == 10 and megabytes(1572864) == 1, "limits are whole megabytes") +testing.assert(assetSizeFailure("a.svg", 10, 10).len == 0, "an asset at the limit is accepted") +object bigAsset = assetSizeFailure("a.svg", 10485761, 10485760) +testing.assert(bigAsset.status == 413 and bigAsset.error == "an asset exceeds your 10 MB limit: a.svg", "an oversized asset names the limit and file") +testing.assert(projectAssetsFailure(5, 5).len == 0 and projectAssetsFailure(6, 5).code == "project_too_large", "project assets are capped") +testing.assert(storageAllowanceFailure(100, -1).len == 0, "a negative allowance means storage is counted elsewhere") +testing.assert(storageAllowanceFailure(100, 100).len == 0, "a project that exactly fits is accepted") +object full = storageAllowanceFailure(150, 100) +testing.assert(full.code == "storage_full" and full.overBytes == 50 and full.status == 413, "a full allowance reports the overflow") diff --git a/src/projects/project-upload-publish.osl b/src/projects/project-upload-publish.osl new file mode 100644 index 0000000..e8cbaf6 --- /dev/null +++ b/src/projects/project-upload-publish.osl @@ -0,0 +1,142 @@ +export {} + +type UploadRollback ( + array previousWorkspaceKeys = [] + string ownerId = "" + object hold = {} + string previousJsonKey = "" +) + +def discardNewWorkspaceKeys(string projectId, array keys, array previousWorkspaceKeys) ( + array added = keysNotIn(keys, previousWorkspaceKeys) + for i added.len ( + removeProjectStorageKeyIfOrphaned(added[i].toStr(), projectId, true) + ) +) + +def rejectProjectFiles(*serve.Context c, object project, object uploadResult, UploadRollback rollback, UploadArchive archive) ( + discardNewWorkspaceKeys(project.id.toStr(), projectWorkspaceKeys(project), rollback.previousWorkspaceKeys) + archive.discard() + if (uploadResult.code ?? "").toStr() == "storage_full" ( + if uploadResult.contains("refusal") ( + roturStorageFullResponse(c, uploadResult.refusal.assertElse(object, {})) + return + ) + storageFullResponse(c, storageDetails(project.owner.toStr()), uploadResult.overBytes.toNum()) + return + ) + c.json(uploadResult.status.toInt(), projectFilesErrorBody(uploadResult)) +) + +def storeUploadExtensions(object project, object extensionResult, UploadRollback rollback) Refusal ( + if !storeProjectExtensions(project.id.toStr(), storedProjectExtensions(project), extensionResult.extensions.assert(array), extensionResult.sources.assert(object)) ( + storageHoldRelease(rollback.ownerId, rollback.hold) + return refuse(500, "failed to store extension sources") + ) + return noRefusal() +) + +def saveUploadThumbnail(*serve.Context c, string projectId) ( + auto thumbResult = c.formFile("thumbnail") + if !thumbResult.isErr() ( + object thumb = thumbResult.unwrap() + if thumb.size.toNum() <= maxThumbBytes ( + void fs.writeFileBytes(thumbnailPath(projectId), thumb.data) + ) + ) +) + +def applyUploadedFiles(object project, object uploadResult, object extensionResult, UploadRollback rollback) ( + project.assets = uploadResult.assets + project.edited = timestamp + rollback.previousJsonKey = project.jsonKey.toStr() + if rollback.previousJsonKey.startsWith("project-json/") or (project.contains("inheritedJson") and project.inheritedJson == true) ( + project.jsonKey = projectJsonKey(project.id.toStr()) + ) + project.inheritedJson = false + project.sizeBytes = uploadResult.assetBytes.toNum() + uploadResult.storedJsonBytes.toNum() + project.jsonBytes = uploadResult.jsonBytes + project.storedJsonBytes = uploadResult.storedJsonBytes + project.assetBytes = uploadResult.assetBytes + project.jsonGzip = true + project.extensions = extensionResult.extensions.assert(array) + project.extensionsIndexed = true + project.delete("pendingExtensionUrls") +) + +def publishProjectJson(object project, object uploadResult) boolean ( + byte[] jsonBody = fs.readFileBytes(uploadResult.gzipPath.toStr()).unwrapOr("".toBytes()) + project.jsonKey = projectJsonSnapshotKey(contentHash(jsonBody)) + boolean storedJson = stageR2Blob(project.jsonKey.toStr(), jsonBody, "application/json", "gzip") + project.lastUploadAt = timestamp + project.pendingJson = false + return storedJson +) + +def abandonUnpublishedUpload(*serve.Context c, object project, UploadRollback rollback, UploadArchive archive) ( + discardNewWorkspaceKeys(project.id.toStr(), storedWorkspaceLayers(project), rollback.previousWorkspaceKeys) + storageHoldRelease(rollback.ownerId, rollback.hold) + c.json(500, {ok: false, error: "failed to store project data, please try saving again"}) + archive.discard() +) + +def abandonUnsavedUpload(*serve.Context c, object project, UploadRollback rollback, UploadArchive archive) ( + if project.jsonKey.toStr() != rollback.previousJsonKey ( + removeProjectStorageKeyIfOrphaned(project.jsonKey.toStr(), project.id.toStr(), false) + ) + discardNewWorkspaceKeys(project.id.toStr(), storedWorkspaceLayers(project), rollback.previousWorkspaceKeys) + storageHoldRelease(rollback.ownerId, rollback.hold) + archive.discard() + c.internalError("failed to save project metadata") +) + +def settleSavedUpload(object project, UploadRollback rollback, string uploader) ( + upsertIndexEntry(project) + thread.new(warmShareCard, project.id.toStr()) + if usesRoturStorage(rollback.ownerId) ( + thread.new(storageSettle, rollback.ownerId, rollback.hold) + ) else ( + releasePendingUploads(uploader, project.assets.assertElse(array, [])) + ) + string replacedJsonKey = if rollback.previousJsonKey != project.jsonKey.toStr() then rollback.previousJsonKey else "" + array replacedKeys = replacedWorkspaceKeys(project, rollback.previousWorkspaceKeys) + if replacedJsonKey != "" or replacedKeys.len > 0 ( + thread.new(cleanupReplacedProjectStorage, project.id.toStr(), replacedJsonKey, replacedKeys) + ) +) + +def projectWithUploadHistory(object project) object ( + if projectHasWorkspace(project) ( + return project + ) + object generatedHistory = ensureUploadedProjectHistory(project) + if !generatedHistory.ok ( + log "WARNING: project " ++ project.id.toStr() ++ " uploaded without history and fallback generation failed: " ++ generatedHistory.error.toStr() + return project + ) + return generatedHistory.project.assert(object) +) + +def completeUploadPull(object project, UploadRequest request, string username) Refusal ( + if request.pullId != "" and !completeNativePull(project, request.pullId, request.expectedHead, username) ( + return refuseWithBody(409, {ok: false, error: "the project saved, but the pull request could not be closed", code: "pull_not_closed"}) + ) + return noRefusal() +) + +def uploadResponse(object project, object uploadResult) object ( + return { + ok: true, + project: { + id: project.id, + gitHead: projectGitHead(project), + gitBranch: project.gitBranch, + edited: project.edited, + remixBaseCommit: project.remixBaseCommit + }, + uploaded: uploadResult.uploaded, + skipped: uploadResult.skipped, + projectJsonUrl: projectJsonUrlFor(project), + thumbUrl: viewerThumbUrlFor(project) + } +) diff --git a/src/projects/project-upload-receive.osl b/src/projects/project-upload-receive.osl new file mode 100644 index 0000000..30329bf --- /dev/null +++ b/src/projects/project-upload-receive.osl @@ -0,0 +1,111 @@ +export {} + +type UploadArchive ( + byte[] data = [] + string workId = "" + string zipPath = "" + string extractDir = "" + number expandedLimit = 0 + + def discard() ( + void fs.remove(self.zipPath) + void fs.remove(self.extractDir) + ) + + def discardOn(Refusal refusal) Refusal ( + if isRefused(refusal) ( + self.discard() + ) + return refusal + ) +) + +def uploadFormValue(*serve.Context c, string name) string ( + return (c.formValue(name) ?? "").strip() +) + +def readUploadRequest(*serve.Context c) UploadRequest ( + UploadRequest request = UploadRequest() + request.replaceHistory = (c.formValue("replaceHistory") ?? "") == "true" + request.expectedHead = uploadFormValue(c, "expectedHead") + request.expectedEdited = uploadFormValue(c, "expectedEdited") + request.restoreCommit = uploadFormValue(c, "restoreCommit") + request.restoreMessage = uploadFormValue(c, "restoreMessage") + request.pullId = uploadFormValue(c, "pullId") + request.mergeSourceBranch = uploadFormValue(c, "mergeSourceBranch") + request.mergeTargetBranch = uploadFormValue(c, "mergeTargetBranch") + request.mergeSourceHead = uploadFormValue(c, "mergeSourceHead") + request.mergeTargetHead = uploadFormValue(c, "mergeTargetHead") + request.git = (c.formValue("git") ?? "") + request.extensions = (c.formValue("extensions") ?? "") + request.workspaceToken = (c.formValue("workspaceToken") ?? "") + request.mergeTreeToken = (c.formValue("mergeTreeToken") ?? "") + return request +) + +def editorReplaceRefusal(*serve.Context c, UploadRequest request, object project) Refusal ( + if request.replaceHistory and isEditorSession(c) and !editorCanDeleteProject(project) ( + return editorScopeRefusal("Open MistWarp to replace this project's saved history.") + ) + return noRefusal() +) + +def receiveProjectArchive(*serve.Context c, UploadArchive archive) Refusal ( + auto fileResult = c.formFile("project") + if fileResult.isErr() ( + return refuse(400, "missing project file") + ) + archive.data = fileResult.unwrap().data + return noRefusal() +) + +def receiveHistoryParts(*serve.Context c, object project, UploadRequest request, UploadHistory history) Refusal ( + auto workspaceResult = c.formFile("workspace") + auto mergeTreeResult = c.formFile("mergeTree") + history.hasWorkspace = !workspaceResult.isErr() + history.hasMergeTree = !mergeTreeResult.isErr() + if history.hasWorkspace ( + object workspaceFile = workspaceResult.unwrap() + history.workspace = workspaceFile.data + history.workspaceSize = workspaceFile.size.toNum() + ) else if request.workspaceToken != "" ( + object stagedWorkspace = consumeStagedUploadPart(request.workspaceToken, project.id.toStr(), "workspace", contextString(c, "userId")) + if !stagedWorkspace.ok ( + return refuse(400, stagedWorkspace.error.toStr()) + ) + object workspaceFile = stagedWorkspace.file.assert(object) + history.workspace = workspaceFile.data.assertElse(byte[], []) + history.workspaceSize = workspaceFile.size.toNum() + history.hasWorkspace = true + ) + if history.hasMergeTree ( + history.mergeTree = mergeTreeResult.unwrap().data + ) else if request.mergeTreeToken != "" ( + object stagedMergeTree = consumeStagedUploadPart(request.mergeTreeToken, project.id.toStr(), "mergeTree", contextString(c, "userId")) + if !stagedMergeTree.ok ( + return refuse(400, stagedMergeTree.error.toStr()) + ) + history.mergeTree = stagedMergeTree.file.assert(object).data.assertElse(byte[], []) + history.hasMergeTree = true + ) + return noRefusal() +) + +def unpackProjectArchive(object project, object benefits, UploadArchive archive) Refusal ( + archive.workId = project.id.toStr() ++ "." ++ randomString(8) + archive.zipPath = tmpDir ++ archive.workId ++ ".sb3" + archive.extractDir = tmpDir ++ archive.workId + if !fs.writeFileBytes(archive.zipPath, archive.data) ( + return refuse(500, "failed to stage project upload") + ) + archive.expandedLimit = maxProjectJsonBytes + benefits.maxProjectAssetsBytes.toNum() + if !validateZipArchive(archive.data, maxProjectUploadRequestBytes, archive.expandedLimit, maxAssetCount + 1, false) ( + void fs.remove(archive.zipPath) + return refuse(400, "invalid or unsafe sb3 file") + ) + if !zip.decompress(archive.zipPath, archive.extractDir, {maxBytes: archive.expandedLimit, maxFiles: maxAssetCount + 1}) ( + archive.discard() + return refuse(400, "invalid sb3 file") + ) + return noRefusal() +) diff --git a/src/projects/project-upload.osl b/src/projects/project-upload.osl index 98429d8..947416c 100644 --- a/src/projects/project-upload.osl +++ b/src/projects/project-upload.osl @@ -1,197 +1,4 @@ -export { - validateUploadedHistory -} - -def scanProjectJsonEvents(*json.Stream stream) object ( - boolean started = false - boolean readingMd5ext = false - number targets = 0 - array md5ext = [] - for event in stream.events() ( - if !started ( - started = true - if event.type != "object-start" ( - return {ok: false} - ) - ) else if readingMd5ext ( - readingMd5ext = false - if event.type == "string" ( - md5ext.append(event.value) - if md5ext.len > maxAssetCount * 50 ( - return {ok: false} - ) - ) - ) else if event.type == "key" ( - string key = event.value.toStr() - if key == "targets" ( - targets += 1 - ) - readingMd5ext = key == "md5ext" - ) - ) - if !started ( - return {ok: false} - ) - return {ok: true, targets: targets, md5ext: md5ext} -) - -def inspectProjectJson(string path) object ( - *json.Stream stream = json.open(path, maxProjectJsonBytes) - object inspected = scanProjectJsonEvents(stream) catch (err) ( - stream.close() - return {ok: false} - ) - stream.close() - return inspected -) - -// storageAllowanceBytes is a student's local limit, or -1. A Rotur owner's -// storage is held in the ledger instead, unless enforceStorage is off -// (admins); the result's hold is committed or released by the caller. -def uploadProjectFiles(object project, string extractDir, object benefits, number storageAllowanceBytes, boolean enforceStorage) object ( - array names = fs.readDir(extractDir).unwrapOr([]) - string jsonPath = extractDir ++ "/project.json" - if !fs.exists(jsonPath) ( - return {ok: false, error: "sb3 has no project.json", status: 400} - ) - number projectJsonBytes = fs.size(jsonPath).unwrapOr(0) - if projectJsonBytes > maxProjectJsonBytes ( - return {ok: false, error: "project.json exceeds the 256 MB expanded limit", code: "project_too_large", status: 413} - ) - object inspected = inspectProjectJson(jsonPath) - if !inspected.ok ( - return {ok: false, error: "invalid project.json", status: 400} - ) - if inspected.targets.toNum() == 0 ( - return {ok: false, error: "invalid project.json", status: 400} - ) - string gzipPath = jsonPath ++ ".gz" - if !zip.gzip(jsonPath, gzipPath, {maxInputBytes: maxProjectJsonBytes, maxBytes: maxStoredProjectJsonBytes}) ( - return {ok: false, error: "compressed project data exceeds the 20 MB limit", code: "project_too_large", status: 413} - ) - - object assetsIndex = loadAssetsIndex() - object knownAssets = assetsIndex.assets - array projectAssets = [] - array pendingUploads = [] - object pendingNames = {} - number assetCount = 0 - - for i names.len ( - string name = names[i].toStr() - if name != "project.json" ( - if isValidMd5Ext(name) ( - assetCount += 1 - if assetCount > maxAssetCount ( - return {ok: false, error: "too many assets", status: 400} - ) - string assetPath = extractDir ++ "/" ++ name - number assetLimit = benefits.maxProjectAssetBytes.toNum() - if fs.size(assetPath).unwrapOr(0) > assetLimit ( - number assetLimitMb = (assetLimit / 1048576).toInt() - return {ok: false, error: "an asset exceeds your " ++ assetLimitMb.toStr() ++ " MB limit: " ++ name, code: "project_too_large", status: 413} - ) - string content = fs.readFile(assetPath).unwrapOr("") - string actualHash = content.hashMD5() - if actualHash != md5PartOf(name) ( - return {ok: false, error: "asset filename does not match its content: " ++ name, status: 400} - ) - projectAssets.append(name) - string localAssetPath = localBlobPath(assetObjectKey(name)) - boolean needsUpload = assetNeedsUpload(knownAssets.contains(name), fs.exists(localAssetPath),!r2Local) - if needsUpload and !pendingNames.contains(name) ( - pendingUploads.append({path: assetPath, name: name}) - pendingNames[name] = true - knownAssets[name] = content.toBytes().len - ) - ) - ) - ) - - object providedSet = {} - for i projectAssets.len ( - providedSet[projectAssets[i].toStr()] = true - ) - array referencedList = inspected.md5ext.assert(array) - object referencedNames = {} - for i referencedList.len ( - string name = referencedList[i].toStr() - if isValidMd5Ext(name) ( - referencedNames[name] = true - ) - ) - array referencedUnique = referencedNames.getKeys() - for i referencedUnique.len ( - string name = referencedUnique[i].toStr() - if !providedSet.contains(name) ( - if !knownAssets.contains(name) ( - return {ok: false, error: "missing asset: " ++ name, status: 400} - ) - projectAssets.append(name) - string localAssetPath = localBlobPath(assetObjectKey(name)) - if !r2Local and fs.exists(localAssetPath) and !pendingNames.contains(name) ( - pendingUploads.append({path: localAssetPath, name: name}) - pendingNames[name] = true - ) - ) - ) - if projectAssets.len > maxAssetCount ( - return {ok: false, error: "too many assets", status: 400} - ) - - number assetBytes = 0 - for i projectAssets.len ( - string name = projectAssets[i].toStr() - if knownAssets.contains(name) ( - assetBytes += knownAssets[name].toNum() - ) - ) - number projectAssetLimit = benefits.maxProjectAssetsBytes.toNum() - if assetBytes > projectAssetLimit ( - number projectAssetLimitMb = (projectAssetLimit / 1048576).toInt() - return {ok: false, error: "project assets exceed your " ++ projectAssetLimitMb.toStr() ++ " MB limit", code: "project_too_large", status: 413} - ) - number projectBytes = assetBytes + fs.size(gzipPath).unwrapOr(0) + projectHistoryBytes(project) - if storageAllowanceBytes >= 0 and projectBytes > storageAllowanceBytes ( - return {ok: false, error: "storage full", code: "storage_full", status: 413, overBytes: projectBytes - storageAllowanceBytes} - ) - object hold = noStorageHold() - string ownerId = projectOwnerId(project) - if usesRoturStorage(ownerId) ( - // Assets uploaded ahead are counted by this save now, not twice. - releasePendingAssetHolds(ownerId, project.owner.toStr(), projectAssets) - object incoming = projectStorageSummary(project, knownAssets) - object incomingJson = {} - incomingJson["incoming:" ++ project.id.toStr()] = fs.size(gzipPath).unwrapOr(0) - incoming.json = incomingJson - incoming.assets = projectAssets - object saving = {} - saving[project.id.toStr()] = incoming - hold = storageHold(ownerId, {projects: saving}, enforceStorage) - if hold.ok != true ( - return {ok: false, error: "storage full", code: "storage_full", status: 413, refusal: hold.refusal} - ) - ) - - number uploaded = 0 - for i pendingUploads.len ( - object upload = pendingUploads[i].assert(object) - string path = upload.path.toStr() - string name = upload.name.toStr() - string content = fs.readFile(path).unwrapOr("") - if !stageR2Blob(assetObjectKey(name), content, contentTypeForExt(extPartOf(name)), "") ( - storageHoldRelease(ownerId, hold) - return {ok: false, error: "asset upload failed", status: 502} - ) - knownAssets[name] = content.toBytes().len - uploaded += 1 - ) - if uploaded > 0 ( - saveAssetsIndex(assetsIndex) - ) - - return {ok: true, assets: projectAssets, uploaded: uploaded, skipped: projectAssets.len - uploaded, gzipPath: gzipPath, jsonBytes: projectJsonBytes, storedJsonBytes: fs.size(gzipPath).unwrapOr(0), assetBytes: assetBytes, hold: hold} -) +export {} def handleCheckProjectAssets(*serve.Context c) ( auto res = loadProject(c.param("id")) @@ -238,569 +45,67 @@ def handleUploadProject(*serve.Context c) ( ) def handleUploadProjectLocked(*serve.Context c) ( - auto res = loadProject(c.param("id")) - if !res.ok ( - c.notFound("project not found") - return - ) - object project = res.project.assert(object) + object? loaded = loadRequestedProject(c) + if loaded == null return + object project = loaded if !canSaveLiveProject(project, c) ( c.forbidden("not your project") return ) + UploadRequest request = readUploadRequest(c) + if refused(c, editorReplaceRefusal(c, request, project)) return + UploadHistory history = newUploadHistory(request, projectGitHead(project)) + if refused(c, liveContributorRefusal(request, canEditProject(project, c))) return + if refused(c, staleUploadRefusal(request, project, history.currentHead)) return + if refused(c, readUploadGitMetadata(history, projectHasWorkspace(project))) return + if refused(c, restoreCommitRefusal(request)) return - boolean replaceHistory = (c.formValue("replaceHistory") ?? "") == "true" - if replaceHistory and contextString(c, "sessionScope") == "editor" and !editorCanDeleteProject(project) ( - c.json(403, {ok: false, code: "editor_scope", error: "Open MistWarp to replace this project's saved history."}) - return - ) - string expectedHead = (c.formValue("expectedHead") ?? "").strip() - string restoreCommit = (c.formValue("restoreCommit") ?? "").strip() - string restoreMessage = (c.formValue("restoreMessage") ?? "").strip() - string pullId = (c.formValue("pullId") ?? "").strip() - string mergeSourceBranch = (c.formValue("mergeSourceBranch") ?? "").strip() - string mergeTargetBranch = (c.formValue("mergeTargetBranch") ?? "").strip() - string requestedMergeSourceHead = (c.formValue("mergeSourceHead") ?? "").strip() - string requestedMergeTargetHead = (c.formValue("mergeTargetHead") ?? "").strip() - string currentHead = projectGitHead(project) - string expectedEdited = (c.formValue("expectedEdited") ?? "").strip() - if !canEditProject(project, c) ( - if expectedEdited == "" or restoreCommit != "" or pullId != "" or mergeSourceBranch != "" or replaceHistory ( - c.forbidden("live contributors can save the current project; history and publishing require a maintainer") - return - ) - ) - if expectedEdited != "" and expectedEdited.toNum() != project.edited.toNum() ( - c.json(409, {ok: false, error: "someone saved newer changes; refresh project history before saving again", code: "head_changed"}) - return - ) - if expectedHead != "" and expectedHead != currentHead ( - c.json(409, {ok: false, error: "the project changed while you were merging", code: "head_changed", currentHead: currentHead}) - return - ) - - if replaceHistory and (expectedHead != currentHead or restoreCommit != "" or pullId != "" or mergeSourceBranch != "") ( - c.json(409, {ok: false, error: "history replacement requires the current saved head and a full workspace", code: "head_changed"}) - return - ) - - string gitMetadata = (c.formValue("git") ?? "") - object gitInfo = {} - if gitMetadata != "" ( - auto parsedGit = json.parse(gitMetadata, {type: "object"}) - if parsedGit.isErr() ( - c.badRequest("invalid MistWarp history metadata") - return - ) - gitInfo = parsedGit.unwrap().assert(object) - if gitInfo.contains("delta") and gitInfo.delta == true ( - string baseHead = (gitInfo.baseHead ?? "").toStr() - if baseHead == "" or baseHead != currentHead or !projectHasWorkspace(project) ( - c.json(409, {ok: false, error: "the saved project history changed", code: "workspace_base_changed", currentHead: currentHead}) - return - ) - ) - ) - - if restoreCommit != "" and !validGitObjectId(restoreCommit) ( - c.badRequest("invalid restore commit") - return - ) - - auto fileResult = c.formFile("project") - if fileResult.isErr() ( - c.badRequest("missing project file") - return - ) - object file = fileResult.unwrap() - object projectBenefits = mistwarpBenefitsForUserId(contextString(c, "userId")) - + UploadArchive archive = UploadArchive() + if refused(c, receiveProjectArchive(c, archive)) return + object benefits = mistwarpBenefitsForUserId(contextString(c, "userId")) string uploader = normalizeUsername(contextString(c, "username")) - auto workspaceResult = c.formFile("workspace") - auto mergeTreeResult = c.formFile("mergeTree") - boolean hasWorkspace = !workspaceResult.isErr() - boolean hasMergeTree = !mergeTreeResult.isErr() - object workspaceFile = {} - object mergeTreeFile = {} - if hasWorkspace ( - workspaceFile = workspaceResult.unwrap() - ) else if (c.formValue("workspaceToken") ?? "") != "" ( - object stagedWorkspace = consumeStagedUploadPart((c.formValue("workspaceToken") ?? ""), project.id.toStr(), "workspace", contextString(c, "userId")) - if !stagedWorkspace.ok ( - c.badRequest(stagedWorkspace.error.toStr()) - return - ) - workspaceFile = stagedWorkspace.file.assert(object) - hasWorkspace = true - ) - if hasMergeTree ( - mergeTreeFile = mergeTreeResult.unwrap() - ) else if (c.formValue("mergeTreeToken") ?? "") != "" ( - object stagedMergeTree = consumeStagedUploadPart((c.formValue("mergeTreeToken") ?? ""), project.id.toStr(), "mergeTree", contextString(c, "userId")) - if !stagedMergeTree.ok ( - c.badRequest(stagedMergeTree.error.toStr()) - return - ) - mergeTreeFile = stagedMergeTree.file.assert(object) - hasMergeTree = true - ) + if refused(c, receiveHistoryParts(c, project, request, history)) return number storageAllowanceBytes = if contextBool(c, "isAdmin") then -1 else projectStorageAllowance(project) - - string id = project.id.toStr() - string workId = id ++ "." ++ randomString(8) - string zipPath = tmpDir ++ workId ++ ".sb3" - string extractDir = tmpDir ++ workId - if !fs.writeFileBytes(zipPath, file.data) ( - c.internalError("failed to stage project upload") - return - ) - - number extractedLimit = maxProjectJsonBytes + projectBenefits.maxProjectAssetsBytes.toNum() - if !validateZipArchive(file.data, maxProjectUploadRequestBytes, extractedLimit, maxAssetCount + 1, false) ( - void fs.remove(zipPath) - c.badRequest("invalid or unsafe sb3 file") - return - ) - if !zip.decompress(zipPath, extractDir, {maxBytes: extractedLimit, maxFiles: maxAssetCount + 1}) ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.badRequest("invalid sb3 file") - return - ) - - if replaceHistory and !hasWorkspace ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.badRequest("history replacement requires a full workspace") - return - ) - - if hasWorkspace ( - if workspaceFile.size.toNum() > maxWorkspaceBytes or !validateZipArchive(workspaceFile.data.assertElse(byte[], []), maxWorkspaceBytes, extractedLimit, maxWorkspaceFiles, true) ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.json(400, {ok: false, code: "invalid_workspace", error: "invalid or unsafe MistWarp history archive"}) - return - ) - object materialized = materializeWorktreeAssets(project, workspaceFile.data.assertElse(byte[], []), extractDir, replaceHistory) - if !materialized.ok ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.json(409, {ok: false, code: "worktree_assets_unavailable", error: "some project files could not be found on the server; save again to upload them"}) - return - ) - if materialized.refs.toNum() > 0 ( - if materialized.size.toNum() > maxWorkspaceBytes or !validateZipArchive(materialized.data.assertElse(byte[], []), maxWorkspaceBytes, extractedLimit, maxWorkspaceFiles, false) ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.json(400, {ok: false, code: "invalid_workspace", error: "invalid or unsafe MistWarp history archive"}) - return - ) - workspaceFile.data = materialized.data - workspaceFile.size = materialized.size - ) - object checked = validateUploadedHistoryMode(project, workspaceFile.data.assertElse(byte[], []), gitMetadata, currentHead, replaceHistory) - if !checked.ok ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.json(409, {ok: false, code: "invalid_workspace", error: checked.error}) - return - ) - workspaceFile.data = checked.data - workspaceFile.size = checked.size - gitInfo = checked.manifest.assert(object) - gitMetadata = json.stringify(gitInfo) - ) - - // An editor with a shallow history can create a new commit object even when - // its project tree is identical to the current version. Compare the commit - // trees on the server before storing project JSON, assets, or history. - boolean ordinaryVersionSave = !replaceHistory and (gitInfo.worktree ?? false) != true and hasWorkspace and restoreCommit == "" and !hasMergeTree and gitMetadata != "" and currentHead != "" - if ordinaryVersionSave ( - string incomingHead = canonicalHistoryHead(gitInfo) - if incomingHead == currentHead ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.json(409, {ok: false, error: "No files changed in this commit.", code: "no_changes"}) - return - ) - if incomingHead != "" ( - byte[] incomingWorkspaceBody = workspaceFile.data.assertElse(byte[], []) - number incomingWorkspaceSize = workspaceFile.size.toNum() - if incomingWorkspaceSize > maxWorkspaceBytes or !validateZipArchive(incomingWorkspaceBody, maxWorkspaceBytes, extractedLimit, maxWorkspaceFiles, false) ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.json(400, {ok: false, error: "invalid or unsafe MistWarp project history", code: "invalid_workspace"}) - return - ) - string compareBasePath = cachedProjectWorkspacePath(project) - string compareIncomingPath = tmpDir ++ workId ++ ".compare-incoming.mwp" - boolean compareReady = compareBasePath != "" and fs.writeFileBytes(compareIncomingPath, incomingWorkspaceBody) - string comparisonJson = "" - if compareReady ( - comparisonJson = gitinspection.CommitTreesDiffer(compareBasePath, currentHead, compareIncomingPath, incomingHead) - ) - void fs.remove(compareIncomingPath) - auto parsedComparison = json.parse(comparisonJson, {type: "object"}) - if !compareReady or parsedComparison.isErr() or parsedComparison.unwrap().assert(object).ok != true ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.json(400, {ok: false, error: "could not validate the MistWarp project history", code: "invalid_workspace"}) - return - ) - if parsedComparison.unwrap().assert(object).changed != true ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.json(409, {ok: false, error: "No files changed in this commit.", code: "no_changes"}) - return - ) - ) - ) - - object extensionResult = collectProjectExtensions(extractDir ++ "/project.json", (c.formValue("extensions") ?? "")) - if !extensionResult.ok ( - void fs.remove(zipPath) - void fs.remove(extractDir) - object extensionError = {ok: false, error: extensionResult.error} - if extensionResult.contains("code") ( - extensionError.code = extensionResult.code - ) - c.json(400, extensionError) - return - ) - - boolean generatedWorkspace = false - byte[] generatedWorkspaceBody = [] - string historyBaseHead = currentHead - if hasMergeTree and !hasWorkspace and restoreCommit == "" ( - object mergeTree = mergeTreeFile - boolean branchMerge = pullId == "" and mergeSourceBranch != "" and mergeTargetBranch != "" and requestedMergeSourceHead != "" and requestedMergeTargetHead != "" - object mergePull = {} - object mergeSource = project - string mergeSourceHead = requestedMergeSourceHead - string mergeTargetHead = requestedMergeTargetHead - if branchMerge ( - string latestSourceHead = projectBranchHead(project, mergeSourceBranch) - string latestTargetHead = projectBranchHead(project, mergeTargetBranch) - if mergeSourceBranch == mergeTargetBranch or latestSourceHead == "" or latestTargetHead == "" or latestSourceHead != requestedMergeSourceHead or latestTargetHead != requestedMergeTargetHead ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.json(409, {ok: false, error: "a branch changed while this merge was processing", code: "branch_head_changed"}) - return - ) - historyBaseHead = mergeTargetHead - ) else ( - mergePull = pullDocument(project.id.toStr(), pullId) - if pullId == "" or mergePull.len == 0 or mergePull.state.toStr() != "open" or mergePull.targetProjectId.toStr() != project.id.toStr() ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.json(409, {ok: false, error: "the pull request is no longer open", code: "pull_not_open"}) - return - ) - auto mergeSourceResult = loadProject(mergePull.sourceProjectId.toStr()) - if !mergeSourceResult.ok ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.notFound("merge source project not found") - return - ) - mergeSource = mergeSourceResult.project.assert(object) - mergeSourceHead = mergePull.headCommit.toStr() - mergeTargetHead = currentHead - ) - if !validateZipArchive(mergeTree.data.assertElse(byte[], []), maxWorkspaceBytes, extractedLimit, maxWorkspaceFiles, false) ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.badRequest("invalid merged project tree") - return - ) - string mergeTargetPath = tmpDir ++ workId ++ ".merge-target.mwp" - string mergeSourcePath = tmpDir ++ workId ++ ".merge-source.mwp" - string mergeTreePath = tmpDir ++ workId ++ ".merge-tree.zip" - string mergeDeltaPath = tmpDir ++ workId ++ ".merge-delta.mwp" - boolean mergeReady = copyProjectWorkspaceToPath(project, mergeTargetPath) - mergeReady = mergeReady and copyProjectWorkspaceToPath(mergeSource, mergeSourcePath) - mergeReady = mergeReady and fs.writeFileBytes(mergeTreePath, mergeTree.data) - string mergeBranch = if branchMerge then mergeTargetBranch else (project.gitBranch ?? "main").toStr() - string mergeRemixParent = if project.contains("remixParent") and project.remixParent != null then project.remixParent.toStr() else "" - string mergeBaseCommit = if project.contains("remixBaseCommit") and project.remixBaseCommit != null then project.remixBaseCommit.toStr() else "" - string mergeProjectId = project.id.toStr() - string mergeMessage = if branchMerge then "Merge branch '" ++ mergeSourceBranch ++ "' into '" ++ mergeTargetBranch ++ "'" else "Merge pull request #" ++ pullId - number mergeTimestamp = timestamp - string mergeJson = "" - if mergeReady ( - mergeJson = gitinspection.CreateMergeArchiveJSON(mergeTargetPath, mergeTargetHead, mergeSourcePath, mergeSourceHead, mergeTreePath, mergeDeltaPath, mergeProjectId, mergeRemixParent, mergeBaseCommit, mergeBranch, uploader, mergeMessage, mergeTimestamp / 1000) - ) - auto parsedMerge = json.parse(mergeJson, {type: "object"}) - if !mergeReady or parsedMerge.isErr() or parsedMerge.unwrap().assert(object).ok != true ( - void fs.remove(mergeTargetPath) - void fs.remove(mergeSourcePath) - void fs.remove(mergeTreePath) - void fs.remove(mergeDeltaPath) - void fs.remove(zipPath) - void fs.remove(extractDir) - c.json(409, {ok: false, error: "could not create the merge commit", code: "merge_unavailable"}) - return - ) - object mergeResult = parsedMerge.unwrap().assert(object) - gitInfo = mergeResult.manifest.assert(object) - gitMetadata = json.stringify(gitInfo) - generatedWorkspaceBody = fs.readFileBytes(mergeDeltaPath).unwrapOr("".toBytes()) - generatedWorkspace = true - void fs.remove(mergeTargetPath) - void fs.remove(mergeSourcePath) - void fs.remove(mergeTreePath) - void fs.remove(mergeDeltaPath) - ) - if restoreCommit != "" and !hasWorkspace ( - string restoreBasePath = tmpDir ++ workId ++ ".restore-base.mwp" - string restoreDeltaPath = tmpDir ++ workId ++ ".restore-delta.mwp" - string restoreBranch = if project.contains("gitBranch") and project.gitBranch != null then project.gitBranch.toStr() else "main" - string restoreRemixParent = if project.contains("remixParent") and project.remixParent != null then project.remixParent.toStr() else "" - string restoreBaseCommit = if project.contains("remixBaseCommit") and project.remixBaseCommit != null then project.remixBaseCommit.toStr() else "" - number restoreTimestamp = timestamp - boolean restoreReady = copyProjectWorkspaceToPath(project, restoreBasePath) - string restoreJson = "" - if restoreReady ( - restoreJson = gitinspection.RestoreArchiveJSON(restoreBasePath, currentHead, restoreCommit, restoreDeltaPath, id, restoreRemixParent, restoreBaseCommit, restoreBranch, uploader, restoreMessage, restoreTimestamp / 1000) - ) - auto parsedRestore = json.parse(restoreJson, {type: "object"}) - if !restoreReady or parsedRestore.isErr() or parsedRestore.unwrap().assert(object).ok != true ( - void fs.remove(restoreBasePath) - void fs.remove(restoreDeltaPath) - void fs.remove(zipPath) - void fs.remove(extractDir) - c.json(409, {ok: false, error: "could not restore that project version", code: "restore_unavailable"}) - return - ) - object restoreResult = parsedRestore.unwrap().assert(object) - gitInfo = restoreResult.manifest.assert(object) - gitMetadata = json.stringify(gitInfo) - generatedWorkspaceBody = fs.readFileBytes(restoreDeltaPath).unwrapOr("".toBytes()) - generatedWorkspace = true - void fs.remove(restoreBasePath) - void fs.remove(restoreDeltaPath) - ) - - if generatedWorkspace ( - object checked = validateUploadedHistory(project, generatedWorkspaceBody, gitMetadata, historyBaseHead) - if !checked.ok ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.json(409, {ok: false, code: "invalid_workspace", error: checked.error}) - return - ) - generatedWorkspaceBody = checked.data.assert(byte[]) - gitInfo = checked.manifest.assert(object) - gitMetadata = json.stringify(gitInfo) - ) - - array previousWorkspaceKeys = projectWorkspaceKeys(project) - if hasWorkspace or generatedWorkspace ( - byte[] incomingWorkspaceBody = generatedWorkspaceBody - number incomingWorkspaceSize = generatedWorkspaceBody.len - if hasWorkspace ( - incomingWorkspaceBody = workspaceFile.data.assertElse(byte[], []) - incomingWorkspaceSize = workspaceFile.size.toNum() - ) - if incomingWorkspaceSize > maxWorkspaceBytes ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.json(413, {ok: false, error: "one history update exceeds the 512 MB safety limit", code: "workspace_update_too_large"}) - return - ) - if !validateZipArchive(incomingWorkspaceBody, maxWorkspaceBytes, extractedLimit, maxWorkspaceFiles, false) ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.json(400, {ok: false, error: "invalid or unsafe MistWarp project history", code: "invalid_workspace"}) - return - ) - // Validation has materialized all required layers and derived metadata. - previousWorkspaceKeys = projectWorkspaceKeys(project) - string workspaceKey = projectWorkspaceSnapshotKey(contentHash(incomingWorkspaceBody)) - if !stageR2Blob(workspaceKey, incomingWorkspaceBody, "application/x-mistwarp-project", "") ( - void fs.remove(zipPath) - void fs.remove(extractDir) - c.internalError("failed to store MistWarp project history") - return - ) - project.workspaceKey = workspaceKey - project.workspaceLayers = [workspaceKey] - project.historyBytes = incomingWorkspaceSize - project.gitHead = gitInfo.head - project.gitBranch = gitInfo.branch - project.gitCommits = gitInfo.commits - project.gitGraph = gitInfo.graph - if replaceHistory ( - project.remixBaseCommit = "" - ) - ) - - object uploadResult = uploadProjectFiles(project, extractDir, projectBenefits, storageAllowanceBytes, !contextBool(c, "isAdmin")) - string storageOwnerId = projectOwnerId(project) - object storageHeld = (uploadResult.hold ?? {}).assertElse(object, {}) - + if refused(c, unpackProjectArchive(project, benefits, archive)) return + + if refused(c, archive.discardOn(replaceWithoutWorkspaceRefusal(request, history))) return + if refused(c, archive.discardOn(checkUploadedWorkspace(project, request, history, archive))) return + if refused(c, archive.discardOn(unchangedVersionCheck(project, request, history, archive))) return + object extensionResult = collectProjectExtensions(archive.extractDir ++ "/project.json", request.extensions) + if refused(c, archive.discardOn(extensionRefusal(extensionResult))) return + if refused(c, archive.discardOn(createMergeCommit(project, request, history, archive, uploader))) return + if refused(c, archive.discardOn(createRestoreCommit(project, request, history, archive, uploader))) return + if refused(c, archive.discardOn(checkGeneratedHistory(project, history))) return + + UploadRollback rollback = UploadRollback() + rollback.previousWorkspaceKeys = projectWorkspaceKeys(project) + if refused(c, archive.discardOn(storeUploadedHistory(project, request, history, archive))) return + object uploadResult = uploadProjectFiles(project, archive.extractDir, benefits, storageAllowanceBytes, !contextBool(c, "isAdmin")) + rollback.ownerId = projectOwnerId(project) + rollback.hold = (uploadResult.hold ?? {}).assertElse(object, {}) if !uploadResult.ok ( - array failedWorkspaceKeys = projectWorkspaceKeys(project) - for i failedWorkspaceKeys.len ( - string failedWorkspaceKey = failedWorkspaceKeys[i].toStr() - if !previousWorkspaceKeys.contains(failedWorkspaceKey) ( - removeProjectStorageKeyIfOrphaned(failedWorkspaceKey, project.id.toStr(), true) - ) - ) - void fs.remove(zipPath) - void fs.remove(extractDir) - if (uploadResult.code ?? "").toStr() == "storage_full" ( - if uploadResult.contains("refusal") ( - roturStorageFullResponse(c, uploadResult.refusal.assertElse(object, {})) - return - ) - storageFullResponse(c, storageDetails(project.owner.toStr()), uploadResult.overBytes.toNum()) - return - ) - object errorBody = {ok: false, error: uploadResult.error} - if uploadResult.contains("code") ( - errorBody.code = uploadResult.code - ) - c.json(uploadResult.status.toInt(), errorBody) + rejectProjectFiles(c, project, uploadResult, rollback, archive) return ) + if refused(c, archive.discardOn(storeUploadExtensions(project, extensionResult, rollback))) return + saveUploadThumbnail(c, project.id.toStr()) - array projectExtensions = extensionResult.extensions.assert(array) - if !storeProjectExtensions(project.id.toStr(), storedProjectExtensions(project), projectExtensions, extensionResult.sources.assert(object)) ( - storageHoldRelease(storageOwnerId, storageHeld) - void fs.remove(zipPath) - void fs.remove(extractDir) - c.internalError("failed to store extension sources") - return - ) - - auto thumbResult = c.formFile("thumbnail") - if !thumbResult.isErr() ( - object thumb = thumbResult.unwrap() - if thumb.size.toNum() <= maxThumbBytes ( - void fs.writeFileBytes(thumbnailPath(id), thumb.data) - ) - ) - - project.assets = uploadResult.assets - project.edited = timestamp - - string gzipPath = uploadResult.gzipPath.toStr() - string previousJsonKey = project.jsonKey.toStr() - if previousJsonKey.startsWith("project-json/") or (project.contains("inheritedJson") and project.inheritedJson == true) ( - project.jsonKey = projectJsonKey(id) - ) - project.inheritedJson = false - project.sizeBytes = uploadResult.assetBytes.toNum() + uploadResult.storedJsonBytes.toNum() - project.jsonBytes = uploadResult.jsonBytes - project.storedJsonBytes = uploadResult.storedJsonBytes - project.assetBytes = uploadResult.assetBytes - project.jsonGzip = true - project.extensions = projectExtensions - project.extensionsIndexed = true - project.delete("pendingExtensionUrls") - - // Use an immutable JSON key so a failed metadata publication cannot - // overwrite the playable content referenced by the previous record. - byte[] jsonBody = fs.readFileBytes(gzipPath).unwrapOr("".toBytes()) - project.jsonKey = projectJsonSnapshotKey(contentHash(jsonBody)) - boolean storedJson = stageR2Blob(project.jsonKey.toStr(), jsonBody, "application/json", "gzip") - project.lastUploadAt = timestamp - project.pendingJson = false - if !storedJson ( - if project.contains("workspaceLayers") and project.workspaceLayers != null ( - array failedWorkspaceKeys = project.workspaceLayers.assertElse(array, []) - for i failedWorkspaceKeys.len ( - string failedWorkspaceKey = failedWorkspaceKeys[i].toStr() - if !previousWorkspaceKeys.contains(failedWorkspaceKey) ( - removeProjectStorageKeyIfOrphaned(failedWorkspaceKey, project.id.toStr(), true) - ) - ) - ) - storageHoldRelease(storageOwnerId, storageHeld) - c.json(500, {ok: false, error: "failed to store project data, please try saving again"}) - void fs.remove(zipPath) - void fs.remove(extractDir) + applyUploadedFiles(project, uploadResult, extensionResult, rollback) + if !publishProjectJson(project, uploadResult) ( + abandonUnpublishedUpload(c, project, rollback, archive) return ) recordCreatingDay(contextString(c, "username")) - - // The wrapper holds the publication lock from the initial version read. - boolean savedProject = saveProject(project) - if !savedProject ( - if project.jsonKey.toStr() != previousJsonKey ( - removeProjectStorageKeyIfOrphaned(project.jsonKey.toStr(), project.id.toStr(), false) - ) - if project.contains("workspaceLayers") and project.workspaceLayers != null ( - array failedWorkspaceKeys = project.workspaceLayers.assertElse(array, []) - for i failedWorkspaceKeys.len ( - string failedWorkspaceKey = failedWorkspaceKeys[i].toStr() - if !previousWorkspaceKeys.contains(failedWorkspaceKey) ( - removeProjectStorageKeyIfOrphaned(failedWorkspaceKey, project.id.toStr(), true) - ) - ) - ) - storageHoldRelease(storageOwnerId, storageHeld) - void fs.remove(zipPath) - void fs.remove(extractDir) - c.internalError("failed to save project metadata") + if !saveProject(project) ( + abandonUnsavedUpload(c, project, rollback, archive) return ) - upsertIndexEntry(project) - thread.new(warmShareCard, project.id.toStr()) - if usesRoturStorage(storageOwnerId) ( - thread.new(storageSettle, storageOwnerId, storageHeld) - ) else ( - releasePendingUploads(uploader, project.assets.assertElse(array, [])) - ) - string replacedJsonKey = if previousJsonKey != project.jsonKey.toStr() then previousJsonKey else "" - array replacedWorkspaceKeys = [] - if project.contains("workspaceLayers") and project.workspaceLayers != null ( - array keptWorkspaceKeys = project.workspaceLayers.assertElse(array, []) - for i previousWorkspaceKeys.len ( - string oldWorkspaceKey = previousWorkspaceKeys[i].toStr() - if !keptWorkspaceKeys.contains(oldWorkspaceKey) ( - replacedWorkspaceKeys.append(oldWorkspaceKey) - ) - ) - ) - if replacedJsonKey != "" or replacedWorkspaceKeys.len > 0 ( - thread.new(cleanupReplacedProjectStorage, project.id.toStr(), replacedJsonKey, replacedWorkspaceKeys) - ) - void fs.remove(zipPath) - void fs.remove(extractDir) - - if !projectHasWorkspace(project) ( - object generatedHistory = ensureUploadedProjectHistory(project) - if generatedHistory.ok ( - project = generatedHistory.project.assert(object) - ) else ( - log "WARNING: project " ++ project.id.toStr() ++ " uploaded without history and fallback generation failed: " ++ generatedHistory.error.toStr() - ) - ) - - if pullId != "" ( - if !completeNativePull(project, pullId, expectedHead, contextString(c, "username")) ( - c.json(409, {ok: false, error: "the project saved, but the pull request could not be closed", code: "pull_not_closed"}) - return - ) - ) + settleSavedUpload(project, rollback, uploader) + archive.discard() - c.json(200, { - ok: true, - project: { - id: project.id, - gitHead: projectGitHead(project), - gitBranch: project.gitBranch, - edited: project.edited, - remixBaseCommit: project.remixBaseCommit - }, - uploaded: uploadResult.uploaded, - skipped: uploadResult.skipped, - projectJsonUrl: projectJsonUrlFor(project), - thumbUrl: viewerThumbUrlFor(project) - }) + project = projectWithUploadHistory(project) + if refused(c, completeUploadPull(project, request, contextString(c, "username"))) return + c.json(200, uploadResponse(project, uploadResult)) ) def handleBootstrapProjectHistory(*serve.Context c) ( @@ -863,89 +168,3 @@ def handleSetThumbnail(*serve.Context c) ( thread.new(warmShareCard, project.id.toStr()) c.json(200, {ok: true, thumbUrl: viewerThumbUrlFor(project)}) ) - -def materializeWorktreeAssets(object project, byte[] body, string extractDir, boolean replaceHistory) object ( - string workId = tmpDir ++ project.id.toStr() ++ "." ++ randomString(8) - string incomingPath = workId ++ ".refs.mwp" - string outputPath = workId ++ ".materialized.mwp" - string fetchDir = workId ++ ".assets" - if !fs.writeFileBytes(incomingPath, body) ( - return {ok: false} - ) - string basePath = "" - if !replaceHistory and projectHasWorkspace(project) ( - basePath = cachedProjectWorkspacePath(project) - ) - string assetDirs = extractDir ++ "\n" ++ localBlobsDir ++ "assets" - auto parsed = json.parse(gitinspection.MaterializeWorktreeAssetsJSON(incomingPath, outputPath, basePath, assetDirs), {type: "object"}) - object res = if parsed.isErr() then {ok: false} else parsed.unwrap().assert(object) - array missing = (res.missing ?? []).assertElse(array, []) - if !res.ok and missing.len > 0 and missing.len <= maxAssetCount and !r2Local ( - void fs.mkdir(fetchDir, {recursive: true}) - for i missing.len ( - string name = missing[i].toStr() - if isValidMd5Ext(name) ( - void copyStorageKeyToPath(assetObjectKey(name), fetchDir ++ "/" ++ name) - ) - ) - parsed = json.parse(gitinspection.MaterializeWorktreeAssetsJSON(incomingPath, outputPath, basePath, assetDirs ++ "\n" ++ fetchDir), {type: "object"}) - res = if parsed.isErr() then {ok: false} else parsed.unwrap().assert(object) - ) - void fs.remove(incomingPath) - void fs.remove(fetchDir) - if res.ok != true ( - void fs.remove(outputPath) - return {ok: false} - ) - number refs = (res.refs ?? 0).toNum() - if refs == 0 ( - return {ok: true, refs: 0} - ) - byte[] data = fs.readFileBytes(outputPath).unwrapOr("".toBytes()) - void fs.remove(outputPath) - return {ok: data.len > 0, refs: refs, data: data, size: data.len} -) - -def validateUploadedHistory(object project, byte[] body, string metadata, string currentHead) object ( - return validateUploadedHistoryMode(project, body, metadata, currentHead, false) -) - -// Only the explicit maintainer upload path may discard the stored graph. -// With no base archive, validation also rejects deltas and incomplete history. -def validateUploadedHistoryMode(object project, byte[] body, string metadata, string currentHead, boolean replaceHistory) object ( - string workId = tmpDir ++ project.id.toStr() ++ "." ++ randomString(8) - string incomingPath = workId ++ ".incoming.mwp" - string outputPath = workId ++ ".validated.mwp" - string basePath = "" - if !replaceHistory and projectHasWorkspace(project) ( - basePath = cachedProjectWorkspacePath(project) - if basePath == "" ( - return {ok: false, error: "stored history unavailable; retry later"} - ) - ) - if !fs.writeFileBytes(incomingPath, body) ( - return {ok: false, error: "could not stage history"} - ) - string forkBase = (project.remixBaseCommit ?? "").toStr() - if replaceHistory ( - currentHead = "" - forkBase = "" - ) - string projectId = project.id.toStr() - string remixParent = (project.remixParent ?? "").toStr() - string resultJson = gitinspection.ValidateHistoryArchiveJSON(basePath, incomingPath, outputPath, currentHead, forkBase, metadata, projectId, remixParent) - void fs.remove(incomingPath) - auto parsed = json.parse(resultJson, {type: "object"}) - if parsed.isErr() ( - void fs.remove(outputPath) - return {ok: false, error: "could not validate history"} - ) - object res = parsed.unwrap().assert(object) - if res.ok ( - byte[] data = fs.readFileBytes(outputPath).unwrapOr("".toBytes()) - res.data = data - res.size = data.len - ) - void fs.remove(outputPath) - return res -) From 2e32ea0647484493cd38008e8bbf67ade654b956 Mon Sep 17 00:00:00 2001 From: Mistium <92952823+Mistium@users.noreply.github.com> Date: Sun, 1 Nov 2026 07:41:44 +0000 Subject: [PATCH 3/8] Send body-less refusals with their own status refused() turned any status other than 400, 403 and 404 into a 500. Give 500 its own branch and send any other status as it is, with an ok: false body. No current refusal uses another status, so responses are unchanged. --- src/projects/project-request.osl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/projects/project-request.osl b/src/projects/project-request.osl index 03a4b9c..9a5da97 100644 --- a/src/projects/project-request.osl +++ b/src/projects/project-request.osl @@ -25,8 +25,10 @@ def refused(*serve.Context c, Refusal refusal) boolean ( c.forbidden(refusal.message) ) else if refusal.status == 404 ( c.notFound(refusal.message) - ) else ( + ) else if refusal.status == 500 ( c.internalError(refusal.message) + ) else ( + c.json(refusal.status, {ok: false, error: refusal.message}) ) return true ) From 36e22e1185e9b1e3b094527a2b8acfd9982b1dba Mon Sep 17 00:00:00 2001 From: Mistium <92952823+Mistium@users.noreply.github.com> Date: Sun, 1 Nov 2026 07:41:57 +0000 Subject: [PATCH 4/8] Give project-update.osl an export block Every source file lists its exports after its imports; project-update.osl had none. Its OwnerTransfer record moves to project-update-policy.osl, next to how the upload step records sit in project-upload-policy.osl. --- src/projects/project-update-policy.osl | 6 ++++++ src/projects/project-update.osl | 6 +----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/projects/project-update-policy.osl b/src/projects/project-update-policy.osl index dc27a53..71d4433 100644 --- a/src/projects/project-update-policy.osl +++ b/src/projects/project-update-policy.osl @@ -1,5 +1,11 @@ export {} +type OwnerTransfer ( + string fromId = "" + string toId = "" + object hold = {} +) + def applyBranding(object project, object body, object creatorBenefits) Refusal ( if !body.contains("branding") ( return noRefusal() diff --git a/src/projects/project-update.osl b/src/projects/project-update.osl index aba371d..b23653f 100644 --- a/src/projects/project-update.osl +++ b/src/projects/project-update.osl @@ -1,8 +1,4 @@ -type OwnerTransfer ( - string fromId = "" - string toId = "" - object hold = {} -) +export {} def handleUpdateProject(*serve.Context c) ( string lockName = "project-upload:" ++ c.param("id") From e1617d205ccbaa47239e75c62750c35f78a4f0c1 Mon Sep 17 00:00:00 2001 From: Mistium <92952823+Mistium@users.noreply.github.com> Date: Sun, 1 Nov 2026 07:43:17 +0000 Subject: [PATCH 5/8] Keep upload helpers next to the steps that use them project-upload-policy.osl was collecting helpers that are not upload policy. keysNotIn, storedWorkspaceLayers, replacedWorkspaceKeys and projectFilesErrorBody move to project-upload-publish.osl, and megabytes with the three size failure helpers move to project-upload-files.osl. Their tests move into new test files beside those sources. --- src/projects/project-upload-files.osl | 25 +++++ src/projects/project-upload-files.test.osl | 74 +++++++++++++ src/projects/project-upload-policy.osl | 58 ---------- src/projects/project-upload-policy.test.osl | 19 ---- src/projects/project-upload-publish.osl | 33 ++++++ src/projects/project-upload-publish.test.osl | 106 +++++++++++++++++++ 6 files changed, 238 insertions(+), 77 deletions(-) create mode 100644 src/projects/project-upload-files.test.osl create mode 100644 src/projects/project-upload-publish.test.osl diff --git a/src/projects/project-upload-files.osl b/src/projects/project-upload-files.osl index e93b9f6..56d1a64 100644 --- a/src/projects/project-upload-files.osl +++ b/src/projects/project-upload-files.osl @@ -188,3 +188,28 @@ def uploadProjectFiles(object project, string extractDir, object benefits, numbe return {ok: true, assets: projectAssets, uploaded: uploaded, skipped: projectAssets.len - uploaded, gzipPath: gzipPath, jsonBytes: projectJsonBytes, storedJsonBytes: fs.size(gzipPath).unwrapOr(0), assetBytes: assetBytes, hold: hold} ) + +def megabytes(number bytes) number ( + return (bytes / 1048576).toInt() +) + +def assetSizeFailure(string name, number size, number limit) object ( + if size > limit ( + return {ok: false, error: "an asset exceeds your " ++ megabytes(limit).toStr() ++ " MB limit: " ++ name, code: "project_too_large", status: 413} + ) + return {} +) + +def projectAssetsFailure(number assetBytes, number limit) object ( + if assetBytes > limit ( + return {ok: false, error: "project assets exceed your " ++ megabytes(limit).toStr() ++ " MB limit", code: "project_too_large", status: 413} + ) + return {} +) + +def storageAllowanceFailure(number projectBytes, number allowanceBytes) object ( + if allowanceBytes >= 0 and projectBytes > allowanceBytes ( + return {ok: false, error: "storage full", code: "storage_full", status: 413, overBytes: projectBytes - allowanceBytes} + ) + return {} +) diff --git a/src/projects/project-upload-files.test.osl b/src/projects/project-upload-files.test.osl new file mode 100644 index 0000000..f837a12 --- /dev/null +++ b/src/projects/project-upload-files.test.osl @@ -0,0 +1,74 @@ +import "std:testing" +import "std:json" +import "std:fs" +import "std:zip" +import "./project-upload-files.osl" + +boolean r2Local = true +number maxAssetCount = 10 +number maxProjectJsonBytes = 268435456 +number maxStoredProjectJsonBytes = 20971520 +def isValidMd5Ext(string name) boolean ( + return true +) +def md5PartOf(string name) string ( + return name +) +def assetObjectKey(string name) string ( + return "assets/" ++ name +) +def localBlobPath(string key) string ( + return "/tmp/" ++ key +) +def assetNeedsUpload(boolean known, boolean localExists, boolean remoteConfigured) boolean ( + return false +) +def loadAssetsIndex() object ( + return {assets: {}} +) +def saveAssetsIndex(object index) boolean ( + return true +) +def contentTypeForExt(string ext) string ( + return "" +) +def extPartOf(string name) string ( + return "" +) +def stageR2Blob(string key, any body, string contentType, string contentEncoding) boolean ( + return true +) +def noStorageHold() object ( + return {} +) +def storageHold(string userId, object changes, boolean enforce) object ( + return {ok: true} +) +def storageHoldRelease(string userId, object hold) ( + return +) +def releasePendingAssetHolds(string ownerId, string username, array names) ( + return +) +def projectHistoryBytes(object project) number ( + return 0 +) +def projectOwnerId(object project) string ( + return "" +) +def projectStorageSummary(object project, object assetSizes) object ( + return {} +) +def usesRoturStorage(string userId) boolean ( + return false +) + +testing.assert(megabytes(10485760) == 10 and megabytes(1572864) == 1, "limits are whole megabytes") +testing.assert(assetSizeFailure("a.svg", 10, 10).len == 0, "an asset at the limit is accepted") +object bigAsset = assetSizeFailure("a.svg", 10485761, 10485760) +testing.assert(bigAsset.status == 413 and bigAsset.error == "an asset exceeds your 10 MB limit: a.svg", "an oversized asset names the limit and file") +testing.assert(projectAssetsFailure(5, 5).len == 0 and projectAssetsFailure(6, 5).code == "project_too_large", "project assets are capped") +testing.assert(storageAllowanceFailure(100, -1).len == 0, "a negative allowance means storage is counted elsewhere") +testing.assert(storageAllowanceFailure(100, 100).len == 0, "a project that exactly fits is accepted") +object full = storageAllowanceFailure(150, 100) +testing.assert(full.code == "storage_full" and full.overBytes == 50 and full.status == 413, "a full allowance reports the overflow") diff --git a/src/projects/project-upload-policy.osl b/src/projects/project-upload-policy.osl index 0333bcb..a970171 100644 --- a/src/projects/project-upload-policy.osl +++ b/src/projects/project-upload-policy.osl @@ -169,61 +169,3 @@ def adoptGeneratedHistory(UploadHistory history, object generated, byte[] genera history.generatedWorkspace = generatedWorkspace history.generated = true ) - -def storedWorkspaceLayers(object project) array ( - if project.contains("workspaceLayers") and project.workspaceLayers != null ( - return project.workspaceLayers.assertElse(array, []) - ) - return [] -) - -def keysNotIn(array keys, array kept) array ( - array missing = [] - for i keys.len ( - string key = keys[i].toStr() - if !kept.contains(key) ( - missing.append(key) - ) - ) - return missing -) - -def replacedWorkspaceKeys(object project, array previousWorkspaceKeys) array ( - if project.contains("workspaceLayers") and project.workspaceLayers != null ( - return keysNotIn(previousWorkspaceKeys, project.workspaceLayers.assertElse(array, [])) - ) - return [] -) - -def projectFilesErrorBody(object uploadResult) object ( - object errorBody = {ok: false, error: uploadResult.error} - if uploadResult.contains("code") ( - errorBody.code = uploadResult.code - ) - return errorBody -) - -def megabytes(number bytes) number ( - return (bytes / 1048576).toInt() -) - -def assetSizeFailure(string name, number size, number limit) object ( - if size > limit ( - return {ok: false, error: "an asset exceeds your " ++ megabytes(limit).toStr() ++ " MB limit: " ++ name, code: "project_too_large", status: 413} - ) - return {} -) - -def projectAssetsFailure(number assetBytes, number limit) object ( - if assetBytes > limit ( - return {ok: false, error: "project assets exceed your " ++ megabytes(limit).toStr() ++ " MB limit", code: "project_too_large", status: 413} - ) - return {} -) - -def storageAllowanceFailure(number projectBytes, number allowanceBytes) object ( - if allowanceBytes >= 0 and projectBytes > allowanceBytes ( - return {ok: false, error: "storage full", code: "storage_full", status: 413, overBytes: projectBytes - allowanceBytes} - ) - return {} -) diff --git a/src/projects/project-upload-policy.test.osl b/src/projects/project-upload-policy.test.osl index 7669d9c..1c85dbe 100644 --- a/src/projects/project-upload-policy.test.osl +++ b/src/projects/project-upload-policy.test.osl @@ -136,22 +136,3 @@ UploadHistory adopted = newUploadHistory(request(), headA) adoptGeneratedHistory(adopted, generated, "delta".toBytes()) testing.assert(adopted.generated and adopted.info.head == headB and adopted.generatedWorkspace.len == 5, "generated history replaces the manifest and body") testing.assert(json.parse(adopted.metadata, {type: "object"}).unwrap().head == headB, "generated metadata matches the manifest") - -testing.assert(storedWorkspaceLayers({}).len == 0 and storedWorkspaceLayers({workspaceLayers: null}).len == 0, "projects without layers have none") -testing.assert(keysNotIn(["a", "b", "c"], ["b"]) == ["a", "c"], "keysNotIn keeps order") -testing.assert(replacedWorkspaceKeys({workspaceLayers: ["n"]}, ["o", "n"]) == ["o"], "replaced layers are the old ones not kept") -testing.assert(replacedWorkspaceKeys({}, ["o"]).len == 0, "nothing is replaced without stored layers") - -object filesError = projectFilesErrorBody({ok: false, error: "too many assets", status: 400}) -testing.assert(filesError.ok == false and filesError.error == "too many assets" and !filesError.contains("status") and !filesError.contains("code"), "file errors expose only ok, error and code") -testing.assert(projectFilesErrorBody({ok: false, error: "big", code: "project_too_large"}).code == "project_too_large", "file errors keep their code") - -testing.assert(megabytes(10485760) == 10 and megabytes(1572864) == 1, "limits are whole megabytes") -testing.assert(assetSizeFailure("a.svg", 10, 10).len == 0, "an asset at the limit is accepted") -object bigAsset = assetSizeFailure("a.svg", 10485761, 10485760) -testing.assert(bigAsset.status == 413 and bigAsset.error == "an asset exceeds your 10 MB limit: a.svg", "an oversized asset names the limit and file") -testing.assert(projectAssetsFailure(5, 5).len == 0 and projectAssetsFailure(6, 5).code == "project_too_large", "project assets are capped") -testing.assert(storageAllowanceFailure(100, -1).len == 0, "a negative allowance means storage is counted elsewhere") -testing.assert(storageAllowanceFailure(100, 100).len == 0, "a project that exactly fits is accepted") -object full = storageAllowanceFailure(150, 100) -testing.assert(full.code == "storage_full" and full.overBytes == 50 and full.status == 413, "a full allowance reports the overflow") diff --git a/src/projects/project-upload-publish.osl b/src/projects/project-upload-publish.osl index e8cbaf6..690100a 100644 --- a/src/projects/project-upload-publish.osl +++ b/src/projects/project-upload-publish.osl @@ -140,3 +140,36 @@ def uploadResponse(object project, object uploadResult) object ( thumbUrl: viewerThumbUrlFor(project) } ) + +def storedWorkspaceLayers(object project) array ( + if project.contains("workspaceLayers") and project.workspaceLayers != null ( + return project.workspaceLayers.assertElse(array, []) + ) + return [] +) + +def keysNotIn(array keys, array kept) array ( + array missing = [] + for i keys.len ( + string key = keys[i].toStr() + if !kept.contains(key) ( + missing.append(key) + ) + ) + return missing +) + +def replacedWorkspaceKeys(object project, array previousWorkspaceKeys) array ( + if project.contains("workspaceLayers") and project.workspaceLayers != null ( + return keysNotIn(previousWorkspaceKeys, project.workspaceLayers.assertElse(array, [])) + ) + return [] +) + +def projectFilesErrorBody(object uploadResult) object ( + object errorBody = {ok: false, error: uploadResult.error} + if uploadResult.contains("code") ( + errorBody.code = uploadResult.code + ) + return errorBody +) diff --git a/src/projects/project-upload-publish.test.osl b/src/projects/project-upload-publish.test.osl new file mode 100644 index 0000000..68ee4e7 --- /dev/null +++ b/src/projects/project-upload-publish.test.osl @@ -0,0 +1,106 @@ +import "std:testing" +import "std:serve" +import "std:json" +import "std:regex" +import "./project-refusal.osl" +import "./project-upload-policy.osl" +import "std:fs" +import "std:thread" +import "./project-upload-publish.osl" + +type UploadArchive ( + def discard() ( + return + ) +) + +number maxThumbBytes = 1048576 +string tmpDir = "/tmp/" +def validGitObjectId(string oid) boolean ( + return regex.match("^[a-f0-9]{40}$", oid) +) +def projectJsonKey(string id) string ( + return "projects/" ++ id ++ ".json" +) +def projectJsonSnapshotKey(string hash) string ( + return "project-json/" ++ hash +) +def projectWorkspaceKeys(object project) array ( + return [] +) +def removeProjectStorageKeyIfOrphaned(string key, string id, boolean workspace) ( + return +) +def roturStorageFullResponse(*serve.Context c, object refusal) ( + return +) +def storageFullResponse(*serve.Context c, object details, number neededBytes) ( + return +) +def storageDetails(string username) object ( + return {} +) +def stageR2Blob(string key, any body, string contentType, string contentEncoding) boolean ( + return true +) +def contentHash(any body) string ( + return "hash" +) +def storageHoldRelease(string userId, object hold) ( + return +) +def storeProjectExtensions(string projectId, array previousExtensions, array extensions, object sources) boolean ( + return true +) +def storedProjectExtensions(object project) array ( + return [] +) +def thumbnailPath(string id) string ( + return tmpDir ++ id ++ ".png" +) +def upsertIndexEntry(object project) ( + return +) +def warmShareCard(string id) ( + return +) +def usesRoturStorage(string userId) boolean ( + return false +) +def storageSettle(string userId, object hold) ( + return +) +def releasePendingUploads(string username, array names) ( + return +) +def cleanupReplacedProjectStorage(string id, string jsonKey, array workspaceKeys) ( + return +) +def projectHasWorkspace(object project) boolean ( + return true +) +def ensureUploadedProjectHistory(object project) object ( + return {ok: true, project: project} +) +def completeNativePull(object project, string pullIndex, string expectedHead, string username) boolean ( + return true +) +def projectGitHead(object project) string ( + return "" +) +def projectJsonUrlFor(object project) string ( + return "" +) +def viewerThumbUrlFor(object project) string ( + return "" +) + +testing.assert(storedWorkspaceLayers({}).len == 0 and storedWorkspaceLayers({workspaceLayers: null}).len == 0, "projects without layers have none") +testing.assert(keysNotIn(["a", "b", "c"], ["b"]) == ["a", "c"], "keysNotIn keeps order") +testing.assert(replacedWorkspaceKeys({workspaceLayers: ["n"]}, ["o", "n"]) == ["o"], "replaced layers are the old ones not kept") +testing.assert(replacedWorkspaceKeys({}, ["o"]).len == 0, "nothing is replaced without stored layers") + +object filesError = projectFilesErrorBody({ok: false, error: "too many assets", status: 400}) +testing.assert(filesError.ok == false and filesError.error == "too many assets" and !filesError.contains("status") and !filesError.contains("code"), "file errors expose only ok, error and code") +testing.assert(projectFilesErrorBody({ok: false, error: "big", code: "project_too_large"}).code == "project_too_large", "file errors keep their code") + From 3c2edf3d49d91f683abc751fc8efd34137dc0c31 Mon Sep 17 00:00:00 2001 From: Mistium <92952823+Mistium@users.noreply.github.com> Date: Sun, 1 Nov 2026 07:43:29 +0000 Subject: [PATCH 6/8] Drop the comments that moved with the upload steps The split carried three comments over unchanged. New code here has no comments, and the function and parameter names already say what they described. --- src/projects/project-upload-files.osl | 4 ---- src/projects/project-upload-history.osl | 2 -- 2 files changed, 6 deletions(-) diff --git a/src/projects/project-upload-files.osl b/src/projects/project-upload-files.osl index 56d1a64..e0a9ea0 100644 --- a/src/projects/project-upload-files.osl +++ b/src/projects/project-upload-files.osl @@ -43,9 +43,6 @@ def inspectProjectJson(string path) object ( return inspected ) -// storageAllowanceBytes is a student's local limit, or -1. A Rotur owner's -// storage is held in the ledger instead, unless enforceStorage is off -// (admins); the result's hold is committed or released by the caller. def uploadProjectFiles(object project, string extractDir, object benefits, number storageAllowanceBytes, boolean enforceStorage) object ( array names = fs.readDir(extractDir).unwrapOr([]) string jsonPath = extractDir ++ "/project.json" @@ -154,7 +151,6 @@ def uploadProjectFiles(object project, string extractDir, object benefits, numbe object hold = noStorageHold() string ownerId = projectOwnerId(project) if usesRoturStorage(ownerId) ( - // Assets uploaded ahead are counted by this save now, not twice. releasePendingAssetHolds(ownerId, project.owner.toStr(), projectAssets) object incoming = projectStorageSummary(project, knownAssets) object incomingJson = {} diff --git a/src/projects/project-upload-history.osl b/src/projects/project-upload-history.osl index 197e156..6a594a7 100644 --- a/src/projects/project-upload-history.osl +++ b/src/projects/project-upload-history.osl @@ -254,8 +254,6 @@ def validateUploadedHistory(object project, byte[] body, string metadata, string return validateUploadedHistoryMode(project, body, metadata, currentHead, false) ) -// Only the explicit maintainer upload path may discard the stored graph. -// With no base archive, validation also rejects deltas and incomplete history. def validateUploadedHistoryMode(object project, byte[] body, string metadata, string currentHead, boolean replaceHistory) object ( string workId = tmpDir ++ project.id.toStr() ++ "." ++ randomString(8) string incomingPath = workId ++ ".incoming.mwp" From 061a36e69da6041d0990bc8472aa98707972909e Mon Sep 17 00:00:00 2001 From: Mistium <92952823+Mistium@users.noreply.github.com> Date: Sun, 1 Nov 2026 07:49:20 +0000 Subject: [PATCH 7/8] Compare exact keys, sizes and stored records in the smoke test The smoke test masked far more than was volatile. Its key pattern was case-insensitive, so suffix rules also hid dataFormat, allowedItems, item and product ids, credit URLs, jsonKey and workspaceKey, and sizes were rounded to KiB, which would hide a character count used as a byte count. Mask only named volatile keys (tokens, view keys, timestamps, history sizes, notification ids and the history-sized quota figures), random access keys in URLs and embedded hashes and times. Sizes are compared exactly, and staged chunk sizes are checked against the bytes sent. Every successful upload and update now records the stored project, and the blob inventory lists the exact asset and project JSON names. A non-ASCII project and description tell characters from bytes. --- scripts/e2e_smoke.py | 79 +++++++++++++++++++++++++++----------------- 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/scripts/e2e_smoke.py b/scripts/e2e_smoke.py index 580f28b..ff6070e 100644 --- a/scripts/e2e_smoke.py +++ b/scripts/e2e_smoke.py @@ -31,8 +31,9 @@ USERS = {"sam": "u_sam", "alice": "u_alice", "bob": "u_bob"} SCOPED_SESSIONS = {"sam_editor": ("sam", "editor")} -VOLATILE_KEYS = re.compile(r"(^|_)(id|_id|at|time|timestamp|created|updated|modified|expires|sha|commit|head|token|etag|key|url)$|At$|Id$|Ms$|Url$|Key$|Sha$", re.I) -APPROX_KEYS = {"bytes", "mistwarp", "sizeBytes", "historyBytes", "storedJsonBytes", "used", "usedBytes", "storageUsed", "received", "size"} +VOLATILE_NAMES = {"token", "viewKey", "expires", "created", "edited", "firstSeen", "historyBytes"} +VOLATILE_FIELDS = {("notifications", "id"), ("largest", "bytes"), ("", "mistwarp")} +SAVE_PATH = re.compile(r"^/v1/projects/([^/]+)(/upload)?$") HEX = re.compile(r"^[0-9a-f]{32,64}$") ISO = re.compile(r"^\d{4}-\d{2}-\d{2}T") @@ -160,25 +161,35 @@ def read_project(self, project_id): return json.load(handle) def blob_inventory(self): - counts = {} + inventory = {} root = os.path.join(self.dir, "data", "blobs") for folder, _, names in os.walk(root): for name in names: - prefix = os.path.relpath(folder, root).split(os.sep)[0] - counts[prefix] = counts.get(prefix, 0) + 1 - return counts + path = os.path.relpath(os.path.join(folder, name), root) + prefix = path.split(os.sep)[0] + if prefix in ("assets", "project-json"): + inventory.setdefault(prefix, []).append(path) + else: + inventory[prefix] = inventory.get(prefix, 0) + 1 + for prefix, value in inventory.items(): + if isinstance(value, list): + value.sort() + return inventory SMOKE_AUTHOR = b"Smoke 1700000000 +0000" ZERO_HEAD = "0" * 40 EMBEDDED_HEX = re.compile(r"[0-9a-f]{32,64}") +CONTENT_NAME = re.compile(r"^(assets/)?[0-9a-f]{32}\.[a-z0-9]+$|^project-json/[0-9a-f]{64}\.json\.gz$") +ACCESS_KEY = re.compile(r"([?&]k=)[A-Za-z0-9]+") +EMBEDDED_TIME = re.compile(r"(?", value) - return value - return scrub(project) - - def committed_workspace(server, ids, alias, who, message, delta=False, same_tree=False): history = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % real_id(ids, alias), who)) tree = history.tree_of(history.head()) @@ -331,7 +330,10 @@ def committed_workspace(server, ids, alias, who, message, delta=False, same_tree def staged_part(server, step, path, who, kind, data, name): status, started = step(name + " start", "POST", path + "/upload-parts", who, {"kind": kind, "name": "", "size": len(data), "chunks": 1}) token = started.get("token", "") if isinstance(started, dict) else "" - step(name + " chunk", "POST", path + "/upload-parts/%s/chunk" % token, who, form=({"index": "0"}, {"chunk": data})) + status, chunk = step(name + " chunk", "POST", path + "/upload-parts/%s/chunk" % token, who, form=({"index": "0"}, {"chunk": data})) + for field in ("received", "size"): + if isinstance(chunk, dict) and chunk.get(field) == len(data): + chunk[field] = "" return token @@ -367,6 +369,10 @@ def head_and_edited(): sparse = sb3_archive(project_with_costumes([asset_name], {"ext": "https://example.com/ext.js"})) upload("upload sparse", {"extensions": json.dumps({"https://example.com/ext.js": "class SmokeExtension {}"})}, {"project": sparse}) upload("upload second asset", files={"project": sb3_archive(project_with_costumes([asset_name, other_name]), {other_name: other_content})}) + wide_name, wide_content = svg_asset("\u00fcn\u00efc\u00f6d\u00e9 \u2713 " * 400) + wide_project = project_with_costumes([asset_name, other_name, wide_name]) + wide_project["targets"][0]["variables"] = {"wide": ["\u00e9\u2713", "\u00e9\u2713\U0001F600" * 3000]} + upload("upload non-ascii project", files={"project": sb3_archive(wide_project, {wide_name: wide_content})}) head, edited = head_and_edited() upload("upload stale edited", {"expectedEdited": "1"}) @@ -474,6 +480,8 @@ def patch(name, body, who="sam"): patch("patch long description", {"description": "d" * 5001}) patch("patch long instructions", {"instructions": "i" * 5001}) patch("patch long notes", {"notes": "n" * 5001}) + patch("patch non-ascii description", {"description": "\u00e9\u2713" * 2400}) + patch("patch long non-ascii notes", {"notes": "\u00e9" * 5001}) patch("patch credits", {"credits": [ {"who": "Ada", "role": "Art", "url": "https://example.com/ada"}, {"who": "Bo", "role": "Music", "url": None}, @@ -554,24 +562,29 @@ def patch(name, body, who="sam"): step("get updated project", "GET", path, "bob") -def normalize(value, ids, key=""): +def volatile_key(key): + return key in VOLATILE_NAMES + + +def normalize(value, ids, key="", parent=""): if isinstance(value, dict): - return {k: normalize(v, ids, k) for k, v in sorted(value.items())} + return {k: normalize(v, ids, k, key) for k, v in sorted(value.items())} if isinstance(value, list): - return [normalize(v, ids, key) for v in value] + return [normalize(v, ids, key, parent) for v in value] + if (parent, key) in VOLATILE_FIELDS: + return "" if isinstance(value, str): value = re.sub(r"127\.0\.0\.1:\d+", "127.0.0.1:", value) for real, alias in ids.items(): value = value.replace(real, alias) - if HEX.match(value) or ISO.match(value): - return "" - if VOLATILE_KEYS.search(key) and not value.startswith("" - return value + value = ACCESS_KEY.sub(r"\1", value) + if CONTENT_NAME.match(value): + return value + return EMBEDDED_TIME.sub("