diff --git a/scripts/e2e_smoke.py b/scripts/e2e_smoke.py index c376c7a..ed88172 100644 --- a/scripts/e2e_smoke.py +++ b/scripts/e2e_smoke.py @@ -11,11 +11,14 @@ import subprocess import sys import tempfile +import threading import time import urllib.error import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import uuid import zipfile +import zlib PROJECT_JSON = { "targets": [{ @@ -26,9 +29,13 @@ "monitors": [], "extensions": [], "meta": {"semver": "3.0.0", "vm": "0.2.0", "agent": "smoke"}, } -USERS = {"sam": "u_sam", "alice": "u_alice"} -VOLATILE_KEYS = re.compile(r"(^|_)(id|_id|at|time|timestamp|created|updated|modified|expires|sha|commit|head|token|etag|key|url)$|At$|Id$|Ms$|Url$|Key$|Sha$", re.I) -APPROX_KEYS = {"bytes", "mistwarp", "sizeBytes"} +USERS = {"sam": "u_sam", "alice": "u_alice", "bob": "u_bob", "dee": "u_dee"} +FULL_STORAGE = {"u_dee"} +EXPECTED_ERRORS = {"upload extension store failure", "upload json store failure", "upload metadata save failure"} +SCOPED_SESSIONS = {"sam_editor": ("sam", "editor")} +VOLATILE_NAMES = {"token", "viewKey", "expires", "created", "edited", "firstSeen", "historyBytes"} +VOLATILE_FIELDS = {("notifications", "id"), ("largest", "bytes"), ("", "mistwarp")} +SAVE_PATH = re.compile(r"^/v1/projects/([^/]+)(/upload)?$") HEX = re.compile(r"^[0-9a-f]{32,64}$") ISO = re.compile(r"^\d{4}-\d{2}-\d{2}T") @@ -46,6 +53,56 @@ def default_sb3(): return buffer.getvalue() +class FakeRotur(BaseHTTPRequestHandler): + def answer(self, status, payload): + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + if self.path.startswith("/v2/storage-ledger/objects"): + self.answer(200, {"objects": []}) + return + if self.path.startswith("/v2/storage-ledger/"): + self.answer(404, {"code": "not_found"}) + return + name = self.path.split("?", 1)[0][len("/profile/"):] if self.path.startswith("/profile/") else "" + if name not in USERS: + self.close_connection = True + return + self.answer(200, {"id": USERS[name], "username": name}) + + def do_POST(self): + if not self.path.startswith("/v2/storage-ledger/"): + self.close_connection = True + return + request = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)) or b"{}") + if self.path == "/v2/storage-ledger/reservations" and request.get("user_id") in FULL_STORAGE: + self.answer(413, {"usage": 1000, "max": 1000, "available": 0, "needed": request.get("bytes", 0)}) + return + self.answer(201 if self.path == "/v2/storage-ledger/reservations" else 200, {"ok": True}) + + def do_DELETE(self): + self.answer(200, {"ok": True}) + + def log_message(self, *args): + pass + + +class Fault: + def __init__(self, path): + self.path = path + + def __enter__(self): + os.chmod(self.path, 0o555) + + def __exit__(self, *args): + os.chmod(self.path, 0o755) + + class Server: def __init__(self, binary, keep): self.binary = os.path.abspath(binary) @@ -53,10 +110,13 @@ def __init__(self, binary, keep): self.dir = tempfile.mkdtemp(prefix="mw-smoke-") self.port = free_port() self.base = "http://127.0.0.1:%d" % self.port + self.rotur = ThreadingHTTPServer(("127.0.0.1", 0), FakeRotur) + threading.Thread(target=self.rotur.serve_forever, daemon=True).start() data = os.path.join(self.dir, "data") os.makedirs(os.path.join(data, "sessions")) with open(os.path.join(self.dir, ".env"), "w") as env: - env.write("PORT=%d\nAPP_URL=http://127.0.0.1:%d\nROTUR_BASE=http://127.0.0.1:9\n" % (self.port, self.port)) + rotur = "http://127.0.0.1:%d" % self.rotur.server_address[1] + env.write("PORT=%d\nAPP_URL=http://127.0.0.1:%d\nROTUR_BASE=%s\nROTUR_LOCAL_API_URL=%s\nROTUR_SERVICE_SECRET=smoke\n" % (self.port, self.port, rotur, rotur)) with open(os.path.join(data, "agreement.json"), "w") as handle: json.dump({"version": 1, "text": "", "updatedAt": 1700000000000}, handle) with open(os.path.join(data, "agreement.md"), "w") as handle: @@ -66,6 +126,9 @@ def __init__(self, binary, keep): for name, uid in USERS.items(): with open(os.path.join(data, "sessions", "tok_%s.json" % name), "w") as handle: json.dump({"username": name, "userId": uid, "createdAt": int(time.time() * 1000)}, handle) + for token, (name, scope) in SCOPED_SESSIONS.items(): + with open(os.path.join(data, "sessions", "tok_%s.json" % token), "w") as handle: + json.dump({"username": name, "userId": USERS[name], "scope": scope, "parent": "tok_" + name, "createdAt": int(time.time() * 1000), "expiresAt": int(time.time() * 1000) + 86400000}, handle) self.log = open(os.path.join(self.dir, "server.log"), "w") self.process = subprocess.Popen([self.binary], cwd=self.dir, stdout=self.log, stderr=subprocess.STDOUT, start_new_session=True) for _ in range(300): @@ -87,6 +150,8 @@ def stop(self): except subprocess.TimeoutExpired: self.process.kill() self.log.close() + self.rotur.shutdown() + self.rotur.server_close() if not self.keep: shutil.rmtree(self.dir) @@ -111,34 +176,498 @@ def call(self, method, path, who=None, body=None, raw=None, content_type="applic return status, parsed, headers def upload(self, project_id, who, sb3): + return self.form("/v1/projects/%s/upload" % project_id, who, {}, {"project": sb3}) + + def form(self, path, who, fields, files): boundary = uuid.uuid4().hex - body = ( - ("--%s\r\nContent-Disposition: form-data; name=\"project\"; filename=\"p.sb3\"\r\n" - "Content-Type: application/octet-stream\r\n\r\n" % boundary).encode() - + sb3 + ("\r\n--%s--\r\n" % boundary).encode() - ) - return self.call("POST", "/v1/projects/%s/upload" % project_id, who, raw=body, - content_type="multipart/form-data; boundary=" + boundary) + body = b"" + for name, value in fields.items(): + body += ("--%s\r\nContent-Disposition: form-data; name=\"%s\"\r\n\r\n" % (boundary, name)).encode() + value.encode() + b"\r\n" + for name, value in files.items(): + body += ("--%s\r\nContent-Disposition: form-data; name=\"%s\"; filename=\"%s.bin\"\r\n" + "Content-Type: application/octet-stream\r\n\r\n" % (boundary, name, name)).encode() + value + b"\r\n" + body += ("--%s--\r\n" % boundary).encode() + return self.call("POST", path, who, raw=body, content_type="multipart/form-data; boundary=" + boundary) + + def faulty(self, folder): + path = os.path.join(self.dir, "data", folder) + os.makedirs(path, exist_ok=True) + return Fault(path) + + def read_project(self, project_id): + with open(os.path.join(self.dir, "data", "projects", project_id + ".json")) as handle: + return json.load(handle) + + def blob_inventory(self): + inventory = {} + root = os.path.join(self.dir, "data", "blobs") + for folder, _, names in os.walk(root): + for name in names: + path = os.path.relpath(os.path.join(folder, name), root) + prefix = path.split(os.sep)[0] + if prefix in ("assets", "project-json"): + inventory.setdefault(prefix, []).append(path) + else: + inventory[prefix] = inventory.get(prefix, 0) + 1 + for prefix, value in inventory.items(): + if isinstance(value, list): + value.sort() + return inventory + + +SMOKE_AUTHOR = b"Smoke 1700000000 +0000" +ZERO_HEAD = "0" * 40 +EMBEDDED_HEX = re.compile(r"[0-9a-f]{32,64}") +CONTENT_NAME = re.compile(r"^(assets/)?[0-9a-f]{32}\.[a-z0-9]+$|^project-json/[0-9a-f]{64}\.json\.gz$") +ACCESS_KEY = re.compile(r"([?&]k=)[A-Za-z0-9]+") +EMBEDDED_TIME = re.compile(r"(?%s" % label).encode() + return hashlib.md5(content).hexdigest() + ".svg", content + + +def project_with_costumes(names, extension_urls=None, targets=True): + project = json.loads(json.dumps(PROJECT_JSON)) + if not targets: + project.pop("targets") + return project + project["targets"][0]["costumes"] = [ + {"name": "c%d" % index, "assetId": name.split(".")[0], "md5ext": name, "dataFormat": "svg", "rotationCenterX": 0, "rotationCenterY": 0} + for index, name in enumerate(names) + ] + if extension_urls: + project["extensionURLs"] = extension_urls + project["extensions"] = list(extension_urls) + return project + + +def git_object(kind, body): + raw = b"%s %d\x00" % (kind.encode(), len(body)) + body + return hashlib.sha1(raw).hexdigest(), zlib.compress(raw) + + +class History: + def __init__(self, data): + with zipfile.ZipFile(io.BytesIO(data)) as archive: + self.entries = {name: archive.read(name) for name in archive.namelist() if not name.endswith("/")} + self.branch = self.entries[".git/HEAD"].decode().strip()[len("ref: refs/heads/"):] + self.refs = {name[len(".git/refs/heads/"):]: value.decode().strip() for name, value in self.entries.items() if name.startswith(".git/refs/heads/")} + self.objects = {name[len(".git/objects/"):].replace("/", ""): value for name, value in self.entries.items() if name.startswith(".git/objects/")} + self.added = {} + + def head(self): + return self.refs[self.branch] + def read(self, oid): + raw = zlib.decompress(self.objects.get(oid) or self.added[oid]) + header, body = raw.split(b"\x00", 1) + return header.split(b" ")[0].decode(), body -def normalize(value, ids, key=""): + def tree_of(self, commit): + return self.read(commit)[1].split(b"\n", 1)[0].decode()[len("tree "):] + + def entries_of(self, tree): + body = self.read(tree)[1] + entries, position = [], 0 + while position < len(body): + space = body.index(b" ", position) + nul = body.index(b"\x00", space) + entries.append((body[position:space], body[space + 1:nul], body[nul + 1:nul + 21])) + position = nul + 21 + return entries + + def files(self, tree, prefix=""): + found = {} + for mode, name, raw_oid in self.entries_of(tree): + path = prefix + name.decode() + if mode in (b"40000", b"040000"): + found.update(self.files(raw_oid.hex(), path + "/")) + else: + found[path] = self.read(raw_oid.hex())[1] + return found + + def add(self, kind, body): + oid, compressed = git_object(kind, body) + if oid not in self.objects: + self.added[oid] = compressed + return oid + + def tree_with(self, tree, name, content): + blob = self.add("blob", content) + entries = [entry for entry in self.entries_of(tree) if entry[1] != name.encode()] + entries.append((b"100644", name.encode(), bytes.fromhex(blob))) + entries.sort(key=lambda entry: entry[1] + (b"/" if entry[0] in (b"40000", b"040000") else b"")) + return self.add("tree", b"".join(mode + b" " + entry_name + b"\x00" + raw_oid for mode, entry_name, raw_oid in entries)) + + def commit(self, tree, parents, message): + lines = [b"tree " + tree.encode()] + [b"parent " + parent.encode() for parent in parents] + lines += [b"author " + SMOKE_AUTHOR, b"committer " + SMOKE_AUTHOR] + return self.add("commit", b"\n".join(lines) + b"\n\n" + message.encode() + b"\n") + + def archive(self, head, delta, extra=None): + refs = dict(self.refs) if not delta else {} + refs[self.branch] = head + manifest = self.metadata(head, delta) + manifest.update(extra or {}) + manifest.update({"format": "mistwarp-project", "version": 1, "commits": [], "graph": {}}) + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("mwp.json", json.dumps(manifest)) + archive.writestr(".git/HEAD", "ref: refs/heads/%s\n" % self.branch) + for branch, oid in sorted(refs.items()): + archive.writestr(".git/refs/heads/" + branch, oid + "\n") + objects = dict(self.added) if delta else dict(self.objects, **self.added) + for oid, compressed in sorted(objects.items()): + archive.writestr(".git/objects/%s/%s" % (oid[:2], oid[2:]), compressed) + return buffer.getvalue() + + def metadata(self, head, delta): + return {"head": head, "branch": self.branch, "delta": delta, "baseHead": self.head() if delta else None, "worktree": False} + + +def raw_get(server, path, who): + request = urllib.request.Request(server.base + path, method="GET") + request.add_header("Authorization", "Bearer tok_" + who) + with urllib.request.urlopen(request, timeout=30) as response: + data = response.read() + return gzip.decompress(data) if data[:2] == b"\x1f\x8b" else data + + +def real_id(ids, alias): + return next(real for real, known in ids.items() if known == alias) + + +def committed_workspace(server, ids, alias, who, message, delta=False, same_tree=False): + history = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % real_id(ids, alias), who)) + tree = history.tree_of(history.head()) + if not same_tree: + tree = history.tree_with(tree, "smoke-notes.txt", message.encode()) + head = history.commit(tree, [history.head()], message) + return history.archive(head, delta), json.dumps(history.metadata(head, delta)) + + +def staged_part(server, step, path, who, kind, data, name): + status, started = step(name + " start", "POST", path + "/upload-parts", who, {"kind": kind, "name": "", "size": len(data), "chunks": 1}) + token = started.get("token", "") if isinstance(started, dict) else "" + status, chunk = step(name + " chunk", "POST", path + "/upload-parts/%s/chunk" % token, who, form=({"index": "0"}, {"chunk": data})) + for field in ("received", "size"): + if isinstance(chunk, dict) and chunk.get(field) == len(data): + chunk[field] = "" + return token + + +def upload_flow(server, step, ids): + status, created = step("history create", "POST", "/v1/projects", "sam", {"title": "History smoke"}) + ids[str(created.get("id", ""))] = "" + path = "/v1/projects/" + asset_name, asset_content = svg_asset("first") + other_name, other_content = svg_asset("second") + plain = sb3_archive(project_with_costumes([])) + + def upload(name, fields=None, files=None, who="sam"): + return step(name, "POST", path + "/upload", who, form=(fields or {}, files if files is not None else {"project": plain})) + + def head_and_edited(): + status, project = server.call("GET", "/v1/projects/" + real_id(ids, ""), "sam")[:2] + return str(project.get("gitHead") or (project.get("project") or {}).get("gitHead") or ""), str(project.get("edited") or (project.get("project") or {}).get("edited") or "") + + upload("upload refused for stranger", who="alice") + upload("upload missing project file", files={}) + upload("upload invalid sb3", files={"project": b"not a zip"}) + upload("upload corrupt sb3", files={"project": corrupt_zip("project.json", json.dumps(project_with_costumes([])))}) + upload("upload without targets", files={"project": sb3_archive(project_with_costumes([], targets=False))}) + upload("upload missing asset", files={"project": sb3_archive(project_with_costumes([asset_name]))}) + upload("upload asset hash mismatch", files={"project": sb3_archive(project_with_costumes([asset_name]), {asset_name: other_content})}) + upload("upload invalid extension sources", {"extensions": "not json"}) + upload("upload unsavable extension url", files={"project": sb3_archive(project_with_costumes([], {"bad": "ftp://example.com/x.js"}))}) + upload("upload unread extension source", files={"project": sb3_archive(project_with_costumes([], {"ext": "https://example.com/ext.js"}))}) + with_asset = sb3_archive(project_with_costumes([asset_name], {"ext": "https://example.com/ext.js"}), {asset_name: asset_content}) + upload("upload with asset and extension", {"extensions": json.dumps({"https://example.com/ext.js": "class SmokeExtension {}"})}, + {"project": with_asset, "thumbnail": b"\x89PNG smoke thumbnail"}) + step("check assets", "POST", path + "/assets/check", "sam", {"assets": [asset_name, other_name]}) + step("check invalid asset name", "POST", path + "/assets/check", "sam", {"assets": ["nope"]}) + sparse = sb3_archive(project_with_costumes([asset_name], {"ext": "https://example.com/ext.js"})) + upload("upload sparse", {"extensions": json.dumps({"https://example.com/ext.js": "class SmokeExtension {}"})}, {"project": sparse}) + upload("upload second asset", files={"project": sb3_archive(project_with_costumes([asset_name, other_name]), {other_name: other_content})}) + wide_name, wide_content = svg_asset("\u00fcn\u00efc\u00f6d\u00e9 \u2713 " * 400) + wide_project = project_with_costumes([asset_name, other_name, wide_name]) + wide_project["targets"][0]["variables"] = {"wide": ["\u00e9\u2713", "\u00e9\u2713\U0001F600" * 3000]} + upload("upload non-ascii project", files={"project": sb3_archive(wide_project, {wide_name: wide_content})}) + big_content = b"\0" * (10 * 1024 * 1024 + 1) + big_name = hashlib.md5(big_content).hexdigest() + ".svg" + upload("upload oversized asset", files={"project": sb3_archive(project_with_costumes([big_name]), {big_name: big_content}, zipfile.ZIP_DEFLATED)}) + with server.faulty("extensions"): + upload("upload extension store failure", {"extensions": json.dumps({"https://example.com/fault.js": "class FaultExtension {}"})}, + {"project": sb3_archive(project_with_costumes([], {"ext": "https://example.com/fault.js"}))}) + unpublished = project_with_costumes([]) + unpublished["meta"]["agent"] = "unpublished" + with server.faulty(os.path.join("blobs", "project-json")): + upload("upload json store failure", files={"project": sb3_archive(unpublished)}) + unsaved = project_with_costumes([]) + unsaved["meta"]["agent"] = "unsaved" + with server.faulty("projects"): + upload("upload metadata save failure", files={"project": sb3_archive(unsaved)}) + + head, edited = head_and_edited() + upload("upload stale edited", {"expectedEdited": "1"}) + upload("upload stale head", {"expectedHead": ZERO_HEAD}) + upload("upload replace without head", {"replaceHistory": "true"}) + upload("upload replace without workspace", {"replaceHistory": "true", "expectedHead": head}) + upload("upload invalid git metadata", {"git": "not json"}) + upload("upload stale delta base", {"git": json.dumps({"delta": True, "baseHead": ZERO_HEAD})}) + upload("upload invalid restore commit", {"restoreCommit": "xyz"}) + upload("upload invalid workspace token", {"workspaceToken": "!!"}) + upload("upload expired workspace token", {"workspaceToken": "abc123"}) + upload("upload invalid merge tree token", {"mergeTreeToken": "!!"}) + upload("upload unsafe workspace", files={"project": plain, "workspace": b"not a zip"}) + missing = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % real_id(ids, ""), "sam")) + missing_head = missing.commit(missing.tree_of(missing.head()), [missing.head()], "Missing worktree asset") + upload("upload missing worktree asset", {"git": json.dumps(missing.metadata(missing_head, False))}, + {"project": plain, "workspace": missing.archive(missing_head, False, {"worktreeAssets": {"costume.svg": "0" * 32 + ".svg"}})}) + unrelated = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % real_id(ids, ""), "sam")) + unrelated_root = unrelated.commit(unrelated.add("tree", b"100644 other.txt\x00" + bytes.fromhex(unrelated.add("blob", b"other"))), [], "Unrelated history") + unrelated.objects = {} + unrelated.refs = {unrelated.branch: unrelated_root} + upload("upload unrelated history", {"git": json.dumps(unrelated.metadata(unrelated_root, False))}, + {"project": plain, "workspace": unrelated.archive(unrelated_root, False)}) + upload("upload replace as editor session", {"replaceHistory": "true", "expectedHead": head}, who="sam_editor") + + unchanged = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % real_id(ids, ""), "sam")) + upload("upload unchanged workspace", {"git": json.dumps(unchanged.metadata(unchanged.head(), False))}, + {"project": plain, "workspace": unchanged.archive(unchanged.head(), False)}) + archive, metadata = committed_workspace(server, ids, "", "sam", "Same tree", same_tree=True) + upload("upload same tree commit", {"git": metadata}, {"project": plain, "workspace": archive}) + archive, metadata = committed_workspace(server, ids, "", "sam", "Full save") + upload("upload full workspace", {"git": metadata}, {"project": plain, "workspace": archive}) + archive, metadata = committed_workspace(server, ids, "", "sam", "Delta save", delta=True) + upload("upload delta workspace", {"git": metadata}, {"project": plain, "workspace": archive}) + archive, metadata = committed_workspace(server, ids, "", "sam", "Staged save", delta=True) + token = staged_part(server, step, path, "sam", "workspace", archive, "stage workspace") + upload("upload staged workspace", {"git": metadata, "workspaceToken": token}) + upload("upload reused workspace token", {"git": metadata, "workspaceToken": token}) + + first = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % real_id(ids, ""), "sam")) + oldest = first.head() + while True: + parents = [line[7:].decode() for line in first.read(oldest)[1].split(b"\n\n", 1)[0].split(b"\n") if line.startswith(b"parent ")] + if not parents: + break + oldest = parents[0] + upload("upload restore", {"restoreCommit": oldest, "restoreMessage": "Restore the first version"}) + upload("upload restore unknown commit", {"restoreCommit": "1" * 40}) + + step("create branch", "POST", path + "/branches", "sam", {"name": "feature"}) + status, listed = step("list branches", "GET", path + "/branches", "sam") + heads = {branch.get("name"): branch.get("head") for branch in (listed.get("branches") or [])} if isinstance(listed, dict) else {} + merged = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % real_id(ids, ""), "sam")) + tree_files = merged.files(merged.tree_of(merged.head())) + tree_files["smoke-merge.txt"] = b"merged" + merge_tree = zip_files(tree_files) + branch_fields = {"mergeSourceBranch": "feature", "mergeTargetBranch": "main", "mergeSourceHead": heads.get("feature", ""), "mergeTargetHead": heads.get("main", "")} + upload("upload branch merge stale", dict(branch_fields, mergeSourceHead=ZERO_HEAD), {"project": plain, "mergeTree": merge_tree}) + upload("upload branch merge same branch", dict(branch_fields, mergeSourceBranch="main", mergeSourceHead=heads.get("main", "")), {"project": plain, "mergeTree": merge_tree}) + upload("upload branch merge invalid tree", branch_fields, {"project": plain, "mergeTree": b"not a zip"}) + upload("upload branch merge corrupt tree", branch_fields, {"project": plain, "mergeTree": corrupt_zip("smoke-merge.txt", "a corrupt merge tree")}) + upload("upload branch merge", branch_fields, {"project": plain, "mergeTree": merge_tree}) + upload("upload pull not open", {"pullId": "99"}, {"project": plain, "mergeTree": merge_tree}) + upload("upload merge tree without target", {}, {"project": plain, "mergeTree": merge_tree}) + + step("share team", "PUT", path, "sam", {"collaborators": [{"username": "alice", "role": "editor"}]}) + upload("upload live editor without edited", who="alice") + upload("upload live editor history refused", {"expectedEdited": "1", "restoreCommit": oldest}, who="alice") + head, edited = head_and_edited() + upload("upload live editor", {"expectedEdited": edited}, who="alice") + + step("share history", "POST", path + "/visibility", "sam", {"visibility": "public"}) + status, remixed = step("remix history", "POST", path + "/remix", "alice", {}) + remix_id = str(remixed.get("id", "") or (remixed.get("project") or {}).get("id", "")) if isinstance(remixed, dict) else "" + ids[remix_id] = "" + archive, metadata = committed_workspace(server, ids, "", "alice", "Remix change") + step("upload remix change", "POST", "/v1/projects//upload", "alice", form=({"git": metadata}, {"project": plain, "workspace": archive})) + status, pull = step("open pull", "POST", path + "/pulls", "alice", {"sourceProjectId": remix_id, "title": "Smoke pull", "body": "From the remix"}) + pull_index = str(((pull.get("pull") or {}).get("index", "")) if isinstance(pull, dict) else "") + status, bob_remixed = step("remix history for bob", "POST", path + "/remix", "bob", {}) + bob_remix_id = str(bob_remixed.get("id", "") or (bob_remixed.get("project") or {}).get("id", "")) if isinstance(bob_remixed, dict) else "" + ids[bob_remix_id] = "" + archive, metadata = committed_workspace(server, ids, "", "bob", "Bob change") + step("upload bob remix change", "POST", "/v1/projects//upload", "bob", form=({"git": metadata}, {"project": plain, "workspace": archive})) + status, bob_pull = step("open bob pull", "POST", path + "/pulls", "bob", {"sourceProjectId": bob_remix_id, "title": "Bob pull", "body": "From a deleted remix"}) + bob_pull_index = str(((bob_pull.get("pull") or {}).get("index", "")) if isinstance(bob_pull, dict) else "") + bob_remix = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % bob_remix_id, "bob")) + bob_tree = zip_files(bob_remix.files(bob_remix.tree_of(bob_remix.head()))) + step("delete bob remix", "DELETE", "/v1/projects/", "bob") + head, edited = head_and_edited() + upload("upload pull merge from deleted remix", {"pullId": bob_pull_index, "expectedHead": head}, {"project": plain, "mergeTree": bob_tree}) + remix = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % remix_id, "alice")) + pull_tree = zip_files(remix.files(remix.tree_of(remix.head()))) + head, edited = head_and_edited() + upload("upload pull merge stale head", {"pullId": pull_index, "expectedHead": ZERO_HEAD}, {"project": plain, "mergeTree": pull_tree}) + tree_token = staged_part(server, step, path, "sam", "mergeTree", pull_tree, "stage merge tree") + upload("upload pull merge with unfinished tree", {"pullId": pull_index, "expectedHead": head, "mergeTreeToken": tree_token}) + upload("upload pull merge", {"pullId": pull_index, "expectedHead": head}, {"project": plain, "mergeTree": pull_tree}) + step("merged pull", "GET", path + "/pulls/" + pull_index, "sam") + upload("upload merged pull again", {"pullId": pull_index}, {"project": plain, "mergeTree": pull_tree}) + + head, edited = head_and_edited() + fresh = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % real_id(ids, ""), "sam")) + root_tree = fresh.add("tree", b"100644 replaced.txt\x00" + bytes.fromhex(fresh.add("blob", b"replaced"))) + root = fresh.commit(root_tree, [], "Replaced history") + fresh.objects = {} + fresh.refs = {fresh.branch: root} + upload("upload replace history", {"replaceHistory": "true", "expectedHead": head, "git": json.dumps(fresh.metadata(root, False))}, + {"project": plain, "workspace": fresh.archive(root, False)}) + step("history after replace", "GET", path + "/branches", "sam") + step("history quota", "GET", "/v1/me/quota", "sam") + step("get history project", "GET", path, "sam") + + +def update_flow(server, step, ids): + status, created = step("update create", "POST", "/v1/projects", "sam", {"title": "Update smoke"}) + ids[str(created.get("id", ""))] = "" + path = "/v1/projects/" + + def patch(name, body, who="sam"): + return step(name, "PUT", path, who, body) + + patch("patch anonymous", {"title": "x"}, who=None) + patch("patch by stranger", {"title": "x"}, who="alice") + patch("patch text", {"title": " Updated smoke ", "description": "About", "instructions": "Press space", "notes": "Thanks"}) + patch("patch empty title", {"title": " "}) + patch("patch long title", {"title": "t" * 101}) + patch("patch long description", {"description": "d" * 5001}) + patch("patch long instructions", {"instructions": "i" * 5001}) + patch("patch long notes", {"notes": "n" * 5001}) + patch("patch non-ascii description", {"description": "\u00e9\u2713" * 2400}) + patch("patch long non-ascii notes", {"notes": "\u00e9" * 5001}) + patch("patch credits", {"credits": [ + {"who": "Ada", "role": "Art", "url": "https://example.com/ada"}, + {"who": "Bo", "role": "Music", "url": None}, + {"who": "", "role": "skipped"}, + {"who": "w" * 61, "role": "skipped"}, + ]}) + patch("patch credit bad url", {"credits": [{"who": "Ada", "role": "Art", "url": "ftp://example.com"}]}) + patch("patch credit long url", {"credits": [{"who": "Ada", "role": "Art", "url": "https://example.com/" + "u" * 500}]}) + patch("patch too many credits", {"credits": [{"who": "A", "role": "B"}] * 51}) + patch("patch flags", {"commentsOff": True, "remixable": False, "seeInside": True}) + patch("patch price", {"price": 5}) + patch("patch zero price", {"price": 0, "remixable": True}) + patch("patch remix share", {"remixShare": 20}) + patch("patch remix share too high", {"remixShare": 80}) + patch("patch tags", {"tags": ["Games", "games", "Puzzle", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]}) + items = [ + {"id": "Sword", "name": "Sword", "visual": {"type": "image", "url": "https://example.com/s.png"}}, + {"id": "gem", "name": "Gem", "image": "https://example.com/g.png"}, + {"id": "hat", "name": "Hat", "visual": {"type": "costume", "assetId": "abcdefgh12", "dataFormat": "PNG"}, "gameAwardable": False}, + {"id": "pixel", "name": "Pixel", "visual": {"type": "image", "url": "data:image/png;base64,AAAA"}}, + ] + product = {"id": "pack", "name": "Pack", "description": "Starter", "price": 10, "grantsItem": "sword", "icon": "https://example.com/i.png"} + patch("patch game items and products", {"gameItems": items, "gameProducts": [product]}) + patch("patch product bad id", {"gameProducts": [dict(product, id="Bad Id")]}) + patch("patch product duplicate id", {"gameProducts": [product, product]}) + patch("patch product bad name", {"gameProducts": [dict(product, name="")]}) + patch("patch product bad price", {"gameProducts": [dict(product, price=2000)]}) + patch("patch product bad icon", {"gameProducts": [dict(product, icon="ftp://example.com/i.png")]}) + patch("patch product big icon", {"gameProducts": [dict(product, icon="data:image/png;base64," + "A" * 100001)]}) + patch("patch product bad grant", {"gameProducts": [dict(product, grantsItem="BAD!")]}) + patch("patch product undefined grant", {"gameProducts": [dict(product, grantsItem="ghost")]}) + patch("patch too many products", {"gameProducts": [product] * 51}) + patch("patch item bad id", {"gameItems": [dict(items[0], id="Bad Id")]}) + patch("patch item bad name", {"gameItems": [dict(items[0], name="")]}) + patch("patch item http image", {"gameItems": [dict(items[0], visual={"type": "image", "url": "http://example.com/s.png"})]}) + patch("patch item big data image", {"gameItems": [dict(items[0], visual={"type": "image", "url": "data:image/png;base64," + "A" * 100001})]}) + patch("patch item bad costume", {"gameItems": [dict(items[0], visual={"type": "costume", "assetId": "short", "dataFormat": "png"})]}) + patch("patch item without visual", {"gameItems": [{"id": "plain", "name": "Plain"}]}) + patch("patch too many items", {"gameItems": [items[1]] * 101}) + patch("patch items dropping a granted item", {"gameItems": [items[1]]}) + patch("patch inventory bad mode", {"inventoryPolicy": {"mode": "some"}}) + patch("patch inventory too large", {"inventoryPolicy": {"mode": "all", "allowedItems": ["p:x"] * 101}}) + patch("patch inventory bad item", {"inventoryPolicy": {"mode": "allowlist", "allowedItems": ["nope"]}}) + patch("patch inventory bad project", {"inventoryPolicy": {"mode": "allowlist", "allowedProjects": ["../x"]}}) + patch("patch inventory", {"inventoryPolicy": {"mode": "Allowlist", "allowedItems": ["p1:sword", "p1:sword"], "allowedProjects": ["p1", "p1"]}}) + patch("patch compatibility", {"compatibility": {"mobile": True, "keyboard": False, "controller": "yes"}}) + patch("patch branding bad accent", {"branding": {"accentColor": "red"}}) + patch("patch branding long tagline", {"branding": {"tagline": "t" * 121}}) + patch("patch branding without plan", {"branding": {"accentColor": "#123456", "tagline": "Hi"}}) + patch("patch branding cleared", {"branding": {}}) + patch("patch vanity without plan", {"vanitySlug": "smoke-game"}) + patch("patch vanity cleared", {"vanitySlug": ""}) + patch("patch group tag invalid", {"groupTag": "!!!"}) + patch("patch group tag not managed", {"groupTag": "somegroup"}) + patch("patch group tag cleared", {"groupTag": " "}) + patch("patch team share too high", {"collaborators": [{"username": "alice", "role": "editor", "share": 60}]}) + patch("patch team unknown user", {"collaborators": [{"username": "ghost_user", "role": "editor"}]}) + patch("patch team duplicate", {"collaborators": [{"username": "alice", "role": "editor"}, {"username": "Alice", "role": "tester"}]}) + patch("patch team total share", {"collaborators": [{"username": "alice", "role": "editor", "share": 40}, {"username": "bob", "role": "tester", "share": 40}]}) + patch("patch team", {"collaborators": [ + {"username": "alice", "role": "Maintainer", "share": 10}, + {"username": "bob", "role": "tester"}, + {"username": "sam", "role": "editor"}, + {"username": "carol", "role": "owner"}, + ]}) + patch("patch team same roles", {"collaborators": [{"username": "alice", "role": "maintainer", "share": 10}, {"username": "bob", "role": "editor"}]}) + step("alice notifications", "GET", "/v1/notifications", "alice") + step("bob notifications", "GET", "/v1/notifications", "bob") + patch("patch remix share as maintainer", {"remixShare": 5}, who="alice") + patch("patch team as maintainer", {"collaborators": []}, who="alice") + patch("patch title as maintainer", {"title": "Maintained"}, who="alice") + patch("patch editor session allowed", {"title": "From the editor"}, who="sam_editor") + patch("patch editor session refused", {"collaborators": []}, who="sam_editor") + patch("patch transfer as maintainer", {"owner": "bob"}, who="alice") + patch("patch transfer unknown", {"owner": "ghost_user"}) + patch("patch transfer", {"owner": "bob", "keepGroup": True, "title": "Given away"}) + patch("patch after transfer", {"title": "Mine again"}) + step("get updated project", "GET", path, "bob") + + +def volatile_key(key): + return key in VOLATILE_NAMES + + +def normalize(value, ids, key="", parent=""): if isinstance(value, dict): - return {k: normalize(v, ids, k) for k, v in sorted(value.items())} + return {k: normalize(v, ids, k, key) for k, v in sorted(value.items())} if isinstance(value, list): - return [normalize(v, ids, key) for v in value] + return [normalize(v, ids, key, parent) for v in value] + if (parent, key) in VOLATILE_FIELDS: + return "" if isinstance(value, str): value = re.sub(r"127\.0\.0\.1:\d+", "127.0.0.1:", value) for real, alias in ids.items(): value = value.replace(real, alias) - if HEX.match(value) or ISO.match(value): - return "" - if VOLATILE_KEYS.search(key) and not value.startswith("" - return value + value = ACCESS_KEY.sub(r"\1", value) + if CONTENT_NAME.match(value): + return value + return EMBEDDED_TIME.sub("