diff --git a/scripts/e2e_smoke.py b/scripts/e2e_smoke.py index ed88172..6031f02 100644 --- a/scripts/e2e_smoke.py +++ b/scripts/e2e_smoke.py @@ -31,9 +31,15 @@ USERS = {"sam": "u_sam", "alice": "u_alice", "bob": "u_bob", "dee": "u_dee"} FULL_STORAGE = {"u_dee"} -EXPECTED_ERRORS = {"upload extension store failure", "upload json store failure", "upload metadata save failure"} +GONE_USERS = {"gone_user"} +BUSY_USERS = {"busy_user"} +BANNED_USERS = {"banned_user"} +SLOW_USERS = {"slow_user": "u_slow"} +ROTUR_DELAY = 4 +EXPECTED_ERRORS = {"patch team unknown user", "patch transfer unknown", "patch team rotur unavailable", "patch transfer rotur unavailable", "upload unsaved first save of the day", "upload extension store failure", "upload extension store failure with history", "upload json store failure", "upload metadata save failure"} SCOPED_SESSIONS = {"sam_editor": ("sam", "editor")} VOLATILE_NAMES = {"token", "viewKey", "expires", "created", "edited", "firstSeen", "historyBytes"} +UNORDERED = {"largest"} VOLATILE_FIELDS = {("notifications", "id"), ("largest", "bytes"), ("", "mistwarp")} SAVE_PATH = re.compile(r"^/v1/projects/([^/]+)(/upload)?$") HEX = re.compile(r"^[0-9a-f]{32,64}$") @@ -70,6 +76,19 @@ def do_GET(self): self.answer(404, {"code": "not_found"}) return name = self.path.split("?", 1)[0][len("/profile/"):] if self.path.startswith("/profile/") else "" + if name in GONE_USERS: + self.answer(404, {"error": "User not found"}) + return + if name in BUSY_USERS: + self.answer(503, {"error": "Service unavailable"}) + return + if name in BANNED_USERS: + self.answer(403, {"error": "User is banned"}) + return + if name in SLOW_USERS: + time.sleep(ROTUR_DELAY) + self.answer(200, {"id": SLOW_USERS[name], "username": name}) + return if name not in USERS: self.close_connection = True return @@ -155,7 +174,7 @@ def stop(self): if not self.keep: shutil.rmtree(self.dir) - def call(self, method, path, who=None, body=None, raw=None, content_type="application/json"): + def call(self, method, path, who=None, body=None, raw=None, content_type="application/json", timeout=30): data = raw if raw is not None else (None if body is None else json.dumps(body).encode("utf-8")) request = urllib.request.Request(self.base + path, data=data, method=method) if who: @@ -163,10 +182,12 @@ def call(self, method, path, who=None, body=None, raw=None, content_type="applic if data is not None: request.add_header("Content-Type", content_type) try: - with urllib.request.urlopen(request, timeout=30) as response: + with urllib.request.urlopen(request, timeout=timeout) as response: status, text, headers = response.status, response.read(), response.headers except urllib.error.HTTPError as error: status, text, headers = error.code, error.read(), error.headers + except (TimeoutError, socket.timeout, urllib.error.URLError) as error: + return 599, {"timedOut": str(error)}, {} if text[:2] == b"\x1f\x8b": text = gzip.decompress(text) try: @@ -178,7 +199,7 @@ def call(self, method, path, who=None, body=None, raw=None, content_type="applic def upload(self, project_id, who, sb3): return self.form("/v1/projects/%s/upload" % project_id, who, {}, {"project": sb3}) - def form(self, path, who, fields, files): + def form(self, path, who, fields, files, timeout=30): boundary = uuid.uuid4().hex body = b"" for name, value in fields.items(): @@ -187,17 +208,40 @@ def form(self, path, who, fields, files): body += ("--%s\r\nContent-Disposition: form-data; name=\"%s\"; filename=\"%s.bin\"\r\n" "Content-Type: application/octet-stream\r\n\r\n" % (boundary, name, name)).encode() + value + b"\r\n" body += ("--%s--\r\n" % boundary).encode() - return self.call("POST", path, who, raw=body, content_type="multipart/form-data; boundary=" + boundary) + return self.call("POST", path, who, raw=body, content_type="multipart/form-data; boundary=" + boundary, timeout=timeout) def faulty(self, folder): path = os.path.join(self.dir, "data", folder) os.makedirs(path, exist_ok=True) return Fault(path) + def thumbnail(self, project_id): + path = os.path.join(self.dir, "data", "thumbnails", project_id + ".png") + if not os.path.exists(path): + return None + with open(path, "rb") as handle: + return handle.read().decode("latin-1") + + def profile(self, username): + path = os.path.join(self.dir, "data", "users", username + ".json") + if not os.path.exists(path): + return {} + with open(path) as handle: + return json.load(handle) + def read_project(self, project_id): with open(os.path.join(self.dir, "data", "projects", project_id + ".json")) as handle: return json.load(handle) + def extension_records(self): + folder = os.path.join(self.dir, "data", "extensions") + records = {} + for name in sorted(os.listdir(folder)) if os.path.isdir(folder) else []: + if name.endswith(".json") and name != "index.json": + with open(os.path.join(folder, name)) as handle: + records[name] = json.load(handle) + return records + def blob_inventory(self): inventory = {} root = os.path.join(self.dir, "data", "blobs") @@ -372,17 +416,28 @@ def committed_workspace(server, ids, alias, who, message, delta=False, same_tree return history.archive(head, delta), json.dumps(history.metadata(head, delta)) -def staged_part(server, step, path, who, kind, data, name): - status, started = step(name + " start", "POST", path + "/upload-parts", who, {"kind": kind, "name": "", "size": len(data), "chunks": 1}) +def staged_part(server, step, path, who, kind, data, name, chunks=1): + status, started = step(name + " start", "POST", path + "/upload-parts", who, {"kind": kind, "name": "", "size": len(data), "chunks": chunks}) token = started.get("token", "") if isinstance(started, dict) else "" - status, chunk = step(name + " chunk", "POST", path + "/upload-parts/%s/chunk" % token, who, form=({"index": "0"}, {"chunk": data})) - for field in ("received", "size"): - if isinstance(chunk, dict) and chunk.get(field) == len(data): - chunk[field] = "" + cut = (len(data) + chunks - 1) // chunks + for index in range(chunks): + part = data[index * cut:(index + 1) * cut] + status, chunk = step("%s chunk %d" % (name, index), "POST", path + "/upload-parts/%s/chunk" % token, who, form=({"index": str(index)}, {"chunk": part})) + for field, sent in (("received", min(len(data), (index + 1) * cut)), ("size", len(data))): + if isinstance(chunk, dict) and chunk.get(field) == sent: + chunk[field] = "" return token -def upload_flow(server, step, ids): +def complete_part(step, path, who, token, name): + step(name, "POST", path + "/upload-parts/%s/complete" % token, who, {}) + + +def upload_flow(server, step, ids, steps): + status, bare = step("create project for an sb3 without project.json", "POST", "/v1/projects", "sam", {"title": "No project.json"}) + ids[str(bare.get("id", "")) if isinstance(bare, dict) else ""] = "" + for name, sb3 in (("upload sb3 without project.json", sb3_archive(None, {"readme.txt": "no project here"})), ("upload after an sb3 without project.json", sb3_archive(project_with_costumes([])))): + step(name, "POST", "/v1/projects//upload", "sam", form=({}, {"project": sb3}), timeout=5) status, created = step("history create", "POST", "/v1/projects", "sam", {"title": "History smoke"}) ids[str(created.get("id", ""))] = "" path = "/v1/projects/" @@ -410,8 +465,10 @@ def head_and_edited(): with_asset = sb3_archive(project_with_costumes([asset_name], {"ext": "https://example.com/ext.js"}), {asset_name: asset_content}) upload("upload with asset and extension", {"extensions": json.dumps({"https://example.com/ext.js": "class SmokeExtension {}"})}, {"project": with_asset, "thumbnail": b"\x89PNG smoke thumbnail"}) + steps.append({"step": "extension records after an upload", "status": 0, "body": server.extension_records()}) step("check assets", "POST", path + "/assets/check", "sam", {"assets": [asset_name, other_name]}) step("check invalid asset name", "POST", path + "/assets/check", "sam", {"assets": ["nope"]}) + step("check assets not a list", "POST", path + "/assets/check", "sam", {"assets": "x"}) sparse = sb3_archive(project_with_costumes([asset_name], {"ext": "https://example.com/ext.js"})) upload("upload sparse", {"extensions": json.dumps({"https://example.com/ext.js": "class SmokeExtension {}"})}, {"project": sparse}) upload("upload second asset", files={"project": sb3_archive(project_with_costumes([asset_name, other_name]), {other_name: other_content})}) @@ -425,14 +482,23 @@ def head_and_edited(): with server.faulty("extensions"): upload("upload extension store failure", {"extensions": json.dumps({"https://example.com/fault.js": "class FaultExtension {}"})}, {"project": sb3_archive(project_with_costumes([], {"ext": "https://example.com/fault.js"}))}) + workspaces_before = server.blob_inventory().get("project-workspaces", 0) + archive, metadata = committed_workspace(server, ids, "", "sam", "Extension store failure") + with server.faulty("extensions"): + upload("upload extension store failure with history", {"git": metadata, "extensions": json.dumps({"https://example.com/fault.js": "class FaultExtension {}"})}, + {"project": sb3_archive(project_with_costumes([], {"ext": "https://example.com/fault.js"})), "workspace": archive}) + steps.append({"step": "history left by the extension store failure", "status": 0, "body": { + "newWorkspaceBlobs": server.blob_inventory().get("project-workspaces", 0) - workspaces_before, + }}) unpublished = project_with_costumes([]) unpublished["meta"]["agent"] = "unpublished" with server.faulty(os.path.join("blobs", "project-json")): - upload("upload json store failure", files={"project": sb3_archive(unpublished)}) + upload("upload json store failure", files={"project": sb3_archive(unpublished), "thumbnail": b"\x89PNG unpublished thumbnail"}) unsaved = project_with_costumes([]) unsaved["meta"]["agent"] = "unsaved" with server.faulty("projects"): - upload("upload metadata save failure", files={"project": sb3_archive(unsaved)}) + upload("upload metadata save failure", files={"project": sb3_archive(unsaved), "thumbnail": b"\x89PNG unsaved thumbnail"}) + steps.append({"step": "thumbnail after failed uploads", "status": 0, "body": server.thumbnail(real_id(ids, ""))}) head, edited = head_and_edited() upload("upload stale edited", {"expectedEdited": "1"}) @@ -468,9 +534,11 @@ def head_and_edited(): archive, metadata = committed_workspace(server, ids, "", "sam", "Delta save", delta=True) upload("upload delta workspace", {"git": metadata}, {"project": plain, "workspace": archive}) archive, metadata = committed_workspace(server, ids, "", "sam", "Staged save", delta=True) - token = staged_part(server, step, path, "sam", "workspace", archive, "stage workspace") + token = staged_part(server, step, path, "sam", "workspace", archive, "stage workspace", chunks=2) + upload("upload unfinished staged workspace", {"git": metadata, "workspaceToken": token}) + complete_part(step, path, "sam", token, "complete staged workspace") upload("upload staged workspace", {"git": metadata, "workspaceToken": token}) - upload("upload reused workspace token", {"git": metadata, "workspaceToken": token}) + upload("upload reused workspace token", {"workspaceToken": token}) first = History(raw_get(server, "/v1/projects/%s/workspace.mwp" % real_id(ids, ""), "sam")) oldest = first.head() @@ -512,6 +580,13 @@ def head_and_edited(): step("upload remix change", "POST", "/v1/projects//upload", "alice", form=({"git": metadata}, {"project": plain, "workspace": archive})) status, pull = step("open pull", "POST", path + "/pulls", "alice", {"sourceProjectId": remix_id, "title": "Smoke pull", "body": "From the remix"}) pull_index = str(((pull.get("pull") or {}).get("index", "")) if isinstance(pull, dict) else "") + status, bob_project = step("bob create", "POST", "/v1/projects", "bob", {"title": "Bob smoke"}) + ids[str(bob_project.get("id", "")) if isinstance(bob_project, dict) else ""] = "" + with server.faulty("projects"): + step("upload unsaved first save of the day", "POST", "/v1/projects//upload", "bob", form=({}, {"project": sb3_archive(unsaved), "thumbnail": b"\x89PNG bob thumbnail"})) + steps.append({"step": "after an unsaved first save of the day", "status": 0, "body": { + "thumbnail": server.thumbnail(real_id(ids, "")), "creatingStreak": server.profile("bob").get("creatingStreak"), + }}) status, bob_remixed = step("remix history for bob", "POST", path + "/remix", "bob", {}) bob_remix_id = str(bob_remixed.get("id", "") or (bob_remixed.get("project") or {}).get("id", "")) if isinstance(bob_remixed, dict) else "" ids[bob_remix_id] = "" @@ -530,7 +605,8 @@ def head_and_edited(): upload("upload pull merge stale head", {"pullId": pull_index, "expectedHead": ZERO_HEAD}, {"project": plain, "mergeTree": pull_tree}) tree_token = staged_part(server, step, path, "sam", "mergeTree", pull_tree, "stage merge tree") upload("upload pull merge with unfinished tree", {"pullId": pull_index, "expectedHead": head, "mergeTreeToken": tree_token}) - upload("upload pull merge", {"pullId": pull_index, "expectedHead": head}, {"project": plain, "mergeTree": pull_tree}) + complete_part(step, path, "sam", tree_token, "complete staged merge tree") + upload("upload pull merge", {"pullId": pull_index, "expectedHead": head, "mergeTreeToken": tree_token}) step("merged pull", "GET", path + "/pulls/" + pull_index, "sam") upload("upload merged pull again", {"pullId": pull_index}, {"project": plain, "mergeTree": pull_tree}) @@ -557,6 +633,8 @@ def patch(name, body, who="sam"): patch("patch anonymous", {"title": "x"}, who=None) patch("patch by stranger", {"title": "x"}, who="alice") + patch("patch null title", {"title": None}) + patch("patch null description", {"description": None, "notes": None}) patch("patch text", {"title": " Updated smoke ", "description": "About", "instructions": "Press space", "notes": "Thanks"}) patch("patch empty title", {"title": " "}) patch("patch long title", {"title": "t" * 101}) @@ -565,6 +643,7 @@ def patch(name, body, who="sam"): patch("patch long notes", {"notes": "n" * 5001}) patch("patch non-ascii description", {"description": "\u00e9\u2713" * 2400}) patch("patch long non-ascii notes", {"notes": "\u00e9" * 5001}) + patch("patch credit without role", {"credits": [{"who": "Ada"}, {"who": None, "role": "Art"}]}) patch("patch credits", {"credits": [ {"who": "Ada", "role": "Art", "url": "https://example.com/ada"}, {"who": "Bo", "role": "Music", "url": None}, @@ -579,6 +658,7 @@ def patch(name, body, who="sam"): patch("patch zero price", {"price": 0, "remixable": True}) patch("patch remix share", {"remixShare": 20}) patch("patch remix share too high", {"remixShare": 80}) + patch("patch null tag", {"tags": [None, "Games"]}) patch("patch tags", {"tags": ["Games", "games", "Puzzle", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]}) items = [ {"id": "Sword", "name": "Sword", "visual": {"type": "image", "url": "https://example.com/s.png"}}, @@ -587,6 +667,7 @@ def patch(name, body, who="sam"): {"id": "pixel", "name": "Pixel", "visual": {"type": "image", "url": "data:image/png;base64,AAAA"}}, ] product = {"id": "pack", "name": "Pack", "description": "Starter", "price": 10, "grantsItem": "sword", "icon": "https://example.com/i.png"} + patch("patch product without icon", {"gameItems": items, "gameProducts": [{"id": "plain", "name": "Plain", "price": 1}]}) patch("patch game items and products", {"gameItems": items, "gameProducts": [product]}) patch("patch product bad id", {"gameProducts": [dict(product, id="Bad Id")]}) patch("patch product duplicate id", {"gameProducts": [product, product]}) @@ -616,10 +697,13 @@ def patch(name, body, who="sam"): patch("patch branding without plan", {"branding": {"accentColor": "#123456", "tagline": "Hi"}}) patch("patch branding cleared", {"branding": {}}) patch("patch vanity without plan", {"vanitySlug": "smoke-game"}) + patch("patch null vanity", {"vanitySlug": None}) patch("patch vanity cleared", {"vanitySlug": ""}) patch("patch group tag invalid", {"groupTag": "!!!"}) patch("patch group tag not managed", {"groupTag": "somegroup"}) + patch("patch null group tag", {"groupTag": None}) patch("patch group tag cleared", {"groupTag": " "}) + patch("patch nameless teammate", {"collaborators": [{"role": "editor"}]}) patch("patch team share too high", {"collaborators": [{"username": "alice", "role": "editor", "share": 60}]}) patch("patch team unknown user", {"collaborators": [{"username": "ghost_user", "role": "editor"}]}) patch("patch team duplicate", {"collaborators": [{"username": "alice", "role": "editor"}, {"username": "Alice", "role": "tester"}]}) @@ -643,17 +727,90 @@ def patch(name, body, who="sam"): patch("patch transfer", {"owner": "bob", "keepGroup": True, "title": "Given away"}) patch("patch after transfer", {"title": "Mine again"}) step("get updated project", "GET", path, "bob") + for name, body in ( + ("patch credits not a list", {"credits": "x"}), + ("patch credit not an object", {"credits": ["Ada"]}), + ("patch tags not a list", {"tags": "games"}), + ("patch compatibility not an object", {"compatibility": "mobile"}), + ("patch team not a list", {"collaborators": "alice"}), + ("patch teammate not an object", {"collaborators": ["alice"]}), + ("patch after bad input", {"title": "Still saving"}), + ): + step(name, "PUT", path, "bob", body, timeout=5) + patch("patch team deleted user", {"collaborators": [{"username": "gone_user", "role": "editor"}]}, who="bob") + patch("patch transfer to deleted user", {"owner": "gone_user"}, who="bob") + patch("patch team rotur unavailable", {"collaborators": [{"username": "busy_user", "role": "editor"}]}, who="bob") + patch("patch transfer rotur unavailable", {"owner": "busy_user"}, who="bob") + patch("patch team banned user", {"collaborators": [{"username": "banned_user", "role": "editor"}]}, who="bob") + patch("patch transfer to banned user", {"owner": "banned_user"}, who="bob") + + +def slow_rotur_flow(server, step, steps, ids): + status, created = step("slow rotur create", "POST", "/v1/projects", "sam", {"title": "Slow Rotur smoke"}) + project_id = str(created.get("id", "")) if isinstance(created, dict) else "" + ids[project_id] = "" + results = {} + + def save_team(): + results["teamSave"] = server.call("PUT", "/v1/projects/" + project_id, "sam", {"collaborators": [{"username": "slow_user", "role": "editor"}]})[0] + + saving = threading.Thread(target=save_team) + saving.start() + time.sleep(0.5) + waited = {} + + def timed(name, send): + started = time.time() + results[name] = send() + waited[name] = time.time() - started >= ROTUR_DELAY / 2 + + others = [ + threading.Thread(target=timed, args=("rename", lambda: server.call("PUT", "/v1/projects/" + project_id, "sam", {"title": "Renamed during a slow lookup"})[0])), + threading.Thread(target=timed, args=("upload", lambda: server.upload(project_id, "sam", sb3_archive(project_with_costumes([])))[0])), + ] + for other in others: + other.start() + for other in others + [saving]: + other.join() + stored = server.read_project(project_id) + steps.append({"step": "saves during a slow Rotur lookup", "status": 0, "body": { + "statuses": results, "waitedForRotur": waited, "team": stored.get("collaborators"), "title": stored.get("title"), + }}) def volatile_key(key): return key in VOLATILE_NAMES +def concurrent_flow(server, step, steps, ids): + status, created = step("concurrent create", "POST", "/v1/projects", "sam", {"title": "Concurrent smoke"}) + project_id = str(created.get("id", "")) if isinstance(created, dict) else "" + ids[project_id] = "" + content = b"" + b" " * (9 * 1024 * 1024) + name = hashlib.md5(content).hexdigest() + ".svg" + sb3 = sb3_archive(project_with_costumes([name]), {name: content}) + results = {} + + def upload(): + results["upload"] = server.form("/v1/projects/%s/upload" % project_id, "sam", {}, {"project": sb3})[0] + + uploading = threading.Thread(target=upload) + uploading.start() + time.sleep(0.05) + results["update"] = server.call("PUT", "/v1/projects/" + project_id, "sam", {"title": "Renamed during upload"})[0] + uploading.join() + stored = server.read_project(project_id) + steps.append({"step": "concurrent update and upload", "status": 0, "body": { + "statuses": results, "title": stored.get("title"), "assets": stored.get("assets"), + }}) + + def normalize(value, ids, key="", parent=""): if isinstance(value, dict): return {k: normalize(v, ids, k, key) for k, v in sorted(value.items())} if isinstance(value, list): - return [normalize(v, ids, key, parent) for v in value] + items = [normalize(v, ids, key, parent) for v in value] + return sorted(items, key=json.dumps) if key in UNORDERED else items if (parent, key) in VOLATILE_FIELDS: return "" if isinstance(value, str): @@ -677,7 +834,7 @@ def run(binary, sb3, keep): ids = {} steps = [] - def step(name, method, path, who=None, body=None, upload=None, headers=(), form=None): + def step(name, method, path, who=None, body=None, upload=None, headers=(), form=None, timeout=30): real_path = path for alias, real in ((a, r) for r, a in ids.items()): real_path = real_path.replace(alias, real) @@ -685,9 +842,9 @@ def step(name, method, path, who=None, body=None, upload=None, headers=(), form= real_path = "/v1/projects/%s/upload" % upload status, parsed, response_headers = server.upload(upload, who, sb3) elif form is not None: - status, parsed, response_headers = server.form(real_path, who, form[0], form[1]) + status, parsed, response_headers = server.form(real_path, who, form[0], form[1], timeout) else: - status, parsed, response_headers = server.call(method, real_path, who, body) + status, parsed, response_headers = server.call(method, real_path, who, body, timeout=timeout) record = {"step": name, "status": status, "body": parsed} for header in headers: record[header] = response_headers.get(header, "") @@ -739,12 +896,15 @@ def step(name, method, path, who=None, body=None, upload=None, headers=(), form= step("wrong method", "PATCH", "/v1/projects/", "sam", {}, headers=("Allow",)) step("legacy prefix", "GET", "/api/agreement") step("health", "GET", "/v1/health") - upload_flow(server, step, ids) + upload_flow(server, step, ids, steps) update_flow(server, step, ids) + concurrent_flow(server, step, steps, ids) + slow_rotur_flow(server, step, steps, ids) time.sleep(1.5) for real, alias in list(ids.items()): if alias in ("", "", ""): steps.append({"step": "stored " + alias, "status": 0, "body": server.read_project(real)}) + steps.append({"step": "extension records", "status": 0, "body": server.extension_records()}) steps.append({"step": "blob inventory", "status": 0, "body": server.blob_inventory()}) finally: server.stop() diff --git a/src/auth/identity-lookup-policy.osl b/src/auth/identity-lookup-policy.osl new file mode 100644 index 0000000..ac4ac1f --- /dev/null +++ b/src/auth/identity-lookup-policy.osl @@ -0,0 +1,11 @@ +export {} + +def profileLookupOutcome(number status) string ( + if status == 200 ( + return "found" + ) + if status == 0 or status == 408 or status == 429 or status >= 500 ( + return "unavailable" + ) + return "missing" +) diff --git a/src/auth/identity-lookup-policy.test.osl b/src/auth/identity-lookup-policy.test.osl new file mode 100644 index 0000000..f6cca17 --- /dev/null +++ b/src/auth/identity-lookup-policy.test.osl @@ -0,0 +1,9 @@ +import "std:testing" +import "./identity-lookup-policy.osl" + +testing.assert(profileLookupOutcome(200) == "found", "a 200 profile is a found user") +testing.assert(profileLookupOutcome(404) == "missing", "a 404 means the user is gone") +testing.assert(profileLookupOutcome(400) == "missing" and profileLookupOutcome(403) == "missing", "a malformed or banned username is not found") +testing.assert(profileLookupOutcome(0) == "unavailable", "an unreachable Rotur is unavailable") +testing.assert(profileLookupOutcome(408) == "unavailable" and profileLookupOutcome(429) == "unavailable", "a timeout or rate limit is unavailable") +testing.assert(profileLookupOutcome(500) == "unavailable" and profileLookupOutcome(502) == "unavailable" and profileLookupOutcome(503) == "unavailable", "server errors are unavailable") diff --git a/src/auth/user-identity.osl b/src/auth/user-identity.osl index 28121b0..03af783 100644 --- a/src/auth/user-identity.osl +++ b/src/auth/user-identity.osl @@ -1,5 +1,6 @@ export { discoverUserIdentity, + discoverUserIdentityOutcome, identityMatches, loadUserIds, purgeUser, @@ -528,12 +529,17 @@ def refreshUsernameForId(string userId) string ( ) def discoverUserIdentity(string username) string ( + return discoverUserIdentityOutcome(username).id.toStr() +) + +def discoverUserIdentityOutcome(string username) object ( string key = normalizeUsername(username) if key == "" or isStudentUsername(key) ( - return "" + return {id: "", outcome: "missing"} ) object resp = requests.get(roturBase ++ "/profile/" ++ url.escape(username) ++ "?include_posts=0", roturServiceOptions()).unwrapOr({body: null, error: "request failed", headers: {}, raw: null, status: 0, success: false}) - if resp.status == 0 ( + string outcome = profileLookupOutcome(resp.status) + if outcome != "found" ( if resp.status == 404 ( string uid = userIdForUsername(username) if uid != "" ( @@ -541,21 +547,21 @@ def discoverUserIdentity(string username) string ( refreshUsernameForId(uid) ) ) - return "" + return {id: "", outcome: outcome} ) auto parsed = json.parse(resp.body, {type: "object"}) if parsed.isErr() ( - return "" + return {id: "", outcome: "unavailable"} ) object profile = parsed.unwrap().assert(object) string userId = profile.id.toStr() string currentUsername = profile.username.toStr() if userId == "" or currentUsername == "" ( - return "" + return {id: "", outcome: "missing"} ) ensureUserIdentity(userId, currentUsername) identityRefreshCache.set(userId, currentUsername) - return userId + return {id: userId, outcome: "found"} ) def usernameForId(string userId) string ( diff --git a/src/history/history-migration.osl b/src/history/history-migration.osl index a7acc2e..bf9e9e0 100644 --- a/src/history/history-migration.osl +++ b/src/history/history-migration.osl @@ -153,9 +153,8 @@ def generateProjectHistory(object project, string parentPath, string parentHead, def ensureUploadedProjectHistory(object project) object ( string lockName = "project-history-" ++ project.id.toStr() sync.unlock(lockName) - object res = ensureGeneratedProjectHistory(project, {}, false) - sync.lock(lockName) - return res + defer sync.lock(lockName) + return ensureGeneratedProjectHistory(project, {}, false) ) def ensureGeneratedProjectHistory(object project, object visiting, boolean allowMissingAssets) object ( @@ -221,6 +220,10 @@ def ensureGeneratedProjectHistory(object project, object visiting, boolean allow ) sync.lock("project-history-" ++ id) + defer ( + sync.unlock("project-history-" ++ id) + void fs.remove(parentPath) + ) auto currentResult = loadProject(id) object res = {ok: false, error: "project disappeared during migration"} if currentResult.ok ( @@ -233,8 +236,6 @@ def ensureGeneratedProjectHistory(object project, object visiting, boolean allow res = generateProjectHistory(current, parentPath, parentHead, allowMissingAssets) ) ) - sync.unlock("project-history-" ++ id) - void fs.remove(parentPath) return res ) diff --git a/src/history/history-migration.test.osl b/src/history/history-migration.test.osl index 5cd1f91..b978f20 100644 --- a/src/history/history-migration.test.osl +++ b/src/history/history-migration.test.osl @@ -15,6 +15,7 @@ number saves = 0 number readFailures = 0 boolean failRead = false boolean failCopy = false +boolean throwRead = false number serial = 0 def randomString(number length) string ( @@ -23,6 +24,9 @@ def randomString(number length) string ( ) def loadProject(string id) object ( + if throwRead ( + throw "project read exploded" + ) if readFailures > 0 ( readFailures -= 1 return {ok: false, retryable: true} @@ -209,4 +213,30 @@ sync.lock("project-history-stale-upload") object staleHistory = ensureUploadedProjectHistory(staleUpload) sync.unlock("project-history-stale-upload") testing.assert(staleHistory.ok == false and saves == before, "history preparation rejects a replaced upload snapshot") +def releaseUploadLock(string lockName) boolean ( + sync.unlock(lockName) + return true +) +def uploadLockFree(string lockName) boolean ( + return thread.new(def() -> ( + boolean taken = sync.tryLock(lockName) + if taken ( + sync.unlock(lockName) + ) + return taken + )).wait() +) +object throwingChild = {id: "throwing-child", owner: "Mist", created: 9000, jsonKey: "throwing-child.json", lastUploadAt: 9000, remixParent: "parent", remixBaseCommit: base, assets: []} +throwRead = true +sync.lock("project-history-throwing-child") +testing.assert(try(ensureUploadedProjectHistory(throwingChild)).isErr(), "a failing ancestor read reaches the upload") +throwRead = false +testing.assert(!uploadLockFree("project-history-throwing-child") and !try(releaseUploadLock("project-history-throwing-child")).isErr(), "a failing ancestor read hands the lock back to the upload") +testing.assert(uploadLockFree("project-history-throwing-child"), "the upload can then release its lock") +object throwingRoot = {id: "throwing-root", owner: "Mist", created: 9000, jsonKey: "throwing-root.json", lastUploadAt: 9000, assets: []} +throwRead = true +sync.lock("project-history-throwing-root") +testing.assert(try(ensureUploadedProjectHistory(throwingRoot)).isErr(), "a failing read under the history lock reaches the upload") +throwRead = false +testing.assert(!try(releaseUploadLock("project-history-throwing-root")).isErr() and uploadLockFree("project-history-throwing-root"), "a failing read under the history lock still hands the lock back once") void fs.remove(tmpDir) diff --git a/src/projects/extensions.osl b/src/projects/extensions.osl index 0866c23..a0fe043 100644 --- a/src/projects/extensions.osl +++ b/src/projects/extensions.osl @@ -199,7 +199,9 @@ def readProjectExtensionUrls(*json.Stream stream) object ( ) def inspectProjectExtensionUrls(string jsonPath) object ( - *json.Stream stream = json.open(jsonPath, maxProjectJsonBytes) + *json.Stream stream = json.open(jsonPath, maxProjectJsonBytes) catch (err) ( + return {ok: false, error: "invalid extension URLs"} + ) object inspected = readProjectExtensionUrls(stream) catch (err) ( stream.close() return {ok: false, error: "invalid extension URLs"} @@ -220,6 +222,9 @@ def parseExtensionSources(string rawSources) object ( ) def collectProjectExtensions(string jsonPath, string rawSources) object ( + if !fs.isFile(jsonPath) ( + return {ok: false, error: "sb3 has no project.json"} + ) object inspected = inspectProjectExtensionUrls(jsonPath) if !inspected.ok ( return inspected @@ -337,6 +342,8 @@ def storeProjectExtensions(string projectId, array previousExtensions, array ext if !projects.contains(projectId) ( projects.append(projectId) ) + record.urls = recordUrls + record.projects = projects if !writeFile(extensionRecordPath(hash), record) ( sync.unlock(lockName) return false @@ -361,7 +368,7 @@ def extensionUsageMigrated(object state) boolean ( if !data.contains("version") ( return false ) - return data.version.toNum() >= 1 + return data.version.toNum() >= 2 ) def migrateExtensionUsage() boolean ( @@ -388,7 +395,7 @@ def migrateExtensionUsage() boolean ( ) ) ) - boolean saved = writeFile(statePath, {version: 1}) + boolean saved = writeFile(statePath, {version: 2}) sync.unlock("extension-index-migration") return saved ) diff --git a/src/projects/extensions.test.osl b/src/projects/extensions.test.osl new file mode 100644 index 0000000..727fe4a --- /dev/null +++ b/src/projects/extensions.test.osl @@ -0,0 +1,110 @@ +import "std:testing" +import "std:json" +import "std:fs" +import "std:regex" +import "std:sync" +import "std:serve" +import "std:zip" +import "./extensions.osl" + +string tmpDir = "/tmp/mistwarp-extensions-test-" ++ timestamp.toStr() ++ "/" +string extensionsDir = tmpDir ++ "extensions/" +string extensionsFile = tmpDir ++ "extensions.json" +string projectsDir = tmpDir ++ "projects/" +number maxProjectJsonBytes = 1048576 +number maxProjectExtensions = 20 +number maxExtensionSourceBytes = 2097152 +number maxProjectExtensionBytes = 10485760 +void fs.mkdir(extensionsDir, {recursive: true}) + +def readFile(string path) object ( + auto text = fs.readFile(path) + if text.isErr() ( + return {ok: false} + ) + auto parsed = json.parse(text.unwrap()) + if parsed.isErr() ( + return {ok: false} + ) + return {ok: true, data: parsed.unwrap()} +) +def writeFile(string path, object data) boolean ( + return fs.writeFile(path, json.stringify(data)) +) +def randomString(number length) string ( + return "r" * length +) +def removeString(array items, string value) array ( + array kept = [] + for i items.len ( + if items[i].toStr() != value ( + kept.append(items[i]) + ) + ) + return kept +) +def loadProject(string id) object ( + return {ok: false, missing: true} +) +def copyStoredProjectJsonToPath(object project, string path) boolean ( + return false +) +def loadProfile(string username) object ( + return {} +) +def saveProfile(object profile) boolean ( + return true +) +def addNotification(string target, string actor, string kind, object payload) ( + return +) +def saveProject(object project) boolean ( + return true +) +def canViewProjectJson(object project, *serve.Context c) boolean ( + return true +) +def requestJSON(*serve.Context c) object ( + return {} +) +def upsertIndexEntry(object project) boolean ( + return true +) + +object noJson = collectProjectExtensions(tmpDir ++ "unpacked/project.json", "") +testing.assert(noJson.ok == false and noJson.error == "sb3 has no project.json", "an sb3 without project.json is refused") +void fs.writeFile(tmpDir ++ "project.json", json.stringify({targets: [], extensionURLs: {ext: "https://example.com/ext.js"}})) +object collected = collectProjectExtensions(tmpDir ++ "project.json", json.stringify({"https://example.com/ext.js": "class Ext {}"})) +testing.assert(collected.ok == true and collected.extensions.len == 1 and collected.extensions[1].url == "https://example.com/ext.js", "extension URLs are collected from project.json") + +object stored = collected.extensions[1].assert(object) +string hash = stored.hash.toStr() +testing.assert(storeProjectExtensions("p1", [], [stored], collected.sources.assert(object)), "extensions are stored") +testing.assert(storeProjectExtensions("p2", [], [stored], {}), "a second project using the extension is stored") +object record = readExtensionRecord(hash) +testing.assert(record.urls == ["https://example.com/ext.js"] and record.projects == ["p1", "p2"], "the record lists the extension's URL and every project using it") +testing.assert(fs.exists(extensionSourcePath(hash)), "the source is kept") +testing.assert(storeProjectExtensions("p1", [stored], [], {}) and readExtensionRecord(hash).projects == ["p2"], "a project that drops the extension leaves the record") +object gallery = {url: "https://extensions.turbowarp.org/a.js", hash: "https://extensions.turbowarp.org/a.js".hashSHA256()} +testing.assert(storeProjectExtensions("p3", [], [gallery], {}) and readExtensionRecord(gallery.hash.toStr()).projects == ["p3"] and readExtensionRecord(gallery.hash.toStr()).gallery == true, "gallery extensions are recorded too") + +string staleUrl = "https://example.com/stale.js" +string staleHash = staleUrl.hashSHA256() +void fs.mkdir(projectsDir, {recursive: true}) +void writeFile(projectsDir ++ "p4.json", {id: "p4", extensions: [{url: staleUrl, hash: staleHash}]}) +void writeFile(projectsDir ++ "p5.json", {id: "p5", extensions: [{url: staleUrl, hash: staleHash}, {url: "https://example.com/ext.js", hash: hash}]}) +void writeFile(extensionRecordPath(staleHash), {status: "untrusted", urls: [], projects: [], firstSeen: 1}) +void writeFile(extensionsDir ++ "index.json", {version: 1}) +testing.assert(migrateExtensionUsage(), "the extension usage migration runs again after the record fix") +object backfilled = readExtensionRecord(staleHash) +array backfilledProjects = backfilled.projects.assert(array) +testing.assert(backfilled.urls == [staleUrl] and backfilledProjects.len == 2 and backfilledProjects.contains("p4") and backfilledProjects.contains("p5") and backfilled.status == "untrusted" and backfilled.firstSeen == 1, "records written before the fix get their URLs and projects back") +testing.assert(readFile(extensionsDir ++ "index.json").data.version == 2, "the migration records its new version") +string afterFirst = json.stringify(readExtensionRecord(staleHash)) +string sharedAfterFirst = json.stringify(readExtensionRecord(hash)) +void writeFile(extensionsDir ++ "index.json", {version: 1}) +testing.assert(migrateExtensionUsage() and json.stringify(readExtensionRecord(staleHash)) == afterFirst and json.stringify(readExtensionRecord(hash)) == sharedAfterFirst, "running the migration again changes nothing") +array sharedProjects = readExtensionRecord(hash).projects.assert(array) +testing.assert(sharedProjects.contains("p2") and sharedProjects.contains("p5"), "the backfill keeps projects the record already listed") + +void fs.remove(tmpDir) diff --git a/src/projects/project-lock.osl b/src/projects/project-lock.osl new file mode 100644 index 0000000..b470097 --- /dev/null +++ b/src/projects/project-lock.osl @@ -0,0 +1,11 @@ +export {} + +def projectSaveLock(string id) string ( + return "project-history-" ++ id +) + +def runLocked(string lockName, function work) ( + sync.lock(lockName) + defer sync.unlock(lockName) + work() +) diff --git a/src/projects/project-lock.test.osl b/src/projects/project-lock.test.osl new file mode 100644 index 0000000..75f027f --- /dev/null +++ b/src/projects/project-lock.test.osl @@ -0,0 +1,34 @@ +import "std:testing" +import "std:sync" +import "std:thread" +import "./project-lock.osl" + +def lockIsFree(string lockName) boolean ( + boolean free = thread.new(def() -> ( + boolean taken = sync.tryLock(lockName) + if taken ( + sync.unlock(lockName) + ) + return taken + )).wait() + return free +) + +testing.assert(projectSaveLock("p1") == "project-history-p1", "saves share the lock the history writers take") + +boolean heldInside = true +runLocked("save-lock-test", def() -> ( + heldInside = !lockIsFree("save-lock-test") +)) +testing.assert(heldInside, "the lock is held while the work runs") +testing.assert(lockIsFree("save-lock-test"), "the lock is released after the work returns") + +def failingSave() ( + throw "bad request body" +) +def lockedFailingSave() boolean ( + runLocked("save-lock-test", failingSave) + return true +) +testing.assert(try(lockedFailingSave()).isErr(), "a failure in the work still reaches the caller") +testing.assert(lockIsFree("save-lock-test"), "the lock is released when the work throws") diff --git a/src/projects/project-team.osl b/src/projects/project-team.osl index a8853e8..fea0813 100644 --- a/src/projects/project-team.osl +++ b/src/projects/project-team.osl @@ -1,13 +1,16 @@ export {} -def applyCollaborators(object project, array incoming) Refusal ( +def applyCollaborators(object project, array incoming, UpdateLookups lookups) Refusal ( array collaborators = [] number totalShare = 0 for i incoming.len ( + if typeof(incoming[i]) != "object" ( + return refuse(400, "each teammate must be an object") + ) if collaborators.len < 25 ( object entry = incoming[i].assert(object) - string username = normalizeUsername(entry.username.toStr()) - string role = entry.role.toStr().toLower().strip() + string username = normalizeUsername((entry.username ?? "").toStr()) + string role = (entry.role ?? "").toStr().toLower().strip() boolean validRole = role == "editor" or role == "maintainer" or role == "contributor" or role == "tester" number share = 0 if entry.contains("share") and entry.share != null ( @@ -17,7 +20,11 @@ def applyCollaborators(object project, array incoming) Refusal ( return refuse(400, "each teammate share must be between 0 and 50 percent") ) if username != "" and validRole and normalizeUsername(username) != normalizeUsername(project.owner.toStr()) ( - string teammateId = discoverUserIdentity(username) + object identity = lookupIdentity(lookups, username) + if identity.outcome == "unavailable" ( + return roturUnavailableRefusal(username) + ) + string teammateId = identity.id.toStr() if teammateId == "" ( return refuse(400, "Rotur user not found: " ++ username) ) diff --git a/src/projects/project-team.test.osl b/src/projects/project-team.test.osl index 4087620..9a70cfa 100644 --- a/src/projects/project-team.test.osl +++ b/src/projects/project-team.test.osl @@ -4,17 +4,50 @@ import "std:regex" import "std:fs" import "go:mistwarp.local/storagefs" import "../core/utils.osl" +import "std:json" +import "std:serve" import "./project-refusal.osl" +import "./project-update-lookups.osl" import "./project-team.osl" array lookups = [] -def discoverUserIdentity(string username) string ( +def discoverUserIdentityOutcome(string username) object ( lookups.append(username) if username == "alice" or username == "bob" ( - return "u_" ++ username + return {id: "u_" ++ username, outcome: "found"} ) + if username == "busy" ( + return {id: "", outcome: "unavailable"} + ) + return {id: "", outcome: "missing"} +) +def mistwarpBenefitsForTier(string tier) object ( + return {} +) +def mistwarpBenefitsForUserId(string userId) object ( + return {} +) +def userIdForUsername(string username) string ( return "" ) +def vanityEligibility(object project) string ( + return "unknown" +) +def canManageGroupTag(string tag, *serve.Context c) boolean ( + return false +) +def groupTagOf(object value) string ( + return "" +) +def noStorageHold() object ( + return {ok: true, reservations: []} +) +def storageHold(string userId, object changes, boolean enforce) object ( + return noStorageHold() +) +def storageHoldRelease(string userId, object hold) ( + return +) array sent = [] def addNotification(string target, string actor, string notifType, object payload) ( @@ -27,15 +60,20 @@ Refusal saved = applyCollaborators(project, [ {username: "bob", role: "tester"}, {username: "sam", role: "editor"}, {username: "carol", role: "owner"} -]) +], UpdateLookups()) testing.assert(!isRefused(saved), "a valid team is saved") testing.assert(project.collaborators.len == 2 and project.collaborators[1].username == "alice" and project.collaborators[1].userId == "u_alice" and project.collaborators[1].role == "maintainer", "teammates are normalized and resolved") testing.assert(project.collaborators[2].share == 0, "missing shares are zero") testing.assert(!lookups.contains("sam") and !lookups.contains("carol"), "the owner and invalid roles are skipped before any lookup") -testing.assert(applyCollaborators(project, [{username: "alice", role: "editor", share: 51}]).message == "each teammate share must be between 0 and 50 percent", "shares are capped per teammate") -testing.assert(applyCollaborators(project, [{username: "ghost", role: "editor"}]).message == "Rotur user not found: ghost", "unknown users are refused") -testing.assert(applyCollaborators(project, [{username: "alice", role: "editor"}, {username: "ALICE", role: "tester"}]).message == "a person can only appear once in the project team", "teammates appear once") -testing.assert(applyCollaborators(project, [{username: "alice", role: "editor", share: 40}, {username: "bob", role: "editor", share: 40}]).message == "teammate revenue shares cannot exceed 70 percent", "total shares are capped") +testing.assert(applyCollaborators(project, [{username: "alice", role: "editor", share: 51}], UpdateLookups()).message == "each teammate share must be between 0 and 50 percent", "shares are capped per teammate") +testing.assert(applyCollaborators(project, [{username: "ghost", role: "editor"}], UpdateLookups()).message == "Rotur user not found: ghost", "unknown users are refused") +testing.assert(applyCollaborators(project, [{username: "alice", role: "editor"}, {username: "ALICE", role: "tester"}], UpdateLookups()).message == "a person can only appear once in the project team", "teammates appear once") +testing.assert(applyCollaborators(project, [{username: "alice", role: "editor", share: 40}, {username: "bob", role: "editor", share: 40}], UpdateLookups()).message == "teammate revenue shares cannot exceed 70 percent", "total shares are capped") +object nullTeam = {id: "p2", title: "Team", owner: "Sam"} +testing.assert(!isRefused(applyCollaborators(nullTeam, [{role: "editor"}, {username: null, role: "editor"}, {username: "bob"}], UpdateLookups())) and nullTeam.collaborators.len == 0 and !lookups.contains("null"), "teammates without a name or role are skipped without a lookup") +Refusal busy = applyCollaborators(project, [{username: "busy", role: "editor"}], UpdateLookups()) +testing.assert(busy.status == 503 and busy.body.code == "rotur_unavailable" and busy.body.error.toStr().contains("busy"), "a Rotur that can't answer is not reported as a missing user") +testing.assert(applyCollaborators(project, ["alice"], UpdateLookups()).message == "each teammate must be an object", "teammates must be objects") testing.assert(project.collaborators.len == 2, "a refused team leaves the saved team alone") array previousTeam = [{username: "Alice", role: "maintainer"}, {username: "bob", role: "editor"}] diff --git a/src/projects/project-update-lookups.osl b/src/projects/project-update-lookups.osl new file mode 100644 index 0000000..2358659 --- /dev/null +++ b/src/projects/project-update-lookups.osl @@ -0,0 +1,135 @@ +export {} + +type UpdateLookups ( + boolean replay = false + boolean missed = false + object answers = {} + array holds = [] +) + +def roturUnavailableRefusal(string username) Refusal ( + return refuseWithBody(503, {ok: false, code: "rotur_unavailable", error: "Rotur can't be reached right now, so " ++ username ++ " couldn't be checked. Try again soon."}) +) + +def lookupKnown(UpdateLookups lookups, string key) boolean ( + if lookups.answers.contains(key) ( + return true + ) + if lookups.replay ( + lookups.missed = true + ) + return false +) + +def lookupIdentity(UpdateLookups lookups, string username) object ( + string key = "identity:" ++ username + if !lookupKnown(lookups, key) ( + if lookups.replay ( + return {id: "", outcome: "unavailable"} + ) + lookups.answers[key] = discoverUserIdentityOutcome(username) + ) + return lookups.answers[key].assertElse(object, {id: "", outcome: "unavailable"}) +) + +def lookupBenefits(UpdateLookups lookups, string userId) object ( + string key = "benefits:" ++ userId + if !lookupKnown(lookups, key) ( + if lookups.replay ( + return mistwarpBenefitsForTier("Free") + ) + lookups.answers[key] = mistwarpBenefitsForUserId(userId) + ) + return lookups.answers[key].assertElse(object, {}) +) + +def lookupProjectBenefits(UpdateLookups lookups, object project) object ( + string ownerId = "" + if project.contains("ownerId") and project.ownerId != null ( + ownerId = project.ownerId.toStr() + ) + if ownerId == "" ( + ownerId = userIdForUsername(project.owner.toStr()) + ) + return lookupBenefits(lookups, ownerId) +) + +def lookupVanityEligibility(UpdateLookups lookups, object holder) string ( + string key = "vanity:" ++ (holder.id ?? "").toStr() ++ ":" ++ (holder.ownerId ?? "").toStr() ++ ":" ++ (holder.owner ?? "").toStr() + if !lookupKnown(lookups, key) ( + if lookups.replay ( + return "unknown" + ) + lookups.answers[key] = vanityEligibility(holder) + ) + return lookups.answers[key].toStr() +) + +def lookupGroupManager(UpdateLookups lookups, *serve.Context c, string tag) boolean ( + string key = "group:" ++ tag + if !lookupKnown(lookups, key) ( + if lookups.replay ( + return false + ) + lookups.answers[key] = canManageGroupTag(tag, c) + ) + return lookups.answers[key] == true +) + +def lookupGroupContentManager(UpdateLookups lookups, *serve.Context c, object project) boolean ( + string tag = groupTagOf(project) + return tag != "" and lookupGroupManager(lookups, c, tag) +) + +def storageHoldKey(string userId, object changes, boolean enforce) string ( + return "hold:" ++ userId ++ ":" ++ enforce.toStr() ++ ":" ++ json.stringify(changes) +) + +def lookupStorageHold(UpdateLookups lookups, string userId, object changes, boolean enforce) object ( + string key = storageHoldKey(userId, changes, enforce) + if !lookupKnown(lookups, key) ( + if lookups.replay ( + return noStorageHold() + ) + lookups.answers[key] = storageHold(userId, changes, enforce) + array holds = lookups.holds + holds.append({userId: userId, key: key}) + lookups.holds = holds + ) + return lookups.answers[key].assertElse(object, noStorageHold()) +) + +def releaseStorageHolds(UpdateLookups lookups, string keptKey) ( + for i lookups.holds.len ( + object entry = lookups.holds[i].assertElse(object, {}) + string key = (entry.key ?? "").toStr() + if key != keptKey ( + storageHoldRelease((entry.userId ?? "").toStr(), lookups.answers[key].assertElse(object, {})) + ) + ) +) + +def lookupsKnowing(object knownIdentities) UpdateLookups ( + UpdateLookups lookups = UpdateLookups() + object answers = {} + array keys = knownIdentities.getKeys() + for i keys.len ( + string key = keys[i].toStr() + answers[key] = knownIdentities[key] + ) + lookups.answers = answers + return lookups +) + +def rememberIdentities(UpdateLookups lookups, object knownIdentities) ( + array keys = lookups.answers.getKeys() + for i keys.len ( + string key = keys[i].toStr() + if key.startsWith("identity:") ( + object answer = lookups.answers[key].assertElse(object, {}) + if answer.outcome == "found" or answer.outcome == "missing" ( + knownIdentities[key] = answer + ) + ) + ) +) diff --git a/src/projects/project-update-lookups.test.osl b/src/projects/project-update-lookups.test.osl new file mode 100644 index 0000000..ad55b48 --- /dev/null +++ b/src/projects/project-update-lookups.test.osl @@ -0,0 +1,95 @@ +import "std:testing" +import "std:json" +import "std:serve" +import "./project-refusal.osl" +import "./project-update-lookups.osl" + +array asked = [] +def discoverUserIdentityOutcome(string username) object ( + asked.append("identity " ++ username) + if username == "alice" ( + return {id: "u_alice", outcome: "found"} + ) + return {id: "", outcome: "unavailable"} +) +def mistwarpBenefitsForTier(string tier) object ( + return {maxProjects: 100} +) +def mistwarpBenefitsForUserId(string userId) object ( + asked.append("benefits " ++ userId) + return {maxProjects: 500} +) +def userIdForUsername(string username) string ( + return "u_" ++ username +) +def vanityEligibility(object project) string ( + asked.append("vanity " ++ project.id.toStr()) + return "yes" +) +def canManageGroupTag(string tag, *serve.Context c) boolean ( + return true +) +def groupTagOf(object value) string ( + return (value.groupTag ?? "").toStr() +) +def noStorageHold() object ( + return {ok: true, reservations: []} +) +def storageHold(string userId, object changes, boolean enforce) object ( + asked.append("hold " ++ userId) + return {ok: true, reservations: [{id: "hold-" ++ userId}]} +) +array released = [] +def storageHoldRelease(string userId, object hold) ( + released.append(userId) +) + +UpdateLookups live = UpdateLookups() +testing.assert(lookupIdentity(live, "alice").id == "u_alice" and lookupIdentity(live, "busy").outcome == "unavailable", "a live pass asks Rotur") +testing.assert(lookupProjectBenefits(live, {owner: "sam"}).maxProjects == 500, "benefits follow the project owner") +testing.assert(lookupVanityEligibility(live, {id: "p2", ownerId: "u_bob"}) == "yes", "vanity eligibility is asked for each holder") +testing.assert(lookupStorageHold(live, "u_bob", {projects: {p1: {bytes: 1}}}, true).ok == true, "a live pass takes the storage hold") +testing.assert(!live.missed and asked.len == 5, "every answer is asked once") + +live.replay = true +testing.assert(lookupIdentity(live, "alice").id == "u_alice" and lookupProjectBenefits(live, {owner: "sam"}).maxProjects == 500, "a replay reuses the answers") +testing.assert(lookupVanityEligibility(live, {id: "p2", ownerId: "u_bob"}) == "yes" and lookupStorageHold(live, "u_bob", {projects: {p1: {bytes: 1}}}, true).reservations.len == 1, "a replay reuses the hold") +testing.assert(!live.missed and asked.len == 5, "a replay with the same project asks Rotur nothing") + +testing.assert(lookupProjectBenefits(live, {owner: "sam", ownerId: "u_carol"}).maxProjects == 100 and live.missed, "a replay for another owner misses instead of asking") +testing.assert(asked.len == 5, "a miss asks Rotur nothing") + +UpdateLookups changed = UpdateLookups() +changed.replay = true +testing.assert(lookupIdentity(changed, "bob").outcome == "unavailable" and changed.missed and asked.len == 5, "a teammate the live pass never looked up is a miss") +UpdateLookups grown = UpdateLookups() +void lookupStorageHold(grown, "u_bob", {projects: {p1: {bytes: 1}}}, true) +grown.replay = true +void lookupStorageHold(grown, "u_bob", {projects: {p1: {bytes: 2}}}, true) +testing.assert(grown.missed, "a project that grew since the hold misses") + +UpdateLookups holds = UpdateLookups() +void lookupStorageHold(holds, "u_a", {projects: {p1: {bytes: 1}}}, true) +void lookupStorageHold(holds, "u_b", {projects: {p1: {bytes: 1}}}, true) +releaseStorageHolds(holds, storageHoldKey("u_b", {projects: {p1: {bytes: 1}}}, true)) +testing.assert(released == ["u_a"], "every hold except the one the save used is released") +released = [] +releaseStorageHolds(holds, "") +testing.assert(released == ["u_a", "u_b"], "an unsaved update releases every hold") + +testing.assert(roturUnavailableRefusal("alice").status == 503 and roturUnavailableRefusal("alice").body.code == "rotur_unavailable", "an unreachable Rotur is a 503") + +object knownIdentities = {} +UpdateLookups firstTry = lookupsKnowing(knownIdentities) +void lookupIdentity(firstTry, "alice") +void lookupIdentity(firstTry, "busy") +void lookupProjectBenefits(firstTry, {owner: "sam"}) +rememberIdentities(firstTry, knownIdentities) +testing.assert(knownIdentities.getKeys() == ["identity:alice"], "only settled identity answers are kept between tries") +number askedBefore = asked.len +UpdateLookups secondTry = lookupsKnowing(knownIdentities) +testing.assert(lookupIdentity(secondTry, "alice").id == "u_alice" and asked.len == askedBefore, "a later try reuses the teammate answers") +void lookupProjectBenefits(secondTry, {owner: "sam"}) +testing.assert(asked.len == askedBefore + 1, "a later try asks for plan benefits again") +secondTry.replay = true +testing.assert(lookupIdentity(secondTry, "alice").id == "u_alice" and !secondTry.missed, "a carried answer replays under the lock") diff --git a/src/projects/project-update-policy.osl b/src/projects/project-update-policy.osl index 71d4433..ccbc221 100644 --- a/src/projects/project-update-policy.osl +++ b/src/projects/project-update-policy.osl @@ -4,6 +4,7 @@ type OwnerTransfer ( string fromId = "" string toId = "" object hold = {} + string holdKey = "" ) def applyBranding(object project, object body, object creatorBenefits) Refusal ( @@ -41,28 +42,28 @@ def vanitySlugRefusal(string slug, string currentSlug, object creatorBenefits) R def applyProjectText(object project, object body) Refusal ( if body.contains("title") ( - string title = body.title.toStr().strip() + string title = (body.title ?? "").toStr().strip() if title == "" or title.len > 100 ( return refuse(400, "invalid title") ) project.title = title ) if body.contains("description") ( - string description = body.description.toStr() + string description = (body.description ?? "").toStr() if description.len > 5000 ( return refuse(400, "description too long") ) project.description = description ) if body.contains("instructions") ( - string instructions = body.instructions.toStr() + string instructions = (body.instructions ?? "").toStr() if instructions.len > 5000 ( return refuse(400, "instructions too long") ) project.instructions = instructions ) if body.contains("notes") ( - string notes = body.notes.toStr() + string notes = (body.notes ?? "").toStr() if notes.len > 5000 ( return refuse(400, "notes too long") ) @@ -75,15 +76,21 @@ def applyCredits(object project, object body) Refusal ( if !body.contains("credits") ( return noRefusal() ) + if typeof(body.credits) != "array" ( + return refuse(400, "credits must be a list") + ) array incoming = body.credits.assert(array) if incoming.len > 50 ( return refuse(400, "too many credits") ) array credits = [] for i incoming.len ( - object entry = incoming[i] - string who = entry.who.toStr().strip() - string role = entry.role.toStr().strip() + if typeof(incoming[i]) != "object" ( + return refuse(400, "each credit must be an object") + ) + object entry = incoming[i].assert(object) + string who = (entry.who ?? "").toStr().strip() + string role = (entry.role ?? "").toStr().strip() string creditUrl = "" if entry.contains("url") and entry.url != null ( creditUrl = entry.url.toStr().strip() @@ -133,21 +140,25 @@ def applyRemixShare(object project, number remixShare) Refusal ( return noRefusal() ) -def applyTags(object project, object body) ( +def applyTags(object project, object body) Refusal ( if !body.contains("tags") ( - return + return noRefusal() + ) + if typeof(body.tags) != "array" ( + return refuse(400, "tags must be a list") ) array incoming = body.tags.assert(array) array tags = [] for i incoming.len ( if tags.len < 10 ( - string t = normalizeTag(incoming[i].toStr()) + string t = normalizeTag((incoming[i] ?? "").toStr()) if t != "" and !tags.contains(t) ( tags.append(t) ) ) ) project.tags = tags + return noRefusal() ) def applyGameProducts(object project, object body) Refusal ( @@ -162,12 +173,12 @@ def applyGameProducts(object project, object body) Refusal ( array productIds = [] for i incomingProducts.len ( object incomingProduct = incomingProducts[i].assertElse(object, {}) - string productId = incomingProduct.id.toStr().toLower().strip() - string productName = incomingProduct.name.toStr().strip() - string productDescription = incomingProduct.description.toStr().strip() - number productPrice = incomingProduct.price.toNum() - string grantsItem = incomingProduct.grantsItem.toStr().toLower().strip() - string productIcon = incomingProduct.icon.toStr().strip() + string productId = (incomingProduct.id ?? "").toStr().toLower().strip() + string productName = (incomingProduct.name ?? "").toStr().strip() + string productDescription = (incomingProduct.description ?? "").toStr().strip() + number productPrice = (incomingProduct.price ?? 0).toNum() + string grantsItem = (incomingProduct.grantsItem ?? "").toStr().toLower().strip() + string productIcon = (incomingProduct.icon ?? "").toStr().strip() if !regex.match("^[a-z0-9_-]{1,40}$", productId) or productIds.contains(productId) ( return refuse(400, "game product ids must be unique lowercase identifiers") ) @@ -194,9 +205,9 @@ def applyGameProducts(object project, object body) Refusal ( ) def portableItemVisual(object visual) object ( - string visualType = visual.type.toStr().toLower().strip() + string visualType = (visual.type ?? "").toStr().toLower().strip() if visualType == "image" ( - string itemImage = visual.url.toStr().strip() + string itemImage = (visual.url ?? "").toStr().strip() if itemImage.startsWith("data:image/") ( if itemImage.toBytes().len > 100000 ( return {error: "portable item images must be smaller than 100KB"} @@ -209,8 +220,8 @@ def portableItemVisual(object visual) object ( return {visual: {type: "image", url: itemImage}} ) if visualType == "costume" ( - string assetId = visual.assetId.toStr().strip() - string dataFormat = visual.dataFormat.toStr().toLower().strip() + string assetId = (visual.assetId ?? "").toStr().strip() + string dataFormat = (visual.dataFormat ?? "").toStr().toLower().strip() if !regex.match("^[A-Za-z0-9_-]{8,128}$", assetId) or (dataFormat != "png" and dataFormat != "jpg" and dataFormat != "jpeg" and dataFormat != "svg") ( return {error: "invalid portable item costume asset"} ) @@ -231,11 +242,12 @@ def applyGameItems(object project, object body) Refusal ( array itemIds = [] for i incomingItems.len ( object incomingItem = incomingItems[i].assertElse(object, {}) - string itemId = incomingItem.id.toStr().toLower().strip() - string itemName = incomingItem.name.toStr().strip() + string itemId = (incomingItem.id ?? "").toStr().toLower().strip() + string itemName = (incomingItem.name ?? "").toStr().strip() object visual = incomingItem.visual.assertElse(object, {}) - if visual.getKeys().len == 0 and incomingItem.image.toStr().strip() != "" ( - visual = {type: "image", url: incomingItem.image.toStr().strip()} + string itemImage = (incomingItem.image ?? "").toStr().strip() + if visual.getKeys().len == 0 and itemImage != "" ( + visual = {type: "image", url: itemImage} ) if !regex.match("^[a-z0-9_-]{1,40}$", itemId) or itemIds.contains(itemId) ( return refuse(400, "portable item ids must be unique lowercase identifiers") @@ -259,7 +271,7 @@ def applyInventoryPolicy(object project, object body) Refusal ( return noRefusal() ) object incomingPolicy = body.inventoryPolicy.assertElse(object, {}) - string mode = incomingPolicy.mode.toStr().toLower().strip() + string mode = (incomingPolicy.mode ?? "").toStr().toLower().strip() if mode != "none" and mode != "allowlist" and mode != "all" ( return refuse(400, "inventory policy must be none, allowlist, or all") ) @@ -270,7 +282,7 @@ def applyInventoryPolicy(object project, object body) Refusal ( ) array cleanItems = [] for i allowedItems.len ( - string globalId = allowedItems[i].toStr().strip() + string globalId = (allowedItems[i] ?? "").toStr().strip() if !regex.match("^[A-Za-z0-9_-]{1,80}:[a-z0-9_-]{1,40}$", globalId) ( return refuse(400, "invalid global portable item id") ) @@ -280,7 +292,7 @@ def applyInventoryPolicy(object project, object body) Refusal ( ) array cleanProjects = [] for i allowedProjects.len ( - string allowedProject = allowedProjects[i].toStr().strip() + string allowedProject = (allowedProjects[i] ?? "").toStr().strip() if !isSafePathPart(allowedProject) ( return refuse(400, "invalid portable item source project") ) @@ -311,13 +323,18 @@ def gameGrantRefusal(object project, object body) Refusal ( return noRefusal() ) -def applyCompatibility(object project, object body) ( - if body.contains("compatibility") and body.compatibility != null ( - object compatibilityInfo = body.compatibility.assert(object) - project.compatibility = { - mobile: compatibilityInfo.contains("mobile") and compatibilityInfo.mobile == true, - keyboard: compatibilityInfo.contains("keyboard") and compatibilityInfo.keyboard == true, - controller: compatibilityInfo.contains("controller") and compatibilityInfo.controller == true - } +def applyCompatibility(object project, object body) Refusal ( + if !body.contains("compatibility") or body.compatibility == null ( + return noRefusal() ) + if typeof(body.compatibility) != "object" ( + return refuse(400, "compatibility must be an object") + ) + object compatibilityInfo = body.compatibility.assert(object) + project.compatibility = { + mobile: compatibilityInfo.contains("mobile") and compatibilityInfo.mobile == true, + keyboard: compatibilityInfo.contains("keyboard") and compatibilityInfo.keyboard == true, + controller: compatibilityInfo.contains("controller") and compatibilityInfo.controller == true + } + return noRefusal() ) diff --git a/src/projects/project-update-policy.test.osl b/src/projects/project-update-policy.test.osl index e6f9635..7dc07da 100644 --- a/src/projects/project-update-policy.test.osl +++ b/src/projects/project-update-policy.test.osl @@ -32,6 +32,8 @@ testing.assert(isRefused(applyProjectText(text, {title: "t" * 101})), "titles ar testing.assert(!isRefused(applyProjectText(text, {title: " New ", description: "d", instructions: "i", notes: "n"})) and text.title == "New" and text.notes == "n", "text fields are saved") testing.assert(applyProjectText(text, {description: "d" * 5001}).message == "description too long" and text.description == "d", "long descriptions are refused") testing.assert(applyProjectText(text, {title: "Kept", notes: "n" * 5001}).message == "notes too long" and text.title == "Kept", "fields before a refused one are already applied") +testing.assert(applyProjectText(text, {title: null}).message == "invalid title" and text.title == "Kept", "a null title is blank, not the word null") +testing.assert(!isRefused(applyProjectText(text, {description: null, instructions: null, notes: null})) and text.description == "" and text.instructions == "" and text.notes == "", "null text fields are cleared") object credited = {id: "p1"} testing.assert(!isRefused(applyCredits(credited, {credits: [{who: " Ada ", role: "Art", url: "https://a.example"}, {who: "", role: "x"}, {who: "Bo", role: "Music", url: null}]})), "credits are accepted") @@ -43,6 +45,10 @@ for i 51 ( manyCredits.append({who: "A", role: "B"}) ) testing.assert(applyCredits(credited, {credits: manyCredits}).message == "too many credits", "credits are capped at 50") +testing.assert(applyCredits(credited, {credits: "x"}).message == "credits must be a list" and applyCredits(credited, {credits: null}).status == 400, "credits must be a list") +testing.assert(applyCredits(credited, {credits: ["Ada"]}).message == "each credit must be an object", "each credit must be an object") +testing.assert(credited.credits.len == 2, "refused credits leave the saved credits alone") +testing.assert(!isRefused(applyCredits(credited, {credits: [{who: "Ada"}, {who: null, role: "Art"}]})) and credited.credits.len == 1 and credited.credits[1].role == "", "a credit without a role has an empty role and one without a name is dropped") object flagged = {id: "p1"} applyProjectFlags(flagged, {commentsOff: "yes", remixable: 0, seeInside: false}) @@ -54,6 +60,10 @@ testing.assert(!isRefused(applyRemixShare(flagged, 50)) and flagged.remixShare = object tagged = {id: "p1"} applyTags(tagged, {tags: ["Games", "games", "#Puzzle", "bad tag!", "a", "b", "c", "d", "e", "f", "g", "h", "i"]}) testing.assert(tagged.tags == ["games", "puzzle", "a", "b", "c", "d", "e", "f", "g", "h"], "tags are normalized, unique and capped at 10") +object nullTagged = {id: "p1"} +applyTags(nullTagged, {tags: [null, "Games"]}) +testing.assert(nullTagged.tags == ["games"], "null tags are skipped") +testing.assert(applyTags(tagged, {tags: "games"}).message == "tags must be a list" and tagged.tags.len == 10, "tags must be a list") object product = {id: "Pack", name: "Pack", description: "d", price: 10, grantsItem: "sword", icon: "https://i.example/i.png"} object shop = {id: "p1"} @@ -74,6 +84,9 @@ testing.assert(applyGameProducts(shop, {gameProducts: [productWith(product, "x", testing.assert(applyGameProducts(shop, {gameProducts: [productWith(product, "x", "X", 1, "data:image/png," ++ ("a" * 100000), "")]}).message == "game product icons must be smaller than 100KB", "product icons are capped in bytes") testing.assert(applyGameProducts(shop, {gameProducts: [productWith(product, "x", "X", 1, "", "Bad!")]}).message == "invalid portable item id on game product", "granted item ids are checked") +object plainProduct = {id: "plain", name: "Plain", price: 3} +testing.assert(!isRefused(applyGameProducts(shop, {gameProducts: [plainProduct]})) and shop.gameProducts[1].icon == "" and shop.gameProducts[1].description == "" and shop.gameProducts[1].grantsItem == "", "a product without an icon, description or grant is accepted") +testing.assert(!isRefused(applyGameProducts(shop, {gameProducts: [{id: "free", name: "Free", icon: null}]})) and shop.gameProducts[1].price == 0 and shop.gameProducts[1].icon == "", "null product fields are empty") testing.assert(portableItemVisual({type: "image", url: "https://i.example/a.png"}).visual.url == "https://i.example/a.png", "HTTPS images are kept") testing.assert(portableItemVisual({type: "image", url: "http://i.example/a.png"}).error == "portable item images must use a short HTTPS URL", "plain HTTP images are refused") testing.assert(portableItemVisual({type: "image", url: "data:image/png," ++ ("a" * 100000)}).error == "portable item images must be smaller than 100KB", "data images are capped") @@ -87,6 +100,8 @@ testing.assert(items.gameItems[1].id == "sword" and items.gameItems[1].visual.ty testing.assert(applyGameItems(items, {gameItems: [{id: "a", name: "A", image: "https://i"}, {id: "A", name: "A", image: "https://i"}]}).message == "portable item ids must be unique lowercase identifiers", "item ids are unique") testing.assert(applyGameItems(items, {gameItems: [{id: "a", name: "", image: ""}]}).message == "invalid portable item definition", "items need a name") testing.assert(applyGameItems(items, {gameItems: [{id: "a", name: "A", image: ""}]}).message == "portable items need an image or costume", "item visual errors reach the caller") +testing.assert(applyGameItems(items, {gameItems: [{id: "a", name: "A"}]}).message == "portable items need an image or costume", "an item without a visual needs one") +testing.assert(portableItemVisual({type: "image"}).error == "portable item images must use a short HTTPS URL", "an image visual without a URL is refused") object granted = {id: "p1", gameProducts: [{id: "pack", grantsItem: "sword"}], gameItems: [{id: "sword"}]} testing.assert(!isRefused(gameGrantRefusal(granted, {gameItems: []})), "products may grant defined items") @@ -96,6 +111,7 @@ testing.assert(gameGrantRefusal({id: "p1", gameProducts: [{grantsItem: "ghost"}] object inventory = {id: "p1"} testing.assert(applyInventoryPolicy(inventory, {inventoryPolicy: {mode: "some"}}).message == "inventory policy must be none, allowlist, or all", "inventory modes are fixed") testing.assert(applyInventoryPolicy(inventory, {inventoryPolicy: {mode: "all", allowedProjects: ["../x"]}}).message == "invalid portable item source project", "source projects must be safe ids") +testing.assert(applyInventoryPolicy(inventory, {inventoryPolicy: {mode: "all", allowedProjects: [null]}}).message == "invalid portable item source project", "a null source project is refused") testing.assert(applyInventoryPolicy(inventory, {inventoryPolicy: {mode: "all", allowedItems: ["nope"]}}).message == "invalid global portable item id", "allowed items are global ids") testing.assert(!isRefused(applyInventoryPolicy(inventory, {inventoryPolicy: {mode: " Allowlist ", allowedItems: ["p1:sword", "p1:sword"], allowedProjects: ["p1", "p1"]}})), "a valid allowlist is accepted") testing.assert(inventory.inventoryPolicy.mode == "allowlist" and inventory.inventoryPolicy.allowedItems == ["p1:sword"] and inventory.inventoryPolicy.allowedProjects == ["p1"], "allowlists are deduplicated") @@ -105,3 +121,4 @@ applyCompatibility(compatible, {compatibility: null}) testing.assert(!compatible.contains("compatibility"), "null compatibility is ignored") applyCompatibility(compatible, {compatibility: {mobile: true, controller: "yes"}}) testing.assert(compatible.compatibility.mobile == true and compatible.compatibility.keyboard == false and compatible.compatibility.controller == false, "compatibility only accepts true") +testing.assert(applyCompatibility(compatible, {compatibility: "mobile"}).message == "compatibility must be an object" and compatible.compatibility.mobile == true, "compatibility must be an object") diff --git a/src/projects/project-update.osl b/src/projects/project-update.osl index b23653f..cef683e 100644 --- a/src/projects/project-update.osl +++ b/src/projects/project-update.osl @@ -1,56 +1,138 @@ export {} +type UpdateOutcome ( + boolean saved = false + object project = {} + array previousTeam = [] + Refusal refusal = Refusal() + OwnerTransfer transfer = OwnerTransfer() +) + def handleUpdateProject(*serve.Context c) ( - string lockName = "project-upload:" ++ c.param("id") - sync.lock(lockName) - handleUpdateProjectLocked(c) - sync.unlock(lockName) -) - -def handleUpdateProjectLocked(*serve.Context c) ( - object? loaded = loadRequestedProject(c) - if loaded == null return - object project = loaded - array previousTeam = project.collaborators.assertElse(array, []) - if !canEditProject(project, c) ( - c.forbidden("not your project") - return - ) object body = requestJSON(c) - if refused(c, updateScopeRefusal(c, project, body)) return + object knownIdentities = {} + for attempt 3 ( + if !updateProjectOnce(c, body, knownIdentities) ( + return + ) + ) + c.json(409, {ok: false, code: "project_changed", error: "The project changed while it was saving. Try again."}) +) - OwnerTransfer transfer = OwnerTransfer() - if refused(c, applyOwnerTransfer(c, project, body, transfer)) return - if refused(c, applyGroupTag(c, project, body)) return - object creatorBenefits = mistwarpBenefitsForProject(project) - if refused(c, applyBranding(project, body, creatorBenefits)) return - if refused(c, applyVanitySlug(project, body, creatorBenefits)) return - if refused(c, applyProjectText(project, body)) return - if refused(c, applyCredits(project, body)) return - applyProjectFlags(project, body) - if refused(c, projectSaleRefusal(body)) return - if refused(c, applyRemixShareUpdate(c, project, body)) return - applyTags(project, body) - if refused(c, applyGameProducts(project, body)) return - if refused(c, applyGameItems(project, body)) return - if refused(c, applyInventoryPolicy(project, body)) return - if refused(c, gameGrantRefusal(project, body)) return - applyCompatibility(project, body) - if refused(c, applyTeamUpdate(c, project, body)) return - - if refused(c, holdTransferredStorage(c, project, transfer)) return +def updateProjectOnce(*serve.Context c, object body, object knownIdentities) boolean ( + object? snapshot = loadRequestedProject(c) + if snapshot == null return false + UpdateLookups lookups = lookupsKnowing(knownIdentities) + UpdateOutcome outcome = UpdateOutcome() + defer ( + releaseStorageHolds(lookups, if outcome.saved then outcome.transfer.holdKey else "") + rememberIdentities(lookups, knownIdentities) + ) + void applyProjectUpdate(c, snapshot, body, lookups, OwnerTransfer()) + lookups.replay = true + runLocked(projectSaveLock(c.param("id")), def() -> ( + saveProjectUpdate(c, body, lookups, outcome) + )) + if lookups.missed ( + return true + ) + finishProjectUpdate(c, body, outcome) + return false +) + +def saveProjectUpdate(*serve.Context c, object body, UpdateLookups lookups, UpdateOutcome outcome) ( + auto loaded = loadProject(c.param("id")) + if !loaded.ok ( + outcome.refusal = refuse(404, "project not found") + return + ) + object project = loaded.project.assert(object) + outcome.previousTeam = project.collaborators.assertElse(array, []) + Refusal refusal = applyProjectUpdate(c, project, body, lookups, outcome.transfer) + if lookups.missed or isRefused(refusal) ( + outcome.refusal = refusal + return + ) project.edited = timestamp if !saveProject(project) ( - storageHoldRelease(transfer.toId, transfer.hold) - c.internalError("could not save project access") + outcome.refusal = refuse(500, "could not save project access") return ) upsertIndexEntry(project) - settleTransferredStorage(transfer) + outcome.saved = true + outcome.project = project +) + +def finishProjectUpdate(*serve.Context c, object body, UpdateOutcome outcome) ( + if !outcome.saved ( + void refused(c, outcome.refusal) + return + ) + settleTransferredStorage(outcome.transfer) if body.contains("collaborators") ( - notifyNewTeammates(project, previousTeam, contextString(c, "username")) + notifyNewTeammates(outcome.project, outcome.previousTeam, contextString(c, "username")) + ) + c.json(200, {ok: true, project: projectResponse(outcome.project, c)}) +) + +def applyProjectUpdate(*serve.Context c, object project, object body, UpdateLookups lookups, OwnerTransfer transfer) Refusal ( + if !canEditProjectWith(c, project, lookups) ( + return refuse(403, "not your project") + ) + Refusal refusal = applyAccessChanges(c, project, body, lookups, transfer) + if isRefused(refusal) return refusal + refusal = applyContentChanges(c, project, body, lookups) + if isRefused(refusal) return refusal + return holdTransferredStorage(c, project, transfer, lookups) +) + +def applyAccessChanges(*serve.Context c, object project, object body, UpdateLookups lookups, OwnerTransfer transfer) Refusal ( + Refusal refusal = updateScopeRefusal(c, project, body) + if isRefused(refusal) return refusal + refusal = applyOwnerTransfer(c, project, body, transfer, lookups) + if isRefused(refusal) return refusal + refusal = applyGroupTag(c, project, body, lookups) + if isRefused(refusal) return refusal + object creatorBenefits = lookupProjectBenefits(lookups, project) + refusal = applyBranding(project, body, creatorBenefits) + if isRefused(refusal) return refusal + return applyVanitySlug(project, body, creatorBenefits, lookups) +) + +def applyContentChanges(*serve.Context c, object project, object body, UpdateLookups lookups) Refusal ( + Refusal refusal = applyProjectText(project, body) + if isRefused(refusal) return refusal + refusal = applyCredits(project, body) + if isRefused(refusal) return refusal + applyProjectFlags(project, body) + refusal = projectSaleRefusal(body) + if isRefused(refusal) return refusal + refusal = applyRemixShareUpdate(c, project, body) + if isRefused(refusal) return refusal + refusal = applyTags(project, body) + if isRefused(refusal) return refusal + refusal = applyGameProducts(project, body) + if isRefused(refusal) return refusal + refusal = applyGameItems(project, body) + if isRefused(refusal) return refusal + refusal = applyInventoryPolicy(project, body) + if isRefused(refusal) return refusal + refusal = gameGrantRefusal(project, body) + if isRefused(refusal) return refusal + refusal = applyCompatibility(project, body) + if isRefused(refusal) return refusal + return applyTeamUpdate(c, project, body, lookups) +) + +def canEditProjectWith(*serve.Context c, object project, UpdateLookups lookups) boolean ( + if !contextBool(c, "authenticated") ( + return false + ) + if contextBool(c, "isAdmin") or lookupGroupContentManager(lookups, c, project) ( + return true ) - c.json(200, {ok: true, project: projectResponse(project, c)}) + string role = projectRole(project, contextString(c, "username")) + return role == "owner" or role == "maintainer" ) def updateScopeRefusal(*serve.Context c, object project, object body) Refusal ( @@ -67,16 +149,20 @@ def isOwnerOrAdmin(*serve.Context c, object project) boolean ( return projectRole(project, contextString(c, "username")) == "owner" or contextBool(c, "isAdmin") ) -def applyOwnerTransfer(*serve.Context c, object project, object body, OwnerTransfer transfer) Refusal ( +def applyOwnerTransfer(*serve.Context c, object project, object body, OwnerTransfer transfer, UpdateLookups lookups) Refusal ( if !body.contains("owner") ( return noRefusal() ) - boolean canTransfer = contextBool(c, "isAdmin") or projectRole(project, contextString(c, "username")) == "owner" or canManageGroupContent(project, c) + boolean canTransfer = contextBool(c, "isAdmin") or projectRole(project, contextString(c, "username")) == "owner" or lookupGroupContentManager(lookups, c, project) if !canTransfer ( return refuse(403, "you cannot transfer this project") ) - string requestedOwner = body.owner.toStr().strip() - string nextOwnerId = discoverUserIdentity(requestedOwner) + string requestedOwner = (body.owner ?? "").toStr().strip() + object identity = lookupIdentity(lookups, requestedOwner) + if identity.outcome == "unavailable" ( + return roturUnavailableRefusal(requestedOwner) + ) + string nextOwnerId = identity.id.toStr() if nextOwnerId == "" ( return refuse(400, "Rotur user not found") ) @@ -84,7 +170,7 @@ def applyOwnerTransfer(*serve.Context c, object project, object body, OwnerTrans if nextOwner == "" ( return refuse(400, "Rotur user not found") ) - number nextOwnerLimit = mistwarpBenefitsForUserId(nextOwnerId).maxProjects.toNum() + number nextOwnerLimit = lookupBenefits(lookups, nextOwnerId).maxProjects.toNum() boolean sameOwner = identityMatches((project.ownerId ?? "").toStr(), project.owner.toStr(), nextOwnerId, nextOwner) if !sameOwner and ownedProjectSlotCount(nextOwnerId, nextOwner) >= nextOwnerLimit ( return refuse(400, "the new owner has reached their project limit") @@ -100,54 +186,55 @@ def applyOwnerTransfer(*serve.Context c, object project, object body, OwnerTrans return noRefusal() ) -def applyGroupTag(*serve.Context c, object project, object body) Refusal ( +def applyGroupTag(*serve.Context c, object project, object body, UpdateLookups lookups) Refusal ( if !body.contains("groupTag") ( return noRefusal() ) - string nextGroupTag = normalizeGroupTag(body.groupTag.toStr()) - if body.groupTag.toStr().strip() == "" ( + string requestedGroupTag = (body.groupTag ?? "").toStr() + string nextGroupTag = normalizeGroupTag(requestedGroupTag) + if requestedGroupTag.strip() == "" ( nextGroupTag = "" ) else if nextGroupTag == "" ( return refuse(400, "invalid group tag") - ) else if !canManageGroupTag(nextGroupTag, c) ( + ) else if !lookupGroupManager(lookups, c, nextGroupTag) ( return refuse(403, "you cannot move this project into that group") ) project.groupTag = nextGroupTag return noRefusal() ) -def vanitySlugHeldElsewhere(object project, string slug) boolean ( +def vanitySlugHeldElsewhere(object project, string slug, UpdateLookups lookups) boolean ( array matches = vanityClaimEntries(slug) for i matches.len ( object holder = matches[i].assert(object) string holderId = holder.id.toStr() - if holderId != project.id.toStr() ( - string holderEligibility = vanityEligibility(holder) - string holderLock = "project-upload:" ++ holderId - boolean held = true - if sync.tryLock(holderLock) ( - held = applyVanityLapseLocked(holderId, slug, holderEligibility) == slug - sync.unlock(holderLock) - ) - if held ( - return true - ) + if holderId != project.id.toStr() and vanityHolderKeepsSlug(holderId, slug, lookupVanityEligibility(lookups, holder)) ( + return true ) ) return false ) -def applyVanitySlug(object project, object body, object creatorBenefits) Refusal ( +def vanityHolderKeepsSlug(string holderId, string slug, string eligibility) boolean ( + string holderLock = projectSaveLock(holderId) + if !sync.tryLock(holderLock) ( + return true + ) + defer sync.unlock(holderLock) + return applyVanityLapseLocked(holderId, slug, eligibility) == slug +) + +def applyVanitySlug(object project, object body, object creatorBenefits, UpdateLookups lookups) Refusal ( if !body.contains("vanitySlug") ( return noRefusal() ) - string slug = body.vanitySlug.toStr().toLower().strip() + string slug = (body.vanitySlug ?? "").toStr().toLower().strip() string currentSlug = if project.contains("vanitySlug") then project.vanitySlug.toStr() else "" Refusal invalidSlug = vanitySlugRefusal(slug, currentSlug, creatorBenefits) if isRefused(invalidSlug) ( return invalidSlug ) - if slug != "" and slug != currentSlug and vanitySlugHeldElsewhere(project, slug) ( + if slug != "" and slug != currentSlug and vanitySlugHeldElsewhere(project, slug, lookups) ( return refuseWithBody(409, {ok: false, error: "that vanity URL is already taken"}) ) if slug != currentSlug ( @@ -167,17 +254,20 @@ def applyRemixShareUpdate(*serve.Context c, object project, object body) Refusal return applyRemixShare(project, body.remixShare.toNum()) ) -def applyTeamUpdate(*serve.Context c, object project, object body) Refusal ( +def applyTeamUpdate(*serve.Context c, object project, object body, UpdateLookups lookups) Refusal ( if !body.contains("collaborators") ( return noRefusal() ) if !isOwnerOrAdmin(c, project) ( return refuse(403, "only the owner can manage the project team") ) - return applyCollaborators(project, body.collaborators.assert(array)) + if typeof(body.collaborators) != "array" ( + return refuse(400, "collaborators must be a list") + ) + return applyCollaborators(project, body.collaborators.assert(array), lookups) ) -def holdTransferredStorage(*serve.Context c, object project, OwnerTransfer transfer) Refusal ( +def holdTransferredStorage(*serve.Context c, object project, OwnerTransfer transfer, UpdateLookups lookups) Refusal ( transfer.hold = noStorageHold() transfer.toId = projectOwnerId(project) if transfer.fromId == "" or !usesRoturStorage(transfer.toId) ( @@ -185,7 +275,10 @@ def holdTransferredStorage(*serve.Context c, object project, OwnerTransfer trans ) object moving = {} moving[project.id.toStr()] = projectStorageSummary(project, loadAssetsIndex().assets.assertElse(object, {})) - transfer.hold = storageHold(transfer.toId, {projects: moving}, !contextBool(c, "isAdmin")) + object changes = {projects: moving} + boolean enforce = !contextBool(c, "isAdmin") + transfer.holdKey = storageHoldKey(transfer.toId, changes, enforce) + transfer.hold = lookupStorageHold(lookups, transfer.toId, changes, enforce) if transfer.hold.ok != true ( object storageRefusal = transfer.hold.refusal.assertElse(object, {}) return refuseWithBody(413, {ok: false, code: "storage_full", error: "The new owner's Rotur storage doesn't have room for this project.", needed: (storageRefusal.needed ?? 0).toNum()}) diff --git a/src/projects/project-upload-publish.osl b/src/projects/project-upload-publish.osl index 690100a..0250cd9 100644 --- a/src/projects/project-upload-publish.osl +++ b/src/projects/project-upload-publish.osl @@ -30,6 +30,7 @@ def rejectProjectFiles(*serve.Context c, object project, object uploadResult, Up def storeUploadExtensions(object project, object extensionResult, UploadRollback rollback) Refusal ( if !storeProjectExtensions(project.id.toStr(), storedProjectExtensions(project), extensionResult.extensions.assert(array), extensionResult.sources.assert(object)) ( + discardNewWorkspaceKeys(project.id.toStr(), storedWorkspaceLayers(project), rollback.previousWorkspaceKeys) storageHoldRelease(rollback.ownerId, rollback.hold) return refuse(500, "failed to store extension sources") ) diff --git a/src/projects/project-upload-publish.test.osl b/src/projects/project-upload-publish.test.osl index 68ee4e7..c932f42 100644 --- a/src/projects/project-upload-publish.test.osl +++ b/src/projects/project-upload-publish.test.osl @@ -28,8 +28,9 @@ def projectJsonSnapshotKey(string hash) string ( def projectWorkspaceKeys(object project) array ( return [] ) +array removedKeys = [] def removeProjectStorageKeyIfOrphaned(string key, string id, boolean workspace) ( - return + removedKeys.append(key) ) def roturStorageFullResponse(*serve.Context c, object refusal) ( return @@ -49,8 +50,9 @@ def contentHash(any body) string ( def storageHoldRelease(string userId, object hold) ( return ) +boolean extensionStoreWorks = true def storeProjectExtensions(string projectId, array previousExtensions, array extensions, object sources) boolean ( - return true + return extensionStoreWorks ) def storedProjectExtensions(object project) array ( return [] @@ -104,3 +106,11 @@ object filesError = projectFilesErrorBody({ok: false, error: "too many assets", testing.assert(filesError.ok == false and filesError.error == "too many assets" and !filesError.contains("status") and !filesError.contains("code"), "file errors expose only ok, error and code") testing.assert(projectFilesErrorBody({ok: false, error: "big", code: "project_too_large"}).code == "project_too_large", "file errors keep their code") +UploadRollback rollback = UploadRollback() +rollback.previousWorkspaceKeys = ["old"] +object uploaded = {id: "p1", workspaceLayers: ["new"]} +object collected = {extensions: [], sources: {}} +testing.assert(!isRefused(storeUploadExtensions(uploaded, collected, rollback)) and removedKeys.len == 0, "stored extensions keep the new history") +extensionStoreWorks = false +Refusal failedExtensions = storeUploadExtensions(uploaded, collected, rollback) +testing.assert(failedExtensions.status == 500 and removedKeys == ["new"], "failing to store extensions removes the new history and keeps the old") diff --git a/src/projects/project-upload.osl b/src/projects/project-upload.osl index 947416c..ed564fd 100644 --- a/src/projects/project-upload.osl +++ b/src/projects/project-upload.osl @@ -12,7 +12,7 @@ def handleCheckProjectAssets(*serve.Context c) ( return ) object body = requestJSON(c) - if !body.contains("assets") or body.assets == null ( + if !body.contains("assets") or typeof(body.assets) != "array" ( c.badRequest("assets list is required") return ) @@ -38,10 +38,9 @@ def handleCheckProjectAssets(*serve.Context c) ( ) def handleUploadProject(*serve.Context c) ( - string lockName = "project-history-" ++ c.param("id") - sync.lock(lockName) - handleUploadProjectLocked(c) - sync.unlock(lockName) + runLocked(projectSaveLock(c.param("id")), def() -> ( + handleUploadProjectLocked(c) + )) ) def handleUploadProjectLocked(*serve.Context c) ( @@ -88,18 +87,18 @@ def handleUploadProjectLocked(*serve.Context c) ( return ) if refused(c, archive.discardOn(storeUploadExtensions(project, extensionResult, rollback))) return - saveUploadThumbnail(c, project.id.toStr()) applyUploadedFiles(project, uploadResult, extensionResult, rollback) if !publishProjectJson(project, uploadResult) ( abandonUnpublishedUpload(c, project, rollback, archive) return ) - recordCreatingDay(contextString(c, "username")) if !saveProject(project) ( abandonUnsavedUpload(c, project, rollback, archive) return ) + saveUploadThumbnail(c, project.id.toStr()) + recordCreatingDay(contextString(c, "username")) settleSavedUpload(project, rollback, uploader) archive.discard() diff --git a/src/projects/project-vanity.osl b/src/projects/project-vanity.osl index 592cb56..be6bb89 100644 --- a/src/projects/project-vanity.osl +++ b/src/projects/project-vanity.osl @@ -42,11 +42,6 @@ def vanityEntriesWhere(string op, string value) array ( return found ) -// Records when the project's vanity URL lapsed, clears that once the owner's -// plan includes vanity URLs again, and releases the URL after 30 days lapsed. -// Call with the project's upload lock held, and its owner's eligibility looked -// up beforehand, so a slow Rotur never holds the lock. Returns the URL it -// still holds. def applyVanityLapseLocked(string id, string slug, string eligibility) string ( auto loaded = loadProject(id) if !loaded.ok ( @@ -69,6 +64,13 @@ def applyVanityLapseLocked(string id, string slug, string eligibility) string ( return after.slug.toStr() ) +def applyVanityLapse(string id, string slug, string eligibility) string ( + string lockName = projectSaveLock(id) + sync.lock(lockName) + defer sync.unlock(lockName) + return applyVanityLapseLocked(id, slug, eligibility) +) + // Notices lapsed vanity URLs and releases them on time, even if nobody visits. def vanityLapseLoop() ( while true ( @@ -81,10 +83,7 @@ def vanityLapseLoop() ( boolean lapsed = (entry.vanitySlug ?? "").toStr() != slug // Nothing changes while Rotur can't be reached, or for a URL in use. if eligibility == "no" or (eligibility == "yes" and lapsed) ( - string lockName = "project-upload:" ++ entry.id.toStr() - sync.lock(lockName) - applyVanityLapseLocked(entry.id.toStr(), slug, eligibility) - sync.unlock(lockName) + void applyVanityLapse(entry.id.toStr(), slug, eligibility) ) ) ) diff --git a/src/projects/upload-parts.osl b/src/projects/upload-parts.osl index 7d2f6e8..6268440 100644 --- a/src/projects/upload-parts.osl +++ b/src/projects/upload-parts.osl @@ -33,13 +33,13 @@ def consumeStagedUploadPart(string token, string projectId, string kind, string if metadata.projectId.toStr() != projectId or metadata.kind.toStr() != kind or metadata.userId.toStr() != userId ( return {ok: false, error: "staged upload does not match this project"} ) + if metadata.contains("complete") and metadata.complete != true and metadata.expires.toNum() >= timestamp ( + return {ok: false, error: "staged upload is incomplete"} + ) if metadata.expires.toNum() < timestamp or !fs.exists(stagedUploadPartDataPath(token)) ( removeStagedUploadPart(token, metadata) return {ok: false, error: "staged upload expired; upload it again"} ) - if metadata.contains("complete") and metadata.complete != true ( - return {ok: false, error: "staged upload is incomplete"} - ) byte[] data = fs.readFileBytes(stagedUploadPartDataPath(token)).unwrapOr("".toBytes()) if data.len != metadata.size.toNum() ( removeStagedUploadPart(token, metadata) @@ -188,7 +188,7 @@ def uploadPartChunkPath(string token, number index) string ( def removeUploadPartChunks(string token, number count) ( for i count ( - void fs.remove(uploadPartChunkPath(token, i)) + void fs.remove(uploadPartChunkPath(token, i - 1)) ) ) @@ -196,7 +196,7 @@ def assembleUploadPart(string token, number count, number expectedBytes) boolean string outputPath = stagedUploadPartDataPath(token) array chunkPaths = [] for i count ( - chunkPaths.append(uploadPartChunkPath(token, i)) + chunkPaths.append(uploadPartChunkPath(token, i - 1)) ) return storagefs.ConcatenateFiles(outputPath, chunkPaths, expectedBytes) ) @@ -213,8 +213,8 @@ def handleCreateProjectUploadPart(*serve.Context c) ( return ) object body = requestJSON(c) - string kind = body.kind.toStr() - string name = body.name.toStr() + string kind = (body.kind ?? "").toStr() + string name = (body.name ?? "").toStr() number size = body.size.toNum().toInt() number chunks = body.chunks.toNum().toInt() if kind != "asset" and kind != "workspace" and kind != "mergeTree" ( diff --git a/src/projects/upload-parts.test.osl b/src/projects/upload-parts.test.osl new file mode 100644 index 0000000..95e1f5e --- /dev/null +++ b/src/projects/upload-parts.test.osl @@ -0,0 +1,101 @@ +import "std:testing" +import "std:fs" +import "std:json" +import "std:regex" +import "std:sync" +import "std:serve" +import "go:mistwarp.local/storagefs" +import "./upload-parts.osl" + +number maxWorkspaceBytes = 1048576 +number maxWorkspaceFiles = 10 +number maxProjectJsonBytes = 1048576 +number maxUploadChunkBytes = 1048576 +def readFile(string path) object ( + auto text = fs.readFile(path) + if text.isErr() ( + return {ok: false} + ) + return {ok: true, data: json.parse(text.unwrap()).unwrap()} +) +def writeFile(string path, object data) boolean ( + return fs.writeFile(path, json.stringify(data)) +) +def randomString(number length) string ( + return "r" * length +) +def loadProject(string id) object ( + return {ok: false} +) +def canSaveLiveProject(object project, *serve.Context c) boolean ( + return false +) +def isValidMd5Ext(string name) boolean ( + return true +) +def md5PartOf(string name) string ( + return name +) +def mistwarpBenefitsForUserId(string userId) object ( + return {maxProjectAssetBytes: 1, maxProjectAssetsBytes: 1} +) +def loadAssetsIndex() object ( + return {assets: {}} +) +def saveAssetsIndex(object index) boolean ( + return true +) +def holdUploadAhead(*serve.Context c, object project, string name, number bytes) boolean ( + return false +) +def releaseUploadAhead(*serve.Context c, object project, array names) ( + return +) +def stageR2Blob(string key, any body, string contentType, string contentEncoding) boolean ( + return false +) +def r2Remove(string key) boolean ( + return true +) +def assetObjectKey(string name) string ( + return name +) +def contentTypeForExt(string ext) string ( + return "" +) +def extPartOf(string name) string ( + return "" +) +def contextString(*serve.Context c, string key) string ( + return "" +) +def requestJSON(*serve.Context c) object ( + return {} +) +def validateZipArchive(byte[] body, number maxCompressedBytes, number maxExpandedBytes, number maxFiles, boolean readContents) boolean ( + return false +) + +string tmpDir = "/tmp/mistwarp-upload-parts-test-" ++ timestamp.toStr() ++ "/" +void fs.mkdir(tmpDir, {recursive: true}) + +for index 3 ( + void fs.writeFile(uploadPartChunkPath("tok", index - 1), "part" ++ (index - 1).toStr() ++ ";") +) +testing.assert(assembleUploadPart("tok", 3, 18), "chunks stored from index 0 assemble") +testing.assert(fs.readFile(stagedUploadPartDataPath("tok")).unwrapOr("") == "part0;part1;part2;", "chunks are joined in order") +removeUploadPartChunks("tok", 3) +testing.assert(!fs.exists(uploadPartChunkPath("tok", 0)) and !fs.exists(uploadPartChunkPath("tok", 2)), "every chunk is removed") + +void writeFile(stagedUploadPartMetaPath("open"), {projectId: "p1", kind: "workspace", userId: "u1", size: 4, chunks: 1, complete: false, expires: timestamp + 60000}) +testing.assert(consumeStagedUploadPart("open", "p1", "workspace", "u1").error == "staged upload is incomplete", "an unfinished upload is refused") +testing.assert(fs.exists(stagedUploadPartMetaPath("open")), "refusing an unfinished upload keeps it so it can still complete") +void fs.writeFile(stagedUploadPartDataPath("open"), "data") +object finished = readFile(stagedUploadPartMetaPath("open")).data.assert(object) +finished.complete = true +void writeFile(stagedUploadPartMetaPath("open"), finished) +object consumed = consumeStagedUploadPart("open", "p1", "workspace", "u1") +testing.assert(consumed.ok == true and consumed.file.size == 4, "a completed upload is consumed") +testing.assert(consumeStagedUploadPart("open", "p1", "workspace", "u1").ok == false, "a consumed upload cannot be used again") + +void fs.remove(tmpDir)