diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b22f980..eee160b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,7 +23,7 @@ jobs: - name: Install OSL env: - OSL_COMMIT: 5895982fa61d3e7c9e9c7f819c9e33f645d04f0f + OSL_COMMIT: 78deaec4824587cf9d71efcf5c594b521e3ab167 run: | mkdir -p "$HOME/.local/bin" git clone --filter=blob:none https://git.rotur.dev/osl/.go /tmp/osl-source diff --git a/AGENTS.md b/AGENTS.md index 96eeae5..e884e65 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,7 +22,9 @@ The editor and site depend on every path, method, middleware order, body limit a ## Checks -`osl test`, `osl lsp check main.osl`, `osl compile main.osl -o main`, and `python3 scripts/check_conventions.py`. The last one rejects code comments and em dashes in the lines you add. CI (`.github/workflows/test.yml`) runs all of them plus the smoke test, with OSL pinned to a commit. +`osl test`, `osl lsp check main.osl`, `osl compile main.osl -o main`, and `python3 scripts/check_conventions.py`. CI (`.github/workflows/test.yml`) runs all of them plus the smoke test, with OSL pinned to a commit. + +OSL style is enforced by the `lint` section of `opal.json`, which OSL's own linter applies in every check, test and compile. It makes comments (`OSL-COMMENT`) and top-level variables without a declared type (`OSL-UNTYPED-GLOBAL`) errors. `OSL-OBJECT-PARAM` and `OSL-LONG-FUNCTION` are warnings for now. The only comments allowed are the `// osl-disable-next-line` and `// osl-disable-file` directives. They must name the codes they turn off and can never turn off `OSL-COMMENT`, which the conventions script checks. The caches that hold whole-collection snapshots stay `auto` behind `// osl-disable-next-line OSL-UNTYPED-GLOBAL`, because a typed `*cache.Cache` converts every value it returns and those values are read on every request. `scripts/check_conventions.py` covers everything else: it rejects code comments in the Go and Python lines you add, and em dashes in the added lines of every text file, OSL included. ## Working here diff --git a/main.osl b/main.osl index a06b1ff..9229350 100644 --- a/main.osl +++ b/main.osl @@ -66,9 +66,6 @@ if !r2Local and appURL == "http://localhost:5627" ( log "WARNING: APP_URL points to localhost but R2 is configured; thumbnails and pending project.json are served from APP_URL and will be unreachable. Set APP_URL to the public https API base." ) -// `main storage-report ` writes what MistWarp stores for everyone, for -// the Rotur API's `claw storage-backfill`, and exits. Run it with the API -// stopped. array commandArgs = process.args() if commandArgs.len >= 1 and commandArgs[1].toStr() == "storage-report" ( if commandArgs.len != 2 ( diff --git a/opal.json b/opal.json index 3da095f..e1069fb 100644 --- a/opal.json +++ b/opal.json @@ -10,5 +10,13 @@ "routes": ["python3", "scripts/route_dump.py", "."], "smoke": ["python3", "scripts/e2e_smoke.py"], "conventions": ["python3", "scripts/check_conventions.py"] + }, + "lint": { + "rules": { + "OSL-COMMENT": "error", + "OSL-UNTYPED-GLOBAL": "error", + "OSL-OBJECT-PARAM": "warning", + "OSL-LONG-FUNCTION": "warning" + } } } diff --git a/scripts/check_conventions.py b/scripts/check_conventions.py index ded89f6..94fa6eb 100644 --- a/scripts/check_conventions.py +++ b/scripts/check_conventions.py @@ -9,6 +9,7 @@ TEXT_FILE = re.compile(r"\.(osl|go|py|md|json|yml|yaml|sh|txt)$") SKIPPED = [re.compile(r"(^|/)go\.sum$"), re.compile(r"^data/")] DIRECTIVE = re.compile(r"^(//|#)\s*(nolint|noqa|go:|lint:|type:\s*ignore)") +OSL_DISABLE = re.compile(r"^\s*//\s*osl-disable-(?:file|next-line)(?P(?:[ \t].*)?)$") def git(*args): @@ -79,7 +80,7 @@ def python_comments(text): return found -SCANNERS = {".osl": c_style_comments, ".go": c_style_comments, ".py": python_comments} +SCANNERS = {".go": c_style_comments, ".py": python_comments} def main(): @@ -97,6 +98,9 @@ def main(): for line, content in added.items(): if EM_DASH in content and content.strip() not in removed: problems.append(f'{path}:{line} em dash: use a comma, parentheses or "to"') + escape = OSL_DISABLE.match(content) if path.endswith(".osl") else None + if escape and (not escape.group("codes").strip() or "OSL-COMMENT" in escape.group("codes")): + problems.append(f"{path}:{line} lint directive: name the codes it turns off, and never OSL-COMMENT") scan = next((scanner for suffix, scanner in SCANNERS.items() if path.endswith(suffix)), None) if scan is None: continue diff --git a/src/admin/admin.osl b/src/admin/admin.osl index ab8ad31..8819839 100644 --- a/src/admin/admin.osl +++ b/src/admin/admin.osl @@ -15,8 +15,9 @@ export { unbanUser } +// osl-disable-next-line OSL-UNTYPED-GLOBAL auto bansCache = cache.new({capacity: 2, ttl: 300000}) -auto adminsCache = cache.new({capacity: 2, ttl: 300000}) +*cache.Cache adminsCache = cache.new({capacity: 2, ttl: 300000}) def loadAdmins() array ( auto res = readFile(adminsFile) @@ -64,13 +65,10 @@ def cachedBans() object ( return data ) -// Rebuilt whenever the bans file is saved (a ban, an unban, a rename, or the -// Rotur sync dropping a ban that ended there), and at most 5 minutes old. def bannedOwners() object ( if bansCache.has("owners") ( return bansCache.get("owners").assert(object) ) - // One rebuild at a time; the rest wait for it and use its result. sync.lock("banned-owners") defer sync.unlock("banned-owners") if bansCache.has("owners") ( @@ -105,7 +103,6 @@ def projectsVisibleTo(array entries,*serve.Context c) array ( return projectEntriesListed(entries, bannedOwners(), shadowBannedSet(), normalizeUsername(contextString(c, "username")), contextString(c, "userId"), contextBool(c, "isAdmin")) ) -// For lists that are cached publicly, so never differ for admins. def projectsVisibleToEveryone(array entries) array ( return projectEntriesListed(entries, bannedOwners(), shadowBannedSet(), "", "", false) ) @@ -137,23 +134,14 @@ def banReasonFor(string username) string ( return "" ) -// MistWarp's bans are also bans from MistWarp's Rotur App, so Rotur enforces -// them: the person's MistWarp tokens stop working, and they can't sign in to -// MistWarp again until the ban is lifted. They see the reason, never who banned -// them. The local list stays the source MistWarp checks on every request. - def roturBanPath(string username) string ( return "/bans/" ++ url.escape(roturUserRef(username)) ) -// "yes" once Rotur has the ban, "refused" if Rotur won't ban them (they are on -// the app's team, a classroom student, or not a Rotur account it can ban), "" -// to try again later. def pushRoturBan(string username, string reason) string ( string shown = reason.strip() - // Older bans from a report only name the report, which means nothing to the - // person banned. - if shown == "" or regex.match("^report r[A-Za-z0-9_-]+$", shown) ( + boolean namesOnlyTheReport = regex.match("^report r[A-Za-z0-9_-]+$", shown) + if shown == "" or namesOnlyTheReport ( shown = "You broke MistWarp's community guidelines." ) object res = roturApp("PUT", roturBanPath(username), {reason: shown.trim(1, 300), note: "Banned in MistWarp"}) @@ -166,7 +154,6 @@ def pushRoturBan(string username, string reason) string ( return "" ) -// Lifts the ban on Rotur. True once Rotur no longer has it. def liftRoturBan(string username) boolean ( object res = roturApp("DELETE", roturBanPath(username), {}) return res.ok or res.status == 404 @@ -205,8 +192,6 @@ def unbanUser(string username) boolean ( if !cachedBans().bans.assertElse(object, {}).contains(key) ( return false ) - // Remembered until a sync has seen Rotur without it, so the sync keeps - // trying if Rotur couldn't be told, and never brings the ban back. boolean lifted = liftRoturBan(username) sync.lock("bans") defer sync.unlock("bans") @@ -218,15 +203,11 @@ def unbanUser(string username) boolean ( return saveBans(data) ) -// Keeps MistWarp's bans and the Rotur App's in step, every few minutes: bans -// made or lifted on rotur.dev/me/developer reach MistWarp, and bans and unbans -// Rotur couldn't be told about yet are sent again. def syncRoturBans() ( number started = timestamp object snapshot = bansSnapshotLocked() object snapshotBans = snapshot.bans.assertElse(object, {}) object snapshotUnbans = snapshot.unbans - // Unbans first, so the list read next already reflects them. object liftedNow = {} array unbanKeys = snapshotUnbans.getKeys() for i unbanKeys.len ( @@ -239,8 +220,6 @@ def syncRoturBans() ( if !res.ok ( return ) - // Rotur lists accounts it has banned itself with no username, so match on - // the ID too. object remote = {} object remoteIds = {} array listed = (res.data.bans ?? []).assertElse(array, []) @@ -277,7 +256,6 @@ def syncRoturBans() ( object unban = unbans[key].assertElse(object, {}) boolean lifted = unban.lifted == true or liftedNow.contains(key) if lifted and (unban.at ?? 0).toNum() < started ( - // Rotur's list was read after it lifted this one. unbans.delete(key) changed = true ) else if lifted and unban.lifted != true ( @@ -286,8 +264,6 @@ def syncRoturBans() ( changed = true ) ) - // Unbanned while the ban was being pushed: Rotur has it again, so lift it - // again next time. array pushedKeys = pushed.getKeys() for i pushedKeys.len ( string key = pushedKeys[i].toStr() @@ -307,7 +283,6 @@ def syncRoturBans() ( ) string state = (ban.rotur ?? "").toStr() if ban.by.toStr() == "rotur_sync" ( - // Rotur enforces its own account bans now; these copies were never lifted. bans.delete(key) changed = true ) else if state == "" and pushed.contains(key) and pushed[key] != "" ( @@ -315,13 +290,10 @@ def syncRoturBans() ( bans[key] = ban changed = true ) else if state == "yes" and ban.created.toNum() < started and !remote.contains(key) and !remoteIds.contains(userIdForUsername(key)) ( - // Lifted on rotur.dev. bans.delete(key) changed = true ) ) - // Banned again while the old ban was being lifted: the lift may have - // removed the new ban from Rotur, so push it again. array liftedKeys = liftedNow.getKeys() for i liftedKeys.len ( string key = liftedKeys[i].toStr() @@ -821,8 +793,6 @@ def handleAdminSearchProjects(*serve.Context c) ( c.json(200, {ok: true, projects: matches}) ) -// Who a report is about, as worked out from the content when it was filed. -// Older reports are worked out now, never from what the reporter said. def reportTargetUser(object found) string ( string subject = (found.subject ?? "").toStr() if subject != "" ( diff --git a/src/admin/banned-owners.osl b/src/admin/banned-owners.osl index 1abf7c1..1f6963a 100644 --- a/src/admin/banned-owners.osl +++ b/src/admin/banned-owners.osl @@ -2,11 +2,6 @@ export { entryOwnerBanned } -// A banned user's projects are hidden everywhere except from admins. Bans are -// kept by username; the set also holds each banned user's ID, so a project -// still matches if its stored owner name is out of date after a rename. -// Nothing is moved or deleted: lifting the ban brings everything back. - def bannedOwnerSet(object bans, object idsByKey) object ( object names = {} object ids = {} @@ -38,13 +33,10 @@ def entryOwnerBanned(object banned, object entry) boolean ( return ownerInBannedSet(banned, (entry.owner ?? "").toStr(), (entry.ownerId ?? "").toStr()) ) -// A direct lookup of a hidden project answers like a missing one. def bannedProjectHidden(object banned, object project, boolean isAdmin) boolean ( return !isAdmin and entryOwnerBanned(banned, project) ) -// Always a new list, so callers can sort it without touching a cached one. -// Admins get every entry, with banned owners' copies marked ownerBanned. def projectEntriesVisible(array entries, object banned, boolean isAdmin) array ( array visible = [] for i entries.len ( diff --git a/src/admin/banned-owners.test.osl b/src/admin/banned-owners.test.osl index 5261731..ac8a13f 100644 --- a/src/admin/banned-owners.test.osl +++ b/src/admin/banned-owners.test.osl @@ -22,7 +22,6 @@ array entries = [ {id: "e", owner: "bob", ownerId: "u-bob"} ] -// Bans are kept by normalized username; IDs are looked up when the set is built. object bans = {spammer: {reason: "spam"}, legacy: {reason: "spam"}} object banned = bannedOwnerSet(bans, {spammer: "u-spam", legacy: ""}) @@ -42,8 +41,6 @@ testing.assert(bannedProjectHidden(banned, entries[2], false), "a direct lookup testing.assert(!bannedProjectHidden(banned, entries[2], true), "a direct lookup by an admin works") testing.assert(!bannedProjectHidden(banned, entries[1], false), "other owners' projects are found") -// Unbanning, or the Rotur sync dropping a ban once it ends there, rewrites the -// bans file and the set is rebuilt from it. object afterUnban = bannedOwnerSet({legacy: {reason: "spam"}}, {legacy: ""}) testing.assert(ids(projectEntriesVisible(entries, afterUnban, false)) == "abce", "an unbanned owner's projects come back") testing.assert(!bannedProjectHidden(afterUnban, entries[2], false), "an unbanned owner's project is found again") diff --git a/src/admin/errors.test.osl b/src/admin/errors.test.osl index 84e6b51..e9aeca9 100644 --- a/src/admin/errors.test.osl +++ b/src/admin/errors.test.osl @@ -7,7 +7,7 @@ import "std:regex" import "../http/request-body.osl" string path = "/tmp/mistwarp-errors-" ++ timestamp.toStr() ++ ".db" -auto testDb = db.open(path) +*db.DB testDb = db.open(path) *db.Collection errorDocs = testDb.collection("site_errors") array errorReportOrigins = ["https://mistwarp.org"] diff --git a/src/admin/project-moderation.osl b/src/admin/project-moderation.osl index 4781f4a..38a3bd0 100644 --- a/src/admin/project-moderation.osl +++ b/src/admin/project-moderation.osl @@ -12,6 +12,7 @@ export { viewerSeesShadowBanned } +// osl-disable-next-line OSL-UNTYPED-GLOBAL auto shadowBansCache = cache.new({capacity: 2, ttl: 300000}) def loadShadowBans() object ( diff --git a/src/auth/account-controls.osl b/src/auth/account-controls.osl index b260033..be48568 100644 --- a/src/auth/account-controls.osl +++ b/src/auth/account-controls.osl @@ -190,10 +190,6 @@ def handleExportMyData(*serve.Context c) ( }) ) -// Delete everything MistWarp holds about someone: their projects and what -// hangs off them, their community records, game data and backpack, then the -// account itself. Used when they ask, and when Rotur says they deleted their -// Rotur account, left MistWarp or were banned from Rotur (rotur-webhooks.osl). def eraseUser(string userId, string username) ( eraseClassroomTeacher(userId) purgeGameDataForUser(userId) @@ -248,7 +244,6 @@ def eraseUser(string userId, string username) ( purgeUser(userId, username) removeUserFromSafetyLists(username) void fs.remove(safetyPath(username)) - // Nothing of theirs is stored any more, so none of it counts. void ledgerFreeAll(userId) ) diff --git a/src/auth/agreement.osl b/src/auth/agreement.osl index ddf64be..63e01ab 100644 --- a/src/auth/agreement.osl +++ b/src/auth/agreement.osl @@ -20,7 +20,6 @@ def loadAgreement() object ( if metaResult.ok ( meta = metaResult.data.assert(object) ) - // Read text from .md file, fall back to the text field in json if fs.exists(agreementTextPath()) ( meta.text = fs.readFile(agreementTextPath()).unwrapOr("") ) @@ -28,11 +27,10 @@ def loadAgreement() object ( ) def saveAgreement(object data) boolean ( - // Write metadata to json, text to md string text = data.text - data.text = "" // don't store the text in json + data.text = "" bool ok = writeFile(agreementMetaPath(), data) - data.text = text // restore + data.text = text if text != "" ( if !fs.writeFile(agreementTextPath(), text) ( ok = false diff --git a/src/auth/auth.osl b/src/auth/auth.osl index e7daf0a..8bdf3b1 100644 --- a/src/auth/auth.osl +++ b/src/auth/auth.osl @@ -7,7 +7,7 @@ export { setAuthCookie } -auto sessionCache = cache.new({capacity: 10000, ttl: 3600000}) +*cache.Cache sessionCache = cache.new({capacity: 10000, ttl: 3600000}) def setAuthCookie(*serve.Context c, string sessionId) ( c.setCookie("auth_token", sessionId, {maxAge: 604800000, path: "/", domain: "", secure: appURL.startsWith("https://"), httpOnly: true}) @@ -135,10 +135,6 @@ def handleRoturAuth(*serve.Context c) ( return ) - // The editor makes validators keyed to MistWarp's Rotur App, which its own - // sign-in token can do without validators:generate; Rotur then also refuses - // anyone the app has banned. Older editors use the old key. A validator - // checked against the wrong key is just invalid, so try the next. array keys = [roturAppKey] if roturClientId != "" ( keys = [roturClientId, roturAppKey] @@ -146,7 +142,6 @@ def handleRoturAuth(*serve.Context c) ( object data = {} for i keys.len ( object resp = requests.get(roturBase ++ "/validate?key=" ++ url.escape(keys[i].toStr()) ++ "&v=" ++ url.escape(validator)).unwrapOr({body: null, error: "request failed", headers: {}, raw: null, status: 0, success: false}) - // Rotur answers every validator with 200, valid or not. number status = (resp.status ?? 0).toNum().toInt() if status < 200 or status >= 300 ( c.json(401, {ok: false, error: "failed to validate token"}) @@ -163,13 +158,10 @@ def handleRoturAuth(*serve.Context c) ( ) ) - // Rotur answers every "no" with valid false and a code. Only a real "yes" - // signs anyone in. if data.valid != true ( string code = (data.code ?? "").toStr() if code != "" and code != "invalid_validator" ( object refusal = {ok: false, code: code, error: (data.error ?? "Rotur refused this sign-in.").toStr(), redirectUrl: (data.redirect_url ?? "https://rotur.dev/me").toStr()} - // An app ban says why, and until when. if (data.reason ?? "").toStr() != "" ( refusal.reason = data.reason.toStr() ) @@ -183,8 +175,6 @@ def handleRoturAuth(*serve.Context c) ( return ) - // Old-key validators skip Rotur's checks for the app. One made by another - // Rotur App's sign-in was passed on to MistWarp, so it isn't accepted. string madeBy = (data.app ?? "").toStr() if roturClientId != "" and madeBy.startsWith("app_") and madeBy != roturClientId ( c.json(401, {ok: false, error: "this sign-in was made for another app", redirectUrl: "https://rotur.dev"}) diff --git a/src/auth/user-identity.osl b/src/auth/user-identity.osl index e9fa4f5..5210f47 100644 --- a/src/auth/user-identity.osl +++ b/src/auth/user-identity.osl @@ -15,8 +15,8 @@ export { usernameForId } -auto userIdsCache = cache.new({capacity: 10000, ttl: 300000}) -auto identityRefreshCache = cache.new({capacity: 10000, ttl: 300000}) +*cache.Cache userIdsCache = cache.new({capacity: 10000, ttl: 300000}) +*cache.Cache identityRefreshCache = cache.new({capacity: 10000, ttl: 300000}) def loadUserIds() object ( auto res = readFile(userIdsFile) @@ -502,13 +502,13 @@ def ensureUserIdentity(string userId, string username) ( userIdsCache.set("username:" ++ normalizeUsername(username), userId) ) -// Requests to Rotur give up after 10 seconds, so a slow Rotur can't hold up -// MistWarp's own requests (perks are read while serving projects). +number roturRequestTimeoutSeconds = 10 + def roturServiceOptions() object ( if mistwarpRoturToken == "" ( - return {timeout: 10} + return {timeout: roturRequestTimeoutSeconds} ) - return {headers: {"Authorization": "Bearer " ++ mistwarpRoturToken}, timeout: 10} + return {headers: {"Authorization": "Bearer " ++ mistwarpRoturToken}, timeout: roturRequestTimeoutSeconds} ) def refreshUsernameForId(string userId) string ( diff --git a/src/classroom/classroom-legal.osl b/src/classroom/classroom-legal.osl index efd9145..86a588f 100644 --- a/src/classroom/classroom-legal.osl +++ b/src/classroom/classroom-legal.osl @@ -259,9 +259,6 @@ def handleAdminPurgeClassroom(*serve.Context c) ( c.json(200, {ok: true, classes: classes.len, students: students}) ) -// Called when a teacher's account is erased. Each class they owned passes to -// a co-teacher, or is deleted with its students when it has none, and they -// leave every class and school they taught in. def eraseClassroomTeacher(string userId) ( if userId == "" or isStudentUserId(userId) ( return diff --git a/src/classroom/classroom-policy.osl b/src/classroom/classroom-policy.osl index b132526..7101107 100644 --- a/src/classroom/classroom-policy.osl +++ b/src/classroom/classroom-policy.osl @@ -277,9 +277,6 @@ def submissionIsLate(number dueAt, number turnedInAt) boolean ( return dueAt > 0 and turnedInAt > dueAt ) -// The class as it should be once a teacher's account is erased: a co-teacher -// takes over a class the teacher owned, and the teacher leaves any class they -// co-taught. An owned class with no co-teacher returns {} and is deleted. def classAfterTeacherErased(object classDoc, string userId) object ( array kept = [] array coTeachers = (classDoc.coTeachers ?? []).assertElse(array, []) @@ -303,8 +300,6 @@ def classAfterTeacherErased(object classDoc, string userId) object ( return next ) -// The school without an erased teacher. When they were its last admin, the -// first remaining teacher becomes admin so the school can still be managed. def schoolAfterTeacherErased(object school, string userId) object ( object next = school.clone() array teachers = (school.teacherIds ?? []).assertElse(array, []).filter((any id) boolean -> id.toStr() != userId) diff --git a/src/classroom/classroom-switch.test.osl b/src/classroom/classroom-switch.test.osl index 2bbae3e..3df262c 100644 --- a/src/classroom/classroom-switch.test.osl +++ b/src/classroom/classroom-switch.test.osl @@ -13,7 +13,7 @@ import "./classroom-policy.osl" import "./classroom-switch-policy.osl" string dbPath = "/tmp/mistwarp-classroom-switch-" ++ timestamp.toStr() ++ ".db" -auto testDb = db.open(dbPath) +*db.DB testDb = db.open(dbPath) *db.Collection classDocs = testDb.collection("classes") number maxStandardRequestBytes = 200 @@ -227,7 +227,7 @@ testing.assert(!classroomSessionAllowed("mwc_other"), "sessions of other student testing.assert(!classroomSessionAllowed("mwc_off"), "sessions of disabled students are refused") testing.assert(!classroomSessionAllowed("mwc_missing"), "sessions of deleted students are refused") -classroomEnabled = true +boolean classroomEnabled = true testing.assert(call("GET", "/v1/classroom", "teen", "").status == 200, "with the flag on, every teacher can use Classroom") testing.assert(call("GET", "/v1/classroom/join/OTHER234", "", "").status == 200, "with the flag on, every join code works") testing.assert(call("POST", "/v1/classroom/login", "", "{\"studentId\":\"mwc_other\"}").status == 200, "with the flag on, every student can sign in") diff --git a/src/classroom/classrooms.osl b/src/classroom/classrooms.osl index e1808c6..1f82f39 100644 --- a/src/classroom/classrooms.osl +++ b/src/classroom/classrooms.osl @@ -6,7 +6,7 @@ export { loadStudent } -auto classroomPoolCache = cache.new({capacity: 10000, ttl: 60000}) +*cache.Cache classroomPoolCache = cache.new({capacity: 10000, ttl: 60000}) def randomFromAlphabet(string alphabet, number length) string ( string out = "" diff --git a/src/community/development-links.osl b/src/community/development-links.osl index f1ea07c..91edf3e 100644 --- a/src/community/development-links.osl +++ b/src/community/development-links.osl @@ -1,10 +1,5 @@ export {} -// Pure helpers behind the development feed: normalising a GitHub search item -// into a feed entry, parsing the pull request references an admin types, and -// resolving which pull requests belong to a roadmap entry. Kept separate from -// development.osl so the linking rules can be tested without HTTP or a cache. - number developmentMaxLinksPerIdea = 12 def pullLinkKey(string repo, number index) string ( @@ -15,8 +10,6 @@ def pullWebUrl(string repo, number index) string ( return "https://github.com/" ++ githubOrg ++ "/" ++ repo ++ "/pull/" ++ index.toStr() ) -// Contributors link their own work by naming the entry in the pull request: -// a /roadmap/entry/ link, "Roadmap: ", or a bare "#". def roadmapReferencesIn(string text) array ( array found = [] array matches = regex.findAll("(roadmap/entry/|[Rr]oadmap: *|#)i[A-Za-z0-9]{14}", text) @@ -102,7 +95,6 @@ def collectDevelopmentPulls(array items) array ( array pulls = [] for i items.len ( object item = items[i].assertElse(object, {}) - // Search also matches issues; only entries carrying a pull_request are ours. if item.contains("pull_request") and item.pull_request != null and item.contains("number") ( object pull = normalizeDevelopmentPull(item) if pull.repo.toStr() != "" ( @@ -133,8 +125,6 @@ def dedupeDevelopmentPulls(array first, array second) array ( return pulls ) -// Accepts "scratch-gui#218", "MistWarp/scratch-gui#218", or a pasted -// https://github.com/MistWarp/scratch-gui/pull/218 link. def parsePullLink(string raw) object ( string value = raw.strip() if value == "" or value.len > 300 ( @@ -146,7 +136,6 @@ def parsePullLink(string raw) object ( if parts.len != 2 ( return {ok: false} ) - // Either "repo" or "owner/repo"; anything else is not a reference we wrote. array repoSegments = parts[1].toStr().strip().split("/") if repoSegments.len < 1 or repoSegments.len > 2 ( return {ok: false} @@ -173,7 +162,6 @@ def normalizePullLinks(array raw) array ( array links = [] array seen = [] for i raw.len ( - // Clients send strings; stored links round-trip as {repo, number} objects. string reference = "" if typeof(raw[i]) == "object" ( object link = raw[i].assertElse(object, {}) @@ -195,9 +183,6 @@ def normalizePullLinks(array raw) array ( return links ) -// A manual link stays visible even when the feed has not seen that pull -// request — an older merge drops out of the window, but the entry still -// deserves to show what shipped it. def developmentPullFor(array pulls, string key, string repo, number index) object ( for i pulls.len ( object pull = pulls[i].assertElse(object, {}) @@ -224,9 +209,6 @@ def developmentPullFor(array pulls, string key, string repo, number index) objec } ) -// Copied rather than tagged in place: the same pull request can be a manual -// link on one entry and a reference match on another, and the feed entries are -// shared with the cached snapshot. def withLinkSource(object pull, string source) object ( return { id: pull.id, @@ -274,11 +256,6 @@ def resolveIdeaPulls(object idea, array pulls) array ( return resolved ) -// GitHub's search quota is separate from the core REST quota: ten requests a -// minute for an unauthenticated caller, and no ETag on search responses, so a -// conditional request cannot buy a free 304. The only way to stay inside the -// budget is to never spend a request we do not need or cannot afford. - def snapshotIsFresh(number updated, number now, number refreshSeconds) boolean ( if updated <= 0 or now < updated ( return false @@ -291,15 +268,12 @@ def quotaFromHeaders(object headers, number nowSeconds) object ( return {ok: false} ) number reset = headers["X-Ratelimit-Reset"].toNum() - // A reset already in the past tells us nothing about the window ahead. if reset < nowSeconds ( reset = nowSeconds ) return {ok: true, remaining: headers["X-Ratelimit-Remaining"].toNum(), reset: reset} ) -// Retry-After is seconds from now; a 403 without one still means the window is -// spent, so hold off until the reset GitHub last reported. def quotaAfterRateLimit(object headers, number nowSeconds, number fallbackSeconds) object ( number reset = nowSeconds + fallbackSeconds if headers.contains("Retry-After") ( @@ -317,7 +291,6 @@ def quotaAfterRateLimit(object headers, number nowSeconds, number fallbackSecond ) def searchAffordable(object quota, number nowSeconds, number calls, number reserve) boolean ( - // Nothing observed yet, or the window has rolled over since we last looked. if !quota.contains("remaining") or quota.remaining.toNum() < 0 ( return true ) diff --git a/src/community/development-links.test.osl b/src/community/development-links.test.osl index 826f029..3e1c300 100644 --- a/src/community/development-links.test.osl +++ b/src/community/development-links.test.osl @@ -10,7 +10,6 @@ testing.assert(pullWebUrl("scratch-vm", 12) == "https://github.com/MistWarp/scra testing.assert(githubRepoName("https://api.github.com/repos/MistWarp/scratch-gui") == "scratch-gui", "the repository name comes off the search result URL") testing.assert(githubRepoName("") == "", "a missing repository URL yields no name") -// Roadmap references written by contributors array linked = roadmapReferencesIn("Fixes https://mistwarp.org/roadmap/entry/iAbCdEf1234567X please review") testing.assert(linked.len == 1 and linked[1].toStr() == "iAbCdEf1234567X", "a roadmap entry link is recognised") array shorthand = roadmapReferencesIn("Roadmap: iAbCdEf1234567X") @@ -22,14 +21,12 @@ testing.assert(repeated.len == 1, "the same entry is only referenced once") testing.assert(roadmapReferencesIn("fixes #218 in the editor").len == 0, "a plain GitHub issue number is not a roadmap reference") testing.assert(roadmapReferencesIn("").len == 0, "an empty body references nothing") -// Pull request state testing.assert(developmentPullState({state: "open", pull_request: {}}) == "open", "an open pull request reads as open") testing.assert(developmentPullState({state: "open", draft: true, pull_request: {}}) == "draft", "drafts are called out separately") testing.assert(developmentPullState({state: "closed", pull_request: {merged_at: "2026-09-01T00:00:00Z"}}) == "merged", "a merged pull request reads as merged") testing.assert(developmentPullState({state: "closed", pull_request: {merged_at: null}}) == "closed", "a closed but unmerged pull request reads as closed") testing.assert(developmentPullState({state: "closed", draft: true, pull_request: {merged_at: "2026-09-01T00:00:00Z"}}) == "merged", "merging wins over a stale draft flag") -// Admin-typed references testing.assert(parsePullLink("scratch-gui#218").ok == true, "the short form parses") testing.assert(parsePullLink("scratch-gui#218").repo.toStr() == "scratch-gui", "the short form keeps the repository") testing.assert(parsePullLink("scratch-gui#218").number.toNum() == 218, "the short form keeps the number") @@ -53,7 +50,6 @@ for i 20 ( ) testing.assert(normalizePullLinks(manyLinks).len == developmentMaxLinksPerIdea, "an entry cannot accumulate unbounded links") -// Feed assembly object openItem = { number: 218, title: "Sprite pane redesign", @@ -82,7 +78,6 @@ testing.assert(collected.len == 1, "search results that are issues rather than p array merged = dedupeDevelopmentPulls([pull], [pull, {id: "scratch-vm#7"}]) testing.assert(merged.len == 2, "a pull request appearing in both searches is listed once") -// Resolving an entry's pull requests object idea = {_id: "iAbCdEf1234567X", pulls: [{repo: "scratch-vm", number: 7}]} array resolved = resolveIdeaPulls(idea, [pull]) testing.assert(resolved.len == 2, "manual links and body references both attach") @@ -99,7 +94,6 @@ testing.assert(once[1].assert(object).linkedBy.toStr() == "manual", "an explicit array otherEntry = resolveIdeaPulls({_id: "iZzZzZz9999999Z", pulls: []}, [pull]) testing.assert(otherEntry.len == 0, "an unrelated entry picks up nothing") -// Staying inside GitHub's search quota testing.assert(snapshotIsFresh(1000000, 1000000 + 60000, 300), "a snapshot a minute old is still fresh") testing.assert(snapshotIsFresh(1000000, 1000000 + 299999, 300), "a snapshot is fresh right up to the refresh interval") testing.assert(!snapshotIsFresh(1000000, 1000000 + 300000, 300), "a snapshot older than the interval is stale") diff --git a/src/community/development.osl b/src/community/development.osl index ec1d3cb..65b33bd 100644 --- a/src/community/development.osl +++ b/src/community/development.osl @@ -3,27 +3,13 @@ export { nowSeconds } +// osl-disable-next-line OSL-UNTYPED-GLOBAL auto developmentCache = cache.new({capacity: 4, ttl: 300000}) -// MistWarp's own source lives in public GitHub repositories, so the work in -// flight there is invisible from inside the site. This pulls the org's open -// and recently merged pull requests into the community API and links them to -// roadmap entries, so a reader can follow a feature from "planned" to the -// change that delivers it. -// -// The GitHub search API answers for the whole org in one request, so a refresh -// costs two calls rather than one per repository. That keeps the unauthorised -// rate limit comfortable; GITHUB_TOKEN only raises the ceiling. - number developmentOpenLimit = 100 number developmentMergedLimit = 40 number developmentRefreshSeconds = 300 -// The loop ticks more often than the refresh window so a tick landing a -// moment early cannot push the next refresh out to a second full interval. -// Ticks that find fresh data cost nothing. number developmentTickSeconds = 60 -// One refresh spends two search requests. The quota is per IP, so leave room -// for anything else on this host that talks to GitHub. number developmentSearchCalls = 2 number developmentRateReserve = 2 number developmentRateLimitBackoffSeconds = 60 @@ -63,8 +49,6 @@ def githubSearchPulls(string query, number limit) object ( object response = requests.get(endpoint, {headers: githubHeaders(), timeoutMs: 10000, maxBytes: 8388608}).unwrapOr({body: null, error: "request failed", headers: {}, raw: null, status: 0, success: false}) object headers = object(response.headers) number status = response.status - // 403 and 429 are how a spent quota arrives. Record when it reopens so the - // next refresh waits instead of asking again. if status == 403 or status == 429 ( storeDevelopmentQuota(quotaAfterRateLimit(headers, nowSeconds(), developmentRateLimitBackoffSeconds)) return {ok: false, items: []} @@ -87,7 +71,6 @@ def githubSearchPulls(string query, number limit) object ( def fetchDevelopmentPulls() object ( object openPulls = githubSearchPulls("org:" ++ githubOrg ++ " is:pr is:open", developmentOpenLimit) object mergedPulls = githubSearchPulls("org:" ++ githubOrg ++ " is:pr is:merged", developmentMergedLimit) - // A total failure must not overwrite the last good snapshot with nothing. if !openPulls.ok and !mergedPulls.ok ( return {ok: false, pulls: []} ) @@ -117,8 +100,6 @@ def developmentPulls() array ( def refreshDevelopmentPullsLocked() object ( object current = developmentSnapshot() - // A restart must not cost a request: the stored snapshot is still good, and - // a crash loop would otherwise burn the whole minute's budget. if snapshotIsFresh((current.updated ?? 0).toNum(), timestamp, developmentRefreshSeconds) ( return current ) diff --git a/src/community/groups.osl b/src/community/groups.osl index 5b0bfc8..e9755f0 100644 --- a/src/community/groups.osl +++ b/src/community/groups.osl @@ -5,7 +5,7 @@ export { normalizeGroupTag } -auto groupSnapshotCache = cache.new({capacity: 10000, ttl: 60000}) +*cache.Cache groupSnapshotCache = cache.new({capacity: 10000, ttl: 60000}) def normalizeGroupTag(string value) string ( string tag = value.toLower().strip() diff --git a/src/community/like-milestone-policy.osl b/src/community/like-milestone-policy.osl index d3f4089..3f32a26 100644 --- a/src/community/like-milestone-policy.osl +++ b/src/community/like-milestone-policy.osl @@ -1,6 +1,5 @@ export {} -// Only new crossings count. Existing likes are not backfilled on rollout. def crossedLikeMilestones(number before, number after, number notifiedThrough) array ( array res = [] array thresholds = [5, 25, 50, 100, 500, 1000] diff --git a/src/community/like-milestones.osl b/src/community/like-milestones.osl index 21eeff6..9b0bf9b 100644 --- a/src/community/like-milestones.osl +++ b/src/community/like-milestones.osl @@ -2,8 +2,6 @@ export { notifyLikeMilestones } -// Claims survive inbox clearing and server restarts. Serialize claims for the -// same item so simultaneous reactions cannot send the same milestone twice. def notifyLikeMilestones(string target, string kind, string contentId, number before, number after, string path, object extra) ( if target == "" or after <= before ( return diff --git a/src/community/like-milestones.test.osl b/src/community/like-milestones.test.osl index 0bebd96..ed83964 100644 --- a/src/community/like-milestones.test.osl +++ b/src/community/like-milestones.test.osl @@ -13,7 +13,7 @@ import "./like-milestone-policy.osl" import "./like-milestones.osl" string path = "/tmp/mistwarp-milestones-" ++ timestamp.toStr() ++ ".db" -auto milestoneDb = db.open(path) +*db.DB milestoneDb = db.open(path) *db.Collection likeMilestoneDocs = milestoneDb.collection("milestones") string roturBase = "http://127.0.0.1:18342" array sent = [] @@ -36,7 +36,6 @@ def normalizeUsername(string name) string ( ) def userIdForUsername(string name) string ( - // Simulate an account rename without changing identity. if name == "alice" or name == "renamed" ( return "u-alice" ) @@ -71,7 +70,6 @@ testing.assert(sent[1].data.path == "/project/p1", "links to the project") notifyLikeMilestones("alice", "project", "p1", 5, 4, "/project/p1", {}) notifyLikeMilestones("alice", "project", "p1", 4, 5, "/project/p1", {}) testing.assert(sent.len == 1, "unlike and relike do not repeat") -// Persistence survives process-style close and reopen, and account renames. milestoneDb.close() milestoneDb = db.open(path) likeMilestoneDocs = milestoneDb.collection("milestones") @@ -104,7 +102,6 @@ testing.assert(sent.len == 12 and sent[12].data.milestone == 25, "sampled counts observeRemoteMilestone("alice", "followers", "remote", 24, "/users/alice/followers", {}) observeRemoteMilestone("alice", "followers", "remote", 25, "/users/alice/followers", {}) testing.assert(sent.len == 12, "follower drops and refollows do not repeat") -// A fake Rotur: 97 followers, of whom only 25 are public. def handleFakeProfile(*serve.Context c) ( if c.param("name") == "hidden" ( c.json(200, {username: "hidden", private: true, restricted: true}) diff --git a/src/community/releases.osl b/src/community/releases.osl index e22f161..f549070 100644 --- a/src/community/releases.osl +++ b/src/community/releases.osl @@ -113,8 +113,6 @@ def handleCreateRelease(*serve.Context c) ( ) project = snapshot.project.assert(object) string jsonUrl = appURL ++ "/v1/projects/" ++ project.id.toStr() ++ "/releases/" ++ url.escape(releaseId) ++ "/project.json" - // jsonBytes is the snapshot's size, which counts against the owner's - // storage for as long as the release keeps it. object release = {_id: releaseId, projectId: project.id, version: version, channel: releaseChannel, notes: notes, commit: commit, jsonKey: project.jsonKey, jsonBytes: (project.storedJsonBytes ?? 0).toNum(), jsonGzip: true, jsonUrl: jsonUrl, assetsBase: assetsBaseUrl(), created: timestamp, creator: contextString(c, "username"), creatorId: contextString(c, "userId")} void releaseDocs.insertOne(release) storageSettleProject(project) diff --git a/src/community/remote-milestones.osl b/src/community/remote-milestones.osl index 543334e..3bd36c5 100644 --- a/src/community/remote-milestones.osl +++ b/src/community/remote-milestones.osl @@ -2,10 +2,8 @@ export { remoteMilestoneChecks } -auto remoteMilestoneChecks = cache.new({capacity: 10000, ttl: 10000}) +*cache.Cache remoteMilestoneChecks = cache.new({capacity: 10000, ttl: 10000}) -// Use existing public Rotur APIs. Counts, recipients and links never come -// from the caller. This records milestones observed through MistWarp. def remoteMilestoneData(string path) object ( return milestoneDataFromUrl(roturBase ++ "/v2" ++ path) ) @@ -27,7 +25,6 @@ def observeRemoteMilestone(string target, string kind, string id, number count, sync.lock(key) defer sync.unlock(key) object previous = (likeMilestoneDocs.findById(key) ?? {}) - // First observation establishes a baseline; an exact milestone still counts. number before = count - 1 if previous.contains("count") ( before = previous.count.toNum() @@ -46,8 +43,6 @@ def syncFollowerMilestones(string username) ( return ) remoteMilestoneChecks.set(key, true) - // The profile's count, not the public follower list: the list leaves out - // private accounts, so its length undercounts. object profile = remoteMilestoneData("/users/" ++ url.escape(name) ++ "?include_posts=0") if !profile.contains("followers") or typeof(profile.followers) != "number" ( return diff --git a/src/community/reports.osl b/src/community/reports.osl index fd11452..ce7fa1d 100644 --- a/src/community/reports.osl +++ b/src/community/reports.osl @@ -70,11 +70,6 @@ def canAppendPendingReport(array reports, string sourceKey) boolean ( return totalPending < 1000 and sourcePending < 10 ) -// Reports also go to the MistWarp Rotur App's report queue. Reports in a -// priority category (child sexual exploitation or abuse, a threat to life, -// encouraging suicide or self-harm, terrorism) reach Rotur's safety team as -// soon as they're filed. MistWarp's own queue and tools are unchanged, and -// closing a report here closes it on Rotur too. array roturReportCategories = ["spam", "harassment", "hate", "sexual", "violence", "scam", "other", "csea", "threat_to_life", "self_harm", "terrorism"] def reportCategory(object body, string reason) string ( @@ -82,7 +77,6 @@ def reportCategory(object body, string reason) string ( if roturReportCategories.contains(category) ( return category ) - // Older clients only send the category's wording at the start of the reason. object worded = { "Inappropriate or explicit content": "sexual", "Harassment or bullying": "harassment", @@ -99,8 +93,6 @@ def reportCategory(object body, string reason) string ( return "other" ) -// The reported comment, by ID: in the store the report's context names, else -// in any store. def findReportedComment(string id, string context) object ( array keys = [] if context.startsWith("project ") ( @@ -122,9 +114,6 @@ def findReportedComment(string id, string context) object ( return {} ) -// What was reported, worked out here and not taken from the reporter: who it -// is about, and what it said, so Rotur's safety team sees it even if it's -// deleted later. user is "" when it can't be worked out. def reportedContent(object report) object ( string type = report.type.toStr() string target = report.target.toStr() @@ -155,7 +144,6 @@ def setReportRotur(string id, string state, string roturReportId) object ( object found = {} for i reports.len ( object stored = reports[i].assertElse(object, {}) - // Only the first filing to finish counts. if (stored.id ?? "").toStr() == id and (stored.rotur ?? "").toStr() == "pending" ( stored.rotur = state if roturReportId != "" ( @@ -169,10 +157,6 @@ def setReportRotur(string id, string state, string roturReportId) object ( return found ) -// Files the report in the Rotur App's queue. Its rotur state is "filed" once -// Rotur has it, "unfiled" if Rotur won't take it (or nobody can be named), and -// stays "pending" to be tried again by the reconcile loop. False if Rotur -// couldn't be reached or failed, so the reconcile loop stops for now. def fileRoturReport(object report) boolean ( string subject = (report.subject ?? "").toStr() if !roturAppReady() ( @@ -200,7 +184,6 @@ def fileRoturReport(object report) boolean ( ) object res = roturApp("POST", "/reports", request) if !res.ok and res.status == 400 and request.contains("reporter") ( - // Only people who have used MistWarp through Rotur can be named. request.delete("reporter") res = roturApp("POST", "/reports", request) ) @@ -217,15 +200,14 @@ def fileRoturReport(object report) boolean ( if state == "filed" and (res.data.escalated ?? false) == true ( void updateReport(report.id.toStr(), {roturEscalated: true, roturEscalatedAt: timestamp, roturEscalatedBy: ""}) ) - // A moderator may have closed it while it was being filed. if state == "filed" and stored.resolved == true ( closeRoturReport(stored, if (stored.action ?? "").toStr() == "dismiss" then "dismissed" else "resolved") ) return true ) -// Tries again to file reports Rotur couldn't be reached for, from the -// reconcile loop. +number roturReportFilingGraceMs = 60000 + def fileUnfiledRoturReports() ( if !roturAppReady() ( return @@ -233,11 +215,7 @@ def fileUnfiledRoturReports() ( array reports = loadReports() for i reports.len ( object report = reports[i].assertElse(object, {}) - // A report made in the last minute is still being filed by its own - // request. - if (report.rotur ?? "").toStr() == "pending" and timestamp - (report.created ?? 0).toNum() > 60000 ( - // Stop at the first sign Rotur is down, rather than waiting on every - // report in turn. + if (report.rotur ?? "").toStr() == "pending" and timestamp - (report.created ?? 0).toNum() > roturReportFilingGraceMs ( if !fileRoturReport(report) ( return ) @@ -245,8 +223,6 @@ def fileUnfiledRoturReports() ( ) ) -// outcome is "resolved" (action taken) or "dismissed". Rotur keeps reports -// it is reviewing, and says no to closing them, which is fine. def closeRoturReport(object report, string outcome) ( string reportId = (report.roturReportId ?? "").toStr() if reportId != "" ( @@ -287,7 +263,6 @@ def handleCreateReport(*serve.Context c) ( if body.contains("context") and body.context != null ( report.context = body.context.toStr().trim(1, 5000) ) - // Who it's about comes from the content, never from the reporter. object reported = reportedContent(report) report.snapshot = reported.snapshot.toStr().trim(1, 4000) if reported.user.toStr() != "" ( @@ -517,7 +492,6 @@ def resolveReportLocked(*serve.Context c, string id) object ( return {found: true, saved: saved, projectTarget: projectTarget, resolved: resolved} ) -// Reports someone filed stay for moderation, without saying who filed them. def anonymiseReportsBy(string username, string userId) ( string key = normalizeUsername(username) string sourceKey = "user:" ++ userId diff --git a/src/config/config.osl b/src/config/config.osl index 8219c38..39cea43 100644 --- a/src/config/config.osl +++ b/src/config/config.osl @@ -118,15 +118,6 @@ def envValue(string name) string ( return value ) -// Configuration is looked up in this order: -// 1. MISTWARP_ in the real environment (an explicit, app-specific override) -// 2. this app's .env -// 3. in the real environment (for deployments that don't ship a .env) -// -// Inherited variables come last on purpose. The API is often started from a -// shared shell that exports another project's variables (for example Rotur's -// R2_ACCESS_KEY_ID and R2_PUBLIC_BASE), and those must not silently replace -// this service's own configuration. def configValue(string name, string defaultValue) string ( string value = envValue("MISTWARP_" ++ name) if value != "" ( diff --git a/src/core/database.osl b/src/core/database.osl index f4c4baf..cb0e704 100644 --- a/src/core/database.osl +++ b/src/core/database.osl @@ -46,9 +46,6 @@ export { } -// The database and its collections are opened once at startup. A failure here -// means the API cannot serve anything, so report the reason and stop instead of -// carrying on with a half-initialized database. def openCommunityDatabase(string path) *db.DB ( *db.DB database = db.open(path) catch (err) ( log "FATAL: could not open " ++ path ++ ": " ++ err.toStr() diff --git a/src/games/game-data.osl b/src/games/game-data.osl index b6aa37c..ed9a397 100644 --- a/src/games/game-data.osl +++ b/src/games/game-data.osl @@ -4,7 +4,7 @@ export { purgeGameDataForUser } -auto gameDataCapabilityCache = cache.new({capacity: 100000, ttl: 300000}) +*cache.Cache gameDataCapabilityCache = cache.new({capacity: 100000, ttl: 300000}) number gameSaveMaxBytes = 65536 number gameDataCapabilityTtlMs = 300000 diff --git a/src/games/game-marketplace.osl b/src/games/game-marketplace.osl index 495902c..2efd032 100644 --- a/src/games/game-marketplace.osl +++ b/src/games/game-marketplace.osl @@ -187,8 +187,6 @@ def handleGameProductIntent(*serve.Context c) ( started.title = product.name c.json(200, started)) -// Delivers a paid game product, once, whether the editor or Rotur's webhook -// gets here first. def completeGameProductPurchase(string key) object ( sync.lock("purchase:" ++ key) defer sync.unlock("purchase:" ++ key) @@ -210,7 +208,6 @@ def completeGameProductPurchase(string key) object ( if !saveGameProductEntitlement(project.id.toStr(), userId, product.id.toStr(), paymentId) or !grantGameProductItem(project, product, userId, paymentId) ( return {ok: false, error: "failed to grant game product"} ) - // What the buyer paid, which a refund pays back. number paid = (entry.price ?? product.price).toNum() addPurchaseRecord(buyer, {projectId: project.id.toStr(), productId: product.id, title: product.name, seller: project.owner, amount: paid, at: timestamp, type: "game_product"}) addNotification(project.owner.toStr(), buyer, "purchase", {projectId: project.id, projectTitle: project.title, productId: product.id, productName: product.name, amount: paid}) @@ -228,7 +225,6 @@ def handleGameProductConfirm(*serve.Context c) ( completed = completeGameProductPurchase(key) ) if !completed.ok and ownsGameProduct(c.param("id"), contextString(c, "userId"), c.param("product")) ( - // Rotur's webhook got here first. auto loaded = loadProject(c.param("id")) object found = findGameProduct(loaded.project.assertElse(object, {}), c.param("product")) if loaded.ok and found.ok ( @@ -247,8 +243,6 @@ def handleGameProductConfirm(*serve.Context c) ( c.json(200, {ok: true, product: gameProductResponse(completed.project.assert(object), completed.product.assert(object), c)}) ) -// What the buyer last paid for this product, from their purchase records, or 0 -// if they never paid for it (a creator grant or a free claim). def gameProductAmountPaid(string buyer, string projectId, string productId) number ( object profile = loadProfile(normalizeUsername(buyer)) array purchases = (profile.purchases ?? []).assertElse(array, []) @@ -261,8 +255,6 @@ def gameProductAmountPaid(string buyer, string projectId, string productId) numb return 0 ) -// The creator pays the buyer back through Rotur before revoking a paid -// product: this asks Rotur for that payment. def handleGameProductRefundIntent(*serve.Context c) ( auto loaded = loadProject(c.param("id")) if !loaded.ok or !canEditProject(loaded.project.assert(object), c) ( @@ -374,7 +366,6 @@ def handleRevokeGameProduct(*serve.Context c) ( refunded = true ) if !removeGameProductEntitlement(project.id.toStr(), targetId, product.id.toStr()) ( - // The refund key stays, so revoking can be tried again with it. c.internalError("failed to revoke game product") return ) diff --git a/src/git/git-inspection.osl b/src/git/git-inspection.osl index ffd5266..cdd65b1 100644 --- a/src/git/git-inspection.osl +++ b/src/git/git-inspection.osl @@ -6,7 +6,7 @@ export { writeForkHistoryArchive } -auto gitInspectionPruneCache = cache.new({capacity: 4, ttl: 300000}) +*cache.Cache gitInspectionPruneCache = cache.new({capacity: 4, ttl: 300000}) def projectWorkspaceFingerprint(object project) string ( array keys = projectWorkspaceKeys(project) diff --git a/src/git/pulls.test.osl b/src/git/pulls.test.osl index 741365b..df84c4d 100644 --- a/src/git/pulls.test.osl +++ b/src/git/pulls.test.osl @@ -9,7 +9,7 @@ import "std:thread" import "go:mistwarp.local/gitinspection" string dbPath = "/tmp/mistwarp-pulls-test-" ++ timestamp.toStr() ++ ".db" -auto testDb = db.open(dbPath) +*db.DB testDb = db.open(dbPath) *db.Collection pullDocs = testDb.collection("project_pulls") string tmpDir = "/tmp/mistwarp-pulls-test-" ++ timestamp.toStr() ++ "/" number maxWorkspaceBytes = 1000 diff --git a/src/history/history-migration.osl b/src/history/history-migration.osl index db68d08..2feb7d5 100644 --- a/src/history/history-migration.osl +++ b/src/history/history-migration.osl @@ -146,10 +146,6 @@ def generateProjectHistory(object project, string parentPath, string parentHead, return {ok: true, project: current, generated: true} ) -// Upload callers hold the publication lock. History preparation takes that -// lock itself after resolving ancestors and revalidates the saved source. -// Drop it during preparation to avoid both self-deadlock and nested locks -// across remix ancestors, then restore the caller's lock ownership. def ensureUploadedProjectHistory(object project) object ( string lockName = "project-history-" ++ project.id.toStr() sync.unlock(lockName) @@ -179,8 +175,6 @@ def ensureGeneratedProjectHistory(object project, object visiting, boolean allow string parentHead = "" string parentId = (project.remixParent ?? "").toStr() if parentId != "" ( - // Absence never authorizes inventing a root or using today's parent HEAD. - // Keep the record retryable until its recorded base can be materialized. auto parentResult = loadProject(parentId) for attempt 2 ( if !parentResult.ok and parentResult.missing != true ( diff --git a/src/history/history-migration.test.osl b/src/history/history-migration.test.osl index b978f20..2773111 100644 --- a/src/history/history-migration.test.osl +++ b/src/history/history-migration.test.osl @@ -168,7 +168,6 @@ testing.assert(cycleResult.ok == false and cycleResult.error == "remix cycle" an object editedFork = {id: "edited", jsonKey: "edited.json", lastUploadAt: 1, remixParent: "parent", remixBaseCommit: base, gitHead: "recorded-edits"} testing.assert(ensureGeneratedProjectHistory(editedFork, {}, false).ok == false, "bootstrap must not discard established edits even when workspace is missing") -// Migration and bootstrap share this code. Contending calls must publish once. def concurrentEnsureHistory(object candidate,*sync.WaitGroup jobs) ( ensureGeneratedProjectHistory(candidate, {}, false) jobs.done() @@ -188,7 +187,6 @@ records["retry-child"] = retryChild readFailures = 1 object retried = ensureGeneratedProjectHistory(retryChild, {}, false) testing.assert(retried.ok == true and readFailures == 0, "transient parent read retries in the same call") -// A first upload already owns the publication lock when preparing history. object uploaded = {id: "uploaded", owner: "Mist", created: 6000, jsonKey: "uploaded.json", lastUploadAt: 6000, assets: []} records.uploaded = uploaded sync.lock("project-history-uploaded") @@ -198,14 +196,11 @@ testing.assert(uploadHistory.ok == true and projectHasWorkspace(uploadHistory.pr sync.lock("project-history-uploaded") sync.unlock("project-history-uploaded") -// Failed preparation must also restore lock ownership for the upload wrapper. sync.lock("project-history-missing") object failedUploadHistory = ensureUploadedProjectHistory(missing) sync.unlock("project-history-missing") testing.assert(failedUploadHistory.ok == false and failedUploadHistory.code == "parent_missing", "upload history failure returns with publication lock restored") -// Releasing the lock must not allow history from an older upload to replace -// a newer saved snapshot. object staleUpload = {id: "stale-upload", owner: "Mist", created: 7000, jsonKey: "old.json", lastUploadAt: 7000, assets: []} records["stale-upload"] = {id: "stale-upload", owner: "Mist", created: 7000, jsonKey: "new.json", lastUploadAt: 8000, assets: []} before = saves diff --git a/src/projects/fork-visibility.test.osl b/src/projects/fork-visibility.test.osl index 752b28e..fe8bff6 100644 --- a/src/projects/fork-visibility.test.osl +++ b/src/projects/fork-visibility.test.osl @@ -189,8 +189,6 @@ def canSeeInsideProjectTest(object project, object ctx) boolean ( return projectIsViewable(project) ) -// Setup projects -// Project A: original project created by Alice, maintained by Dan object projectA = { id: "proj_original", owner: "alice", @@ -200,7 +198,6 @@ object projectA = { collaborators: [{username: "maintainer_dan", userId: "u_dan", role: "maintainer"}] } -// Project B: private fork of Project A created by Bob object projectB = { id: "p178804999473398600081XWkq", owner: "bob", @@ -212,7 +209,6 @@ object projectB = { collaborators: [] } -// Project C: chained fork of Project B created by Charlie object projectC = { id: "proj_chained_fork", owner: "charlie", @@ -236,7 +232,6 @@ object danCtx = {authenticated: true, username: "maintainer_dan", userId: "u_dan object anonCtx = {authenticated: false, username: "", userId: "", isAdmin: false} object strangerCtx = {authenticated: true, username: "stranger", userId: "u_stranger", isAdmin: false} -// Tests for Project B (fork of A) testing.assert(canViewProjectTest(projectB, aliceCtx), "Alice (creator of forked project) can view private fork") testing.assert(canViewProjectTest(projectB, danCtx), "Dan (maintainer of forked project) can view private fork") testing.assert(canViewProjectTest(projectB, bobCtx), "Bob (fork owner) can view own fork") @@ -253,7 +248,6 @@ testing.assert(canViewProjectJsonTest(projectB, danCtx), "Dan can view project J testing.assert(canViewProjectJsonTest(projectB, bobCtx), "Bob can view project JSON for private fork") testing.assert(!canViewProjectJsonTest(projectB, strangerCtx), "Stranger cannot view project JSON for private fork") -// Tests for chained fork C (remixRoot = proj_original) testing.assert(canViewProjectTest(projectC, charlieCtx), "Charlie (creator of chained fork) can view own fork") testing.assert(canViewProjectTest(projectC, aliceCtx), "Alice (creator of root project) can view chained fork") testing.assert(canViewProjectTest(projectC, bobCtx), "Bob (creator of direct parent fork) can view chained fork") diff --git a/src/projects/live-collaboration.osl b/src/projects/live-collaboration.osl index 0feb977..eefdf86 100644 --- a/src/projects/live-collaboration.osl +++ b/src/projects/live-collaboration.osl @@ -3,9 +3,8 @@ import "./live-collaboration-policy.osl" export {} -// Temporary approvals are leases; explicitly shared Editor access persists. -auto liveEditorSessions = cache.new({capacity: 10000, ttl: 90000}) -auto projectEditorPresence = cache.new({capacity: 10000, ttl: 90000}) +*cache.Cache liveEditorSessions = cache.new({capacity: 10000, ttl: 90000}) +*cache.Cache projectEditorPresence = cache.new({capacity: 10000, ttl: 90000}) def handleGetSharedProjects(*serve.Context c) ( string username = normalizeUsername(contextString(c, "username")) diff --git a/src/projects/monetize.osl b/src/projects/monetize.osl index 6381cdc..ea6bc44 100644 --- a/src/projects/monetize.osl +++ b/src/projects/monetize.osl @@ -146,8 +146,6 @@ def projectCommerceSplits(object project) array ( if platformPoints > 0 ( platformUser = mistwarpFeeAccount() if platformUser == "" ( - // Without the fee account its share would go to the creator. No splits - // means the sale can't start. return [] ) ) @@ -252,8 +250,9 @@ def renameUserMonetization(string oldKey, string newName) boolean ( return savePurchaseKeys(purchaseStore) ) -// Unpaid keys go after an hour (Rotur's requests expire after 15 minutes). -// Paid ones still waiting for their comment or revoke are kept for 30 days. +number unpaidPurchaseKeyLifetimeMs = 3600000 +number paidPurchaseKeyLifetimeMs = 2592000000 + def prunePurchaseKeys(object store) object ( object keys = store.keys array names = keys.getKeys() @@ -262,7 +261,7 @@ def prunePurchaseKeys(object store) object ( string k = names[i].toStr() object entry = keys[k] number age = timestamp - entry.at.toNum() - if age < 3600000 or (entry.paid == true and age < 2592000000) ( + if age < unpaidPurchaseKeyLifetimeMs or (entry.paid == true and age < paidPurchaseKeyLifetimeMs) ( kept[k] = entry ) ) @@ -443,9 +442,6 @@ def recordSale(object project, string buyer, number price, number sellerAmount) project.saleHistory = sales ) -// An editor or packaged game from before payment requests sends no returnUrl. -// It would pay MistWarp the old way, which nothing delivers any more, so it's -// told to reload before anything is asked of Rotur. def refuseOldPaymentClient(*serve.Context c) boolean ( if requestJSON(c).contains("returnUrl") ( return false @@ -471,7 +467,6 @@ def completeProjectPurchase(string key) object ( if !hasBought(project, buyer) ( number owed = entry.price.toNum() string seller = project.owner.toStr() - // The splits Rotur paid, as they were when the purchase started. array splits = (entry.splits ?? projectCommerceSplits(project)).assertElse(array, []) recordSale(project, buyer, owed, ownerAmountFromSplits(project, owed, splits)) if addToLibrary(buyer, project.id.toStr()) ( diff --git a/src/projects/project-access.osl b/src/projects/project-access.osl index efa0402..a532294 100644 --- a/src/projects/project-access.osl +++ b/src/projects/project-access.osl @@ -138,15 +138,11 @@ def ensureViewKey(object project) ( project.bannedViewKey = randomString(16) saveProject(project) ) else if !ownerBanned and bannedViewKey(project) != "" ( - // A later ban gets a new key. project.delete("bannedViewKey") saveProject(project) ) ) -// While the owner is banned, a project's JSON is only served with this key, -// which only admins are handed (the player fetches the JSON signed out). -// Older view keys stop working until the ban is lifted. def bannedViewKey(object project) string ( return (project.bannedViewKey ?? "").toStr() ) diff --git a/src/projects/project-create.osl b/src/projects/project-create.osl index 3251dea..e8463e0 100644 --- a/src/projects/project-create.osl +++ b/src/projects/project-create.osl @@ -97,7 +97,6 @@ def ownedProjectSlotCount(string userId, string username) number ( return ownerIndexCount(username, true) + trashedProjectsOwnedBy(userId, username).len ) -// A project is shared for the first time, from either way of sharing it. def projectFirstShared(object project) ( addActivity(project.owner.toStr(), "share", {projectId: project.id, projectTitle: project.title}) notifyPublishedRemix(project) diff --git a/src/projects/project-discovery.osl b/src/projects/project-discovery.osl index 982d27d..a91284a 100644 --- a/src/projects/project-discovery.osl +++ b/src/projects/project-discovery.osl @@ -2,6 +2,7 @@ export { sharedProjectsCache } +// osl-disable-next-line OSL-UNTYPED-GLOBAL auto sharedProjectsCache = cache.new({capacity: 2, ttl: 60000}) def sharedIndexEntries() array ( diff --git a/src/projects/project-publishing.osl b/src/projects/project-publishing.osl index 12b0e64..3c38ab9 100644 --- a/src/projects/project-publishing.osl +++ b/src/projects/project-publishing.osl @@ -1,5 +1,5 @@ -auto viewDedup = cache.new({capacity: 100000, ttl: 21600000}) -auto viewIndexFlush = cache.new({capacity: 100000, ttl: 600000}) +*cache.Cache viewDedup = cache.new({capacity: 100000, ttl: 21600000}) +*cache.Cache viewIndexFlush = cache.new({capacity: 100000, ttl: 600000}) def handlePublishProject(*serve.Context c) ( auto res = loadProject(c.param("id")) if !res.ok ( @@ -164,7 +164,6 @@ def handleReactProjectLocked(*serve.Context c) ( broken = project.brokenhearts.assert(array) ) number previousLikes = loves.len - // Projects from before everLoved count everyone who loves them now. if !project.contains("everLoved") or project.everLoved == null ( project.everLoved = loves.clone() ) @@ -177,7 +176,6 @@ def handleReactProjectLocked(*serve.Context c) ( if reactionType == "heart" and !hadHeart ( loves.append(viewer) addedHeart = true - // Badges count each person's love once, however often it's toggled. array everLoved = (project.everLoved ?? []).assertElse(array, []) if !everLoved.contains(viewer) ( everLoved.append(viewer) diff --git a/src/projects/project-quota.osl b/src/projects/project-quota.osl index 169254c..e874a52 100644 --- a/src/projects/project-quota.osl +++ b/src/projects/project-quota.osl @@ -24,8 +24,6 @@ def handleGetMyQuota(*serve.Context c) ( projects.append({id: project.id, title: project.title, bytes: projectStorageBytes(project), trashed: true}) ) projects.sortBy("bytes", "descending") - // shared: used, limit and remaining are the person's whole Rotur storage, - // and mistwarp is what MistWarp holds of it. c.json(200, { ok: true, shared: details.shared, @@ -90,9 +88,6 @@ def ownerStorageBytes(string username) number ( return total ) -// shared is true for Rotur accounts, whose used, limit and remaining are their -// whole Rotur storage, shared with Rotur's other apps; mistwarp is MistWarp's -// part of it. known is false when Rotur couldn't say. def storageDetails(string username) object ( string userId = userIdForUsername(username) if usesRoturStorage(userId) ( @@ -114,7 +109,6 @@ def storageDetails(string username) object ( string roturMembershipUrl = "https://rotur.dev/membership" -// The ledger refused something because the person's Rotur storage is full. def roturStorageFullResponse(*serve.Context c, object refusal) ( c.json(413, { ok: false, @@ -137,8 +131,6 @@ def storageFullResponse(*serve.Context c, object details, number neededBytes) ( c.json(413, {ok: false, code: "storage_full", error: "Your projects and backpack use " ++ usedMb.toStr() ++ " MB of your " ++ limitMb.toStr() ++ " MB of storage, and this needs about " ++ neededMb.toStr() ++ " MB more. Delete projects or backpack items you no longer need and empty the trash, or upgrade your plan.", used: details.used, pending: details.pending, limit: details.limit, remaining: details.remaining}) ) -// Assets uploaded ahead of a save count against storage until a saved -// project references them or they age out. def reservePendingUpload(string username, string name, number bytes, number limit, boolean enforceLimit) object ( string lockName = "upload-quota-" ++ normalizeUsername(username) sync.lock(lockName) @@ -166,8 +158,6 @@ def releasePendingUploads(string username, array names) ( ) ) -// The most a student's project may occupy after a save, or -1 when the -// owner's storage is counted by Rotur instead. def projectStorageAllowance(object project) number ( if usesRoturStorage(projectOwnerId(project)) ( return -1 @@ -176,8 +166,6 @@ def projectStorageAllowance(object project) number ( return storageAllowance(limit, ownerStorageBytes(project.owner.toStr()), projectStorageBytes(project)) ) -// Holds room for an asset uploaded ahead of the save that uses it, or -// answers that there isn't any. def holdUploadAhead(*serve.Context c, object project, string name, number bytes) boolean ( boolean enforce = !contextBool(c, "isAdmin") string ownerId = projectOwnerId(project) diff --git a/src/projects/project-remix.test.osl b/src/projects/project-remix.test.osl index 2e42a8f..5580c2c 100644 --- a/src/projects/project-remix.test.osl +++ b/src/projects/project-remix.test.osl @@ -14,7 +14,7 @@ import "./project-update-lookups.osl" string work = "/tmp/mistwarp-remix-test-" ++ timestamp.toStr() void fs.mkdir(work, {recursive: true}) string tmpDir = work ++ "/" -auto testDb = db.open(work ++ "/community.db") +*db.DB testDb = db.open(work ++ "/community.db") *db.Collection projectSearchDocs = testDb.collection("project_search") def lockIsFree(string lockName) boolean ( diff --git a/src/projects/project-response.osl b/src/projects/project-response.osl index a843dbd..d66d9cb 100644 --- a/src/projects/project-response.osl +++ b/src/projects/project-response.osl @@ -197,7 +197,6 @@ def projectResponse(object project,*serve.Context c) object ( sizeBytes: projectSizeBytes(project), isOwner: isOwner } - // Owners see a lapsed vanity URL, so they can keep it or clear it. if isOwner and activeVanity.lapsed == true ( response.vanityReserved = project.vanitySlug response.vanityLapsed = true diff --git a/src/projects/project-snapshots.osl b/src/projects/project-snapshots.osl index 1a7902a..4cb99ca 100644 --- a/src/projects/project-snapshots.osl +++ b/src/projects/project-snapshots.osl @@ -106,8 +106,6 @@ def ensureProjectJsonSnapshot(object project) object ( project.jsonGzip = true project.pendingJson = false void fs.remove(tempPath) - // Return an immutable snapshot to the caller without rewriting the source - // record. Another save may have advanced that record while storage was read. return {ok: true, project: project} ) diff --git a/src/projects/project-vanity.osl b/src/projects/project-vanity.osl index be6bb89..84299e4 100644 --- a/src/projects/project-vanity.osl +++ b/src/projects/project-vanity.osl @@ -2,8 +2,6 @@ export { vanityLapseLoop } -// The vanity URL that works for a project right now (slug, or "" while the -// owner's plan doesn't include one), and whether one is held but lapsed. def vanityState(object project) object ( string slug = (project.vanitySlug ?? "").toStr() if slug == "" ( @@ -20,8 +18,6 @@ def activeVanitySlug(object project) string ( return vanityState(project).slug.toStr() ) -// Projects holding a vanity URL, including lapsed ones. Entries indexed before -// vanityClaim existed only have vanitySlug. def vanityClaimEntries(string slug) array ( return vanityEntriesWhere("==", slug) ) @@ -71,7 +67,6 @@ def applyVanityLapse(string id, string slug, string eligibility) string ( return applyVanityLapseLocked(id, slug, eligibility) ) -// Notices lapsed vanity URLs and releases them on time, even if nobody visits. def vanityLapseLoop() ( while true ( wait 900 @@ -81,7 +76,6 @@ def vanityLapseLoop() ( string slug = (entry.vanityClaim ?? entry.vanitySlug ?? "").toStr() string eligibility = if slug == "" then "unknown" else vanityEligibility(entry) boolean lapsed = (entry.vanitySlug ?? "").toStr() != slug - // Nothing changes while Rotur can't be reached, or for a URL in use. if eligibility == "no" or (eligibility == "yes" and lapsed) ( void applyVanityLapse(entry.id.toStr(), slug, eligibility) ) diff --git a/src/realtime/game-realtime.osl b/src/realtime/game-realtime.osl index 185a0ca..48d7d54 100644 --- a/src/realtime/game-realtime.osl +++ b/src/realtime/game-realtime.osl @@ -4,8 +4,8 @@ export { setupGameRealtime } -auto multiplayerTicketCache = cache.new({capacity: 100000, ttl: 120000}) -auto realtimeHandshakeCache = cache.new({capacity: 100000, ttl: 30000}) +*cache.Cache multiplayerTicketCache = cache.new({capacity: 100000, ttl: 120000}) +*cache.Cache realtimeHandshakeCache = cache.new({capacity: 100000, ttl: 30000}) def realtimeRoomKey(object identity) string ( return identity.projectId.toStr() ++ ":" ++ identity.projectVersion.toStr() ++ ":" ++ identity.context.toStr() ++ ":" ++ identity.room.toStr() diff --git a/src/rotur/perks.osl b/src/rotur/perks.osl index e55ee3d..ab6af7a 100644 --- a/src/rotur/perks.osl +++ b/src/rotur/perks.osl @@ -9,17 +9,14 @@ export { vanityEligibility } -auto paidPerksCache = cache.new({capacity: 10000, ttl: 300000}) -auto lastKnownTiers = cache.new({capacity: 10000, ttl: 2592000000}) -auto roturTierUnreachable = cache.new({capacity: 10000, ttl: 60000}) +*cache.Cache paidPerksCache = cache.new({capacity: 10000, ttl: 300000}) +*cache.Cache lastKnownTiers = cache.new({capacity: 10000, ttl: 2592000000}) +*cache.Cache roturTierUnreachable = cache.new({capacity: 10000, ttl: 60000}) def roturTierForUser(string userId) string ( return roturTierLookup(userId).tier.toStr() ) -// The user's Rotur tier, and whether Rotur gave it in the last 5 minutes. -// While Rotur can't be reached, the last tier it gave is kept (known: false), -// so an outage doesn't take anyone's perks away. def roturTierLookup(string userId) object ( if userId == "" or isStudentUserId(userId) ( return {tier: "Free", known: true} @@ -30,7 +27,6 @@ def roturTierLookup(string userId) object ( return {tier: cached.tier.toStr(), known: true} ) ) - // Rotur failed in the last minute: don't make every request wait on it again. if roturTierUnreachable.has(userId) ( return lastKnownRoturTier(userId) ) @@ -48,7 +44,6 @@ def roturTierLookup(string userId) object ( def profileRoturTier(string userId) object ( object response = requests.get(roturBase ++ "/profile?id=" ++ url.escape(userId) ++ "&include_posts=0", roturServiceOptions()).unwrapOr({body: null, error: "request failed", headers: {}, raw: null, status: 0, success: false}) if response.status == 404 ( - // No such Rotur account, so no plan. return {reached: true, tier: "Free"} ) if response.status != 200 ( @@ -72,8 +67,6 @@ def lastKnownRoturTier(string userId) object ( return {tier: "Free", known: false} ) -// "yes" or "no" for whether the project owner's plan includes vanity URLs, -// or "unknown" while Rotur can't be reached. def vanityEligibility(object project) string ( string ownerId = (project.ownerId ?? "").toStr() if ownerId == "" ( @@ -94,9 +87,6 @@ def vanityEligibility(object project) string ( def mistwarpBenefitsForTier(string rawTier) object ( string tier = normalizeRoturTier(rawTier) - // Storage isn't a MistWarp benefit: projects count against the person's - // one Rotur storage allowance (storage-ledger.osl), shown in - // roturBenefitsForTier. object benefits = { maxProjectAssetBytes: 10485760, maxProjectAssetsBytes: 52428800, @@ -187,8 +177,6 @@ def mistwarpBenefitsForProject(object project) object ( def roturBenefitsForTier(string rawTier) object ( string tier = normalizeRoturTier(rawTier) - // storageBytes is the one Rotur storage allowance, which MistWarp projects - // share with Rotur's other apps. object benefits = { storageBytes: 1000000000, accessProducts: 5, diff --git a/src/rotur/rotur-app.osl b/src/rotur/rotur-app.osl index 33f455e..53b81f7 100644 --- a/src/rotur/rotur-app.osl +++ b/src/rotur/rotur-app.osl @@ -5,12 +5,6 @@ export { roturUserRef } -// MistWarp's Rotur App, called with its client ID and secret: -// https://docs.rotur.dev/build-an-app/apps-api -// -// This is how mistwarp-api talks to Rotur about MistWarp itself: bans, -// reports, safety signals, badges and payment requests. - string roturClientId = configValue("ROTUR_CLIENT_ID", "") string roturClientSecret = configValue("ROTUR_CLIENT_SECRET", "") @@ -18,8 +12,6 @@ def roturAppReady() boolean ( return roturClientId != "" and roturClientSecret != "" ) -// How the apps API names someone: their Rotur ID when MistWarp knows it, -// which never changes, else their username. def roturUserRef(string username) string ( string userId = userIdForUsername(username) if userId != "" ( @@ -28,8 +20,6 @@ def roturUserRef(string username) string ( return username ) -// Calls https://api.rotur.dev/v2/apps/. Returns {ok, status, -// data}: ok is false when the app isn't set up or Rotur didn't say yes. def roturApp(string method, string path, object body) object ( if !roturAppReady() ( return {ok: false, status: 0, data: {}} diff --git a/src/rotur/rotur-badges.osl b/src/rotur/rotur-badges.osl index 90f4c73..3b68a4a 100644 --- a/src/rotur/rotur-badges.osl +++ b/src/rotur/rotur-badges.osl @@ -3,25 +3,6 @@ export { roturBadgeEvent } -// Badges on people's Rotur profiles, given by MistWarp's Rotur App: -// https://docs.rotur.dev/build-an-app/badges -// -// Which badges exist, and what earns them, is data, not code. Define the -// badges on rotur.dev/me/developer, then map MistWarp's events to them in -// data/rotur-badges.json, for example: -// -// {"events": { -// "project_shared": [{"badge": "creator", "delta": 1}], -// "streak": [{"badge": "streak", "progress": "value"}] -// }} -// -// "delta" adds to someone's progress. "progress": "value" sets it to the -// event's value. The events are project_shared (a project shared for the first -// time), love_received (once per person per project), remixed (a remix of -// your project was shared), comment_posted, and streak (days in a row you've -// saved a project). A rule with neither is skipped. Without the file, nothing -// is sent. - string roturBadgesFile = "data/rotur-badges.json" def roturBadgeRules(string event) array ( @@ -45,15 +26,12 @@ def awardRoturBadges(string username, string event, number value) ( ) else if (rule.progress ?? "").toStr() == "value" ( change = {progress: value} ) - // A rule that says neither is a mistake; sending it would reset progress. if badge != "" and change.len > 0 ( roturApp("PUT", "/badges/" ++ url.escape(badge) ++ "/users/" ++ person, change) ) ) ) -// Badge updates go out in the background, so they never slow a request down. -// Rotur only gives badges to MistWarp's own users, and ignores anyone else. def roturBadgeEvent(string username, string event, number value) ( if username == "" or isStudentUsername(username) or !roturAppReady() or roturBadgeRules(event).len == 0 ( return @@ -61,7 +39,6 @@ def roturBadgeEvent(string username, string event, number value) ( thread.new(awardRoturBadges, username, event, value) ) -// Counts days in a row on which someone saved a project. def recordCreatingDay(string username) ( if username == "" or isStudentUsername(username) ( return diff --git a/src/rotur/rotur-badges.test.osl b/src/rotur/rotur-badges.test.osl index 7d3894d..903b41b 100644 --- a/src/rotur/rotur-badges.test.osl +++ b/src/rotur/rotur-badges.test.osl @@ -37,8 +37,7 @@ def readFile(string path) object ( import "./rotur-badges.osl" -// Days in a row count up; a missed day starts again; the same day counts once. -timestamp = (86400000 * 100) + 5000 +number timestamp = (86400000 * 100) + 5000 recordCreatingDay("sam") recordCreatingDay("sam") timestamp = (86400000 * 101) + 5000 diff --git a/src/rotur/rotur-payments.osl b/src/rotur/rotur-payments.osl index 252548e..e45d462 100644 --- a/src/rotur/rotur-payments.osl +++ b/src/rotur/rotur-payments.osl @@ -5,24 +5,10 @@ export { startPurchase } -auto roturAppOwnerCache = cache.new({capacity: 4, ttl: 3600000}) +*cache.Cache roturAppOwnerCache = cache.new({capacity: 4, ttl: 3600000}) +number roturPaymentRequestLifetimeMs = 900000 +number paymentRequestReuseMarginMs = 60000 -// Payments through Rotur payment requests: -// https://docs.rotur.dev/build-an-app/payments -// -// mistwarp-api asks Rotur, with the app's credentials, for someone to pay. They -// approve it on rotur.dev with their password, Rotur moves the credits (shared -// between everyone a sale pays), and Rotur tells MistWarp with the -// payment.completed webhook. MistWarp never holds a permission to spend -// anyone's credits. -// -// Each purchase key (data/purchase-keys.json) remembers its payment request. -// The key is the request's reference, so the webhook knows which purchase was -// paid, and the editor's confirm call asks Rotur directly if the webhook hasn't -// come yet. - -// MistWarp's share of a sale goes to its own Rotur account: MISTWARP_ROTUR_USER, -// or the MistWarp Rotur App's owner. def mistwarpFeeAccount() string ( if mistwarpRoturUser != "" ( return mistwarpRoturUser @@ -41,9 +27,6 @@ def mistwarpFeeAccount() string ( return owner ) -// Asks Rotur for a payment of amount from payer to payee, shared by splits -// ([{username, basis_points}]) when there is more than one. Returns {ok, id, -// approveUrl} or {ok: false, error}. def requestRoturPayment(string payer, string payee, array splits, number amount, string note, string reference, string returnUrl) object ( object body = {payer: roturUserRef(payer), "to": roturUserRef(payee), amount: amount, note: note.trim(1, 160), reference: reference} if splits.len > 1 ( @@ -59,8 +42,6 @@ def requestRoturPayment(string payer, string payee, array splits, number amount, ) object res = roturApp("POST", "/payment-requests", body) if !res.ok and (res.data.code ?? "").toStr() == "invalid_return_url" ( - // The editor's page isn't on one of the app's origins: no way back, but - // the editor still sees the window close. body.delete("return_url") res = roturApp("POST", "/payment-requests", body) ) @@ -69,15 +50,11 @@ def requestRoturPayment(string payer, string payee, array splits, number amount, ) number expiresAt = (res.data.expires_at ?? 0).toNum() if expiresAt <= 0 ( - // Rotur's requests last 15 minutes. - expiresAt = timestamp + 900000 + expiresAt = timestamp + roturPaymentRequestLifetimeMs ) return {ok: true, id: res.data.id.toStr(), approveUrl: res.data.approve_url.toStr(), expiresAt: expiresAt} ) -// Rotur refuses splits that pay the payer, as when someone buys a remix of -// their own project, or the fee account buys something. Their share goes to -// the next person in the splits instead. def splitsWithoutPayer(array splits, string payer) array ( array kept = [] number carried = 0 @@ -99,10 +76,8 @@ def splitsWithoutPayer(array splits, string payer) array ( return kept ) -// The same purchase, started again before it was paid: everything the caller -// said about it matches, and Rotur's request has a minute or more left. def samePurchase(object stored, object entry, string prefix, string key) boolean ( - if !key.startsWith(prefix) or stored.paid == true or (stored.expiresAt ?? 0).toNum() < timestamp + 60000 or (stored.approveUrl ?? "").toStr() == "" ( + if !key.startsWith(prefix) or stored.paid == true or (stored.expiresAt ?? 0).toNum() < timestamp + paymentRequestReuseMarginMs or (stored.approveUrl ?? "").toStr() == "" ( return false ) array fields = entry.getKeys() @@ -115,8 +90,6 @@ def samePurchase(object stored, object entry, string prefix, string key) boolean return true ) -// Rotur's status for a payment request: "pending", "paid", "declined", -// "cancelled" or "expired", or "" if it couldn't be read. def roturPaymentStatus(string requestId) string ( object res = roturApp("GET", "/payment-requests/" ++ url.escape(requestId), {}) if !res.ok ( @@ -133,15 +106,9 @@ def roturPaymentPaid(string requestId, string reference) boolean ( return res.ok and res.data.status == "paid" and (res.data.reference ?? "").toStr() == reference ) -// Starts a purchase: saves its key and asks Rotur for the payment. Returns what -// the editor needs to open Rotur's window, with the splits Rotur was given, or -// {ok: false, error}. The same purchase started again while its request is -// still open gets that request back, so one purchase is only ever one payment. def startPurchase(string prefix, object entry, string payer, string payee, array splits, number amount, string note, string returnUrl) object ( array shares = splitsWithoutPayer(splits, payer) entry.splits = shares - // One start at a time for the same purchase, so two clicks can't both miss - // the open request and make two. string lock = "purchase-start:" ++ prefix ++ json.stringify(entry) sync.lock(lock) defer sync.unlock(lock) @@ -150,8 +117,6 @@ def startPurchase(string prefix, object entry, string payer, string payee, array for i openKeys.len ( string openKey = openKeys[i].toStr() object stored = open[openKey].assertElse(object, {}) - // Only a request still waiting for an answer is reused. A declined one - // would leave the person unable to try again until it expired. if samePurchase(stored, entry, prefix, openKey) and roturPaymentStatus(stored.requestId.toStr()) == "pending" ( return {ok: true, key: openKey, requestId: stored.requestId.toStr(), approveUrl: stored.approveUrl.toStr(), splits: shares} ) @@ -179,8 +144,6 @@ def storePurchaseKeyLocked(string key, object entry) ( savePurchaseKeys(store) ) -// The purchase key's entry, once it has been paid, else {}. Rotur is asked if -// the webhook hasn't said so yet. def paidPurchase(string key) object ( object keys = loadPurchaseKeys().keys if key == "" or !keys.contains(key) ( @@ -204,9 +167,6 @@ def forgetPurchase(string key) ( savePurchaseKeys(store) ) -// payment.completed: mark the purchase paid, and deliver what needs no more -// from the buyer. Donations and refunds wait for the comment or the revoke -// that started them. def completeRoturPayment(string reference, string requestId) ( if !markPurchasePaidLocked(reference, requestId) ( return diff --git a/src/rotur/rotur-payments.test.osl b/src/rotur/rotur-payments.test.osl index ca36d9d..36c52f2 100644 --- a/src/rotur/rotur-payments.test.osl +++ b/src/rotur/rotur-payments.test.osl @@ -51,7 +51,6 @@ def roturUserRef(string username) string ( return username ) -// A pretend Rotur: payment requests by ID, and every call made. array calls = [] object requests = {} def roturApp(string method, string path, object body) object ( @@ -92,7 +91,6 @@ def completeGameProductPurchase(string key) object ( import "./rotur-payments.osl" -// MistWarp's share goes to the app owner unless MISTWARP_ROTUR_USER says otherwise. testing.assert(mistwarpFeeAccount() == "mist", "the app owner gets MistWarp's share") mistwarpFeeAccount() number ownerLookups = 0 @@ -103,7 +101,6 @@ for i calls.len ( ) testing.assert(ownerLookups == 1, "the owner is looked up once") -// A sale shared between people asks Rotur for one request with splits. object sale = startPurchase("mwbuy_", {projectId: "p1", buyer: "sam", price: 10}, "sam", "kit", [{username: "kit", basis_points: 9000}, {username: "mist", basis_points: 1000}], 10, "A game", "https://mistwarp.org/projects/p1") testing.assert(sale.ok and sale.key == "mwbuy_k1" and sale.approveUrl == "https://rotur.dev/pay/approve/" ++ sale.requestId, "starts a purchase: " ++ json.stringify(sale)) object asked = calls[calls.len].body @@ -111,17 +108,13 @@ testing.assert(asked.payer == "u-sam" and asked["to"] == "kit" and asked.amount testing.assert(asked.splits.len == 2 and asked.splits[1].basis_points == 9000 and asked.splits[2]["to"] == "mist", "shares the sale") testing.assert(asked.return_url == "https://mistwarp.org/projects/p1", "comes back to the page") -// One person paid needs no splits. object tip = startPurchase("mwdonate_", {kind: "comment_donation", projectId: "p1", buyer: "sam", amount: 2}, "sam", "kit", [], 2, "Donation", "") testing.assert(tip.ok and !calls[calls.len].body.assert(object).contains("splits") and !calls[calls.len].body.assert(object).contains("return_url"), "a donation goes to one person") -// Starting the same purchase again while its request is open gets that -// request back, not a second one to pay. number postsBefore = calls.len object again = startPurchase("mwbuy_", {projectId: "p1", buyer: "sam", price: 10}, "sam", "kit", [{username: "kit", basis_points: 9000}, {username: "mist", basis_points: 1000}], 10, "A game", "") testing.assert(again.ok and again.key == sale.key and again.requestId == sale.requestId and again.approveUrl == sale.approveUrl, "reuses the open request: " ++ json.stringify(again)) testing.assert(calls.len == postsBefore + 1 and calls[calls.len].method == "GET", "only checks the open request is still pending") -// A declined request isn't reused, so the person can try again straight away. requests[sale.requestId].status = "declined" object afterDecline = startPurchase("mwbuy_", {projectId: "p1", buyer: "sam", price: 10}, "sam", "kit", [{username: "kit", basis_points: 9000}, {username: "mist", basis_points: 1000}], 10, "A game", "") testing.assert(afterDecline.ok and afterDecline.key != sale.key and afterDecline.requestId != sale.requestId, "a declined request gets a new one") @@ -134,7 +127,6 @@ savePurchaseKeys(store) object afterExpiry = startPurchase("mwbuy_", {projectId: "p1", buyer: "sam", price: 12}, "sam", "kit", [], 12, "A game", "") testing.assert(afterExpiry.ok and afterExpiry.key != otherPrice.key, "a request about to expire isn't reused") -// The payer is never in the splits: their share goes to the next person. array shares = splitsWithoutPayer([{username: "mist", basis_points: 1000}, {username: "Sam", basis_points: 1000}, {username: "kit", basis_points: 8000}], "sam") testing.assert(json.stringify(shares) == json.stringify([{username: "mist", basis_points: 1000}, {username: "kit", basis_points: 9000}]), "a remix parent buying gives their share on: " ++ json.stringify(shares)) shares = splitsWithoutPayer([{username: "kit", basis_points: 9000}, {username: "mist", basis_points: 1000}], "mist") @@ -142,7 +134,6 @@ testing.assert(json.stringify(shares) == json.stringify([{username: "kit", basis object ownSale = startPurchase("mwbuy_", {projectId: "p2", buyer: "mist", price: 10}, "mist", "kit", [{username: "kit", basis_points: 9000}, {username: "mist", basis_points: 1000}], 10, "A game", "") testing.assert(ownSale.ok and !calls[calls.len].body.assert(object).contains("splits") and loadPurchaseKeys().keys[ownSale.key].splits.len == 1, "with only the creator left, no splits are sent, and the key keeps what was used") -// A page Rotur won't send people back to still gets a request. object elsewhere = startPurchase("mwbuy_", {projectId: "p1", buyer: "sam", price: 1}, "sam", "kit", [], 1, "A game", "https://evil.example/") testing.assert(elsewhere.ok and !calls[calls.len].body.assert(object).contains("return_url"), "retries without the return URL") @@ -150,19 +141,16 @@ object refused = startPurchase("mwbuy_", {projectId: "p1", buyer: "sam", price: testing.assert(!refused.ok and refused.error == "amount must be positive", "passes Rotur's refusal on") testing.assert(!loadPurchaseKeys().keys.assert(object).contains("mwbuy_k" ++ keyCount.toStr()), "a refused request saves no key") -// Nothing counts as paid until Rotur says so. testing.assert(paidPurchase(sale.key).len == 0, "an unpaid purchase is not paid") testing.assert(paidPurchase("mwbuy_nope").len == 0, "an unknown key is not paid") testing.assert(paidPurchase("").len == 0, "no key is not paid") -// The editor's confirm can ask Rotur before the webhook comes. requests[sale.requestId].status = "paid" testing.assert(paidPurchase(sale.key).paid == true, "a paid request is paid") requests[tip.requestId].status = "paid" requests[tip.requestId].reference = "mwdonate_other" testing.assert(paidPurchase(tip.key).len == 0, "a paid request for another reference doesn't count") -// The webhook marks the purchase paid and delivers sales. completeRoturPayment(sale.key, "payreq_wrong") testing.assert(delivered.len == 0 and loadPurchaseKeys().keys[sale.key].paid != true, "ignores a request that isn't the key's") completeRoturPayment(sale.key, sale.requestId) diff --git a/src/rotur/rotur-signals.osl b/src/rotur/rotur-signals.osl index c44ba92..7a44bad 100644 --- a/src/rotur/rotur-signals.osl +++ b/src/rotur/rotur-signals.osl @@ -5,18 +5,7 @@ export { roturPurchaseRefusal } -auto roturMessageSignals = cache.new({capacity: 10000, ttl: 60000}) - -// Rotur's safety signals: https://docs.rotur.dev/build-an-app/safety -// -// MistWarp declares that people can talk to each other and can spend credits, -// so it asks Rotur about comments and purchases. Rotur answers from blocks, -// parental controls, teen defaults and people's own message settings, without -// saying which, so MistWarp shows Rotur's sentence and offers no way round it. -// -// Rotur only answers about MistWarp's own users: people who signed in through -// Sign in with Rotur, or made their account on MistWarp. For anyone else, or if -// Rotur can't be reached, MistWarp carries on as it did before. +*cache.Cache roturMessageSignals = cache.new({capacity: 10000, ttl: 60000}) def roturMessageSignal(string sender, string recipient) object ( string senderKey = normalizeUsername(sender) @@ -43,7 +32,6 @@ def roturMessageSignal(string sender, string recipient) object ( return signal ) -// "" when the message is allowed, else the sentence to show the sender. def roturMessageRefusal(string sender, string recipient) string ( object signal = roturMessageSignal(sender, recipient) if signal.allowed == true ( @@ -56,9 +44,6 @@ def isMessageNotification(string notifType) boolean ( return ["comment", "reply", "mention", "profile_comment", "space_comment", "roadmap_comment", "donation"].contains(notifType) ) -// "" when the purchase is allowed, else the sentence to show the buyer. Rotur -// counts credits towards a parent's monthly limit when it moves them, so this -// only asks. def roturPurchaseRefusal(string buyer, number amount, string note) string ( if buyer == "" or amount <= 0 ( return "" @@ -70,8 +55,6 @@ def roturPurchaseRefusal(string buyer, number amount, string note) string ( return (res.data.reason ?? "Rotur settings don't allow this purchase.").toStr() ) -// Who a comment reaches: the owner of the project or profile it's on, and the -// author of the comment it replies to. def commentRecipients(string key, string parentId) array ( array recipients = [] if key.startsWith("project-") ( diff --git a/src/rotur/rotur-webhooks.osl b/src/rotur/rotur-webhooks.osl index 21f2285..0c85fb3 100644 --- a/src/rotur/rotur-webhooks.osl +++ b/src/rotur/rotur-webhooks.osl @@ -2,39 +2,22 @@ export { processRoturWebhooks } -// Webhooks from MistWarp's Rotur App: https://docs.rotur.dev/build-an-app/webhooks -// -// Rotur tells MistWarp when someone deletes their Rotur account (user.deleted), -// leaves MistWarp from rotur.dev/me/apps (user.left) or is banned from Rotur -// (user.banned). MistWarp must then delete what it holds about them within 30 -// days, so it does it straight away. With privacy requests turned on, people -// can also ask for a copy of their data or for it to be deleted. When someone -// approves a payment request, payment.completed says which purchase was paid -// (rotur-payments.osl). -// -// Rotur waits 10 seconds for an answer, and erasing someone can take longer, -// so each delivery is saved to disk, answered, then handled by a worker. Saved -// events also stop a retried delivery from being handled twice. - string roturWebhookSecret = configValue("ROTUR_WEBHOOK_SECRET", "") string roturWebhooksFile = "data/rotur-webhooks.json" -// Handled events are kept this long, longer than Rotur retries (36 minutes). number roturWebhookRetention = 604800000 -// Rotur-Signature is v1= and the hex HMAC-SHA256 of "..", keyed with the whole signing secret. def roturWebhookSignature(string secret, string id, string sentAt, string body) string ( return "v1=" ++ crypto.hmac("sha256", secret, id ++ "." ++ sentAt ++ "." ++ body) ) -// A delivery is genuine if it is signed with the secret and was sent within -// five minutes of now (in Unix seconds), so an old one can't be replayed. +number roturWebhookMaxClockSkewSeconds = 300 + def roturWebhookValid(string secret, string id, string sentAt, string body, string signature, number now) boolean ( if secret == "" or id == "" or signature == "" or !regex.match("^[0-9]{1,12}$", sentAt) ( return false ) number sent = sentAt.toNum() - if sent < now - 300 or sent > now + 300 ( + if sent < now - roturWebhookMaxClockSkewSeconds or sent > now + roturWebhookMaxClockSkewSeconds ( return false ) return crypto.constantTimeCompare(signature, roturWebhookSignature(secret, id, sentAt, body)) @@ -48,7 +31,6 @@ def loadRoturWebhooks() object ( return res.data.assert(object) ) -// Saves a verified delivery. Returns false if it was already saved. def queueRoturWebhook(object event) boolean ( string id = event.id.toStr() object data = {} @@ -121,7 +103,6 @@ def handleRoturEvent(object event) ( ) ) -// Handles every saved event that hasn't been handled yet, one at a time. def processRoturWebhooks() ( sync.lock("rotur-webhooks-worker") defer sync.unlock("rotur-webhooks-worker") diff --git a/src/rotur/rotur-webhooks.test.osl b/src/rotur/rotur-webhooks.test.osl index 4f3ec41..dce374c 100644 --- a/src/rotur/rotur-webhooks.test.osl +++ b/src/rotur/rotur-webhooks.test.osl @@ -56,11 +56,9 @@ def completeRoturPayment(string reference, string requestId) ( ) import "./rotur-webhooks.osl" -roturWebhooksFile = dir ++ "rotur-webhooks.json" +string roturWebhooksFile = dir ++ "rotur-webhooks.json" -// Signatures are checked the way Rotur makes them. string body = "{\"id\":\"evt_1\",\"type\":\"user.left\",\"data\":{\"user_id\":\"u-sam\"}}" -// Made the way Rotur makes it (internal/core/app_webhooks.go, WebhookSignature). string signature = "v1=" ++ crypto.hmac("sha256", "whsec_test", "evt_1.1790000000." ++ body) testing.assert(signature == roturWebhookSignature("whsec_test", "evt_1", "1790000000", body), "signs like Rotur") testing.assert(signature.len == 67, "a hex SHA-256 HMAC") @@ -73,7 +71,6 @@ testing.assert(!roturWebhookValid("whsec_test", "evt_1", "1790000000", body, sig testing.assert(!roturWebhookValid("whsec_test", "evt_1", "soon", body, signature, 1790000060), "refuses a malformed timestamp") testing.assert(!roturWebhookValid("", "evt_1", "1790000000", body, signature, 1790000060), "refuses everything without a secret") -// Each event is handled once, even when Rotur retries it. testing.assert(queueRoturWebhook({id: "evt_1", type: "user.left", data: {user_id: "u-sam"}}), "queues a new event") testing.assert(!queueRoturWebhook({id: "evt_1", type: "user.left", data: {user_id: "u-sam"}}), "ignores a retried event") queueRoturWebhook({id: "evt_2", type: "user.deleted", data: {user_id: "u-unknown"}}) diff --git a/src/rotur/vanity-policy.osl b/src/rotur/vanity-policy.osl index 2304e37..a8b8b6f 100644 --- a/src/rotur/vanity-policy.osl +++ b/src/rotur/vanity-policy.osl @@ -2,16 +2,8 @@ export { vanityAfter } -// A vanity project URL needs a plan that includes one. Once the owner's plan -// stops including it, the URL stops working but stays reserved for them for -// 30 days, so resubscribing brings it straight back. After that it is -// released for anyone to claim. number vanityReserveMs = 2592000000 -// What a project's vanity URL becomes, given whether its owner's plan includes -// one right now: "yes", "no", or "unknown" when Rotur couldn't be asked. An -// unknown plan changes nothing, so a Rotur outage never starts or advances -// the 30 days. def vanityAfter(string slug, number lapsedAt, string eligibility, number now) object ( if slug == "" ( return {slug: "", lapsedAt: 0, active: false} diff --git a/src/storage/archive-security.test.osl b/src/storage/archive-security.test.osl index 53ee55e..0357b1d 100644 --- a/src/storage/archive-security.test.osl +++ b/src/storage/archive-security.test.osl @@ -5,6 +5,7 @@ import "go:mistwarp.local/storagefs" import "./archive-security.osl" string fixtureDir = "/tmp/mistwarp-zip-fixtures-" ++ timestamp.toStr() +array zipCentralDirectoryHeader = [80, 75, 1, 2] void fs.mkdir(fixtureDir, {recursive: true}) def stageParent(string stage, string name) ( @@ -38,8 +39,6 @@ def compressFixture(string kind) byte[] ( return fs.readFileBytes(outPath) ) -// Little-endian reads over 1-based byte arrays. Positions are 1-based, so -// 0-based offset N of a header starting at H becomes H + N. def read16le(byte[] data, number pos) number ( return data[pos].toNum() + (data[(pos + 1)].toNum() * 256) ) @@ -48,8 +47,6 @@ def read32le(byte[] data, number pos) number ( return read16le(data, pos) + (read16le(data, (pos + 2)) * 65536) ) -// Byte search over 1-based OSL byte arrays. Returns the first match at or after -// from, or 0. def findBytes(byte[] data, array needle, number from) number ( for i in 1 to ((data.len - needle.len - from) + 2) ( number pos = (from + i) - 1 @@ -67,8 +64,6 @@ def findBytes(byte[] data, array needle, number from) number ( return 0 ) -// Overwrite every occurrence of orig with replacement. Both stay the same -// length, so offsets elsewhere in the archive are untouched. def patchAll(byte[] data, array orig, array replacement) byte[] ( number pos = findBytes(data, orig, 1) while pos != 0 ( @@ -80,8 +75,6 @@ def patchAll(byte[] data, array orig, array replacement) byte[] ( return data ) -// Malformed fixtures start as honest archives and get byte patches, since no -// honest writer emits duplicates, traversal names, or symlink entries. def zipFixture(string kind) byte[] ( if kind == "valid" ( stageText(kind, "project.json", "{}") @@ -99,7 +92,7 @@ def zipFixture(string kind) byte[] ( if kind == "symlink" ( stageText(kind, "link", "target") byte[] data = compressFixture(kind) - number header = findBytes(data, [80, 75, 1, 2], 1) + number header = findBytes(data, zipCentralDirectoryHeader, 1) while header != 0 ( number nameLen = read16le(data, (header + 28)) number nameEnd = ((header + 46) + nameLen) - 1 @@ -111,7 +104,7 @@ def zipFixture(string kind) byte[] ( data[(header + 41)] = 161 return data ) - header = findBytes(data, [80, 75, 1, 2], header + 1) + header = findBytes(data, zipCentralDirectoryHeader, header + 1) ) testing.assert(false, "symlink entry not found") return data @@ -142,9 +135,7 @@ def zipFixture(string kind) byte[] ( ) stageText(kind, "corrupt", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx") byte[] data = compressFixture(kind) - // Flip the first payload byte of the file entry: directories have no payload - // and only corrupt payload bytes trip the CRC check on read. - number entry = findBytes(data, [80, 75, 1, 2], 1) + number entry = findBytes(data, zipCentralDirectoryHeader, 1) while entry != 0 ( number entryNameLen = read16le(data, (entry + 28)) number entryNameEnd = ((entry + 46) + entryNameLen) - 1 @@ -154,7 +145,7 @@ def zipFixture(string kind) byte[] ( data[payloadStart] = 255 - data[payloadStart] return data ) - entry = findBytes(data, [80, 75, 1, 2], (entry + 1)) + entry = findBytes(data, zipCentralDirectoryHeader, (entry + 1)) ) testing.assert(false, "file entry not found") return data diff --git a/src/storage/quota-policy.osl b/src/storage/quota-policy.osl index 5a1e6b6..431cffc 100644 --- a/src/storage/quota-policy.osl +++ b/src/storage/quota-policy.osl @@ -25,8 +25,6 @@ def quotaTotalBytes(array events, number nowMs, number windowMs) number ( return total ) -// Drop pending uploads that a saved project now references, so their bytes -// are counted once, as part of that project. def quotaWithoutNames(array events, array names, number nowMs, number windowMs) array ( object saved = {} for i names.len ( @@ -43,8 +41,6 @@ def quotaWithoutNames(array events, array names, number nowMs, number windowMs) return kept ) -// The most a project may occupy after a save. A user over their limit can -// still save a project at its current size or smaller. def storageAllowance(number limit, number used, number currentBytes) number ( number allowance = limit - (used - currentBytes) if allowance < currentBytes ( diff --git a/src/storage/r2-snapshot-serving.osl b/src/storage/r2-snapshot-serving.osl index 1e3e943..801a340 100644 --- a/src/storage/r2-snapshot-serving.osl +++ b/src/storage/r2-snapshot-serving.osl @@ -28,11 +28,8 @@ def serveProjectJsonStorageKey(*serve.Context c, string key, boolean gzip) ( return ) string signedUrl = r2Client.presignURL(key, {expires: 60000}) - // An explicit encoding prevents the HTTP client from decompressing R2's body. auto fetched = requests.get(signedUrl, {headers: {"Accept-Encoding": "identity"}}).unwrapOr({body: null, error: "request failed", headers: {}, raw: null, status: 0, success: false}) if fetched.status != 200 ( - // A missing object is a real 404, but a refused credential or an R2 - // outage must not be reported as "this project has no snapshot". log "WARNING: project snapshot fetch failed for " ++ key ++ ": status " ++ fetched.status.toStr() if fetched.status == 404 ( c.notFound("project snapshot not found") @@ -41,7 +38,6 @@ def serveProjectJsonStorageKey(*serve.Context c, string key, boolean gzip) ( ) return ) - // HTTP responses contain a raw binary string; asserting byte[] would decode it as base64. string snapshotBody = fetched.body ?? "" if snapshotBody.len == 0 or snapshotBody.toBytes().len > maxStoredProjectJsonBytes ( log "WARNING: project snapshot fetch for " ++ key ++ " returned an empty or oversized body" diff --git a/src/storage/r2-snapshot-serving.test.osl b/src/storage/r2-snapshot-serving.test.osl index 7a1cbab..b296849 100644 --- a/src/storage/r2-snapshot-serving.test.osl +++ b/src/storage/r2-snapshot-serving.test.osl @@ -13,7 +13,6 @@ void fs.mkdir(testDir, {recursive: true}) boolean r2Local = false number maxStoredProjectJsonBytes = 20971520 -// Fake R2: serves whatever the current case stages. number stubStatus = 200 boolean stubGzip = false string stubBodyPath = testDir ++ "/stub-body.bin" @@ -34,7 +33,7 @@ def runStubServer() ( stubApp.run(":18331") ) -auto r2Client = s3.new({endpoint: "http://127.0.0.1:18331", bucket: "test", access_key_id: "test", secret_access_key: "test"}) +*s3.Client r2Client = s3.new({endpoint: "http://127.0.0.1:18331", bucket: "test", access_key_id: "test", secret_access_key: "test"}) def isSafeBlobKey(string key) boolean ( return key != "" and !key.contains("..") diff --git a/src/storage/r2-upload-queue.osl b/src/storage/r2-upload-queue.osl index f650bd9..0c25410 100644 --- a/src/storage/r2-upload-queue.osl +++ b/src/storage/r2-upload-queue.osl @@ -11,8 +11,6 @@ def projectStorageActivityAt(object project) number ( return if edited > 0 then edited else created ) -// Shared blobs must wait for every project that references them, including trash. -// Read under the queue lock, which also serializes project metadata publication. def loadR2BlobActivity() object ( object activity = {} array directories = [projectsDir, trashDir] @@ -86,7 +84,6 @@ def queueR2UploadLocked(string key, string contentType, string contentEncoding) for i items.len ( if items[i].assertElse(object, {}).key.toStr() == key ( item.firstQueuedAt = items[i].assert(object).firstQueuedAt ?? items[i].assert(object).queuedAt ?? timestamp - // Keep upload age while restarting the local-cache inactivity window. items[i] = item found = true ) @@ -138,7 +135,6 @@ def flushR2UploadQueue() ( queued[items[i].assert(object).key.toStr()] = true ) number now = timestamp - // Older flushes retained their local copies. Reclaim those too, without PUTs. array storedKeys = stored.getKeys() for i storedKeys.len ( string key = storedKeys[i].toStr() @@ -158,7 +154,6 @@ def flushR2UploadQueue() ( boolean removed = !isSafeBlobKey(key) number uploadAge = now - (item.firstQueuedAt ?? item.queuedAt ?? now).toNum() if !removed and (r2UploadReady(item, activity, now) or uploadAge >= 3600000) ( - // Only immutable keys can use the inventory to skip an upload. boolean uploaded = (item.uploaded ?? false).assert(boolean) or (stored.contains(key) and !key.startsWith("projects/")) if !uploaded and uploadedItems < 64 and uploadedBytes < 268435456 and fs.exists(path) ( byte[] body = fs.readFileBytes(path).unwrapOr("".toBytes()) @@ -170,7 +165,6 @@ def flushR2UploadQueue() ( ) if uploaded ( item.uploaded = true - // Active projects retain their fast local copy after cloud backup. removed = !fs.exists(path) or !r2UploadReady(item, activity, now) or fs.remove(path) ) ) diff --git a/src/storage/r2-upload-queue.test.osl b/src/storage/r2-upload-queue.test.osl index 06cd945..7666c99 100644 --- a/src/storage/r2-upload-queue.test.osl +++ b/src/storage/r2-upload-queue.test.osl @@ -89,7 +89,6 @@ writeFile(projectsDir ++ "active.json", active) flushR2UploadQueue() testing.assert(puts.len == 0 and loadR2UploadQueue().len == 3, "new projects upload nothing on the five-minute flush") -// Simulate a persisted queue older than seven days, followed by another project save. array queuedItems = loadR2UploadQueue() for i queuedItems.len ( object queuedItem = queuedItems[i].assert(object) @@ -128,7 +127,6 @@ for i keys.len ( testing.assert(!fs.exists(localBlobPath(keys[i].toStr())), "successful uploads remove local copies") ) -// Existing deployments already have uploaded blobs that were never evicted. for i keys.len ( void fs.writeFile(localBlobPath(keys[i].toStr()), "payload") ) @@ -154,7 +152,6 @@ flushR2UploadQueue() testing.assert(fs.exists(localBlobPath(keys[1].toStr())), "unreadable project metadata prevents unsafe eviction") void fs.remove(projectsDir ++ "unreadable.json") -// A busy project gets a backup after one hour without losing its local cache. writeFile(r2UploadQueueFile, {items: []}) active.edited = timestamp writeFile(projectsDir ++ "active.json", active) diff --git a/src/storage/storage-ledger-policy.osl b/src/storage/storage-ledger-policy.osl index 1b4c6bf..4ec53c0 100644 --- a/src/storage/storage-ledger-policy.osl +++ b/src/storage/storage-ledger-policy.osl @@ -2,28 +2,8 @@ export { projectStorageSummary } -// What MistWarp counts against a person's one Rotur storage allowance, which -// the Rotur API keeps in its storage ledger. -// -// It's every blob their projects (live and in the trash) and backpack keep in -// storage, each counted once for that person however many of their projects -// use it: a costume in ten of their projects counts once, and so does a remix's -// project.json until the remix is first saved. Blobs are only ever shared -// within one person's count; two people using the same asset each count it. -// Thumbnails and share cards are made from projects, so they don't count, and -// neither do backpack previews. -// -// The ledger holds one object per kind, named by ledgerKinds, at that kind's -// total: "projects" is project.json snapshots (including those releases keep) -// and history archives, "assets" is costumes and sounds, and "backpack" is -// backpack items. A change to any of them replaces the total, so it only needs -// room for what it adds. - array ledgerKinds = ["projects", "assets", "backpack"] -// The blobs one project keeps: { json: {key: bytes}, history: {key: bytes}, -// assets: [md5ext] }. assetSizes is the assets index, for working out the -// size of project.json on projects saved before it was recorded. def projectStorageSummary(object project, object assetSizes) object ( array assets = (project.assets ?? []).assertElse(array, []) object jsonBlobs = {} @@ -49,7 +29,6 @@ def projectStorageSummary(object project, object assetSizes) object ( return {json: jsonBlobs, history: historyBlobs, assets: assets} ) -// Adds a release's project.json snapshot to its project's summary. def addReleaseBlob(object summary, string key, number bytes) object ( if key != "" ( object jsonBlobs = (summary.json ?? {}).assertElse(object, {}) @@ -59,8 +38,6 @@ def addReleaseBlob(object summary, string key, number bytes) object ( return summary ) -// Each kind's total for one person: summaries are their projects', and -// backpack their backpack items. A blob in more than one summary counts once. def storageTotals(array summaries, object assetSizes, array backpack) object ( object jsonSeen = {} object historySeen = {} @@ -105,7 +82,6 @@ def storageTotals(array summaries, object assetSizes, array backpack) object ( return {projects: projectBytes, assets: assetBytes, backpack: backpackBytes} ) -// The kinds whose total would grow from what the ledger counts. def ledgerGrowth(object counted, object totals) array ( array grown = [] for i ledgerKinds.len ( @@ -117,8 +93,6 @@ def ledgerGrowth(object counted, object totals) array ( return grown ) -// What brings the ledger in line with the totals: { kind, bytes } to set a -// kind's total, with bytes 0 to stop counting it. def ledgerChanges(object counted, object totals) array ( array changes = [] for i ledgerKinds.len ( diff --git a/src/storage/storage-ledger-policy.test.osl b/src/storage/storage-ledger-policy.test.osl index 23ed2a1..259f116 100644 --- a/src/storage/storage-ledger-policy.test.osl +++ b/src/storage/storage-ledger-policy.test.osl @@ -24,7 +24,6 @@ testing.equal(projectStorageSummary(older, sizes).json, {"projects/p0/project.js testing.equal(projectStorageSummary(older, sizes).history, {}, "and with no history, none counts") testing.equal(projectStorageSummary({id: "p9"}, sizes), {json: {}, history: {}, assets: []}, "an empty project keeps nothing") -// An untouched remix of their own project shares its project.json and assets. object remix = { id: "p2", jsonKey: "project-json/j1.json.gz", @@ -39,7 +38,6 @@ testing.equal(totals.projects, 50 + 400 + 300, "a shared project.json counts onc testing.equal(totals.assets, 1100, "an asset in two projects counts once") testing.equal(totals.backpack, 0) -// A release keeps an older project.json after the project moves on. object released = addReleaseBlob(projectStorageSummary(game, sizes), "project-json/j0.json.gz", 45) testing.equal(storageTotals([released], sizes, []).projects, 50 + 400 + 45, "a release's snapshot counts") testing.equal(storageTotals([addReleaseBlob(projectStorageSummary(game, sizes), "project-json/j1.json.gz", 50)], sizes, []).projects, 450, "a release of the current version adds nothing") @@ -51,7 +49,6 @@ array backpack = [ testing.equal(storageTotals([], sizes, backpack).backpack, 100, "backpack items count, their previews don't") testing.equal(storageTotals([{assets: ["missing.png"]}], sizes, []).assets, 0, "an asset that isn't stored counts nothing") -// What the ledger is asked to change. object counted = {projects: 750, assets: 1100} testing.equal(ledgerGrowth(counted, {projects: 800, assets: 1100, backpack: 100}), ["projects", "backpack"], "only what grows needs room") testing.equal(ledgerGrowth(counted, {projects: 10, assets: 0, backpack: 0}), [], "shrinking needs none") diff --git a/src/storage/storage-ledger.osl b/src/storage/storage-ledger.osl index 4af3f15..159b8d0 100644 --- a/src/storage/storage-ledger.osl +++ b/src/storage/storage-ledger.osl @@ -16,34 +16,11 @@ export { usesRoturStorage } -// MistWarp's projects, assets and backpack count against each person's one -// Rotur storage allowance (storage-ledger-policy.osl says what counts). The -// Rotur API keeps the count in its storage ledger: -// -// - Before something is stored, the kinds it makes bigger are reserved at -// their new totals (storageHold). The ledger only holds what they grow by, -// and answers 413 when that doesn't fit. Making things smaller always fits. -// - Once it's stored, the totals are worked out again from what's really -// there and the ledger is set to them (storageSettle), committing what was -// reserved. Deleting, emptying the trash and anything else that only frees -// space just settles. -// - Every six hours everyone is settled, which catches anything a settle -// missed while Rotur couldn't be reached. -// -// When Rotur can't be asked, MistWarp saves anyway rather than lose someone's -// work, and the next settle counts it. Classroom students aren't Rotur -// accounts: their class's plan gives them storage, worked out here as before. -// -// mistwarp-api runs on the same machine as the Rotur API, so it calls it -// directly as the official app "mistwarp" with ROTUR_SERVICE_SECRET. -// ROTUR_LOCAL_API_URL replaces http://127.0.0.1:5602 and has to be a loopback -// address, so the secret never leaves the machine. - string ledgerSecret = configValue("ROTUR_SERVICE_SECRET", "") string ledgerBaseSetting = configValue("ROTUR_LOCAL_API_URL", "") number ledgerPendingTtlSeconds = 604800 -auto projectStorageSummaries = cache.new({capacity: 200000, ttl: 86400000}) -auto storageSettleQueued = cache.new({capacity: 100000, ttl: 60000}) +*cache.Cache projectStorageSummaries = cache.new({capacity: 200000, ttl: 86400000}) +*cache.Cache storageSettleQueued = cache.new({capacity: 100000, ttl: 60000}) def ledgerBase() string ( string base = ledgerBaseSetting.strip() @@ -56,8 +33,6 @@ def ledgerBase() string ( return base ) -// http://127.0.0.1:, http://localhost: or http://[::1]:, -// with nothing after the port. def ledgerIsLoopback(string base) boolean ( array prefixes = ["http://127.0.0.1:", "http://localhost:", "http://[::1]:"] for i prefixes.len ( @@ -82,8 +57,6 @@ def logLedgerSetup() ( ) ) -// Rotur accounts count through the ledger. Students and projects without a -// known Rotur owner don't. def usesRoturStorage(string userId) boolean ( return userId != "" and !isStudentUserId(userId) ) @@ -96,7 +69,6 @@ def projectOwnerId(object project) string ( return ownerId ) -// { status, body }: status 0 when the ledger couldn't be asked at all. def ledgerCall(string method, string path, object body) object ( if !ledgerReady() ( return {status: 0, body: {}} @@ -128,8 +100,6 @@ def ledgerCall(string method, string path, object body) object ( return {status: status, body: data} ) -// The account's whole Rotur storage: { ok, used, max, available, bytes, -// reserved }, where bytes and reserved are MistWarp's part of it. def ledgerAllowance(string userId) object ( object resp = ledgerCall("GET", "/v2/storage-ledger/usage?user_id=" ++ url.escape(userId), {}) if resp.status != 200 ( @@ -146,7 +116,6 @@ def ledgerAllowance(string userId) object ( } ) -// What the ledger counts for MistWarp: { ok, objects: {name: bytes} }. def ledgerCounted(string userId) object ( object resp = ledgerCall("GET", "/v2/storage-ledger/objects?user_id=" ++ url.escape(userId), {}) if resp.status != 200 ( @@ -161,9 +130,6 @@ def ledgerCounted(string userId) object ( return {ok: true, objects: objects} ) -// Holds room for object at bytes. { ok: true }, or { ok: false, status } with -// used, max, available and needed when the allowance is full (413). ttl is -// in seconds, 0 for the ledger's hour. def ledgerReserve(string userId, string reservation, string objectName, number bytes, number ttl) object ( object body = {id: reservation, user_id: userId, object: objectName, bytes: bytes} if ttl > 0 ( @@ -190,8 +156,6 @@ def ledgerReserve(string userId, string reservation, string objectName, number b return {ok: false, status: resp.status} ) -// Counts a reservation as stored. Sent again, the ledger answers with the -// first commit, so it's tried a few times. def ledgerCommit(string userId, string reservation, number bytes) boolean ( for attempt 3 ( object resp = ledgerCall("POST", "/v2/storage-ledger/reservations/" ++ reservation ++ "/commit", {user_id: userId, bytes: bytes}) @@ -211,7 +175,6 @@ def ledgerCommit(string userId, string reservation, number bytes) boolean ( return false ) -// Gives a reservation back. One that isn't released expires on its own. def ledgerRelease(string userId, string reservation) ( object resp = ledgerCall("DELETE", "/v2/storage-ledger/reservations/" ++ reservation ++ "?user_id=" ++ url.escape(userId), {}) if resp.status != 200 and resp.status != 0 ( @@ -223,7 +186,6 @@ def ledgerRemove(string userId, object body) boolean ( body.user_id = userId for attempt 3 ( object resp = ledgerCall("POST", "/v2/storage-ledger/objects/remove", body) - // A deleted account's ledger went with it. if resp.status == 200 or (resp.status == 404 and (resp.body.code ?? "").toStr() == "account_not_found") ( return true ) @@ -240,7 +202,6 @@ def ledgerRemove(string userId, object body) boolean ( return false ) -// Frees everything MistWarp counted for an account, when it's deleted. def ledgerFreeAll(string userId) boolean ( if !usesRoturStorage(userId) ( return true @@ -248,10 +209,6 @@ def ledgerFreeAll(string userId) boolean ( return ledgerRemove(userId, {all: true}) ) -// ---------- What someone stores ---------- - -// A project file's summary, kept while the file is unchanged, with whose it -// is: { ownerId, owner, id, summary }. def storedProjectSummary(string path, object assetSizes) object ( if !fs.exists(path) ( return {} @@ -291,7 +248,6 @@ def storedFor(object stored, string userId, string username) boolean ( return username != "" and normalizeUsername((stored.owner ?? "").toStr()) == normalizeUsername(username) ) -// The project.json snapshots a project's releases keep. def addReleaseBlobs(string projectId, object summary) object ( array releases = releaseDocs.where("projectId", "==", projectId).all() for i releases.len ( @@ -309,8 +265,6 @@ def addReleaseBlobs(string projectId, object summary) object ( return summary ) -// A release's snapshot size: recorded when it was made, or the project's own -// when it's the same snapshot, or the copy on this machine. def releaseJsonBytes(object release, object summary) number ( if release.contains("jsonBytes") and release.jsonBytes != null ( return release.jsonBytes.toNum() @@ -327,9 +281,6 @@ def releaseJsonBytes(object release, object summary) number ( return 0 ) -// One person's totals. changes.projects swaps in summaries by project id, -// for a project about to be saved, and changes.backpack adds backpack items -// about to be stored. def ownerStorageTotals(string userId, object changes) object ( object assetSizes = loadAssetsIndex().assets.assertElse(object, {}) string username = usernameForId(userId) @@ -370,8 +321,6 @@ def ownerStorageTotals(string userId, object changes) object ( return storageTotals(summaries, assetSizes, backpack) ) -// Everyone's totals at once, for the storage report and the six-hourly -// settle: { userId: totals } for every Rotur account with anything stored. def everyoneStorageTotals() object ( object assetSizes = loadAssetsIndex().assets.assertElse(object, {}) object summariesByOwner = {} @@ -419,15 +368,10 @@ def everyoneStorageTotals() object ( return totals ) -// ---------- Holding and settling ---------- - def noStorageHold() object ( return {ok: true, reservations: []} ) -// Reserves what a change would add for userId. { ok: true, reservations }, -// or { ok: false, refusal } when their Rotur storage is full and enforce is -// set. Without enforce (admins), a full allowance doesn't stop the change. def storageHold(string userId, object changes, boolean enforce) object ( if !usesRoturStorage(userId) or !ledgerReady() ( return noStorageHold() @@ -466,7 +410,6 @@ def storageHoldRelease(string userId, object hold) ( ) ) -// Sets the ledger to what userId really stores now, committing hold. def storageSettle(string userId, object hold) ( if !usesRoturStorage(userId) or !ledgerReady() ( return @@ -481,7 +424,6 @@ def storageSettle(string userId, object hold) ( def settleTotalsLocked(string userId, object totals, object hold) ( object listed = ledgerCounted(userId) if !listed.ok ( - // What was reserved expires, and the six-hourly settle counts it. return ) object counted = listed.objects @@ -509,16 +451,12 @@ def settleTotalsLocked(string userId, object totals, object hold) ( if held.ok ( void ledgerCommit(userId, id, bytes) ) else if held.status == 413 ( - // Over their allowance: they keep what they have, uncounted past it, - // and can't add more until they're under it. log "[ledger] " ++ userId ++ " stores more than their Rotur storage allows; " ++ kind ++ " stays at what was counted" ) ) ) ) -// Settles userId shortly, once however many changes arrive meanwhile, for -// changes that only free space. def storageSettleSoon(string userId) ( if !usesRoturStorage(userId) or !ledgerReady() ( return @@ -602,10 +540,6 @@ def storageSettleLoop() ( ) ) -// ---------- Files uploaded ahead of a save ---------- - -// An asset uploaded before the project that uses it is saved holds room for -// up to a week, released when the project is saved. def pendingAssetHold(string ownerId, string username, string name, number bytes, boolean enforce) object ( object held = ledgerReserve(ownerId, "pending-" ++ name, "pending-" ++ name, bytes, ledgerPendingTtlSeconds) if !held.ok and held.status == 413 and enforce ( @@ -626,7 +560,6 @@ def recordPendingAssetLocked(string username, string name, number bytes) ( writeFile(quotaPath(username), {pending: events}) ) -// Releases the holds of assets a save now counts, or that failed to upload. def releasePendingAssetHolds(string ownerId, string username, array names) ( if names.len == 0 ( return @@ -659,10 +592,6 @@ def forgetPendingAssetsLocked(string username, array names) array ( return releasing ) -// ---------- Backfill ---------- - -// What MistWarp stores for everyone, for the Rotur API's one-off -// `claw storage-backfill`: { service, accounts: { id: { kind: bytes } } }. def storageReport() object ( object totals = everyoneStorageTotals() object accounts = {} diff --git a/src/storage/storage-ledger.test.osl b/src/storage/storage-ledger.test.osl index cb7c253..f87508d 100644 --- a/src/storage/storage-ledger.test.osl +++ b/src/storage/storage-ledger.test.osl @@ -12,7 +12,6 @@ import "std:regex" import "std:db" import "go:mistwarp.local/storagefs" -// The accounting works in a folder of its own. string work = "/tmp/mistwarp-storage-ledger-" ++ timestamp.toStr() void fs.mkdir(work, {recursive: true}) string projectsDir = work ++ "/projects/" @@ -24,7 +23,7 @@ void fs.mkdir(trashDir, {recursive: true}) void fs.mkdir(quotaDir, {recursive: true}) number quotaWindowMs = 604800000 -auto testDb = db.open(work ++ "/community.db") +*db.DB testDb = db.open(work ++ "/community.db") *db.Collection projectSearchDocs = testDb.collection("project_search") *db.Collection releaseDocs = testDb.collection("releases") @@ -118,8 +117,6 @@ testing.assert(!ledgerReady(), "without the service secret, nothing is counted") testing.equal(storageHold("u-ana", {}, true), {ok: true, reservations: []}, "and nothing is refused") testing.assert(usesRoturStorage("u-ana") and !usesRoturStorage("mwc_123") and !usesRoturStorage(""), "students and unknown owners aren't Rotur accounts") -// ---------- A stand-in for the Rotur API's storage ledger ---------- - object stub = {max: 0, failing: false, calls: 0, lastApp: ""} object stubObjects = {} object stubReservations = {} @@ -239,8 +236,8 @@ thread.new(def() -> ( fakeLedger.run("127.0.0.1:18761") )) wait 0.3 -ledgerSecret = secret -ledgerBaseSetting = "http://127.0.0.1:18761/" +string ledgerSecret = secret +string ledgerBaseSetting = "http://127.0.0.1:18761/" testing.assert(ledgerReady(), "with the secret and a loopback address, storage counts") def countedFor(string userId) object ( @@ -270,8 +267,6 @@ def putProject(string dir, object project) ( ) ) -// ---------- What Ana stores ---------- - string a1 = "a" * 32 ++ ".png" string a2 = "b" * 32 ++ ".wav" string a3 = "c" * 32 ++ ".svg" @@ -286,26 +281,22 @@ putProject(projectsDir, { workspaceKey: "project-workspaces/h1.mwp", workspaceLayers: ["project-workspaces/h1.mwp"], historyBytes: 4000, assets: [a1, a2] }) -// Her untouched remix of it shares its project.json and assets. putProject(projectsDir, { id: "p2", owner: "ana", ownerId: "u-ana", remixParent: "p1", inheritedJson: true, jsonKey: "project-json/j1.json.gz", storedJsonBytes: 500, workspaceLayers: ["project-workspaces/h2.mwp"], historyBytes: 3000, assets: [a1, a2] }) -// An older project from before ownerIds, in the trash. putProject(trashDir, { id: "p3", owner: "Ana", jsonKey: "projects/p3/project.json", sizeBytes: 1300, assets: [a3] }) -// Bea's project uses one of the same assets; that counts for her too. putProject(projectsDir, { id: "p4", owner: "bea", ownerId: "u-bea", jsonKey: "project-json/j4.json.gz", storedJsonBytes: 70, assets: [a1] }) -// A release keeps p1's previous project.json. void releaseDocs.insertOne({_id: "p1:r1", projectId: "p1", jsonKey: "project-json/j0.json.gz", jsonBytes: 450}) backpacks["u-ana"] = [{bodyKey: "backpack/x/body.png", bodyBytes: 800, thumbnailKey: "backpack/x/thumbnail.png", thumbnailBytes: 90}] @@ -320,8 +311,6 @@ testing.equal(report.accounts["u-ana"], {projects: 8950, assets: 21300, backpack testing.equal(report.accounts["u-bea"], {projects: 70, assets: 1000}, "and Bea's, with her own copy of the shared asset") testing.assert(!report.accounts.assertElse(object, {}).contains("u-cy"), "someone who stores nothing isn't in it") -// ---------- Settling sets the ledger to what's stored ---------- - stub.max = 100000 storageSettle("u-ana", noStorageHold()) testing.equal(countedFor("u-ana"), {projects: 8950, assets: 21300, backpack: 800}, "settling counts what she stores") @@ -330,8 +319,6 @@ number before = stub.calls.toNum() storageSettle("u-ana", noStorageHold()) testing.equal(stub.calls.toNum() - before, 1, "settling again only asks what's counted") -// ---------- A save holds what it adds ---------- - object grown = projectStorageSummary(readFile(projectsDir ++ "p1.json").data.assertElse(object, {}), assetIndex.assets) grown.json = {"incoming:p1": 600} grown.assets = [a1, a2, a3] @@ -342,7 +329,6 @@ testing.equal(hold.reservations.len, 1, "only the kind that grows") testing.equal(hold.reservations[1].kind, "projects", "its project.json is new; the asset is already counted from the trash") testing.equal(stubReservations[hold.reservations[1].id.toStr()].held, 600, "the ledger holds only the growth") -// The save lands: p1's project.json is replaced. putProject(projectsDir, { id: "p1", owner: "ana", ownerId: "u-ana", jsonKey: "project-json/j2.json.gz", storedJsonBytes: 600, @@ -353,8 +339,6 @@ storageSettle("u-ana", hold) testing.equal(heldFor("u-ana"), [], "settling commits what was held") testing.equal(countedFor("u-ana").projects, 9550, "at what was really stored: the remix still keeps the old project.json") -// ---------- Full ---------- - stub.max = stubUsed("u-ana") + 100 object bigger = projectStorageSummary(readFile(projectsDir ++ "p1.json").data.assertElse(object, {}), assetIndex.assets) bigger.history = {"incoming:p1": 4500} @@ -374,8 +358,6 @@ smaller.history = {"incoming:p1": 10} testing.assert(storageHold("u-ana", {projects: {p1: smaller}}, true).ok, "over her allowance, she can still make a project smaller") testing.assert(storageHold("u-ana", {}, true).ok, "or save without adding anything") -// ---------- Deleting frees ---------- - stub.max = 100000 void fs.remove(trashDir ++ "p3.json") storageSettle("u-ana", noStorageHold()) @@ -385,8 +367,6 @@ backpacks["u-ana"] = [] storageSettle("u-ana", noStorageHold()) testing.assert(!countedFor("u-ana").contains("backpack"), "an empty backpack stops counting") -// ---------- Uploads ahead of a save ---------- - string d1 = "d" * 32 ++ ".png" string e1 = "e" * 32 ++ ".png" object ahead = pendingAssetHold("u-ana", "ana", d1, 5000, true) @@ -403,16 +383,12 @@ testing.assert(!stubReservations.contains("pending-" ++ d1), "the save that uses testing.equal(pendingUploadEvents("ana").len, 0) stub.max = 100000 -// ---------- Rotur can't be reached ---------- - stub.failing = true object down = storageHold("u-ana", {projects: {p9: grown}}, true) testing.equal(down, {ok: true, reservations: []}, "MistWarp saves anyway rather than lose work") storageSettle("u-ana", noStorageHold()) stub.failing = false -// ---------- Everyone, every six hours ---------- - stubObjects[stubKey("u-cy", "projects")] = 999 putProject(projectsDir, {id: "p5", owner: "ana", ownerId: "u-ana", jsonKey: "project-json/j5.json.gz", storedJsonBytes: 40, assets: []}) number settled = settleEveryoneOnce() @@ -422,8 +398,6 @@ testing.equal(countedFor("u-bea"), {projects: 70, assets: 1000}) testing.equal(countedFor("u-cy"), {}, "what's counted for someone who stores nothing is freed") testing.equal(settleEveryoneOnce(), 0, "and once in line, nothing changes") -// ---------- Students and deleted accounts ---------- - before = stub.calls.toNum() storageSettle("mwc_student", noStorageHold()) testing.assert(storageHold("mwc_student", {}, true).ok, "a student's storage comes from their class") diff --git a/src/storage/storage-usage.osl b/src/storage/storage-usage.osl index 117d646..5368252 100644 --- a/src/storage/storage-usage.osl +++ b/src/storage/storage-usage.osl @@ -68,8 +68,6 @@ def backfillTrashedProjectHistoryBytes(string file, object inventory) boolean ( return writeFile(trashDir ++ file, project) ) -// What R2 holds, by key, with sizes: { ok, objects } (none when blobs are -// only on this machine), or { ok: false, error }. def storageInventory() object ( if r2Local ( return {ok: true, objects: {}} @@ -85,8 +83,6 @@ def storageInventory() object ( return {ok: true, objects: (loaded.data.assertElse(object, {}).objects ?? {}).assertElse(object, {})} ) -// Projects saved before storage limits have no recorded history size. -// Measure them once from local copies or the R2 inventory. def backfillProjectHistoryBytes() ( string migrationId = "migration:project-history-bytes-v1" if (diagnosticDocs.findById(migrationId) ?? {}).len > 0 ( @@ -121,9 +117,6 @@ def backfillProjectHistoryBytes() ( log "Storage size backfill complete" ) -// Releases made before their snapshot's size was recorded, which counts -// against the owner's Rotur storage. Measured once from local copies or the -// R2 inventory. def backfillReleaseJsonBytes() boolean ( string migrationId = "migration:release-json-bytes-v1" if (diagnosticDocs.findById(migrationId) ?? {}).len > 0 ( diff --git a/src/storage/storage.osl b/src/storage/storage.osl index 33c1d1e..6b2abc9 100644 --- a/src/storage/storage.osl +++ b/src/storage/storage.osl @@ -187,8 +187,6 @@ def indexEntryFromProject(object project) object ( if project.contains("vanitySlug") and project.vanitySlug != null ( vanitySlug = project.vanitySlug.toStr() ) - // vanityClaim holds the URL while it's lapsed. vanitySlug is what lists link - // to, so it's empty until the owner's plan includes vanity URLs again. return { id: project.id, owner: project.owner, @@ -272,7 +270,6 @@ def saveProject(object project) boolean ( if !isSafePathPart(id) ( return false ) - // Keep the inactivity scan and eviction consistent with published edits. sync.lock("r2-upload-queue") defer sync.unlock("r2-upload-queue") boolean saved = writeFile(projectsDir ++ id ++ ".json", project)