From 7d399ff29a3e6ddf525afbf8789d810efbf6bc42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Mon, 19 Oct 2026 20:18:23 +0000 Subject: [PATCH 1/3] feat(cmd): the commands that change, sum and check files, as rclone New commands, with the output, the log and the exit codes of rclone: - mkdir, delete (with --rmdirs), deletefile, purge, rmdir and rmdirs (with --leave-root). - copy and copyto, move and moveto: a file goes when the target has no file with its name, or another size, or another time in the precision of the two remotes. The flags --size-only, --ignore-times, --ignore-existing, --create-empty-src-dirs, --delete-empty-src-dirs, --transfers and --dry-run work. A move is a copy, then the delete of each source file. - size (with --json), tree (with the flags of the tree of rclone, and its quirks: no names with "." at the start, and no count of the folders with no file), touch, listremotes (with --long and --json), version and cryptdecode (with --reverse). - md5sum, sha1sum and hashsum: MD5, SHA-1, SHA-256, SHA-512 and CRC-32, with --base64, --output-file and --checkfile. The backends give no hash: thus rfetch reads each file, as --download of rclone. - check, checksum and cryptcheck: the lists of --combined, --match, --differ, --missing-on-src, --missing-on-dst and --error. check compares the sizes, as rclone with no common hash, and the bytes with --download. copy now has the semantics of rclone: the contents of a source folder, or a source file, go in the target folder. The old copy of one file to a name is copyto. The backends cannot set the time of a file yet: thus touch of a file that exists gives the error of rclone, and a second copy sends the files again, unless --size-only. The shared code is in cmd/operations.js: stat, walk, copyFile, remove, mkdir, the log on the standard error, and OperationError. lsjson gives its walk to the other commands with listItems. A command with an output and an error gives the error in statusText. The command line gives the exit code 4 for "object not found". cmd/rclone_test.js runs each command with rfetch and with rclone on the same local folders, and compares the outputs, the exit codes and the files after the command. --- CONTRIBUTING.md | 20 ++ README.md | 20 +- backend/crypt/main.js | 19 ++ cmd/check/main.js | 431 +++++++++++++++++++++++++++++++++++ cmd/check/main_test.js | 115 ++++++++++ cmd/copy/main.js | 275 ++++++++++++++++++++-- cmd/copy/main_test.js | 90 +++++++- cmd/cryptdecode/main.js | 50 ++++ cmd/cryptdecode/main_test.js | 37 +++ cmd/delete/main.js | 258 +++++++++++++++++++++ cmd/delete/main_test.js | 104 +++++++++ cmd/flags.js | 92 +++++++- cmd/folders.js | 40 ++++ cmd/hashsum/main.js | 229 +++++++++++++++++++ cmd/hashsum/main_test.js | 61 +++++ cmd/listremotes/main.js | 63 +++++ cmd/listremotes/main_test.js | 42 ++++ cmd/lsjson/main.js | 110 +++++---- cmd/main.js | 13 +- cmd/mkdir/main.js | 22 ++ cmd/mkdir/main_test.js | 16 ++ cmd/move/main.js | 45 ++++ cmd/move/main_test.js | 42 ++++ cmd/operations.js | 396 ++++++++++++++++++++++++++++++++ cmd/rclone_test.js | 259 +++++++++++++++++++++ cmd/size/main.js | 75 ++++++ cmd/size/main_test.js | 40 ++++ cmd/testing.js | 65 ++++++ cmd/touch/main.js | 104 +++++++++ cmd/touch/main_test.js | 36 +++ cmd/tree/main.js | 325 ++++++++++++++++++++++++++ cmd/tree/main_test.js | 75 ++++++ cmd/version/main.js | 23 ++ cmd/version/main_test.js | 11 + main.js | 19 +- main_test.js | 9 +- 36 files changed, 3542 insertions(+), 89 deletions(-) create mode 100644 cmd/check/main.js create mode 100644 cmd/check/main_test.js create mode 100644 cmd/cryptdecode/main.js create mode 100644 cmd/cryptdecode/main_test.js create mode 100644 cmd/delete/main.js create mode 100644 cmd/delete/main_test.js create mode 100644 cmd/hashsum/main.js create mode 100644 cmd/hashsum/main_test.js create mode 100644 cmd/listremotes/main.js create mode 100644 cmd/listremotes/main_test.js create mode 100644 cmd/mkdir/main.js create mode 100644 cmd/mkdir/main_test.js create mode 100644 cmd/move/main.js create mode 100644 cmd/move/main_test.js create mode 100644 cmd/operations.js create mode 100644 cmd/rclone_test.js create mode 100644 cmd/size/main.js create mode 100644 cmd/size/main_test.js create mode 100644 cmd/testing.js create mode 100644 cmd/touch/main.js create mode 100644 cmd/touch/main_test.js create mode 100644 cmd/tree/main.js create mode 100644 cmd/tree/main_test.js create mode 100644 cmd/version/main.js create mode 100644 cmd/version/main_test.js diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9722349..916a527 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -287,6 +287,26 @@ the local time zone, and the size and the count with `sizeField` and `countField` of `@rfetch/lib/format`, in fields of the widths of rclone. `cmd/lsf/rclone_test.js` compares the lines with the lines of rclone. +The other commands use the operations of `cmd/operations.js`, as the package +`operations` of rclone: `stat` gives the information of a path, `walk` gives the +files and the folders in a folder, and `copyFile`, `remove` and `mkdir` change +the files. An operation that fails throws an `OperationError` with the status of +the answer, and `failure` gives the answer of the command for it. `log` writes +the errors and the notices of rclone on the standard error, for example +`2006/01/02 15:04:05 ERROR : a.txt: sizes differ`, and `--quiet` keeps only the +errors. A command that has an output and an error, for example +`check --combined -`, gives the output in the body and the error in +`statusText`: the command line writes the body on the standard output, and the +error on the standard error. + +The command line gives the exit codes of rclone: 3 for the status 404, 4 for the +status 404 with a message that ends with `object not found`, and 1 for another +status that is not 2xx. + +`cmd/rclone_test.js` runs each command with rfetch and with rclone on the same +local folders, and compares the outputs, the exit codes and the files after the +command. + ## Tests of a backend Each backend must pass the shared tests of `@rfetch/lib/testing/suite`. These diff --git a/README.md b/README.md index 2fd87a2..cf2fe10 100644 --- a/README.md +++ b/README.md @@ -63,16 +63,30 @@ remote. ## Commands +The commands give the output of rclone, and its exit codes: 3 for a folder that +does not exist, 4 for a file that does not exist, and 1 for another error. + - `backend` - `cat` +- `check`, `checksum` and `cryptcheck`: compare the files of two places, or the + files and a sum file. The backends give no hash: `check` compares the sizes, + and with `--download` the bytes. - `config` -- `copy` +- `copy` and `copyto`: copy the files that the target does not have, or that + differ by size or by time. - `copyurl` +- `cryptdecode` +- `delete`, `deletefile`, `purge`, `rmdir` and `rmdirs` +- `hashsum`, `md5sum` and `sha1sum`: MD5, SHA-1, SHA-256, SHA-512 and CRC-32. + rfetch reads each file to compute its hash. +- `listremotes` - `ls` - `lsd` - `lsf` - `lsjson` - `lsl` +- `mkdir` +- `move` and `moveto`: copy, then delete each source file. - `obscure` - `rcat` - `reveal` @@ -81,6 +95,10 @@ remote. - `serve webdav`: serve a remote over WebDAV, as `rclone serve webdav`. A client reads and writes, unless `--read-only`. With `--user` and `--pass`, a client must log in with them. +- `size` +- `touch`: make an empty file. The backends cannot set the time of a file yet. +- `tree` +- `version` ## Runtimes diff --git a/backend/crypt/main.js b/backend/crypt/main.js index 8ba83d6..8d9d798 100644 --- a/backend/crypt/main.js +++ b/backend/crypt/main.js @@ -547,6 +547,25 @@ export async function decode(options, ...args) { return new Response(decoded.join("")); } +/** + * The cipher of the names of a crypt remote, for the command `cryptdecode`: + * it makes the keys one time for all the names. + * + * ```js + * import { nameCipher } from "./main.js"; + * const names = await nameCipher({ password, password2 }); + * names.decrypt(encryptedName); + * ``` + * @param {Object} options + * @param {string} options.password The password, obscured. + * @param {string} [options.password2] The salt, obscured. + * @returns {Promise<{ encrypt: (path: string) => string, decrypt: (path: string) => string }>} + */ +export async function nameCipher(options) { + const { names } = await cipherOf(options.password, options.password2 ?? ""); + return names; +} + /** * Makes the keys from a password and a salt, as `Cipher.Key` of rclone: 32 * bytes for the data, 32 bytes for the names, and 16 bytes of tweak for the diff --git a/cmd/check/main.js b/cmd/check/main.js new file mode 100644 index 0000000..4d720d3 --- /dev/null +++ b/cmd/check/main.js @@ -0,0 +1,431 @@ +/** + * The checks of files, as `rclone check`, `rclone checksum` and `rclone + * cryptcheck`. Each problem is an error in the log, and the summary is in + * notices, as rclone. The lists of the flags `--combined`, `--match`, + * `--differ`, `--missing-on-src`, `--missing-on-dst` and `--error` go to the + * location of the flag, or to the output for `-`: + * + * - `= path`: the file is the same in the two places. + * - `- path`: the file is only in the destination. + * - `+ path`: the file is only in the source. + * - `* path`: the files differ. + * - `! path`: the check had an error. + */ +import { isTrue } from "@rfetch/lib/options"; + +import { fetch } from "../../fetch.js"; +import { hashFile, hashName } from "../hashsum/main.js"; +import { + each, + failure, + log, + nameOf, + numberOf, + OperationError, + remoteOf, + stat, + walk, + withFlags, +} from "../operations.js"; + +/** The flags of the lists, and the sign of each list in `--combined`. */ +const REPORTS = { + match: "=", + differ: "*", + missing_on_src: "-", + missing_on_dst: "+", + error: "!", +}; + +/** + * The results of a check. + */ +class Results { + differences = 0; + errors = 0; + matches = 0; + noHashes = 0; + missingOnDst = 0; + missingOnSrc = 0; + /** @type {Record} */ + lists = { + combined: [], + ...Object.fromEntries(Object.keys(REPORTS).map((k) => [k, []])), + }; + + /** + * Adds a path to a list, and to the combined list. + * @param {keyof typeof REPORTS} kind + * @param {string} path + */ + report(kind, path) { + this.lists[kind].push(path); + this.lists.combined.push(`${REPORTS[kind]} ${path}`); + } +} + +/** + * The files of a location: a file, or the files in a folder. + * @param {string} location + * @param {Object} flags + * @returns {Promise>} + * @throws {OperationError} With 404 when the location does not exist. + */ +async function filesOf(location, flags) { + const info = await stat(location, flags); + if (!info) throw new OperationError("directory not found", 404); + /** @type {Map} */ + const files = new Map(); + if (!info.isDir) { + files.set(nameOf(location), { location, size: info.size }); + return files; + } + for await (const item of await walk(location, flags)) { + if (!item.IsDir) { + files.set(item.Path, { location: item.location, size: item.Size }); + } + } + return files; +} + +/** + * Checks the files in the source and destination match, as `rclone check`. + * It compares the sizes. The backends of rfetch give no hash, thus as rclone + * with no common hash, the files with the same size match, and the hashes + * could not be checked. With `--download`, it reads the files and compares + * their bytes. With `--one-way`, it does not check the files that are only + * in the destination. + * + * ```js + * import { check } from "./main.js"; + * await check("source:path", "dest:path", { download: true }); + * ``` + * @param {string} source + * @param {string} target + * @param {Object} [flags] + * @returns {Promise} + */ +export async function check(source, target, flags = {}) { + return await compare(source, target, flags, isTrue(flags.download)); +} + +/** + * Checks the integrity of an encrypted remote, as `rclone cryptcheck`: the + * files of the source, and the files of the crypt remote. rfetch reads and + * decrypts each file of the crypt remote, and compares its bytes with the + * source file. + * @param {string} source + * @param {string} target A crypt remote. + * @param {Object} [flags] + * @returns {Promise} + */ +export async function cryptcheck(source, target, flags = {}) { + try { + const { type } = await remoteOf(target, flags); + if (type !== "crypt") { + throw new OperationError( + `${target} is not a crypt remote`, + 400, + ); + } + } catch (error) { + return failure(error); + } + return await compare(source, target, flags, true); +} + +/** + * Compares the files of two places. + * @param {string} source + * @param {string} target + * @param {Object} flags + * @param {boolean} download Compares the bytes of the files. + * @returns {Promise} + */ +async function compare(source, target, flags, download) { + try { + const [sources, targets] = await Promise.all([ + filesOf(source, flags), + filesOf(target, flags), + ]); + const results = new Results(); + const paths = [...new Set([...sources.keys(), ...targets.keys()])].sort(); + const both = []; + for (const path of paths) { + const src = sources.get(path); + const dst = targets.get(path); + if (src && !dst) { + log("ERROR", path, `file not in ${target}`, flags); + results.differences++; + results.errors++; + results.missingOnDst++; + results.report("missing_on_dst", path); + } else if (!src && dst) { + if (isTrue(flags.one_way)) continue; + log("ERROR", path, `file not in ${source}`, flags); + results.differences++; + results.errors++; + results.missingOnSrc++; + results.report("missing_on_src", path); + } else if (src && dst) { + both.push({ path, src, dst }); + } + } + await each( + both, + numberOf(flags.checkers, 8), + async ({ path, src, dst }) => { + if (src.size >= 0 && dst.size >= 0 && src.size !== dst.size) { + log("ERROR", path, "sizes differ", flags); + results.differences++; + results.errors++; + results.report("differ", path); + return; + } + if (!download) { + results.noHashes++; + results.matches++; + results.report("match", path); + return; + } + try { + if (await sameBytes(src.location, dst.location, flags)) { + results.matches++; + results.report("match", path); + } else { + log("ERROR", path, "contents differ", flags); + results.differences++; + results.errors++; + results.report("differ", path); + } + } catch (error) { + log("ERROR", path, `failed to download: ${error.message}`, flags); + results.errors++; + results.report("error", path); + } + }, + ); + return await summary(results, source, target, flags, "files"); + } catch (error) { + return failure(error); + } +} + +/** + * Tells if two files have the same bytes. + * @param {string} a + * @param {string} b + * @param {Object} flags + */ +async function sameBytes(a, b, flags) { + const [first, second] = await Promise.all([ + hashFile(a, "sha256", flags), + hashFile(b, "sha256", flags), + ]); + return first === second; +} + +/** + * Logs the summary of a check, writes its lists, and gives the answer: an + * error when there are differences. + * @param {Results} results + * @param {string} source + * @param {string} target + * @param {Object} flags + * @param {"files" | "hashes"} missing What the source does not have. + * @returns {Promise} + */ +async function summary(results, source, target, flags, missing) { + if (results.missingOnDst) { + log("NOTICE", target, `${results.missingOnDst} files missing`, flags); + } + if (results.missingOnSrc) { + // As rclone: a sum file is not a place of files. + log( + "NOTICE", + missing === "hashes" ? "" : source, + `${results.missingOnSrc} ${missing} missing`, + flags, + ); + } + log("NOTICE", target, `${results.differences} differences found`, flags); + if (results.errors) { + log("NOTICE", target, `${results.errors} errors while checking`, flags); + } + if (results.noHashes) { + log( + "NOTICE", + target, + `${results.noHashes} hashes could not be checked`, + flags, + ); + } + if (results.matches) { + log("NOTICE", target, `${results.matches} matching files`, flags); + } + + let output = ""; + for (const kind of ["combined", ...Object.keys(REPORTS)]) { + const where = flags[kind]; + if (typeof where !== "string") continue; + const text = results.lists[kind].map((line) => `${line}\n`).join(""); + if (where === "-") { + output += text; + continue; + } + const response = await fetch(where, { + method: "PUT", + headers: { "Content-Length": `${new TextEncoder().encode(text).length}` }, + body: text, + }); + await response.body?.cancel(); + if (!response.ok) { + throw new OperationError( + `failed to write ${where}: ${response.status}`, + response.status, + ); + } + } + // The lists go to the output, and the error to the error output. + if (results.differences) { + return new Response(output, { + status: 500, + statusText: `${results.differences} differences found`, + }); + } + if (results.errors) { + return new Response(output, { + status: 500, + statusText: `${results.errors} errors while checking`, + }); + } + return new Response(output); +} + +/** + * Reads a sum file, as `ParseSumFile` of rclone: on each line, a hash, a + * space, a space or a `*`, and a path. + * @param {string} text + * @param {string} name The name of the sum file, for the log. + * @param {Object} flags + * @returns {Map} + */ +export function parseSums(text, name, flags = {}) { + /** @type {Map} */ + const sums = new Map(); + let warnings = 0; + const lines = text.split(/\r?\n/); + lines.forEach((line, number) => { + if (line === "") return; + const match = line.match(/^([^ ]+) [ *](.+)$/); + if (!match) { + if (++warnings <= 3) { + log( + "NOTICE", + name, + `improperly formatted checksum line ${number}`, + flags, + ); + } + return; + } + const [, sum, file] = match; + if (sums.has(file)) { + if (++warnings <= 3) { + log("NOTICE", name, `duplicate file on checksum line ${number}`, flags); + } + return; + } + sums.set(file, sum.toLowerCase()); + }); + if (warnings > 3) { + log("NOTICE", name, `${warnings - 3} warning(s) suppressed...`, flags); + } + return sums; +} + +/** + * Checks the files in the destination against a SUM file, as `rclone + * checksum`. rfetch reads each file to compute its hash, as `--download` + * of rclone. + * + * ```js + * import { checksum } from "./main.js"; + * await checksum("md5", "/path/to/MD5SUMS", "remote:path"); + * ``` + * @param {string} type The type of the hash, for example `md5` or `SHA-1`. + * @param {string} sumfile The location of the sum file. + * @param {string} target + * @param {Object} [flags] + * @returns {Promise} + */ +export async function checksum(type, sumfile, target, flags = {}) { + try { + const name = hashName(type); + if (!name) { + throw new OperationError(`${type}: hash type is not supported`, 400); + } + const response = await fetch(withFlags(sumfile, flags)); + if (!response.ok) { + await response.body?.cancel(); + throw new OperationError( + `cannot open sum file: ${response.status} ${response.statusText}` + .trim(), + response.status === 404 ? 404 : 400, + ); + } + const sums = parseSums(await response.text(), sumfile, flags); + const files = await filesOf(target, flags); + const results = new Results(); + const base64 = isTrue(flags.base64); + await each( + [...files].sort(), + numberOf(flags.checkers, 8), + async ([path, file]) => { + const sum = sums.get(path); + if (sum === undefined) { + if (isTrue(flags.one_way)) return; + log("ERROR", path, "sum not found", flags); + results.differences++; + results.errors++; + results.missingOnSrc++; + results.report("missing_on_src", path); + return; + } + sums.delete(path); + let hash; + try { + hash = await hashFile(file.location, name, { ...flags, base64 }); + } catch (error) { + log( + "ERROR", + path, + `Failed to calculate hash: ${error.message}`, + flags, + ); + results.errors++; + results.report("error", path); + return; + } + if ((base64 ? hash : hash.toLowerCase()) === sum || hash === sum) { + results.matches++; + results.report("match", path); + } else { + log("ERROR", path, "files differ", flags); + results.differences++; + results.errors++; + results.report("differ", path); + } + }, + ); + for (const path of sums.keys()) { + log("ERROR", path, `file not in ${target}`, flags); + results.errors++; + results.missingOnDst++; + results.report("missing_on_dst", path); + } + return await summary(results, sumfile, target, flags, "hashes"); + } catch (error) { + return failure(error); + } +} diff --git a/cmd/check/main_test.js b/cmd/check/main_test.js new file mode 100644 index 0000000..ee6ca2a --- /dev/null +++ b/cmd/check/main_test.js @@ -0,0 +1,115 @@ +import { beforeEach, describe, test } from "node:test"; +import assert from "node:assert/strict"; + +import { obscure } from "@rfetch/lib/obscure"; +import { fetch } from "../../fetch.js"; +import { captureLog, putFiles } from "../testing.js"; +import { check, checksum, cryptcheck } from "./main.js"; + +const SOURCE = ":memory:/check/source/"; +const TARGET = ":memory:/check/target/"; + +describe("check, checksum and cryptcheck", () => { + beforeEach(async () => { + await (await fetch(":memory:/check/", { method: "DELETE" })).body?.cancel(); + await putFiles(SOURCE, { + "f.txt": "hello", + "a/g.txt": "world!", + "a/zero": "", + "big.txt": "1234", + }); + await putFiles(TARGET, { + "f.txt": "HELLO", + "a/zero": "", + "big.txt": "12345", + "extra": "x", + }); + }); + + test("check compares the sizes, as rclone with no common hash", async (t) => { + const lines = captureLog(t); + const response = await check(SOURCE, TARGET, { combined: "-" }); + assert.equal(response.status, 500); + assert.equal(response.statusText, "3 differences found"); + assert.equal( + await response.text(), + "+ a/g.txt\n- extra\n= a/zero\n* big.txt\n= f.txt\n", + ); + assert.deepEqual(lines, [ + `ERROR : a/g.txt: file not in ${TARGET}`, + `ERROR : extra: file not in ${SOURCE}`, + "ERROR : big.txt: sizes differ", + `NOTICE: ${TARGET}: 1 files missing`, + `NOTICE: ${SOURCE}: 1 files missing`, + `NOTICE: ${TARGET}: 3 differences found`, + `NOTICE: ${TARGET}: 3 errors while checking`, + `NOTICE: ${TARGET}: 2 hashes could not be checked`, + `NOTICE: ${TARGET}: 2 matching files`, + ]); + }); + + test("check --download compares the bytes", async (t) => { + const lines = captureLog(t); + const response = await check(SOURCE, TARGET, { + download: true, + one_way: true, + match: "-", + differ: "-", + }); + assert.equal(response.statusText, "3 differences found"); + assert.ok(lines.includes("ERROR : f.txt: contents differ")); + assert.ok(!lines.some((line) => line.includes("extra"))); + assert.equal(await response.text(), "a/zero\nbig.txt\nf.txt\n"); + }); + + test("check of the same files gives no error", async (t) => { + captureLog(t); + const response = await check(SOURCE, SOURCE, { download: true }); + assert.equal(response.status, 200); + }); + + test("checksum checks the files with a sum file", async (t) => { + const lines = captureLog(t); + await putFiles(":memory:/check/", { + "MD5SUMS": "5d41402abc4b2a76b9719d911017c592 f.txt\n" + + "00000000000000000000000000000000 *a/g.txt\n" + + "d41d8cd98f00b204e9800998ecf8427e gone.txt\n" + + "not a line\n", + }); + const response = await checksum("MD5", ":memory:/check/MD5SUMS", SOURCE); + assert.equal(response.statusText, "3 differences found"); + for ( + const line of [ + "NOTICE: :memory:/check/MD5SUMS: improperly formatted checksum line 3", + "ERROR : a/g.txt: files differ", + "ERROR : a/zero: sum not found", + "ERROR : big.txt: sum not found", + `ERROR : gone.txt: file not in ${SOURCE}`, + "NOTICE: 2 hashes missing", + `NOTICE: ${SOURCE}: 1 matching files`, + ] + ) { + assert.ok(lines.includes(line), `${line}\n${lines.join("\n")}`); + } + }); + + test("cryptcheck compares a crypt remote with its source", async (t) => { + captureLog(t); + const password = await (await obscure("cryptcheck")).text(); + const crypted = + `:crypt,remote=':memory:/check/crypted',password='${password}':`; + await putFiles(crypted, { "f.txt": "hello", "a/g.txt": "world!" }); + await putFiles(":memory:/check/plain/", { + "f.txt": "hello", + "a/g.txt": "world!", + }); + assert.equal( + (await cryptcheck(":memory:/check/plain/", crypted)).status, + 200, + ); + await putFiles(crypted, { "f.txt": "HELLO" }); + const differ = await cryptcheck(":memory:/check/plain/", crypted); + assert.equal(differ.statusText, "1 differences found"); + assert.equal((await cryptcheck(SOURCE, TARGET)).status, 400); + }); +}); diff --git a/cmd/copy/main.js b/cmd/copy/main.js index 834e261..79d53bd 100644 --- a/cmd/copy/main.js +++ b/cmd/copy/main.js @@ -1,31 +1,266 @@ -import { fetch } from "../../fetch.js"; /** - * Copy files from source to dest, skipping identical files. + * The copy of files, as `rclone copy` and `rclone copyto`. `move` and + * `moveto` use it too. * - * Copy the source to the destination. Does not transfer files that are - * identical on source and destination, testing by size and modification time - * or MD5SUM. Doesn't delete files from the destination. If you want to also - * delete files from destination, to make it match source, use the `sync` - * command instead. + * A file goes when the target has no file with its name, or when the two + * files differ, as rclone: another size, or another time. A time that a + * backend does not give is not compared. The flags of rclone: * - * Note that it is always the contents of the directory that is synced, not - * the directory itself. So when `source:path` is a directory, it's the - * contents of source:path that are copied, not the directory name and - * contents. + * - `--size-only`: compare only the sizes. + * - `--ignore-times`: copy each file, also the same files. + * - `--ignore-existing`: copy no file that the target has. + * - `--create-empty-src-dirs`: make the folders with no file too. + * - `--dry-run`: copy nothing, and tell what it would copy. + * - `--transfers N`: copy N files at the same time, 4 by default. * - * To copy single files, use the `copyto` command instead. - * - * If `dest:path` doesn't exist, it is created and the source:path contents - * go there. + * The backends do not keep the time of a source file yet: the time of a + * copy is the time of the copy. Thus a second copy sends the files again, + * unless `--size-only`. + */ +import { isTrue } from "@rfetch/lib/options"; + +import { + copyFile, + each, + failure, + folderOf, + join, + log, + mkdir, + nameOf, + numberOf, + OperationError, + remove, + stat, + walk, +} from "../operations.js"; + +/** + * Copy files from source to dest, skipping identical files, as `rclone + * copy`. The contents of the source folder go in the dest folder. A source + * file goes in the dest folder, with its name. * + * ```js + * import { copy } from "./main.js"; + * await copy("source:path", "dest:path"); + * ``` + * @param {string} source + * @param {string} target + * @param {Object} [flags] + * @returns {Promise} + */ +export async function copy(source, target, flags = {}) { + try { + await transfer(source, target, flags); + return new Response(null); + } catch (error) { + return failure(error); + } +} + +/** + * Copy files from source to dest, skipping identical files, as `rclone + * copyto`: a source file goes to the path of dest, with the name of dest. + * A source folder is as `copy`. * @param {string} source * @param {string} target * @param {Object} [flags] * @returns {Promise} */ -export async function copy(source, target, flags) { - /** @TODO: Handle copy folder */ - const params = new URLSearchParams(flags); - const { body } = await fetch(`${source}?${params}`); - return fetch(`${target}?${params}`, { method: "PUT", body }); +export async function copyto(source, target, flags = {}) { + try { + await transfer(source, target, flags, { to: true }); + return new Response(null); + } catch (error) { + return failure(error); + } +} + +/** + * A file of a transfer. + * @typedef {Object} File + * @property {string} path The path from the folder of the transfer. + * @property {string} location + * @property {number} size + * @property {string} modTime + */ + +/** + * Copies, or moves, a file or a folder. + * @param {string} source + * @param {string} target + * @param {Object} flags + * @param {{ move?: boolean, to?: boolean }} [mode] `move` deletes each source + * file after its copy. `to` puts a source file at the path of the target, + * not in the target folder. + * @throws {OperationError} + */ +export async function transfer(source, target, flags, mode = {}) { + const info = await stat(source, flags); + if (!info) throw new OperationError("directory not found", 404); + if (!info.isDir) { + const location = mode.to ? target : join(target, nameOf(source)); + if (location === source) { + throw new OperationError("can't copy or move a file onto itself", 400); + } + const existing = await stat(location, flags); + await transferFile( + { + path: nameOf(location), + location: source, + size: info.size, + modTime: info.modTime, + }, + location, + existing?.isDir ? null : existing, + flags, + mode, + ); + return; + } + + /** @type {File[]} */ + const files = []; + /** @type {string[]} */ + const folders = []; + for await (const item of await walk(source, flags)) { + if (item.IsDir) folders.push(item.Path); + else { + files.push({ + path: item.Path, + location: item.location, + size: item.Size, + modTime: item.ModTime, + }); + } + } + /** @type {Map} */ + const existing = new Map(); + try { + for await (const item of await walk(target, flags)) { + if (!item.IsDir) existing.set(item.Path, item); + } + } catch (error) { + // A target that does not exist gets all the files. + if (!(error instanceof OperationError && error.status === 404)) throw error; + } + + if (isTrue(flags.create_empty_src_dirs) && !isTrue(flags.dry_run)) { + await mkdir(target, flags); + for (const path of folders) await mkdir(join(target, path, true), flags); + } + + let errors = 0; + let last = ""; + await each(files, numberOf(flags.transfers, 4), async (file) => { + const other = existing.get(file.path); + try { + await transferFile( + file, + join(target, file.path), + other + ? { isDir: false, size: other.Size, modTime: other.ModTime } + : null, + flags, + mode, + ); + } catch (error) { + errors++; + last = error.message; + log( + "ERROR", + file.path, + `Failed to ${mode.move ? "move" : "copy"}: ${error.message}`, + flags, + ); + } + }); + if (errors) { + throw new OperationError( + `failed to ${ + mode.move ? "move" : "copy" + } ${errors} files: last error was: ${last}`, + ); + } + if ( + mode.move && isTrue(flags.delete_empty_src_dirs) && !isTrue(flags.dry_run) + ) { + await removeFolders(source, folders, flags); + } +} + +/** + * Copies, or moves, a file, unless the target has the same file. + * @param {File} file + * @param {string} target The location of the target file. + * @param {import("../operations.js").Info | null} existing The target file, + * or null. + * @param {Object} flags + * @param {{ move?: boolean }} mode + */ +async function transferFile(file, target, existing, flags, mode) { + const verb = mode.move ? "move" : "copy"; + const same = existing && (isTrue(flags.ignore_existing) || + (!isTrue(flags.ignore_times) && isSame(file, existing, flags))); + if (isTrue(flags.dry_run)) { + if (!same || mode.move) { + log( + "NOTICE", + file.path, + `Skipped ${verb} as --dry-run is set (size ${file.size})`, + flags, + ); + } + return; + } + if (!same) await copyFile(file.location, target, flags); + // As rclone: a move deletes the source, also when the target has the same + // file. + if (mode.move) await remove(file.location, flags); +} + +/** + * Tells if two files are the same, as `Equal` of rclone: the same size, and + * the same time in the precision of the two remotes. A time with no part of + * a second comes from a remote with a precision of 1 s, as WebDAV: then the + * times can differ by less than 1 s. Else they can differ by less than 1 ms. + * A time that a backend does not give is not compared. + * @param {{ size: number, modTime: string }} a + * @param {{ size: number, modTime: string }} b + * @param {Object} flags + */ +function isSame(a, b, flags) { + if (a.size >= 0 && b.size >= 0 && a.size !== b.size) return false; + if (isTrue(flags.size_only)) return true; + const first = Date.parse(a.modTime); + const second = Date.parse(b.modTime); + if (isNaN(first) || isNaN(second)) return true; + const fraction = /:\d\d\.\d/; + const window = fraction.test(a.modTime) && fraction.test(b.modTime) + ? 1 + : 1000; + return Math.abs(first - second) < window; +} + +/** + * Deletes the folders of a source that have no file after a move, from the + * deepest folder up. The folder of the source stays. + * @param {string} source + * @param {string[]} folders + * @param {Object} flags + */ +async function removeFolders(source, folders, flags) { + const deepest = folders.toSorted((a, b) => + b.split("/").length - a.split("/").length + ); + for (const path of deepest) { + const location = join(folderOf(source), path, true); + // A folder with a file that did not move stays. + let empty = true; + for await (const _ of await walk(location, flags, { maxDepth: 1 })) { + empty = false; + break; + } + if (empty) await remove(location, flags); + } } diff --git a/cmd/copy/main_test.js b/cmd/copy/main_test.js index cbfba36..683d33d 100644 --- a/cmd/copy/main_test.js +++ b/cmd/copy/main_test.js @@ -1,7 +1,89 @@ -import { test } from "node:test"; +import { beforeEach, describe, test } from "node:test"; import assert from "node:assert/strict"; -import { copy } from "./main.js"; -test("copy", () => { - assert(copy); +import { fetch } from "../../fetch.js"; +import { captureLog, contents, putFiles } from "../testing.js"; +import { copy, copyto } from "./main.js"; + +const SOURCE = ":memory:/copy/source/"; +const TARGET = ":memory:/copy/target/"; + +describe("copy and copyto", () => { + beforeEach(async () => { + await (await fetch(":memory:/copy/", { method: "DELETE" })).body?.cancel(); + await putFiles(SOURCE, { + "f.txt": "hello", + "a/g.txt": "world!", + "a/b/zero": "", + "n?m/#1.txt": "name", + }); + await (await fetch(`${SOURCE}empty/`, { method: "PUT" })).body?.cancel(); + }); + + test("copy puts the contents of a folder in the target", async () => { + assert.equal((await copy(SOURCE, TARGET)).status, 200); + assert.deepEqual(await contents(TARGET), { + files: { + "f.txt": "hello", + "a/g.txt": "world!", + "a/b/zero": "", + "n?m/#1.txt": "name", + }, + folders: ["a", "a/b", "n?m"], + }); + }); + + test("copy --create-empty-src-dirs makes the empty folders too", async () => { + await copy(SOURCE, TARGET, { create_empty_src_dirs: true }); + assert.ok((await contents(TARGET)).folders.includes("empty")); + }); + + test("copy of a file puts it in the target folder", async () => { + assert.equal((await copy(`${SOURCE}a/g.txt`, TARGET)).status, 200); + assert.deepEqual((await contents(TARGET)).files, { "g.txt": "world!" }); + }); + + test("copyto gives the file the name of the target", async () => { + assert.equal( + (await copyto(`${SOURCE}a/g.txt`, `${TARGET}x/new.txt`)).status, + 200, + ); + assert.deepEqual((await contents(TARGET)).files, { "x/new.txt": "world!" }); + const self = await copyto(`${SOURCE}f.txt`, `${SOURCE}f.txt`); + assert.equal(self.status, 400); + }); + + test("copy skips the same files, as rclone", async () => { + await putFiles(TARGET, { "f.txt": "HELLO", "a/g.txt": "other!!" }); + // Same size: with --size-only, the file is the same. + await copy(SOURCE, TARGET, { size_only: true }); + let files = (await contents(TARGET)).files; + assert.equal(files["f.txt"], "HELLO"); + assert.equal(files["a/g.txt"], "world!"); + + // The times differ: the file goes again. + await copy(SOURCE, TARGET); + files = (await contents(TARGET)).files; + assert.equal(files["f.txt"], "hello"); + + await putFiles(TARGET, { "f.txt": "HELLO" }); + await copy(SOURCE, TARGET, { ignore_existing: true }); + assert.equal((await contents(TARGET)).files["f.txt"], "HELLO"); + }); + + test("copy --dry-run copies nothing", async (t) => { + const lines = captureLog(t); + assert.equal((await copy(SOURCE, TARGET, { dry_run: true })).status, 200); + assert.equal(await contents(TARGET), null); + assert.ok( + lines.includes( + "NOTICE: f.txt: Skipped copy as --dry-run is set (size 5)", + ), + lines.join("\n"), + ); + }); + + test("copy of a source that does not exist gives 404", async () => { + assert.equal((await copy(`${SOURCE}missing`, TARGET)).status, 404); + }); }); diff --git a/cmd/cryptdecode/main.js b/cmd/cryptdecode/main.js new file mode 100644 index 0000000..aac8def --- /dev/null +++ b/cmd/cryptdecode/main.js @@ -0,0 +1,50 @@ +import { nameCipher } from "@rfetch/crypt"; +import { isTrue } from "@rfetch/lib/options"; + +import { failure, OperationError, remoteOf } from "../operations.js"; + +/** + * Cryptdecode returns unencrypted file names, as `rclone cryptdecode`: a + * line for each name, with the name, a tab and the decrypted name, or + * `Failed to decrypt`. With `--reverse`, it encrypts the names. + * + * ```js + * import { cryptdecode } from "./main.js"; + * await cryptdecode("encryptedremote:", "encryptedname1", {}); + * ``` + * @param {string} remote A crypt remote. + * @param {...(string | Object)} args The names, then the flags. + * @returns {Promise} + */ +export async function cryptdecode(remote, ...args) { + const flags = typeof args.at(-1) === "object" ? args.pop() : {}; + try { + const { type, options } = await remoteOf(remote, flags); + if (type !== "crypt") { + throw new OperationError( + 'the remote needs to be of type "crypt"', + 400, + ); + } + if (!options.password) throw new OperationError("Missing password", 400); + let names; + try { + names = await nameCipher(options); + } catch { + throw new OperationError("The password is not obscured", 400); + } + const reverse = isTrue(flags.reverse); + const lines = args.map((name) => { + try { + return reverse + ? `${name} \t ${names.encrypt(name)}` + : `${name} \t ${names.decrypt(name)}`; + } catch { + return `${name} \t Failed to ${reverse ? "encrypt" : "decrypt"}`; + } + }); + return new Response(lines.map((line) => `${line}\n`).join("")); + } catch (error) { + return failure(error); + } +} diff --git a/cmd/cryptdecode/main_test.js b/cmd/cryptdecode/main_test.js new file mode 100644 index 0000000..cbb8c99 --- /dev/null +++ b/cmd/cryptdecode/main_test.js @@ -0,0 +1,37 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { obscure } from "@rfetch/lib/obscure"; +import { fetch } from "../../fetch.js"; +import { contents, putFiles } from "../testing.js"; +import { cryptdecode } from "./main.js"; + +test("cryptdecode decrypts and encrypts the names of a crypt remote", async () => { + const password = await (await obscure("cryptdecode")).text(); + const remote = `:crypt,remote=':memory:/cryptdecode',password='${password}':`; + await putFiles(remote, { "a/g.txt": "world!" }); + const [encrypted] = Object.keys( + (await contents(":memory:/cryptdecode/")).files, + ); + const [folder, file] = encrypted.split("/"); + + assert.equal( + await (await cryptdecode(remote, folder, encrypted, "bad-name", {})).text(), + `${folder} \t a\n${encrypted} \t a/g.txt\nbad-name \t Failed to decrypt\n`, + ); + assert.equal( + await (await cryptdecode(remote, "a/g.txt", { reverse: true })).text(), + `a/g.txt \t ${folder}/${file}\n`, + ); + // The options can come in the flags of the command too. + const flagged = await cryptdecode(":crypt:", folder, { + crypt_remote: ":memory:/cryptdecode", + crypt_password: password, + }); + assert.equal(await flagged.text(), `${folder} \t a\n`); + + const plain = await cryptdecode(":memory:", "x", {}); + assert.equal(plain.status, 400); + await (await fetch(":memory:/cryptdecode/", { method: "DELETE" })).body + ?.cancel(); +}); diff --git a/cmd/delete/main.js b/cmd/delete/main.js new file mode 100644 index 0000000..27869c4 --- /dev/null +++ b/cmd/delete/main.js @@ -0,0 +1,258 @@ +/** + * The commands that delete, as rclone: `delete`, `deletefile`, `purge`, + * `rmdir` and `rmdirs`. With `--dry-run`, they delete nothing, and tell + * what they would delete. + */ +import { isTrue } from "@rfetch/lib/options"; + +import { fetch } from "../../fetch.js"; +import { + each, + failure, + folderOf, + log, + numberOf, + OperationError, + remove, + stat, + walk, + withFlags, +} from "../operations.js"; + +/** + * Remove the files in the path, as `rclone delete`. The folders stay, unless + * `--rmdirs`: then the folders with no file go too, but not the folder of the + * path. A path of a file deletes only that file. + * + * ```js + * import { delete as remove } from "./main.js"; + * await remove("remote:path", { rmdirs: true }); + * ``` + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +async function deleteFiles(location, flags = {}) { + try { + const info = await stat(location, flags); + if (!info) throw new OperationError("directory not found", 404); + if (!info.isDir) { + await deleteOne(location, location, info.size, flags); + return new Response(null); + } + const files = []; + for await (const item of await walk(location, flags)) { + if (!item.IsDir) files.push(item); + } + let errors = 0; + let last = ""; + await each(files, numberOf(flags.checkers, 8), async (item) => { + try { + await deleteOne(item.location, item.Path, item.Size, flags); + } catch (error) { + errors++; + last = error.message; + log("ERROR", item.Path, `Couldn't delete: ${error.message}`, flags); + } + }); + if (errors) { + throw new OperationError( + `failed to delete ${errors} files: last error was: ${last}`, + ); + } + if (isTrue(flags.rmdirs)) { + await removeEmptyFolders(location, flags, true); + } + return new Response(null); + } catch (error) { + return failure(error); + } +} + +export { deleteFiles as delete }; + +/** + * Deletes a file, or tells it with `--dry-run`. + * @param {string} location + * @param {string} name The name for the log. + * @param {number} size + * @param {Object} flags + */ +async function deleteOne(location, name, size, flags) { + if (isTrue(flags.dry_run)) { + log( + "NOTICE", + name, + `Skipped delete as --dry-run is set (size ${size})`, + flags, + ); + return; + } + await remove(location, flags); +} + +/** + * Remove a single file from remote, as `rclone deletefile`. A folder, or a + * path that does not exist, is an error. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function deletefile(location, flags = {}) { + try { + const info = await stat(location, flags); + if (!info || info.isDir) { + throw new OperationError( + `${location} is a directory or doesn't exist: object not found`, + 404, + ); + } + await deleteOne(location, location, info.size, flags); + return new Response(null); + } catch (error) { + return failure(error); + } +} + +/** + * Remove the path and all of its contents, as `rclone purge`. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function purge(location, flags = {}) { + try { + const info = await stat(location, flags); + if (!info) throw new OperationError("directory not found", 404); + if (!info.isDir) { + throw new OperationError(`${location} is a file not a directory`, 400); + } + if (isTrue(flags.dry_run)) { + log("NOTICE", location, "Skipped purge as --dry-run is set", flags); + } else { + await remove(folderOf(location), flags); + } + return new Response(null); + } catch (error) { + return failure(error); + } +} + +/** + * Remove the empty folder of the path, as `rclone rmdir`. A folder with a + * file or a folder in it is an error. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function rmdir(location, flags = {}) { + try { + const folder = folderOf(location); + const response = await fetch(withFlags(folder, flags)); + if (response.status === 404) { + await response.body?.cancel(); + throw new OperationError(`${location}: directory not found`, 404); + } + if (!response.ok) { + await response.body?.cancel(); + throw new OperationError( + `${location}: ${response.status} ${response.statusText}`.trim(), + response.status, + ); + } + await response.body?.cancel(); + for await (const _ of await walk(folder, flags, { maxDepth: 1 })) { + throw new OperationError(`remove ${location}: directory not empty`, 409); + } + if (isTrue(flags.dry_run)) { + log( + "NOTICE", + location, + "Skipped remove directory as --dry-run is set", + flags, + ); + } else { + await remove(folder, flags); + } + return new Response(null); + } catch (error) { + return failure(error); + } +} + +/** + * Remove the folders with no file in the path, as `rclone rmdirs`: a folder + * with only folders that have no file goes too. The folder of the path goes + * when it has no file, unless `--leave-root`. The backends delete a folder + * with all that is in it: thus a file that comes into such a folder during + * the command goes with the folder. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function rmdirs(location, flags = {}) { + try { + await removeEmptyFolders(location, flags, isTrue(flags.leave_root)); + return new Response(null); + } catch (error) { + return failure(error); + } +} + +/** + * Deletes the folders with no file in a folder. + * @param {string} location + * @param {Object} flags + * @param {boolean} leaveRoot When true, the folder of the location stays. + */ +async function removeEmptyFolders(location, flags, leaveRoot) { + /** @type {Map} The locations of the folders, by path. */ + const folders = new Map(); + /** The folders with a file in them, also below. */ + const full = new Set([""]); + let hasFile = false; + for await (const item of await walk(location, flags)) { + if (item.IsDir) { + folders.set(item.Path, item.location); + continue; + } + hasFile = true; + const names = item.Path.split("/"); + for (let i = 1; i < names.length; i++) { + full.add(names.slice(0, i).join("/")); + } + } + const dryRun = isTrue(flags.dry_run); + if (!hasFile && !leaveRoot) { + if (dryRun) { + log( + "NOTICE", + location, + "Skipped remove directory as --dry-run is set", + flags, + ); + } else { + await remove(folderOf(location), flags); + } + return; + } + // The highest folders with no file: the delete of a folder deletes the + // folders in it. + for (const [path, folder] of [...folders].sort()) { + if (full.has(path)) continue; + const parent = path.includes("/") + ? path.slice(0, path.lastIndexOf("/")) + : ""; + if (!full.has(parent)) continue; + if (dryRun) { + log( + "NOTICE", + path, + "Skipped remove directory as --dry-run is set", + flags, + ); + } else { + await remove(folder, flags); + } + } +} diff --git a/cmd/delete/main_test.js b/cmd/delete/main_test.js new file mode 100644 index 0000000..c40ac31 --- /dev/null +++ b/cmd/delete/main_test.js @@ -0,0 +1,104 @@ +import { beforeEach, describe, test } from "node:test"; +import assert from "node:assert/strict"; + +import { fetch } from "../../fetch.js"; +import { captureLog, contents, putFiles } from "../testing.js"; +import { delete as remove, deletefile, purge, rmdir, rmdirs } from "./main.js"; + +const ROOT = ":memory:/delete/"; + +describe("the commands that delete", () => { + beforeEach(async () => { + await (await fetch(ROOT, { method: "DELETE" })).body?.cancel(); + await putFiles(ROOT, { + "f.txt": "hello", + "a/g.txt": "world!", + "a/b/zero": "", + "a?b/#c.txt": "x", + }); + await (await fetch(`${ROOT}empty/inner/`, { method: "PUT" })).body + ?.cancel(); + }); + + test("delete removes the files, and keeps the folders", async () => { + assert.equal((await remove(ROOT)).status, 200); + assert.deepEqual(await contents(ROOT), { + files: {}, + folders: ["a", "a/b", "a?b", "empty", "empty/inner"], + }); + }); + + test("delete --rmdirs removes the folders too, but not the root", async () => { + assert.equal((await remove(ROOT, { rmdirs: true })).status, 200); + assert.deepEqual(await contents(ROOT), { files: {}, folders: [] }); + }); + + test("delete of a file removes only that file", async () => { + assert.equal((await remove(`${ROOT}a/g.txt`)).status, 200); + assert.deepEqual(Object.keys((await contents(ROOT)).files).sort(), [ + "a/b/zero", + "a?b/#c.txt", + "f.txt", + ]); + }); + + test("delete --dry-run removes nothing, and tells what", async (t) => { + const lines = captureLog(t); + assert.equal((await remove(ROOT, { dry_run: true })).status, 200); + assert.equal(Object.keys((await contents(ROOT)).files).length, 4); + assert.ok( + lines.includes( + "NOTICE: f.txt: Skipped delete as --dry-run is set (size 5)", + ), + lines.join("\n"), + ); + }); + + test("deletefile removes a file, and refuses a folder", async () => { + assert.equal((await deletefile(`${ROOT}f.txt`)).status, 200); + for (const path of ["a", "missing.txt"]) { + const response = await deletefile(`${ROOT}${path}`); + assert.equal(response.status, 404); + assert.match(await response.text(), /object not found$/); + } + }); + + test("purge removes a folder and all in it", async () => { + assert.equal((await purge(`${ROOT}a`)).status, 200); + assert.deepEqual((await contents(ROOT)).folders, [ + "a?b", + "empty", + "empty/inner", + ]); + assert.equal((await purge(`${ROOT}f.txt`)).status, 400); + assert.equal((await purge(`${ROOT}missing`)).status, 404); + }); + + test("rmdir removes only an empty folder", async () => { + const full = await rmdir(`${ROOT}a`); + assert.equal(full.status, 409); + assert.match(await full.text(), /directory not empty/); + assert.equal((await rmdir(`${ROOT}empty/inner`)).status, 200); + assert.equal((await rmdir(`${ROOT}missing`)).status, 404); + assert.deepEqual((await contents(ROOT)).folders, [ + "a", + "a/b", + "a?b", + "empty", + ]); + }); + + test("rmdirs removes the folders with no file", async () => { + await (await fetch(`${ROOT}a/b/c/`, { method: "PUT" })).body?.cancel(); + assert.equal((await rmdirs(ROOT)).status, 200); + assert.deepEqual((await contents(ROOT)).folders, ["a", "a/b", "a?b"]); + + // The root goes when it has no file, unless --leave-root. + await (await fetch(ROOT, { method: "DELETE" })).body?.cancel(); + await (await fetch(`${ROOT}x/y/`, { method: "PUT" })).body?.cancel(); + assert.equal((await rmdirs(ROOT, { leave_root: true })).status, 200); + assert.deepEqual(await contents(ROOT), { files: {}, folders: [] }); + assert.equal((await rmdirs(ROOT)).status, 200); + assert.equal(await contents(ROOT), null); + }); +}); diff --git a/cmd/flags.js b/cmd/flags.js index cda2dad..7edcb40 100644 --- a/cmd/flags.js +++ b/cmd/flags.js @@ -32,29 +32,47 @@ function variable(name) { export const FLAGS = { // `serve` "addr": { type: "string" }, - // `serve webdav` - "pass": { type: "string" }, - // `serve webdav` - "read-only": { type: "boolean" }, - // `serve webdav` - "user": { type: "string" }, + // `tree` + "all": { type: "boolean" }, // `copyurl` "auto-filename": { type: "boolean", short: "a" }, + // `hashsum`, `checksum` + "base64": { type: "boolean" }, + // The checks, the deletes and the sums + "checkers": { type: "string", default: variable("CHECKERS") || "8" }, + // `hashsum`, `md5sum`, `sha1sum` + "checkfile": { type: "string" }, + // `check` + "combined": { type: "string" }, + // `copy`, `move` + "create-empty-src-dirs": { type: "boolean" }, // `lsf` "csv": { type: "boolean" }, + // `move` + "delete-empty-src-dirs": { type: "boolean" }, + // `check` + "differ": { type: "string" }, // `lsf` "dir-slash": { type: "boolean", short: "d" }, - // `lsf` + // `tree` + "dirsfirst": { type: "boolean" }, + // `lsf`, `tree` "dirs-only": { type: "boolean" }, + // `check`, `checksum`, the sums + "download": { type: "boolean" }, "dry-run": { type: "boolean", short: "n", default: variable("DRY_RUN") === "true", }, + // `check` + "error": { type: "string" }, // `lsf` "files-only": { type: "boolean" }, // `lsf` "format": { type: "string", short: "F" }, + // `tree` + "full-path": { type: "boolean" }, "header": { type: "string", multiple: true }, "header-download": { type: "string", multiple: true }, // `copyurl` @@ -62,10 +80,44 @@ export const FLAGS = { "header-upload": { type: "string", multiple: true }, "help": { type: "boolean", short: "h" }, "human-readable": { type: "boolean" }, + // `copy`, `move` + "ignore-existing": { type: "boolean" }, + // `copy`, `move` + "ignore-times": { type: "boolean", short: "I" }, + // `size`, `listremotes` + "json": { type: "boolean" }, + // `rmdirs` + "leave-root": { type: "boolean" }, + // `tree` + "level": { type: "string" }, + // `touch` + "localtime": { type: "boolean" }, + // `listremotes` + "long": { type: "boolean" }, + // `check` + "match": { type: "string" }, // The lists "max-depth": { type: "string" }, + // `check` + "missing-on-dst": { type: "string" }, + // `check` + "missing-on-src": { type: "string" }, + // `tree` + "modtime": { type: "boolean" }, // `copyurl` "no-clobber": { type: "boolean" }, + // `touch` + "no-create": { type: "boolean", short: "C" }, + // `tree` + "noindent": { type: "boolean" }, + // `tree` + "noreport": { type: "boolean" }, + // `check`, `checksum` + "one-way": { type: "boolean" }, + // `hashsum` + "output-file": { type: "string" }, + // `serve webdav` + "pass": { type: "string" }, // `copyurl` "print-filename": { type: "boolean", short: "p" }, "progress": { @@ -73,13 +125,37 @@ export const FLAGS = { short: "P", default: variable("PROGRESS") === "true", }, - // The lists + // Only the errors in the log. + "quiet": { type: "boolean", short: "q" }, + // `tree` + "quote": { type: "boolean" }, + // `serve webdav` + "read-only": { type: "boolean" }, + // The lists, `touch` "recursive": { type: "boolean", short: "R" }, + // `cryptdecode` + "reverse": { type: "boolean" }, + // `delete` + "rmdirs": { type: "boolean" }, // `lsf` "separator": { type: "string", short: "s" }, + // `tree` + "size": { type: "boolean" }, + // `copy`, `move`, `check` + "size-only": { type: "boolean" }, + // `tree` + "sort": { type: "string" }, + // `tree` + "sort-modtime": { type: "boolean" }, + // `tree` + "sort-reverse": { type: "boolean", short: "r" }, // `copyurl` "stdout": { type: "boolean" }, + // `touch` + "timestamp": { type: "string", short: "t" }, "transfers": { type: "string", default: variable("TRANSFERS") || "4" }, + // `serve webdav` + "user": { type: "string" }, }; /** diff --git a/cmd/folders.js b/cmd/folders.js index 371e3ff..ead6da6 100644 --- a/cmd/folders.js +++ b/cmd/folders.js @@ -5,19 +5,40 @@ export const FOLDERS = { backend: "backend", cat: "cat", + check: "check", + checksum: "check", config: "config", copy: "copy", + copyto: "copy", copyurl: "copyurl", + cryptcheck: "check", + cryptdecode: "cryptdecode", + delete: "delete", + deletefile: "delete", + hashsum: "hashsum", help: "help", + listremotes: "listremotes", ls: "ls", lsd: "lsd", lsf: "lsf", lsjson: "lsjson", lsl: "lsl", + md5sum: "hashsum", + mkdir: "mkdir", + move: "move", + moveto: "move", obscure: "obscure", + purge: "delete", rcat: "rcat", reveal: "obscure", + rmdir: "delete", + rmdirs: "delete", serve: "serve", + sha1sum: "hashsum", + size: "size", + touch: "touch", + tree: "tree", + version: "version", }; /** @@ -30,13 +51,32 @@ export const FOLDERS = { export const PATHS = { backend: [1], cat: [0], + check: [0, 1], + checksum: [1, 2], copy: [0, 1], + copyto: [0, 1], copyurl: [1], + cryptcheck: [0, 1], + cryptdecode: [0], + delete: [0], + deletefile: [0], + hashsum: [1], ls: [0], lsd: [0], lsf: [0], lsjson: [0], lsl: [0], + md5sum: [0], + mkdir: [0], + move: [0, 1], + moveto: [0, 1], + purge: [0], rcat: [0], + rmdir: [0], + rmdirs: [0], serve: [1], + sha1sum: [0], + size: [0], + touch: [0], + tree: [0], }; diff --git a/cmd/hashsum/main.js b/cmd/hashsum/main.js new file mode 100644 index 0000000..150d323 --- /dev/null +++ b/cmd/hashsum/main.js @@ -0,0 +1,229 @@ +/** + * The sums of the files, as `rclone hashsum`, `rclone md5sum` and `rclone + * sha1sum`. The backends of rfetch give no hash: thus the commands read each + * file and compute its hash, as `--download` of rclone. + */ +import { createHash } from "node:crypto"; +import { crc32 } from "@rfetch/lib/crc32"; +import { isTrue } from "@rfetch/lib/options"; + +import { fetch } from "../../fetch.js"; +import { + each, + failure, + nameOf, + numberOf, + OperationError, + stat, + walk, + withFlags, +} from "../operations.js"; + +/** + * The hashes of rfetch, by their names in rclone. The name of `node:crypto`, + * or null for CRC-32. + * @type {Record} + */ +const HASHES = { + md5: "md5", + sha1: "sha1", + crc32: null, + sha256: "sha256", + sha512: "sha512", +}; + +/** + * Gives the name of a hash in rclone, for a name of a person: for example + * `MD5`, `SHA-1` or `sha256`. + * @param {string} name + * @returns {string | null} null for a hash that rfetch does not know. + */ +export function hashName(name) { + const key = `${name}`.toLowerCase().replaceAll("-", ""); + return Object.hasOwn(HASHES, key) ? key : null; +} + +/** + * Computes the hash of a stream. + * @param {ReadableStream} stream + * @param {string} type A name of `hashName`. + * @param {boolean} [base64] Gives the hash in base64, not in hexadecimal. + * @returns {Promise} + */ +export async function hashStream(stream, type, base64 = false) { + /** @type {Uint8Array} */ + let bytes; + if (HASHES[type] === null) { + let crc = 0; + for await (const chunk of stream) crc = crc32(chunk, crc); + bytes = new Uint8Array(4); + new DataView(bytes.buffer).setUint32(0, crc); + } else { + const hash = createHash(HASHES[type]); + for await (const chunk of stream) hash.update(chunk); + bytes = hash.digest(); + } + return Buffer.from(bytes).toString(base64 ? "base64" : "hex"); +} + +/** + * Computes the hash of a file of a remote. + * @param {string} location + * @param {string} type + * @param {Object} flags With `base64`. + * @returns {Promise} + * @throws {OperationError} When the remote gives an error. + */ +export async function hashFile(location, type, flags = {}) { + const response = await fetch(withFlags(location, flags)); + if (!response.ok) { + await response.body?.cancel(); + throw new OperationError( + `${response.status} ${response.statusText}`.trim(), + response.status, + ); + } + return await hashStream( + response.body ?? new Blob([]).stream(), + type, + isTrue(flags.base64), + ); +} + +/** + * Produces a hashsum file for all the objects in the path, as `rclone + * hashsum`: a line for each file, with its hash, two spaces and its path. + * With no hash type, it gives the hashes that rfetch knows. + * + * The flags of rclone: `--base64`, `--output-file` (the location of a file + * for the lines), `--checkfile` (the location of a sum file to check the + * files with, as `checksum` with `--combined -`) and `--download` (always, + * in rfetch). + * + * ```js + * import { hashsum } from "./main.js"; + * await hashsum("SHA-256", "remote:path"); + * ``` + * @param {string | Object} [type] + * @param {string | Object} [location] + * @param {Object} [flags] + * @returns {Promise} + */ +export async function hashsum(type, location, flags = {}) { + if (typeof type !== "string") { + return new Response( + `Supported hashes are:\n${ + Object.keys(HASHES).map((name) => `- ${name}\n`).join("") + }`, + ); + } + if (typeof location !== "string") { + return new Response("Command hashsum needs 2 arguments", { status: 400 }); + } + const name = hashName(type); + if (!name) { + return new Response( + `${type}: hash type is not supported, use one of: ${ + Object.keys(HASHES).join(", ") + }`, + { status: 400 }, + ); + } + return await sums(name, location, flags); +} + +/** + * Produces an md5sum file for all the objects in the path, as `rclone + * md5sum`. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export function md5sum(location, flags = {}) { + return sums("md5", location, flags); +} + +/** + * Produces an sha1sum file for all the objects in the path, as `rclone + * sha1sum`. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export function sha1sum(location, flags = {}) { + return sums("sha1", location, flags); +} + +/** + * Gives the sums of the files of a path. + * @param {string} type + * @param {string} location + * @param {Object} flags + * @returns {Promise} + */ +async function sums(type, location, flags) { + if (typeof flags.checkfile === "string") { + // As rclone: the list of the check goes to the output. + const { checksum } = await import("../check/main.js"); + return checksum(type, flags.checkfile, location, { + combined: "-", + ...flags, + }); + } + try { + const info = await stat(location, flags); + if (!info) throw new OperationError("directory not found", 404); + /** @type {{ path: string, location: string }[]} */ + const files = []; + if (info.isDir) { + for await (const item of await walk(location, flags)) { + if (!item.IsDir) { + files.push({ path: item.Path, location: item.location }); + } + } + } else { + files.push({ path: nameOf(location), location }); + } + let text = ""; + let errors = 0; + let last = ""; + await each(files, numberOf(flags.checkers, 8), async (file) => { + try { + // The hash first: else `text` is the text before the wait. + const hash = await hashFile(file.location, type, flags); + text += `${hash} ${file.path}\n`; + } catch (error) { + errors++; + last = error.message; + // As rclone: the line of a file that fails has the error. + text += `${"ERROR".padEnd(type === "crc32" ? 8 : 32)} ${file.path}\n`; + } + }); + if (typeof flags.output_file === "string") { + const response = await fetch(flags.output_file, { + method: "PUT", + headers: { + "Content-Length": `${new TextEncoder().encode(text).length}`, + }, + body: text, + }); + await response.body?.cancel(); + if (!response.ok) { + throw new OperationError( + `failed to write the output file: ${response.status}`, + response.status, + ); + } + text = ""; + } + if (errors) { + return new Response(text, { + status: 500, + statusText: `failed to hash ${errors} files: last error was: ${last}`, + }); + } + return new Response(text); + } catch (error) { + return failure(error); + } +} diff --git a/cmd/hashsum/main_test.js b/cmd/hashsum/main_test.js new file mode 100644 index 0000000..2b3876f --- /dev/null +++ b/cmd/hashsum/main_test.js @@ -0,0 +1,61 @@ +import { before, test } from "node:test"; +import assert from "node:assert/strict"; + +import { fetch } from "../../fetch.js"; +import { putFiles } from "../testing.js"; +import { hashsum, md5sum, sha1sum } from "./main.js"; + +const ROOT = ":memory:/hashsum/"; + +before(async () => { + await putFiles(ROOT, { "f.txt": "hello", "a/g.txt": "world!", "a/zero": "" }); +}); + +/** @param {Response} response */ +async function lines(response) { + assert.equal(response.status, 200); + return (await response.text()).split("\n").filter(Boolean).sort(); +} + +test("md5sum and sha1sum give the sums of the files, as rclone", async () => { + assert.deepEqual(await lines(await md5sum(ROOT)), [ + "08cf82251c975a5e9734699fadf5e9c0 a/g.txt", + "5d41402abc4b2a76b9719d911017c592 f.txt", + "d41d8cd98f00b204e9800998ecf8427e a/zero", + ]); + assert.deepEqual(await lines(await sha1sum(`${ROOT}f.txt`)), [ + "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d f.txt", + ]); +}); + +test("hashsum gives the hashes of rclone", async () => { + assert.deepEqual(await lines(await hashsum("SHA-256", `${ROOT}f.txt`)), [ + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 f.txt", + ]); + assert.deepEqual(await lines(await hashsum("crc32", ROOT)), [ + "00000000 a/zero", + "3610a686 f.txt", + "718498e8 a/g.txt", + ]); + assert.deepEqual( + await lines(await hashsum("md5", `${ROOT}f.txt`, { base64: true })), + ["XUFAKrxLKna5cZ2REBfFkg== f.txt"], + ); + assert.match( + await (await hashsum({})).text(), + /^Supported hashes are:\n- md5\n/, + ); + assert.equal((await hashsum("whirlpool", ROOT)).status, 400); +}); + +test("hashsum writes an output file, and checks a sum file", async () => { + const out = `${ROOT}sums/MD5SUMS`; + assert.equal( + await (await md5sum(`${ROOT}a/`, { output_file: out })).text(), + "", + ); + const sums = await (await fetch(out)).text(); + assert.match(sums, /^08cf82251c975a5e9734699fadf5e9c0 {2}g\.txt$/m); + const checked = await md5sum(`${ROOT}a/`, { checkfile: out, quiet: true }); + assert.equal(checked.status, 200); +}); diff --git a/cmd/listremotes/main.js b/cmd/listremotes/main.js new file mode 100644 index 0000000..4e21830 --- /dev/null +++ b/cmd/listremotes/main.js @@ -0,0 +1,63 @@ +import { env } from "node:process"; +import { isTrue } from "@rfetch/lib/options"; + +import { config } from "../config/main.js"; + +/** + * List all the remotes in the config file and defined in environment + * variables, as `rclone listremotes`: one `name:` on each line. With + * `--long`, the type too. With `--json`, the name, the type, the source + * (`file` or `environment`) and the description of each remote. + * + * A remote of the variables has a variable `RFETCH_CONFIG_NAME_TYPE`, or + * `RCLONE_CONFIG_NAME_TYPE`: its name is in lower case. + * @param {string | Object} [filter] Only the remotes with this text in their + * names, as the argument of rclone. + * @param {Object} [flags] + * @returns {Promise} + */ +export async function listremotes(filter, flags = {}) { + if (typeof filter !== "string") [filter, flags] = ["", filter ?? {}]; + const response = await config("dump"); + /** @type {Record>} */ + const sections = response.ok ? await response.json() : {}; + /** @type {{ name: string, type: string, source: string, description: string }[]} */ + const remotes = []; + for (const [name, options] of Object.entries(sections)) { + remotes.push({ + name, + type: options.type ?? "", + source: "file", + description: options.description ?? "", + }); + } + for (const [key, value] of Object.entries(env)) { + const match = key.match(/^(?:RFETCH|RCLONE)_CONFIG_(.+)_TYPE$/); + if (!match || typeof value !== "string") continue; + const name = match[1].toLowerCase(); + const known = remotes.find((remote) => remote.name === name); + if (known) { + if (known.source === "environment") known.type = value; + continue; + } + remotes.push({ name, type: value, source: "environment", description: "" }); + } + const shown = remotes.filter((remote) => remote.name.includes(filter)); + + if (isTrue(flags.json)) { + const lines = shown.map((remote) => JSON.stringify(remote)); + return new Response( + `[\n${lines.join(",\n")}${lines.length ? "\n" : ""}]\n`, + ); + } + if (isTrue(flags.long)) { + const names = Math.max(0, ...shown.map(({ name }) => name.length + 1)); + const types = Math.max(0, ...shown.map(({ type }) => type.length)); + return new Response( + shown.map(({ name, type, description }) => + `${`${name}:`.padEnd(names)} ${type.padEnd(types)} ${description}\n` + ).join(""), + ); + } + return new Response(shown.map(({ name }) => `${name}:\n`).join("")); +} diff --git a/cmd/listremotes/main_test.js b/cmd/listremotes/main_test.js new file mode 100644 index 0000000..96a3dce --- /dev/null +++ b/cmd/listremotes/main_test.js @@ -0,0 +1,42 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import process from "node:process"; + +import { listremotes } from "./main.js"; + +test("listremotes gives the remotes of the config file and of the variables", async (t) => { + const folder = await mkdtemp(join(tmpdir(), "rfetch-listremotes-")); + const saved = { ...process.env }; + t.after(async () => { + for (const key of Object.keys(process.env)) { + if (!(key in saved)) delete process.env[key]; + } + Object.assign(process.env, saved); + await rm(folder, { recursive: true, force: true }); + }); + await mkdir(join(folder, "rfetch")); + await writeFile( + join(folder, "rfetch", "rfetch.conf"), + "[mem]\ntype = memory\n\n[enc]\ntype = crypt\ndescription = mine\n", + ); + process.env.XDG_CONFIG_HOME = folder; + process.env.RFETCH_CONFIG_FROMENV_TYPE = "s3"; + + assert.equal(await (await listremotes({})).text(), "mem:\nenc:\nfromenv:\n"); + assert.equal( + await (await listremotes({ long: true })).text(), + "mem: memory \nenc: crypt mine\nfromenv: s3 \n", + ); + assert.deepEqual( + JSON.parse(await (await listremotes({ json: true })).text()), + [ + { name: "mem", type: "memory", source: "file", description: "" }, + { name: "enc", type: "crypt", source: "file", description: "mine" }, + { name: "fromenv", type: "s3", source: "environment", description: "" }, + ], + ); + assert.equal(await (await listremotes("en", {})).text(), "enc:\nfromenv:\n"); +}); diff --git a/cmd/lsjson/main.js b/cmd/lsjson/main.js index e3de9b0..d04239e 100644 --- a/cmd/lsjson/main.js +++ b/cmd/lsjson/main.js @@ -60,6 +60,65 @@ const FOLDER = new URL("http://remote/folder/"); * @returns {Promise} */ export async function lsjson(location, flags = {}, init = {}) { + const items = await listItems(location, flags, init); + if (items instanceof Response) return items; + /** + * The JSON of the item before, with no comma: only the last item has no + * comma after it. + * @type {string | null} + */ + let previous = null; + + /** + * A stream of JSON-encoded file objects, one per chunk and line, in an + * array. + * @type {ReadableStream} + */ + const body = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode("[\n")); + }, + async pull(controller) { + for (;;) { + const { done, value } = await items.next(); + if (done) { + if (previous !== null) { + controller.enqueue(encoder.encode(`${previous}\n`)); + } + controller.enqueue(encoder.encode("]\n")); + controller.close(); + return; + } + const line = JSON.stringify(value); + if (previous !== null) { + controller.enqueue(encoder.encode(`${previous},\n`)); + previous = line; + return; + } + previous = line; + } + }, + async cancel() { + await items.return(undefined); + }, + }); + + return new Response(body, { + headers: { + "Content-Type": "application/json", + }, + }); +} + +/** + * Gives the items of a path, as `lsjson` lists them, or the answer of the + * error. The other commands use it to walk a folder. + * @param {string} location + * @param {Object} [flags={}] As the flags of `lsjson`. + * @param {RequestInit} [init] + * @returns {Promise | Response>} + */ +export async function listItems(location, flags = {}, init = {}) { // A path that ends with `.` or `..` is a folder, as in rclone. if (/(^|[:/])\.\.?$/.test(location)) location = `${location}/`; // Only the path in the remote has folders: a value of the connection @@ -103,7 +162,7 @@ export async function lsjson(location, flags = {}, init = {}) { return new Response("directory not found", { status: 404 }); } if (named.every(({ IsDir }) => IsDir)) { - return lsjson(`${location}/`, flags, init); + return listItems(`${location}/`, flags, init); } } @@ -120,54 +179,7 @@ export async function lsjson(location, flags = {}, init = {}) { : readListing(response, FOLDER).then((list) => folderOf(list, options)); // The error comes when the stream gets to the folder. listed.catch(() => {}); - const items = walk(listed, ""); - - /** - * The JSON of the item before, with no comma: only the last item has no - * comma after it. - * @type {string | null} - */ - let previous = null; - - /** - * A stream of JSON-encoded file objects, one per chunk and line, in an - * array. - * @type {ReadableStream} - */ - const body = new ReadableStream({ - start(controller) { - controller.enqueue(encoder.encode("[\n")); - }, - async pull(controller) { - for (;;) { - const { done, value } = await items.next(); - if (done) { - if (previous !== null) { - controller.enqueue(encoder.encode(`${previous}\n`)); - } - controller.enqueue(encoder.encode("]\n")); - controller.close(); - return; - } - const line = JSON.stringify(value); - if (previous !== null) { - controller.enqueue(encoder.encode(`${previous},\n`)); - previous = line; - return; - } - previous = line; - } - }, - async cancel() { - await items.return(undefined); - }, - }); - - return new Response(body, { - headers: { - "Content-Type": "application/json", - }, - }); + return walk(listed, ""); } /** diff --git a/cmd/main.js b/cmd/main.js index 6c7454f..2d7e65d 100644 --- a/cmd/main.js +++ b/cmd/main.js @@ -8,7 +8,18 @@ export { lsl } from "./lsl/main.js"; export { lsd } from "./lsd/main.js"; export { cat } from "./cat/main.js"; export { rcat } from "./rcat/main.js"; -export { copy } from "./copy/main.js"; +export { copy, copyto } from "./copy/main.js"; +export { check, checksum, cryptcheck } from "./check/main.js"; +export { cryptdecode } from "./cryptdecode/main.js"; +export { delete, deletefile, purge, rmdir, rmdirs } from "./delete/main.js"; +export { hashsum, md5sum, sha1sum } from "./hashsum/main.js"; +export { listremotes } from "./listremotes/main.js"; +export { mkdir } from "./mkdir/main.js"; +export { move, moveto } from "./move/main.js"; +export { size } from "./size/main.js"; +export { touch } from "./touch/main.js"; +export { tree } from "./tree/main.js"; +export { version } from "./version/main.js"; export { copyurl } from "./copyurl/main.js"; export { obscure, reveal } from "./obscure/main.js"; export { serve } from "./serve/main.js"; diff --git a/cmd/mkdir/main.js b/cmd/mkdir/main.js new file mode 100644 index 0000000..7eefc44 --- /dev/null +++ b/cmd/mkdir/main.js @@ -0,0 +1,22 @@ +import { failure, mkdir as makeFolder } from "../operations.js"; + +/** + * Make the path if it doesn't already exist, as `rclone mkdir`. The folders + * above the path come too. + * + * ```js + * import { mkdir } from "./main.js"; + * await mkdir("remote:path/to/folder"); + * ``` + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function mkdir(location, flags = {}) { + try { + await makeFolder(location, flags); + return new Response(null); + } catch (error) { + return failure(error); + } +} diff --git a/cmd/mkdir/main_test.js b/cmd/mkdir/main_test.js new file mode 100644 index 0000000..de822cc --- /dev/null +++ b/cmd/mkdir/main_test.js @@ -0,0 +1,16 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { contents } from "../testing.js"; +import { mkdir } from "./main.js"; + +test("mkdir makes a folder and the folders above it", async () => { + assert.equal((await mkdir(":memory:/mkdir/a/b c/d")).status, 200); + assert.deepEqual((await contents(":memory:/mkdir/")).folders, [ + "a", + "a/b c", + "a/b c/d", + ]); + // A folder that exists is not an error. + assert.equal((await mkdir(":memory:/mkdir/a")).status, 200); +}); diff --git a/cmd/move/main.js b/cmd/move/main.js new file mode 100644 index 0000000..5fc0cdb --- /dev/null +++ b/cmd/move/main.js @@ -0,0 +1,45 @@ +import { failure } from "../operations.js"; +import { transfer } from "../copy/main.js"; + +/** + * Move files from source to dest, as `rclone move`: the copy of `copy`, + * then the delete of each source file. The folders of the source stay, + * unless `--delete-empty-src-dirs`. + * + * The backends cannot move a file yet: each file goes through rfetch. + * + * ```js + * import { move } from "./main.js"; + * await move("source:path", "dest:path"); + * ``` + * @param {string} source + * @param {string} target + * @param {Object} [flags] + * @returns {Promise} + */ +export async function move(source, target, flags = {}) { + try { + await transfer(source, target, flags, { move: true }); + return new Response(null); + } catch (error) { + return failure(error); + } +} + +/** + * Move file or directory from source to dest, as `rclone moveto`: a source + * file goes to the path of dest, with the name of dest. A source folder is + * as `move`. + * @param {string} source + * @param {string} target + * @param {Object} [flags] + * @returns {Promise} + */ +export async function moveto(source, target, flags = {}) { + try { + await transfer(source, target, flags, { move: true, to: true }); + return new Response(null); + } catch (error) { + return failure(error); + } +} diff --git a/cmd/move/main_test.js b/cmd/move/main_test.js new file mode 100644 index 0000000..df09ca7 --- /dev/null +++ b/cmd/move/main_test.js @@ -0,0 +1,42 @@ +import { beforeEach, describe, test } from "node:test"; +import assert from "node:assert/strict"; + +import { fetch } from "../../fetch.js"; +import { contents, putFiles } from "../testing.js"; +import { move, moveto } from "./main.js"; + +const SOURCE = ":memory:/move/source/"; +const TARGET = ":memory:/move/target/"; + +describe("move and moveto", () => { + beforeEach(async () => { + await (await fetch(":memory:/move/", { method: "DELETE" })).body?.cancel(); + await putFiles(SOURCE, { "f.txt": "hello", "a/g.txt": "world!" }); + }); + + test("move copies the files, and deletes them in the source", async () => { + assert.equal((await move(SOURCE, TARGET)).status, 200); + assert.deepEqual((await contents(TARGET)).files, { + "f.txt": "hello", + "a/g.txt": "world!", + }); + // The folders of the source stay, as rclone. + assert.deepEqual(await contents(SOURCE), { files: {}, folders: ["a"] }); + }); + + test("move --delete-empty-src-dirs deletes the folders too", async () => { + await move(SOURCE, TARGET, { delete_empty_src_dirs: true }); + assert.deepEqual(await contents(SOURCE), { files: {}, folders: [] }); + }); + + test("moveto renames a file", async () => { + assert.equal( + (await moveto(`${SOURCE}f.txt`, `${SOURCE}renamed.txt`)).status, + 200, + ); + assert.deepEqual((await contents(SOURCE)).files, { + "renamed.txt": "hello", + "a/g.txt": "world!", + }); + }); +}); diff --git a/cmd/operations.js b/cmd/operations.js new file mode 100644 index 0000000..f4670bc --- /dev/null +++ b/cmd/operations.js @@ -0,0 +1,396 @@ +/** + * The operations that the commands share, as the package `operations` of + * rclone: the information of a path, the files of a folder, the copy of a + * file and the log of the messages. + * + * A location is a path of rclone for `fetch`, with percent-encoding, as the + * command line gives it: for example `remote:a%20b/c.txt`. + */ +import process from "node:process"; +import { isRemoteRoot, parse } from "@rfetch/lib/fspath"; +import { readListing } from "@rfetch/lib/listing"; +import { encodePath } from "@rfetch/lib/path"; + +import { fetch } from "../fetch.js"; +import { listItems } from "./lsjson/main.js"; + +/** The URL of a folder: the links of its list are relative to it. */ +const FOLDER = new URL("http://remote/folder/"); + +/** The flags of the commands that are not options of a backend. */ +const OWN_FLAGS = new Set(["recursive", "max_depth", "max-depth"]); + +/** + * An error of an operation, with the status of its answer: 404 for a folder + * that does not exist, as the exit code 3 of rclone. + */ +export class OperationError extends Error { + /** @param {string} message @param {number} [status] */ + constructor(message, status = 500) { + super(message); + this.status = status; + } +} + +/** + * The answer of a command for an error. + * @param {unknown} error + * @returns {Response} + */ +export function failure(error) { + if (error instanceof OperationError) { + return new Response(error.message, { status: error.status }); + } + if (error instanceof SyntaxError) { + return new Response(error.message, { status: 400 }); + } + throw error; +} + +/** + * Writes the messages of the commands on the standard error, as the log of + * rclone: `2006/01/02 15:04:05 NOTICE: text`. The tests replace `write`. + */ +export const logger = { + /** @param {string} line */ + write(line) { + process.stderr.write(line); + }, +}; + +/** + * Writes a message of a level, about an object, as rclone. + * @param {"ERROR" | "NOTICE" | "INFO"} level + * @param {string} object The object of the message, for example a path, or + * "". + * @param {string} text + * @param {Object} [flags] With `quiet`, only the errors. + */ +export function log(level, object, text, flags = {}) { + if (level !== "ERROR" && (flags.quiet === true || flags.quiet === "true")) { + return; + } + const now = new Date(); + const pad = (number) => `${number}`.padStart(2, "0"); + const time = `${now.getFullYear()}/${pad(now.getMonth() + 1)}/${ + pad(now.getDate()) + } ${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`; + logger.write( + `${time} ${level.padEnd(6)}: ${object ? `${object}: ` : ""}${text}\n`, + ); +} + +/** + * Gives a location with the flags of the command in its query, as the + * other commands do: the fetch of rfetch gives each backend its flags. + * @param {string} location + * @param {Object} [flags] + */ +export function withFlags(location, flags = {}) { + const params = new URLSearchParams(); + for (const [name, value] of Object.entries(flags)) { + if (OWN_FLAGS.has(name)) continue; + if (typeof value === "string" || typeof value === "boolean") { + params.set(name, `${value}`); + } + } + const query = `${params}`; + return query ? `${location}?${query}` : location; +} + +/** + * Splits a location into its remote, and the path in the remote. + * @param {string} location + * @returns {{ remote: string, path: string }} For example `remote:` and + * `a/b.txt`. + */ +export function split(location) { + const { path } = parse(location); + return { remote: location.slice(0, location.length - path.length), path }; +} + +/** + * Tells if a location is a folder by its form: the root of a remote, or a + * path with `/` at the end. + * @param {string} location + */ +export function isFolderForm(location) { + const { path } = split(location); + return path === "" || path.endsWith("/") || isRemoteRoot(path); +} + +/** + * Gives the location of a folder, with `/` at the end of its path. + * @param {string} location + */ +export function folderOf(location) { + const { path } = split(location); + if (path === "" || path.endsWith("/")) return location; + return `${location}/`; +} + +/** + * Gives the location of a path in a folder. + * @param {string} folder The location of the folder. + * @param {string} path A path with no percent-encoding, for example + * `a b/c.txt`. + * @param {boolean} [isDir] + */ +export function join(folder, path, isDir = false) { + return `${folderOf(folder)}${encodePath(path)}${isDir ? "/" : ""}`; +} + +/** + * The name of the last part of a location, with no percent-encoding. + * @param {string} location + */ +export function nameOf(location) { + const { path } = split(location); + const name = path.replace(/\/$/, "").split("/").at(-1) ?? ""; + try { + return decodeURIComponent(name); + } catch { + return name; + } +} + +/** + * The information of a file or a folder. + * @typedef {Object} Info + * @property {boolean} isDir + * @property {number} size -1 for a folder. + * @property {string} modTime The time in ISO 8601, or "". + * @property {string} mimeType + */ + +/** + * Gives the information of a path, or null when it does not exist. As in + * rclone, a path with no `/` at the end is a file, or else a folder. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + * @throws {OperationError} When the remote gives an error. + */ +export async function stat(location, flags = {}) { + if (isFolderForm(location)) { + const response = await fetch(withFlags(folderOf(location), flags), { + method: "HEAD", + }); + await response.body?.cancel(); + if (response.ok) { + return { isDir: true, size: -1, modTime: "", mimeType: "" }; + } + if (response.status === 404) return null; + throw new OperationError( + `${location}: ${response.status} ${response.statusText}`.trim(), + response.status, + ); + } + // The list of the folder tells if the name is a file or a folder. + const { remote, path } = split(location); + const slash = path.lastIndexOf("/") + 1; + const parent = `${remote}${path.slice(0, slash)}`; + const name = nameOf(location); + const response = await fetch(withFlags(parent, flags)); + if (response.status === 404) { + await response.body?.cancel(); + return null; + } + if (!response.ok) { + await response.body?.cancel(); + throw new OperationError( + `${location}: ${response.status} ${response.statusText}`.trim(), + response.status, + ); + } + const entries = (await readListing(response, FOLDER)) + .filter((entry) => entry.Name === name); + const file = entries.find((entry) => !entry.IsDir); + if (file) { + return { + isDir: false, + size: file.Size, + modTime: file.ModTime, + mimeType: file.MimeType, + }; + } + if (entries.length) { + return { isDir: true, size: -1, modTime: entries[0].ModTime, mimeType: "" }; + } + return null; +} + +/** + * An item of a walk: a file or a folder below the folder of the walk. + * @typedef {import("./lsjson/main.js").Item & { location: string }} WalkItem + */ + +/** + * Gives the files and the folders in a folder, and in its folders. + * @param {string} location The location of the folder. + * @param {Object} [flags] + * @param {{ maxDepth?: number }} [options] + * @returns {Promise>} + * @throws {OperationError} With 404 when the folder does not exist. + */ +export async function walk(location, flags = {}, { maxDepth = Infinity } = {}) { + const folder = folderOf(location); + const items = await listItems(folder, { + ...flags, + recursive: "true", + max_depth: String(maxDepth), + }); + if (items instanceof Response) { + const text = (await items.text()).trim(); + throw new OperationError( + items.status === 404 ? "directory not found" : text || + `${items.status} ${items.statusText}`.trim(), + items.status, + ); + } + return (async function* () { + for await (const item of items) { + yield { ...item, location: join(folder, item.Path, item.IsDir) }; + } + })(); +} + +/** + * Runs a function on each value, with `concurrency` at the same time. + * @template T + * @param {AsyncIterable | Iterable} values + * @param {number} concurrency + * @param {(value: T) => Promise} run + */ +export async function each(values, concurrency, run) { + const iterator = values[Symbol.asyncIterator]?.() ?? + values[Symbol.iterator](); + const next = async () => { + for (;;) { + const { done, value } = await iterator.next(); + if (done) return; + await run(value); + } + }; + await Promise.all( + Array.from({ length: Math.max(1, concurrency) }, next), + ); +} + +/** + * The number of a flag, or its default. + * @param {unknown} value + * @param {number} fallback + */ +export function numberOf(value, fallback) { + const number = Number(value); + return Number.isFinite(number) && number > 0 ? number : fallback; +} + +/** + * Copies a file: it reads the source, and puts it at the target. + * @param {string} source + * @param {string} target + * @param {Object} [flags] + * @throws {OperationError} When the read or the write fails. + */ +export async function copyFile(source, target, flags = {}) { + const response = await fetch(withFlags(source, flags)); + if (!response.ok) { + await response.body?.cancel(); + throw new OperationError( + `${source}: ${response.status} ${response.statusText}`.trim(), + response.status, + ); + } + /** @type {Record} */ + const headers = {}; + for (const name of ["Content-Type", "Content-Length"]) { + const value = response.headers.get(name); + if (value) headers[name] = value; + } + const put = await fetch(withFlags(target, flags), { + method: "PUT", + headers, + body: response.body ?? new Uint8Array(0), + }); + if (!put.ok) { + const text = (await put.text()).trim(); + throw new OperationError( + `${target}: ${text || `${put.status} ${put.statusText}`.trim()}`, + put.status, + ); + } + await put.body?.cancel(); +} + +/** + * Deletes a file, or a folder with all that is in it. + * @param {string} location For a folder, with `/` at the end. + * @param {Object} [flags] + * @throws {OperationError} When the remote gives an error. + */ +export async function remove(location, flags = {}) { + const response = await fetch(withFlags(location, flags), { + method: "DELETE", + }); + await response.body?.cancel(); + if (!response.ok) { + throw new OperationError( + response.status === 404 + ? `${location}: not found` + : `${location}: ${response.status} ${response.statusText}`.trim(), + response.status, + ); + } +} + +/** + * Makes a folder, and the folders above it. + * @param {string} location + * @param {Object} [flags] + * @throws {OperationError} When the remote gives an error. + */ +export async function mkdir(location, flags = {}) { + const response = await fetch(withFlags(folderOf(location), flags), { + method: "PUT", + }); + await response.body?.cancel(); + if (!response.ok) { + throw new OperationError( + `${location}: ${response.status} ${response.statusText}`.trim(), + response.status, + ); + } +} + +/** + * Gives the type and the options of the remote of a location, from TRACE of + * the fetch of rfetch: the options of the path, of the connection string, + * of the flags, of the variables and of the config file. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise<{ type: string, options: Record }>} + */ +export async function remoteOf(location, flags = {}) { + const response = await fetch(withFlags(location, flags), { + method: "TRACE", + }); + const [, type = ""] = response.headers.get("Via")?.match(/^(.*?)(\/.*)?$/) ?? + []; + const lines = (await response.text()).split("\r"); + /** @type {Record} */ + const options = {}; + const [, query = ""] = lines[0]?.match(/^TRACE [^?]*\?(.*) HTTP\/1\.1$/) ?? + []; + for (const [name, value] of new URLSearchParams(query)) options[name] = value; + // The other options are in the variables `RFETCH_TYPE_NAME`. + const prefix = `RFETCH_${type.toUpperCase()}_`; + for (const line of lines.slice(1)) { + const colon = line.indexOf(": "); + if (colon === -1 || !line.startsWith(prefix)) continue; + const name = line.slice(prefix.length, colon).toLowerCase(); + options[name] ??= line.slice(colon + 2); + } + return { type, options }; +} diff --git a/cmd/rclone_test.js b/cmd/rclone_test.js new file mode 100644 index 0000000..57ac262 --- /dev/null +++ b/cmd/rclone_test.js @@ -0,0 +1,259 @@ +/** + * The tests of the commands against the commands of rclone, on the same + * local folders: the output, the exit code, and the files after the command. + * The tests run when rclone is in the PATH, or when RCLONE is the path of + * rclone. + */ +import { after, before, describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { + cpSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, relative } from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +const RCLONE = process.env.RCLONE || "rclone"; +const hasRclone = spawnSync(RCLONE, ["version"]).status === 0; +const MAIN = join(dirname(fileURLToPath(import.meta.url)), "..", "main.js"); + +/** The files of the tests, with hard names. */ +const FILES = { + "f.txt": "hello", + "B.txt": "b", + "é.txt": "e", + "a/g.txt": "world!", + "a/b/zero": "", + "sp ace/#1 & 2?.txt": "name", + ".hidden/x": "x", +}; + +/** + * The files and the folders of a local folder. + * @param {string} folder + * @returns {string[]} The paths, with a `/` at the end of a folder, and the + * text of each file. + */ +function treeOf(folder) { + /** @type {string[]} */ + const paths = []; + /** @param {string} path */ + const visit = (path) => { + for (const entry of readdirSync(path, { withFileTypes: true })) { + const full = join(path, entry.name); + const name = relative(folder, full); + if (entry.isDirectory()) { + paths.push(`${name}/`); + visit(full); + } else { + paths.push(`${name} ${readFileSync(full, "utf8")}`); + } + } + }; + try { + statSync(folder); + } catch { + return ["(none)"]; + } + visit(folder); + return paths.sort(); +} + +describe("the commands and the commands of rclone", { + skip: !hasRclone && "rclone is not installed. Set RCLONE to its path.", +}, () => { + const home = mkdtempSync(join(tmpdir(), "rfetch-commands-")); + const source = join(home, "source"); + + /** + * Runs a program in the folder of the tests. + * @param {string} program + * @param {string[]} args + */ + const run = (program, args) => { + const { status, stdout, stderr } = spawnSync(program, args, { + cwd: home, + env: { PATH: process.env.PATH, HOME: home, TZ: "UTC" }, + encoding: "utf8", + }); + return { status, stdout, stderr }; + }; + /** @param {...string} args */ + const rclone = (...args) => run(RCLONE, args); + /** @param {...string} args */ + const rfetch = (...args) => run(process.execPath, [MAIN, ...args]); + + before(() => { + for (const [path, text] of Object.entries(FILES)) { + mkdirSync(dirname(join(source, path)), { recursive: true }); + writeFileSync(join(source, path), text); + } + mkdirSync(join(source, "empty", "inner"), { recursive: true }); + const password = rclone("obscure", "commands").stdout.trim(); + mkdirSync(join(home, ".config", "rclone"), { recursive: true }); + writeFileSync( + join(home, ".config", "rclone", "rclone.conf"), + `[mem]\ntype = memory\n\n[enc]\ntype = crypt\nremote = ${ + join(home, "encrypted") + }\npassword = ${password}\n`, + ); + }); + after(() => rmSync(home, { recursive: true, force: true })); + + /** + * Runs a command with rclone and rfetch, and compares the outputs and + * the exit codes. + * @param {string[]} args + * @param {{ sort?: boolean }} [options] `sort` compares the lines in any + * order: rclone writes them as its checks end. + */ + const same = (args, { sort = false } = {}) => { + const expected = rclone(...args); + const actual = rfetch(...args); + const lines = (text) => sort ? text.split("\n").sort() : text; + assert.equal(actual.status, expected.status, `${args.join(" ")}: status`); + assert.deepEqual( + lines(actual.stdout), + lines(expected.stdout), + args.join(" "), + ); + }; + + test("size", () => { + same(["size", "source"]); + same(["size", "--json", "source"]); + same(["size", "source/a/g.txt"]); + same(["size", "missing"]); + }); + + test("tree", () => { + same(["tree", "source"]); + same(["tree", "--dirs-only", "source"]); + same(["tree", "--level", "1", "source"]); + same(["tree", "--dirsfirst", "source"]); + same(["tree", "--noreport", "--all", "source"]); + same(["tree", "--size", "source"]); + same(["tree", "source/f.txt"]); + }); + + test("the sums", () => { + same(["md5sum", "source"], { sort: true }); + same(["sha1sum", "source"], { sort: true }); + same(["hashsum", "sha256", "source"], { sort: true }); + same(["hashsum", "SHA-1", "source"], { sort: true }); + same(["hashsum", "crc32", "source"], { sort: true }); + same(["hashsum", "md5", "--base64", "source/f.txt"]); + same(["md5sum", "missing"]); + }); + + test("checksum, and hashsum --checkfile", () => { + writeFileSync(join(home, "MD5SUMS"), rclone("md5sum", "source").stdout); + same(["checksum", "md5", "MD5SUMS", "source"]); + same(["md5sum", "--checkfile", "MD5SUMS", "source"], { sort: true }); + writeFileSync( + join(home, "BAD"), + "00000000000000000000000000000000 f.txt\n", + ); + same(["checksum", "md5", "BAD", "source"]); + }); + + test("check", () => { + cpSync(source, join(home, "other"), { recursive: true }); + rmSync(join(home, "other", "a", "g.txt")); + writeFileSync(join(home, "other", "extra"), "x"); + writeFileSync(join(home, "other", "B.txt"), "longer"); + same(["check", "source", "other", "--combined", "-"], { sort: true }); + same(["check", "source", "other", "--one-way", "--missing-on-dst", "-"]); + same(["check", "source", "source"]); + // Same size, other bytes: rclone sees it with MD5, rfetch with the + // bytes. + writeFileSync(join(home, "other", "f.txt"), "HELLO"); + same(["check", "--download", "source", "other", "--differ", "-"], { + sort: true, + }); + }); + + test("listremotes", () => { + same(["listremotes"]); + same(["listremotes", "--long"]); + same(["listremotes", "--json"]); + }); + + test("cryptdecode of the names of rclone", () => { + assert.equal(rclone("copy", "source", "enc:").status, 0); + const names = readdirSync(join(home, "encrypted")); + same(["cryptdecode", "enc:", ...names, "not-a-name"]); + same(["cryptdecode", "--reverse", "enc:", "f.txt", "a/g.txt"]); + }); + + /** + * Runs a command that changes files with rclone and rfetch, each on its + * own copy of the files, and compares the files after it. + * @param {(folder: string) => string[]} args The arguments, for a folder. + * @param {string[]} [only] Other arguments for rclone only. + */ + const sameEffect = (args, only = []) => { + const results = []; + for ( + const [name, tool] of [ + ["by-rclone", (...a) => rclone(...only, ...a)], + ["by-rfetch", rfetch], + ] + ) { + const folder = join(home, name); + rmSync(folder, { recursive: true, force: true }); + cpSync(source, join(folder, "source"), { recursive: true }); + const { status } = tool(...args(name)); + results.push({ status, tree: treeOf(folder) }); + } + assert.deepEqual(results[1], results[0], args("folder").join(" ")); + }; + + test("the commands that delete", () => { + sameEffect((f) => ["delete", `${f}/source`]); + sameEffect((f) => ["delete", "--rmdirs", `${f}/source`]); + sameEffect((f) => ["delete", `${f}/source/a/g.txt`]); + sameEffect((f) => ["deletefile", `${f}/source/f.txt`]); + sameEffect((f) => ["deletefile", `${f}/source/a`]); + sameEffect((f) => ["purge", `${f}/source/a`]); + sameEffect((f) => ["rmdir", `${f}/source/empty/inner`]); + sameEffect((f) => ["rmdir", `${f}/source/a`]); + sameEffect((f) => ["rmdirs", `${f}/source`]); + sameEffect((f) => ["rmdirs", "--leave-root", `${f}/source/empty`]); + sameEffect((f) => ["delete", "--dry-run", `${f}/source`]); + }); + + test("the commands that copy and move", () => { + sameEffect((f) => ["copy", `${f}/source`, `${f}/target`]); + sameEffect((f) => ["copy", `${f}/source/f.txt`, `${f}/target`]); + sameEffect((f) => ["copyto", `${f}/source/f.txt`, `${f}/target/new.txt`]); + // rclone moves a folder of a local remote with one rename. rfetch moves + // each file, as rclone does with no DirMove. + const noDirMove = ["--disable", "DirMove"]; + sameEffect((f) => ["move", `${f}/source`, `${f}/target`], noDirMove); + sameEffect((f) => [ + "move", + "--delete-empty-src-dirs", + `${f}/source`, + `${f}/target`, + ], noDirMove); + sameEffect((f) => [ + "move", + "--delete-empty-src-dirs", + `${f}/source/a`, + `${f}/target`, + ], noDirMove); + sameEffect((f) => ["moveto", `${f}/source/f.txt`, `${f}/source/g.txt`]); + sameEffect((f) => ["mkdir", `${f}/source/x/y`]); + sameEffect((f) => ["touch", `${f}/source/new.txt`]); + }); +}); diff --git a/cmd/size/main.js b/cmd/size/main.js new file mode 100644 index 0000000..6b1fe14 --- /dev/null +++ b/cmd/size/main.js @@ -0,0 +1,75 @@ +import { countString, sizeString } from "@rfetch/lib/format"; +import { isTrue } from "@rfetch/lib/options"; + +import { failure, log, OperationError, stat, walk } from "../operations.js"; + +/** + * Prints the total size and number of objects in remote:path, as `rclone + * size`: + * + * ```text + * Total objects: 3 + * Total size: 11 B (11 Byte) + * ``` + * + * With `--json`, it gives `{"count":3,"bytes":11,"sizeless":0}`. A file with + * no size counts as 0 bytes, and in `sizeless`. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function size(location, flags = {}) { + try { + const info = await stat(location, flags); + if (!info) throw new OperationError("directory not found", 404); + let count = 0, bytes = 0, sizeless = 0; + /** @param {number} value */ + const add = (value) => { + count++; + if (value >= 0) bytes += value; + else sizeless++; + }; + if (info.isDir) { + for await (const item of await walk(location, flags)) { + if (!item.IsDir) add(item.Size); + } + } else { + add(info.size); + } + if (sizeless > 0) { + log( + "NOTICE", + location, + `Size may be underestimated due to ${sizeless} objects with unknown size`, + flags, + ); + } + if (isTrue(flags.json)) { + return new Response(`${JSON.stringify({ count, bytes, sizeless })}\n`); + } + const suffix = countString(count, true); + let text = suffix === `${count}` + ? `Total objects: ${count}\n` + : `Total objects: ${suffix} (${count})\n`; + text += `Total size: ${byteUnit(bytes)} (${bytes} Byte)\n`; + if (sizeless > 0) { + text += `Total objects with unknown size: ${ + countString(sizeless, true) + } (${sizeless})\n`; + } + return new Response(text); + } catch (error) { + return failure(error); + } +} + +/** + * Writes a size as `SizeSuffix.ByteUnit` of rclone, for example `11 B` or + * `1.500 KiB`. + * @param {number} bytes + */ +export function byteUnit(bytes) { + const [, number, suffix] = sizeString(bytes, true).match(/^([\d.]+)(\D*)$/) ?? + []; + return `${number} ${suffix}B`; +} diff --git a/cmd/size/main_test.js b/cmd/size/main_test.js new file mode 100644 index 0000000..d9231e1 --- /dev/null +++ b/cmd/size/main_test.js @@ -0,0 +1,40 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { putFiles } from "../testing.js"; +import { byteUnit, size } from "./main.js"; + +test("size gives the count and the size of the files, as rclone", async () => { + await putFiles(":memory:/size/", { + "f.txt": "hello", + "a/g.txt": "world!", + "a/b/zero": "", + }); + assert.equal( + await (await size(":memory:/size")).text(), + "Total objects: 3\nTotal size: 11 B (11 Byte)\n", + ); + assert.equal( + await (await size(":memory:/size", { json: true })).text(), + '{"count":3,"bytes":11,"sizeless":0}\n', + ); + assert.equal( + await (await size(":memory:/size/a/g.txt")).text(), + "Total objects: 1\nTotal size: 6 B (6 Byte)\n", + ); + assert.equal((await size(":memory:/size/missing")).status, 404); +}); + +test("byteUnit writes the sizes of rclone", () => { + for ( + const [bytes, text] of [ + [0, "0 B"], + [11, "11 B"], + [1024, "1 KiB"], + [1536, "1.500 KiB"], + [5 * 1024 ** 3 + 1, "5.000 GiB"], + ] + ) { + assert.equal(byteUnit(bytes), text, `${bytes}`); + } +}); diff --git a/cmd/testing.js b/cmd/testing.js new file mode 100644 index 0000000..e362414 --- /dev/null +++ b/cmd/testing.js @@ -0,0 +1,65 @@ +/** + * The helpers of the tests of the commands, on the memory remote. + */ +import { encodePath } from "@rfetch/lib/path"; + +import { fetch } from "../fetch.js"; +import { logger, walk } from "./operations.js"; + +/** + * Puts files in a folder of a remote. + * @param {string} folder For example `:memory:/test/`. + * @param {Record} files The texts of the files, by path. + */ +export async function putFiles(folder, files) { + for (const [path, text] of Object.entries(files)) { + const bytes = new TextEncoder().encode(text); + const response = await fetch(`${folder}${encodePath(path)}`, { + method: "PUT", + headers: { "Content-Length": `${bytes.length}` }, + body: bytes, + }); + await response.body?.cancel(); + if (!response.ok) throw new Error(`PUT ${path}: ${response.status}`); + } +} + +/** + * Gives the texts of the files, and the folders, of a folder of a remote. + * @param {string} folder + * @returns {Promise<{ files: Record, folders: string[] } | null>} + * null when the folder does not exist. + */ +export async function contents(folder) { + /** @type {Record} */ + const files = {}; + const folders = []; + let items; + try { + items = await walk(folder); + } catch { + return null; + } + for await (const item of items) { + if (item.IsDir) { + folders.push(item.Path); + } else { + files[item.Path] = await (await fetch(item.location)).text(); + } + } + return { files, folders: folders.sort() }; +} + +/** + * Keeps the lines of the log of the commands during a test. + * @param {import("node:test").TestContext} t + * @returns {string[]} The lines, with no time. + */ +export function captureLog(t) { + /** @type {string[]} */ + const lines = []; + t.mock.method(logger, "write", (line) => { + lines.push(line.replace(/^\S+ \S+ /, "").trimEnd()); + }); + return lines; +} diff --git a/cmd/touch/main.js b/cmd/touch/main.js new file mode 100644 index 0000000..7c55088 --- /dev/null +++ b/cmd/touch/main.js @@ -0,0 +1,104 @@ +import { isTrue } from "@rfetch/lib/options"; + +import { fetch } from "../../fetch.js"; +import { + failure, + log, + OperationError, + stat, + walk, + withFlags, +} from "../operations.js"; + +/** The formats of `--timestamp`, as rclone. */ +const TIMESTAMP = /^\d{4}-\d\d-\d\d(T\d\d:\d\d:\d\d(\.\d{1,9})?)?$/; + +/** + * Create new file or change file modification time, as `rclone touch`. + * + * A file that does not exist gets 0 bytes, unless `--no-create` or + * `--recursive`. The backends cannot set the time of a file yet: thus a file + * that exists, and `--timestamp`, give the error of rclone for a remote that + * cannot set the time. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function touch(location, flags = {}) { + try { + const timestamp = flags.timestamp; + if (timestamp !== undefined && !TIMESTAMP.test(`${timestamp}`)) { + throw new OperationError( + `failed to parse timestamp argument: ${timestamp}`, + 400, + ); + } + const info = await stat(location, flags); + if (!info) { + if (isTrue(flags.no_create)) { + log( + "NOTICE", + location, + "Not touching nonexistent file due to --no-create", + flags, + ); + return new Response(null); + } + if (isTrue(flags.recursive)) { + log( + "NOTICE", + location, + "Not touching nonexistent file due to --recursive", + flags, + ); + return new Response(null); + } + if (timestamp !== undefined) { + throw new OperationError( + "failed to touch (create): can't set modified time", + 501, + ); + } + if (isTrue(flags.dry_run)) { + log( + "NOTICE", + location, + "Skipped touch (create) as --dry-run is set", + flags, + ); + return new Response(null); + } + const response = await fetch(withFlags(location, flags), { + method: "PUT", + headers: { "Content-Length": "0" }, + body: new Uint8Array(0), + }); + await response.body?.cancel(); + if (!response.ok) { + throw new OperationError( + `failed to touch (create): ${response.status} ${response.statusText}` + .trim(), + response.status, + ); + } + return new Response(null); + } + // The files of a folder, as rclone: with `--recursive`, also the files + // of its folders. + let files = info.isDir ? 0 : 1; + if (info.isDir) { + const depth = isTrue(flags.recursive) ? Infinity : 1; + for await ( + const item of await walk(location, flags, { maxDepth: depth }) + ) { + if (!item.IsDir) files++; + } + } + if (files > 0 && !isTrue(flags.dry_run)) { + throw new OperationError("failed to touch: can't set modified time", 501); + } + return new Response(null); + } catch (error) { + return failure(error); + } +} diff --git a/cmd/touch/main_test.js b/cmd/touch/main_test.js new file mode 100644 index 0000000..e052b55 --- /dev/null +++ b/cmd/touch/main_test.js @@ -0,0 +1,36 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { contents } from "../testing.js"; +import { touch } from "./main.js"; + +test("touch makes an empty file, as rclone", async () => { + assert.equal((await touch(":memory:/touch/new.txt")).status, 200); + assert.deepEqual((await contents(":memory:/touch/")).files, { + "new.txt": "", + }); + assert.equal( + (await touch(":memory:/touch/none.txt", { no_create: true })).status, + 200, + ); + assert.deepEqual(Object.keys((await contents(":memory:/touch/")).files), [ + "new.txt", + ]); +}); + +test("touch tells that the backends cannot set the time yet", async () => { + const response = await touch(":memory:/touch/new.txt"); + assert.equal(response.status, 501); + assert.equal( + await response.text(), + "failed to touch: can't set modified time", + ); + const timestamp = await touch(":memory:/touch/t.txt", { + timestamp: "2020-01-02T03:04:05", + }); + assert.equal(timestamp.status, 501); + assert.equal( + (await touch(":memory:/touch/t.txt", { timestamp: "yesterday" })).status, + 400, + ); +}); diff --git a/cmd/tree/main.js b/cmd/tree/main.js new file mode 100644 index 0000000..d8b0cf2 --- /dev/null +++ b/cmd/tree/main.js @@ -0,0 +1,325 @@ +import { isTrue } from "@rfetch/lib/options"; + +import { failure, nameOf, OperationError, stat, walk } from "../operations.js"; + +/** + * A file or a folder of the tree. + * @typedef {Object} Node + * @property {string} name + * @property {string} path The path from the root, with a `/` at the start. + * @property {boolean} isDir + * @property {number} size + * @property {string} modTime + * @property {Node[]} children + */ + +/** + * List the contents of the remote in a tree like fashion, as `rclone tree` + * and the `tree` command of Unix: + * + * ```text + * / + * ├── file1 + * └── subdir + * └── file2 + * + * 1 directories, 2 files + * ``` + * + * As rclone, the names that start with `.` are not in the tree, unless + * `--all`. The flags of rclone: `--all`, `--dirs-only`, `--dirsfirst`, + * `--full-path`, `--level` (or `--max-depth`), `--modtime`, `--noindent`, + * `--noreport`, `--quote`, `--size` (with `--human-readable`), `--sort` + * (name, version, size or mtime), `--sort-modtime` and `--sort-reverse`. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function tree(location, flags = {}) { + try { + const info = await stat(location, flags); + if (!info) throw new OperationError("directory not found", 404); + /** @type {Node} */ + const root = { + name: "/", + path: "/", + isDir: true, + size: -1, + modTime: "", + children: [], + }; + const all = isTrue(flags.all); + if (!info.isDir) { + const name = nameOf(location); + root.children.push({ + name, + path: `/${name}`, + isDir: false, + size: info.size, + modTime: info.modTime, + children: [], + }); + } else { + const level = Number(flags.level || flags.max_depth || Infinity); + /** @type {Map} */ + const folders = new Map([["", root]]); + const items = []; + for await ( + const item of await walk(location, flags, { + maxDepth: level > 0 ? level : Infinity, + }) + ) { + items.push(item); + } + // A folder comes before the files in it in the walk, but a child can + // come before the node of a folder of the same level. + for (const item of items) { + if (!item.IsDir) continue; + folders.set(item.Path, nodeOf(item)); + } + for (const item of items) { + const slash = item.Path.lastIndexOf("/"); + const parent = folders.get( + slash === -1 ? "" : item.Path.slice(0, slash), + ); + if (!parent) continue; + if (!all && item.Name.startsWith(".")) continue; + if (!all && hidden(item.Path)) continue; + parent.children.push( + item.IsDir ? folders.get(item.Path) : nodeOf(item), + ); + } + } + + const dirsOnly = isTrue(flags.dirs_only); + const counts = { dirs: 0, files: 0 }; + prune(root, dirsOnly, counts); + sortTree(root, flags); + let text = ""; + print(root, "", flags, (line) => text += line); + if (!isTrue(flags.noreport)) { + // As rclone: with no file, the count of the folders is 0 too. + if (counts.files === 0) counts.dirs = 0; + text += `\n${counts.dirs} directories${ + dirsOnly ? "" : `, ${counts.files} files` + }\n`; + } + return new Response(text); + } catch (error) { + return failure(error); + } +} + +/** + * Tells if a path has a folder with a name that starts with `.`. + * @param {string} path + */ +function hidden(path) { + return path.split("/").some((name) => name.startsWith(".")); +} + +/** + * The node of an item of a walk. + * @param {import("../operations.js").WalkItem} item + * @returns {Node} + */ +function nodeOf(item) { + return { + name: item.Name, + path: `/${item.Path}`, + isDir: item.IsDir, + size: item.Size, + modTime: item.ModTime ?? "", + children: [], + }; +} + +/** + * Removes the files for `--dirs-only`, and counts the folders and the files + * below the root. + * @param {Node} node + * @param {boolean} dirsOnly + * @param {{ dirs: number, files: number }} counts + */ +function prune(node, dirsOnly, counts) { + if (dirsOnly) node.children = node.children.filter((child) => child.isDir); + for (const child of node.children) { + if (child.isDir) { + counts.dirs++; + prune(child, dirsOnly, counts); + } else { + counts.files++; + } + } +} + +/** + * Compares two names as Go compares two strings: by the bytes of UTF-8, that + * is by the code points. + * @param {string} a + * @param {string} b + */ +function compareNames(a, b) { + const x = [...a]; + const y = [...b]; + for (let i = 0; i < x.length && i < y.length; i++) { + const difference = x[i].codePointAt(0) - y[i].codePointAt(0); + if (difference) return difference; + } + return x.length - y.length; +} + +/** + * Tells if a name comes before another in the natural order of versions, as + * `NaturalLess` of the tree package: the numbers by their values. + * @param {string} a + * @param {string} b + */ +function naturalLess(a, b) { + const digit = (c) => c >= "0" && c <= "9"; + let i = 0, j = 0; + while (i < a.length && j < b.length) { + const c1 = a[i], c2 = b[j]; + const d1 = digit(c1), d2 = digit(c2); + if (d1 !== d2) return d1; + if (!d1) { + if (c1 !== c2) return c1 < c2; + i++; + j++; + continue; + } + while (i < a.length && a[i] === "0") i++; + while (j < b.length && b[j] === "0") j++; + const start1 = i, start2 = j; + while (i < a.length && digit(a[i])) i++; + while (j < b.length && digit(b[j])) j++; + const n1 = a.slice(start1, i), n2 = b.slice(start2, j); + if (n1.length !== n2.length) return n1.length < n2.length; + if (n1 !== n2) return n1 < n2; + if (start1 !== start2) return start1 < start2; + } + return a.length < b.length; +} + +/** + * Sorts the children of each folder, as the tree package: by name, or by the + * flags. `--dirsfirst` puts the folders first, and keeps the order of the + * names in each part. + * @param {Node} node + * @param {Object} flags + */ +function sortTree(node, flags) { + const sort = `${flags.sort ?? ""}`; + /** @type {(a: Node, b: Node) => number} */ + let compare; + const time = (n) => Date.parse(n.modTime) || 0; + if (isTrue(flags.sort_modtime) || sort === "mtime") { + compare = (a, b) => time(a) - time(b); + } else if (isTrue(flags.dirsfirst)) { + compare = (a, b) => Number(b.isDir) - Number(a.isDir); + } else if (sort === "version") { + compare = (a, b) => + naturalLess(a.name, b.name) ? -1 : naturalLess(b.name, a.name) ? 1 : 0; + } else if (sort === "size") { + compare = (a, b) => a.size - b.size; + } else { + compare = (a, b) => compareNames(a.name, b.name); + } + const reverse = isTrue(flags.sort_reverse); + const visit = (node) => { + // The order of the names before the sort, as the list of rclone. + node.children.sort((a, b) => compareNames(a.name, b.name)); + node.children.sort((a, b) => reverse ? compare(b, a) : compare(a, b)); + for (const child of node.children) visit(child); + }; + visit(node); +} + +/** + * Formats a size as the tree package, for `--human-readable`: for example + * `52`, `64K` or `2.0M`. + * @param {number} bytes + */ +function formatBytes(bytes) { + const units = ["K", "M", "G", "T", "P", "E"]; + let unit = ""; + let value = bytes; + for (let i = units.length; i > 0; i--) { + const scale = 2 ** (10 * i); + if (bytes > scale) { + unit = units[i - 1]; + value = bytes / scale; + break; + } + } + if (!unit) return `${Math.round(bytes)}`; + return `${value >= 10 ? Math.round(value) : value.toFixed(1)}${unit}`; +} + +/** + * The size of a folder: the sizes of all the files in it. + * @param {Node} node + * @returns {number} + */ +function folderSize(node) { + let total = 0; + for (const child of node.children) { + total += child.isDir ? folderSize(child) : Math.max(child.size, 0); + } + return total; +} + +/** + * Writes the time of `--modtime`: `Jan 02 15:04` in this year, else + * `Jan 02 2006`. + * @param {string} modTime + */ +function formatTime(modTime) { + const date = new Date(modTime); + if (isNaN(date.getTime())) return ""; + const month = date.toLocaleString("en-US", { month: "short" }); + const day = `${date.getDate()}`.padStart(2, "0"); + if (date.getFullYear() !== new Date().getFullYear()) { + return `${month} ${day} ${date.getFullYear()}`; + } + const hours = `${date.getHours()}`.padStart(2, "0"); + const minutes = `${date.getMinutes()}`.padStart(2, "0"); + return `${month} ${day} ${hours}:${minutes}`; +} + +/** + * Writes a node and the nodes in it. + * @param {Node} node + * @param {string} indent + * @param {Object} flags + * @param {(text: string) => void} write + */ +function print(node, indent, flags, write) { + const props = []; + const human = isTrue(flags.human_readable); + if (isTrue(flags.size)) { + const bytes = node.isDir ? folderSize(node) : node.size; + props.push( + human ? formatBytes(bytes).padStart(4) : `${bytes}`.padStart(11), + ); + } + if (!node.isDir && isTrue(flags.modtime)) { + props.push(formatTime(node.modTime)); + } + if (props.length) write(`[${props.join(" ")}] `); + let name = node.path === "/" || isTrue(flags.full_path) + ? node.path + : node.name; + if (isTrue(flags.quote)) name = `"${name}"`; + write(`${name}\n`); + const noIndent = isTrue(flags.noindent); + node.children.forEach((child, index) => { + let add = ""; + if (!noIndent) { + const last = index === node.children.length - 1; + write(`${indent}${last ? "└── " : "├── "}`); + add = last ? " " : "│ "; + } + print(child, indent + add, flags, write); + }); +} diff --git a/cmd/tree/main_test.js b/cmd/tree/main_test.js new file mode 100644 index 0000000..442a725 --- /dev/null +++ b/cmd/tree/main_test.js @@ -0,0 +1,75 @@ +import { before, test } from "node:test"; +import assert from "node:assert/strict"; + +import { fetch } from "../../fetch.js"; +import { putFiles } from "../testing.js"; +import { tree } from "./main.js"; + +const ROOT = ":memory:/tree/"; + +before(async () => { + await putFiles(ROOT, { + "f.txt": "hello", + "a/g.txt": "world!", + "a/b/zero": "", + ".hidden/x": "x", + "B.txt": "b", + "é.txt": "e", + }); + await (await fetch(`${ROOT}empty/inner/`, { method: "PUT" })).body?.cancel(); +}); + +/** @param {Object} [flags] */ +async function text(flags) { + return await (await tree(ROOT, flags)).text(); +} + +test("tree draws the folders and the files, as rclone", async () => { + assert.equal( + await text(), + `/ +├── B.txt +├── a +│ ├── b +│ │ └── zero +│ └── g.txt +├── empty +│ └── inner +├── f.txt +└── é.txt + +4 directories, 5 files +`, + ); +}); + +test("tree takes the flags of rclone", async () => { + assert.equal( + await text({ dirs_only: true, noreport: true }), + "/\n├── a\n│ └── b\n└── empty\n └── inner\n", + ); + assert.equal( + await text({ level: "1", dirsfirst: true }), + `/ +├── a +├── empty +├── B.txt +├── f.txt +└── é.txt + +2 directories, 3 files +`, + ); + assert.match(await text({ all: true }), /├── \.hidden\n│ └── x\n/); + assert.match(await text({ size: true }), /\[ {10}6\] {2}g\.txt\n/); + assert.match(await text({ full_path: true }), /└── \/a\/g\.txt\n/); + assert.match(await text({ sort_reverse: true }), /^\/\n├── é\.txt\n/); +}); + +test("tree of a file, and of a path that does not exist", async () => { + assert.equal( + await (await tree(`${ROOT}a/g.txt`)).text(), + "/\n└── g.txt\n\n0 directories, 1 files\n", + ); + assert.equal((await tree(`${ROOT}missing`)).status, 404); +}); diff --git a/cmd/version/main.js b/cmd/version/main.js new file mode 100644 index 0000000..448b281 --- /dev/null +++ b/cmd/version/main.js @@ -0,0 +1,23 @@ +import process from "node:process"; + +import info from "../../package.json" with { type: "json" }; + +/** + * Show the version number, as `rclone version`: the version of rfetch, and + * the runtime, the system and the processor that run it. + * @returns {Response} + */ +export function version() { + const { versions } = process; + const runtime = versions.bun + ? `bun ${versions.bun}` + : versions.deno + ? `deno ${versions.deno}` + : `node ${versions.node}`; + return new Response( + `rfetch v${info.version}\n` + + `- os/type: ${process.platform}\n` + + `- os/arch: ${process.arch}\n` + + `- runtime: ${runtime}\n`, + ); +} diff --git a/cmd/version/main_test.js b/cmd/version/main_test.js new file mode 100644 index 0000000..75cee6b --- /dev/null +++ b/cmd/version/main_test.js @@ -0,0 +1,11 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import info from "../../package.json" with { type: "json" }; +import { version } from "./main.js"; + +test("version gives the version of rfetch and the runtime", async () => { + const lines = (await version().text()).split("\n"); + assert.equal(lines[0], `rfetch v${info.version}`); + assert.match(lines[3], /^- runtime: (node|deno|bun) \d/); +}); diff --git a/main.js b/main.js index 2f39be3..2269780 100755 --- a/main.js +++ b/main.js @@ -59,12 +59,23 @@ import { encodeLocation } from "@rfetch/lib/fspath"; /** TODO merge global flags into config */ const response = await module[name](...args, flags); if (!response.ok) { - // As rclone: the exit code 3 is "directory not found", and 1 is an - // other error. - const text = (await response.text()).trim() || response.statusText || + // As rclone: the exit code 3 is "directory not found", 4 is "object not + // found", and 1 is an other error. A command with an output and an + // error, for example `check --combined -`, gives the error in + // `statusText`, and the output in the body. + let text = (await response.text()).trim(); + if (response.statusText && text) { + stdout.write(`${text}\n`); + text = response.statusText; + } + text ||= response.statusText || (response.status === 404 ? "not found" : `status ${response.status}`); stderr.write(`Error: ${text}\n`); - process.exitCode = response.status === 404 ? 3 : 1; + process.exitCode = response.status !== 404 + ? 1 + : /object not found$/.test(text) + ? 4 + : 3; return; } if (response.body) { diff --git a/main_test.js b/main_test.js index fbaaef3..d845d58 100644 --- a/main_test.js +++ b/main_test.js @@ -594,10 +594,15 @@ test("the command line", async (t) => { assert.deepEqual(JSON.parse(lsjson.stdout).map(({ Name }) => Name), [ "a#b.txt", ]); - const copy = await run(folder, ["copy", "c?d.txt", "copy #1.txt"]); - assert.equal(copy.code, 0, copy.stderr); + const copyto = await run(folder, ["copyto", "c?d.txt", "copy #1.txt"]); + assert.equal(copyto.code, 0, copyto.stderr); const cat = await run(folder, ["cat", "copy #1.txt"]); assert.equal(cat.stdout, "c?d.txt"); + // As rclone: `copy` puts a file in the target folder. + const copy = await run(folder, ["copy", "a#b.txt", "in #2"]); + assert.equal(copy.code, 0, copy.stderr); + const inside = await run(folder, ["cat", "in #2/a#b.txt"]); + assert.equal(inside.stdout, "a#b.txt"); }); await t.test("serve reads --addr, and a TRACE does not stop it", async () => { From ba0f9dea3aa77405fe001e319f4d56f0bfb8c161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Tue, 20 Oct 2026 00:01:45 +0000 Subject: [PATCH 2/3] test: content that is not text in the name tests of the suite The name tests put a file with no extension. rclone serve webdav guesses the type of such a file from its bytes. When the 64 random bytes have no control byte, the type is text/plain, and the test fails. This occurs for about 1 file in 1250, thus CI failed on some runs. Now the content is the bytes 0 to 63. They are not text, and they are not the start of a known type of file: the type is application/octet-stream each time. --- lib/testing/suite.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/testing/suite.js b/lib/testing/suite.js index f34dd7a..96d779b 100644 --- a/lib/testing/suite.js +++ b/lib/testing/suite.js @@ -147,7 +147,11 @@ export function backendSuite( it(`keeps the name ${JSON.stringify(name)}`, { skip }, async () => { // The name is a folder and a file in it. const path = `${name}/${name}`; - const content = random(64); + // The name has no extension. A server can guess the type of the + // file from its bytes, as `rclone serve webdav` does, and random + // bytes can look like text. The bytes 0 to 63 are not text, and + // they are not the start of a known type of file. + const content = Uint8Array.from({ length: 64 }, (_, i) => i); try { const start = Date.now(); await put(path, content); From 1e2b4c38af698fce8f0e6ffcacaf6513114ca952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Tue, 20 Oct 2026 00:13:53 +0000 Subject: [PATCH 3/3] test(union): rclone reads no free space in the tests against rclone With the policy epmfs, the default create policy, rclone takes the upstream with the most free space. The upstreams of the tests are on one disk. rclone reads the free space of a, then of b. When another test deletes a file between the two reads, b has more free space, and rclone takes b. rfetch reads no free space and takes a, thus CI failed on some runs. Now rclone runs with --disable About. Then it reads no free space, and the policies of space take the first upstream, as rfetch. With b on a disk with more free space, the old tests fail each time, and the new tests pass. --- backend/union/rclone_test.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/union/rclone_test.js b/backend/union/rclone_test.js index 76eccdc..344bee4 100644 --- a/backend/union/rclone_test.js +++ b/backend/union/rclone_test.js @@ -12,6 +12,12 @@ * The policy `epff` of rclone takes the upstream that answers first, thus * the tests cannot compare it. rfetch takes the first upstream in the order * of the upstreams, as the documentation of rclone tells. + * + * rclone runs with `--disable About`: then it reads no free space, and the + * policies of space, as `epmfs`, take the first upstream, as rfetch. Else + * rclone takes the upstream with the most free space. The upstreams are on + * one disk, but other tests write on it at the same time, thus rclone can + * take `b`. */ import { after, describe, test } from "node:test"; import assert from "node:assert/strict"; @@ -85,7 +91,8 @@ describe("union and rclone", { * @param {string} [input] */ function rclone(args, input) { - return spawnSync(RCLONE, ["--config", join(folder, "none.conf"), ...args], { + const config = ["--config", join(folder, "none.conf")]; + return spawnSync(RCLONE, [...config, "--disable", "About", ...args], { env: { PATH: process.env.PATH, HOME: folder }, encoding: "utf8", input,