diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9722349..916a527 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -287,6 +287,26 @@ the local time zone, and the size and the count with `sizeField` and `countField` of `@rfetch/lib/format`, in fields of the widths of rclone. `cmd/lsf/rclone_test.js` compares the lines with the lines of rclone. +The other commands use the operations of `cmd/operations.js`, as the package +`operations` of rclone: `stat` gives the information of a path, `walk` gives the +files and the folders in a folder, and `copyFile`, `remove` and `mkdir` change +the files. An operation that fails throws an `OperationError` with the status of +the answer, and `failure` gives the answer of the command for it. `log` writes +the errors and the notices of rclone on the standard error, for example +`2006/01/02 15:04:05 ERROR : a.txt: sizes differ`, and `--quiet` keeps only the +errors. A command that has an output and an error, for example +`check --combined -`, gives the output in the body and the error in +`statusText`: the command line writes the body on the standard output, and the +error on the standard error. + +The command line gives the exit codes of rclone: 3 for the status 404, 4 for the +status 404 with a message that ends with `object not found`, and 1 for another +status that is not 2xx. + +`cmd/rclone_test.js` runs each command with rfetch and with rclone on the same +local folders, and compares the outputs, the exit codes and the files after the +command. + ## Tests of a backend Each backend must pass the shared tests of `@rfetch/lib/testing/suite`. These diff --git a/README.md b/README.md index 2fd87a2..cf2fe10 100644 --- a/README.md +++ b/README.md @@ -63,16 +63,30 @@ remote. ## Commands +The commands give the output of rclone, and its exit codes: 3 for a folder that +does not exist, 4 for a file that does not exist, and 1 for another error. + - `backend` - `cat` +- `check`, `checksum` and `cryptcheck`: compare the files of two places, or the + files and a sum file. The backends give no hash: `check` compares the sizes, + and with `--download` the bytes. - `config` -- `copy` +- `copy` and `copyto`: copy the files that the target does not have, or that + differ by size or by time. - `copyurl` +- `cryptdecode` +- `delete`, `deletefile`, `purge`, `rmdir` and `rmdirs` +- `hashsum`, `md5sum` and `sha1sum`: MD5, SHA-1, SHA-256, SHA-512 and CRC-32. + rfetch reads each file to compute its hash. +- `listremotes` - `ls` - `lsd` - `lsf` - `lsjson` - `lsl` +- `mkdir` +- `move` and `moveto`: copy, then delete each source file. - `obscure` - `rcat` - `reveal` @@ -81,6 +95,10 @@ remote. - `serve webdav`: serve a remote over WebDAV, as `rclone serve webdav`. A client reads and writes, unless `--read-only`. With `--user` and `--pass`, a client must log in with them. +- `size` +- `touch`: make an empty file. The backends cannot set the time of a file yet. +- `tree` +- `version` ## Runtimes diff --git a/backend/crypt/main.js b/backend/crypt/main.js index 8ba83d6..8d9d798 100644 --- a/backend/crypt/main.js +++ b/backend/crypt/main.js @@ -547,6 +547,25 @@ export async function decode(options, ...args) { return new Response(decoded.join("")); } +/** + * The cipher of the names of a crypt remote, for the command `cryptdecode`: + * it makes the keys one time for all the names. + * + * ```js + * import { nameCipher } from "./main.js"; + * const names = await nameCipher({ password, password2 }); + * names.decrypt(encryptedName); + * ``` + * @param {Object} options + * @param {string} options.password The password, obscured. + * @param {string} [options.password2] The salt, obscured. + * @returns {Promise<{ encrypt: (path: string) => string, decrypt: (path: string) => string }>} + */ +export async function nameCipher(options) { + const { names } = await cipherOf(options.password, options.password2 ?? ""); + return names; +} + /** * Makes the keys from a password and a salt, as `Cipher.Key` of rclone: 32 * bytes for the data, 32 bytes for the names, and 16 bytes of tweak for the diff --git a/backend/union/rclone_test.js b/backend/union/rclone_test.js index 76eccdc..344bee4 100644 --- a/backend/union/rclone_test.js +++ b/backend/union/rclone_test.js @@ -12,6 +12,12 @@ * The policy `epff` of rclone takes the upstream that answers first, thus * the tests cannot compare it. rfetch takes the first upstream in the order * of the upstreams, as the documentation of rclone tells. + * + * rclone runs with `--disable About`: then it reads no free space, and the + * policies of space, as `epmfs`, take the first upstream, as rfetch. Else + * rclone takes the upstream with the most free space. The upstreams are on + * one disk, but other tests write on it at the same time, thus rclone can + * take `b`. */ import { after, describe, test } from "node:test"; import assert from "node:assert/strict"; @@ -85,7 +91,8 @@ describe("union and rclone", { * @param {string} [input] */ function rclone(args, input) { - return spawnSync(RCLONE, ["--config", join(folder, "none.conf"), ...args], { + const config = ["--config", join(folder, "none.conf")]; + return spawnSync(RCLONE, [...config, "--disable", "About", ...args], { env: { PATH: process.env.PATH, HOME: folder }, encoding: "utf8", input, diff --git a/cmd/check/main.js b/cmd/check/main.js new file mode 100644 index 0000000..4d720d3 --- /dev/null +++ b/cmd/check/main.js @@ -0,0 +1,431 @@ +/** + * The checks of files, as `rclone check`, `rclone checksum` and `rclone + * cryptcheck`. Each problem is an error in the log, and the summary is in + * notices, as rclone. The lists of the flags `--combined`, `--match`, + * `--differ`, `--missing-on-src`, `--missing-on-dst` and `--error` go to the + * location of the flag, or to the output for `-`: + * + * - `= path`: the file is the same in the two places. + * - `- path`: the file is only in the destination. + * - `+ path`: the file is only in the source. + * - `* path`: the files differ. + * - `! path`: the check had an error. + */ +import { isTrue } from "@rfetch/lib/options"; + +import { fetch } from "../../fetch.js"; +import { hashFile, hashName } from "../hashsum/main.js"; +import { + each, + failure, + log, + nameOf, + numberOf, + OperationError, + remoteOf, + stat, + walk, + withFlags, +} from "../operations.js"; + +/** The flags of the lists, and the sign of each list in `--combined`. */ +const REPORTS = { + match: "=", + differ: "*", + missing_on_src: "-", + missing_on_dst: "+", + error: "!", +}; + +/** + * The results of a check. + */ +class Results { + differences = 0; + errors = 0; + matches = 0; + noHashes = 0; + missingOnDst = 0; + missingOnSrc = 0; + /** @type {Record} */ + lists = { + combined: [], + ...Object.fromEntries(Object.keys(REPORTS).map((k) => [k, []])), + }; + + /** + * Adds a path to a list, and to the combined list. + * @param {keyof typeof REPORTS} kind + * @param {string} path + */ + report(kind, path) { + this.lists[kind].push(path); + this.lists.combined.push(`${REPORTS[kind]} ${path}`); + } +} + +/** + * The files of a location: a file, or the files in a folder. + * @param {string} location + * @param {Object} flags + * @returns {Promise>} + * @throws {OperationError} With 404 when the location does not exist. + */ +async function filesOf(location, flags) { + const info = await stat(location, flags); + if (!info) throw new OperationError("directory not found", 404); + /** @type {Map} */ + const files = new Map(); + if (!info.isDir) { + files.set(nameOf(location), { location, size: info.size }); + return files; + } + for await (const item of await walk(location, flags)) { + if (!item.IsDir) { + files.set(item.Path, { location: item.location, size: item.Size }); + } + } + return files; +} + +/** + * Checks the files in the source and destination match, as `rclone check`. + * It compares the sizes. The backends of rfetch give no hash, thus as rclone + * with no common hash, the files with the same size match, and the hashes + * could not be checked. With `--download`, it reads the files and compares + * their bytes. With `--one-way`, it does not check the files that are only + * in the destination. + * + * ```js + * import { check } from "./main.js"; + * await check("source:path", "dest:path", { download: true }); + * ``` + * @param {string} source + * @param {string} target + * @param {Object} [flags] + * @returns {Promise} + */ +export async function check(source, target, flags = {}) { + return await compare(source, target, flags, isTrue(flags.download)); +} + +/** + * Checks the integrity of an encrypted remote, as `rclone cryptcheck`: the + * files of the source, and the files of the crypt remote. rfetch reads and + * decrypts each file of the crypt remote, and compares its bytes with the + * source file. + * @param {string} source + * @param {string} target A crypt remote. + * @param {Object} [flags] + * @returns {Promise} + */ +export async function cryptcheck(source, target, flags = {}) { + try { + const { type } = await remoteOf(target, flags); + if (type !== "crypt") { + throw new OperationError( + `${target} is not a crypt remote`, + 400, + ); + } + } catch (error) { + return failure(error); + } + return await compare(source, target, flags, true); +} + +/** + * Compares the files of two places. + * @param {string} source + * @param {string} target + * @param {Object} flags + * @param {boolean} download Compares the bytes of the files. + * @returns {Promise} + */ +async function compare(source, target, flags, download) { + try { + const [sources, targets] = await Promise.all([ + filesOf(source, flags), + filesOf(target, flags), + ]); + const results = new Results(); + const paths = [...new Set([...sources.keys(), ...targets.keys()])].sort(); + const both = []; + for (const path of paths) { + const src = sources.get(path); + const dst = targets.get(path); + if (src && !dst) { + log("ERROR", path, `file not in ${target}`, flags); + results.differences++; + results.errors++; + results.missingOnDst++; + results.report("missing_on_dst", path); + } else if (!src && dst) { + if (isTrue(flags.one_way)) continue; + log("ERROR", path, `file not in ${source}`, flags); + results.differences++; + results.errors++; + results.missingOnSrc++; + results.report("missing_on_src", path); + } else if (src && dst) { + both.push({ path, src, dst }); + } + } + await each( + both, + numberOf(flags.checkers, 8), + async ({ path, src, dst }) => { + if (src.size >= 0 && dst.size >= 0 && src.size !== dst.size) { + log("ERROR", path, "sizes differ", flags); + results.differences++; + results.errors++; + results.report("differ", path); + return; + } + if (!download) { + results.noHashes++; + results.matches++; + results.report("match", path); + return; + } + try { + if (await sameBytes(src.location, dst.location, flags)) { + results.matches++; + results.report("match", path); + } else { + log("ERROR", path, "contents differ", flags); + results.differences++; + results.errors++; + results.report("differ", path); + } + } catch (error) { + log("ERROR", path, `failed to download: ${error.message}`, flags); + results.errors++; + results.report("error", path); + } + }, + ); + return await summary(results, source, target, flags, "files"); + } catch (error) { + return failure(error); + } +} + +/** + * Tells if two files have the same bytes. + * @param {string} a + * @param {string} b + * @param {Object} flags + */ +async function sameBytes(a, b, flags) { + const [first, second] = await Promise.all([ + hashFile(a, "sha256", flags), + hashFile(b, "sha256", flags), + ]); + return first === second; +} + +/** + * Logs the summary of a check, writes its lists, and gives the answer: an + * error when there are differences. + * @param {Results} results + * @param {string} source + * @param {string} target + * @param {Object} flags + * @param {"files" | "hashes"} missing What the source does not have. + * @returns {Promise} + */ +async function summary(results, source, target, flags, missing) { + if (results.missingOnDst) { + log("NOTICE", target, `${results.missingOnDst} files missing`, flags); + } + if (results.missingOnSrc) { + // As rclone: a sum file is not a place of files. + log( + "NOTICE", + missing === "hashes" ? "" : source, + `${results.missingOnSrc} ${missing} missing`, + flags, + ); + } + log("NOTICE", target, `${results.differences} differences found`, flags); + if (results.errors) { + log("NOTICE", target, `${results.errors} errors while checking`, flags); + } + if (results.noHashes) { + log( + "NOTICE", + target, + `${results.noHashes} hashes could not be checked`, + flags, + ); + } + if (results.matches) { + log("NOTICE", target, `${results.matches} matching files`, flags); + } + + let output = ""; + for (const kind of ["combined", ...Object.keys(REPORTS)]) { + const where = flags[kind]; + if (typeof where !== "string") continue; + const text = results.lists[kind].map((line) => `${line}\n`).join(""); + if (where === "-") { + output += text; + continue; + } + const response = await fetch(where, { + method: "PUT", + headers: { "Content-Length": `${new TextEncoder().encode(text).length}` }, + body: text, + }); + await response.body?.cancel(); + if (!response.ok) { + throw new OperationError( + `failed to write ${where}: ${response.status}`, + response.status, + ); + } + } + // The lists go to the output, and the error to the error output. + if (results.differences) { + return new Response(output, { + status: 500, + statusText: `${results.differences} differences found`, + }); + } + if (results.errors) { + return new Response(output, { + status: 500, + statusText: `${results.errors} errors while checking`, + }); + } + return new Response(output); +} + +/** + * Reads a sum file, as `ParseSumFile` of rclone: on each line, a hash, a + * space, a space or a `*`, and a path. + * @param {string} text + * @param {string} name The name of the sum file, for the log. + * @param {Object} flags + * @returns {Map} + */ +export function parseSums(text, name, flags = {}) { + /** @type {Map} */ + const sums = new Map(); + let warnings = 0; + const lines = text.split(/\r?\n/); + lines.forEach((line, number) => { + if (line === "") return; + const match = line.match(/^([^ ]+) [ *](.+)$/); + if (!match) { + if (++warnings <= 3) { + log( + "NOTICE", + name, + `improperly formatted checksum line ${number}`, + flags, + ); + } + return; + } + const [, sum, file] = match; + if (sums.has(file)) { + if (++warnings <= 3) { + log("NOTICE", name, `duplicate file on checksum line ${number}`, flags); + } + return; + } + sums.set(file, sum.toLowerCase()); + }); + if (warnings > 3) { + log("NOTICE", name, `${warnings - 3} warning(s) suppressed...`, flags); + } + return sums; +} + +/** + * Checks the files in the destination against a SUM file, as `rclone + * checksum`. rfetch reads each file to compute its hash, as `--download` + * of rclone. + * + * ```js + * import { checksum } from "./main.js"; + * await checksum("md5", "/path/to/MD5SUMS", "remote:path"); + * ``` + * @param {string} type The type of the hash, for example `md5` or `SHA-1`. + * @param {string} sumfile The location of the sum file. + * @param {string} target + * @param {Object} [flags] + * @returns {Promise} + */ +export async function checksum(type, sumfile, target, flags = {}) { + try { + const name = hashName(type); + if (!name) { + throw new OperationError(`${type}: hash type is not supported`, 400); + } + const response = await fetch(withFlags(sumfile, flags)); + if (!response.ok) { + await response.body?.cancel(); + throw new OperationError( + `cannot open sum file: ${response.status} ${response.statusText}` + .trim(), + response.status === 404 ? 404 : 400, + ); + } + const sums = parseSums(await response.text(), sumfile, flags); + const files = await filesOf(target, flags); + const results = new Results(); + const base64 = isTrue(flags.base64); + await each( + [...files].sort(), + numberOf(flags.checkers, 8), + async ([path, file]) => { + const sum = sums.get(path); + if (sum === undefined) { + if (isTrue(flags.one_way)) return; + log("ERROR", path, "sum not found", flags); + results.differences++; + results.errors++; + results.missingOnSrc++; + results.report("missing_on_src", path); + return; + } + sums.delete(path); + let hash; + try { + hash = await hashFile(file.location, name, { ...flags, base64 }); + } catch (error) { + log( + "ERROR", + path, + `Failed to calculate hash: ${error.message}`, + flags, + ); + results.errors++; + results.report("error", path); + return; + } + if ((base64 ? hash : hash.toLowerCase()) === sum || hash === sum) { + results.matches++; + results.report("match", path); + } else { + log("ERROR", path, "files differ", flags); + results.differences++; + results.errors++; + results.report("differ", path); + } + }, + ); + for (const path of sums.keys()) { + log("ERROR", path, `file not in ${target}`, flags); + results.errors++; + results.missingOnDst++; + results.report("missing_on_dst", path); + } + return await summary(results, sumfile, target, flags, "hashes"); + } catch (error) { + return failure(error); + } +} diff --git a/cmd/check/main_test.js b/cmd/check/main_test.js new file mode 100644 index 0000000..ee6ca2a --- /dev/null +++ b/cmd/check/main_test.js @@ -0,0 +1,115 @@ +import { beforeEach, describe, test } from "node:test"; +import assert from "node:assert/strict"; + +import { obscure } from "@rfetch/lib/obscure"; +import { fetch } from "../../fetch.js"; +import { captureLog, putFiles } from "../testing.js"; +import { check, checksum, cryptcheck } from "./main.js"; + +const SOURCE = ":memory:/check/source/"; +const TARGET = ":memory:/check/target/"; + +describe("check, checksum and cryptcheck", () => { + beforeEach(async () => { + await (await fetch(":memory:/check/", { method: "DELETE" })).body?.cancel(); + await putFiles(SOURCE, { + "f.txt": "hello", + "a/g.txt": "world!", + "a/zero": "", + "big.txt": "1234", + }); + await putFiles(TARGET, { + "f.txt": "HELLO", + "a/zero": "", + "big.txt": "12345", + "extra": "x", + }); + }); + + test("check compares the sizes, as rclone with no common hash", async (t) => { + const lines = captureLog(t); + const response = await check(SOURCE, TARGET, { combined: "-" }); + assert.equal(response.status, 500); + assert.equal(response.statusText, "3 differences found"); + assert.equal( + await response.text(), + "+ a/g.txt\n- extra\n= a/zero\n* big.txt\n= f.txt\n", + ); + assert.deepEqual(lines, [ + `ERROR : a/g.txt: file not in ${TARGET}`, + `ERROR : extra: file not in ${SOURCE}`, + "ERROR : big.txt: sizes differ", + `NOTICE: ${TARGET}: 1 files missing`, + `NOTICE: ${SOURCE}: 1 files missing`, + `NOTICE: ${TARGET}: 3 differences found`, + `NOTICE: ${TARGET}: 3 errors while checking`, + `NOTICE: ${TARGET}: 2 hashes could not be checked`, + `NOTICE: ${TARGET}: 2 matching files`, + ]); + }); + + test("check --download compares the bytes", async (t) => { + const lines = captureLog(t); + const response = await check(SOURCE, TARGET, { + download: true, + one_way: true, + match: "-", + differ: "-", + }); + assert.equal(response.statusText, "3 differences found"); + assert.ok(lines.includes("ERROR : f.txt: contents differ")); + assert.ok(!lines.some((line) => line.includes("extra"))); + assert.equal(await response.text(), "a/zero\nbig.txt\nf.txt\n"); + }); + + test("check of the same files gives no error", async (t) => { + captureLog(t); + const response = await check(SOURCE, SOURCE, { download: true }); + assert.equal(response.status, 200); + }); + + test("checksum checks the files with a sum file", async (t) => { + const lines = captureLog(t); + await putFiles(":memory:/check/", { + "MD5SUMS": "5d41402abc4b2a76b9719d911017c592 f.txt\n" + + "00000000000000000000000000000000 *a/g.txt\n" + + "d41d8cd98f00b204e9800998ecf8427e gone.txt\n" + + "not a line\n", + }); + const response = await checksum("MD5", ":memory:/check/MD5SUMS", SOURCE); + assert.equal(response.statusText, "3 differences found"); + for ( + const line of [ + "NOTICE: :memory:/check/MD5SUMS: improperly formatted checksum line 3", + "ERROR : a/g.txt: files differ", + "ERROR : a/zero: sum not found", + "ERROR : big.txt: sum not found", + `ERROR : gone.txt: file not in ${SOURCE}`, + "NOTICE: 2 hashes missing", + `NOTICE: ${SOURCE}: 1 matching files`, + ] + ) { + assert.ok(lines.includes(line), `${line}\n${lines.join("\n")}`); + } + }); + + test("cryptcheck compares a crypt remote with its source", async (t) => { + captureLog(t); + const password = await (await obscure("cryptcheck")).text(); + const crypted = + `:crypt,remote=':memory:/check/crypted',password='${password}':`; + await putFiles(crypted, { "f.txt": "hello", "a/g.txt": "world!" }); + await putFiles(":memory:/check/plain/", { + "f.txt": "hello", + "a/g.txt": "world!", + }); + assert.equal( + (await cryptcheck(":memory:/check/plain/", crypted)).status, + 200, + ); + await putFiles(crypted, { "f.txt": "HELLO" }); + const differ = await cryptcheck(":memory:/check/plain/", crypted); + assert.equal(differ.statusText, "1 differences found"); + assert.equal((await cryptcheck(SOURCE, TARGET)).status, 400); + }); +}); diff --git a/cmd/copy/main.js b/cmd/copy/main.js index 834e261..79d53bd 100644 --- a/cmd/copy/main.js +++ b/cmd/copy/main.js @@ -1,31 +1,266 @@ -import { fetch } from "../../fetch.js"; /** - * Copy files from source to dest, skipping identical files. + * The copy of files, as `rclone copy` and `rclone copyto`. `move` and + * `moveto` use it too. * - * Copy the source to the destination. Does not transfer files that are - * identical on source and destination, testing by size and modification time - * or MD5SUM. Doesn't delete files from the destination. If you want to also - * delete files from destination, to make it match source, use the `sync` - * command instead. + * A file goes when the target has no file with its name, or when the two + * files differ, as rclone: another size, or another time. A time that a + * backend does not give is not compared. The flags of rclone: * - * Note that it is always the contents of the directory that is synced, not - * the directory itself. So when `source:path` is a directory, it's the - * contents of source:path that are copied, not the directory name and - * contents. + * - `--size-only`: compare only the sizes. + * - `--ignore-times`: copy each file, also the same files. + * - `--ignore-existing`: copy no file that the target has. + * - `--create-empty-src-dirs`: make the folders with no file too. + * - `--dry-run`: copy nothing, and tell what it would copy. + * - `--transfers N`: copy N files at the same time, 4 by default. * - * To copy single files, use the `copyto` command instead. - * - * If `dest:path` doesn't exist, it is created and the source:path contents - * go there. + * The backends do not keep the time of a source file yet: the time of a + * copy is the time of the copy. Thus a second copy sends the files again, + * unless `--size-only`. + */ +import { isTrue } from "@rfetch/lib/options"; + +import { + copyFile, + each, + failure, + folderOf, + join, + log, + mkdir, + nameOf, + numberOf, + OperationError, + remove, + stat, + walk, +} from "../operations.js"; + +/** + * Copy files from source to dest, skipping identical files, as `rclone + * copy`. The contents of the source folder go in the dest folder. A source + * file goes in the dest folder, with its name. * + * ```js + * import { copy } from "./main.js"; + * await copy("source:path", "dest:path"); + * ``` + * @param {string} source + * @param {string} target + * @param {Object} [flags] + * @returns {Promise} + */ +export async function copy(source, target, flags = {}) { + try { + await transfer(source, target, flags); + return new Response(null); + } catch (error) { + return failure(error); + } +} + +/** + * Copy files from source to dest, skipping identical files, as `rclone + * copyto`: a source file goes to the path of dest, with the name of dest. + * A source folder is as `copy`. * @param {string} source * @param {string} target * @param {Object} [flags] * @returns {Promise} */ -export async function copy(source, target, flags) { - /** @TODO: Handle copy folder */ - const params = new URLSearchParams(flags); - const { body } = await fetch(`${source}?${params}`); - return fetch(`${target}?${params}`, { method: "PUT", body }); +export async function copyto(source, target, flags = {}) { + try { + await transfer(source, target, flags, { to: true }); + return new Response(null); + } catch (error) { + return failure(error); + } +} + +/** + * A file of a transfer. + * @typedef {Object} File + * @property {string} path The path from the folder of the transfer. + * @property {string} location + * @property {number} size + * @property {string} modTime + */ + +/** + * Copies, or moves, a file or a folder. + * @param {string} source + * @param {string} target + * @param {Object} flags + * @param {{ move?: boolean, to?: boolean }} [mode] `move` deletes each source + * file after its copy. `to` puts a source file at the path of the target, + * not in the target folder. + * @throws {OperationError} + */ +export async function transfer(source, target, flags, mode = {}) { + const info = await stat(source, flags); + if (!info) throw new OperationError("directory not found", 404); + if (!info.isDir) { + const location = mode.to ? target : join(target, nameOf(source)); + if (location === source) { + throw new OperationError("can't copy or move a file onto itself", 400); + } + const existing = await stat(location, flags); + await transferFile( + { + path: nameOf(location), + location: source, + size: info.size, + modTime: info.modTime, + }, + location, + existing?.isDir ? null : existing, + flags, + mode, + ); + return; + } + + /** @type {File[]} */ + const files = []; + /** @type {string[]} */ + const folders = []; + for await (const item of await walk(source, flags)) { + if (item.IsDir) folders.push(item.Path); + else { + files.push({ + path: item.Path, + location: item.location, + size: item.Size, + modTime: item.ModTime, + }); + } + } + /** @type {Map} */ + const existing = new Map(); + try { + for await (const item of await walk(target, flags)) { + if (!item.IsDir) existing.set(item.Path, item); + } + } catch (error) { + // A target that does not exist gets all the files. + if (!(error instanceof OperationError && error.status === 404)) throw error; + } + + if (isTrue(flags.create_empty_src_dirs) && !isTrue(flags.dry_run)) { + await mkdir(target, flags); + for (const path of folders) await mkdir(join(target, path, true), flags); + } + + let errors = 0; + let last = ""; + await each(files, numberOf(flags.transfers, 4), async (file) => { + const other = existing.get(file.path); + try { + await transferFile( + file, + join(target, file.path), + other + ? { isDir: false, size: other.Size, modTime: other.ModTime } + : null, + flags, + mode, + ); + } catch (error) { + errors++; + last = error.message; + log( + "ERROR", + file.path, + `Failed to ${mode.move ? "move" : "copy"}: ${error.message}`, + flags, + ); + } + }); + if (errors) { + throw new OperationError( + `failed to ${ + mode.move ? "move" : "copy" + } ${errors} files: last error was: ${last}`, + ); + } + if ( + mode.move && isTrue(flags.delete_empty_src_dirs) && !isTrue(flags.dry_run) + ) { + await removeFolders(source, folders, flags); + } +} + +/** + * Copies, or moves, a file, unless the target has the same file. + * @param {File} file + * @param {string} target The location of the target file. + * @param {import("../operations.js").Info | null} existing The target file, + * or null. + * @param {Object} flags + * @param {{ move?: boolean }} mode + */ +async function transferFile(file, target, existing, flags, mode) { + const verb = mode.move ? "move" : "copy"; + const same = existing && (isTrue(flags.ignore_existing) || + (!isTrue(flags.ignore_times) && isSame(file, existing, flags))); + if (isTrue(flags.dry_run)) { + if (!same || mode.move) { + log( + "NOTICE", + file.path, + `Skipped ${verb} as --dry-run is set (size ${file.size})`, + flags, + ); + } + return; + } + if (!same) await copyFile(file.location, target, flags); + // As rclone: a move deletes the source, also when the target has the same + // file. + if (mode.move) await remove(file.location, flags); +} + +/** + * Tells if two files are the same, as `Equal` of rclone: the same size, and + * the same time in the precision of the two remotes. A time with no part of + * a second comes from a remote with a precision of 1 s, as WebDAV: then the + * times can differ by less than 1 s. Else they can differ by less than 1 ms. + * A time that a backend does not give is not compared. + * @param {{ size: number, modTime: string }} a + * @param {{ size: number, modTime: string }} b + * @param {Object} flags + */ +function isSame(a, b, flags) { + if (a.size >= 0 && b.size >= 0 && a.size !== b.size) return false; + if (isTrue(flags.size_only)) return true; + const first = Date.parse(a.modTime); + const second = Date.parse(b.modTime); + if (isNaN(first) || isNaN(second)) return true; + const fraction = /:\d\d\.\d/; + const window = fraction.test(a.modTime) && fraction.test(b.modTime) + ? 1 + : 1000; + return Math.abs(first - second) < window; +} + +/** + * Deletes the folders of a source that have no file after a move, from the + * deepest folder up. The folder of the source stays. + * @param {string} source + * @param {string[]} folders + * @param {Object} flags + */ +async function removeFolders(source, folders, flags) { + const deepest = folders.toSorted((a, b) => + b.split("/").length - a.split("/").length + ); + for (const path of deepest) { + const location = join(folderOf(source), path, true); + // A folder with a file that did not move stays. + let empty = true; + for await (const _ of await walk(location, flags, { maxDepth: 1 })) { + empty = false; + break; + } + if (empty) await remove(location, flags); + } } diff --git a/cmd/copy/main_test.js b/cmd/copy/main_test.js index cbfba36..683d33d 100644 --- a/cmd/copy/main_test.js +++ b/cmd/copy/main_test.js @@ -1,7 +1,89 @@ -import { test } from "node:test"; +import { beforeEach, describe, test } from "node:test"; import assert from "node:assert/strict"; -import { copy } from "./main.js"; -test("copy", () => { - assert(copy); +import { fetch } from "../../fetch.js"; +import { captureLog, contents, putFiles } from "../testing.js"; +import { copy, copyto } from "./main.js"; + +const SOURCE = ":memory:/copy/source/"; +const TARGET = ":memory:/copy/target/"; + +describe("copy and copyto", () => { + beforeEach(async () => { + await (await fetch(":memory:/copy/", { method: "DELETE" })).body?.cancel(); + await putFiles(SOURCE, { + "f.txt": "hello", + "a/g.txt": "world!", + "a/b/zero": "", + "n?m/#1.txt": "name", + }); + await (await fetch(`${SOURCE}empty/`, { method: "PUT" })).body?.cancel(); + }); + + test("copy puts the contents of a folder in the target", async () => { + assert.equal((await copy(SOURCE, TARGET)).status, 200); + assert.deepEqual(await contents(TARGET), { + files: { + "f.txt": "hello", + "a/g.txt": "world!", + "a/b/zero": "", + "n?m/#1.txt": "name", + }, + folders: ["a", "a/b", "n?m"], + }); + }); + + test("copy --create-empty-src-dirs makes the empty folders too", async () => { + await copy(SOURCE, TARGET, { create_empty_src_dirs: true }); + assert.ok((await contents(TARGET)).folders.includes("empty")); + }); + + test("copy of a file puts it in the target folder", async () => { + assert.equal((await copy(`${SOURCE}a/g.txt`, TARGET)).status, 200); + assert.deepEqual((await contents(TARGET)).files, { "g.txt": "world!" }); + }); + + test("copyto gives the file the name of the target", async () => { + assert.equal( + (await copyto(`${SOURCE}a/g.txt`, `${TARGET}x/new.txt`)).status, + 200, + ); + assert.deepEqual((await contents(TARGET)).files, { "x/new.txt": "world!" }); + const self = await copyto(`${SOURCE}f.txt`, `${SOURCE}f.txt`); + assert.equal(self.status, 400); + }); + + test("copy skips the same files, as rclone", async () => { + await putFiles(TARGET, { "f.txt": "HELLO", "a/g.txt": "other!!" }); + // Same size: with --size-only, the file is the same. + await copy(SOURCE, TARGET, { size_only: true }); + let files = (await contents(TARGET)).files; + assert.equal(files["f.txt"], "HELLO"); + assert.equal(files["a/g.txt"], "world!"); + + // The times differ: the file goes again. + await copy(SOURCE, TARGET); + files = (await contents(TARGET)).files; + assert.equal(files["f.txt"], "hello"); + + await putFiles(TARGET, { "f.txt": "HELLO" }); + await copy(SOURCE, TARGET, { ignore_existing: true }); + assert.equal((await contents(TARGET)).files["f.txt"], "HELLO"); + }); + + test("copy --dry-run copies nothing", async (t) => { + const lines = captureLog(t); + assert.equal((await copy(SOURCE, TARGET, { dry_run: true })).status, 200); + assert.equal(await contents(TARGET), null); + assert.ok( + lines.includes( + "NOTICE: f.txt: Skipped copy as --dry-run is set (size 5)", + ), + lines.join("\n"), + ); + }); + + test("copy of a source that does not exist gives 404", async () => { + assert.equal((await copy(`${SOURCE}missing`, TARGET)).status, 404); + }); }); diff --git a/cmd/cryptdecode/main.js b/cmd/cryptdecode/main.js new file mode 100644 index 0000000..aac8def --- /dev/null +++ b/cmd/cryptdecode/main.js @@ -0,0 +1,50 @@ +import { nameCipher } from "@rfetch/crypt"; +import { isTrue } from "@rfetch/lib/options"; + +import { failure, OperationError, remoteOf } from "../operations.js"; + +/** + * Cryptdecode returns unencrypted file names, as `rclone cryptdecode`: a + * line for each name, with the name, a tab and the decrypted name, or + * `Failed to decrypt`. With `--reverse`, it encrypts the names. + * + * ```js + * import { cryptdecode } from "./main.js"; + * await cryptdecode("encryptedremote:", "encryptedname1", {}); + * ``` + * @param {string} remote A crypt remote. + * @param {...(string | Object)} args The names, then the flags. + * @returns {Promise} + */ +export async function cryptdecode(remote, ...args) { + const flags = typeof args.at(-1) === "object" ? args.pop() : {}; + try { + const { type, options } = await remoteOf(remote, flags); + if (type !== "crypt") { + throw new OperationError( + 'the remote needs to be of type "crypt"', + 400, + ); + } + if (!options.password) throw new OperationError("Missing password", 400); + let names; + try { + names = await nameCipher(options); + } catch { + throw new OperationError("The password is not obscured", 400); + } + const reverse = isTrue(flags.reverse); + const lines = args.map((name) => { + try { + return reverse + ? `${name} \t ${names.encrypt(name)}` + : `${name} \t ${names.decrypt(name)}`; + } catch { + return `${name} \t Failed to ${reverse ? "encrypt" : "decrypt"}`; + } + }); + return new Response(lines.map((line) => `${line}\n`).join("")); + } catch (error) { + return failure(error); + } +} diff --git a/cmd/cryptdecode/main_test.js b/cmd/cryptdecode/main_test.js new file mode 100644 index 0000000..cbb8c99 --- /dev/null +++ b/cmd/cryptdecode/main_test.js @@ -0,0 +1,37 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { obscure } from "@rfetch/lib/obscure"; +import { fetch } from "../../fetch.js"; +import { contents, putFiles } from "../testing.js"; +import { cryptdecode } from "./main.js"; + +test("cryptdecode decrypts and encrypts the names of a crypt remote", async () => { + const password = await (await obscure("cryptdecode")).text(); + const remote = `:crypt,remote=':memory:/cryptdecode',password='${password}':`; + await putFiles(remote, { "a/g.txt": "world!" }); + const [encrypted] = Object.keys( + (await contents(":memory:/cryptdecode/")).files, + ); + const [folder, file] = encrypted.split("/"); + + assert.equal( + await (await cryptdecode(remote, folder, encrypted, "bad-name", {})).text(), + `${folder} \t a\n${encrypted} \t a/g.txt\nbad-name \t Failed to decrypt\n`, + ); + assert.equal( + await (await cryptdecode(remote, "a/g.txt", { reverse: true })).text(), + `a/g.txt \t ${folder}/${file}\n`, + ); + // The options can come in the flags of the command too. + const flagged = await cryptdecode(":crypt:", folder, { + crypt_remote: ":memory:/cryptdecode", + crypt_password: password, + }); + assert.equal(await flagged.text(), `${folder} \t a\n`); + + const plain = await cryptdecode(":memory:", "x", {}); + assert.equal(plain.status, 400); + await (await fetch(":memory:/cryptdecode/", { method: "DELETE" })).body + ?.cancel(); +}); diff --git a/cmd/delete/main.js b/cmd/delete/main.js new file mode 100644 index 0000000..27869c4 --- /dev/null +++ b/cmd/delete/main.js @@ -0,0 +1,258 @@ +/** + * The commands that delete, as rclone: `delete`, `deletefile`, `purge`, + * `rmdir` and `rmdirs`. With `--dry-run`, they delete nothing, and tell + * what they would delete. + */ +import { isTrue } from "@rfetch/lib/options"; + +import { fetch } from "../../fetch.js"; +import { + each, + failure, + folderOf, + log, + numberOf, + OperationError, + remove, + stat, + walk, + withFlags, +} from "../operations.js"; + +/** + * Remove the files in the path, as `rclone delete`. The folders stay, unless + * `--rmdirs`: then the folders with no file go too, but not the folder of the + * path. A path of a file deletes only that file. + * + * ```js + * import { delete as remove } from "./main.js"; + * await remove("remote:path", { rmdirs: true }); + * ``` + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +async function deleteFiles(location, flags = {}) { + try { + const info = await stat(location, flags); + if (!info) throw new OperationError("directory not found", 404); + if (!info.isDir) { + await deleteOne(location, location, info.size, flags); + return new Response(null); + } + const files = []; + for await (const item of await walk(location, flags)) { + if (!item.IsDir) files.push(item); + } + let errors = 0; + let last = ""; + await each(files, numberOf(flags.checkers, 8), async (item) => { + try { + await deleteOne(item.location, item.Path, item.Size, flags); + } catch (error) { + errors++; + last = error.message; + log("ERROR", item.Path, `Couldn't delete: ${error.message}`, flags); + } + }); + if (errors) { + throw new OperationError( + `failed to delete ${errors} files: last error was: ${last}`, + ); + } + if (isTrue(flags.rmdirs)) { + await removeEmptyFolders(location, flags, true); + } + return new Response(null); + } catch (error) { + return failure(error); + } +} + +export { deleteFiles as delete }; + +/** + * Deletes a file, or tells it with `--dry-run`. + * @param {string} location + * @param {string} name The name for the log. + * @param {number} size + * @param {Object} flags + */ +async function deleteOne(location, name, size, flags) { + if (isTrue(flags.dry_run)) { + log( + "NOTICE", + name, + `Skipped delete as --dry-run is set (size ${size})`, + flags, + ); + return; + } + await remove(location, flags); +} + +/** + * Remove a single file from remote, as `rclone deletefile`. A folder, or a + * path that does not exist, is an error. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function deletefile(location, flags = {}) { + try { + const info = await stat(location, flags); + if (!info || info.isDir) { + throw new OperationError( + `${location} is a directory or doesn't exist: object not found`, + 404, + ); + } + await deleteOne(location, location, info.size, flags); + return new Response(null); + } catch (error) { + return failure(error); + } +} + +/** + * Remove the path and all of its contents, as `rclone purge`. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function purge(location, flags = {}) { + try { + const info = await stat(location, flags); + if (!info) throw new OperationError("directory not found", 404); + if (!info.isDir) { + throw new OperationError(`${location} is a file not a directory`, 400); + } + if (isTrue(flags.dry_run)) { + log("NOTICE", location, "Skipped purge as --dry-run is set", flags); + } else { + await remove(folderOf(location), flags); + } + return new Response(null); + } catch (error) { + return failure(error); + } +} + +/** + * Remove the empty folder of the path, as `rclone rmdir`. A folder with a + * file or a folder in it is an error. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function rmdir(location, flags = {}) { + try { + const folder = folderOf(location); + const response = await fetch(withFlags(folder, flags)); + if (response.status === 404) { + await response.body?.cancel(); + throw new OperationError(`${location}: directory not found`, 404); + } + if (!response.ok) { + await response.body?.cancel(); + throw new OperationError( + `${location}: ${response.status} ${response.statusText}`.trim(), + response.status, + ); + } + await response.body?.cancel(); + for await (const _ of await walk(folder, flags, { maxDepth: 1 })) { + throw new OperationError(`remove ${location}: directory not empty`, 409); + } + if (isTrue(flags.dry_run)) { + log( + "NOTICE", + location, + "Skipped remove directory as --dry-run is set", + flags, + ); + } else { + await remove(folder, flags); + } + return new Response(null); + } catch (error) { + return failure(error); + } +} + +/** + * Remove the folders with no file in the path, as `rclone rmdirs`: a folder + * with only folders that have no file goes too. The folder of the path goes + * when it has no file, unless `--leave-root`. The backends delete a folder + * with all that is in it: thus a file that comes into such a folder during + * the command goes with the folder. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function rmdirs(location, flags = {}) { + try { + await removeEmptyFolders(location, flags, isTrue(flags.leave_root)); + return new Response(null); + } catch (error) { + return failure(error); + } +} + +/** + * Deletes the folders with no file in a folder. + * @param {string} location + * @param {Object} flags + * @param {boolean} leaveRoot When true, the folder of the location stays. + */ +async function removeEmptyFolders(location, flags, leaveRoot) { + /** @type {Map} The locations of the folders, by path. */ + const folders = new Map(); + /** The folders with a file in them, also below. */ + const full = new Set([""]); + let hasFile = false; + for await (const item of await walk(location, flags)) { + if (item.IsDir) { + folders.set(item.Path, item.location); + continue; + } + hasFile = true; + const names = item.Path.split("/"); + for (let i = 1; i < names.length; i++) { + full.add(names.slice(0, i).join("/")); + } + } + const dryRun = isTrue(flags.dry_run); + if (!hasFile && !leaveRoot) { + if (dryRun) { + log( + "NOTICE", + location, + "Skipped remove directory as --dry-run is set", + flags, + ); + } else { + await remove(folderOf(location), flags); + } + return; + } + // The highest folders with no file: the delete of a folder deletes the + // folders in it. + for (const [path, folder] of [...folders].sort()) { + if (full.has(path)) continue; + const parent = path.includes("/") + ? path.slice(0, path.lastIndexOf("/")) + : ""; + if (!full.has(parent)) continue; + if (dryRun) { + log( + "NOTICE", + path, + "Skipped remove directory as --dry-run is set", + flags, + ); + } else { + await remove(folder, flags); + } + } +} diff --git a/cmd/delete/main_test.js b/cmd/delete/main_test.js new file mode 100644 index 0000000..c40ac31 --- /dev/null +++ b/cmd/delete/main_test.js @@ -0,0 +1,104 @@ +import { beforeEach, describe, test } from "node:test"; +import assert from "node:assert/strict"; + +import { fetch } from "../../fetch.js"; +import { captureLog, contents, putFiles } from "../testing.js"; +import { delete as remove, deletefile, purge, rmdir, rmdirs } from "./main.js"; + +const ROOT = ":memory:/delete/"; + +describe("the commands that delete", () => { + beforeEach(async () => { + await (await fetch(ROOT, { method: "DELETE" })).body?.cancel(); + await putFiles(ROOT, { + "f.txt": "hello", + "a/g.txt": "world!", + "a/b/zero": "", + "a?b/#c.txt": "x", + }); + await (await fetch(`${ROOT}empty/inner/`, { method: "PUT" })).body + ?.cancel(); + }); + + test("delete removes the files, and keeps the folders", async () => { + assert.equal((await remove(ROOT)).status, 200); + assert.deepEqual(await contents(ROOT), { + files: {}, + folders: ["a", "a/b", "a?b", "empty", "empty/inner"], + }); + }); + + test("delete --rmdirs removes the folders too, but not the root", async () => { + assert.equal((await remove(ROOT, { rmdirs: true })).status, 200); + assert.deepEqual(await contents(ROOT), { files: {}, folders: [] }); + }); + + test("delete of a file removes only that file", async () => { + assert.equal((await remove(`${ROOT}a/g.txt`)).status, 200); + assert.deepEqual(Object.keys((await contents(ROOT)).files).sort(), [ + "a/b/zero", + "a?b/#c.txt", + "f.txt", + ]); + }); + + test("delete --dry-run removes nothing, and tells what", async (t) => { + const lines = captureLog(t); + assert.equal((await remove(ROOT, { dry_run: true })).status, 200); + assert.equal(Object.keys((await contents(ROOT)).files).length, 4); + assert.ok( + lines.includes( + "NOTICE: f.txt: Skipped delete as --dry-run is set (size 5)", + ), + lines.join("\n"), + ); + }); + + test("deletefile removes a file, and refuses a folder", async () => { + assert.equal((await deletefile(`${ROOT}f.txt`)).status, 200); + for (const path of ["a", "missing.txt"]) { + const response = await deletefile(`${ROOT}${path}`); + assert.equal(response.status, 404); + assert.match(await response.text(), /object not found$/); + } + }); + + test("purge removes a folder and all in it", async () => { + assert.equal((await purge(`${ROOT}a`)).status, 200); + assert.deepEqual((await contents(ROOT)).folders, [ + "a?b", + "empty", + "empty/inner", + ]); + assert.equal((await purge(`${ROOT}f.txt`)).status, 400); + assert.equal((await purge(`${ROOT}missing`)).status, 404); + }); + + test("rmdir removes only an empty folder", async () => { + const full = await rmdir(`${ROOT}a`); + assert.equal(full.status, 409); + assert.match(await full.text(), /directory not empty/); + assert.equal((await rmdir(`${ROOT}empty/inner`)).status, 200); + assert.equal((await rmdir(`${ROOT}missing`)).status, 404); + assert.deepEqual((await contents(ROOT)).folders, [ + "a", + "a/b", + "a?b", + "empty", + ]); + }); + + test("rmdirs removes the folders with no file", async () => { + await (await fetch(`${ROOT}a/b/c/`, { method: "PUT" })).body?.cancel(); + assert.equal((await rmdirs(ROOT)).status, 200); + assert.deepEqual((await contents(ROOT)).folders, ["a", "a/b", "a?b"]); + + // The root goes when it has no file, unless --leave-root. + await (await fetch(ROOT, { method: "DELETE" })).body?.cancel(); + await (await fetch(`${ROOT}x/y/`, { method: "PUT" })).body?.cancel(); + assert.equal((await rmdirs(ROOT, { leave_root: true })).status, 200); + assert.deepEqual(await contents(ROOT), { files: {}, folders: [] }); + assert.equal((await rmdirs(ROOT)).status, 200); + assert.equal(await contents(ROOT), null); + }); +}); diff --git a/cmd/flags.js b/cmd/flags.js index cda2dad..7edcb40 100644 --- a/cmd/flags.js +++ b/cmd/flags.js @@ -32,29 +32,47 @@ function variable(name) { export const FLAGS = { // `serve` "addr": { type: "string" }, - // `serve webdav` - "pass": { type: "string" }, - // `serve webdav` - "read-only": { type: "boolean" }, - // `serve webdav` - "user": { type: "string" }, + // `tree` + "all": { type: "boolean" }, // `copyurl` "auto-filename": { type: "boolean", short: "a" }, + // `hashsum`, `checksum` + "base64": { type: "boolean" }, + // The checks, the deletes and the sums + "checkers": { type: "string", default: variable("CHECKERS") || "8" }, + // `hashsum`, `md5sum`, `sha1sum` + "checkfile": { type: "string" }, + // `check` + "combined": { type: "string" }, + // `copy`, `move` + "create-empty-src-dirs": { type: "boolean" }, // `lsf` "csv": { type: "boolean" }, + // `move` + "delete-empty-src-dirs": { type: "boolean" }, + // `check` + "differ": { type: "string" }, // `lsf` "dir-slash": { type: "boolean", short: "d" }, - // `lsf` + // `tree` + "dirsfirst": { type: "boolean" }, + // `lsf`, `tree` "dirs-only": { type: "boolean" }, + // `check`, `checksum`, the sums + "download": { type: "boolean" }, "dry-run": { type: "boolean", short: "n", default: variable("DRY_RUN") === "true", }, + // `check` + "error": { type: "string" }, // `lsf` "files-only": { type: "boolean" }, // `lsf` "format": { type: "string", short: "F" }, + // `tree` + "full-path": { type: "boolean" }, "header": { type: "string", multiple: true }, "header-download": { type: "string", multiple: true }, // `copyurl` @@ -62,10 +80,44 @@ export const FLAGS = { "header-upload": { type: "string", multiple: true }, "help": { type: "boolean", short: "h" }, "human-readable": { type: "boolean" }, + // `copy`, `move` + "ignore-existing": { type: "boolean" }, + // `copy`, `move` + "ignore-times": { type: "boolean", short: "I" }, + // `size`, `listremotes` + "json": { type: "boolean" }, + // `rmdirs` + "leave-root": { type: "boolean" }, + // `tree` + "level": { type: "string" }, + // `touch` + "localtime": { type: "boolean" }, + // `listremotes` + "long": { type: "boolean" }, + // `check` + "match": { type: "string" }, // The lists "max-depth": { type: "string" }, + // `check` + "missing-on-dst": { type: "string" }, + // `check` + "missing-on-src": { type: "string" }, + // `tree` + "modtime": { type: "boolean" }, // `copyurl` "no-clobber": { type: "boolean" }, + // `touch` + "no-create": { type: "boolean", short: "C" }, + // `tree` + "noindent": { type: "boolean" }, + // `tree` + "noreport": { type: "boolean" }, + // `check`, `checksum` + "one-way": { type: "boolean" }, + // `hashsum` + "output-file": { type: "string" }, + // `serve webdav` + "pass": { type: "string" }, // `copyurl` "print-filename": { type: "boolean", short: "p" }, "progress": { @@ -73,13 +125,37 @@ export const FLAGS = { short: "P", default: variable("PROGRESS") === "true", }, - // The lists + // Only the errors in the log. + "quiet": { type: "boolean", short: "q" }, + // `tree` + "quote": { type: "boolean" }, + // `serve webdav` + "read-only": { type: "boolean" }, + // The lists, `touch` "recursive": { type: "boolean", short: "R" }, + // `cryptdecode` + "reverse": { type: "boolean" }, + // `delete` + "rmdirs": { type: "boolean" }, // `lsf` "separator": { type: "string", short: "s" }, + // `tree` + "size": { type: "boolean" }, + // `copy`, `move`, `check` + "size-only": { type: "boolean" }, + // `tree` + "sort": { type: "string" }, + // `tree` + "sort-modtime": { type: "boolean" }, + // `tree` + "sort-reverse": { type: "boolean", short: "r" }, // `copyurl` "stdout": { type: "boolean" }, + // `touch` + "timestamp": { type: "string", short: "t" }, "transfers": { type: "string", default: variable("TRANSFERS") || "4" }, + // `serve webdav` + "user": { type: "string" }, }; /** diff --git a/cmd/folders.js b/cmd/folders.js index 371e3ff..ead6da6 100644 --- a/cmd/folders.js +++ b/cmd/folders.js @@ -5,19 +5,40 @@ export const FOLDERS = { backend: "backend", cat: "cat", + check: "check", + checksum: "check", config: "config", copy: "copy", + copyto: "copy", copyurl: "copyurl", + cryptcheck: "check", + cryptdecode: "cryptdecode", + delete: "delete", + deletefile: "delete", + hashsum: "hashsum", help: "help", + listremotes: "listremotes", ls: "ls", lsd: "lsd", lsf: "lsf", lsjson: "lsjson", lsl: "lsl", + md5sum: "hashsum", + mkdir: "mkdir", + move: "move", + moveto: "move", obscure: "obscure", + purge: "delete", rcat: "rcat", reveal: "obscure", + rmdir: "delete", + rmdirs: "delete", serve: "serve", + sha1sum: "hashsum", + size: "size", + touch: "touch", + tree: "tree", + version: "version", }; /** @@ -30,13 +51,32 @@ export const FOLDERS = { export const PATHS = { backend: [1], cat: [0], + check: [0, 1], + checksum: [1, 2], copy: [0, 1], + copyto: [0, 1], copyurl: [1], + cryptcheck: [0, 1], + cryptdecode: [0], + delete: [0], + deletefile: [0], + hashsum: [1], ls: [0], lsd: [0], lsf: [0], lsjson: [0], lsl: [0], + md5sum: [0], + mkdir: [0], + move: [0, 1], + moveto: [0, 1], + purge: [0], rcat: [0], + rmdir: [0], + rmdirs: [0], serve: [1], + sha1sum: [0], + size: [0], + touch: [0], + tree: [0], }; diff --git a/cmd/hashsum/main.js b/cmd/hashsum/main.js new file mode 100644 index 0000000..150d323 --- /dev/null +++ b/cmd/hashsum/main.js @@ -0,0 +1,229 @@ +/** + * The sums of the files, as `rclone hashsum`, `rclone md5sum` and `rclone + * sha1sum`. The backends of rfetch give no hash: thus the commands read each + * file and compute its hash, as `--download` of rclone. + */ +import { createHash } from "node:crypto"; +import { crc32 } from "@rfetch/lib/crc32"; +import { isTrue } from "@rfetch/lib/options"; + +import { fetch } from "../../fetch.js"; +import { + each, + failure, + nameOf, + numberOf, + OperationError, + stat, + walk, + withFlags, +} from "../operations.js"; + +/** + * The hashes of rfetch, by their names in rclone. The name of `node:crypto`, + * or null for CRC-32. + * @type {Record} + */ +const HASHES = { + md5: "md5", + sha1: "sha1", + crc32: null, + sha256: "sha256", + sha512: "sha512", +}; + +/** + * Gives the name of a hash in rclone, for a name of a person: for example + * `MD5`, `SHA-1` or `sha256`. + * @param {string} name + * @returns {string | null} null for a hash that rfetch does not know. + */ +export function hashName(name) { + const key = `${name}`.toLowerCase().replaceAll("-", ""); + return Object.hasOwn(HASHES, key) ? key : null; +} + +/** + * Computes the hash of a stream. + * @param {ReadableStream} stream + * @param {string} type A name of `hashName`. + * @param {boolean} [base64] Gives the hash in base64, not in hexadecimal. + * @returns {Promise} + */ +export async function hashStream(stream, type, base64 = false) { + /** @type {Uint8Array} */ + let bytes; + if (HASHES[type] === null) { + let crc = 0; + for await (const chunk of stream) crc = crc32(chunk, crc); + bytes = new Uint8Array(4); + new DataView(bytes.buffer).setUint32(0, crc); + } else { + const hash = createHash(HASHES[type]); + for await (const chunk of stream) hash.update(chunk); + bytes = hash.digest(); + } + return Buffer.from(bytes).toString(base64 ? "base64" : "hex"); +} + +/** + * Computes the hash of a file of a remote. + * @param {string} location + * @param {string} type + * @param {Object} flags With `base64`. + * @returns {Promise} + * @throws {OperationError} When the remote gives an error. + */ +export async function hashFile(location, type, flags = {}) { + const response = await fetch(withFlags(location, flags)); + if (!response.ok) { + await response.body?.cancel(); + throw new OperationError( + `${response.status} ${response.statusText}`.trim(), + response.status, + ); + } + return await hashStream( + response.body ?? new Blob([]).stream(), + type, + isTrue(flags.base64), + ); +} + +/** + * Produces a hashsum file for all the objects in the path, as `rclone + * hashsum`: a line for each file, with its hash, two spaces and its path. + * With no hash type, it gives the hashes that rfetch knows. + * + * The flags of rclone: `--base64`, `--output-file` (the location of a file + * for the lines), `--checkfile` (the location of a sum file to check the + * files with, as `checksum` with `--combined -`) and `--download` (always, + * in rfetch). + * + * ```js + * import { hashsum } from "./main.js"; + * await hashsum("SHA-256", "remote:path"); + * ``` + * @param {string | Object} [type] + * @param {string | Object} [location] + * @param {Object} [flags] + * @returns {Promise} + */ +export async function hashsum(type, location, flags = {}) { + if (typeof type !== "string") { + return new Response( + `Supported hashes are:\n${ + Object.keys(HASHES).map((name) => `- ${name}\n`).join("") + }`, + ); + } + if (typeof location !== "string") { + return new Response("Command hashsum needs 2 arguments", { status: 400 }); + } + const name = hashName(type); + if (!name) { + return new Response( + `${type}: hash type is not supported, use one of: ${ + Object.keys(HASHES).join(", ") + }`, + { status: 400 }, + ); + } + return await sums(name, location, flags); +} + +/** + * Produces an md5sum file for all the objects in the path, as `rclone + * md5sum`. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export function md5sum(location, flags = {}) { + return sums("md5", location, flags); +} + +/** + * Produces an sha1sum file for all the objects in the path, as `rclone + * sha1sum`. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export function sha1sum(location, flags = {}) { + return sums("sha1", location, flags); +} + +/** + * Gives the sums of the files of a path. + * @param {string} type + * @param {string} location + * @param {Object} flags + * @returns {Promise} + */ +async function sums(type, location, flags) { + if (typeof flags.checkfile === "string") { + // As rclone: the list of the check goes to the output. + const { checksum } = await import("../check/main.js"); + return checksum(type, flags.checkfile, location, { + combined: "-", + ...flags, + }); + } + try { + const info = await stat(location, flags); + if (!info) throw new OperationError("directory not found", 404); + /** @type {{ path: string, location: string }[]} */ + const files = []; + if (info.isDir) { + for await (const item of await walk(location, flags)) { + if (!item.IsDir) { + files.push({ path: item.Path, location: item.location }); + } + } + } else { + files.push({ path: nameOf(location), location }); + } + let text = ""; + let errors = 0; + let last = ""; + await each(files, numberOf(flags.checkers, 8), async (file) => { + try { + // The hash first: else `text` is the text before the wait. + const hash = await hashFile(file.location, type, flags); + text += `${hash} ${file.path}\n`; + } catch (error) { + errors++; + last = error.message; + // As rclone: the line of a file that fails has the error. + text += `${"ERROR".padEnd(type === "crc32" ? 8 : 32)} ${file.path}\n`; + } + }); + if (typeof flags.output_file === "string") { + const response = await fetch(flags.output_file, { + method: "PUT", + headers: { + "Content-Length": `${new TextEncoder().encode(text).length}`, + }, + body: text, + }); + await response.body?.cancel(); + if (!response.ok) { + throw new OperationError( + `failed to write the output file: ${response.status}`, + response.status, + ); + } + text = ""; + } + if (errors) { + return new Response(text, { + status: 500, + statusText: `failed to hash ${errors} files: last error was: ${last}`, + }); + } + return new Response(text); + } catch (error) { + return failure(error); + } +} diff --git a/cmd/hashsum/main_test.js b/cmd/hashsum/main_test.js new file mode 100644 index 0000000..2b3876f --- /dev/null +++ b/cmd/hashsum/main_test.js @@ -0,0 +1,61 @@ +import { before, test } from "node:test"; +import assert from "node:assert/strict"; + +import { fetch } from "../../fetch.js"; +import { putFiles } from "../testing.js"; +import { hashsum, md5sum, sha1sum } from "./main.js"; + +const ROOT = ":memory:/hashsum/"; + +before(async () => { + await putFiles(ROOT, { "f.txt": "hello", "a/g.txt": "world!", "a/zero": "" }); +}); + +/** @param {Response} response */ +async function lines(response) { + assert.equal(response.status, 200); + return (await response.text()).split("\n").filter(Boolean).sort(); +} + +test("md5sum and sha1sum give the sums of the files, as rclone", async () => { + assert.deepEqual(await lines(await md5sum(ROOT)), [ + "08cf82251c975a5e9734699fadf5e9c0 a/g.txt", + "5d41402abc4b2a76b9719d911017c592 f.txt", + "d41d8cd98f00b204e9800998ecf8427e a/zero", + ]); + assert.deepEqual(await lines(await sha1sum(`${ROOT}f.txt`)), [ + "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d f.txt", + ]); +}); + +test("hashsum gives the hashes of rclone", async () => { + assert.deepEqual(await lines(await hashsum("SHA-256", `${ROOT}f.txt`)), [ + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 f.txt", + ]); + assert.deepEqual(await lines(await hashsum("crc32", ROOT)), [ + "00000000 a/zero", + "3610a686 f.txt", + "718498e8 a/g.txt", + ]); + assert.deepEqual( + await lines(await hashsum("md5", `${ROOT}f.txt`, { base64: true })), + ["XUFAKrxLKna5cZ2REBfFkg== f.txt"], + ); + assert.match( + await (await hashsum({})).text(), + /^Supported hashes are:\n- md5\n/, + ); + assert.equal((await hashsum("whirlpool", ROOT)).status, 400); +}); + +test("hashsum writes an output file, and checks a sum file", async () => { + const out = `${ROOT}sums/MD5SUMS`; + assert.equal( + await (await md5sum(`${ROOT}a/`, { output_file: out })).text(), + "", + ); + const sums = await (await fetch(out)).text(); + assert.match(sums, /^08cf82251c975a5e9734699fadf5e9c0 {2}g\.txt$/m); + const checked = await md5sum(`${ROOT}a/`, { checkfile: out, quiet: true }); + assert.equal(checked.status, 200); +}); diff --git a/cmd/listremotes/main.js b/cmd/listremotes/main.js new file mode 100644 index 0000000..4e21830 --- /dev/null +++ b/cmd/listremotes/main.js @@ -0,0 +1,63 @@ +import { env } from "node:process"; +import { isTrue } from "@rfetch/lib/options"; + +import { config } from "../config/main.js"; + +/** + * List all the remotes in the config file and defined in environment + * variables, as `rclone listremotes`: one `name:` on each line. With + * `--long`, the type too. With `--json`, the name, the type, the source + * (`file` or `environment`) and the description of each remote. + * + * A remote of the variables has a variable `RFETCH_CONFIG_NAME_TYPE`, or + * `RCLONE_CONFIG_NAME_TYPE`: its name is in lower case. + * @param {string | Object} [filter] Only the remotes with this text in their + * names, as the argument of rclone. + * @param {Object} [flags] + * @returns {Promise} + */ +export async function listremotes(filter, flags = {}) { + if (typeof filter !== "string") [filter, flags] = ["", filter ?? {}]; + const response = await config("dump"); + /** @type {Record>} */ + const sections = response.ok ? await response.json() : {}; + /** @type {{ name: string, type: string, source: string, description: string }[]} */ + const remotes = []; + for (const [name, options] of Object.entries(sections)) { + remotes.push({ + name, + type: options.type ?? "", + source: "file", + description: options.description ?? "", + }); + } + for (const [key, value] of Object.entries(env)) { + const match = key.match(/^(?:RFETCH|RCLONE)_CONFIG_(.+)_TYPE$/); + if (!match || typeof value !== "string") continue; + const name = match[1].toLowerCase(); + const known = remotes.find((remote) => remote.name === name); + if (known) { + if (known.source === "environment") known.type = value; + continue; + } + remotes.push({ name, type: value, source: "environment", description: "" }); + } + const shown = remotes.filter((remote) => remote.name.includes(filter)); + + if (isTrue(flags.json)) { + const lines = shown.map((remote) => JSON.stringify(remote)); + return new Response( + `[\n${lines.join(",\n")}${lines.length ? "\n" : ""}]\n`, + ); + } + if (isTrue(flags.long)) { + const names = Math.max(0, ...shown.map(({ name }) => name.length + 1)); + const types = Math.max(0, ...shown.map(({ type }) => type.length)); + return new Response( + shown.map(({ name, type, description }) => + `${`${name}:`.padEnd(names)} ${type.padEnd(types)} ${description}\n` + ).join(""), + ); + } + return new Response(shown.map(({ name }) => `${name}:\n`).join("")); +} diff --git a/cmd/listremotes/main_test.js b/cmd/listremotes/main_test.js new file mode 100644 index 0000000..96a3dce --- /dev/null +++ b/cmd/listremotes/main_test.js @@ -0,0 +1,42 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import process from "node:process"; + +import { listremotes } from "./main.js"; + +test("listremotes gives the remotes of the config file and of the variables", async (t) => { + const folder = await mkdtemp(join(tmpdir(), "rfetch-listremotes-")); + const saved = { ...process.env }; + t.after(async () => { + for (const key of Object.keys(process.env)) { + if (!(key in saved)) delete process.env[key]; + } + Object.assign(process.env, saved); + await rm(folder, { recursive: true, force: true }); + }); + await mkdir(join(folder, "rfetch")); + await writeFile( + join(folder, "rfetch", "rfetch.conf"), + "[mem]\ntype = memory\n\n[enc]\ntype = crypt\ndescription = mine\n", + ); + process.env.XDG_CONFIG_HOME = folder; + process.env.RFETCH_CONFIG_FROMENV_TYPE = "s3"; + + assert.equal(await (await listremotes({})).text(), "mem:\nenc:\nfromenv:\n"); + assert.equal( + await (await listremotes({ long: true })).text(), + "mem: memory \nenc: crypt mine\nfromenv: s3 \n", + ); + assert.deepEqual( + JSON.parse(await (await listremotes({ json: true })).text()), + [ + { name: "mem", type: "memory", source: "file", description: "" }, + { name: "enc", type: "crypt", source: "file", description: "mine" }, + { name: "fromenv", type: "s3", source: "environment", description: "" }, + ], + ); + assert.equal(await (await listremotes("en", {})).text(), "enc:\nfromenv:\n"); +}); diff --git a/cmd/lsjson/main.js b/cmd/lsjson/main.js index e3de9b0..d04239e 100644 --- a/cmd/lsjson/main.js +++ b/cmd/lsjson/main.js @@ -60,6 +60,65 @@ const FOLDER = new URL("http://remote/folder/"); * @returns {Promise} */ export async function lsjson(location, flags = {}, init = {}) { + const items = await listItems(location, flags, init); + if (items instanceof Response) return items; + /** + * The JSON of the item before, with no comma: only the last item has no + * comma after it. + * @type {string | null} + */ + let previous = null; + + /** + * A stream of JSON-encoded file objects, one per chunk and line, in an + * array. + * @type {ReadableStream} + */ + const body = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode("[\n")); + }, + async pull(controller) { + for (;;) { + const { done, value } = await items.next(); + if (done) { + if (previous !== null) { + controller.enqueue(encoder.encode(`${previous}\n`)); + } + controller.enqueue(encoder.encode("]\n")); + controller.close(); + return; + } + const line = JSON.stringify(value); + if (previous !== null) { + controller.enqueue(encoder.encode(`${previous},\n`)); + previous = line; + return; + } + previous = line; + } + }, + async cancel() { + await items.return(undefined); + }, + }); + + return new Response(body, { + headers: { + "Content-Type": "application/json", + }, + }); +} + +/** + * Gives the items of a path, as `lsjson` lists them, or the answer of the + * error. The other commands use it to walk a folder. + * @param {string} location + * @param {Object} [flags={}] As the flags of `lsjson`. + * @param {RequestInit} [init] + * @returns {Promise | Response>} + */ +export async function listItems(location, flags = {}, init = {}) { // A path that ends with `.` or `..` is a folder, as in rclone. if (/(^|[:/])\.\.?$/.test(location)) location = `${location}/`; // Only the path in the remote has folders: a value of the connection @@ -103,7 +162,7 @@ export async function lsjson(location, flags = {}, init = {}) { return new Response("directory not found", { status: 404 }); } if (named.every(({ IsDir }) => IsDir)) { - return lsjson(`${location}/`, flags, init); + return listItems(`${location}/`, flags, init); } } @@ -120,54 +179,7 @@ export async function lsjson(location, flags = {}, init = {}) { : readListing(response, FOLDER).then((list) => folderOf(list, options)); // The error comes when the stream gets to the folder. listed.catch(() => {}); - const items = walk(listed, ""); - - /** - * The JSON of the item before, with no comma: only the last item has no - * comma after it. - * @type {string | null} - */ - let previous = null; - - /** - * A stream of JSON-encoded file objects, one per chunk and line, in an - * array. - * @type {ReadableStream} - */ - const body = new ReadableStream({ - start(controller) { - controller.enqueue(encoder.encode("[\n")); - }, - async pull(controller) { - for (;;) { - const { done, value } = await items.next(); - if (done) { - if (previous !== null) { - controller.enqueue(encoder.encode(`${previous}\n`)); - } - controller.enqueue(encoder.encode("]\n")); - controller.close(); - return; - } - const line = JSON.stringify(value); - if (previous !== null) { - controller.enqueue(encoder.encode(`${previous},\n`)); - previous = line; - return; - } - previous = line; - } - }, - async cancel() { - await items.return(undefined); - }, - }); - - return new Response(body, { - headers: { - "Content-Type": "application/json", - }, - }); + return walk(listed, ""); } /** diff --git a/cmd/main.js b/cmd/main.js index 6c7454f..2d7e65d 100644 --- a/cmd/main.js +++ b/cmd/main.js @@ -8,7 +8,18 @@ export { lsl } from "./lsl/main.js"; export { lsd } from "./lsd/main.js"; export { cat } from "./cat/main.js"; export { rcat } from "./rcat/main.js"; -export { copy } from "./copy/main.js"; +export { copy, copyto } from "./copy/main.js"; +export { check, checksum, cryptcheck } from "./check/main.js"; +export { cryptdecode } from "./cryptdecode/main.js"; +export { delete, deletefile, purge, rmdir, rmdirs } from "./delete/main.js"; +export { hashsum, md5sum, sha1sum } from "./hashsum/main.js"; +export { listremotes } from "./listremotes/main.js"; +export { mkdir } from "./mkdir/main.js"; +export { move, moveto } from "./move/main.js"; +export { size } from "./size/main.js"; +export { touch } from "./touch/main.js"; +export { tree } from "./tree/main.js"; +export { version } from "./version/main.js"; export { copyurl } from "./copyurl/main.js"; export { obscure, reveal } from "./obscure/main.js"; export { serve } from "./serve/main.js"; diff --git a/cmd/mkdir/main.js b/cmd/mkdir/main.js new file mode 100644 index 0000000..7eefc44 --- /dev/null +++ b/cmd/mkdir/main.js @@ -0,0 +1,22 @@ +import { failure, mkdir as makeFolder } from "../operations.js"; + +/** + * Make the path if it doesn't already exist, as `rclone mkdir`. The folders + * above the path come too. + * + * ```js + * import { mkdir } from "./main.js"; + * await mkdir("remote:path/to/folder"); + * ``` + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function mkdir(location, flags = {}) { + try { + await makeFolder(location, flags); + return new Response(null); + } catch (error) { + return failure(error); + } +} diff --git a/cmd/mkdir/main_test.js b/cmd/mkdir/main_test.js new file mode 100644 index 0000000..de822cc --- /dev/null +++ b/cmd/mkdir/main_test.js @@ -0,0 +1,16 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { contents } from "../testing.js"; +import { mkdir } from "./main.js"; + +test("mkdir makes a folder and the folders above it", async () => { + assert.equal((await mkdir(":memory:/mkdir/a/b c/d")).status, 200); + assert.deepEqual((await contents(":memory:/mkdir/")).folders, [ + "a", + "a/b c", + "a/b c/d", + ]); + // A folder that exists is not an error. + assert.equal((await mkdir(":memory:/mkdir/a")).status, 200); +}); diff --git a/cmd/move/main.js b/cmd/move/main.js new file mode 100644 index 0000000..5fc0cdb --- /dev/null +++ b/cmd/move/main.js @@ -0,0 +1,45 @@ +import { failure } from "../operations.js"; +import { transfer } from "../copy/main.js"; + +/** + * Move files from source to dest, as `rclone move`: the copy of `copy`, + * then the delete of each source file. The folders of the source stay, + * unless `--delete-empty-src-dirs`. + * + * The backends cannot move a file yet: each file goes through rfetch. + * + * ```js + * import { move } from "./main.js"; + * await move("source:path", "dest:path"); + * ``` + * @param {string} source + * @param {string} target + * @param {Object} [flags] + * @returns {Promise} + */ +export async function move(source, target, flags = {}) { + try { + await transfer(source, target, flags, { move: true }); + return new Response(null); + } catch (error) { + return failure(error); + } +} + +/** + * Move file or directory from source to dest, as `rclone moveto`: a source + * file goes to the path of dest, with the name of dest. A source folder is + * as `move`. + * @param {string} source + * @param {string} target + * @param {Object} [flags] + * @returns {Promise} + */ +export async function moveto(source, target, flags = {}) { + try { + await transfer(source, target, flags, { move: true, to: true }); + return new Response(null); + } catch (error) { + return failure(error); + } +} diff --git a/cmd/move/main_test.js b/cmd/move/main_test.js new file mode 100644 index 0000000..df09ca7 --- /dev/null +++ b/cmd/move/main_test.js @@ -0,0 +1,42 @@ +import { beforeEach, describe, test } from "node:test"; +import assert from "node:assert/strict"; + +import { fetch } from "../../fetch.js"; +import { contents, putFiles } from "../testing.js"; +import { move, moveto } from "./main.js"; + +const SOURCE = ":memory:/move/source/"; +const TARGET = ":memory:/move/target/"; + +describe("move and moveto", () => { + beforeEach(async () => { + await (await fetch(":memory:/move/", { method: "DELETE" })).body?.cancel(); + await putFiles(SOURCE, { "f.txt": "hello", "a/g.txt": "world!" }); + }); + + test("move copies the files, and deletes them in the source", async () => { + assert.equal((await move(SOURCE, TARGET)).status, 200); + assert.deepEqual((await contents(TARGET)).files, { + "f.txt": "hello", + "a/g.txt": "world!", + }); + // The folders of the source stay, as rclone. + assert.deepEqual(await contents(SOURCE), { files: {}, folders: ["a"] }); + }); + + test("move --delete-empty-src-dirs deletes the folders too", async () => { + await move(SOURCE, TARGET, { delete_empty_src_dirs: true }); + assert.deepEqual(await contents(SOURCE), { files: {}, folders: [] }); + }); + + test("moveto renames a file", async () => { + assert.equal( + (await moveto(`${SOURCE}f.txt`, `${SOURCE}renamed.txt`)).status, + 200, + ); + assert.deepEqual((await contents(SOURCE)).files, { + "renamed.txt": "hello", + "a/g.txt": "world!", + }); + }); +}); diff --git a/cmd/operations.js b/cmd/operations.js new file mode 100644 index 0000000..f4670bc --- /dev/null +++ b/cmd/operations.js @@ -0,0 +1,396 @@ +/** + * The operations that the commands share, as the package `operations` of + * rclone: the information of a path, the files of a folder, the copy of a + * file and the log of the messages. + * + * A location is a path of rclone for `fetch`, with percent-encoding, as the + * command line gives it: for example `remote:a%20b/c.txt`. + */ +import process from "node:process"; +import { isRemoteRoot, parse } from "@rfetch/lib/fspath"; +import { readListing } from "@rfetch/lib/listing"; +import { encodePath } from "@rfetch/lib/path"; + +import { fetch } from "../fetch.js"; +import { listItems } from "./lsjson/main.js"; + +/** The URL of a folder: the links of its list are relative to it. */ +const FOLDER = new URL("http://remote/folder/"); + +/** The flags of the commands that are not options of a backend. */ +const OWN_FLAGS = new Set(["recursive", "max_depth", "max-depth"]); + +/** + * An error of an operation, with the status of its answer: 404 for a folder + * that does not exist, as the exit code 3 of rclone. + */ +export class OperationError extends Error { + /** @param {string} message @param {number} [status] */ + constructor(message, status = 500) { + super(message); + this.status = status; + } +} + +/** + * The answer of a command for an error. + * @param {unknown} error + * @returns {Response} + */ +export function failure(error) { + if (error instanceof OperationError) { + return new Response(error.message, { status: error.status }); + } + if (error instanceof SyntaxError) { + return new Response(error.message, { status: 400 }); + } + throw error; +} + +/** + * Writes the messages of the commands on the standard error, as the log of + * rclone: `2006/01/02 15:04:05 NOTICE: text`. The tests replace `write`. + */ +export const logger = { + /** @param {string} line */ + write(line) { + process.stderr.write(line); + }, +}; + +/** + * Writes a message of a level, about an object, as rclone. + * @param {"ERROR" | "NOTICE" | "INFO"} level + * @param {string} object The object of the message, for example a path, or + * "". + * @param {string} text + * @param {Object} [flags] With `quiet`, only the errors. + */ +export function log(level, object, text, flags = {}) { + if (level !== "ERROR" && (flags.quiet === true || flags.quiet === "true")) { + return; + } + const now = new Date(); + const pad = (number) => `${number}`.padStart(2, "0"); + const time = `${now.getFullYear()}/${pad(now.getMonth() + 1)}/${ + pad(now.getDate()) + } ${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`; + logger.write( + `${time} ${level.padEnd(6)}: ${object ? `${object}: ` : ""}${text}\n`, + ); +} + +/** + * Gives a location with the flags of the command in its query, as the + * other commands do: the fetch of rfetch gives each backend its flags. + * @param {string} location + * @param {Object} [flags] + */ +export function withFlags(location, flags = {}) { + const params = new URLSearchParams(); + for (const [name, value] of Object.entries(flags)) { + if (OWN_FLAGS.has(name)) continue; + if (typeof value === "string" || typeof value === "boolean") { + params.set(name, `${value}`); + } + } + const query = `${params}`; + return query ? `${location}?${query}` : location; +} + +/** + * Splits a location into its remote, and the path in the remote. + * @param {string} location + * @returns {{ remote: string, path: string }} For example `remote:` and + * `a/b.txt`. + */ +export function split(location) { + const { path } = parse(location); + return { remote: location.slice(0, location.length - path.length), path }; +} + +/** + * Tells if a location is a folder by its form: the root of a remote, or a + * path with `/` at the end. + * @param {string} location + */ +export function isFolderForm(location) { + const { path } = split(location); + return path === "" || path.endsWith("/") || isRemoteRoot(path); +} + +/** + * Gives the location of a folder, with `/` at the end of its path. + * @param {string} location + */ +export function folderOf(location) { + const { path } = split(location); + if (path === "" || path.endsWith("/")) return location; + return `${location}/`; +} + +/** + * Gives the location of a path in a folder. + * @param {string} folder The location of the folder. + * @param {string} path A path with no percent-encoding, for example + * `a b/c.txt`. + * @param {boolean} [isDir] + */ +export function join(folder, path, isDir = false) { + return `${folderOf(folder)}${encodePath(path)}${isDir ? "/" : ""}`; +} + +/** + * The name of the last part of a location, with no percent-encoding. + * @param {string} location + */ +export function nameOf(location) { + const { path } = split(location); + const name = path.replace(/\/$/, "").split("/").at(-1) ?? ""; + try { + return decodeURIComponent(name); + } catch { + return name; + } +} + +/** + * The information of a file or a folder. + * @typedef {Object} Info + * @property {boolean} isDir + * @property {number} size -1 for a folder. + * @property {string} modTime The time in ISO 8601, or "". + * @property {string} mimeType + */ + +/** + * Gives the information of a path, or null when it does not exist. As in + * rclone, a path with no `/` at the end is a file, or else a folder. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + * @throws {OperationError} When the remote gives an error. + */ +export async function stat(location, flags = {}) { + if (isFolderForm(location)) { + const response = await fetch(withFlags(folderOf(location), flags), { + method: "HEAD", + }); + await response.body?.cancel(); + if (response.ok) { + return { isDir: true, size: -1, modTime: "", mimeType: "" }; + } + if (response.status === 404) return null; + throw new OperationError( + `${location}: ${response.status} ${response.statusText}`.trim(), + response.status, + ); + } + // The list of the folder tells if the name is a file or a folder. + const { remote, path } = split(location); + const slash = path.lastIndexOf("/") + 1; + const parent = `${remote}${path.slice(0, slash)}`; + const name = nameOf(location); + const response = await fetch(withFlags(parent, flags)); + if (response.status === 404) { + await response.body?.cancel(); + return null; + } + if (!response.ok) { + await response.body?.cancel(); + throw new OperationError( + `${location}: ${response.status} ${response.statusText}`.trim(), + response.status, + ); + } + const entries = (await readListing(response, FOLDER)) + .filter((entry) => entry.Name === name); + const file = entries.find((entry) => !entry.IsDir); + if (file) { + return { + isDir: false, + size: file.Size, + modTime: file.ModTime, + mimeType: file.MimeType, + }; + } + if (entries.length) { + return { isDir: true, size: -1, modTime: entries[0].ModTime, mimeType: "" }; + } + return null; +} + +/** + * An item of a walk: a file or a folder below the folder of the walk. + * @typedef {import("./lsjson/main.js").Item & { location: string }} WalkItem + */ + +/** + * Gives the files and the folders in a folder, and in its folders. + * @param {string} location The location of the folder. + * @param {Object} [flags] + * @param {{ maxDepth?: number }} [options] + * @returns {Promise>} + * @throws {OperationError} With 404 when the folder does not exist. + */ +export async function walk(location, flags = {}, { maxDepth = Infinity } = {}) { + const folder = folderOf(location); + const items = await listItems(folder, { + ...flags, + recursive: "true", + max_depth: String(maxDepth), + }); + if (items instanceof Response) { + const text = (await items.text()).trim(); + throw new OperationError( + items.status === 404 ? "directory not found" : text || + `${items.status} ${items.statusText}`.trim(), + items.status, + ); + } + return (async function* () { + for await (const item of items) { + yield { ...item, location: join(folder, item.Path, item.IsDir) }; + } + })(); +} + +/** + * Runs a function on each value, with `concurrency` at the same time. + * @template T + * @param {AsyncIterable | Iterable} values + * @param {number} concurrency + * @param {(value: T) => Promise} run + */ +export async function each(values, concurrency, run) { + const iterator = values[Symbol.asyncIterator]?.() ?? + values[Symbol.iterator](); + const next = async () => { + for (;;) { + const { done, value } = await iterator.next(); + if (done) return; + await run(value); + } + }; + await Promise.all( + Array.from({ length: Math.max(1, concurrency) }, next), + ); +} + +/** + * The number of a flag, or its default. + * @param {unknown} value + * @param {number} fallback + */ +export function numberOf(value, fallback) { + const number = Number(value); + return Number.isFinite(number) && number > 0 ? number : fallback; +} + +/** + * Copies a file: it reads the source, and puts it at the target. + * @param {string} source + * @param {string} target + * @param {Object} [flags] + * @throws {OperationError} When the read or the write fails. + */ +export async function copyFile(source, target, flags = {}) { + const response = await fetch(withFlags(source, flags)); + if (!response.ok) { + await response.body?.cancel(); + throw new OperationError( + `${source}: ${response.status} ${response.statusText}`.trim(), + response.status, + ); + } + /** @type {Record} */ + const headers = {}; + for (const name of ["Content-Type", "Content-Length"]) { + const value = response.headers.get(name); + if (value) headers[name] = value; + } + const put = await fetch(withFlags(target, flags), { + method: "PUT", + headers, + body: response.body ?? new Uint8Array(0), + }); + if (!put.ok) { + const text = (await put.text()).trim(); + throw new OperationError( + `${target}: ${text || `${put.status} ${put.statusText}`.trim()}`, + put.status, + ); + } + await put.body?.cancel(); +} + +/** + * Deletes a file, or a folder with all that is in it. + * @param {string} location For a folder, with `/` at the end. + * @param {Object} [flags] + * @throws {OperationError} When the remote gives an error. + */ +export async function remove(location, flags = {}) { + const response = await fetch(withFlags(location, flags), { + method: "DELETE", + }); + await response.body?.cancel(); + if (!response.ok) { + throw new OperationError( + response.status === 404 + ? `${location}: not found` + : `${location}: ${response.status} ${response.statusText}`.trim(), + response.status, + ); + } +} + +/** + * Makes a folder, and the folders above it. + * @param {string} location + * @param {Object} [flags] + * @throws {OperationError} When the remote gives an error. + */ +export async function mkdir(location, flags = {}) { + const response = await fetch(withFlags(folderOf(location), flags), { + method: "PUT", + }); + await response.body?.cancel(); + if (!response.ok) { + throw new OperationError( + `${location}: ${response.status} ${response.statusText}`.trim(), + response.status, + ); + } +} + +/** + * Gives the type and the options of the remote of a location, from TRACE of + * the fetch of rfetch: the options of the path, of the connection string, + * of the flags, of the variables and of the config file. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise<{ type: string, options: Record }>} + */ +export async function remoteOf(location, flags = {}) { + const response = await fetch(withFlags(location, flags), { + method: "TRACE", + }); + const [, type = ""] = response.headers.get("Via")?.match(/^(.*?)(\/.*)?$/) ?? + []; + const lines = (await response.text()).split("\r"); + /** @type {Record} */ + const options = {}; + const [, query = ""] = lines[0]?.match(/^TRACE [^?]*\?(.*) HTTP\/1\.1$/) ?? + []; + for (const [name, value] of new URLSearchParams(query)) options[name] = value; + // The other options are in the variables `RFETCH_TYPE_NAME`. + const prefix = `RFETCH_${type.toUpperCase()}_`; + for (const line of lines.slice(1)) { + const colon = line.indexOf(": "); + if (colon === -1 || !line.startsWith(prefix)) continue; + const name = line.slice(prefix.length, colon).toLowerCase(); + options[name] ??= line.slice(colon + 2); + } + return { type, options }; +} diff --git a/cmd/rclone_test.js b/cmd/rclone_test.js new file mode 100644 index 0000000..57ac262 --- /dev/null +++ b/cmd/rclone_test.js @@ -0,0 +1,259 @@ +/** + * The tests of the commands against the commands of rclone, on the same + * local folders: the output, the exit code, and the files after the command. + * The tests run when rclone is in the PATH, or when RCLONE is the path of + * rclone. + */ +import { after, before, describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { + cpSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, relative } from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +const RCLONE = process.env.RCLONE || "rclone"; +const hasRclone = spawnSync(RCLONE, ["version"]).status === 0; +const MAIN = join(dirname(fileURLToPath(import.meta.url)), "..", "main.js"); + +/** The files of the tests, with hard names. */ +const FILES = { + "f.txt": "hello", + "B.txt": "b", + "é.txt": "e", + "a/g.txt": "world!", + "a/b/zero": "", + "sp ace/#1 & 2?.txt": "name", + ".hidden/x": "x", +}; + +/** + * The files and the folders of a local folder. + * @param {string} folder + * @returns {string[]} The paths, with a `/` at the end of a folder, and the + * text of each file. + */ +function treeOf(folder) { + /** @type {string[]} */ + const paths = []; + /** @param {string} path */ + const visit = (path) => { + for (const entry of readdirSync(path, { withFileTypes: true })) { + const full = join(path, entry.name); + const name = relative(folder, full); + if (entry.isDirectory()) { + paths.push(`${name}/`); + visit(full); + } else { + paths.push(`${name} ${readFileSync(full, "utf8")}`); + } + } + }; + try { + statSync(folder); + } catch { + return ["(none)"]; + } + visit(folder); + return paths.sort(); +} + +describe("the commands and the commands of rclone", { + skip: !hasRclone && "rclone is not installed. Set RCLONE to its path.", +}, () => { + const home = mkdtempSync(join(tmpdir(), "rfetch-commands-")); + const source = join(home, "source"); + + /** + * Runs a program in the folder of the tests. + * @param {string} program + * @param {string[]} args + */ + const run = (program, args) => { + const { status, stdout, stderr } = spawnSync(program, args, { + cwd: home, + env: { PATH: process.env.PATH, HOME: home, TZ: "UTC" }, + encoding: "utf8", + }); + return { status, stdout, stderr }; + }; + /** @param {...string} args */ + const rclone = (...args) => run(RCLONE, args); + /** @param {...string} args */ + const rfetch = (...args) => run(process.execPath, [MAIN, ...args]); + + before(() => { + for (const [path, text] of Object.entries(FILES)) { + mkdirSync(dirname(join(source, path)), { recursive: true }); + writeFileSync(join(source, path), text); + } + mkdirSync(join(source, "empty", "inner"), { recursive: true }); + const password = rclone("obscure", "commands").stdout.trim(); + mkdirSync(join(home, ".config", "rclone"), { recursive: true }); + writeFileSync( + join(home, ".config", "rclone", "rclone.conf"), + `[mem]\ntype = memory\n\n[enc]\ntype = crypt\nremote = ${ + join(home, "encrypted") + }\npassword = ${password}\n`, + ); + }); + after(() => rmSync(home, { recursive: true, force: true })); + + /** + * Runs a command with rclone and rfetch, and compares the outputs and + * the exit codes. + * @param {string[]} args + * @param {{ sort?: boolean }} [options] `sort` compares the lines in any + * order: rclone writes them as its checks end. + */ + const same = (args, { sort = false } = {}) => { + const expected = rclone(...args); + const actual = rfetch(...args); + const lines = (text) => sort ? text.split("\n").sort() : text; + assert.equal(actual.status, expected.status, `${args.join(" ")}: status`); + assert.deepEqual( + lines(actual.stdout), + lines(expected.stdout), + args.join(" "), + ); + }; + + test("size", () => { + same(["size", "source"]); + same(["size", "--json", "source"]); + same(["size", "source/a/g.txt"]); + same(["size", "missing"]); + }); + + test("tree", () => { + same(["tree", "source"]); + same(["tree", "--dirs-only", "source"]); + same(["tree", "--level", "1", "source"]); + same(["tree", "--dirsfirst", "source"]); + same(["tree", "--noreport", "--all", "source"]); + same(["tree", "--size", "source"]); + same(["tree", "source/f.txt"]); + }); + + test("the sums", () => { + same(["md5sum", "source"], { sort: true }); + same(["sha1sum", "source"], { sort: true }); + same(["hashsum", "sha256", "source"], { sort: true }); + same(["hashsum", "SHA-1", "source"], { sort: true }); + same(["hashsum", "crc32", "source"], { sort: true }); + same(["hashsum", "md5", "--base64", "source/f.txt"]); + same(["md5sum", "missing"]); + }); + + test("checksum, and hashsum --checkfile", () => { + writeFileSync(join(home, "MD5SUMS"), rclone("md5sum", "source").stdout); + same(["checksum", "md5", "MD5SUMS", "source"]); + same(["md5sum", "--checkfile", "MD5SUMS", "source"], { sort: true }); + writeFileSync( + join(home, "BAD"), + "00000000000000000000000000000000 f.txt\n", + ); + same(["checksum", "md5", "BAD", "source"]); + }); + + test("check", () => { + cpSync(source, join(home, "other"), { recursive: true }); + rmSync(join(home, "other", "a", "g.txt")); + writeFileSync(join(home, "other", "extra"), "x"); + writeFileSync(join(home, "other", "B.txt"), "longer"); + same(["check", "source", "other", "--combined", "-"], { sort: true }); + same(["check", "source", "other", "--one-way", "--missing-on-dst", "-"]); + same(["check", "source", "source"]); + // Same size, other bytes: rclone sees it with MD5, rfetch with the + // bytes. + writeFileSync(join(home, "other", "f.txt"), "HELLO"); + same(["check", "--download", "source", "other", "--differ", "-"], { + sort: true, + }); + }); + + test("listremotes", () => { + same(["listremotes"]); + same(["listremotes", "--long"]); + same(["listremotes", "--json"]); + }); + + test("cryptdecode of the names of rclone", () => { + assert.equal(rclone("copy", "source", "enc:").status, 0); + const names = readdirSync(join(home, "encrypted")); + same(["cryptdecode", "enc:", ...names, "not-a-name"]); + same(["cryptdecode", "--reverse", "enc:", "f.txt", "a/g.txt"]); + }); + + /** + * Runs a command that changes files with rclone and rfetch, each on its + * own copy of the files, and compares the files after it. + * @param {(folder: string) => string[]} args The arguments, for a folder. + * @param {string[]} [only] Other arguments for rclone only. + */ + const sameEffect = (args, only = []) => { + const results = []; + for ( + const [name, tool] of [ + ["by-rclone", (...a) => rclone(...only, ...a)], + ["by-rfetch", rfetch], + ] + ) { + const folder = join(home, name); + rmSync(folder, { recursive: true, force: true }); + cpSync(source, join(folder, "source"), { recursive: true }); + const { status } = tool(...args(name)); + results.push({ status, tree: treeOf(folder) }); + } + assert.deepEqual(results[1], results[0], args("folder").join(" ")); + }; + + test("the commands that delete", () => { + sameEffect((f) => ["delete", `${f}/source`]); + sameEffect((f) => ["delete", "--rmdirs", `${f}/source`]); + sameEffect((f) => ["delete", `${f}/source/a/g.txt`]); + sameEffect((f) => ["deletefile", `${f}/source/f.txt`]); + sameEffect((f) => ["deletefile", `${f}/source/a`]); + sameEffect((f) => ["purge", `${f}/source/a`]); + sameEffect((f) => ["rmdir", `${f}/source/empty/inner`]); + sameEffect((f) => ["rmdir", `${f}/source/a`]); + sameEffect((f) => ["rmdirs", `${f}/source`]); + sameEffect((f) => ["rmdirs", "--leave-root", `${f}/source/empty`]); + sameEffect((f) => ["delete", "--dry-run", `${f}/source`]); + }); + + test("the commands that copy and move", () => { + sameEffect((f) => ["copy", `${f}/source`, `${f}/target`]); + sameEffect((f) => ["copy", `${f}/source/f.txt`, `${f}/target`]); + sameEffect((f) => ["copyto", `${f}/source/f.txt`, `${f}/target/new.txt`]); + // rclone moves a folder of a local remote with one rename. rfetch moves + // each file, as rclone does with no DirMove. + const noDirMove = ["--disable", "DirMove"]; + sameEffect((f) => ["move", `${f}/source`, `${f}/target`], noDirMove); + sameEffect((f) => [ + "move", + "--delete-empty-src-dirs", + `${f}/source`, + `${f}/target`, + ], noDirMove); + sameEffect((f) => [ + "move", + "--delete-empty-src-dirs", + `${f}/source/a`, + `${f}/target`, + ], noDirMove); + sameEffect((f) => ["moveto", `${f}/source/f.txt`, `${f}/source/g.txt`]); + sameEffect((f) => ["mkdir", `${f}/source/x/y`]); + sameEffect((f) => ["touch", `${f}/source/new.txt`]); + }); +}); diff --git a/cmd/size/main.js b/cmd/size/main.js new file mode 100644 index 0000000..6b1fe14 --- /dev/null +++ b/cmd/size/main.js @@ -0,0 +1,75 @@ +import { countString, sizeString } from "@rfetch/lib/format"; +import { isTrue } from "@rfetch/lib/options"; + +import { failure, log, OperationError, stat, walk } from "../operations.js"; + +/** + * Prints the total size and number of objects in remote:path, as `rclone + * size`: + * + * ```text + * Total objects: 3 + * Total size: 11 B (11 Byte) + * ``` + * + * With `--json`, it gives `{"count":3,"bytes":11,"sizeless":0}`. A file with + * no size counts as 0 bytes, and in `sizeless`. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function size(location, flags = {}) { + try { + const info = await stat(location, flags); + if (!info) throw new OperationError("directory not found", 404); + let count = 0, bytes = 0, sizeless = 0; + /** @param {number} value */ + const add = (value) => { + count++; + if (value >= 0) bytes += value; + else sizeless++; + }; + if (info.isDir) { + for await (const item of await walk(location, flags)) { + if (!item.IsDir) add(item.Size); + } + } else { + add(info.size); + } + if (sizeless > 0) { + log( + "NOTICE", + location, + `Size may be underestimated due to ${sizeless} objects with unknown size`, + flags, + ); + } + if (isTrue(flags.json)) { + return new Response(`${JSON.stringify({ count, bytes, sizeless })}\n`); + } + const suffix = countString(count, true); + let text = suffix === `${count}` + ? `Total objects: ${count}\n` + : `Total objects: ${suffix} (${count})\n`; + text += `Total size: ${byteUnit(bytes)} (${bytes} Byte)\n`; + if (sizeless > 0) { + text += `Total objects with unknown size: ${ + countString(sizeless, true) + } (${sizeless})\n`; + } + return new Response(text); + } catch (error) { + return failure(error); + } +} + +/** + * Writes a size as `SizeSuffix.ByteUnit` of rclone, for example `11 B` or + * `1.500 KiB`. + * @param {number} bytes + */ +export function byteUnit(bytes) { + const [, number, suffix] = sizeString(bytes, true).match(/^([\d.]+)(\D*)$/) ?? + []; + return `${number} ${suffix}B`; +} diff --git a/cmd/size/main_test.js b/cmd/size/main_test.js new file mode 100644 index 0000000..d9231e1 --- /dev/null +++ b/cmd/size/main_test.js @@ -0,0 +1,40 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { putFiles } from "../testing.js"; +import { byteUnit, size } from "./main.js"; + +test("size gives the count and the size of the files, as rclone", async () => { + await putFiles(":memory:/size/", { + "f.txt": "hello", + "a/g.txt": "world!", + "a/b/zero": "", + }); + assert.equal( + await (await size(":memory:/size")).text(), + "Total objects: 3\nTotal size: 11 B (11 Byte)\n", + ); + assert.equal( + await (await size(":memory:/size", { json: true })).text(), + '{"count":3,"bytes":11,"sizeless":0}\n', + ); + assert.equal( + await (await size(":memory:/size/a/g.txt")).text(), + "Total objects: 1\nTotal size: 6 B (6 Byte)\n", + ); + assert.equal((await size(":memory:/size/missing")).status, 404); +}); + +test("byteUnit writes the sizes of rclone", () => { + for ( + const [bytes, text] of [ + [0, "0 B"], + [11, "11 B"], + [1024, "1 KiB"], + [1536, "1.500 KiB"], + [5 * 1024 ** 3 + 1, "5.000 GiB"], + ] + ) { + assert.equal(byteUnit(bytes), text, `${bytes}`); + } +}); diff --git a/cmd/testing.js b/cmd/testing.js new file mode 100644 index 0000000..e362414 --- /dev/null +++ b/cmd/testing.js @@ -0,0 +1,65 @@ +/** + * The helpers of the tests of the commands, on the memory remote. + */ +import { encodePath } from "@rfetch/lib/path"; + +import { fetch } from "../fetch.js"; +import { logger, walk } from "./operations.js"; + +/** + * Puts files in a folder of a remote. + * @param {string} folder For example `:memory:/test/`. + * @param {Record} files The texts of the files, by path. + */ +export async function putFiles(folder, files) { + for (const [path, text] of Object.entries(files)) { + const bytes = new TextEncoder().encode(text); + const response = await fetch(`${folder}${encodePath(path)}`, { + method: "PUT", + headers: { "Content-Length": `${bytes.length}` }, + body: bytes, + }); + await response.body?.cancel(); + if (!response.ok) throw new Error(`PUT ${path}: ${response.status}`); + } +} + +/** + * Gives the texts of the files, and the folders, of a folder of a remote. + * @param {string} folder + * @returns {Promise<{ files: Record, folders: string[] } | null>} + * null when the folder does not exist. + */ +export async function contents(folder) { + /** @type {Record} */ + const files = {}; + const folders = []; + let items; + try { + items = await walk(folder); + } catch { + return null; + } + for await (const item of items) { + if (item.IsDir) { + folders.push(item.Path); + } else { + files[item.Path] = await (await fetch(item.location)).text(); + } + } + return { files, folders: folders.sort() }; +} + +/** + * Keeps the lines of the log of the commands during a test. + * @param {import("node:test").TestContext} t + * @returns {string[]} The lines, with no time. + */ +export function captureLog(t) { + /** @type {string[]} */ + const lines = []; + t.mock.method(logger, "write", (line) => { + lines.push(line.replace(/^\S+ \S+ /, "").trimEnd()); + }); + return lines; +} diff --git a/cmd/touch/main.js b/cmd/touch/main.js new file mode 100644 index 0000000..7c55088 --- /dev/null +++ b/cmd/touch/main.js @@ -0,0 +1,104 @@ +import { isTrue } from "@rfetch/lib/options"; + +import { fetch } from "../../fetch.js"; +import { + failure, + log, + OperationError, + stat, + walk, + withFlags, +} from "../operations.js"; + +/** The formats of `--timestamp`, as rclone. */ +const TIMESTAMP = /^\d{4}-\d\d-\d\d(T\d\d:\d\d:\d\d(\.\d{1,9})?)?$/; + +/** + * Create new file or change file modification time, as `rclone touch`. + * + * A file that does not exist gets 0 bytes, unless `--no-create` or + * `--recursive`. The backends cannot set the time of a file yet: thus a file + * that exists, and `--timestamp`, give the error of rclone for a remote that + * cannot set the time. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function touch(location, flags = {}) { + try { + const timestamp = flags.timestamp; + if (timestamp !== undefined && !TIMESTAMP.test(`${timestamp}`)) { + throw new OperationError( + `failed to parse timestamp argument: ${timestamp}`, + 400, + ); + } + const info = await stat(location, flags); + if (!info) { + if (isTrue(flags.no_create)) { + log( + "NOTICE", + location, + "Not touching nonexistent file due to --no-create", + flags, + ); + return new Response(null); + } + if (isTrue(flags.recursive)) { + log( + "NOTICE", + location, + "Not touching nonexistent file due to --recursive", + flags, + ); + return new Response(null); + } + if (timestamp !== undefined) { + throw new OperationError( + "failed to touch (create): can't set modified time", + 501, + ); + } + if (isTrue(flags.dry_run)) { + log( + "NOTICE", + location, + "Skipped touch (create) as --dry-run is set", + flags, + ); + return new Response(null); + } + const response = await fetch(withFlags(location, flags), { + method: "PUT", + headers: { "Content-Length": "0" }, + body: new Uint8Array(0), + }); + await response.body?.cancel(); + if (!response.ok) { + throw new OperationError( + `failed to touch (create): ${response.status} ${response.statusText}` + .trim(), + response.status, + ); + } + return new Response(null); + } + // The files of a folder, as rclone: with `--recursive`, also the files + // of its folders. + let files = info.isDir ? 0 : 1; + if (info.isDir) { + const depth = isTrue(flags.recursive) ? Infinity : 1; + for await ( + const item of await walk(location, flags, { maxDepth: depth }) + ) { + if (!item.IsDir) files++; + } + } + if (files > 0 && !isTrue(flags.dry_run)) { + throw new OperationError("failed to touch: can't set modified time", 501); + } + return new Response(null); + } catch (error) { + return failure(error); + } +} diff --git a/cmd/touch/main_test.js b/cmd/touch/main_test.js new file mode 100644 index 0000000..e052b55 --- /dev/null +++ b/cmd/touch/main_test.js @@ -0,0 +1,36 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { contents } from "../testing.js"; +import { touch } from "./main.js"; + +test("touch makes an empty file, as rclone", async () => { + assert.equal((await touch(":memory:/touch/new.txt")).status, 200); + assert.deepEqual((await contents(":memory:/touch/")).files, { + "new.txt": "", + }); + assert.equal( + (await touch(":memory:/touch/none.txt", { no_create: true })).status, + 200, + ); + assert.deepEqual(Object.keys((await contents(":memory:/touch/")).files), [ + "new.txt", + ]); +}); + +test("touch tells that the backends cannot set the time yet", async () => { + const response = await touch(":memory:/touch/new.txt"); + assert.equal(response.status, 501); + assert.equal( + await response.text(), + "failed to touch: can't set modified time", + ); + const timestamp = await touch(":memory:/touch/t.txt", { + timestamp: "2020-01-02T03:04:05", + }); + assert.equal(timestamp.status, 501); + assert.equal( + (await touch(":memory:/touch/t.txt", { timestamp: "yesterday" })).status, + 400, + ); +}); diff --git a/cmd/tree/main.js b/cmd/tree/main.js new file mode 100644 index 0000000..d8b0cf2 --- /dev/null +++ b/cmd/tree/main.js @@ -0,0 +1,325 @@ +import { isTrue } from "@rfetch/lib/options"; + +import { failure, nameOf, OperationError, stat, walk } from "../operations.js"; + +/** + * A file or a folder of the tree. + * @typedef {Object} Node + * @property {string} name + * @property {string} path The path from the root, with a `/` at the start. + * @property {boolean} isDir + * @property {number} size + * @property {string} modTime + * @property {Node[]} children + */ + +/** + * List the contents of the remote in a tree like fashion, as `rclone tree` + * and the `tree` command of Unix: + * + * ```text + * / + * ├── file1 + * └── subdir + * └── file2 + * + * 1 directories, 2 files + * ``` + * + * As rclone, the names that start with `.` are not in the tree, unless + * `--all`. The flags of rclone: `--all`, `--dirs-only`, `--dirsfirst`, + * `--full-path`, `--level` (or `--max-depth`), `--modtime`, `--noindent`, + * `--noreport`, `--quote`, `--size` (with `--human-readable`), `--sort` + * (name, version, size or mtime), `--sort-modtime` and `--sort-reverse`. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function tree(location, flags = {}) { + try { + const info = await stat(location, flags); + if (!info) throw new OperationError("directory not found", 404); + /** @type {Node} */ + const root = { + name: "/", + path: "/", + isDir: true, + size: -1, + modTime: "", + children: [], + }; + const all = isTrue(flags.all); + if (!info.isDir) { + const name = nameOf(location); + root.children.push({ + name, + path: `/${name}`, + isDir: false, + size: info.size, + modTime: info.modTime, + children: [], + }); + } else { + const level = Number(flags.level || flags.max_depth || Infinity); + /** @type {Map} */ + const folders = new Map([["", root]]); + const items = []; + for await ( + const item of await walk(location, flags, { + maxDepth: level > 0 ? level : Infinity, + }) + ) { + items.push(item); + } + // A folder comes before the files in it in the walk, but a child can + // come before the node of a folder of the same level. + for (const item of items) { + if (!item.IsDir) continue; + folders.set(item.Path, nodeOf(item)); + } + for (const item of items) { + const slash = item.Path.lastIndexOf("/"); + const parent = folders.get( + slash === -1 ? "" : item.Path.slice(0, slash), + ); + if (!parent) continue; + if (!all && item.Name.startsWith(".")) continue; + if (!all && hidden(item.Path)) continue; + parent.children.push( + item.IsDir ? folders.get(item.Path) : nodeOf(item), + ); + } + } + + const dirsOnly = isTrue(flags.dirs_only); + const counts = { dirs: 0, files: 0 }; + prune(root, dirsOnly, counts); + sortTree(root, flags); + let text = ""; + print(root, "", flags, (line) => text += line); + if (!isTrue(flags.noreport)) { + // As rclone: with no file, the count of the folders is 0 too. + if (counts.files === 0) counts.dirs = 0; + text += `\n${counts.dirs} directories${ + dirsOnly ? "" : `, ${counts.files} files` + }\n`; + } + return new Response(text); + } catch (error) { + return failure(error); + } +} + +/** + * Tells if a path has a folder with a name that starts with `.`. + * @param {string} path + */ +function hidden(path) { + return path.split("/").some((name) => name.startsWith(".")); +} + +/** + * The node of an item of a walk. + * @param {import("../operations.js").WalkItem} item + * @returns {Node} + */ +function nodeOf(item) { + return { + name: item.Name, + path: `/${item.Path}`, + isDir: item.IsDir, + size: item.Size, + modTime: item.ModTime ?? "", + children: [], + }; +} + +/** + * Removes the files for `--dirs-only`, and counts the folders and the files + * below the root. + * @param {Node} node + * @param {boolean} dirsOnly + * @param {{ dirs: number, files: number }} counts + */ +function prune(node, dirsOnly, counts) { + if (dirsOnly) node.children = node.children.filter((child) => child.isDir); + for (const child of node.children) { + if (child.isDir) { + counts.dirs++; + prune(child, dirsOnly, counts); + } else { + counts.files++; + } + } +} + +/** + * Compares two names as Go compares two strings: by the bytes of UTF-8, that + * is by the code points. + * @param {string} a + * @param {string} b + */ +function compareNames(a, b) { + const x = [...a]; + const y = [...b]; + for (let i = 0; i < x.length && i < y.length; i++) { + const difference = x[i].codePointAt(0) - y[i].codePointAt(0); + if (difference) return difference; + } + return x.length - y.length; +} + +/** + * Tells if a name comes before another in the natural order of versions, as + * `NaturalLess` of the tree package: the numbers by their values. + * @param {string} a + * @param {string} b + */ +function naturalLess(a, b) { + const digit = (c) => c >= "0" && c <= "9"; + let i = 0, j = 0; + while (i < a.length && j < b.length) { + const c1 = a[i], c2 = b[j]; + const d1 = digit(c1), d2 = digit(c2); + if (d1 !== d2) return d1; + if (!d1) { + if (c1 !== c2) return c1 < c2; + i++; + j++; + continue; + } + while (i < a.length && a[i] === "0") i++; + while (j < b.length && b[j] === "0") j++; + const start1 = i, start2 = j; + while (i < a.length && digit(a[i])) i++; + while (j < b.length && digit(b[j])) j++; + const n1 = a.slice(start1, i), n2 = b.slice(start2, j); + if (n1.length !== n2.length) return n1.length < n2.length; + if (n1 !== n2) return n1 < n2; + if (start1 !== start2) return start1 < start2; + } + return a.length < b.length; +} + +/** + * Sorts the children of each folder, as the tree package: by name, or by the + * flags. `--dirsfirst` puts the folders first, and keeps the order of the + * names in each part. + * @param {Node} node + * @param {Object} flags + */ +function sortTree(node, flags) { + const sort = `${flags.sort ?? ""}`; + /** @type {(a: Node, b: Node) => number} */ + let compare; + const time = (n) => Date.parse(n.modTime) || 0; + if (isTrue(flags.sort_modtime) || sort === "mtime") { + compare = (a, b) => time(a) - time(b); + } else if (isTrue(flags.dirsfirst)) { + compare = (a, b) => Number(b.isDir) - Number(a.isDir); + } else if (sort === "version") { + compare = (a, b) => + naturalLess(a.name, b.name) ? -1 : naturalLess(b.name, a.name) ? 1 : 0; + } else if (sort === "size") { + compare = (a, b) => a.size - b.size; + } else { + compare = (a, b) => compareNames(a.name, b.name); + } + const reverse = isTrue(flags.sort_reverse); + const visit = (node) => { + // The order of the names before the sort, as the list of rclone. + node.children.sort((a, b) => compareNames(a.name, b.name)); + node.children.sort((a, b) => reverse ? compare(b, a) : compare(a, b)); + for (const child of node.children) visit(child); + }; + visit(node); +} + +/** + * Formats a size as the tree package, for `--human-readable`: for example + * `52`, `64K` or `2.0M`. + * @param {number} bytes + */ +function formatBytes(bytes) { + const units = ["K", "M", "G", "T", "P", "E"]; + let unit = ""; + let value = bytes; + for (let i = units.length; i > 0; i--) { + const scale = 2 ** (10 * i); + if (bytes > scale) { + unit = units[i - 1]; + value = bytes / scale; + break; + } + } + if (!unit) return `${Math.round(bytes)}`; + return `${value >= 10 ? Math.round(value) : value.toFixed(1)}${unit}`; +} + +/** + * The size of a folder: the sizes of all the files in it. + * @param {Node} node + * @returns {number} + */ +function folderSize(node) { + let total = 0; + for (const child of node.children) { + total += child.isDir ? folderSize(child) : Math.max(child.size, 0); + } + return total; +} + +/** + * Writes the time of `--modtime`: `Jan 02 15:04` in this year, else + * `Jan 02 2006`. + * @param {string} modTime + */ +function formatTime(modTime) { + const date = new Date(modTime); + if (isNaN(date.getTime())) return ""; + const month = date.toLocaleString("en-US", { month: "short" }); + const day = `${date.getDate()}`.padStart(2, "0"); + if (date.getFullYear() !== new Date().getFullYear()) { + return `${month} ${day} ${date.getFullYear()}`; + } + const hours = `${date.getHours()}`.padStart(2, "0"); + const minutes = `${date.getMinutes()}`.padStart(2, "0"); + return `${month} ${day} ${hours}:${minutes}`; +} + +/** + * Writes a node and the nodes in it. + * @param {Node} node + * @param {string} indent + * @param {Object} flags + * @param {(text: string) => void} write + */ +function print(node, indent, flags, write) { + const props = []; + const human = isTrue(flags.human_readable); + if (isTrue(flags.size)) { + const bytes = node.isDir ? folderSize(node) : node.size; + props.push( + human ? formatBytes(bytes).padStart(4) : `${bytes}`.padStart(11), + ); + } + if (!node.isDir && isTrue(flags.modtime)) { + props.push(formatTime(node.modTime)); + } + if (props.length) write(`[${props.join(" ")}] `); + let name = node.path === "/" || isTrue(flags.full_path) + ? node.path + : node.name; + if (isTrue(flags.quote)) name = `"${name}"`; + write(`${name}\n`); + const noIndent = isTrue(flags.noindent); + node.children.forEach((child, index) => { + let add = ""; + if (!noIndent) { + const last = index === node.children.length - 1; + write(`${indent}${last ? "└── " : "├── "}`); + add = last ? " " : "│ "; + } + print(child, indent + add, flags, write); + }); +} diff --git a/cmd/tree/main_test.js b/cmd/tree/main_test.js new file mode 100644 index 0000000..442a725 --- /dev/null +++ b/cmd/tree/main_test.js @@ -0,0 +1,75 @@ +import { before, test } from "node:test"; +import assert from "node:assert/strict"; + +import { fetch } from "../../fetch.js"; +import { putFiles } from "../testing.js"; +import { tree } from "./main.js"; + +const ROOT = ":memory:/tree/"; + +before(async () => { + await putFiles(ROOT, { + "f.txt": "hello", + "a/g.txt": "world!", + "a/b/zero": "", + ".hidden/x": "x", + "B.txt": "b", + "é.txt": "e", + }); + await (await fetch(`${ROOT}empty/inner/`, { method: "PUT" })).body?.cancel(); +}); + +/** @param {Object} [flags] */ +async function text(flags) { + return await (await tree(ROOT, flags)).text(); +} + +test("tree draws the folders and the files, as rclone", async () => { + assert.equal( + await text(), + `/ +├── B.txt +├── a +│ ├── b +│ │ └── zero +│ └── g.txt +├── empty +│ └── inner +├── f.txt +└── é.txt + +4 directories, 5 files +`, + ); +}); + +test("tree takes the flags of rclone", async () => { + assert.equal( + await text({ dirs_only: true, noreport: true }), + "/\n├── a\n│ └── b\n└── empty\n └── inner\n", + ); + assert.equal( + await text({ level: "1", dirsfirst: true }), + `/ +├── a +├── empty +├── B.txt +├── f.txt +└── é.txt + +2 directories, 3 files +`, + ); + assert.match(await text({ all: true }), /├── \.hidden\n│ └── x\n/); + assert.match(await text({ size: true }), /\[ {10}6\] {2}g\.txt\n/); + assert.match(await text({ full_path: true }), /└── \/a\/g\.txt\n/); + assert.match(await text({ sort_reverse: true }), /^\/\n├── é\.txt\n/); +}); + +test("tree of a file, and of a path that does not exist", async () => { + assert.equal( + await (await tree(`${ROOT}a/g.txt`)).text(), + "/\n└── g.txt\n\n0 directories, 1 files\n", + ); + assert.equal((await tree(`${ROOT}missing`)).status, 404); +}); diff --git a/cmd/version/main.js b/cmd/version/main.js new file mode 100644 index 0000000..448b281 --- /dev/null +++ b/cmd/version/main.js @@ -0,0 +1,23 @@ +import process from "node:process"; + +import info from "../../package.json" with { type: "json" }; + +/** + * Show the version number, as `rclone version`: the version of rfetch, and + * the runtime, the system and the processor that run it. + * @returns {Response} + */ +export function version() { + const { versions } = process; + const runtime = versions.bun + ? `bun ${versions.bun}` + : versions.deno + ? `deno ${versions.deno}` + : `node ${versions.node}`; + return new Response( + `rfetch v${info.version}\n` + + `- os/type: ${process.platform}\n` + + `- os/arch: ${process.arch}\n` + + `- runtime: ${runtime}\n`, + ); +} diff --git a/cmd/version/main_test.js b/cmd/version/main_test.js new file mode 100644 index 0000000..75cee6b --- /dev/null +++ b/cmd/version/main_test.js @@ -0,0 +1,11 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import info from "../../package.json" with { type: "json" }; +import { version } from "./main.js"; + +test("version gives the version of rfetch and the runtime", async () => { + const lines = (await version().text()).split("\n"); + assert.equal(lines[0], `rfetch v${info.version}`); + assert.match(lines[3], /^- runtime: (node|deno|bun) \d/); +}); diff --git a/lib/testing/suite.js b/lib/testing/suite.js index f34dd7a..96d779b 100644 --- a/lib/testing/suite.js +++ b/lib/testing/suite.js @@ -147,7 +147,11 @@ export function backendSuite( it(`keeps the name ${JSON.stringify(name)}`, { skip }, async () => { // The name is a folder and a file in it. const path = `${name}/${name}`; - const content = random(64); + // The name has no extension. A server can guess the type of the + // file from its bytes, as `rclone serve webdav` does, and random + // bytes can look like text. The bytes 0 to 63 are not text, and + // they are not the start of a known type of file. + const content = Uint8Array.from({ length: 64 }, (_, i) => i); try { const start = Date.now(); await put(path, content); diff --git a/main.js b/main.js index 2f39be3..2269780 100755 --- a/main.js +++ b/main.js @@ -59,12 +59,23 @@ import { encodeLocation } from "@rfetch/lib/fspath"; /** TODO merge global flags into config */ const response = await module[name](...args, flags); if (!response.ok) { - // As rclone: the exit code 3 is "directory not found", and 1 is an - // other error. - const text = (await response.text()).trim() || response.statusText || + // As rclone: the exit code 3 is "directory not found", 4 is "object not + // found", and 1 is an other error. A command with an output and an + // error, for example `check --combined -`, gives the error in + // `statusText`, and the output in the body. + let text = (await response.text()).trim(); + if (response.statusText && text) { + stdout.write(`${text}\n`); + text = response.statusText; + } + text ||= response.statusText || (response.status === 404 ? "not found" : `status ${response.status}`); stderr.write(`Error: ${text}\n`); - process.exitCode = response.status === 404 ? 3 : 1; + process.exitCode = response.status !== 404 + ? 1 + : /object not found$/.test(text) + ? 4 + : 3; return; } if (response.body) { diff --git a/main_test.js b/main_test.js index fbaaef3..d845d58 100644 --- a/main_test.js +++ b/main_test.js @@ -594,10 +594,15 @@ test("the command line", async (t) => { assert.deepEqual(JSON.parse(lsjson.stdout).map(({ Name }) => Name), [ "a#b.txt", ]); - const copy = await run(folder, ["copy", "c?d.txt", "copy #1.txt"]); - assert.equal(copy.code, 0, copy.stderr); + const copyto = await run(folder, ["copyto", "c?d.txt", "copy #1.txt"]); + assert.equal(copyto.code, 0, copyto.stderr); const cat = await run(folder, ["cat", "copy #1.txt"]); assert.equal(cat.stdout, "c?d.txt"); + // As rclone: `copy` puts a file in the target folder. + const copy = await run(folder, ["copy", "a#b.txt", "in #2"]); + assert.equal(copy.code, 0, copy.stderr); + const inside = await run(folder, ["cat", "in #2/a#b.txt"]); + assert.equal(inside.stdout, "a#b.txt"); }); await t.test("serve reads --addr, and a TRACE does not stop it", async () => {