diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f425cb8..19556f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,8 +2,8 @@ # # - npm test: the tests, with the oldest Node.js of package.json and the # newer versions. The job installs rclone, 7-Zip, Go, go-pmtiles, -# versatiles and Azurite, thus the tests that compare rfetch with these -# tools, or that run on them, run too. +# versatiles, Azurite and pyftpdlib, thus the tests that compare rfetch +# with these tools, or that run on them, run too. # - deno fmt --check: the format of the files. # - wasm: the tests of the crate of backend/crypt/secretbox.wasm, and its # build from its source gives the same bytes as the file in the repository. @@ -119,16 +119,30 @@ jobs: "$RUNNER_TEMP/azurite/node_modules/.bin/azurite-blob" --version echo "AZURITE=$RUNNER_TEMP/azurite/node_modules/.bin/azurite-blob" >> "$GITHUB_ENV" + # pyftpdlib is a server of FTP, with explicit and implicit TLS through + # pyOpenSSL: the tests of the ftp backend run on it. The versions of + # all the packages are fixed. + - name: Install pyftpdlib + run: | + python3 -m venv "$RUNNER_TEMP/pyftpdlib" + "$RUNNER_TEMP/pyftpdlib/bin/pip" install --quiet --no-deps \ + pyftpdlib==2.2.0 pyasynchat==1.0.5 pyasyncore==1.0.5 \ + pyOpenSSL==26.4.0 cryptography==50.0.2 cffi==2.1.1 \ + pycparser==3.11 typing_extensions==4.16.0 + "$RUNNER_TEMP/pyftpdlib/bin/pip" check + "$RUNNER_TEMP/pyftpdlib/bin/python" -I -c "from pyftpdlib.handlers import TLS_FTPHandler; import pyftpdlib.servers, pyftpdlib, OpenSSL; print(pyftpdlib.__ver__, OpenSSL.__version__)" + echo "PYFTPDLIB_PYTHON=$RUNNER_TEMP/pyftpdlib/bin/python" >> "$GITHUB_ENV" + - run: npm ci - # A test against rclone, 7-Zip, Go, go-pmtiles, versatiles or Azurite - # skips when the tool is not installed. + # A test against rclone, 7-Zip, Go, go-pmtiles, versatiles, Azurite or + # pyftpdlib skips when the tool is not installed. # Here the tools are installed, thus such a skip is an error. - name: npm test shell: bash run: | npm test 2>&1 | tee "$RUNNER_TEMP/test.log" - if grep -E "((rclone|7-Zip|Go|go-pmtiles|versatiles|Azurite|openssl) is not installed|node:sqlite is not in)" "$RUNNER_TEMP/test.log"; then + if grep -E "((rclone|7-Zip|Go|go-pmtiles|versatiles|Azurite|openssl|pyftpdlib) is not installed|pyftpdlib, pyOpenSSL or openssl is not installed|node:sqlite is not in)" "$RUNNER_TEMP/test.log"; then echo "::error::A test skipped because a tool is not installed." exit 1 fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ba3895..45b51a0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -279,9 +279,9 @@ A backend that uploads a large file in parts, for example s3 and azureblob, reads the body with `Parts` of `@rfetch/lib/parts`: each part has the size of the option `chunk_size`, and only one part is in memory. -The path of a remote is from its root. For sftp, as in rclone, the root is the -login folder of the user, and a path that starts with `/` is from the root of -the server: `fetch.js` gives the same pathname for `sftp:a.txt` and for +The path of a remote is from its root. For sftp and ftp, as in rclone, the root +is the login folder of the user, and a path that starts with `/` is from the +root of the server: `fetch.js` gives the same pathname for `sftp:a.txt` and for `sftp:/a.txt`, thus it tells a backend with `RFETCH_TYPE_ROOT` of `env`, for example `RFETCH_SFTP_ROOT`, which is `/` for a path that starts with `/`. A config file, a variable, a header or the query cannot set it. @@ -479,6 +479,18 @@ They also compare rfetch with the sftp backend of rclone on the same server, for example with keys in the format of OpenSSH that the tests write and rclone reads. The tests in `backend/sftp/ssh/ssh_test.js` need no server. +The tests of `ftp` use `rclone serve ftp`, with no TLS, and pyftpdlib, with no +TLS, with explicit TLS and with implicit TLS, and with MFMT, which sets the time +of a file. The certificates come from `openssl` for each run. The tests on +pyftpdlib run when the Python of the variable `PYFTPDLIB_PYTHON`, or `python3`, +has pyftpdlib and pyOpenSSL: get them with +`pip install pyftpdlib==2.2.0 pyOpenSSL==26.4.0`, for example in a venv. From +Python 3.12, pip also installs pyasynchat and pyasyncore for pyftpdlib. The +tests also compare rfetch with the ftp backend of rclone on the same servers. +`rclone serve ftp` has no MFMT, and its implicit TLS refuses PBSZ and PROT: the +ftp backend of rclone and of rfetch cannot use it. The tests in +`backend/ftp/list_test.js` need no server. + The tests of `azureblob` use Azurite, the emulator of Azure Storage, with its data in memory: they run when `azurite-blob` is in the `PATH`, or when the variable `AZURITE` is its path. Get it with `npm install azurite@3.37.0`. The @@ -537,8 +549,43 @@ message of the commit. ## Continuous integration The workflow `.github/workflows/ci.yml` runs for each push and each pull -request. It installs rclone, 7-Zip, Go, go-pmtiles, versatiles and Azurite (the -runner has openssl), runs `npm test` with Node.js 22, 24 and 26, and runs -`deno fmt --check`. A test that needs one of these tools skips when the tool is -not installed: in the workflow, such a skip is an error. Before a push, run -`npm test` and `deno fmt --check`. +request. It installs rclone, 7-Zip, Go, go-pmtiles, versatiles, Azurite and +pyftpdlib (the runner has openssl), runs `npm test` with Node.js 22, 24 and 26, +and runs `deno fmt --check`. A test that needs one of these tools skips when the +tool is not installed: in the workflow, such a skip is an error. Before a push, +run `npm test` and `deno fmt --check`. + +## Problems in other tools + +These are problems that the tests of rfetch found in other tools. rfetch has a +workaround for each of them. + +- rclone 1.75.1: the ftp backend loses the last bytes of an upload with explicit + TLS 1.3. With pyftpdlib 2.2.0, `rclone copyto` of a file of 1 MiB, 8 MiB or 64 + MiB gives `corrupted on transfer: sizes differ`. With `disable_tls13=true`, + the upload is correct. The cause is probably the close of the data connection + while data of TLS 1.3 that the client did not read waits: then the connection + resets, and the server loses the bytes that it did not read. rfetch had the + same problem. Now it ends the writes, reads the answer 226, and then closes + the data connection. +- rclone 1.75.1: `rclone serve ftp` with `--cert` and `--key` is implicit TLS, + and it answers 550 to PBSZ and to PROT. The ftp backend of rclone stops at + PBSZ, with `550 Action not taken`. rfetch also stops: with no `PROT P`, the + data connections are not encrypted. +- workerd 1.20260926.1 to 1.20261019.1: `tls.connect({ socket })` on a socket of + `node:net` throws + `Cannot call releaseLock() on a reader with outstanding + read promises` when + a read waits on the socket. A socket waits for data all the time after it + starts to read. No compatibility date or flag changes this: the internal + streams of workerd always throw in `releaseLock()` when a read waits. In + workerd, the ftp backend reads the control connection of explicit FTPS with + the option `onread` of `net.connect`, and its callback stops the reads after + each chunk (`backend/ftp/socket.js`). +- Deno 2.9.6: a socket of `node:tls` loses its end when the server closes the + connection while the socket is paused for some time, for example 200 ms: the + events `end` and `close` do not come. Node.js and Bun are correct. The ftp + backend ends a download at the answer 226 of the server, and does not pause + the data connection of a list. In Deno, `tls.connect({ socket })` also does + not start TLS on a socket that reads with `onread`: thus only workerd uses + `onread`. diff --git a/README.md b/README.md index 61d23dd..46fdc68 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ A [Rclone wrapper for WebAssembly build](./rclone.js) is also available. [:page_facing_up:](https://rclone.org/azureblob/) - DOI: read the files of a dataset with a DOI, on Zenodo, InvenioRDM and Dataverse [:page_facing_up:](https://rclone.org/doi/) +- FTP: the files of an FTP server, also with TLS (FTPS), in Node.js, Deno, Bun + and Cloudflare Workers [:page_facing_up:](https://rclone.org/ftp/) - Google Drive [:page_facing_up:](https://rclone.org/drive/) - HTTP [:page_facing_up:](https://rclone.org/http/) - The local filesystem [:page_facing_up:](https://rclone.org/local/) diff --git a/backend/ftp/client.js b/backend/ftp/client.js new file mode 100644 index 0000000..c3de429 --- /dev/null +++ b/backend/ftp/client.js @@ -0,0 +1,612 @@ +/** + * A client of FTP, as RFC 959, with the passive mode (EPSV of RFC 2428, and + * PASV), the lists and the times of RFC 3659, and TLS of RFC 4217: from the + * start (implicit FTPS) or after `AUTH TLS` (explicit FTPS). + */ + +import { Connection, ConnectionError } from "./socket.js"; +import { parseLIST, parseMLSD, parseTime } from "./list.js"; + +/** An answer of the server that is an error, with its code. */ +export class FTPError extends Error { + /** @param {number} code @param {string} message */ + constructor(code, message) { + super(`${code} ${message}`); + this.code = code; + } +} + +/** The largest line of an answer. */ +const MAX_LINE = 64 * 1024; + +/** + * The options of a connection. + * @typedef {Object} ClientOptions + * @property {string} host + * @property {number} port + * @property {string} user + * @property {string} password + * @property {boolean} [tls] True for implicit FTPS. + * @property {boolean} [explicitTls] True for explicit FTPS. + * @property {boolean} [checkCertificate] False for no check of the + * certificate. + * @property {boolean} [disableEpsv] + * @property {boolean} [disableMlsd] + * @property {boolean} [disableUtf8] + * @property {boolean} [forceListHidden] `LIST -a` in place of MLSD. + * @property {boolean} [writingMdtm] `MDTM` with a time sets the time of a + * file, as vsftpd. + * @property {boolean} [disableTls13] TLS 1.2 at most. + * @property {boolean} [tlsCache] False for no TLS session of the control + * connection on the data connections. + * @property {number} [timeout] The time to connect, in milliseconds. + * @property {number} [shutTimeout] The time to wait for the answer at the + * end of a transfer, in milliseconds. 0 is no limit. + * @property {number} [closeTimeout] The time to wait for the answer at the + * end of a download, in milliseconds. 0 is no limit. + */ + +/** + * An answer of the server. + * @typedef {{ code: number, lines: string[] }} Reply + */ + +/** A connection of FTP, with the user signed in. */ +export class FTPClient { + /** @type {Connection} */ + #control; + /** @type {ClientOptions} */ + #options; + /** The features of the server, from FEAT, by their names in upper case. */ + /** @type {Map} */ + features = new Map(); + /** The commands, one after the other. */ + /** @type {Promise} */ + #queue = Promise.resolve(); + /** True when the connection is not in a known state. */ + broken = false; + #epsv = true; + + /** + * @param {Connection} control + * @param {ClientOptions} options + */ + constructor(control, options) { + this.#control = control; + this.#options = options; + this.#epsv = !options.disableEpsv; + } + + /** + * Opens a connection, and signs in the user. + * @param {ClientOptions} options + * @returns {Promise} + * @throws {FTPError | ConnectionError} + */ + static async connect(options) { + const control = await Connection.open({ + host: options.host, + port: options.port, + tls: options.tls, + startTls: options.explicitTls, + checkCertificate: options.checkCertificate, + disableTls13: options.disableTls13, + timeout: options.timeout, + }); + const client = new FTPClient(control, options); + try { + await client.#start(); + } catch (error) { + control.close(); + throw error; + } + return client; + } + + async #start() { + expect(await this.#reply(), 220); + if (this.#options.explicitTls) { + expect(await this.#send("AUTH TLS"), 234); + await this.#control.startTls({ + host: this.#options.host, + port: this.#options.port, + checkCertificate: this.#options.checkCertificate, + disableTls13: this.#options.disableTls13, + timeout: this.#options.timeout, + }); + } + let reply = await this.#send(`USER ${this.#options.user}`); + if (reply.code === 331) { + reply = await this.#send(`PASS ${this.#options.password}`); + } + expect(reply, 230); + if (this.#options.tls || this.#options.explicitTls) { + expect(await this.#send("PBSZ 0"), 200); + expect(await this.#send("PROT P"), 200); + } + reply = await this.#send("FEAT"); + if (reply.code === 211) { + for (const line of reply.lines.slice(1, -1)) { + const text = line.trim(); + const space = text.indexOf(" "); + const name = (space === -1 ? text : text.slice(0, space)).toUpperCase(); + this.features.set(name, space === -1 ? "" : text.slice(space + 1)); + } + } + if (this.features.has("UTF8") && !this.#options.disableUtf8) { + await this.#send("OPTS UTF8 ON"); + } + expect(await this.#send("TYPE I"), 200); + } + + /** True when the server gives the entry of a path with MLST. */ + get mlst() { + return this.features.has("MLST") && !this.#options.disableMlsd; + } + + /** True when the server lists with MLSD. */ + get mlsd() { + return this.mlst && !this.#options.forceListHidden; + } + + /** True when the server gives the time of a file with MDTM. */ + get canGetTime() { + return this.features.has("MDTM"); + } + + /** True when the server sets the time of a file. */ + get canSetTime() { + return this.features.has("MFMT") || + Boolean(this.#options.writingMdtm && this.features.has("MDTM")); + } + + /** True when the connection cannot take more commands. */ + get closed() { + return this.broken || this.#control.ended; + } + + /** + * Reads an answer, with its lines: `123-text` up to `123 text`. + * @returns {Promise} + */ + async #reply() { + const first = await this.#control.readLine(MAX_LINE); + const match = /^(\d{3})([ -])/.exec(first); + if (!match) { + this.broken = true; + throw new ConnectionError(`The answer ${first} is not of FTP`); + } + const lines = [first]; + if (match[2] === "-") { + while (true) { + const line = await this.#control.readLine(MAX_LINE); + lines.push(line); + if (line.startsWith(`${match[1]} `)) break; + } + } + return { code: Number(match[1]), lines }; + } + + /** + * Sends a command, and reads its answer. + * @param {string} command + * @returns {Promise} + */ + async #send(command) { + if (/[\r\n]/.test(command)) { + throw new FTPError(501, "A name cannot have a line break"); + } + await this.#control.write(`${command}\r\n`); + return await this.#reply(); + } + + /** + * Runs work with the control connection, after the work before it. + * @template T + * @param {() => Promise} work + * @returns {Promise} + */ + #run(work) { + const result = this.#queue.then(async () => { + if (this.broken) throw new ConnectionError("The connection is broken"); + try { + return await work(); + } catch (error) { + // An error of the connection leaves it in a state that is not known. + if (!(error instanceof FTPError)) this.broken = true; + throw error; + } + }); + this.#queue = result.then(() => {}, () => {}); + return result; + } + + /** + * Sends a command, and gives its answer. An answer of 4xx or 5xx throws. + * @param {string} command + * @returns {Promise} + */ + command(command) { + return this.#run(async () => { + const reply = await this.#send(command); + if (reply.code >= 400) { + throw new FTPError(reply.code, message(reply)); + } + return reply; + }); + } + + /** + * Opens a data connection with EPSV, or with PASV. The host of PASV is + * not used: the data connection goes to the host of the control + * connection, as the clients of FTP do now. + * @param {boolean} [backpressure] False for data that the client reads to + * its end at once, for example a list. + * @returns {Promise} + */ + async #openData(backpressure = true) { + let port = 0; + if (this.#epsv) { + const reply = await this.#send("EPSV"); + const match = /\(([^\d\s])\1\1(\d+)\1\)/.exec(reply.lines.at(-1) ?? ""); + if (reply.code === 229 && match) port = Number(match[2]); + else this.#epsv = false; + } + if (!port) { + const reply = await this.#send("PASV"); + const match = /(\d+),(\d+),(\d+),(\d+),(\d+),(\d+)/.exec( + reply.lines.at(-1) ?? "", + ); + if (reply.code !== 227 || !match) { + throw new FTPError(reply.code, message(reply)); + } + port = Number(match[5]) * 256 + Number(match[6]); + } + const secure = Boolean(this.#options.tls || this.#options.explicitTls); + return await Connection.open({ + host: this.#options.host, + port, + tls: secure, + checkCertificate: this.#options.checkCertificate, + disableTls13: this.#options.disableTls13, + // Many servers want the TLS session of the control connection. + session: secure && this.#options.tlsCache !== false + ? this.#control.session + : undefined, + timeout: this.#options.timeout, + backpressure, + }); + } + + /** + * Reads the answer at the end of a transfer, for a time at most. + * @param {number} [time] In milliseconds, 0 for no limit. + * @returns {Promise} + * @throws {ConnectionError} After the time. + */ + async #endReply(time = 0) { + if (!time) return await this.#reply(); + /** @type {ReturnType | undefined} */ + let timer; + try { + return await Promise.race([ + this.#reply(), + new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new ConnectionError("The server did not end the transfer"), + ), + time, + ); + }), + ]); + } finally { + clearTimeout(timer); + } + } + + /** + * Runs a command with a data connection: the data connection opens before + * the command, as the passive mode needs. It closes after the answer at + * the end of the transfer: a close before, with data to read, for example + * a ticket of TLS 1.3, can reset the connection, and then the server can + * lose the last bytes of an upload. + * @template T + * @param {string} command + * @param {(data: Connection) => Promise} transfer Reads the data to its + * end, or writes the data and ends the writes. + * @param {boolean} [backpressure] False for data that the client reads to + * its end at once. + * @returns {Promise} + */ + transfer(command, transfer, backpressure = true) { + return this.#run(async () => { + const data = await this.#openData(backpressure); + try { + const reply = await this.#send(command); + if (reply.code !== 125 && reply.code !== 150) { + throw new FTPError(reply.code, message(reply)); + } + let result; + try { + result = await transfer(data); + } catch (error) { + // The state of the control connection is not known. + this.broken = true; + throw error; + } + const end = await this.#endReply(this.#options.shutTimeout); + if (end.code >= 400) throw new FTPError(end.code, message(end)); + return result; + } finally { + data.close(); + } + }); + } + + /** + * Starts a download with RETR, from an offset. The control connection + * waits until the end of the stream. A download that stops before the end + * of the file leaves the connection broken: the server can give 426, 451 + * or 226 for it. + * @param {string} path + * @param {number} offset + * @param {number} length The number of bytes to read, Infinity for all + * the bytes after the offset. + * @param {boolean} toEnd True when the bytes go to the end of the file. + * @returns {Promise>} + */ + retrieve(path, offset, length, toEnd) { + return new Promise((resolve, reject) => { + this.#run(async () => { + const data = await this.#openData(); + // REST comes just before RETR: some servers forget it after EPSV. + if (offset > 0) { + const reply = await this.#send(`REST ${offset}`).catch((error) => { + data.close(); + throw error; + }); + if (reply.code !== 350) { + data.close(); + throw new FTPError(reply.code, message(reply)); + } + } + const reply = await this.#send(`RETR ${path}`); + if (reply.code !== 125 && reply.code !== 150) { + data.close(); + throw new FTPError(reply.code, message(reply)); + } + let left = length; + let finished = false; + /** @type {(value?: unknown) => void} */ + let ended = () => {}; + const end = new Promise((done) => ended = done); + /** @param {boolean} complete */ + const finish = async (complete) => { + if (finished) return; + finished = true; + data.close(); + if (complete) { + const last = await this.#endReply(this.#options.closeTimeout) + .catch(() => null); + if (!last || last.code >= 400) this.broken = true; + } else { + this.broken = true; + } + ended(); + }; + // The end of the file: the end of the data connection, or the + // answer at the end of the transfer, which comes after the last + // byte. Deno 2.9 loses the end of a TLS connection that ends while + // the socket is paused. + const finishAtEnd = async () => { + if (finished) return; + finished = true; + const last = this.#endReply(this.#options.closeTimeout) + .catch(() => null); + const drained = (async () => { + try { + while (await data.read() !== null) { + // A file that grows: the bytes after its size go. + } + } catch { + // The data connection closes below. + } + })(); + await Promise.race([drained, last]); + data.close(); + const reply = await last; + if (!reply || reply.code >= 400) this.broken = true; + ended(); + }; + resolve( + new ReadableStream({ + async pull(controller) { + try { + if (left > 0) { + const chunk = await data.read(); + if (chunk === null) { + await finish(true); + if (left === Infinity) controller.close(); + else { + controller.error( + new Error("The file is shorter than its size"), + ); + } + return; + } + const part = chunk.length > left + ? chunk.subarray(0, left) + : chunk; + left -= part.length; + controller.enqueue(part); + if (left > 0) return; + } + // All the bytes of the range are read. + await (toEnd ? finishAtEnd() : finish(false)); + controller.close(); + } catch (error) { + await finish(false); + controller.error(error); + } + }, + async cancel() { + await finish(false); + }, + }), + ); + await end; + }).catch(reject); + }); + } + + /** + * Gives the time of a file with MDTM. + * @param {string} path + * @returns {Promise} The time in ISO 8601, or "". + */ + async modTime(path) { + if (!this.canGetTime) return ""; + const reply = await this.command(`MDTM ${path}`); + return parseTime((reply.lines.at(-1) ?? "").slice(4).trim()); + } + + /** + * Sets the time of a file, to the second, with MFMT, or with MDTM as + * vsftpd. + * @param {string} path + * @param {Date} time In the years 1000 to 9999. + */ + async setModTime(path, time) { + const text = time.toISOString().replace(/\.\d+Z$/, "").replace(/\D/g, ""); + const command = this.features.has("MFMT") ? "MFMT" : "MDTM"; + await this.command(`${command} ${text} ${path}`); + } + + /** + * Gives the size of a file with SIZE. + * @param {string} path + * @returns {Promise} -1 when the server does not give it. + */ + async size(path) { + if (!this.features.has("SIZE")) return -1; + const reply = await this.command(`SIZE ${path}`); + const size = Number((reply.lines.at(-1) ?? "").slice(4).trim()); + return Number.isSafeInteger(size) && size >= 0 ? size : -1; + } + + /** + * The entry of a path, with MLST. + * @param {string} path + * @returns {Promise} null for a path + * that does not exist. + */ + async stat(path) { + try { + const reply = await this.command(`MLST ${path}`); + for (const line of reply.lines.slice(1, -1)) { + const item = parseMLSD(line.replace(/^ /, "").replace(/ .*$/, " x")); + if (item) return { ...item, name: path }; + } + return null; + } catch (error) { + // 550 and 450: no such file. 501: a name that the server does not + // take, as rclone. + if ( + error instanceof FTPError && (error.code >= 500 || error.code === 450) + ) { + return null; + } + throw error; + } + } + + /** + * The entries of a folder, with MLSD, or with LIST. + * @param {string} path + * @returns {Promise} + */ + async list(path) { + const mlsd = this.mlsd; + const verb = mlsd + ? "MLSD" + : this.#options.forceListHidden + ? "LIST -a" + : "LIST"; + // No path is the current folder: the login folder. + const command = path === "" ? verb : `${verb} ${path}`; + const text = await this.transfer(command, async (data) => { + /** @type {Uint8Array[]} */ + const chunks = []; + for (let chunk = await data.read(); chunk; chunk = await data.read()) { + chunks.push(chunk); + } + return new TextDecoder().decode(await new Blob(chunks).arrayBuffer()); + }, false); + /** @type {import("./list.js").Item[]} */ + const items = []; + for (const line of text.split(/\r?\n/)) { + if (line === "") continue; + const item = mlsd ? parseMLSD(line) : parseLIST(line); + if (item) items.push(item); + } + return items; + } + + /** + * Uploads a file with STOR. + * @param {string} path + * @param {ReadableStream} body + * @returns {Promise} The number of bytes. + */ + async store(path, body) { + let size = 0; + await this.transfer(`STOR ${path}`, async (data) => { + for await (const chunk of body) { + size += chunk.length; + await data.write(chunk); + } + // The end of the writes is the end of the file. + await data.end(); + }); + return size; + } + + /** + * Tells if the connection keeps the process of Node.js running. + * @param {boolean} active + */ + setActive(active) { + this.#control.setActive(active); + } + + /** Closes the connection, with QUIT. */ + async close() { + if (!this.broken) { + await Promise.race([ + this.#run(() => this.#send("QUIT")).catch(() => {}), + new Promise((resolve) => setTimeout(resolve, 1000)), + ]); + } + this.#control.close(); + } +} + +/** + * Makes sure that an answer has a code, else throws its error. + * @param {Reply} reply + * @param {number} code + */ +function expect(reply, code) { + if (reply.code !== code) throw new FTPError(reply.code, message(reply)); +} + +/** + * The text of an answer. + * @param {Reply} reply + */ +function message(reply) { + return reply.lines.map((line) => line.replace(/^\d{3}[ -]/, "")).join(" ") + .trim(); +} diff --git a/backend/ftp/list.js b/backend/ftp/list.js new file mode 100644 index 0000000..dca4e6f --- /dev/null +++ b/backend/ftp/list.js @@ -0,0 +1,141 @@ +/** + * Reads the lines of the lists of FTP: MLSD and MLST of RFC 3659, and LIST, + * which has the form of `ls -l` on Unix, or the form of MS-DOS. + */ + +/** + * An entry of a list. + * @typedef {Object} Item + * @property {string} name + * @property {"file" | "dir" | "link"} type + * @property {number} size -1 when the server does not give it. + * @property {string} modTime The time in ISO 8601, to the second, or "" + * when the server does not give an exact time. + * @property {string} [target] The target of a link of LIST. + */ + +/** + * Reads a line of MLSD or of MLST: facts, a space and the name. + * @param {string} line For example + * `type=file;size=3;modify=20121127181505; a.txt`. + * @returns {Item | null} null for `.`, `..` and a line that is not correct. + */ +export function parseMLSD(line) { + // MLST gives a space before the facts. + const text = line.replace(/^ /, ""); + const space = text.indexOf(" "); + if (space === -1) return null; + const name = text.slice(space + 1); + /** @type {Record} */ + const facts = {}; + for (const fact of text.slice(0, space).split(";")) { + const equal = fact.indexOf("="); + if (equal > 0) { + facts[fact.slice(0, equal).toLowerCase()] = fact.slice(equal + 1); + } + } + const type = (facts.type ?? "").toLowerCase(); + if (type === "cdir" || type === "pdir" || name === "." || name === "..") { + return null; + } + /** @type {Item["type"]} */ + let kind = "file"; + if (type === "dir") kind = "dir"; + else if (/^os\.unix=s(?:ym)?link/.test(type)) kind = "link"; + else if (type !== "file" && type !== "") return null; + const size = facts.size ?? facts.sizd ?? ""; + return { + name, + type: kind, + size: kind !== "dir" && /^\d+$/.test(size) ? Number(size) : -1, + modTime: parseTime(facts.modify ?? ""), + }; +} + +/** + * Reads a time of MLSD or of MDTM: `YYYYMMDDHHMMSS`, with an optional part + * of a second, in UTC. + * @param {string} text + * @returns {string} The time in ISO 8601 to the second, or "". + */ +export function parseTime(text) { + const match = /^(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(?:\.\d+)?$/.exec(text); + if (!match) return ""; + const [, year, month, day, hour, minute, second] = match; + const time = Date.UTC( + Number(year), + Number(month) - 1, + Number(day), + Number(hour), + Number(minute), + Number(second), + ); + if (Number.isNaN(time)) return ""; + // A list with a precision of 1 s has no part of a second. + return new Date(time).toISOString().replace(/\.000Z$/, "Z"); +} + +/** A line of `ls -l`: the type and the permissions, then the fields. */ +const UNIX = /^([-dlbcpsD])[-rwxsStTlL+.@]{9}\S*\s+(.*)$/; + +/** The date of `ls -l`: a month, a day, and a time or a year. */ +const UNIX_DATE = + /^(?:(\S+)\s+)*?(\d+)\s+([A-Za-z]{3})\s+(\d{1,2})\s+(\d{1,2}:\d\d|\d{4})\s(.*)$/; + +/** A line of MS-DOS: `01-02-23 03:04PM name`. */ +const DOS = + /^(\d\d)-(\d\d)-(\d\d(?:\d\d)?)\s+(\d\d):(\d\d)([AP]M)?\s+(|\d+)\s+(.+)$/i; + +/** + * Reads a line of LIST. The times of LIST are not exact, for example with + * no second or no year: the item has no time. + * @param {string} line + * @returns {Item | null} null for `.`, `..`, a total and a line that is not + * correct. + */ +export function parseLIST(line) { + const unix = UNIX.exec(line); + if (unix) { + const [, letter, rest] = unix; + // The fields before the date: links, user, group and size. The size is + // the last number before the month. + const date = UNIX_DATE.exec(rest); + if (!date) return null; + const size = Number(date[2]); + let name = date[6]; + /** @type {Item["type"]} */ + let type = "file"; + /** @type {string | undefined} */ + let target; + if (letter === "d") type = "dir"; + else if (letter === "l") { + type = "link"; + const arrow = name.indexOf(" -> "); + if (arrow !== -1) { + target = name.slice(arrow + 4); + name = name.slice(0, arrow); + } + } else if (letter !== "-") return null; + if (name === "." || name === "..") return null; + return { + name, + type, + size: type === "dir" ? -1 : size, + modTime: "", + ...(target === undefined ? {} : { target }), + }; + } + const dos = DOS.exec(line); + if (dos) { + const name = dos[8]; + if (name === "." || name === "..") return null; + const folder = dos[7].toUpperCase() === ""; + return { + name, + type: folder ? "dir" : "file", + size: folder ? -1 : Number(dos[7]), + modTime: "", + }; + } + return null; +} diff --git a/backend/ftp/list_test.js b/backend/ftp/list_test.js new file mode 100644 index 0000000..3f49dd2 --- /dev/null +++ b/backend/ftp/list_test.js @@ -0,0 +1,148 @@ +/** + * The tests of the lists of FTP, with no server: MLSD, the times of MDTM, + * and LIST of Unix and of MS-DOS. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fc from "fast-check"; + +import { parseLIST, parseMLSD, parseTime } from "./list.js"; + +test("parseMLSD reads the facts of RFC 3659", () => { + for ( + const [line, expected] of [ + [ + "type=file;size=3;modify=20121127181505; a.txt", + { + name: "a.txt", + type: "file", + size: 3, + modTime: "2012-11-27T18:15:05Z", + }, + ], + [ + "Type=dir;Modify=20121127181505.123;Perm=el; my folder", + { + name: "my folder", + type: "dir", + size: -1, + modTime: "2012-11-27T18:15:05Z", + }, + ], + [ + "type=OS.unix=slink:/a/b;size=4; link", + { name: "link", type: "link", size: 4, modTime: "" }, + ], + [ + // MLST gives a space before the facts. + " type=file;sizd=10; name with two spaces ", + { name: "name with two spaces ", type: "file", size: 10, modTime: "" }, + ], + ["type=file; ;x", { name: ";x", type: "file", size: -1, modTime: "" }], + ["type=cdir; .", null], + ["type=pdir; ..", null], + ["type=file; ..", null], + ["type=OS.unix=chr-13/29; tty", null], + ["no-space", null], + ] + ) { + assert.deepEqual(parseMLSD(line), expected, line); + } +}); + +test("parseTime reads the times of MLSD and MDTM, to the second", () => { + assert.equal(parseTime("20121127181505"), "2012-11-27T18:15:05Z"); + assert.equal(parseTime("20121127181505.999"), "2012-11-27T18:15:05Z"); + assert.equal(parseTime("19700101000000"), "1970-01-01T00:00:00Z"); + for (const text of ["", "2012112718150", "2012-11-27", "20121127181505Z"]) { + assert.equal(parseTime(text), "", text); + } + fc.assert( + fc.property( + fc.date({ + min: new Date("1000-01-01T00:00:00Z"), + max: new Date("9999-12-31T23:59:59Z"), + noInvalidDate: true, + }), + (date) => { + const second = new Date(Math.floor(date.getTime() / 1000) * 1000); + const text = second.toISOString().slice(0, 19).replace(/\D/g, ""); + assert.equal( + parseTime(text), + second.toISOString().replace(".000Z", "Z"), + ); + }, + ), + ); +}); + +test("parseLIST reads the lines of ls -l and of MS-DOS", () => { + for ( + const [line, expected] of [ + [ + "-rw-r--r-- 1 user group 1024 Nov 27 18:15 a b.txt", + { name: "a b.txt", type: "file", size: 1024, modTime: "" }, + ], + [ + "drwxr-xr-x 2 user group 4096 Jan 2 2020 folder", + { name: "folder", type: "dir", size: -1, modTime: "" }, + ], + [ + "lrwxrwxrwx 1 user group 7 Jan 2 2020 link -> a b.txt", + { + name: "link", + type: "link", + size: 7, + modTime: "", + target: "a b.txt", + }, + ], + [ + // No group, as some servers. + "-rw-r--r-- 1 owner 5 Mar 3 09:07 leading space", + { name: " leading space", type: "file", size: 5, modTime: "" }, + ], + [ + "-rw-r--r--+ 1 user group 12 Dec 31 23:59 acl.txt", + { name: "acl.txt", type: "file", size: 12, modTime: "" }, + ], + [ + "01-02-23 03:04PM My Folder", + { name: "My Folder", type: "dir", size: -1, modTime: "" }, + ], + [ + "01-02-2023 03:04 1234 file.bin", + { name: "file.bin", type: "file", size: 1234, modTime: "" }, + ], + ["drwxr-xr-x 2 user group 4096 Jan 2 2020 .", null], + ["drwxr-xr-x 2 user group 4096 Jan 2 2020 ..", null], + ["crw-rw-rw- 1 root root 1, 3 Jan 2 2020 null", null], + ["total 12", null], + ["", null], + ] + ) { + assert.deepEqual(parseLIST(line), expected, line); + } +}); + +test("parseLIST keeps each name of ls -l", () => { + fc.assert( + fc.property( + fc.string({ minLength: 1 }).filter((name) => + !/[\r\n]/.test(name) && name !== "." && name !== ".." && + !name.includes(" -> ") + ), + fc.nat(), + (name, size) => { + const line = + `-rw-r--r-- 1 user group ${size} Nov 27 18:15 ${name}`; + assert.deepEqual(parseLIST(line), { + name, + type: "file", + size, + modTime: "", + }); + }, + ), + ); +}); diff --git a/backend/ftp/main.js b/backend/ftp/main.js new file mode 100644 index 0000000..e9b52f3 --- /dev/null +++ b/backend/ftp/main.js @@ -0,0 +1,941 @@ +/** + * FTP backend + * + * The ftp backend reads and writes the files of an FTP server, as the ftp + * backend of rclone, also with TLS (FTPS). Its client of FTP is in + * JavaScript, on `node:net` and `node:tls`, thus it runs in Node.js, Deno, + * Bun and workerd. + * + * A path is from the login folder of the user, as in rclone, for example + * `:ftp,host=example.com:docs/a.txt`. A path that starts with `/` is from + * the root of the server, for example `:ftp,host=example.com:/pub/a.txt`. + * The fetch of rfetch tells this to the backend with `RFETCH_FTP_ROOT` of + * `env`. + * + * With no TLS, the password and the files go in clear on the network: use + * `tls` or `explicit_tls` when the server has TLS. + * + * The options, with the names of rclone: + * + * - `host`, `port` (21 by default) and `user` (by default the user of the + * system, when there is one). For anonymous FTP, set `user=anonymous`. + * - `pass`: the password, obscured as in rclone. + * - `tls`: TLS from the start (implicit FTPS), usually on the port 990. + * - `explicit_tls`: TLS after `AUTH TLS` (explicit FTPS). The client sends + * the user and the password only after TLS starts. + * - `no_check_certificate`: no check of the certificate of the server. Then + * a server in the middle can get the password. workerd always checks the + * certificate: there, this option gives 501. + * - `disable_tls13`: TLS 1.2 at most. In workerd, this option gives 501. + * - `tls_cache_size`: 0 sends no TLS session of the control connection on + * the data connections. Other values send it, which many servers want. + * - `concurrency`: the number of connections at the same time, 0 for no + * limit. A request waits for a free connection. + * - `disable_epsv`, `disable_mlsd`, `disable_utf8` and `force_list_hidden` + * (`LIST -a` in place of MLSD). + * - `writing_mdtm`: `MDTM` with a time sets the time of a file, as + * vsftpd. Else only a server with `MFMT` can set the time of a file. + * - `idle_timeout`: the time that a connection waits for a next request, + * 60 s by default, and 0 for no limit. + * - `shut_timeout` and `close_timeout`: the time to wait for the answer of + * the server at the end of a transfer and of a download, 60 s by default. + * - `no_check_upload`: no check of the size of a file after a `PUT`. + * - `encoding`: the names that change on the server, as rclone, by default + * `Slash,Del,Ctl,RightSpace,Dot`. A CR or an LF in a name always changes, + * as in rclone: a name cannot add a command of FTP. + * + * rfetch has no proxy and asks for no password: `socks_proxy`, `http_proxy` + * and `ask_password` give 501. `allow_insecure_tls_ciphers` does nothing: + * the ciphers are the ciphers of the runtime. + * + * Other than GET, HEAD, PUT and DELETE, the backend knows `MOVE` of a file + * or of a folder (a rename on the server, as rclone), and `PATCH` with + * `X-OC-Mtime` on a server that can set the time of a file. FTP has no + * copy on the server: a `COPY` gives 501. + * + * A list has the times of the files only with MLSD: the times of LIST are + * not exact. A `GET` of a file gives `Last-Modified` with MLST or MDTM. + * As rclone, a symbolic link is a file in a list. + * + * The data connections go to the host of the control connection, not to + * the address in the answer of PASV: a server cannot send the client to + * another host. + * + * Each connection does one transfer at a time. In Node.js, Deno and Bun, a + * connection stays open for the next requests of the same remote, for + * `idle_timeout`. In workerd, each request has its own connection, as a + * connection cannot go from a request to another. There, a data connection + * cannot use the TLS session of the control connection, as `node:tls` of + * workerd does not resume sessions: a server that wants it refuses the + * transfers from workerd. + */ +import { + decodeName, + EncodeCrLf, + encodeName, + parseEncoding, +} from "@rfetch/lib/encoder"; +import { renderListing } from "@rfetch/lib/listing"; +import { mimeType } from "@rfetch/lib/mime"; +import { reveal } from "@rfetch/lib/obscure"; +import { isTrue, optionsOf } from "@rfetch/lib/options"; +import { decodePath } from "@rfetch/lib/path"; +import { notSatisfiable, parseRange, partialHeaders } from "@rfetch/lib/range"; +import { sized } from "@rfetch/lib/sized"; +import { modTimeOf, parseDuration } from "@rfetch/lib/time"; +import { FTPClient, FTPError } from "./client.js"; +import { WORKERD } from "./socket.js"; + +/** + * The options of the backend, with their types, for example for the flag + * `--ftp-host` of the command line. + */ +export const options = { + string: [ + "host", + "user", + "port", + "pass", + "concurrency", + "idle_timeout", + "close_timeout", + "tls_cache_size", + "shut_timeout", + "socks_proxy", + "http_proxy", + "encoding", + ], + boolean: [ + "tls", + "explicit_tls", + "no_check_certificate", + "disable_epsv", + "disable_mlsd", + "disable_utf8", + "writing_mdtm", + "force_list_hidden", + "disable_tls13", + "allow_insecure_tls_ciphers", + "ask_password", + "no_check_upload", + ], + default: { + port: "21", + concurrency: "0", + idle_timeout: "1m0s", + close_timeout: "1m0s", + tls_cache_size: "32", + shut_timeout: "1m0s", + encoding: "Slash,Del,Ctl,RightSpace,Dot", + }, +}; + +/** + * The options that are secret. They are not in the links of a list, which a + * browser can keep in its history and send to other sites. + */ +const SECRETS = ["user", "pass"]; + +/** The options of rclone that rfetch does not know, when they are set. */ +const UNKNOWN = ["socks_proxy", "http_proxy", "ask_password"]; + +/** The time to connect to a server, in milliseconds. */ +const CONNECT_TIMEOUT = 60000; + +/** + * The configuration of the requests. + * @typedef {Object} Config + * @property {import("./client.js").ClientOptions} client The options of + * each connection. + * @property {boolean} absolute True for a path from the root of the server. + * @property {number} encoding The mask of the encoding of the names. + * @property {number} concurrency The connections at the same time, 0 for + * no limit. + * @property {number} idleTimeout In milliseconds, 0 for no limit. + * @property {boolean} noCheckUpload + * @property {string} key The key of the connections in the pool. + */ + +/** An error with the status of the answer. */ +class FTPBackendError extends Error { + /** @param {string} message @param {number} status */ + constructor(message, status) { + super(message); + this.status = status; + } +} + +/** + * Serves an FTP remote. + * @param {Request} request + * @param {Record} [env] The options in the variables + * `RFETCH_FTP_*` or `RCLONE_FTP_*`, and `RFETCH_FTP_ROOT` of rfetch. + * @returns {Promise} + */ +async function ftpBackend(request, env = {}) { + const { method } = request; + const { pathname, searchParams } = new URL(request.url); + + let config; + try { + config = await configOf(optionsOf(request, "ftp", env), env); + } catch (error) { + await request.body?.cancel(); + if (error instanceof FTPBackendError) { + return new Response(error.message, { status: error.status }); + } + throw error; + } + const path = decodePath(pathname); + if (path === null) { + await request.body?.cancel(); + return new Response(null, { status: 400 }); + } + const isFolder = path.endsWith("/"); + const modTime = modTimeOf(request); + if (modTime === undefined) { + await request.body?.cancel(); + return new Response("X-OC-Mtime is not a time", { status: 400 }); + } + const remote = remotePath(config, path); + + if ( + !["GET", "HEAD", "PUT", "DELETE", "COPY", "MOVE", "PATCH"].includes(method) + ) { + await request.body?.cancel(); + return new Response(null, { + status: 405, + headers: { Allow: "GET, HEAD, PUT, DELETE, COPY, MOVE, PATCH" }, + }); + } + if (method === "COPY") { + await request.body?.cancel(); + return new Response("can't copy on the server", { status: 501 }); + } + + /** @type {FTPClient | null} */ + let client = null; + // A GET of a file streams its body: the connection goes back at its end. + let streaming = false; + try { + client = await acquire(config); + if (method === "GET" || method === "HEAD") { + if (isFolder) { + const query = new URLSearchParams(searchParams); + for (const name of SECRETS) query.delete(name); + return await getFolder(config, client, remote, method, `${query}`); + } + const response = await getFile(config, client, remote, path, request); + streaming = response.body !== null; + return response; + } + if (method === "PUT") { + /** @type {Record} */ + const headers = { "Content-Location": pathname }; + if (isFolder) { + await request.body?.cancel(); + await mkdirAll(client, remote); + return new Response(null, { status: 201, headers }); + } + await mkdirAll(client, parentOf(remote)); + let size; + try { + size = await client.store( + remote, + request.body ?? new Blob([]).stream(), + ); + } catch (error) { + await request.body?.cancel().catch(() => {}); + // A failed upload leaves a part of the file: it goes, as rclone. + if (client.closed) { + release(config, client); + client = null; + // The server can still write the part, as rclone says. + await new Promise((resolve) => setTimeout(resolve, 1000)); + client = await acquire(config).catch(() => null); + } + await client?.command(`DELE ${remote}`).catch(() => {}); + throw error; + } + if (!config.noCheckUpload) { + const item = await stat(client, remote); + if (!item || (item.size >= 0 && item.size !== size)) { + await client.command(`DELE ${remote}`).catch(() => {}); + throw new FTPBackendError( + `The server has ${item?.size ?? "no"} bytes, not ${size}`, + 502, + ); + } + } + if (modTime !== null && client.canSetTime) { + await setTime(client, remote, modTime); + headers["X-OC-Mtime"] = "accepted"; + } + return new Response(null, { status: 201, headers }); + } + await request.body?.cancel(); + if (method === "DELETE") { + if (isFolder) await removeFolder(client, remote); + else await client.command(`DELE ${remote}`); + return new Response(null, { status: 204 }); + } + if (method === "PATCH") { + if (isFolder) return new Response(null, { status: 405 }); + if (modTime === null) { + return new Response("X-OC-Mtime is missing", { status: 400 }); + } + if (!client.canSetTime) { + return new Response("can't set modified time", { status: 501 }); + } + const item = await stat(client, remote); + if (!item || item.type === "dir") { + return new Response(null, { status: 404 }); + } + await setTime(client, remote, modTime); + return new Response(null, { + status: 204, + headers: { "X-OC-Mtime": "accepted" }, + }); + } + // MOVE. + const target = decodePath(request.headers.get("Destination") ?? ""); + if (target === null || target === "/") { + return new Response("Destination is not a path", { status: 400 }); + } + if (target.endsWith("/") !== isFolder) { + return new Response("Destination is not of the same type", { + status: 409, + }); + } + const destination = remotePath(config, target); + return await move(client, remote, destination, isFolder, modTime); + } catch (error) { + await request.body?.cancel().catch(() => {}); + return errorResponse(error); + } finally { + if (client && !streaming) release(config, client); + } +} + +/** + * The answer of an error. + * @param {unknown} error + * @returns {Response} + */ +function errorResponse(error) { + if (error instanceof FTPBackendError) { + return new Response(error.message, { status: error.status }); + } + if (error instanceof FTPError) { + const status = { + // The file is not available: it does not exist, or the user cannot + // see it, as rclone. + 450: 404, + 550: 404, + // Not signed in. + 530: 403, + 532: 403, + // No space. + 452: 507, + 552: 507, + // A name that the server does not take. + 553: 400, + // A command that the server does not know. + 500: 501, + 502: 501, + 504: 501, + // The server stops. + 421: 503, + }[error.code] ?? 502; + return new Response(error.message, { status }); + } + if (error instanceof Error) { + // A server that does not answer, or that does not speak FTP. + return new Response(error.message, { status: 502 }); + } + throw error; +} + +/** + * Reads the options of a request, as `NewFs` of the ftp backend of rclone. + * @param {URLSearchParams} values + * @param {Record} env + * @returns {Promise} + * @throws {FTPBackendError} When an option is not correct. + */ +async function configOf(values, env) { + /** @param {string} name */ + const get = (name) => values.get(name) ?? options.default[name] ?? ""; + for (const name of UNKNOWN) { + const value = get(name); + if (value && value !== "false") { + throw new FTPBackendError(`rfetch does not know the option ${name}`, 501); + } + } + const host = get("host"); + if (!host) throw new FTPBackendError("The host option is missing", 400); + const port = Number(get("port") || "21"); + if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new FTPBackendError("The port option is not a port", 400); + } + const user = get("user") || await systemUser(); + if (!user) throw new FTPBackendError("The user option is missing", 400); + let password = ""; + const obscured = get("pass"); + if (obscured) { + try { + password = await (await reveal(obscured)).text(); + } catch { + throw new FTPBackendError("couldn't decrypt password", 400); + } + } + const tls = isTrue(get("tls")); + const explicitTls = isTrue(get("explicit_tls")); + if (tls && explicitTls) { + throw new FTPBackendError( + "implicit TLS and explicit TLS are mutually incompatible", + 400, + ); + } + const checkCertificate = !isTrue(get("no_check_certificate")); + const disableTls13 = isTrue(get("disable_tls13")); + if (WORKERD && (tls || explicitTls)) { + if (!checkCertificate) { + throw new FTPBackendError( + "workerd always checks the certificate: no_check_certificate is not possible there", + 501, + ); + } + if (disableTls13) { + throw new FTPBackendError( + "workerd chooses the version of TLS: disable_tls13 is not possible there", + 501, + ); + } + } + + let encoding; + try { + encoding = parseEncoding(get("encoding")); + } catch (error) { + throw new FTPBackendError(error.message, 400); + } + /** + * @param {string} name + * @returns {number} + */ + const count = (name) => { + const value = Number(get(name)); + if (!Number.isInteger(value) || value < 0) { + throw new FTPBackendError(`${name} must be 0 or more`, 400); + } + return value; + }; + /** + * @param {string} name + * @returns {number} + */ + const duration = (name) => { + const value = parseDuration(get(name)); + if (Number.isNaN(value)) { + throw new FTPBackendError(`${name} is not a duration`, 400); + } + return value; + }; + + /** @type {import("./client.js").ClientOptions} */ + const client = { + host, + port, + user, + password, + tls, + explicitTls, + checkCertificate, + disableTls13, + tlsCache: count("tls_cache_size") > 0, + disableEpsv: isTrue(get("disable_epsv")), + disableMlsd: isTrue(get("disable_mlsd")), + disableUtf8: isTrue(get("disable_utf8")), + writingMdtm: isTrue(get("writing_mdtm")), + forceListHidden: isTrue(get("force_list_hidden")), + timeout: CONNECT_TIMEOUT, + shutTimeout: duration("shut_timeout"), + closeTimeout: duration("close_timeout"), + }; + return { + client, + absolute: env.RFETCH_FTP_ROOT === "/", + // A CR or an LF in a name could add a command, as rclone says. + encoding: encoding | EncodeCrLf, + concurrency: count("concurrency"), + idleTimeout: duration("idle_timeout"), + noCheckUpload: isTrue(get("no_check_upload")), + // The connections of the pool, by a hash of all that makes a + // connection. + key: await hash(JSON.stringify(client)), + }; +} + +/** + * The SHA-256 of a text, in hexadecimal. + * @param {string} text + */ +async function hash(text) { + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(text), + ); + return Array.from( + new Uint8Array(digest), + (byte) => byte.toString(16).padStart(2, "0"), + ).join(""); +} + +/** + * The user of the system, as the default user of rclone. + * @returns {Promise} "" when there is no such user, for example in + * workerd. + */ +async function systemUser() { + try { + const { userInfo } = await import("node:os"); + return userInfo().username; + } catch { + return ""; + } +} + +/** + * The path on the server of a path of a request: each name with the + * encoding, from the login folder or from the root. + * @param {Config} config + * @param {string} path For example `/a/b.txt` or `/a/`. + * @returns {string} "" is the login folder, and "/" the root. + */ +function remotePath(config, path) { + const names = path.slice(1).replace(/\/$/, ""); + const encoded = names === "" + ? "" + : names.split("/").map((name) => encodeName(name, config.encoding)) + .join("/"); + return config.absolute ? `/${encoded}` : encoded; +} + +/** + * Tells if a path of the server is the root of the remote. + * @param {string} path + */ +function isRoot(path) { + return path === "" || path === "/"; +} + +/** + * The folder of a path of the server. + * @param {string} path + */ +function parentOf(path) { + const slash = path.lastIndexOf("/"); + if (slash === -1) return ""; + return slash === 0 ? "/" : path.slice(0, slash); +} + +/** + * Joins a folder of the server and a name. + * @param {string} folder + * @param {string} name + */ +function join(folder, name) { + if (folder === "") return name; + return folder.endsWith("/") ? `${folder}${name}` : `${folder}/${name}`; +} + +/** + * The connections of a configuration. + * @typedef {Object} Pool + * @property {{ client: FTPClient, timer: ReturnType | null }[]} idle + * @property {number} active The connections that requests use. + * @property {(() => void)[]} waiting The requests that wait for a + * connection. + */ + +/** + * The connections, by the keys of their configurations. workerd keeps no + * connection: a connection cannot go from a request to another. + * @type {Map} + */ +const pools = new Map(); + +/** + * The pool of each connection that a request uses. + * @type {WeakMap} + */ +const owners = new WeakMap(); + +/** + * Removes a pool with no connection and no request from the pools. + * @param {string} key + * @param {Pool} pool + */ +function forget(key, pool) { + if ( + pool.active === 0 && pool.idle.length === 0 && + pool.waiting.length === 0 && pools.get(key) === pool + ) { + pools.delete(key); + } +} + +/** + * Gives a connection for a request: a connection of the pool, or a new one. + * With `concurrency`, the request waits for a free connection. + * @param {Config} config + * @returns {Promise} + */ +async function acquire(config) { + let pool = pools.get(config.key); + if (!pool) { + pool = { idle: [], active: 0, waiting: [] }; + pools.set(config.key, pool); + } + while (config.concurrency > 0 && pool.active >= config.concurrency) { + await new Promise((resolve) => pool.waiting.push(() => resolve(undefined))); + } + pool.active++; + for (let entry = pool.idle.pop(); entry; entry = pool.idle.pop()) { + if (entry.timer) clearTimeout(entry.timer); + if (!entry.client.closed) { + entry.client.setActive(true); + owners.set(entry.client, pool); + return entry.client; + } + entry.client.close().catch(() => {}); + } + try { + const client = await FTPClient.connect(config.client); + owners.set(client, pool); + return client; + } catch (error) { + pool.active--; + const next = pool.waiting.shift(); + if (next) next(); + else forget(config.key, pool); + throw error; + } +} + +/** + * Gives a connection back after a request. In the pool, it waits for the + * next request for `idle_timeout`, and does not keep the process running. + * Else it closes. + * @param {Config} config + * @param {FTPClient} client + */ +function release(config, client) { + const pool = /** @type {Pool} */ (owners.get(client)); + owners.delete(client); + pool.active--; + const next = pool.waiting.shift(); + next?.(); + if (WORKERD || client.closed) { + client.close().catch(() => {}); + if (!next) forget(config.key, pool); + return; + } + client.setActive(false); + /** @type {Pool["idle"][number]} */ + const entry = { client, timer: null }; + if (config.idleTimeout > 0) { + entry.timer = setTimeout(() => { + const index = pool.idle.indexOf(entry); + if (index !== -1) pool.idle.splice(index, 1); + client.close().catch(() => {}); + forget(config.key, pool); + }, config.idleTimeout); + // @ts-ignore: the timer of Node.js does not keep the process running. + entry.timer.unref?.(); + } + pool.idle.push(entry); +} + +/** + * The entry of a path: with MLST, or in the list of its folder, as rclone. + * @param {FTPClient} client + * @param {string} path + * @returns {Promise} null for a path that + * does not exist. + */ +async function stat(client, path) { + if (isRoot(path)) return { name: path, type: "dir", size: -1, modTime: "" }; + if (client.mlst) return await client.stat(path); + const name = path.slice(path.lastIndexOf("/") + 1); + try { + const items = await client.list(parentOf(path)); + return items.find((item) => item.name === name) ?? null; + } catch (error) { + if ( + error instanceof FTPError && (error.code === 550 || error.code === 450) + ) { + return null; + } + throw error; + } +} + +/** + * Sets the time of a file, to the second. + * @param {FTPClient} client + * @param {string} path + * @param {bigint} nanoseconds + */ +async function setTime(client, path, nanoseconds) { + const seconds = Number(nanoseconds / 1_000_000_000n) - + (nanoseconds < 0n && nanoseconds % 1_000_000_000n !== 0n ? 1 : 0); + const time = new Date(seconds * 1000); + const year = time.getUTCFullYear(); + if (Number.isNaN(year) || year < 1000 || year > 9999) { + throw new FTPBackendError("FTP cannot keep this time", 501); + } + await client.setModTime(path, time); +} + +/** + * Gets the list of a folder, or only its headers for HEAD. + * @param {Config} config + * @param {FTPClient} client + * @param {string} path + * @param {string} method + * @param {string} query The query of each link. + * @returns {Promise} + */ +async function getFolder(config, client, path, method, query) { + const item = await stat(client, path); + if (!item || item.type !== "dir") return new Response(null, { status: 404 }); + const headers = { "Content-Type": "text/html;charset=utf-8" }; + if (method === "HEAD") return new Response(null, { headers }); + /** @type {import("@rfetch/lib/listing").Entry[]} */ + const entries = (await client.list(path)).map((item) => { + const Name = decodeName(item.name, config.encoding); + // As rclone, a link is a file. + const folder = item.type === "dir"; + return { + Name, + IsDir: folder, + Size: folder ? -1 : item.size, + MimeType: folder ? "" : mimeType(Name), + ModTime: item.modTime, + }; + }); + entries.sort((a, b) => a.Name < b.Name ? -1 : a.Name > b.Name ? 1 : 0); + return new Response(renderListing(entries, { query }), { headers }); +} + +/** + * Gets a file, or only its headers for HEAD, with the `Range` of the + * request, as RFC 9110. + * @param {Config} config + * @param {FTPClient} client + * @param {string} remote + * @param {string} path The path of the request. + * @param {Request} request + * @returns {Promise} + */ +async function getFile(config, client, remote, path, request) { + const item = await stat(client, remote); + if (!item || item.type === "dir") return new Response(null, { status: 404 }); + let size = item.size; + if (size < 0) size = await client.size(remote).catch(() => -1); + let modTime = item.modTime; + if (!modTime) modTime = await client.modTime(remote).catch(() => ""); + const name = path.slice(path.lastIndexOf("/") + 1); + const headers = new Headers({ "Content-Type": mimeType(name) }); + if (modTime) headers.set("Last-Modified", new Date(modTime).toUTCString()); + if (size < 0) { + // With no size, no range: the file goes to its end. + if (request.method === "HEAD") return new Response(null, { headers }); + const body = await client.retrieve(remote, 0, Infinity, true); + return new Response(onEnd(body, () => release(config, client)), { + headers, + }); + } + headers.set("Accept-Ranges", "bytes"); + let range = null; + if (request.method === "GET") { + try { + range = parseRange(request.headers.get("Range"), size); + } catch { + return notSatisfiable(size); + } + } + const start = range?.start ?? 0; + const length = range ? range.end - range.start + 1 : size; + headers.set("Content-Length", `${length}`); + if (range) { + for (const [key, value] of Object.entries(partialHeaders(range, size))) { + headers.set(key, value); + } + } + const status = range ? 206 : 200; + if (request.method === "HEAD" || length === 0) { + return new Response(null, { status, headers }); + } + const body = await client.retrieve( + remote, + start, + length, + start + length === size, + ); + return new Response( + sized(onEnd(body, () => release(config, client)), length), + { status, headers }, + ); +} + +/** + * Runs a function one time at the end of a stream: at its last byte, at an + * error, or when its reader cancels it. + * @param {ReadableStream} stream + * @param {() => void} done + * @returns {ReadableStream} + */ +function onEnd(stream, done) { + const reader = stream.getReader(); + let ended = false; + const end = () => { + if (ended) return; + ended = true; + done(); + }; + return new ReadableStream({ + async pull(controller) { + try { + const { done, value } = await reader.read(); + if (done) { + end(); + controller.close(); + } else { + controller.enqueue(value); + } + } catch (error) { + end(); + controller.error(error); + } + }, + async cancel(reason) { + await reader.cancel(reason).catch(() => {}); + end(); + }, + }); +} + +/** + * Makes a folder and its parent folders, as `mkdir` of rclone. A folder + * that exists is not an error. + * @param {FTPClient} client + * @param {string} path + */ +async function mkdirAll(client, path) { + if (isRoot(path)) return; + const item = await stat(client, path); + if (item) { + if (item.type === "dir") return; + throw new FTPBackendError(`${path} is a file, not a folder`, 409); + } + await mkdirAll(client, parentOf(path)); + try { + await client.command(`MKD ${path}`); + } catch (error) { + // 550 and 521: the folder exists, maybe from another request at the + // same time, as rclone. + if (!(error instanceof FTPError && [550, 521].includes(error.code))) { + throw error; + } + } +} + +/** + * Deletes a folder and all that is in it. The root of the remote stays. + * @param {FTPClient} client + * @param {string} path + */ +async function removeFolder(client, path) { + const item = await stat(client, path); + if (!item || item.type !== "dir") { + throw new FTPBackendError(`${path} is not a folder`, 404); + } + await removeContent(client, path); + if (!isRoot(path)) await client.command(`RMD ${path}`); +} + +/** + * Deletes all that is in a folder: a link is deleted, not what it points + * to. + * @param {FTPClient} client + * @param {string} path + */ +async function removeContent(client, path) { + for (const item of await client.list(path)) { + const child = join(path, item.name); + if (item.type === "dir") { + await removeContent(client, child); + await client.command(`RMD ${child}`); + } else { + await client.command(`DELE ${child}`); + } + } +} + +/** + * Renames a path on the server, with RNFR and RNTO. + * @param {FTPClient} client + * @param {string} source + * @param {string} target + */ +async function rename(client, source, target) { + await client.command(`RNFR ${source}`); + await client.command(`RNTO ${target}`); +} + +/** + * Moves a file, or a folder to a path that does not exist, with a rename on + * the server, as rclone. + * @param {FTPClient} client + * @param {string} source + * @param {string} target + * @param {boolean} isFolder + * @param {bigint | null} modTime + * @returns {Promise} + */ +async function move(client, source, target, isFolder, modTime) { + const item = await stat(client, source); + if (!item || (item.type === "dir") !== isFolder) { + return new Response(null, { status: 404 }); + } + const existing = await stat(client, target); + if (existing && (isFolder || existing.type === "dir")) { + return new Response( + isFolder ? "Destination exists" : "Destination is a folder", + { status: isFolder ? 412 : 409 }, + ); + } + await mkdirAll(client, parentOf(target)); + try { + await rename(client, source, target); + } catch (error) { + // Some servers do not rename to a file that exists. + if (!(existing && error instanceof FTPError)) throw error; + await client.command(`DELE ${target}`); + await rename(client, source, target); + } + /** @type {Record} */ + const headers = {}; + if (!isFolder && modTime !== null && client.canSetTime) { + await setTime(client, target, modTime); + headers["X-OC-Mtime"] = "accepted"; + } + return new Response(null, { status: existing ? 204 : 201, headers }); +} + +export default { + fetch: ftpBackend, +}; diff --git a/backend/ftp/main_test.js b/backend/ftp/main_test.js new file mode 100644 index 0000000..8775292 --- /dev/null +++ b/backend/ftp/main_test.js @@ -0,0 +1,536 @@ +/** + * The tests of the ftp backend: the shared tests on `rclone serve ftp`, + * and on pyftpdlib with no TLS, with explicit FTPS and with implicit FTPS; + * the check of the certificate; LIST in place of MLSD; and the comparison + * with the ftp backend of rclone on the same servers. See `testing.js` for + * the tools that the tests need. + */ +import { after, describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import process from "node:process"; + +import { obscure } from "@rfetch/lib/obscure"; +import { backendSuite } from "@rfetch/lib/testing/suite"; +import { remote } from "@rfetch/lib/testing/remote"; +// Sets the global `fetch` that reads the paths of remotes. +import { lsjson } from "../../mod.js"; +import ftp from "./main.js"; +import { + hasOpenSSL, + hasPyftpdlib, + hasRclone, + LOGIN, + servePyftpdlib, + serveRclone, +} from "./testing.js"; + +const RCLONE = process.env.RCLONE || "rclone"; + +/** The password of the test servers, obscured as in the config of rclone. */ +const pass = await (await obscure(LOGIN.pass)).text(); + +/** + * Runs a command, with no spawnSync: the server is another process, but + * the tests wait for the command with no block. + * @param {string} file + * @param {string[]} args + * @param {Record} [env] + * @returns {Promise<{ status: number, stdout: string, stderr: string }>} + */ +function run(file, args, env = {}) { + return new Promise((resolve) => { + execFile(file, args, { + encoding: "utf8", + env: { PATH: process.env.PATH, HOME: "/nonexistent", TZ: "UTC", ...env }, + }, (error, stdout, stderr) => + resolve({ + status: error ? Number(error.code ?? 1) || 1 : 0, + stdout, + stderr, + })); + }); +} + +/** + * The parameters of a test server. + * @param {import("./testing.js").TestServer} server + * @param {Record} [more] + * @returns {Record} + */ +function paramsOf(server, more = {}) { + return { + host: server.host, + port: `${server.port}`, + user: LOGIN.user, + pass, + ...more, + }; +} + +/** + * A remote of rfetch and of rclone for a test server. + * @param {Record} params + * @param {string} path + */ +function locationOf(params, path) { + const options = Object.entries(params).map(([name, value]) => + `${name}='${value}'` + ).join(","); + return `:ftp,${options}:${path}`; +} + +/** The features of a server with no MFMT, as `rclone serve ftp`. */ +const NO_TIME = { move: true, dirMove: true }; +/** The features of a server with MFMT. */ +const TIME = { move: true, dirMove: true, modTime: true, setModTime: true }; + +if (hasRclone) { + const plain = await serveRclone(); + after(() => plain.stop()); + + backendSuite("ftp", ftp, { + type: "ftp", + params: paramsOf(plain), + features: NO_TIME, + }); + backendSuite("ftp with one connection", ftp, { + params: paramsOf(plain, { concurrency: "1", idle_timeout: "0" }), + features: NO_TIME, + }); + + describe("ftp", () => { + const params = paramsOf(plain); + const fs = remote(ftp, { params, root: "/ftp-test/" }); + + test("gives 403 for a wrong password", async () => { + const wrong = await (await obscure("wrong")).text(); + const response = await remote(ftp, { params: { ...params, pass: wrong } }) + .get(""); + await response.body?.cancel(); + assert.equal(response.status, 403); + }); + + test("puts no secret in the links of a list", async () => { + await fs.put("a b.txt", new TextEncoder().encode("a")).then((r) => + r.body?.cancel() + ); + const html = await fs.get("").then((response) => response.text()); + assert.ok(html.includes(' href="a%20b.txt?host=')); + assert.ok(!html.includes("pass="), "the password is in a link"); + assert.ok(!html.includes("user="), "the user is in a link"); + }); + + test("gives 400 and 501 for options that are not correct", async () => { + for ( + const [options, status] of [ + [{ host: "" }, 400], + [{ port: "70000" }, 400], + [{ tls: "true", explicit_tls: "true" }, 400], + [{ concurrency: "-1" }, 400], + [{ idle_timeout: "1 minute" }, 400], + [{ encoding: "Nothing" }, 400], + [{ socks_proxy: "localhost:1080" }, 501], + [{ ask_password: "true" }, 501], + ] + ) { + const response = await remote(ftp, { + params: { ...params, ...options }, + }).get(""); + await response.body?.cancel(); + assert.equal(response.status, status, JSON.stringify(options)); + } + }); + + test("gives 501 for COPY", async () => { + const response = await fs.send("a b.txt", { + method: "COPY", + headers: { Destination: fs.url("c.txt").pathname }, + }); + await response.body?.cancel(); + assert.equal(response.status, 501); + }); + + test("sends no line break in a command", async () => { + // The encoding always changes a CR and an LF: a name cannot add a + // command, as in rclone. + const response = await fs.put( + "a\r\nDELE a b.txt", + new TextEncoder().encode("x"), + { "Content-Length": "1" }, + ); + await response.body?.cancel(); + assert.equal(response.status, 201); + const names = (await fs.list("")).map(({ Name }) => Name).sort(); + assert.deepEqual(names, ["a\r\nDELE a b.txt", "a b.txt"]); + const theirs = await fs.get("a b.txt"); + assert.equal(await theirs.text(), "a"); + }); + + test("reads a path from the root of the server", async () => { + // The login folder of rclone serve ftp is the root of the server. + await fs.put("root.txt", new TextEncoder().encode("root")); + const response = await fetch(locationOf(params, "/ftp-test/root.txt")); + assert.equal(await response.text(), "root"); + }); + + test("stops a download in the middle, and uses the server again", async () => { + const data = new Uint8Array(4 << 20).map((_, i) => i * 7); + await fs.put("big.bin", data).then((r) => r.body?.cancel()); + for (let i = 0; i < 3; i++) { + const response = await fs.get("big.bin"); + const reader = /** @type {ReadableStream} */ (response.body) + .getReader(); + await reader.read(); + await reader.cancel(); + } + const response = await fs.get("big.bin", { Range: "bytes=1000-1999" }); + assert.equal(response.status, 206); + assert.deepEqual( + new Uint8Array(await response.arrayBuffer()), + data.subarray(1000, 2000), + ); + }); + + test("deletes the files of the tests", async () => { + const response = await fs.delete(""); + assert.equal(response.status, 204); + }); + + describe("and rclone", () => { + /** The same remote for rclone and for rfetch. */ + const location = (path = "") => locationOf(params, `both/${path}`); + + test("rclone reads the files of rfetch", async () => { + for ( + const [path, text] of [["a.txt", "hello"], ["d/b c.txt", "world!"]] + ) { + const response = await fetch(location(path.replace(" ", "%20")), { + method: "PUT", + body: text, + }); + assert.equal(response.status, 201); + } + const { status, stdout, stderr } = await run(RCLONE, [ + "lsjson", + "-R", + "--files-only", + location(), + ]); + assert.equal(status, 0, stderr); + assert.deepEqual( + JSON.parse(stdout).map(({ Path, Size }) => [Path, Size]).sort(), + [["a.txt", 5], ["d/b c.txt", 6]], + ); + assert.equal( + (await run(RCLONE, ["cat", location("d/b c.txt")])).stdout, + "world!", + ); + }); + + test("rfetch reads the files and the times of rclone", async () => { + const { status, stderr } = await run(RCLONE, [ + "copyto", + fileURLToPath(new URL("./package.json", import.meta.url)), + location("c/new.json"), + ]); + assert.equal(status, 0, stderr); + const listed = await (await lsjson(location(), { recursive: true })) + .json(); + const theirList = JSON.parse( + (await run(RCLONE, ["lsjson", "-R", location()])).stdout, + ); + const shape = (list) => + list.map(({ Path, Size, IsDir, ModTime }) => [ + Path, + Size, + IsDir, + IsDir ? "" : Date.parse(ModTime), + ]).sort(); + assert.deepEqual(shape(listed), shape(theirList)); + }); + + test("rfetch and rclone encode the names in the same way", async () => { + // The default encoding changes a control character and a space at + // the end. + for (const name of ["a%0Bb.txt", "c%20"]) { + const response = await fetch(location(`names/${name}`), { + method: "PUT", + body: "x", + }); + assert.equal(response.status, 201); + } + // rclone shows a control character as its picture. + const theirs = JSON.parse( + (await run(RCLONE, ["lsjson", location("names")])).stdout, + ); + assert.deepEqual(theirs.map(({ Name }) => Name).sort(), [ + "a␋b.txt", + "c ", + ]); + const ours = await (await lsjson(location("names"))).json(); + assert.deepEqual(ours.map(({ Name }) => Name).sort(), [ + "a\u000bb.txt", + "c ", + ]); + await fetch(location(""), { method: "DELETE" }).then((r) => + r.body?.cancel() + ); + }); + }); + }); +} else { + test("ftp with rclone", { + skip: "rclone is not installed. Set RCLONE to the path of rclone.", + }); +} + +if (hasPyftpdlib && hasOpenSSL) { + const plain = await servePyftpdlib(); + const secure = await servePyftpdlib({ tls: "explicit" }); + const implicit = await servePyftpdlib({ tls: "implicit" }); + after(() => { + plain.stop(); + secure.stop(); + implicit.stop(); + }); + + backendSuite("ftp on pyftpdlib", ftp, { + params: paramsOf(plain), + features: TIME, + }); + // The certificates of the test servers are signed by themselves. + backendSuite("ftp on pyftpdlib with implicit TLS", ftp, { + params: paramsOf(implicit, { tls: "true", no_check_certificate: "true" }), + features: TIME, + }); + backendSuite("ftp on pyftpdlib with explicit TLS", ftp, { + params: paramsOf(secure, { + explicit_tls: "true", + no_check_certificate: "true", + }), + features: TIME, + }); + backendSuite("ftp on pyftpdlib with TLS 1.2 and no EPSV", ftp, { + params: paramsOf(secure, { + explicit_tls: "true", + no_check_certificate: "true", + disable_tls13: "true", + disable_epsv: "true", + tls_cache_size: "0", + }), + features: TIME, + }); + + describe("ftp on pyftpdlib", () => { + test("refuses a certificate that it does not trust", async () => { + const response = await remote(ftp, { + params: paramsOf(implicit, { tls: "true" }), + }).get(""); + const text = await response.text(); + assert.equal(response.status, 502); + assert.match(text, /certificate/i); + }); + + test("trusts the certificate of a known authority", async () => { + // The authorities of Node.js come at its start: another process + // trusts the certificate of the test server. The data connections + // also check it. + const main = fileURLToPath(new URL("./main.js", import.meta.url)); + const query = new URLSearchParams( + paramsOf(implicit, { host: "localhost", tls: "true" }), + ); + const script = ` + const { default: ftp } = await import(${JSON.stringify(main)}); + const request = new Request("http://rfetch.test/?${query}"); + const response = await ftp.fetch(request); + process.stdout.write(response.status + " " + await response.text()); + `; + const { stdout, stderr } = await run( + process.execPath, + ["--input-type=module", "-e", script], + { NODE_EXTRA_CA_CERTS: /** @type {string} */ (implicit.cert) }, + ); + assert.match(stdout, /^200 /, stderr); + }); + + test("refuses explicit TLS with a certificate that it does not trust", async () => { + const response = await remote(ftp, { + params: paramsOf(secure, { explicit_tls: "true" }), + }).get(""); + const text = await response.text(); + assert.equal(response.status, 502); + assert.match(text, /certificate/i); + }); + + test("downloads a large file with TLS, then uses the connection again", async () => { + // More than 1 MiB waits in the client: the data connection pauses. + const data = new Uint8Array(4 << 20).map((_, i) => i * 13); + for ( + const params of [ + paramsOf(implicit, { tls: "true", no_check_certificate: "true" }), + paramsOf(secure, { + explicit_tls: "true", + no_check_certificate: "true", + }), + ] + ) { + const fs = remote(ftp, { params, root: "/ftp-large/" }); + const put = await fs.put("big.bin", data); + await put.body?.cancel(); + assert.equal(put.status, 201); + const response = await fs.get("big.bin"); + const reader = /** @type {ReadableStream} */ (response.body) + .getReader(); + /** @type {Uint8Array[]} */ + const chunks = []; + for ( + let read = await reader.read(); + !read.done; + read = await reader.read() + ) { + chunks.push(read.value); + // A slow reader: the server ends the transfer while the client + // waits. + if (chunks.length === 1) { + await new Promise((resolve) => setTimeout(resolve, 300)); + } + } + assert.deepEqual( + new Uint8Array(await new Blob(chunks).arrayBuffer()), + data, + ); + assert.equal((await fs.delete("")).status, 204); + } + }); + + test("does not sign in with no TLS when the server wants TLS", async () => { + const response = await remote(ftp, { params: paramsOf(secure) }).get(""); + await response.body?.cancel(); + assert.ok(response.status >= 400, `${response.status}`); + }); + + test("lists with LIST, and gives the time with MDTM", async () => { + const fs = remote(ftp, { + params: paramsOf(plain, { disable_mlsd: "true" }), + root: "/ftp-list/", + }); + const put = await fs.put("d/a b.txt", new TextEncoder().encode("hello"), { + "X-OC-Mtime": "1354040105", + }); + await put.body?.cancel(); + assert.equal(put.status, 201); + assert.equal(put.headers.get("X-OC-Mtime"), "accepted"); + const entries = await fs.list("d/"); + assert.deepEqual( + entries.map(({ Name, Size, IsDir, ModTime }) => ({ + Name, + Size, + IsDir, + ModTime, + })), + // The times of LIST are not exact: the list has no time. + [{ Name: "a b.txt", Size: 5, IsDir: false, ModTime: "" }], + ); + assert.deepEqual( + (await fs.list("")).map(({ Name, IsDir }) => [Name, IsDir]), + [["d", true]], + ); + const response = await fs.get("d/a b.txt"); + assert.equal( + response.headers.get("Last-Modified"), + "Tue, 27 Nov 2012 18:15:05 GMT", + ); + assert.equal(await response.text(), "hello"); + const moved = await fs.send("d/a b.txt", { + method: "MOVE", + headers: { Destination: fs.url("e/f.txt").pathname }, + }); + await moved.body?.cancel(); + assert.equal(moved.status, 201); + assert.equal((await fs.get("e/f.txt")).status, 200); + assert.equal((await fs.delete("")).status, 204); + }); + + describe("and rclone", { + skip: !hasRclone && "rclone is not installed", + }, () => { + const params = paramsOf(plain); + /** The same remote for rclone and for rfetch. */ + const location = (path = "") => locationOf(params, `both/${path}`); + + test("rfetch and rclone use the same implicit TLS", async () => { + const secureParams = paramsOf(implicit, { + tls: "true", + no_check_certificate: "true", + }); + const response = await fetch(locationOf(secureParams, "tls/a.txt"), { + method: "PUT", + body: "secret", + }); + assert.equal(response.status, 201); + const { status, stdout, stderr } = await run(RCLONE, [ + "cat", + locationOf(secureParams, "tls/a.txt"), + ]); + assert.equal(status, 0, stderr); + assert.equal(stdout, "secret"); + await fetch(locationOf(secureParams, "tls/"), { method: "DELETE" }) + .then((r) => r.body?.cancel()); + }); + + test("rclone reads the times of rfetch", async () => { + const response = await fetch(location("a.txt"), { + method: "PUT", + headers: { "X-OC-Mtime": "1354040105.5" }, + body: "hello", + }); + assert.equal(response.status, 201); + assert.equal(response.headers.get("X-OC-Mtime"), "accepted"); + const { status, stdout, stderr } = await run(RCLONE, [ + "lsjson", + location(), + ]); + assert.equal(status, 0, stderr); + assert.deepEqual( + JSON.parse(stdout).map(({ Path, Size, ModTime }) => [ + Path, + Size, + ModTime, + ]), + [["a.txt", 5, "2012-11-27T18:15:05Z"]], + ); + }); + + test("rfetch reads the times of rclone", async () => { + const { status, stderr } = await run(RCLONE, [ + "touch", + "--timestamp", + "2020-01-02T03:04:05", + location("new.txt"), + ]); + assert.equal(status, 0, stderr); + const listed = await (await lsjson(location())).json(); + const item = listed.find((item) => item.Path === "new.txt"); + assert.equal( + Date.parse(item?.ModTime), + Date.parse("2020-01-02T03:04:05Z"), + ); + const response = await fetch(location("new.txt"), { method: "HEAD" }); + assert.equal( + response.headers.get("Last-Modified"), + "Thu, 02 Jan 2020 03:04:05 GMT", + ); + await fetch(location(""), { method: "DELETE" }).then((r) => + r.body?.cancel() + ); + }); + }); + }); +} else { + test("ftp with pyftpdlib", { + skip: + "pyftpdlib, pyOpenSSL or openssl is not installed. Set PYFTPDLIB_PYTHON to a Python with pyftpdlib and pyOpenSSL.", + }); +} diff --git a/backend/ftp/package.json b/backend/ftp/package.json new file mode 100644 index 0000000..9596308 --- /dev/null +++ b/backend/ftp/package.json @@ -0,0 +1,28 @@ +{ + "name": "@rfetch/ftp", + "version": "0.1.0", + "description": "The ftp backend of rfetch: the files of an FTP server, also with TLS, with a client of FTP on node:net and node:tls.", + "type": "module", + "exports": { + ".": "./main.js" + }, + "scripts": { + "test": "node --test" + }, + "engines": { + "node": ">=22.2.0" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/sntran/rfetch.git", + "directory": "backend/ftp" + }, + "author": "Trần Nguyễn Sơn", + "license": "ISC", + "dependencies": { + "@rfetch/lib": "^0.1.0" + }, + "devDependencies": { + "fast-check": "^3.19.0" + } +} diff --git a/backend/ftp/socket.js b/backend/ftp/socket.js new file mode 100644 index 0000000..8aa8a56 --- /dev/null +++ b/backend/ftp/socket.js @@ -0,0 +1,390 @@ +/** + * The TCP connections of FTP, with or without TLS, of `node:net` and + * `node:tls`, in Node.js, Deno, Bun and workerd. + * + * In workerd, `tls.connect({ socket })` starts TLS on a socket of + * `node:net` only when no read waits on the socket: else `releaseLock()` + * of the reader of the socket throws "Cannot call releaseLock() on a + * reader with outstanding read promises". A socket of `node:net` reads all + * the time. Thus in workerd, the control connection of explicit FTPS reads + * with `onread`, and its callback stops the reads after each chunk, as + * Node.js does when the callback gives false. A read that waits starts them + * again. Deno does not start TLS on such a socket: the other runtimes read + * with the events of the socket. + */ + +/** An error of a connection, for example a timeout. */ +export class ConnectionError extends Error {} + +/** True in workerd, the runtime of Cloudflare Workers. */ +export const WORKERD = globalThis.navigator?.userAgent === "Cloudflare-Workers"; + +/** + * The options of a connection. + * @typedef {Object} ConnectOptions + * @property {string} host + * @property {number} port + * @property {boolean} [tls] True for TLS from the start. + * @property {boolean} [startTls] True for TLS later, with `startTls()`. + * @property {boolean} [checkCertificate] False for no check of the + * certificate. workerd does not know this option. + * @property {boolean} [disableTls13] TLS 1.2 at most. workerd does not + * know this option. + * @property {unknown} [session] A TLS session to resume, of `node:tls`. + * @property {boolean} [backpressure] False for no pause of the socket when + * much data waits, for data that the user reads to its end at once. + * @property {number} [timeout] The time to connect, in milliseconds. + */ + +/** A connection that reads lines and bytes. */ +export class Connection { + /** @type {Uint8Array} */ + #buffer = new Uint8Array(0); + #ended = false; + /** @type {Transport} */ + #transport; + + /** @param {Transport} transport */ + constructor(transport) { + this.#transport = transport; + } + + /** + * Opens a connection. + * @param {ConnectOptions} options + * @returns {Promise} + * @throws {ConnectionError} + */ + static async open(options) { + const transport = new Transport(); + await transport.open(options); + return new Connection(transport); + } + + /** + * Reads more data into the buffer. + * @returns {Promise} False at the end of the connection. + */ + async #more() { + if (this.#ended) return false; + const chunk = await this.#transport.read(); + if (chunk === null) { + this.#ended = true; + return false; + } + if (this.#buffer.length === 0) this.#buffer = chunk; + else { + const joined = new Uint8Array(this.#buffer.length + chunk.length); + joined.set(this.#buffer); + joined.set(chunk, this.#buffer.length); + this.#buffer = joined; + } + return true; + } + + /** + * Reads a line that ends with CR LF, or with LF, with no line break. + * @param {number} max The largest number of bytes of the line. + * @returns {Promise} + * @throws {ConnectionError} At the end of the connection. + */ + async readLine(max) { + let from = 0; + while (true) { + const at = this.#buffer.indexOf(0x0a, from); + if (at !== -1) { + let line = this.#buffer.subarray(0, at); + this.#buffer = this.#buffer.subarray(at + 1); + if (line.at(-1) === 0x0d) line = line.subarray(0, line.length - 1); + return new TextDecoder().decode(line); + } + if (this.#buffer.length > max) { + throw new ConnectionError("A line is too long"); + } + from = this.#buffer.length; + if (!await this.#more()) { + throw new ConnectionError("The server closed the connection"); + } + } + } + + /** + * Reads the next bytes. + * @returns {Promise} null at the end of the + * connection. + */ + async read() { + if (this.#buffer.length === 0 && !await this.#more()) return null; + const bytes = this.#buffer; + this.#buffer = new Uint8Array(0); + return bytes; + } + + /** + * Writes bytes, or a text in UTF-8. + * @param {Uint8Array | string} data + */ + async write(data) { + const bytes = typeof data === "string" + ? new TextEncoder().encode(data) + : data; + await this.#transport.write(bytes); + } + + /** + * Starts TLS on the connection, as `AUTH TLS` of FTP. + * @param {ConnectOptions} options + */ + async startTls(options) { + if (this.#buffer.length > 0) { + throw new ConnectionError("The server sent data before TLS"); + } + await this.#transport.startTls(options); + } + + /** The TLS session, to resume it on another connection. */ + get session() { + return this.#transport.session; + } + + /** + * True when the connection ended, maybe with data that nobody read, for + * example `421` before the server closes an idle connection. + */ + get ended() { + return this.#ended || this.#transport.ended; + } + + /** + * Tells if the connection keeps the process of Node.js running. + * @param {boolean} active + */ + setActive(active) { + this.#transport.setActive(active); + } + + /** Ends the writes of the connection: the server sees the end. */ + async end() { + await this.#transport.end(); + } + + /** Closes the connection. */ + close() { + this.#ended = true; + this.#transport.close(); + } +} + +/** A connection of `node:net` and `node:tls`. */ +class Transport { + /** @type {any} */ + #socket = null; + /** @type {Uint8Array[]} */ + #chunks = []; + #queued = 0; + #ended = false; + /** @type {Error | null} */ + #error = null; + /** @type {(() => void) | null} */ + #wake = null; + /** True when the socket reads only while a read waits, with `onread`. */ + #pull = false; + /** True when the socket pauses when much data waits. */ + #backpressure = true; + /** @type {unknown} */ + session = undefined; + + get ended() { + return this.#ended || this.#error !== null; + } + + /** @param {ConnectOptions} options */ + async open(options) { + const net = await import("node:net"); + const tls = await import("node:tls"); + const { host, port, timeout = 60000 } = options; + const socket = options.tls + ? tls.connect({ + host, + port, + servername: isIP(host) ? undefined : host, + rejectUnauthorized: options.checkCertificate !== false, + session: /** @type {any} */ (options.session), + ...(options.disableTls13 ? { maxVersion: "TLSv1.2" } : {}), + }) + : net.connect({ + host, + port, + noDelay: true, + // TLS starts later: the socket reads only while a read waits. + ...(WORKERD && options.startTls + ? { + onread: { + buffer: () => new Uint8Array(65536), + callback: ( + /** @type {number} */ size, + /** @type {Uint8Array} */ buffer, + ) => this.#onread(size, buffer), + }, + } + : {}), + }); + this.#pull = Boolean(WORKERD && options.startTls && !options.tls); + this.#backpressure = options.backpressure !== false; + await this.#ready( + socket, + options.tls ? "secureConnect" : "connect", + timeout, + ); + this.#listen(socket); + } + + /** + * Keeps a chunk of `onread`, and stops the reads of the socket. + * @param {number} size + * @param {Uint8Array} buffer + * @returns {false} + */ + #onread(size, buffer) { + this.#chunks.push(Uint8Array.prototype.slice.call(buffer, 0, size)); + this.#queued += size; + this.#notify(); + return false; + } + + /** + * Waits until a socket connects. + * @param {any} socket + * @param {string} event + * @param {number} timeout + */ + #ready(socket, event, timeout) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + socket.destroy(); + reject(new ConnectionError("The connection timed out")); + }, timeout); + /** @param {Error} error */ + const failed = (error) => { + clearTimeout(timer); + reject(new ConnectionError(error.message)); + }; + socket.once("error", failed); + socket.once(event, () => { + clearTimeout(timer); + socket.off("error", failed); + resolve(undefined); + }); + }); + } + + /** @param {any} socket */ + #listen(socket) { + this.#socket = socket; + if (!this.#pull) { + socket.on("data", (chunk) => { + this.#chunks.push( + new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.length), + ); + this.#queued += chunk.length; + // Too much data that nobody reads stops the reads of the socket. + if (this.#backpressure && this.#queued > 1 << 20) socket.pause(); + this.#notify(); + }); + } + socket.on("end", () => { + this.#ended = true; + this.#notify(); + }); + socket.on("close", () => { + this.#ended = true; + this.#notify(); + }); + socket.on("error", (error) => { + this.#error ??= new ConnectionError(error.message); + this.#notify(); + }); + socket.on("session", (session) => this.session = session); + } + + #notify() { + const wake = this.#wake; + this.#wake = null; + wake?.(); + } + + async read() { + while (this.#chunks.length === 0) { + if (this.#error) throw this.#error; + if (this.#ended) return null; + this.#socket.resume(); + await new Promise((resolve) => this.#wake = resolve); + } + const chunk = /** @type {Uint8Array} */ (this.#chunks.shift()); + this.#queued -= chunk.length; + if (!this.#pull && this.#queued < 1 << 19) this.#socket.resume(); + return chunk; + } + + /** @param {Uint8Array} bytes */ + async write(bytes) { + if (this.#error) throw this.#error; + const socket = this.#socket; + if (socket.write(bytes)) return; + await new Promise((resolve) => { + const done = () => { + socket.off("drain", done); + socket.off("close", done); + resolve(undefined); + }; + socket.on("drain", done); + socket.on("close", done); + }); + if (this.#error) throw this.#error; + } + + /** @param {ConnectOptions} options */ + async startTls(options) { + if (this.#chunks.length > 0) { + throw new ConnectionError("The server sent data before TLS"); + } + const tls = await import("node:tls"); + const plain = this.#socket; + for (const event of ["data", "end", "close", "error", "session"]) { + plain.removeAllListeners(event); + } + this.#pull = false; + const socket = tls.connect({ + socket: plain, + servername: isIP(options.host) ? undefined : options.host, + rejectUnauthorized: options.checkCertificate !== false, + ...(options.disableTls13 ? { maxVersion: "TLSv1.2" } : {}), + }); + await this.#ready(socket, "secureConnect", options.timeout ?? 60000); + this.#listen(socket); + } + + /** @param {boolean} active */ + setActive(active) { + if (active) this.#socket?.ref?.(); + else this.#socket?.unref?.(); + } + + async end() { + await new Promise((resolve) => this.#socket.end(resolve)); + } + + close() { + this.#socket?.destroy(); + this.#ended = true; + this.#notify(); + } +} + +/** + * Tells if a host is an IP address: TLS sends no name for it. + * @param {string} host + */ +function isIP(host) { + return /^[\d.]+$/.test(host) || host.includes(":"); +} diff --git a/backend/ftp/testing.js b/backend/ftp/testing.js new file mode 100644 index 0000000..417eb58 --- /dev/null +++ b/backend/ftp/testing.js @@ -0,0 +1,252 @@ +/** + * The test servers of FTP, on a local folder, with a user and a password: + * + * - `rclone serve ftp`, with no TLS. It runs when rclone is in the PATH, or + * when RCLONE is the path of rclone. Its TLS from the start refuses PBSZ + * and PROT, thus the ftp backend of rclone cannot use it. + * - pyftpdlib, with no TLS, with `AUTH TLS` (explicit FTPS) or with TLS + * from the start (implicit FTPS), and with MFMT, which sets the time of a + * file. It runs when the Python of PYFTPDLIB_PYTHON, or `python3`, has + * pyftpdlib and pyOpenSSL. From Python 3.12, pyftpdlib also needs + * pyasynchat and pyasyncore. + * + * The certificates of TLS come from openssl. + */ +import { spawn, spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createConnection, createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import process from "node:process"; + +const RCLONE = process.env.RCLONE || "rclone"; +const PYTHON = process.env.PYFTPDLIB_PYTHON || "python3"; + +/** Tells if rclone is installed. */ +export const hasRclone = spawnSync(RCLONE, ["version"]).status === 0; + +/** Tells if openssl is installed. */ +export const hasOpenSSL = spawnSync("openssl", ["version"]).status === 0; + +/** Tells if pyftpdlib and pyOpenSSL are installed, with the servers. */ +export const hasPyftpdlib = spawnSync(PYTHON, [ + "-I", + "-c", + "from pyftpdlib.handlers import TLS_FTPHandler; import pyftpdlib.servers", +]).status === 0; + +/** The user and the password of the test servers. */ +export const LOGIN = { user: "rfetch-user", pass: "rfetch pass é" }; + +/** + * A server of pyftpdlib: the arguments are the folder, the user, the + * password, the TLS (`explicit`, `implicit` or ""), and the files of the + * certificate and of the key. The server takes a free port, and writes it + * on a line: no other process can take the port before the server. + */ +const PYFTPDLIB_SERVER = ` +import sys +from pyftpdlib.authorizers import DummyAuthorizer +from pyftpdlib.handlers import FTPHandler, TLS_FTPHandler +from pyftpdlib.servers import FTPServer + + +class ImplicitTLSHandler(TLS_FTPHandler): + # TLS starts before the first answer, as after AUTH TLS. + def handle(self): + self.secure_connection(self.ssl_context) + super().handle() + + +folder, user, password, tls, cert, key = sys.argv[1:7] +authorizer = DummyAuthorizer() +authorizer.add_user(user, password, folder, perm="elradfmwMT") +handler = { + "explicit": TLS_FTPHandler, + "implicit": ImplicitTLSHandler, +}.get(tls, FTPHandler) +if tls: + handler.certfile = cert + handler.keyfile = key + handler.tls_control_required = True + handler.tls_data_required = True +handler.authorizer = authorizer +handler.encoding = "utf8" +server = FTPServer(("127.0.0.1", 0), handler) +print(server.address[1], flush=True) +server.serve_forever() +`; + +/** + * Gives a free port of 127.0.0.1. + * @returns {Promise} + */ +function freePort() { + return new Promise((resolve) => { + const server = createServer().listen(0, "127.0.0.1", () => { + const { port } = server.address(); + server.close(() => resolve(port)); + }); + }); +} + +/** + * Makes a certificate for `localhost` and `127.0.0.1`, signed by itself. + * @param {string} folder + * @returns {{ cert: string, key: string }} The files of the certificate and + * of its key, in PEM. + */ +export function makeCertificate(folder) { + const cert = join(folder, "cert.pem"); + const key = join(folder, "key.pem"); + const { status, stderr } = spawnSync("openssl", [ + "req", + "-x509", + "-newkey", + "ec", + "-pkeyopt", + "ec_paramgen_curve:P-256", + "-nodes", + "-keyout", + key, + "-out", + cert, + "-days", + "1", + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost,IP:127.0.0.1", + ], { encoding: "utf8" }); + if (status !== 0) throw new Error(`openssl: ${stderr}`); + return { cert, key }; +} + +/** + * Waits until a port takes connections. + * @param {number} port + */ +async function waitForPort(port) { + for (let i = 0; i < 100; i++) { + const open = await new Promise((resolve) => { + const socket = createConnection({ host: "127.0.0.1", port }); + socket.once("connect", () => { + socket.destroy(); + resolve(true); + }); + socket.once("error", () => resolve(false)); + }); + if (open) return; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(`No server on the port ${port}`); +} + +/** + * A test server. + * @typedef {Object} TestServer + * @property {string} host + * @property {number} port + * @property {string} folder The local folder of the server. + * @property {string} [cert] The file of the certificate of TLS. + * @property {() => void} stop + */ + +/** + * Starts `rclone serve ftp` on a free port. + * @returns {Promise} + */ +export async function serveRclone() { + const home = mkdtempSync(join(tmpdir(), "rfetch-ftp-")); + const folder = join(home, "data"); + mkdirSync(folder); + const port = await freePort(); + const args = [ + "serve", + "ftp", + "--user", + LOGIN.user, + "--pass", + LOGIN.pass, + "--addr", + `127.0.0.1:${port}`, + ]; + args.push(folder); + const child = spawn(RCLONE, args, { + env: { PATH: process.env.PATH, HOME: home }, + stdio: "ignore", + }); + await waitForPort(port); + return { + host: "127.0.0.1", + port, + folder, + stop() { + child.kill(); + rmSync(home, { recursive: true, force: true }); + }, + }; +} + +/** + * Starts a server of pyftpdlib on a free port. + * @param {Object} [options] + * @param {"explicit" | "implicit"} [options.tls] Explicit or implicit + * FTPS: the server wants TLS for the control and the data connections. + * @returns {Promise} + */ +export async function servePyftpdlib({ tls } = {}) { + const home = mkdtempSync(join(tmpdir(), "rfetch-ftp-")); + const folder = join(home, "data"); + mkdirSync(folder); + const script = join(home, "server.py"); + writeFileSync(script, PYFTPDLIB_SERVER); + const files = tls ? makeCertificate(home) : { cert: "", key: "" }; + const child = spawn( + PYTHON, + [ + "-I", + script, + folder, + LOGIN.user, + LOGIN.pass, + tls ?? "", + files.cert, + files.key, + ], + { + env: { PATH: process.env.PATH, HOME: home, TZ: "UTC" }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + // The log of the server, for the error of a server that does not start. + let log = ""; + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (text) => { + if (log.length < 16384) log += text; + }); + const port = await new Promise((resolve, reject) => { + let output = ""; + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (text) => { + output += text; + const match = /^(\d+)\n/.exec(output); + if (match) resolve(Number(match[1])); + }); + child.once("error", reject); + child.once("exit", (code) => { + rmSync(home, { recursive: true, force: true }); + reject(new Error(`pyftpdlib stopped with the code ${code}: ${log}`)); + }); + }); + return { + host: "127.0.0.1", + port, + folder, + cert: files.cert || undefined, + stop() { + child.kill(); + rmSync(home, { recursive: true, force: true }); + }, + }; +} diff --git a/backend/main.js b/backend/main.js index ee4b640..f1f9dd0 100644 --- a/backend/main.js +++ b/backend/main.js @@ -31,6 +31,7 @@ export const backends = { crypt: () => import("@rfetch/crypt"), doi: () => import("@rfetch/doi"), drive: () => import("@rfetch/drive"), + ftp: () => import("@rfetch/ftp"), http: () => import("@rfetch/http"), local: () => import("@rfetch/local"), memory: () => import("@rfetch/memory"), diff --git a/backend/sftp/main.js b/backend/sftp/main.js index cfc9968..71e9050 100644 --- a/backend/sftp/main.js +++ b/backend/sftp/main.js @@ -40,7 +40,7 @@ * - `ciphers`, `key_exchange`, `macs` and `host_key_algorithms`: the * algorithms of SSH, with spaces between them. * - `idle_timeout`: the time that a connection waits for a next request, - * 60 s by default. + * 60 s by default, and 0 for no limit. * - `encoding`: the names that change on the server, as rclone, by default * `Slash,Ctl,Del,Dot`. * @@ -83,7 +83,7 @@ import { decodePath } from "@rfetch/lib/path"; import { notSatisfiable, parseRange, partialHeaders } from "@rfetch/lib/range"; import { parseSize } from "@rfetch/lib/size"; import { sized } from "@rfetch/lib/sized"; -import { modTimeOf } from "@rfetch/lib/time"; +import { modTimeOf, parseDuration } from "@rfetch/lib/time"; import { findHostKey } from "./knownhosts.js"; import { isDirectory, isLink, OPEN, SFTP, SFTPError, STATUS } from "./sftp.js"; import { CIPHERS, MACS } from "./ssh/cipher.js"; @@ -581,32 +581,6 @@ function chooseAlgorithms(value, known, name) { return chosen; } -/** - * Reads a duration of Go, for example `1m0s` or `30s`. - * @param {string} text - * @returns {number} Milliseconds, or NaN. - */ -function parseDuration(text) { - if (text === "0") return 0; - const units = { - ns: 1e-6, - us: 1e-3, - "µs": 1e-3, - ms: 1, - s: 1000, - m: 60000, - h: 3600000, - }; - const parts = [...text.matchAll(/(\d+(?:\.\d+)?)(ns|us|µs|ms|s|m|h)/g)]; - if (parts.length === 0 || parts.map((part) => part[0]).join("") !== text) { - return NaN; - } - return parts.reduce( - (sum, [, number, unit]) => sum + Number(number) * units[unit], - 0, - ); -} - /** * The SHA-256 of a text, in hexadecimal. * @param {string} text @@ -769,18 +743,19 @@ async function checkHostKey(config, key) { /** * Gives a session back after a request. In the pool, it waits for the - * next request for `idle_timeout`, and does not keep the process running. - * Else it closes. + * next request for `idle_timeout`, or with no limit for 0, as rclone, and + * does not keep the process running. Else it closes. * @param {Session} session */ function release(session) { session.users--; if (session.users > 0) return; - if (!POOLED || session.config.idleTimeout === 0) { + if (!POOLED) { close(session); return; } session.client.setActive(false); + if (session.config.idleTimeout === 0) return; session.timer = setTimeout(() => close(session), session.config.idleTimeout); // @ts-ignore: the timer of Node.js does not keep the process running. session.timer.unref?.(); diff --git a/lib/testing/suite.js b/lib/testing/suite.js index faf51b7..4e60287 100644 --- a/lib/testing/suite.js +++ b/lib/testing/suite.js @@ -418,7 +418,7 @@ export function backendSuite( await response.body?.cancel(); if (!setModTime) { assert.ok( - response.status === 405 || response.status === 403, + [403, 405, 501, 502].includes(response.status), `PATCH gave ${response.status}, not 405`, ); return; diff --git a/lib/time.js b/lib/time.js index 90f5c68..4ae509d 100644 --- a/lib/time.js +++ b/lib/time.js @@ -145,3 +145,30 @@ export function modTimeOf(request) { const value = request.headers.get("X-OC-Mtime"); return value === null ? null : parseUnixTime(value); } + +/** + * Reads a duration of Go, as the options of rclone, for example `1m0s`, + * `30s` or `1h30m`. `0` is no time. + * @param {string} text + * @returns {number} Milliseconds, or NaN for a text that is not correct. + */ +export function parseDuration(text) { + if (text === "0") return 0; + const units = { + ns: 1e-6, + us: 1e-3, + "µs": 1e-3, + ms: 1, + s: 1000, + m: 60000, + h: 3600000, + }; + const parts = [...text.matchAll(/(\d+(?:\.\d+)?)(ns|us|µs|ms|s|m|h)/g)]; + if (parts.length === 0 || parts.map((part) => part[0]).join("") !== text) { + return NaN; + } + return parts.reduce( + (sum, [, number, unit]) => sum + Number(number) * units[unit], + 0, + ); +} diff --git a/lib/time_test.js b/lib/time_test.js index 405aa8e..14f9f8c 100644 --- a/lib/time_test.js +++ b/lib/time_test.js @@ -6,6 +6,7 @@ import { isoStringOf, modTimeOf, nanosecondsOf, + parseDuration, parseUnixTime, toListTime, toLocaleISOString, @@ -144,3 +145,23 @@ test("modTimeOf reads the header X-OC-Mtime", () => { assert.equal(modTimeOf(request("1354040105")), 1354040105000000000n); assert.equal(modTimeOf(request("soon")), undefined); }); + +test("parseDuration reads the durations of Go", () => { + for ( + const [text, expected] of [ + ["0", 0], + ["1m0s", 60000], + ["30s", 30000], + ["1h30m", 5400000], + ["1.5s", 1500], + ["250ms", 250], + ["10us", 0.01], + ["", NaN], + ["1x", NaN], + ["s", NaN], + ["1m 0s", NaN], + ] + ) { + assert.equal(parseDuration(text), expected, text); + } +}); diff --git a/package-lock.json b/package-lock.json index d6ff81e..c70af67 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "@rfetch/crypt": "^0.1.0", "@rfetch/doi": "^0.1.0", "@rfetch/drive": "^0.1.0", + "@rfetch/ftp": "^0.1.0", "@rfetch/http": "^0.1.0", "@rfetch/lib": "^0.1.0", "@rfetch/local": "^0.1.0", @@ -148,6 +149,20 @@ "node": ">=22.2.0" } }, + "backend/ftp": { + "name": "@rfetch/ftp", + "version": "0.1.0", + "license": "ISC", + "dependencies": { + "@rfetch/lib": "^0.1.0" + }, + "devDependencies": { + "fast-check": "^3.19.0" + }, + "engines": { + "node": ">=22.2.0" + } + }, "backend/http": { "name": "@rfetch/http", "version": "0.1.0", @@ -302,6 +317,10 @@ "resolved": "backend/drive", "link": true }, + "node_modules/@rfetch/ftp": { + "resolved": "backend/ftp", + "link": true + }, "node_modules/@rfetch/http": { "resolved": "backend/http", "link": true diff --git a/package.json b/package.json index 14a3584..a3d8a5a 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "@rfetch/crypt": "^0.1.0", "@rfetch/doi": "^0.1.0", "@rfetch/drive": "^0.1.0", + "@rfetch/ftp": "^0.1.0", "@rfetch/http": "^0.1.0", "@rfetch/lib": "^0.1.0", "@rfetch/local": "^0.1.0",