From affd8692193654c725dc52ff1bc5a034cf846c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Mon, 19 Oct 2026 21:51:42 +0000 Subject: [PATCH 1/3] fix: obscure as rclone, parts with no copy, errors in the middle of an answer obscure wrote the standard base64, with "+", "/" and "=". rclone reads only the base64 of URLs with no padding: rclone could not reveal a password of rfetch. Now obscure writes the base64 of rclone, and the counter of AES-CTR has 128 bits, as the counter of Go. reveal reads the two forms, as before. A test checks the alphabet. Parts copied the rest of a large chunk for each part. The local backend reads chunks of 1 MiB: an upload in parts of 32 KiB copied about 4 GB for 64 MB. Now a part is a view of the chunk when the chunk has the part, and the parts of s3 and azureblob have no such copy too. The command line had no handler for an error after the start of an answer, for example a server that loses a file in the middle of a cat: Node.js wrote the stack of the error. Now the error is one line "Error: ...", with the exit code 1, as rclone. --- lib/obscure.js | 12 +++++++---- lib/obscure_test.js | 10 +++++++++ lib/parts.js | 50 ++++++++++++++++++++++++++------------------- main.js | 8 +++++++- 4 files changed, 54 insertions(+), 26 deletions(-) diff --git a/lib/obscure.js b/lib/obscure.js index 9d20f48..8738e0e 100644 --- a/lib/obscure.js +++ b/lib/obscure.js @@ -74,15 +74,19 @@ export async function obscure(password) { { name: ALGORITHM, counter: iv, - length: 64, + length: 128, }, await getKey(), plaintext, ); // Copies the encrypted buffer into the ciphertext buffer. ciphertext.set(new Uint8Array(encrypted), BlockSize); - // Encodes the ciphertext buffer into a base64 string. - const encoded = btoa(String.fromCharCode(...ciphertext)); + // Encodes the ciphertext buffer in the base64 of URLs with no padding, + // as rclone: rclone cannot reveal the standard base64. + const encoded = btoa(String.fromCharCode(...ciphertext)) + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=+$/, ""); return new Response(encoded); } @@ -104,7 +108,7 @@ export async function reveal(cipherText) { { name: ALGORITHM, counter: iv, - length: 64, + length: 128, }, await getKey(), data, diff --git a/lib/obscure_test.js b/lib/obscure_test.js index 1b21e32..65eb939 100644 --- a/lib/obscure_test.js +++ b/lib/obscure_test.js @@ -27,3 +27,13 @@ test("obscure", async () => { }), ); }); + +test("obscure writes the base64 of rclone", async () => { + // rclone reads only the base64 of URLs, with no padding. + await fc.assert( + fc.asyncProperty(fc.string(), async (plaintext) => { + const ciphertext = await obscure(plaintext).then((r) => r.text()); + assert.match(ciphertext, /^[A-Za-z0-9_-]+$/); + }), + ); +}); diff --git a/lib/parts.js b/lib/parts.js index a91ffe7..c9c17c0 100644 --- a/lib/parts.js +++ b/lib/parts.js @@ -19,35 +19,43 @@ export class Parts { } /** - * The next part, or null at the end. + * The next part, or null at the end. A part can be a view of a chunk of + * the stream: a large chunk is not copied for each part. * @returns {Promise} */ async next() { - const chunks = this.pending.length ? [this.pending] : []; - let length = this.pending.length; - this.pending = new Uint8Array(0); - while (length < this.size && !this.done) { - const { done, value } = await this.reader.read(); - if (done) { - this.done = true; - break; + /** @type {Uint8Array[]} */ + const chunks = []; + let length = 0; + // The chunks before the last chunk of the part. + while (length + this.pending.length < this.size && !this.done) { + if (this.pending.length) { + chunks.push(this.pending); + length += this.pending.length; + this.pending = new Uint8Array(0); } - chunks.push(value); - length += value.length; + const { done, value } = await this.reader.read(); + if (done) this.done = true; + else this.pending = value; } - if (length === 0 && this.done) return null; - const bytes = new Uint8Array(length); - let offset = 0; - for (const chunk of chunks) { - bytes.set(chunk, offset); - offset += chunk.length; + const take = Math.min(this.size - length, this.pending.length); + if (take > 0) { + chunks.push(this.pending.subarray(0, take)); + length += take; + this.pending = this.pending.subarray(take); } - if (length > this.size) { - this.pending = bytes.slice(this.size); - return bytes.subarray(0, this.size); + if (length === 0) return null; + let bytes = chunks[0]; + if (chunks.length > 1) { + bytes = new Uint8Array(length); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.length; + } } // A part of the size can be the last part: the next read tells it. - if (!this.done && this.pending.length === 0) { + while (!this.done && this.pending.length === 0) { const { done, value } = await this.reader.read(); if (done) this.done = true; else this.pending = value; diff --git a/main.js b/main.js index 2269780..10ba581 100755 --- a/main.js +++ b/main.js @@ -89,4 +89,10 @@ import { encodeLocation } from "@rfetch/lib/fspath"; if (!stdout.write(chunk)) await once(stdout, "drain"); } } -})(); +})().catch((error) => { + // An error after the start of the answer, for example a connection that + // closes in the middle of a file, as rclone. + stderr.write(`Error: ${error?.message ?? error} +`); + process.exitCode = 1; +}); From 6dbd84867a65c8044bd0c2fe454f8f452fa576d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Mon, 19 Oct 2026 21:52:00 +0000 Subject: [PATCH 2/3] feat: the sftp backend, with a client of SSH on WebCrypto and node:net The sftp backend reads and writes the files of an SFTP server, as the sftp backend of rclone. Its client of SSH is in JavaScript: it runs in Node.js, Deno, Bun and workerd. The client of SSH (backend/sftp/ssh): - The transport of RFC 4253: the key exchanges curve25519-sha256 and ecdh-sha2-nistp256, -nistp384 and -nistp521, the strict key exchange of OpenSSH against the Terrapin attack, and new keys when the server asks for them. - The ciphers aes128-gcm@openssh.com, aes256-gcm@openssh.com and AES-CTR with HMAC-SHA2-256 or HMAC-SHA2-512, also as encrypt-then-MAC, with WebCrypto. - The host keys ssh-ed25519, ECDSA and RSA with SHA-2. - The user signs in with a password, with keyboard-interactive, or with a key of OpenSSH, PKCS #8, PKCS #1 or SEC 1, of Ed25519, ECDSA or RSA. - The channels of RFC 4254, with their windows. The client of SFTP (version 3) sends the reads and the writes of a file at the same time, as rclone: chunk_size and concurrency. The backend: - A path is from the login folder, as rclone, and a path that starts with "/" is from the root of the server. fetch.js gives the backend RFETCH_SFTP_ROOT=/ for such a path: a config file, a variable, a header or the query cannot set it. - host_keys and known_hosts_file check the host key of the server, before the password goes to the server. known_hosts_file reads the hashed names, the patterns and @revoked of OpenSSH. With no option, the backend accepts each host key, as rclone. - key_file and known_hosts_file are locations that the backend reads with the fetch of rfetch. - MOVE of a file or of a folder is a rename on the server, PATCH sets the time to the second, and COPY makes a hard link only with copy_is_hardlink, as rclone. - The names have the encoding of rclone: Slash, Ctl, Del and Dot. - In Node.js, Deno and Bun, a connection waits for the next request for idle_timeout, and does not keep the process running. In workerd, each request has its own connection. - The options of rclone that rfetch does not know, for example key_use_agent and ssh, give 501. The tests run the shared suite on rclone serve sftp with a password and with a key, and with small chunks. They sign in with each type of key, use each host key, cipher and key exchange, check host_keys and known_hosts_file, and compare rfetch with the sftp backend of rclone on the same server: the files, the times, the encoded names, and keys in the format of OpenSSH that the tests write and rclone reads. The keys come from openssl and WebCrypto for each run. On this machine, a copy of 64 MB to rclone serve sftp takes 1.9 s up and 1.2 s down; rclone takes 0.4 s and 0.5 s. The calls of WebCrypto for each packet take most of the time. --- .github/workflows/ci.yml | 2 +- CONTRIBUTING.md | 22 +- README.md | 3 + backend/main.js | 1 + backend/sftp/knownhosts.js | 118 ++++ backend/sftp/main.js | 1185 +++++++++++++++++++++++++++++++++ backend/sftp/main_test.js | 394 +++++++++++ backend/sftp/package.json | 28 + backend/sftp/sftp.js | 584 ++++++++++++++++ backend/sftp/ssh/cipher.js | 282 ++++++++ backend/sftp/ssh/client.js | 625 +++++++++++++++++ backend/sftp/ssh/der.js | 125 ++++ backend/sftp/ssh/keys.js | 588 ++++++++++++++++ backend/sftp/ssh/socket.js | 169 +++++ backend/sftp/ssh/ssh_test.js | 216 ++++++ backend/sftp/ssh/transport.js | 568 ++++++++++++++++ backend/sftp/ssh/wire.js | 253 +++++++ backend/sftp/testing.js | 205 ++++++ fetch.js | 8 + package-lock.json | 19 + package.json | 1 + 21 files changed, 5391 insertions(+), 5 deletions(-) create mode 100644 backend/sftp/knownhosts.js create mode 100644 backend/sftp/main.js create mode 100644 backend/sftp/main_test.js create mode 100644 backend/sftp/package.json create mode 100644 backend/sftp/sftp.js create mode 100644 backend/sftp/ssh/cipher.js create mode 100644 backend/sftp/ssh/client.js create mode 100644 backend/sftp/ssh/der.js create mode 100644 backend/sftp/ssh/keys.js create mode 100644 backend/sftp/ssh/socket.js create mode 100644 backend/sftp/ssh/ssh_test.js create mode 100644 backend/sftp/ssh/transport.js create mode 100644 backend/sftp/ssh/wire.js create mode 100644 backend/sftp/testing.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a12f4e..f425cb8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -128,7 +128,7 @@ jobs: shell: bash run: | npm test 2>&1 | tee "$RUNNER_TEMP/test.log" - if grep -E "((rclone|7-Zip|Go|go-pmtiles|versatiles|Azurite) is not installed|node:sqlite is not in)" "$RUNNER_TEMP/test.log"; then + if grep -E "((rclone|7-Zip|Go|go-pmtiles|versatiles|Azurite|openssl) is not installed|node:sqlite is not in)" "$RUNNER_TEMP/test.log"; then echo "::error::A test skipped because a tool is not installed." exit 1 fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a7d08c3..0ba3895 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -279,6 +279,13 @@ A backend that uploads a large file in parts, for example s3 and azureblob, reads the body with `Parts` of `@rfetch/lib/parts`: each part has the size of the option `chunk_size`, and only one part is in memory. +The path of a remote is from its root. For sftp, as in rclone, the root is the +login folder of the user, and a path that starts with `/` is from the root of +the server: `fetch.js` gives the same pathname for `sftp:a.txt` and for +`sftp:/a.txt`, thus it tells a backend with `RFETCH_TYPE_ROOT` of `env`, for +example `RFETCH_SFTP_ROOT`, which is `/` for a path that starts with `/`. A +config file, a variable, a header or the query cannot set it. + A backend on another remote, for example alias, gives `fetch` the location `encodeLocation(remote) + encodePath(path)`. The option `remote` is a path of rclone: `encodeLocation` of `@rfetch/lib/fspath` keeps its `%`, `?` and `#` in @@ -466,6 +473,12 @@ default). Put the credentials in `RFETCH_S3_ACCESS_KEY_ID` and `RCLONE_`. Use a bucket only for the tests: the tests make folders in it and delete them. +The tests of `sftp` use `rclone serve sftp`, with host keys and user keys that +openssl makes for each run: they run when rclone and `openssl` are installed. +They also compare rfetch with the sftp backend of rclone on the same server, for +example with keys in the format of OpenSSH that the tests write and rclone +reads. The tests in `backend/sftp/ssh/ssh_test.js` need no server. + The tests of `azureblob` use Azurite, the emulator of Azure Storage, with its data in memory: they run when `azurite-blob` is in the `PATH`, or when the variable `AZURITE` is its path. Get it with `npm install azurite@3.37.0`. The @@ -524,7 +537,8 @@ message of the commit. ## Continuous integration The workflow `.github/workflows/ci.yml` runs for each push and each pull -request. It installs rclone, 7-Zip, Go, go-pmtiles, versatiles and Azurite, runs -`npm test` with Node.js 22, 24 and 26, and runs `deno fmt --check`. A test that -needs one of these tools skips when the tool is not installed: in the workflow, -such a skip is an error. Before a push, run `npm test` and `deno fmt --check`. +request. It installs rclone, 7-Zip, Go, go-pmtiles, versatiles and Azurite (the +runner has openssl), runs `npm test` with Node.js 22, 24 and 26, and runs +`deno fmt --check`. A test that needs one of these tools skips when the tool is +not installed: in the workflow, such a skip is an error. Before a push, run +`npm test` and `deno fmt --check`. diff --git a/README.md b/README.md index 993a00f..61d23dd 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,9 @@ A [Rclone wrapper for WebAssembly build](./rclone.js) is also available. - Memory [:page_facing_up:](https://rclone.org/memory/) - S3: Amazon S3 and the services with the same API, for example Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/) +- SFTP: the files of an SSH server, with a client of SSH in JavaScript that runs + in Node.js, Deno, Bun and Cloudflare Workers + [:page_facing_up:](https://rclone.org/sftp/) - WebDAV: Nextcloud, ownCloud, Fastmail Files, `rclone serve webdav` and other WebDAV servers [:page_facing_up:](https://rclone.org/webdav/) diff --git a/backend/main.js b/backend/main.js index a1d4ae0..ee4b640 100644 --- a/backend/main.js +++ b/backend/main.js @@ -35,6 +35,7 @@ export const backends = { local: () => import("@rfetch/local"), memory: () => import("@rfetch/memory"), s3: () => import("@rfetch/s3"), + sftp: () => import("@rfetch/sftp"), tiles: () => import("@rfetch/tiles"), union: () => import("@rfetch/union"), webdav: () => import("@rfetch/webdav"), diff --git a/backend/sftp/knownhosts.js b/backend/sftp/knownhosts.js new file mode 100644 index 0000000..334dbf4 --- /dev/null +++ b/backend/sftp/knownhosts.js @@ -0,0 +1,118 @@ +/** + * Reads a file `known_hosts` of OpenSSH, as the option `known_hosts_file` + * of rclone, and finds the host key of a server in it. + */ + +import { equalBytes } from "./ssh/wire.js"; +import { fromBase64 } from "./ssh/keys.js"; + +/** + * The name of a host in a file `known_hosts`: the host for the port 22, + * else `[host]:port`. + * @param {string} host + * @param {number} port + */ +function nameOf(host, port) { + return port === 22 ? host : `[${host}]:${port}`; +} + +/** + * Tells if a pattern of OpenSSH matches a name: `*` is any text, and `?` + * is one character. + * @param {string} pattern + * @param {string} name + */ +function matches(pattern, name) { + const source = pattern + .toLowerCase() + .replace(/[.+^${}()|[\]\\]/g, "\\$&") + .replace(/\*/g, ".*") + .replace(/\?/g, "."); + return new RegExp(`^${source}$`).test(name.toLowerCase()); +} + +/** + * Tells if a hashed name, `|1|salt|hash`, is the name: the hash is the + * HMAC-SHA1 of the name with the salt as its key. + * @param {string} entry + * @param {string} name + */ +async function matchesHash(entry, name) { + const [, , salt, hash] = entry.split("|"); + if (!salt || !hash) return false; + const key = await crypto.subtle.importKey( + "raw", + fromBase64(salt), + { name: "HMAC", hash: "SHA-1" }, + false, + ["sign"], + ); + const mac = new Uint8Array( + await crypto.subtle.sign("HMAC", key, new TextEncoder().encode(name)), + ); + return equalBytes(mac, fromBase64(hash)); +} + +/** + * Tells if the hosts of a line match the name of a server, as OpenSSH: a + * pattern with `!` before it excludes the name. + * @param {string} hosts The hosts of the line, with `,` between them. + * @param {string} name + */ +async function hostMatches(hosts, name) { + let found = false; + for (const pattern of hosts.split(",")) { + if (pattern.startsWith("|1|")) { + if (await matchesHash(pattern, name)) found = true; + } else if (pattern.startsWith("!")) { + if (matches(pattern.slice(1), name)) return false; + } else if (matches(pattern, name)) { + found = true; + } + } + return found; +} + +/** + * Finds the host key of a server in a file `known_hosts`. + * @param {string} text The content of the file. + * @param {string} host + * @param {number} port + * @param {Uint8Array} blob The host key of the server. + * @returns {Promise<"known" | "unknown" | "changed" | "revoked">} `known` + * when the file has the key for the host, `changed` when the file has + * other keys for the host, `revoked` when the file revokes the key. + */ +export async function findHostKey(text, host, port, blob) { + const name = nameOf(host, port); + let other = false; + let known = false; + for (const raw of text.split(/\r?\n/)) { + const line = raw.trim(); + if (line === "" || line.startsWith("#")) continue; + const fields = line.split(/\s+/); + let marker = ""; + if (fields[0].startsWith("@")) marker = fields.shift() ?? ""; + const [hosts, , key] = fields; + if (!hosts || !key) continue; + let keyBlob; + try { + keyBlob = fromBase64(key); + } catch { + continue; + } + const same = equalBytes(keyBlob, blob); + // A revoked key is refused for each host. + if (marker === "@revoked") { + if (same) return "revoked"; + continue; + } + // rfetch does not know the certificates of hosts. + if (marker !== "") continue; + if (!await hostMatches(hosts, name)) continue; + if (same) known = true; + else other = true; + } + if (known) return "known"; + return other ? "changed" : "unknown"; +} diff --git a/backend/sftp/main.js b/backend/sftp/main.js new file mode 100644 index 0000000..36c80b3 --- /dev/null +++ b/backend/sftp/main.js @@ -0,0 +1,1185 @@ +/** + * SFTP backend + * + * The sftp backend reads and writes the files of an SFTP server, as the + * sftp backend of rclone. Its client of SSH is in JavaScript, on WebCrypto + * and `node:net`, thus it runs in Node.js, Deno, Bun and workerd. + * + * A path is from the login folder of the user, as in rclone, for example + * `:sftp,host=example.com:docs/a.txt`. A path that starts with `/` is from + * the root of the server, for example `:sftp,host=example.com:/srv/a.txt`. + * The fetch of rfetch tells this to the backend with `RFETCH_SFTP_ROOT` of + * `env`. + * + * The options, with the names of rclone: + * + * - `host`, `port` (22 by default) and `user` (by default the user of the + * system, when there is one). + * - `pass`: the password, obscured as in rclone. The backend tries it with + * `password`, then with `keyboard-interactive`. + * - `key_pem`: a private key in PEM, with `\n` in place of the line breaks, + * or `key_file`: the location of such a file, which the backend reads with + * the fetch of rfetch. The key can be of OpenSSH, of PKCS #8, of PKCS #1 + * or of SEC 1, for Ed25519, ECDSA or RSA. A key with a passphrase gives + * 501. + * - `host_keys`: the host keys of the server, as `algorithm base64` with + * `,` between them, as in a file `known_hosts`. The server must have one + * of them. + * - `known_hosts_file`: the location of a file `known_hosts` of OpenSSH. + * The file must have the host key of the server. `none` is no check. + * - `set_modtime`: `true` by default. Then a `PUT`, a `MOVE` and a `PATCH` + * with `X-OC-Mtime` set the time of the file, to the second. + * - `skip_links`: with `true`, a list has no symbolic link. Else a link is + * the file or the folder that it points to, as in rclone. + * - `chunk_size` (32 KiB by default) and `concurrency` (64 by default): + * the size of each read and write, and the number of reads or writes at + * the same time. + * - `copy_is_hardlink`: with `true`, a COPY makes a hard link, with the + * extension `hardlink@openssh.com`. Else a COPY gives 501, as rclone has + * no copy on the server. + * - `ciphers`, `key_exchange`, `macs` and `host_key_algorithms`: the + * algorithms of SSH, with spaces between them. + * - `idle_timeout`: the time that a connection waits for a next request, + * 60 s by default. + * - `encoding`: the names that change on the server, as rclone, by default + * `Slash,Ctl,Del,Dot`. + * + * Security: with no `host_keys` and no `known_hosts_file`, the backend + * accepts each host key of the server, as rclone does. Then a server in the + * middle can get the password. Set `host_keys` to the key of the server. + * + * The algorithms: the key exchanges `curve25519-sha256` and + * `ecdh-sha2-nistp256`, `-nistp384` and `-nistp521`, the host keys + * `ssh-ed25519`, ECDSA and RSA with SHA-2, the ciphers + * `aes128-gcm@openssh.com`, `aes256-gcm@openssh.com` and AES-CTR with the + * MACs HMAC-SHA2-256 and HMAC-SHA2-512, as WebCrypto has them. The strict + * key exchange of OpenSSH protects against the Terrapin attack. + * + * rfetch has no ssh-agent, no certificate of user, no command on the + * server, and no proxy: `key_use_agent`, `pubkey`, `pubkey_file`, `ssh`, + * `server_command`, `socks_proxy`, `http_proxy`, `pin_host_key` and + * `use_insecure_cipher` give 501. rfetch accepts the other options of + * rclone, for example the commands of the hashes, but they do nothing. + * + * Other than GET, HEAD, PUT and DELETE, the backend knows `MOVE` of a file + * or of a folder (a rename on the server, as rclone), `PATCH` with + * `X-OC-Mtime`, and `COPY` with `copy_is_hardlink`. + * + * In Node.js, Deno and Bun, a connection stays open for the next requests + * of the same remote, for `idle_timeout`. In workerd, each request has its + * own connection, as a connection cannot go from a request to another. + */ +import { decodeName, encodeName, parseEncoding } from "@rfetch/lib/encoder"; +import { renderListing } from "@rfetch/lib/listing"; +import { mimeType } from "@rfetch/lib/mime"; +import { reveal } from "@rfetch/lib/obscure"; +import { isTrue, optionsOf } from "@rfetch/lib/options"; +import { Parts } from "@rfetch/lib/parts"; +import { decodePath } from "@rfetch/lib/path"; +import { notSatisfiable, parseRange, partialHeaders } from "@rfetch/lib/range"; +import { parseSize } from "@rfetch/lib/size"; +import { sized } from "@rfetch/lib/sized"; +import { modTimeOf } from "@rfetch/lib/time"; +import { findHostKey } from "./knownhosts.js"; +import { isDirectory, isLink, OPEN, SFTP, SFTPError, STATUS } from "./sftp.js"; +import { CIPHERS, MACS } from "./ssh/cipher.js"; +import { AuthenticationError, SSHClient } from "./ssh/client.js"; +import { readKeyLine, readPrivateKey, toBase64 } from "./ssh/keys.js"; +import { HOST_KEYS, KEX } from "./ssh/transport.js"; + +/** + * The options of the backend, with their types, for example for the flag + * `--sftp-host` of the command line. + */ +export const options = { + string: [ + "host", + "user", + "port", + "pass", + "key_pem", + "key_file", + "key_file_pass", + "pubkey", + "pubkey_file", + "known_hosts_file", + "host_keys", + "path_override", + "encoding", + "shell_type", + "hashes", + "md5sum_command", + "sha1sum_command", + "crc32sum_command", + "sha256sum_command", + "blake3sum_command", + "xxh3sum_command", + "xxh128sum_command", + "subsystem", + "server_command", + "idle_timeout", + "chunk_size", + "concurrency", + "connections", + "set_env", + "ciphers", + "key_exchange", + "macs", + "host_key_algorithms", + "ssh", + "socks_proxy", + "http_proxy", + ], + boolean: [ + "pin_host_key", + "key_use_agent", + "use_insecure_cipher", + "disable_hashcheck", + "ask_password", + "set_modtime", + "skip_links", + "use_fstat", + "disable_concurrent_reads", + "disable_concurrent_writes", + "copy_is_hardlink", + ], + default: { + port: "22", + subsystem: "sftp", + set_modtime: "true", + chunk_size: "32Ki", + concurrency: "64", + idle_timeout: "1m0s", + encoding: "Slash,Ctl,Del,Dot", + }, +}; + +/** + * The options that are secret. They are not in the links of a list, which a + * browser can keep in its history and send to other sites. + */ +const SECRETS = ["user", "pass", "key_pem", "key_file_pass"]; + +/** The options of rclone that rfetch does not know, when they are set. */ +const UNKNOWN = [ + "key_use_agent", + "pubkey", + "pubkey_file", + "ssh", + "server_command", + "socks_proxy", + "http_proxy", + "pin_host_key", + "use_insecure_cipher", +]; + +/** The time to connect to a server, in milliseconds. */ +const CONNECT_TIMEOUT = 60000; + +/** The number of files or folders that a DELETE of a folder deletes at once. */ +const DELETE_CONCURRENCY = 16; + +/** + * The configuration of the requests. + * @typedef {Object} Config + * @property {string} host + * @property {number} port + * @property {string} user + * @property {string | undefined} password + * @property {import("./ssh/keys.js").PrivateKey | undefined} privateKey + * @property {{ type: string, blob: Uint8Array }[]} hostKeys + * @property {string | null} knownHosts The content of `known_hosts_file`, + * or null for no file. + * @property {{ ciphers?: string[], macs?: string[], kex?: string[], hostKeyAlgorithms?: string[] }} algorithms + * @property {string} subsystem + * @property {boolean} absolute True for a path from the root of the server. + * @property {number} encoding The mask of the encoding of the names. + * @property {boolean} setModTime + * @property {boolean} skipLinks + * @property {boolean} copyIsHardlink + * @property {number} chunkSize + * @property {number} concurrency + * @property {number} idleTimeout In milliseconds. + * @property {string} key The key of the connection in the pool. + */ + +/** An error with the status of the answer. */ +class SFTPBackendError extends Error { + /** @param {string} message @param {number} status */ + constructor(message, status) { + super(message); + this.status = status; + } +} + +/** + * Serves an SFTP remote. + * @param {Request} request + * @param {Record} [env] The options in the variables + * `RFETCH_SFTP_*` or `RCLONE_SFTP_*`, and `RFETCH_SFTP_ROOT` of rfetch. + * @returns {Promise} + */ +async function sftpBackend(request, env = {}) { + const { method } = request; + const { pathname, searchParams } = new URL(request.url); + + let config; + try { + config = await configOf(optionsOf(request, "sftp", env), env); + } catch (error) { + await request.body?.cancel(); + if (error instanceof SFTPBackendError) { + return new Response(error.message, { status: error.status }); + } + throw error; + } + const path = decodePath(pathname); + if (path === null) { + await request.body?.cancel(); + return new Response(null, { status: 400 }); + } + const isFolder = path.endsWith("/"); + const modTime = modTimeOf(request); + if (modTime === undefined) { + await request.body?.cancel(); + return new Response("X-OC-Mtime is not a time", { status: 400 }); + } + const remote = remotePath(config, path); + + if ( + !["GET", "HEAD", "PUT", "DELETE", "COPY", "MOVE", "PATCH"].includes(method) + ) { + await request.body?.cancel(); + return new Response(null, { + status: 405, + headers: { Allow: "GET, HEAD, PUT, DELETE, COPY, MOVE, PATCH" }, + }); + } + + /** @type {Session} */ + let session; + try { + session = await acquire(config); + } catch (error) { + await request.body?.cancel().catch(() => {}); + return errorResponse(error); + } + // A GET of a file streams its body: the session goes back at its end. + let streaming = false; + try { + const { sftp } = session; + if (method === "GET" || method === "HEAD") { + if (isFolder) { + const query = new URLSearchParams(searchParams); + for (const name of SECRETS) query.delete(name); + return await getFolder(config, sftp, remote, method, `${query}`); + } + const response = await getFile(config, session, remote, path, request); + streaming = response.body !== null; + return response; + } + if (method === "PUT") { + /** @type {Record} */ + const headers = { "Content-Location": pathname }; + if (isFolder) { + await request.body?.cancel(); + await mkdirAll(sftp, remote); + } else { + await putFile(config, sftp, remote, request); + if (modTime !== null && config.setModTime) { + await setTime(sftp, remote, modTime); + headers["X-OC-Mtime"] = "accepted"; + } + } + return new Response(null, { status: 201, headers }); + } + await request.body?.cancel(); + if (method === "DELETE") { + await (isFolder ? removeFolder(sftp, remote) : sftp.remove(remote)); + return new Response(null, { status: 204 }); + } + if (method === "PATCH") { + if (isFolder) return new Response(null, { status: 405 }); + if (modTime === null) { + return new Response("X-OC-Mtime is missing", { status: 400 }); + } + if (!config.setModTime) { + return new Response("can't set modified time", { status: 501 }); + } + await sftp.stat(remote); + await setTime(sftp, remote, modTime); + return new Response(null, { + status: 204, + headers: { "X-OC-Mtime": "accepted" }, + }); + } + // COPY and MOVE. + const target = decodePath(request.headers.get("Destination") ?? ""); + if (target === null || target === "/") { + return new Response("Destination is not a path", { status: 400 }); + } + if (target.endsWith("/") !== isFolder) { + return new Response("Destination is not of the same type", { + status: 409, + }); + } + const destination = remotePath(config, target); + if (method === "COPY") { + if (isFolder || !config.copyIsHardlink) { + return new Response("can't copy on the server", { status: 501 }); + } + return await copyFile(sftp, remote, destination); + } + return await move(config, sftp, remote, destination, isFolder, modTime); + } catch (error) { + await request.body?.cancel().catch(() => {}); + return errorResponse(error); + } finally { + if (!streaming) release(session); + } +} + +/** + * The answer of an error. + * @param {unknown} error + * @returns {Response} + */ +function errorResponse(error) { + if (error instanceof SFTPBackendError) { + return new Response(error.message, { status: error.status }); + } + if (error instanceof SFTPError) { + const status = { + [STATUS.NO_SUCH_FILE]: 404, + [STATUS.PERMISSION_DENIED]: 403, + [STATUS.OP_UNSUPPORTED]: 501, + }[error.code] ?? 500; + return new Response(error.message, { status }); + } + if (error instanceof AuthenticationError) { + return new Response(error.message, { status: 403 }); + } + if (error instanceof Error) { + // A server that does not answer, or that does not speak SSH. + return new Response(error.message, { status: 502 }); + } + throw error; +} + +/** + * Reads the options of a request, as `NewFs` of the sftp backend of rclone. + * @param {URLSearchParams} values + * @param {Record} env + * @returns {Promise} + * @throws {SFTPBackendError} When an option is not correct. + */ +async function configOf(values, env) { + /** @param {string} name */ + const get = (name) => values.get(name) ?? options.default[name] ?? ""; + for (const name of UNKNOWN) { + const value = get(name); + if (value && value !== "false") { + throw new SFTPBackendError( + `rfetch does not know the option ${name}`, + 501, + ); + } + } + const host = get("host"); + if (!host) throw new SFTPBackendError("The host option is missing", 400); + const port = Number(get("port")); + if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new SFTPBackendError("The port option is not a port", 400); + } + const user = get("user") || await systemUser(); + if (!user) throw new SFTPBackendError("The user option is missing", 400); + + let password; + const obscured = get("pass"); + if (obscured) { + try { + password = await (await reveal(obscured)).text(); + } catch { + throw new SFTPBackendError("couldn't decrypt password", 400); + } + } + + let pem = get("key_pem").replaceAll("\\n", "\n"); + if (!pem && get("key_file")) pem = await readLocation(get("key_file")); + let privateKey; + if (pem) { + try { + privateKey = await readPrivateKey(pem); + } catch (error) { + const passphrase = /passphrase/.test(error.message); + throw new SFTPBackendError( + `The private key is not correct: ${error.message}`, + passphrase ? 501 : 400, + ); + } + } + if (password === undefined && !privateKey) { + throw new SFTPBackendError("The pass, key_pem or key_file is missing", 400); + } + + /** @type {{ type: string, blob: Uint8Array }[]} */ + const hostKeys = []; + for (const line of get("host_keys").split(",")) { + if (line.trim() === "") continue; + try { + hostKeys.push(readKeyLine(line)); + } catch { + throw new SFTPBackendError( + `The host key ${line.trim()} is not correct`, + 400, + ); + } + } + const knownHostsFile = get("known_hosts_file"); + const knownHosts = knownHostsFile && knownHostsFile !== "none" + ? await readLocation(knownHostsFile) + : null; + + const algorithms = { + ciphers: chooseAlgorithms(get("ciphers"), Object.keys(CIPHERS), "ciphers"), + macs: chooseAlgorithms(get("macs"), Object.keys(MACS), "macs"), + kex: chooseAlgorithms( + get("key_exchange"), + Object.keys(KEX), + "key_exchange", + ), + hostKeyAlgorithms: chooseAlgorithms( + get("host_key_algorithms"), + HOST_KEYS, + "host_key_algorithms", + ), + }; + + let encoding; + try { + encoding = parseEncoding(get("encoding")); + } catch (error) { + throw new SFTPBackendError(error.message, 400); + } + let chunkSize; + try { + chunkSize = parseSize(get("chunk_size")); + } catch { + chunkSize = NaN; + } + if (!(chunkSize >= 1 && chunkSize <= 256 * 1024)) { + throw new SFTPBackendError( + "chunk_size must be from 1 byte to 256 KiB", + 400, + ); + } + const concurrency = Number(get("concurrency")); + if (!Number.isInteger(concurrency) || concurrency < 1) { + throw new SFTPBackendError("concurrency must be 1 or more", 400); + } + const idleTimeout = parseDuration(get("idle_timeout")); + if (Number.isNaN(idleTimeout)) { + throw new SFTPBackendError("idle_timeout is not a duration", 400); + } + + const subsystem = get("subsystem") || "sftp"; + const absolute = env.RFETCH_SFTP_ROOT === "/"; + // The connections of the pool, by a hash of all that makes a connection. + const key = await hash(JSON.stringify([ + host, + port, + user, + password ?? null, + pem, + get("host_keys"), + knownHosts, + algorithms, + subsystem, + ])); + return { + host, + port, + user, + password, + privateKey, + hostKeys, + knownHosts, + algorithms, + subsystem, + absolute, + encoding, + setModTime: isTrue(get("set_modtime")), + skipLinks: isTrue(get("skip_links")), + copyIsHardlink: isTrue(get("copy_is_hardlink")), + chunkSize, + concurrency, + idleTimeout, + key, + }; +} + +/** + * The user of the system, as the default user of rclone. + * @returns {Promise} "" when there is no such user, for example in + * workerd. + */ +async function systemUser() { + try { + const { userInfo } = await import("node:os"); + return userInfo().username; + } catch { + return ""; + } +} + +/** + * Reads a file of an option, for example `key_file`, with the fetch of + * rfetch: a local path in the command line, or a location of a remote. + * @param {string} location `~/` is the home folder. + * @returns {Promise} + * @throws {SFTPBackendError} + */ +async function readLocation(location) { + if (location.startsWith("~/")) { + try { + const { homedir } = await import("node:os"); + location = `${homedir()}${location.slice(1)}`; + } catch { + // No home folder, for example in workerd. + } + } + try { + const response = await fetch(location); + if (response.ok) return await response.text(); + await response.body?.cancel(); + } catch { + // The error below. + } + throw new SFTPBackendError(`rfetch cannot read ${location}`, 400); +} + +/** + * The algorithms of an option, in the order of the option. + * @param {string} value The names, with spaces between them. + * @param {string[]} known The algorithms of rfetch. + * @param {string} name The name of the option. + * @returns {string[] | undefined} No list for no option. + * @throws {SFTPBackendError} When rfetch knows none of them. + */ +function chooseAlgorithms(value, known, name) { + const names = value.split(/[\s,]+/).filter(Boolean); + if (names.length === 0) return undefined; + const chosen = names.filter((algorithm) => known.includes(algorithm)); + if (chosen.length === 0) { + throw new SFTPBackendError( + `rfetch knows none of the ${name}: it knows ${known.join(" ")}`, + 501, + ); + } + return chosen; +} + +/** + * Reads a duration of Go, for example `1m0s` or `30s`. + * @param {string} text + * @returns {number} Milliseconds, or NaN. + */ +function parseDuration(text) { + if (text === "0") return 0; + const units = { + ns: 1e-6, + us: 1e-3, + "µs": 1e-3, + ms: 1, + s: 1000, + m: 60000, + h: 3600000, + }; + const parts = [...text.matchAll(/(\d+(?:\.\d+)?)(ns|us|µs|ms|s|m|h)/g)]; + if (parts.length === 0 || parts.map((part) => part[0]).join("") !== text) { + return NaN; + } + return parts.reduce( + (sum, [, number, unit]) => sum + Number(number) * units[unit], + 0, + ); +} + +/** + * The SHA-256 of a text, in hexadecimal. + * @param {string} text + */ +async function hash(text) { + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(text), + ); + return Array.from( + new Uint8Array(digest), + (byte) => byte.toString(16).padStart(2, "0"), + ).join(""); +} + +/** + * The path on the server of a path of a request: each name with the + * encoding, from the login folder or from the root. + * @param {Config} config + * @param {string} path For example `/a/b.txt` or `/a/`. + */ +function remotePath(config, path) { + const names = path.slice(1).replace(/\/$/, ""); + const encoded = names === "" + ? "" + : names.split("/").map((name) => encodeName(name, config.encoding)) + .join("/"); + if (config.absolute) return `/${encoded}`; + return encoded === "" ? "." : encoded; +} + +/** + * A connection of SSH with a session of SFTP, which requests can share. + * @typedef {Object} Session + * @property {SSHClient} client + * @property {SFTP} sftp + * @property {number} users The requests that use the session. + * @property {ReturnType | null} timer + * @property {Config} config + */ + +/** + * The sessions of the pool, by the keys of their configurations. workerd + * has no pool: a connection cannot go from a request to another. + * @type {Map>} + */ +const pool = new Map(); +const POOLED = globalThis.navigator?.userAgent !== "Cloudflare-Workers"; + +/** + * Gives a session for a request: a session of the pool, or a new one. + * @param {Config} config + * @returns {Promise} + */ +async function acquire(config) { + if (POOLED) { + const pending = pool.get(config.key); + if (pending) { + const session = await pending.catch(() => null); + if (session && !session.client.isClosed) { + session.users++; + if (session.timer) clearTimeout(session.timer); + session.timer = null; + session.client.setActive(true); + return session; + } + if (pool.get(config.key) === pending) pool.delete(config.key); + } + } + const pending = connect(config); + if (POOLED) { + pool.set(config.key, pending); + pending.catch(() => { + if (pool.get(config.key) === pending) pool.delete(config.key); + }); + } + return await pending; +} + +/** + * Opens a connection, and starts SFTP on it. + * @param {Config} config + * @returns {Promise} + */ +async function connect(config) { + const client = await SSHClient.connect({ + host: config.host, + port: config.port, + username: config.user, + password: config.password, + privateKey: config.privateKey, + hostKeyTypes: config.hostKeys.map((key) => key.type), + algorithms: config.algorithms, + timeout: CONNECT_TIMEOUT, + checkHostKey: (key) => checkHostKey(config, key), + }); + try { + const sftp = await SFTP.start(await client.subsystem(config.subsystem)); + /** @type {Session} */ + const session = { client, sftp, users: 1, timer: null, config }; + // A connection that closes leaves the pool. + client.closed.then(() => { + for (const [key, pending] of pool) { + pending.then((other) => { + if (other === session && pool.get(key) === pending) pool.delete(key); + }, () => {}); + } + }); + return session; + } catch (error) { + await client.close(); + throw error; + } +} + +/** + * Checks the host key of the server, with `host_keys` or with + * `known_hosts_file`. + * @param {Config} config + * @param {import("./ssh/transport.js").HostKey} key + * @returns {Promise} + * @throws {SFTPBackendError} When the server has another key. + */ +async function checkHostKey(config, key) { + const line = `${key.type} ${toBase64(key.blob)}`; + if (config.hostKeys.length) { + const known = config.hostKeys.some((known) => + known.type === key.type && toBase64(known.blob) === toBase64(key.blob) + ); + if (!known) { + throw new SFTPBackendError( + `The host key ${key.fingerprint} of the server is not in host_keys: ${line}`, + 502, + ); + } + return true; + } + if (config.knownHosts !== null) { + const found = await findHostKey( + config.knownHosts, + config.host, + config.port, + key.blob, + ); + if (found !== "known") { + throw new SFTPBackendError( + `The host key ${key.fingerprint} of the server is ${found} in known_hosts_file`, + 502, + ); + } + } + // With no host_keys and no known_hosts_file, each key, as rclone. + return true; +} + +/** + * Gives a session back after a request. In the pool, it waits for the + * next request for `idle_timeout`, and does not keep the process running. + * Else it closes. + * @param {Session} session + */ +function release(session) { + session.users--; + if (session.users > 0) return; + if (!POOLED || session.config.idleTimeout === 0) { + close(session); + return; + } + session.client.setActive(false); + session.timer = setTimeout(() => close(session), session.config.idleTimeout); + // @ts-ignore: the timer of Node.js does not keep the process running. + session.timer.unref?.(); +} + +/** @param {Session} session */ +function close(session) { + for (const [key, pending] of pool) { + pending.then((other) => { + if (other === session && pool.get(key) === pending) pool.delete(key); + }, () => {}); + } + session.client.close().catch(() => {}); +} + +/** + * The time of a file in ISO 8601, to the second: the list of a remote with + * a precision of 1 s has no part of a second. + * @param {number | undefined} seconds + */ +function isoTime(seconds) { + if (seconds === undefined) return ""; + return new Date(seconds * 1000).toISOString().replace(/\.000Z$/, "Z"); +} + +/** + * Sets the time of a file, to the second. + * @param {SFTP} sftp + * @param {string} path + * @param {bigint} nanoseconds + */ +async function setTime(sftp, path, nanoseconds) { + const seconds = Number(nanoseconds / 1_000_000_000n) - + (nanoseconds < 0n && nanoseconds % 1_000_000_000n !== 0n ? 1 : 0); + if (seconds < 0 || seconds > 0xffffffff) { + throw new SFTPBackendError("SFTP cannot keep this time", 501); + } + await sftp.setstat(path, { atime: seconds, mtime: seconds }); +} + +/** + * Gets the list of a folder, or only its headers for HEAD. + * @param {Config} config + * @param {SFTP} sftp + * @param {string} path + * @param {string} method + * @param {string} query The query of each link. + * @returns {Promise} + */ +async function getFolder(config, sftp, path, method, query) { + const attrs = await sftp.stat(path); + if (!isDirectory(attrs)) return new Response(null, { status: 404 }); + const headers = { "Content-Type": "text/html;charset=utf-8" }; + if (method === "HEAD") return new Response(null, { headers }); + const items = await sftp.readdir(path); + /** @type {import("@rfetch/lib/listing").Entry[]} */ + const entries = []; + await Promise.all(items.map(async ({ filename, attrs }) => { + if (isLink(attrs)) { + if (config.skipLinks) return; + // A link is the file or the folder that it points to. A link to + // nothing is not in the list. + try { + attrs = await sftp.stat(join(path, filename)); + } catch { + return; + } + } + const Name = decodeName(filename, config.encoding); + const folder = isDirectory(attrs); + entries.push({ + Name, + IsDir: folder, + Size: folder ? -1 : Number(attrs.size ?? 0), + MimeType: folder ? "" : mimeType(Name), + ModTime: isoTime(attrs.mtime), + }); + })); + entries.sort((a, b) => a.Name < b.Name ? -1 : a.Name > b.Name ? 1 : 0); + return new Response(renderListing(entries, { query }), { headers }); +} + +/** + * Joins a folder of the server and a name. + * @param {string} folder + * @param {string} name + */ +function join(folder, name) { + if (folder === ".") return name; + return folder.endsWith("/") ? `${folder}${name}` : `${folder}/${name}`; +} + +/** + * Gets a file, or only its headers for HEAD, with the `Range` of the + * request, as RFC 9110. + * @param {Config} config + * @param {Session} session + * @param {string} remote + * @param {string} path The path of the request. + * @param {Request} request + * @returns {Promise} + */ +async function getFile(config, session, remote, path, request) { + const { sftp } = session; + const attrs = await sftp.stat(remote); + if (isDirectory(attrs)) return new Response(null, { status: 404 }); + const size = Number(attrs.size ?? 0); + const name = path.slice(path.lastIndexOf("/") + 1); + const headers = new Headers({ + "Content-Type": mimeType(name), + "Accept-Ranges": "bytes", + }); + if (attrs.mtime !== undefined) { + headers.set("Last-Modified", new Date(attrs.mtime * 1000).toUTCString()); + } + let range = null; + if (request.method === "GET") { + try { + range = parseRange(request.headers.get("Range"), size); + } catch { + return notSatisfiable(size); + } + } + const start = range?.start ?? 0; + const length = range ? range.end - range.start + 1 : size; + headers.set("Content-Length", `${length}`); + if (range) { + for (const [key, value] of Object.entries(partialHeaders(range, size))) { + headers.set(key, value); + } + } + const status = range ? 206 : 200; + if (request.method === "HEAD" || length === 0) { + return new Response(null, { status, headers }); + } + const handle = await sftp.open(remote, OPEN.READ); + const body = readStream(config, sftp, handle, start, length, () => { + release(session); + }); + return new Response(sized(body, length), { status, headers }); +} + +/** + * Reads bytes of an open file, with reads of `chunk_size` at the same time, + * in their order. + * @param {Config} config + * @param {SFTP} sftp + * @param {Uint8Array} handle + * @param {number} start + * @param {number} length + * @param {() => void} done Runs one time, at the end of the stream. + * @returns {ReadableStream} + */ +function readStream(config, sftp, handle, start, length, done) { + const end = start + length; + let offset = start; + /** @type {{ offset: number, size: number, data: Promise }[]} */ + const queue = []; + let finished = false; + const finish = async () => { + if (finished) return; + finished = true; + // The reads that wait must end before the handle closes. + await Promise.allSettled(queue.map((item) => item.data)); + await sftp.close(handle).catch(() => {}); + done(); + }; + /** @param {number} at @param {number} size */ + const read = (at, size) => { + const data = sftp.read(handle, at, size); + data.catch(() => {}); + return { offset: at, size, data }; + }; + const fill = () => { + while (queue.length < config.concurrency && offset < end) { + const size = Math.min(config.chunkSize, end - offset); + queue.push(read(offset, size)); + offset += size; + } + }; + return new ReadableStream({ + async pull(controller) { + try { + fill(); + const item = queue.shift(); + if (!item) { + await finish(); + controller.close(); + return; + } + const data = await item.data; + if (data === null || data.length === 0) { + throw new Error("The file is shorter than its size"); + } + // A server can give less than the size of a read: the rest is the + // next read. + if (data.length < item.size) { + queue.unshift( + read(item.offset + data.length, item.size - data.length), + ); + } + controller.enqueue(data); + } catch (error) { + await finish(); + controller.error(error); + } + }, + async cancel() { + await finish(); + }, + }); +} + +/** + * Makes a folder and its parent folders, as `mkdir` of rclone. A folder + * that exists is not an error. + * @param {SFTP} sftp + * @param {string} path + */ +async function mkdirAll(sftp, path) { + if (path === "." || path === "/") return; + try { + const attrs = await sftp.stat(path); + if (isDirectory(attrs)) return; + throw new SFTPBackendError(`${path} is a file, not a folder`, 409); + } catch (error) { + if (!(error instanceof SFTPError && error.code === STATUS.NO_SUCH_FILE)) { + throw error; + } + } + const slash = path.lastIndexOf("/"); + if (slash > 0) await mkdirAll(sftp, path.slice(0, slash)); + try { + await sftp.mkdir(path); + } catch (error) { + // Another request made the folder at the same time. + const attrs = await sftp.stat(path).catch(() => null); + if (!attrs || !isDirectory(attrs)) throw error; + } +} + +/** + * The folder of a path of the server. + * @param {string} path + */ +function parentOf(path) { + const slash = path.lastIndexOf("/"); + if (slash === -1) return "."; + return slash === 0 ? "/" : path.slice(0, slash); +} + +/** + * Puts a file, after its folders, with writes of `chunk_size` at the same + * time. A failed upload deletes the part of the file, as rclone. + * @param {Config} config + * @param {SFTP} sftp + * @param {string} path + * @param {Request} request + */ +async function putFile(config, sftp, path, request) { + await mkdirAll(sftp, parentOf(path)); + const handle = await sftp.open( + path, + OPEN.WRITE | OPEN.CREAT | OPEN.TRUNC, + ); + const parts = new Parts( + request.body ?? new Blob([]).stream(), + config.chunkSize, + ); + /** @type {Set>} */ + const writes = new Set(); + /** @type {unknown} */ + let failure = null; + try { + let offset = 0; + for (let part = await parts.next(); part; part = await parts.next()) { + const write = sftp.write(handle, offset, part); + offset += part.length; + const tracked = write.then( + () => { + writes.delete(tracked); + }, + (error) => { + writes.delete(tracked); + failure ??= error; + }, + ); + writes.add(tracked); + if (writes.size >= config.concurrency) await Promise.race(writes); + if (failure) throw failure; + } + await Promise.all(writes); + if (failure) throw failure; + await sftp.close(handle); + } catch (error) { + await Promise.allSettled(writes); + await sftp.close(handle).catch(() => {}); + await sftp.remove(path).catch(() => {}); + throw error; + } +} + +/** + * Deletes a folder and all that is in it. The root of the remote stays. + * @param {SFTP} sftp + * @param {string} path + */ +async function removeFolder(sftp, path) { + const attrs = await sftp.stat(path); + if (!isDirectory(attrs)) { + throw new SFTPBackendError(`${path} is not a folder`, 404); + } + await removeContent(sftp, path); + if (path !== "." && path !== "/") await sftp.rmdir(path); +} + +/** + * Deletes all that is in a folder: a link is deleted, not what it points + * to. + * @param {SFTP} sftp + * @param {string} path + */ +async function removeContent(sftp, path) { + const items = await sftp.readdir(path); + for (let i = 0; i < items.length; i += DELETE_CONCURRENCY) { + await Promise.all( + items.slice(i, i + DELETE_CONCURRENCY).map( + async ({ filename, attrs }) => { + const child = join(path, filename); + if (isDirectory(attrs) && !isLink(attrs)) { + await removeContent(sftp, child); + await sftp.rmdir(child); + } else { + await sftp.remove(child); + } + }, + ), + ); + } +} + +/** + * Tells if a path exists on the server. + * @param {SFTP} sftp + * @param {string} path + */ +async function exists(sftp, path) { + try { + await sftp.lstat(path); + return true; + } catch (error) { + if (error instanceof SFTPError && error.code === STATUS.NO_SUCH_FILE) { + return false; + } + throw error; + } +} + +/** + * Copies a file with a hard link, as `copy_is_hardlink` of rclone. + * @param {SFTP} sftp + * @param {string} source + * @param {string} target + * @returns {Promise} + */ +async function copyFile(sftp, source, target) { + const attrs = await sftp.stat(source); + if (isDirectory(attrs)) return new Response(null, { status: 404 }); + await mkdirAll(sftp, parentOf(target)); + const existed = await exists(sftp, target); + if (existed) await sftp.remove(target); + await sftp.hardlink(source, target); + return new Response(null, { status: existed ? 204 : 201 }); +} + +/** + * Moves a file, or a folder to a path that does not exist, with a rename on + * the server, as rclone. + * @param {Config} config + * @param {SFTP} sftp + * @param {string} source + * @param {string} target + * @param {boolean} isFolder + * @param {bigint | null} modTime + * @returns {Promise} + */ +async function move(config, sftp, source, target, isFolder, modTime) { + const attrs = await sftp.stat(source); + if (isDirectory(attrs) !== isFolder) { + return new Response(null, { status: 404 }); + } + const existed = await exists(sftp, target); + if (isFolder && existed) { + return new Response("Destination exists", { status: 412 }); + } + await mkdirAll(sftp, parentOf(target)); + await sftp.rename(source, target, !isFolder); + /** @type {Record} */ + const headers = {}; + if (!isFolder && modTime !== null && config.setModTime) { + await setTime(sftp, target, modTime); + headers["X-OC-Mtime"] = "accepted"; + } + return new Response(null, { status: existed ? 204 : 201, headers }); +} + +export default { + fetch: sftpBackend, +}; diff --git a/backend/sftp/main_test.js b/backend/sftp/main_test.js new file mode 100644 index 0000000..ff31406 --- /dev/null +++ b/backend/sftp/main_test.js @@ -0,0 +1,394 @@ +/** + * The tests of the sftp backend, on `rclone serve sftp`: the shared tests, + * the host keys, the keys of the user, and the comparison with the sftp + * backend of rclone on the same server. They run when rclone is in the PATH, + * or when RCLONE is the path of rclone. The keys come from openssl. + */ +import { after, describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import process from "node:process"; + +import { obscure } from "@rfetch/lib/obscure"; +import { backendSuite } from "@rfetch/lib/testing/suite"; +import { remote } from "@rfetch/lib/testing/remote"; +// Sets the global `fetch` that reads the paths of remotes. +import { lsjson } from "../../mod.js"; +import sftp from "./main.js"; +import { readPrivateKey, toBase64 } from "./ssh/keys.js"; +import { + hasOpenSSL, + hasRclone, + KEY_PARAMS, + LOGIN, + makeKey, + makeKeys, + serveSFTP, +} from "./testing.js"; + +const RCLONE = process.env.RCLONE || "rclone"; + +/** The password of the test server, obscured as in the config of rclone. */ +const pass = await (await obscure(LOGIN.pass)).text(); + +/** + * The line of a public key, as in `authorized_keys` and in `host_keys`. + * @param {string} pem + */ +async function keyLine(pem) { + const key = await readPrivateKey(pem); + return `${key.type} ${toBase64(key.blob)}`; +} + +if (hasRclone && hasOpenSSL) { + const hostKey = makeKey("ed25519"); + const userKeys = { + ed25519: makeKey("ed25519"), + ecdsa: makeKey("ecdsa", "traditional"), + rsa: makeKey("rsa", "traditional"), + "rsa pkcs8": makeKey("rsa"), + }; + // Keys of WebCrypto, also in the format of OpenSSH. + const openSSHKeys = await Promise.all( + Object.values(KEY_PARAMS).map((params) => makeKeys(params)), + ); + const server = await serveSFTP({ + hostKeys: [hostKey, makeKey("ecdsa"), makeKey("rsa")], + authorizedKeys: [ + ...await Promise.all(Object.values(userKeys).map(keyLine)), + ...await Promise.all(openSSHKeys.map(({ pem }) => keyLine(pem))), + ], + }); + after(() => server.stop()); + const hostKeys = await keyLine(hostKey); + const params = { + host: server.host, + port: `${server.port}`, + user: LOGIN.user, + pass, + host_keys: hostKeys, + }; + + backendSuite("sftp", sftp, { + type: "sftp", + params, + features: { + move: true, + dirMove: true, + modTime: true, + setModTime: true, + }, + }); + // One read and one write at a time, of 1000 bytes. + backendSuite("sftp with small chunks", sftp, { + params: { ...params, chunk_size: "1000b", concurrency: "1" }, + features: { move: true, dirMove: true, modTime: true, setModTime: true }, + }); + backendSuite("sftp with a key", sftp, { + params: { + host: server.host, + port: `${server.port}`, + user: LOGIN.user, + key_pem: userKeys.ed25519.replaceAll("\n", "\\n"), + host_keys: hostKeys, + }, + features: { move: true, dirMove: true, modTime: true, setModTime: true }, + }); + + describe("sftp", () => { + const fs = remote(sftp, { params, root: "/sftp-test/" }); + + test("signs in with each type of key", async () => { + for (const [name, pem] of Object.entries(userKeys)) { + const response = await remote(sftp, { + params: { + host: server.host, + port: `${server.port}`, + user: LOGIN.user, + key_pem: pem, + host_keys: hostKeys, + }, + }).get(""); + await response.body?.cancel(); + assert.equal(response.status, 200, name); + } + }); + + test("uses each host key and each cipher", async () => { + for ( + const options of [ + { host_key_algorithms: "ecdsa-sha2-nistp256" }, + { host_key_algorithms: "rsa-sha2-512" }, + { host_key_algorithms: "rsa-sha2-256" }, + { ciphers: "aes256-gcm@openssh.com" }, + { ciphers: "aes128-ctr", macs: "hmac-sha2-256-etm@openssh.com" }, + { ciphers: "aes256-ctr", macs: "hmac-sha2-512" }, + { key_exchange: "ecdh-sha2-nistp256" }, + { key_exchange: "ecdh-sha2-nistp384" }, + { key_exchange: "ecdh-sha2-nistp521" }, + ] + ) { + const data = new Uint8Array(100000).map((_, i) => i * 13); + const other = remote(sftp, { + params: { ...params, host_keys: "", ...options }, + root: "/sftp-test/", + }); + const put = await other.put("cipher.bin", data); + assert.equal(put.status, 201, JSON.stringify(options)); + const response = await other.get("cipher.bin"); + assert.deepEqual( + new Uint8Array(await response.arrayBuffer()), + data, + JSON.stringify(options), + ); + } + }); + + test("refuses a host key that is not in host_keys", async () => { + const response = await remote(sftp, { + params: { ...params, host_keys: await keyLine(makeKey("ed25519")) }, + }).get(""); + const text = await response.text(); + assert.equal(response.status, 502); + assert.match( + text, + /The host key SHA256:\S+ of the server is not in host_keys/, + ); + }); + + test("checks the host key with known_hosts_file", async () => { + const file = join(server.folder, "..", "known_hosts"); + const line = await keyLine(hostKey); + for ( + const [hosts, status] of [ + [`[${server.host}]:${server.port}`, 200], + [`[127.0.0.*]:${server.port}`, 200], + [`[${server.host}]:1`, 502], + [server.host, 502], + ] + ) { + writeFileSync(file, `# A comment\n${hosts} ${line}\n`); + const response = await remote(sftp, { + params: { ...params, host_keys: "", known_hosts_file: file }, + }).get(""); + await response.body?.cancel(); + assert.equal(response.status, status, hosts); + } + // A hashed name, as `ssh-keygen -H` writes it. + const salt = crypto.getRandomValues(new Uint8Array(20)); + const key = await crypto.subtle.importKey( + "raw", + salt, + { name: "HMAC", hash: "SHA-1" }, + false, + ["sign"], + ); + const hash = new Uint8Array( + await crypto.subtle.sign( + "HMAC", + key, + new TextEncoder().encode(`[${server.host}]:${server.port}`), + ), + ); + writeFileSync(file, `|1|${toBase64(salt)}|${toBase64(hash)} ${line}\n`); + let response = await remote(sftp, { + params: { ...params, host_keys: "", known_hosts_file: file }, + }).get(""); + await response.body?.cancel(); + assert.equal(response.status, 200, "hashed name"); + // A revoked key. + writeFileSync( + file, + `@revoked * ${line}\n[${server.host}]:${server.port} ${line}\n`, + ); + response = await remote(sftp, { + params: { ...params, host_keys: "", known_hosts_file: file }, + }).get(""); + await response.body?.cancel(); + assert.equal(response.status, 502, "revoked key"); + }); + + test("gives 403 for a wrong password", async () => { + const wrong = await (await obscure("wrong")).text(); + const response = await remote(sftp, { + params: { ...params, pass: wrong }, + }) + .get(""); + await response.body?.cancel(); + assert.equal(response.status, 403); + }); + + test("puts no secret in the links of a list", async () => { + await fs.put("a b.txt", new TextEncoder().encode("a")).then((r) => + r.body?.cancel() + ); + const html = await fs.get("").then((response) => response.text()); + assert.ok(html.includes(' href="a%20b.txt?host=')); + assert.ok(!html.includes("pass="), "the password is in a link"); + assert.ok(!html.includes("user="), "the user is in a link"); + }); + + test("gives 400 and 501 for options that are not correct", async () => { + for ( + const [options, status] of [ + [{ host: "" }, 400], + [{ port: "70000" }, 400], + [{ pass: "", key_pem: "" }, 400], + [{ chunk_size: "1M" }, 400], + [{ key_use_agent: "true" }, 501], + [{ ciphers: "chacha20-poly1305@openssh.com" }, 501], + [{ + key_pem: + "-----BEGIN OPENSSH PRIVATE KEY-----\nAAAA\n-----END OPENSSH PRIVATE KEY-----", + }, 400], + ] + ) { + const response = await remote(sftp, { + params: { ...params, ...options }, + }) + .get(""); + await response.body?.cancel(); + assert.equal(response.status, status, JSON.stringify(options)); + } + }); + + test("reads a path from the root of the server", async () => { + // The login folder of rclone serve sftp is the root of the server. + await fs.put("root.txt", new TextEncoder().encode("root")); + const response = await fetch( + `:sftp,host=${server.host},port=${server.port},user=${LOGIN.user},pass=${pass},host_keys='${hostKeys}':/sftp-test/root.txt`, + ); + assert.equal(await response.text(), "root"); + }); + + test("deletes the files of the tests", async () => { + const response = await fs.delete(""); + assert.equal(response.status, 204); + }); + + describe("and rclone", () => { + /** + * Runs rclone, with no spawnSync: the server is another process, but + * the tests wait for rclone with no block. + * @param {...string} args + */ + const rclone = (...args) => + new Promise((resolve) => { + execFile(RCLONE, args, { + encoding: "utf8", + env: { PATH: process.env.PATH, HOME: "/nonexistent", TZ: "UTC" }, + }, (error, stdout, stderr) => + resolve({ status: error ? error.code ?? 1 : 0, stdout, stderr })); + }); + /** The same remote for rclone and for rfetch. */ + const location = (path = "") => + `:sftp,host=${server.host},port=${server.port},user=${LOGIN.user},pass=${pass},host_keys='${hostKeys}':both/${path}`; + + test("rfetch and rclone sign in with the same OpenSSH keys", async () => { + // rclone reads the key with Go: the key of the test has the format + // of OpenSSH. + const names = Object.keys(KEY_PARAMS); + for (const [index, { openssh }] of openSSHKeys.entries()) { + const file = join(server.folder, "..", `id_${index}`); + writeFileSync(file, openssh, { mode: 0o600 }); + const remoteOf = (option) => + `:sftp,host=${server.host},port=${server.port},user=${LOGIN.user},${option},host_keys='${hostKeys}':`; + const theirs = await rclone( + "lsjson", + remoteOf(`key_file='${file}'`), + ); + assert.equal(theirs.status, 0, `${names[index]}: ${theirs.stderr}`); + const ours = await fetch(remoteOf(`key_file='${file}'`)); + await ours.body?.cancel(); + assert.equal(ours.status, 200, names[index]); + } + }); + + test("rclone reads the files and the times of rfetch", async () => { + for ( + const [path, text] of [["a.txt", "hello"], ["d/b c.txt", "world!"]] + ) { + const response = await fetch(location(path.replace(" ", "%20")), { + method: "PUT", + headers: { "X-OC-Mtime": "1354040105.5" }, + body: text, + }); + assert.equal(response.status, 201); + assert.equal(response.headers.get("X-OC-Mtime"), "accepted"); + } + const { status, stdout, stderr } = await rclone( + "lsjson", + "-R", + "--files-only", + location(), + ); + assert.equal(status, 0, stderr); + assert.deepEqual( + JSON.parse(stdout).map(({ Path, Size, ModTime }) => [ + Path, + Size, + ModTime, + ]).sort(), + [ + ["a.txt", 5, "2012-11-27T18:15:05Z"], + ["d/b c.txt", 6, "2012-11-27T18:15:05Z"], + ], + ); + assert.equal( + (await rclone("cat", location("d/b c.txt"))).stdout, + "world!", + ); + }); + + test("rfetch reads the files and the times of rclone", async () => { + const { status, stderr } = await rclone( + "touch", + "--timestamp", + "2020-01-02T03:04:05", + location("c/new.txt"), + ); + assert.equal(status, 0, stderr); + const listed = await (await lsjson(location(), { recursive: true })) + .json(); + const item = listed.find((item) => item.Path === "c/new.txt"); + assert.equal( + Date.parse(item?.ModTime), + Date.parse("2020-01-02T03:04:05Z"), + ); + assert.equal(item?.Size, 0); + const theirList = JSON.parse( + (await rclone("lsjson", "-R", location())).stdout, + ); + const shape = (list) => + list.map(({ Path, Size, IsDir }) => [Path, Size, IsDir]).sort(); + assert.deepEqual(shape(listed), shape(theirList)); + }); + + test("rfetch and rclone encode the names in the same way", async () => { + // The default encoding changes a control character. + const response = await fetch(location("names/a%0Bb.txt"), { + method: "PUT", + body: "x", + }); + assert.equal(response.status, 201); + // rclone shows a control character as its picture. + const theirs = JSON.parse( + (await rclone("lsjson", location("names"))).stdout, + ); + assert.deepEqual(theirs.map(({ Name }) => Name), ["a␋b.txt"]); + const ours = await (await lsjson(location("names"))).json(); + assert.deepEqual(ours.map(({ Name }) => Name), ["a\u000bb.txt"]); + await fetch(location(""), { method: "DELETE" }).then((r) => + r.body?.cancel() + ); + }); + }); + }); +} else { + test("sftp", { + skip: + "rclone or openssl is not installed. Set RCLONE to the path of rclone.", + }); +} diff --git a/backend/sftp/package.json b/backend/sftp/package.json new file mode 100644 index 0000000..447fa86 --- /dev/null +++ b/backend/sftp/package.json @@ -0,0 +1,28 @@ +{ + "name": "@rfetch/sftp", + "version": "0.1.0", + "description": "The sftp backend of rfetch: the files of an SFTP server, with a client of SSH on WebCrypto and node:net.", + "type": "module", + "exports": { + ".": "./main.js" + }, + "scripts": { + "test": "node --test" + }, + "engines": { + "node": ">=22.2.0" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/sntran/rfetch.git", + "directory": "backend/sftp" + }, + "author": "Trần Nguyễn Sơn", + "license": "ISC", + "dependencies": { + "@rfetch/lib": "^0.1.0" + }, + "devDependencies": { + "fast-check": "^3.19.0" + } +} diff --git a/backend/sftp/sftp.js b/backend/sftp/sftp.js new file mode 100644 index 0000000..69eab5f --- /dev/null +++ b/backend/sftp/sftp.js @@ -0,0 +1,584 @@ +/** + * A client of the SFTP protocol, version 3, as + * draft-ietf-secsh-filexfer-02, with the extensions of OpenSSH + * `posix-rename@openssh.com` and `hardlink@openssh.com`. + */ + +import { ProtocolError, Reader, Writer } from "./ssh/wire.js"; + +/** The numbers of the messages of SFTP. */ +const FXP = { + INIT: 1, + VERSION: 2, + OPEN: 3, + CLOSE: 4, + READ: 5, + WRITE: 6, + LSTAT: 7, + FSTAT: 8, + SETSTAT: 9, + FSETSTAT: 10, + OPENDIR: 11, + READDIR: 12, + REMOVE: 13, + MKDIR: 14, + RMDIR: 15, + REALPATH: 16, + STAT: 17, + RENAME: 18, + READLINK: 19, + SYMLINK: 20, + STATUS: 101, + HANDLE: 102, + DATA: 103, + NAME: 104, + ATTRS: 105, + EXTENDED: 200, + EXTENDED_REPLY: 201, +}; + +/** The flags of OPEN. */ +export const OPEN = { + READ: 0x01, + WRITE: 0x02, + APPEND: 0x04, + CREAT: 0x08, + TRUNC: 0x10, + EXCL: 0x20, +}; + +/** The codes of STATUS. */ +export const STATUS = { + OK: 0, + EOF: 1, + NO_SUCH_FILE: 2, + PERMISSION_DENIED: 3, + FAILURE: 4, + BAD_MESSAGE: 5, + NO_CONNECTION: 6, + CONNECTION_LOST: 7, + OP_UNSUPPORTED: 8, +}; + +/** The flags of the attributes. */ +const ATTR = { + SIZE: 0x01, + UIDGID: 0x02, + PERMISSIONS: 0x04, + ACMODTIME: 0x08, + EXTENDED: 0x80000000, +}; + +/** The type bits of the permissions of a file, as `st_mode`. */ +const S_IFMT = 0o170000; +const S_IFDIR = 0o040000; +const S_IFREG = 0o100000; +const S_IFLNK = 0o120000; + +/** The largest packet of SFTP that the client reads. */ +const MAX_PACKET = 256 * 1024 + 64; + +/** An error of the server, with its code of STATUS. */ +export class SFTPError extends Error { + /** @param {number} code @param {string} message */ + constructor(code, message) { + super(message || `SFTP error ${code}`); + this.code = code; + } +} + +/** + * The attributes of a file. + * @typedef {Object} Attributes + * @property {bigint} [size] + * @property {number} [uid] + * @property {number} [gid] + * @property {number} [permissions] + * @property {number} [atime] Seconds after 1970. + * @property {number} [mtime] Seconds after 1970. + */ + +/** + * Tells if attributes are of a folder. + * @param {Attributes} attrs + */ +export function isDirectory(attrs) { + return attrs.permissions !== undefined && + (attrs.permissions & S_IFMT) === S_IFDIR; +} + +/** + * Tells if attributes are of a regular file. + * @param {Attributes} attrs + */ +export function isRegular(attrs) { + return attrs.permissions === undefined || + (attrs.permissions & S_IFMT) === S_IFREG; +} + +/** + * Tells if attributes are of a symbolic link. + * @param {Attributes} attrs + */ +export function isLink(attrs) { + return attrs.permissions !== undefined && + (attrs.permissions & S_IFMT) === S_IFLNK; +} + +/** + * Reads the attributes of a message. + * @param {Reader} reader + * @returns {Attributes} + */ +function readAttributes(reader) { + const flags = reader.uint32(); + /** @type {Attributes} */ + const attrs = {}; + if (flags & ATTR.SIZE) attrs.size = reader.uint64(); + if (flags & ATTR.UIDGID) { + attrs.uid = reader.uint32(); + attrs.gid = reader.uint32(); + } + if (flags & ATTR.PERMISSIONS) attrs.permissions = reader.uint32(); + if (flags & ATTR.ACMODTIME) { + attrs.atime = reader.uint32(); + attrs.mtime = reader.uint32(); + } + if (flags & ATTR.EXTENDED) { + const count = reader.uint32(); + for (let i = 0; i < count; i++) { + reader.bytes(); + reader.bytes(); + } + } + return attrs; +} + +/** + * Writes attributes. + * @param {Writer} writer + * @param {Attributes} attrs + */ +function writeAttributes(writer, attrs) { + let flags = 0; + if (attrs.size !== undefined) flags |= ATTR.SIZE; + if (attrs.uid !== undefined && attrs.gid !== undefined) flags |= ATTR.UIDGID; + if (attrs.permissions !== undefined) flags |= ATTR.PERMISSIONS; + if (attrs.atime !== undefined && attrs.mtime !== undefined) { + flags |= ATTR.ACMODTIME; + } + writer.uint32(flags); + if (attrs.size !== undefined) writer.uint64(attrs.size); + if (flags & ATTR.UIDGID) { + writer.uint32(/** @type {number} */ (attrs.uid)); + writer.uint32(/** @type {number} */ (attrs.gid)); + } + if (attrs.permissions !== undefined) writer.uint32(attrs.permissions); + if (flags & ATTR.ACMODTIME) { + writer.uint32(/** @type {number} */ (attrs.atime)); + writer.uint32(/** @type {number} */ (attrs.mtime)); + } +} + +/** + * An entry of a folder. + * @typedef {Object} Entry + * @property {string} filename + * @property {Attributes} attrs + */ + +/** A session of SFTP on a channel of SSH. */ +export class SFTP { + /** @type {import("./ssh/client.js").Channel} */ + #channel; + #nextId = 1; + /** @type {Map void, reject: (error: Error) => void }>} */ + #pending = new Map(); + /** The extensions of the server, by their names. */ + /** @type {Map} */ + extensions = new Map(); + /** @type {Error | null} */ + #error = null; + /** The bytes of a packet that is not complete. */ + #buffer = new Uint8Array(0); + + /** @param {import("./ssh/client.js").Channel} channel */ + constructor(channel) { + this.#channel = channel; + } + + /** + * Starts SFTP on a channel with the subsystem `sftp`. + * @param {import("./ssh/client.js").Channel} channel + * @returns {Promise} + */ + static async start(channel) { + const sftp = new SFTP(channel); + await sftp.#writePacket(new Writer().byte(FXP.INIT).uint32(3).bytes()); + const reader = new Reader(await sftp.#readPacket()); + if (reader.byte() !== FXP.VERSION) { + throw new ProtocolError("The server did not send its SFTP version"); + } + const version = reader.uint32(); + if (version < 3) { + throw new ProtocolError(`SFTP version ${version} is not supported`); + } + while (reader.remaining > 0) { + const name = reader.string(); + sftp.extensions.set(name, new TextDecoder().decode(reader.bytes())); + } + sftp.#readLoop(); + return sftp; + } + + /** The channel of the session. */ + get channel() { + return this.#channel; + } + + /** @param {Uint8Array} payload */ + async #writePacket(payload) { + await this.#channel.write( + new Writer(payload.length + 4).uint32(payload.length).raw(payload) + .bytes(), + ); + } + + /** + * Reads one packet of SFTP from the channel. + * @returns {Promise} + */ + async #readPacket() { + while (true) { + if (this.#buffer.length >= 4) { + const length = new DataView( + this.#buffer.buffer, + this.#buffer.byteOffset, + 4, + ).getUint32(0); + if (length === 0 || length > MAX_PACKET) { + throw new ProtocolError(`An SFTP packet of ${length} bytes is wrong`); + } + if (this.#buffer.length >= 4 + length) { + const packet = this.#buffer.subarray(4, 4 + length); + this.#buffer = this.#buffer.subarray(4 + length); + return packet; + } + } + const data = await this.#channel.read(); + if (data === null) { + throw new ProtocolError("The SFTP server closed the channel"); + } + if (this.#buffer.length === 0) { + this.#buffer = data; + } else { + const joined = new Uint8Array(this.#buffer.length + data.length); + joined.set(this.#buffer); + joined.set(data, this.#buffer.length); + this.#buffer = joined; + } + } + } + + /** Reads the answers, and gives each one to its request. */ + async #readLoop() { + try { + while (true) { + const reader = new Reader(await this.#readPacket()); + const type = reader.byte(); + const id = reader.uint32(); + const pending = this.#pending.get(id); + if (!pending) throw new ProtocolError("An answer has no request"); + this.#pending.delete(id); + pending.resolve({ type, reader }); + } + } catch (error) { + this.#error = error instanceof Error ? error : new Error(String(error)); + for (const pending of this.#pending.values()) pending.reject(this.#error); + this.#pending.clear(); + } + } + + /** + * Sends a request, and gives its answer. A STATUS with an error throws + * its SFTPError. + * @param {number} type + * @param {(writer: Writer) => void} write Writes the data after the ID. + * @returns {Promise} + */ + async #call(type, write) { + if (this.#error) throw this.#error; + const id = this.#nextId; + this.#nextId = ((this.#nextId + 1) >>> 0) || 1; + const writer = new Writer(type === FXP.WRITE ? 64 * 1024 : 256) + .byte(type) + .uint32(id); + write(writer); + /** @type {Promise} */ + const answer = new Promise((resolve, reject) => { + this.#pending.set(id, { resolve, reject }); + }); + await this.#writePacket(writer.bytes()); + const result = await answer; + if (result.type === FXP.STATUS) { + const code = result.reader.uint32(); + let message = ""; + try { + message = result.reader.string(); + } catch { + // Some servers send no message. + } + if (code !== STATUS.OK) throw new SFTPError(code, message); + } + return result; + } + + /** + * The attributes of a path, with the links followed. + * @param {string} path + * @returns {Promise} + */ + async stat(path) { + const { type, reader } = await this.#call(FXP.STAT, (w) => w.string(path)); + expect(type, FXP.ATTRS); + return readAttributes(reader); + } + + /** + * The attributes of a path, with no link followed. + * @param {string} path + * @returns {Promise} + */ + async lstat(path) { + const { type, reader } = await this.#call(FXP.LSTAT, (w) => w.string(path)); + expect(type, FXP.ATTRS); + return readAttributes(reader); + } + + /** + * The attributes of an open file. + * @param {Uint8Array} handle + * @returns {Promise} + */ + async fstat(handle) { + const { type, reader } = await this.#call( + FXP.FSTAT, + (w) => w.string(handle), + ); + expect(type, FXP.ATTRS); + return readAttributes(reader); + } + + /** + * Changes the attributes of a path, for example its time. + * @param {string} path + * @param {Attributes} attrs + */ + async setstat(path, attrs) { + await this.#call( + FXP.SETSTAT, + (w) => writeAttributes(w.string(path), attrs), + ); + } + + /** + * Opens a file. + * @param {string} path + * @param {number} flags The flags of OPEN. + * @param {Attributes} [attrs] The attributes of a new file. + * @returns {Promise} The handle of the file. + */ + async open(path, flags, attrs = {}) { + const { type, reader } = await this.#call( + FXP.OPEN, + (w) => writeAttributes(w.string(path).uint32(flags), attrs), + ); + expect(type, FXP.HANDLE); + return reader.bytes().slice(); + } + + /** + * Closes a handle of a file or of a folder. + * @param {Uint8Array} handle + */ + async close(handle) { + await this.#call(FXP.CLOSE, (w) => w.string(handle)); + } + + /** + * Reads bytes of an open file. + * @param {Uint8Array} handle + * @param {bigint | number} offset + * @param {number} length + * @returns {Promise} null at the end of the file. + */ + async read(handle, offset, length) { + try { + const { type, reader } = await this.#call( + FXP.READ, + (w) => w.string(handle).uint64(offset).uint32(length), + ); + expect(type, FXP.DATA); + return reader.bytes().slice(); + } catch (error) { + if (error instanceof SFTPError && error.code === STATUS.EOF) return null; + throw error; + } + } + + /** + * Writes bytes in an open file. + * @param {Uint8Array} handle + * @param {bigint | number} offset + * @param {Uint8Array} data + */ + async write(handle, offset, data) { + await this.#call( + FXP.WRITE, + (w) => w.string(handle).uint64(offset).string(data), + ); + } + + /** + * The entries of a folder, with no `.` and no `..`. + * @param {string} path + * @returns {Promise} + */ + async readdir(path) { + const { type, reader } = await this.#call( + FXP.OPENDIR, + (w) => w.string(path), + ); + expect(type, FXP.HANDLE); + const handle = reader.bytes().slice(); + /** @type {Entry[]} */ + const entries = []; + try { + while (true) { + let answer; + try { + answer = await this.#call(FXP.READDIR, (w) => w.string(handle)); + } catch (error) { + if (error instanceof SFTPError && error.code === STATUS.EOF) break; + throw error; + } + expect(answer.type, FXP.NAME); + const count = answer.reader.uint32(); + for (let i = 0; i < count; i++) { + const filename = answer.reader.string(); + answer.reader.string(); + const attrs = readAttributes(answer.reader); + if (filename !== "." && filename !== "..") { + entries.push({ filename, attrs }); + } + } + } + } finally { + await this.close(handle).catch(() => {}); + } + return entries; + } + + /** + * Makes a folder. + * @param {string} path + */ + async mkdir(path) { + await this.#call(FXP.MKDIR, (w) => writeAttributes(w.string(path), {})); + } + + /** + * Deletes an empty folder. + * @param {string} path + */ + async rmdir(path) { + await this.#call(FXP.RMDIR, (w) => w.string(path)); + } + + /** + * Deletes a file. + * @param {string} path + */ + async remove(path) { + await this.#call(FXP.REMOVE, (w) => w.string(path)); + } + + /** + * Renames a file or a folder. With `posix-rename@openssh.com`, the new + * path can exist: then the file at that path is replaced. + * @param {string} from + * @param {string} to + * @param {boolean} [replace] True to replace a file at the new path. + */ + async rename(from, to, replace = false) { + if (replace && this.extensions.has("posix-rename@openssh.com")) { + await this.#call( + FXP.EXTENDED, + (w) => w.string("posix-rename@openssh.com").string(from).string(to), + ); + return; + } + if (replace) { + // As rclone: with no POSIX rename, the old file goes first. + await this.remove(to).catch((error) => { + if ( + !(error instanceof SFTPError && error.code === STATUS.NO_SUCH_FILE) + ) { + throw error; + } + }); + } + await this.#call(FXP.RENAME, (w) => w.string(from).string(to)); + } + + /** + * Makes a hard link, with `hardlink@openssh.com`. + * @param {string} from + * @param {string} to + */ + async hardlink(from, to) { + if (!this.extensions.has("hardlink@openssh.com")) { + throw new SFTPError(STATUS.OP_UNSUPPORTED, "No hard links"); + } + await this.#call( + FXP.EXTENDED, + (w) => w.string("hardlink@openssh.com").string(from).string(to), + ); + } + + /** + * The absolute path of a path, as the server knows it. + * @param {string} path For example `.` for the login folder. + * @returns {Promise} + */ + async realpath(path) { + const { type, reader } = await this.#call( + FXP.REALPATH, + (w) => w.string(path), + ); + expect(type, FXP.NAME); + if (reader.uint32() < 1) throw new ProtocolError("REALPATH gave no name"); + return reader.string(); + } + + /** Closes the channel of the session. */ + async end() { + await this.#channel.close(); + } +} + +/** + * An answer of the server. + * @typedef {{ type: number, reader: Reader }} Answer + */ + +/** + * Makes sure that an answer has a type. + * @param {number} type + * @param {number} expected + */ +function expect(type, expected) { + if (type !== expected) { + throw new ProtocolError(`The answer ${type} is not the answer ${expected}`); + } +} diff --git a/backend/sftp/ssh/cipher.js b/backend/sftp/ssh/cipher.js new file mode 100644 index 0000000..013bb49 --- /dev/null +++ b/backend/sftp/ssh/cipher.js @@ -0,0 +1,282 @@ +/** + * The ciphers and the MACs of the packets of SSH, with WebCrypto, as + * RFC 4253 section 6, RFC 4344 (AES-CTR), RFC 5647 and the + * `aes*-gcm@openssh.com` of OpenSSH (AES-GCM), and the `*-etm@openssh.com` + * MACs of OpenSSH. + */ + +import { concat, ProtocolError } from "./wire.js"; + +const { subtle } = globalThis.crypto; + +/** The largest packet that the client reads, as OpenSSH. */ +export const MAX_PACKET = 256 * 1024; + +/** The ciphers, with the sizes of their keys and of their IVs. */ +export const CIPHERS = { + "aes128-gcm@openssh.com": { key: 16, iv: 12, gcm: true }, + "aes256-gcm@openssh.com": { key: 32, iv: 12, gcm: true }, + "aes128-ctr": { key: 16, iv: 16, gcm: false }, + "aes192-ctr": { key: 24, iv: 16, gcm: false }, + "aes256-ctr": { key: 32, iv: 16, gcm: false }, +}; + +/** The MACs, with the sizes of their keys. */ +export const MACS = { + "hmac-sha2-256-etm@openssh.com": { key: 32, hash: "SHA-256", etm: true }, + "hmac-sha2-512-etm@openssh.com": { key: 64, hash: "SHA-512", etm: true }, + "hmac-sha2-256": { key: 32, hash: "SHA-256", etm: false }, + "hmac-sha2-512": { key: 64, hash: "SHA-512", etm: false }, +}; + +/** + * The reads of the bytes of a connection. + * @typedef {{ read(size: number): Promise }} Source + */ + +/** + * Seals and opens the packets of one direction. + * @typedef {Object} PacketCipher + * @property {number} blockSize + * @property {boolean} lengthInClear True when the length of a packet is not + * encrypted: then the padding does not count the length. + * @property {(sequence: number, packet: Uint8Array) => Promise} seal + * Makes the bytes of a packet from `padding_length || payload || padding`. + * @property {(sequence: number, source: Source) => Promise} open + * Reads a packet, and gives `padding_length || payload || padding`. + */ + +/** + * The length of a packet, as 4 bytes. + * @param {number} length + */ +function lengthBytes(length) { + const bytes = new Uint8Array(4); + new DataView(bytes.buffer).setUint32(0, length); + return bytes; +} + +/** + * Reads the length of a packet, and checks it. + * @param {Uint8Array} bytes + * @param {number} blockSize The length must be a multiple of it. + */ +function readLength(bytes, blockSize) { + const length = new DataView(bytes.buffer, bytes.byteOffset, 4).getUint32(0); + if (length < 5 || length > MAX_PACKET || length % blockSize !== 0) { + throw new ProtocolError(`The length ${length} of a packet is not correct`); + } + return length; +} + +/** + * The cipher of the start of a connection: no encryption, and no MAC. + * @returns {PacketCipher} + */ +export function noCipher() { + return { + blockSize: 8, + lengthInClear: false, + seal: async (_sequence, packet) => + concat(lengthBytes(packet.length), packet), + open: async (_sequence, source) => { + const header = await source.read(4); + const view = new DataView(header.buffer, header.byteOffset, 4); + const length = view.getUint32(0); + if (length < 5 || length > MAX_PACKET) { + throw new ProtocolError(`The length ${length} of a packet is wrong`); + } + return await source.read(length); + }, + }; +} + +/** + * The cipher of one direction, with its keys. + * @param {string} name The name of the cipher. + * @param {string | null} macName The name of the MAC, or null for AES-GCM. + * @param {{ key: Uint8Array, iv: Uint8Array, mac: Uint8Array }} keys + * @returns {Promise} + */ +export async function makeCipher(name, macName, keys) { + const spec = CIPHERS[name]; + if (!spec) throw new ProtocolError(`The cipher ${name} is not known`); + if (spec.gcm) return await gcmCipher(keys.key, keys.iv); + const mac = MACS[macName ?? ""]; + if (!mac) throw new ProtocolError(`The MAC ${macName} is not known`); + return await ctrCipher(keys.key, keys.iv, mac, keys.mac); +} + +/** + * AES-GCM, as RFC 5647: the length is not encrypted, and it is the + * additional data. The IV has a fixed part of 4 bytes and a counter of 8 + * bytes, which goes up by one for each packet. + * @param {Uint8Array} keyBytes + * @param {Uint8Array} ivBytes + * @returns {Promise} + */ +async function gcmCipher(keyBytes, ivBytes) { + const key = await subtle.importKey("raw", keyBytes, "AES-GCM", false, [ + "encrypt", + "decrypt", + ]); + const iv = ivBytes.slice(); + const view = new DataView(iv.buffer); + const next = () => { + view.setBigUint64(4, (view.getBigUint64(4) + 1n) & 0xffffffffffffffffn); + }; + return { + blockSize: 16, + lengthInClear: true, + seal: async (_sequence, packet) => { + const length = lengthBytes(packet.length); + const sealed = new Uint8Array( + await subtle.encrypt( + { name: "AES-GCM", iv, additionalData: length, tagLength: 128 }, + key, + packet, + ), + ); + next(); + return concat(length, sealed); + }, + open: async (_sequence, source) => { + const header = await source.read(4); + const length = readLength(header, 16); + const sealed = await source.read(length + 16); + let packet; + try { + packet = await subtle.decrypt( + { name: "AES-GCM", iv, additionalData: header, tagLength: 128 }, + key, + sealed, + ); + } catch { + throw new ProtocolError("The MAC of a packet is not correct"); + } + next(); + return new Uint8Array(packet); + }, + }; +} + +/** + * Adds a number of blocks to the counter of AES-CTR, a number of 128 bits. + * @param {Uint8Array} counter + * @param {number} blocks + */ +function addBlocks(counter, blocks) { + let carry = blocks; + for (let i = 15; i >= 0 && carry > 0; i--) { + const sum = counter[i] + (carry % 256); + counter[i] = sum & 0xff; + carry = Math.floor(carry / 256) + (sum >> 8); + } +} + +/** + * AES-CTR with an HMAC, as RFC 4344, or with the MAC of the encrypted + * packet (`*-etm@openssh.com`). + * @param {Uint8Array} keyBytes + * @param {Uint8Array} ivBytes + * @param {typeof MACS[keyof typeof MACS]} mac + * @param {Uint8Array} macBytes + * @returns {Promise} + */ +async function ctrCipher(keyBytes, ivBytes, mac, macBytes) { + const key = await subtle.importKey("raw", keyBytes, "AES-CTR", false, [ + "encrypt", + "decrypt", + ]); + const macKey = await subtle.importKey( + "raw", + macBytes.subarray(0, mac.key), + { name: "HMAC", hash: mac.hash }, + false, + ["sign", "verify"], + ); + const macSize = mac.key; + const counter = ivBytes.slice(0, 16); + /** + * Encrypts or decrypts bytes from the counter, and moves the counter. + * @param {Uint8Array} bytes A multiple of 16 bytes. + */ + const crypt = async (bytes) => { + const result = new Uint8Array( + await subtle.encrypt( + { name: "AES-CTR", counter, length: 128 }, + key, + bytes, + ), + ); + addBlocks(counter, bytes.length / 16); + return result; + }; + /** @param {number} sequence @param {Uint8Array[]} parts */ + const sign = async (sequence, ...parts) => + new Uint8Array( + await subtle.sign( + "HMAC", + macKey, + concat(lengthBytes(sequence), ...parts), + ), + ); + /** + * @param {number} sequence + * @param {Uint8Array} tag + * @param {Uint8Array[]} parts + */ + const verify = async (sequence, tag, ...parts) => { + const ok = await subtle.verify( + "HMAC", + macKey, + tag, + concat(lengthBytes(sequence), ...parts), + ); + if (!ok) throw new ProtocolError("The MAC of a packet is not correct"); + }; + + if (mac.etm) { + return { + blockSize: 16, + lengthInClear: true, + seal: async (sequence, packet) => { + const length = lengthBytes(packet.length); + const encrypted = await crypt(packet); + return concat( + length, + encrypted, + await sign(sequence, length, encrypted), + ); + }, + open: async (sequence, source) => { + const header = await source.read(4); + const length = readLength(header, 16); + const encrypted = await source.read(length); + // The MAC is checked before the decryption. + await verify(sequence, await source.read(macSize), header, encrypted); + return await crypt(encrypted); + }, + }; + } + return { + blockSize: 16, + lengthInClear: false, + seal: async (sequence, packet) => { + const plain = concat(lengthBytes(packet.length), packet); + const tag = await sign(sequence, plain); + return concat(await crypt(plain), tag); + }, + open: async (sequence, source) => { + const first = await crypt(await source.read(16)); + const length = readLength(first, 1); + if ((length + 4) % 16 !== 0) { + throw new ProtocolError("The length of a packet is not correct"); + } + const rest = await crypt(await source.read(length + 4 - 16)); + const plain = concat(first, rest); + await verify(sequence, await source.read(macSize), plain); + return plain.subarray(4); + }, + }; +} diff --git a/backend/sftp/ssh/client.js b/backend/sftp/ssh/client.js new file mode 100644 index 0000000..65b60de --- /dev/null +++ b/backend/sftp/ssh/client.js @@ -0,0 +1,625 @@ +/** + * A client of SSH: the authentication of the user, as RFC 4252, and the + * channels of the connection, as RFC 4254. + */ + +import { Connection } from "./socket.js"; +import { DisconnectError, Transport } from "./transport.js"; +import { ProtocolError, Reader, Writer } from "./wire.js"; + +/** The numbers of the messages of the authentication and the channels. */ +const MSG = { + SERVICE_REQUEST: 5, + SERVICE_ACCEPT: 6, + USERAUTH_REQUEST: 50, + USERAUTH_FAILURE: 51, + USERAUTH_SUCCESS: 52, + USERAUTH_BANNER: 53, + USERAUTH_INFO_REQUEST: 60, + USERAUTH_INFO_RESPONSE: 61, + GLOBAL_REQUEST: 80, + REQUEST_SUCCESS: 81, + REQUEST_FAILURE: 82, + CHANNEL_OPEN: 90, + CHANNEL_OPEN_CONFIRMATION: 91, + CHANNEL_OPEN_FAILURE: 92, + CHANNEL_WINDOW_ADJUST: 93, + CHANNEL_DATA: 94, + CHANNEL_EXTENDED_DATA: 95, + CHANNEL_EOF: 96, + CHANNEL_CLOSE: 97, + CHANNEL_REQUEST: 98, + CHANNEL_SUCCESS: 99, + CHANNEL_FAILURE: 100, +}; + +/** The window of a channel: the bytes that the server can send. */ +const WINDOW = 2 * 1024 * 1024; + +/** + * The largest data of one message of the server on a channel: a packet of + * SFTP with 32 KiB of data fits in one message. + */ +const MAX_DATA = 64 * 1024; + +/** The largest data of one message of the client, as OpenSSH. */ +const MAX_SEND = 32 * 1024; + +/** An error of the authentication of the user. */ +export class AuthenticationError extends Error {} + +/** + * The options of a connection. + * @typedef {Object} ClientOptions + * @property {string} host + * @property {number} port + * @property {string} username + * @property {string} [password] + * @property {import("./keys.js").PrivateKey} [privateKey] + * @property {(key: import("./transport.js").HostKey) => boolean | Promise} checkHostKey + * @property {string[]} [hostKeyTypes] The types of the keys that the + * client knows for the server. + * @property {{ ciphers?: string[], macs?: string[], kex?: string[], hostKeyAlgorithms?: string[] }} [algorithms] + * The algorithms that the client can use, by preference. + * @property {number} [timeout] The time to connect, in milliseconds. + */ + +/** A connection of SSH, with the user signed in. */ +export class SSHClient { + /** @type {Transport} */ + #transport; + /** @type {Map} */ + #channels = new Map(); + #nextId = 0; + /** @type {Error | null} */ + #error = null; + /** @type {Promise} */ + closed; + + /** @param {Transport} transport */ + constructor(transport) { + this.#transport = transport; + this.closed = this.#readLoop(); + } + + /** + * Opens a connection, and signs in the user. + * @param {ClientOptions} options + * @returns {Promise} + * @throws {AuthenticationError | ProtocolError | Error} + */ + static async connect(options) { + const socket = await Connection.open( + options.host, + options.port, + options.timeout ?? 60000, + ); + const transport = new Transport(socket, { + checkHostKey: options.checkHostKey, + hostKeyTypes: options.hostKeyTypes, + ...options.algorithms, + }); + try { + await transport.start(); + await authenticate(transport, options); + } catch (error) { + await transport.close(); + throw error; + } + return new SSHClient(transport); + } + + /** Reads the messages of the server, and gives them to the channels. */ + async #readLoop() { + try { + while (true) { + const payload = await this.#transport.receive(); + this.#dispatch(payload); + } + } catch (error) { + this.#error = error instanceof Error ? error : new Error(String(error)); + for (const channel of this.#channels.values()) channel.fail(this.#error); + this.#channels.clear(); + } + } + + /** @param {Uint8Array} payload */ + #dispatch(payload) { + const reader = new Reader(payload); + const type = reader.byte(); + if (type === MSG.GLOBAL_REQUEST) { + reader.string(); + if (reader.boolean()) { + this.#send(new Uint8Array([MSG.REQUEST_FAILURE])); + } + return; + } + if (type < MSG.CHANNEL_OPEN || type > MSG.CHANNEL_FAILURE) { + // An unexpected message, for example of the authentication. + return; + } + const channel = this.#channels.get(reader.uint32()); + if (!channel) { + throw new ProtocolError("A message came for a channel that is closed"); + } + channel.receive(type, reader); + if (channel.isClosed) this.#channels.delete(channel.id); + } + + /** + * Sends a message. An error of the write ends the connection. + * @param {Uint8Array} payload + */ + #send(payload) { + return this.#transport.send(payload); + } + + /** + * Opens a channel of a session. + * @returns {Promise} + */ + async openSession() { + if (this.#error) throw this.#error; + const id = this.#nextId++; + const channel = new Channel(id, (payload) => this.#send(payload)); + this.#channels.set(id, channel); + await this.#send( + new Writer() + .byte(MSG.CHANNEL_OPEN) + .string("session") + .uint32(id) + .uint32(WINDOW) + .uint32(MAX_DATA) + .bytes(), + ); + await channel.opened; + return channel; + } + + /** + * Opens a channel with a subsystem, for example `sftp`. + * @param {string} name + * @returns {Promise} + */ + async subsystem(name) { + const channel = await this.openSession(); + const ok = await channel.request( + "subsystem", + true, + (writer) => writer.string(name), + ); + if (!ok) { + await channel.close(); + throw new ProtocolError(`The server has no subsystem ${name}`); + } + return channel; + } + + /** True when the connection cannot send more. */ + get isClosed() { + return this.#error !== null; + } + + /** + * Tells if the connection keeps the process of Node.js running. + * @param {boolean} active + */ + setActive(active) { + this.#transport.setActive(active); + } + + /** Closes the connection. */ + async close() { + await this.#transport.close(); + await this.closed; + } +} + +/** + * Signs in the user: with the key, then with the password, then with + * keyboard-interactive and the password, as rclone. + * @param {Transport} transport + * @param {ClientOptions} options + */ +async function authenticate(transport, options) { + await transport.send( + new Writer().byte(MSG.SERVICE_REQUEST).string("ssh-userauth").bytes(), + ); + const accept = await transport.receive(); + if (accept[0] !== MSG.SERVICE_ACCEPT) { + throw new ProtocolError("The server did not accept the authentication"); + } + const { username, password, privateKey } = options; + /** @param {string} method */ + const request = (method) => + new Writer() + .byte(MSG.USERAUTH_REQUEST) + .string(username) + .string("ssh-connection") + .string(method); + + /** + * Reads the answer of a request of authentication. + * @returns {Promise<{ ok: boolean, methods: string[] }>} + */ + const answer = async () => { + while (true) { + const payload = await transport.receive(); + const reader = new Reader(payload); + const type = reader.byte(); + if (type === MSG.USERAUTH_SUCCESS) return { ok: true, methods: [] }; + if (type === MSG.USERAUTH_FAILURE) { + return { ok: false, methods: reader.nameList() }; + } + if (type === MSG.USERAUTH_BANNER) continue; + if (type === MSG.USERAUTH_INFO_REQUEST) { + // The answer of each prompt of keyboard-interactive is the password. + reader.string(); + reader.string(); + reader.string(); + const count = reader.uint32(); + const writer = new Writer().byte(MSG.USERAUTH_INFO_RESPONSE).uint32( + count, + ); + for (let i = 0; i < count; i++) writer.string(password ?? ""); + await transport.send(writer.bytes()); + continue; + } + throw new ProtocolError(`The message ${type} came in the authentication`); + } + }; + + await transport.send(request("none").bytes()); + let { ok, methods } = await answer(); + if (ok) return; + + if (privateKey && methods.includes("publickey")) { + for (const algorithm of signatureAlgorithms(privateKey, transport)) { + const signed = request("publickey") + .boolean(true) + .string(algorithm) + .string(privateKey.blob) + .bytes(); + const data = new Writer() + .string(/** @type {Uint8Array} */ (transport.sessionId)) + .raw(signed) + .bytes(); + await transport.send( + new Writer() + .raw(signed) + .string(await privateKey.sign(algorithm, data)) + .bytes(), + ); + ({ ok, methods } = await answer()); + if (ok) return; + if (!methods.includes("publickey")) break; + } + } + if (password !== undefined && methods.includes("password")) { + await transport.send( + request("password").boolean(false).string(password).bytes(), + ); + ({ ok, methods } = await answer()); + if (ok) return; + } + if (password !== undefined && methods.includes("keyboard-interactive")) { + await transport.send( + request("keyboard-interactive").string("").string("").bytes(), + ); + ({ ok, methods } = await answer()); + if (ok) return; + } + throw new AuthenticationError( + `The server refused the user ${username}: it accepts ${ + methods.join(", ") || "no method" + }`, + ); +} + +/** + * The algorithms of the signatures of a key, that the server accepts. + * @param {import("./keys.js").PrivateKey} key + * @param {Transport} transport + */ +function signatureAlgorithms(key, transport) { + if (key.type !== "ssh-rsa") return key.algorithms; + const accepted = transport.serverSigAlgs; + if (accepted) { + const found = key.algorithms.find((name) => accepted.includes(name)); + return found ? [found] : []; + } + // A server with no list of its algorithms can be old. + return ["rsa-sha2-256", "ssh-rsa"]; +} + +/** A channel of a connection. */ +export class Channel { + /** @type {number} */ + id; + /** The number of the channel for the server. */ + remoteId = -1; + #remoteWindow = 0; + #remoteMax = MAX_SEND; + #localWindow = WINDOW; + /** The bytes that the client read, and did not give to the window. */ + #consumed = 0; + /** @type {(payload: Uint8Array) => Promise} */ + #send; + /** @type {Uint8Array[]} */ + #data = []; + /** The data of the standard error, for `exec`. */ + /** @type {Uint8Array[]} */ + stderr = []; + /** @type {number | null} */ + exitStatus = null; + #eof = false; + isClosed = false; + #sentClose = false; + /** @type {Error | null} */ + #error = null; + /** @type {(() => void) | null} */ + #wakeRead = null; + /** @type {(() => void) | null} */ + #wakeWrite = null; + /** The writes of the channel, in their order. */ + /** @type {Promise} */ + #writes = Promise.resolve(); + /** @type {((ok: boolean) => void)[]} */ + #replies = []; + /** @type {Promise} */ + opened; + /** @type {(() => void) | null} */ + #resolveOpen = null; + /** @type {((error: Error) => void) | null} */ + #rejectOpen = null; + /** @type {Promise} */ + closedPromise; + /** @type {(() => void) | null} */ + #resolveClosed = null; + + /** + * @param {number} id + * @param {(payload: Uint8Array) => Promise} send + */ + constructor(id, send) { + this.id = id; + this.#send = send; + this.opened = new Promise((resolve, reject) => { + this.#resolveOpen = resolve; + this.#rejectOpen = reject; + }); + this.opened.catch(() => {}); + this.closedPromise = new Promise((resolve) => + this.#resolveClosed = resolve + ); + } + + /** + * Takes a message of the server for this channel. + * @param {number} type + * @param {Reader} reader + */ + receive(type, reader) { + switch (type) { + case MSG.CHANNEL_OPEN_CONFIRMATION: + this.remoteId = reader.uint32(); + this.#remoteWindow = reader.uint32(); + this.#remoteMax = Math.min(reader.uint32(), MAX_SEND); + this.#resolveOpen?.(); + break; + case MSG.CHANNEL_OPEN_FAILURE: { + reader.uint32(); + const message = reader.string(); + this.isClosed = true; + this.#rejectOpen?.( + new ProtocolError(`The server refused a channel: ${message}`), + ); + this.#resolveClosed?.(); + break; + } + case MSG.CHANNEL_WINDOW_ADJUST: + this.#remoteWindow = Math.min( + this.#remoteWindow + reader.uint32(), + 0xffffffff, + ); + this.#wake("write"); + break; + case MSG.CHANNEL_DATA: + case MSG.CHANNEL_EXTENDED_DATA: { + const extended = type === MSG.CHANNEL_EXTENDED_DATA + ? reader.uint32() + : null; + const data = reader.bytes(); + if (data.length > this.#localWindow) { + throw new ProtocolError("The server sent more than the window"); + } + this.#localWindow -= data.length; + if (extended === null) { + this.#data.push(data); + this.#wake("read"); + } else { + this.stderr.push(data); + this.#consume(data.length); + } + break; + } + case MSG.CHANNEL_EOF: + this.#eof = true; + this.#wake("read"); + break; + case MSG.CHANNEL_CLOSE: + this.#eof = true; + this.isClosed = true; + if (!this.#sentClose) { + this.#sentClose = true; + this.#send( + new Writer().byte(MSG.CHANNEL_CLOSE).uint32(this.remoteId).bytes(), + ).catch(() => {}); + } + for (const reply of this.#replies.splice(0)) reply(false); + this.#wake("read"); + this.#wake("write"); + this.#resolveClosed?.(); + break; + case MSG.CHANNEL_REQUEST: { + const name = reader.string(); + const wantReply = reader.boolean(); + if (name === "exit-status") this.exitStatus = reader.uint32(); + if (wantReply) { + this.#send( + new Writer().byte(MSG.CHANNEL_FAILURE).uint32(this.remoteId) + .bytes(), + ).catch(() => {}); + } + break; + } + case MSG.CHANNEL_SUCCESS: + case MSG.CHANNEL_FAILURE: + this.#replies.shift()?.(type === MSG.CHANNEL_SUCCESS); + break; + } + } + + /** @param {"read" | "write"} side */ + #wake(side) { + if (side === "read") { + const wake = this.#wakeRead; + this.#wakeRead = null; + wake?.(); + } else { + const wake = this.#wakeWrite; + this.#wakeWrite = null; + wake?.(); + } + } + + /** + * Ends the channel with an error of the connection. + * @param {Error} error + */ + fail(error) { + this.#error = error; + this.isClosed = true; + this.#rejectOpen?.(error); + for (const reply of this.#replies.splice(0)) reply(false); + this.#wake("read"); + this.#wake("write"); + this.#resolveClosed?.(); + } + + /** + * Gives the read bytes back to the window of the server. + * @param {number} size + */ + #consume(size) { + this.#consumed += size; + if (this.#consumed >= WINDOW / 2 && !this.isClosed) { + const add = this.#consumed; + this.#consumed = 0; + this.#localWindow += add; + this.#send( + new Writer() + .byte(MSG.CHANNEL_WINDOW_ADJUST) + .uint32(this.remoteId) + .uint32(add) + .bytes(), + ).catch(() => {}); + } + } + + /** + * Sends a request of the channel. + * @param {string} name For example `subsystem` or `exec`. + * @param {boolean} wantReply + * @param {(writer: Writer) => void} [write] Writes the data of the request. + * @returns {Promise} True when the server accepts the request. + */ + async request(name, wantReply, write) { + const writer = new Writer() + .byte(MSG.CHANNEL_REQUEST) + .uint32(this.remoteId) + .string(name) + .boolean(wantReply); + write?.(writer); + const reply = wantReply + ? new Promise((resolve) => this.#replies.push(resolve)) + : Promise.resolve(true); + await this.#send(writer.bytes()); + return await reply; + } + + /** + * Reads the next bytes of the channel. + * @returns {Promise} null at the end of the data. + */ + async read() { + while (this.#data.length === 0) { + if (this.#error) throw this.#error; + if (this.#eof) return null; + await new Promise((resolve) => this.#wakeRead = resolve); + } + const data = /** @type {Uint8Array} */ (this.#data.shift()); + this.#consume(data.length); + return data; + } + + /** + * Writes bytes to the channel, in the window of the server. The writes + * go one after the other: the bytes of two writes do not mix. + * @param {Uint8Array} bytes + * @returns {Promise} + */ + write(bytes) { + const write = this.#writes.then(() => this.#write(bytes)); + this.#writes = write.catch(() => {}); + return write; + } + + /** @param {Uint8Array} bytes */ + async #write(bytes) { + let offset = 0; + while (offset < bytes.length) { + if (this.#error) throw this.#error; + if (this.isClosed) throw new ProtocolError("The channel is closed"); + if (this.#remoteWindow === 0) { + await new Promise((resolve) => this.#wakeWrite = resolve); + continue; + } + const size = Math.min( + bytes.length - offset, + this.#remoteWindow, + this.#remoteMax, + ); + this.#remoteWindow -= size; + await this.#send( + new Writer(size + 9) + .byte(MSG.CHANNEL_DATA) + .uint32(this.remoteId) + .string(bytes.subarray(offset, offset + size)) + .bytes(), + ); + offset += size; + } + } + + /** Sends the end of the data of the client. */ + async eof() { + if (this.isClosed) return; + await this.#send( + new Writer().byte(MSG.CHANNEL_EOF).uint32(this.remoteId).bytes(), + ); + } + + /** Closes the channel, and waits for the close of the server. */ + async close() { + if (!this.isClosed && !this.#sentClose) { + this.#sentClose = true; + await this.#send( + new Writer().byte(MSG.CHANNEL_CLOSE).uint32(this.remoteId).bytes(), + ).catch(() => {}); + } + await this.closedPromise; + } +} + +export { DisconnectError }; diff --git a/backend/sftp/ssh/der.js b/backend/sftp/ssh/der.js new file mode 100644 index 0000000..3e01a55 --- /dev/null +++ b/backend/sftp/ssh/der.js @@ -0,0 +1,125 @@ +/** + * A small reader and writer of DER, for the private keys of PEM files: + * PKCS #1, SEC 1 and PKCS #8. + */ + +import { ProtocolError } from "./wire.js"; + +/** + * An element of DER. + * @typedef {Object} Element + * @property {number} tag + * @property {Uint8Array} value + */ + +/** + * Reads the elements of DER in a value, as the children of a SEQUENCE. + * @param {Uint8Array} bytes + * @returns {Element[]} + * @throws {ProtocolError} When the DER is not correct. + */ +export function readDER(bytes) { + /** @type {Element[]} */ + const elements = []; + let offset = 0; + while (offset < bytes.length) { + if (offset + 2 > bytes.length) throw new ProtocolError("DER is too short"); + const tag = bytes[offset++]; + let length = bytes[offset++]; + if (length & 0x80) { + const count = length & 0x7f; + if (count === 0 || count > 4 || offset + count > bytes.length) { + throw new ProtocolError("A DER length is not correct"); + } + length = 0; + for (let i = 0; i < count; i++) length = length * 256 + bytes[offset++]; + } + if (offset + length > bytes.length) { + throw new ProtocolError("DER is too short"); + } + elements.push({ tag, value: bytes.subarray(offset, offset + length) }); + offset += length; + } + return elements; +} + +/** + * Reads the one element of DER of a value, with the tag that it must have. + * @param {Uint8Array} bytes + * @param {number} tag + */ +export function readOne(bytes, tag) { + const elements = readDER(bytes); + if (elements.length !== 1 || elements[0].tag !== tag) { + throw new ProtocolError("DER is not the expected type"); + } + return elements[0].value; +} + +/** The tags of DER. */ +export const TAG = { + INTEGER: 0x02, + BIT_STRING: 0x03, + OCTET_STRING: 0x04, + NULL: 0x05, + OID: 0x06, + SEQUENCE: 0x30, +}; + +/** + * Writes an element of DER. + * @param {number} tag + * @param {...Uint8Array} parts The value, in parts. + */ +export function writeDER(tag, ...parts) { + const length = parts.reduce((sum, part) => sum + part.length, 0); + /** @type {number[]} */ + const header = [tag]; + if (length < 0x80) header.push(length); + else { + /** @type {number[]} */ + const digits = []; + for (let rest = length; rest > 0; rest = Math.floor(rest / 256)) { + digits.unshift(rest % 256); + } + header.push(0x80 | digits.length, ...digits); + } + const bytes = new Uint8Array(header.length + length); + bytes.set(header); + let offset = header.length; + for (const part of parts) { + bytes.set(part, offset); + offset += part.length; + } + return bytes; +} + +/** + * Writes an OID of DER from its text, for example `1.3.101.112`. + * @param {string} text + */ +export function oid(text) { + const numbers = text.split(".").map(Number); + /** @type {number[]} */ + const bytes = [numbers[0] * 40 + numbers[1]]; + for (const number of numbers.slice(2)) { + /** @type {number[]} */ + const digits = [number & 0x7f]; + for (let rest = number >>> 7; rest > 0; rest >>>= 7) { + digits.unshift(0x80 | (rest & 0x7f)); + } + bytes.push(...digits); + } + return writeDER(TAG.OID, Uint8Array.from(bytes)); +} + +/** + * Tells if a value of DER is an OID. + * @param {Uint8Array} value The value of the element, with no tag. + * @param {string} text + */ +export function isOID(value, text) { + const expected = oid(text).subarray(2); + return value.length === expected.length && + value.every((byte, i) => byte === expected[i]); +} diff --git a/backend/sftp/ssh/keys.js b/backend/sftp/ssh/keys.js new file mode 100644 index 0000000..6f2fb99 --- /dev/null +++ b/backend/sftp/ssh/keys.js @@ -0,0 +1,588 @@ +/** + * The keys of SSH: the host keys of a server, and the private keys of a + * user, with WebCrypto. + * + * The algorithms: `ssh-ed25519`, `ecdsa-sha2-nistp256`, + * `ecdsa-sha2-nistp384`, `ecdsa-sha2-nistp521`, and RSA with + * `rsa-sha2-512`, `rsa-sha2-256` and `ssh-rsa`, as RFC 8709, RFC 5656 and + * RFC 8332. + */ + +import { isOID, oid, readDER, readOne, TAG, writeDER } from "./der.js"; +import { concat, mpintBytes, ProtocolError, Reader, Writer } from "./wire.js"; + +const { subtle } = globalThis.crypto; + +/** The curves of ECDSA, by their names in SSH. */ +const CURVES = { + nistp256: { + name: "P-256", + hash: "SHA-256", + size: 32, + oid: "1.2.840.10045.3.1.7", + }, + nistp384: { name: "P-384", hash: "SHA-384", size: 48, oid: "1.3.132.0.34" }, + nistp521: { name: "P-521", hash: "SHA-512", size: 66, oid: "1.3.132.0.35" }, +}; + +/** The hashes of the signatures of RSA, by their names in SSH. */ +const RSA_HASHES = { + "rsa-sha2-512": "SHA-512", + "rsa-sha2-256": "SHA-256", + "ssh-rsa": "SHA-1", +}; + +/** The smallest size of an RSA key, in bits, as OpenSSH. */ +const MIN_RSA_BITS = 1024; + +/** + * Encodes bytes in base64url, for JWK. + * @param {Uint8Array} bytes + */ +function base64url(bytes) { + let text = ""; + for (const byte of bytes) text += String.fromCharCode(byte); + return btoa(text).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +/** + * Decodes base64url, of JWK. + * @param {string} text + */ +function fromBase64url(text) { + const binary = atob(text.replace(/-/g, "+").replace(/_/g, "/")); + return Uint8Array.from(binary, (character) => character.charCodeAt(0)); +} + +/** + * Decodes base64. + * @param {string} text + * @throws {ProtocolError} For text that is not base64. + */ +export function fromBase64(text) { + try { + return Uint8Array.from(atob(text), (character) => character.charCodeAt(0)); + } catch { + throw new ProtocolError("The text is not base64"); + } +} + +/** + * Encodes bytes in base64. + * @param {Uint8Array} bytes + */ +export function toBase64(bytes) { + let text = ""; + for (const byte of bytes) text += String.fromCharCode(byte); + return btoa(text); +} + +/** + * The bytes of a number with no 0 byte before it, with a size. + * @param {Uint8Array} bytes A number that is not negative, big-endian. + * @param {number} size + */ +function fixed(bytes, size) { + let start = 0; + while (start < bytes.length - 1 && bytes[start] === 0) start++; + bytes = bytes.subarray(start); + if (bytes.length > size) throw new ProtocolError("A number is too large"); + const result = new Uint8Array(size); + result.set(bytes, size - bytes.length); + return result; +} + +/** + * A public key of SSH. + * @typedef {Object} PublicKey + * @property {string} type For example `ssh-ed25519` or `ssh-rsa`. + * @property {Uint8Array} blob The key in the format of SSH. + * @property {(algorithm: string, signature: Uint8Array, data: Uint8Array) => Promise} verify + * Checks a signature in the format of SSH. + */ + +/** + * Reads a public key in the format of SSH, as the host key of a server. + * @param {Uint8Array} blob + * @returns {Promise} + * @throws {ProtocolError} For a key that rfetch does not know. + */ +export async function readPublicKey(blob) { + const reader = new Reader(blob); + const type = reader.string(); + if (type === "ssh-ed25519") { + const raw = reader.bytes(); + if (raw.length !== 32) throw new ProtocolError("An Ed25519 key is wrong"); + const key = await subtle.importKey("raw", raw, "Ed25519", true, ["verify"]); + return { + type, + blob, + verify: async (algorithm, signature, data) => { + const sig = readSignature(signature, algorithm); + if (algorithm !== type || sig.length !== 64) return false; + return await subtle.verify("Ed25519", key, sig, data); + }, + }; + } + if (type.startsWith("ecdsa-sha2-")) { + const name = reader.string(); + const curve = CURVES[name]; + if (!curve || type !== `ecdsa-sha2-${name}`) { + throw new ProtocolError(`The curve of ${type} is not known`); + } + const key = await subtle.importKey( + "raw", + reader.bytes(), + { name: "ECDSA", namedCurve: curve.name }, + true, + ["verify"], + ); + return { + type, + blob, + verify: async (algorithm, signature, data) => { + if (algorithm !== type) return false; + const numbers = new Reader(readSignature(signature, algorithm)); + let raw; + try { + raw = concat( + fixed(numbers.bytes(), curve.size), + fixed(numbers.bytes(), curve.size), + ); + } catch { + return false; + } + return await subtle.verify( + { name: "ECDSA", hash: curve.hash }, + key, + raw, + data, + ); + }, + }; + } + if (type === "ssh-rsa") { + const e = reader.bytes(); + const n = reader.bytes(); + const bits = bitLength(n); + if (bits < MIN_RSA_BITS) { + throw new ProtocolError(`An RSA key of ${bits} bits is too small`); + } + const jwk = { + kty: "RSA", + n: base64url(fixed(n, Math.ceil(bits / 8))), + e: base64url(fixed(e, e.length)), + }; + /** @type {Map} */ + const keys = new Map(); + return { + type, + blob, + verify: async (algorithm, signature, data) => { + const hash = RSA_HASHES[algorithm]; + if (!hash) return false; + let key = keys.get(hash); + if (!key) { + key = await subtle.importKey( + "jwk", + jwk, + { name: "RSASSA-PKCS1-v1_5", hash }, + false, + ["verify"], + ); + keys.set(hash, key); + } + return await subtle.verify( + "RSASSA-PKCS1-v1_5", + key, + readSignature(signature, algorithm), + data, + ); + }, + }; + } + throw new ProtocolError(`The key type ${type} is not known`); +} + +/** + * The bytes of a signature in the format of SSH. + * @param {Uint8Array} signature `string algorithm, string bytes`. + * @param {string} algorithm The algorithm that the signature must have. + * @returns {Uint8Array} No bytes for another algorithm. + */ +function readSignature(signature, algorithm) { + const reader = new Reader(signature); + if (reader.string() !== algorithm) return new Uint8Array(0); + const bytes = reader.bytes(); + return reader.remaining === 0 ? bytes : new Uint8Array(0); +} + +/** + * The number of bits of a number. + * @param {Uint8Array} bytes Big-endian. + */ +function bitLength(bytes) { + let start = 0; + while (start < bytes.length && bytes[start] === 0) start++; + if (start === bytes.length) return 0; + return (bytes.length - start - 1) * 8 + (32 - Math.clz32(bytes[start])); +} + +/** + * The fingerprint of a public key, as OpenSSH: `SHA256:` and the hash in + * base64 with no `=`. + * @param {Uint8Array} blob + */ +export async function fingerprint(blob) { + const hash = new Uint8Array(await subtle.digest("SHA-256", blob)); + return `SHA256:${toBase64(hash).replace(/=+$/, "")}`; +} + +/** + * A private key of a user. + * @typedef {Object} PrivateKey + * @property {string} type For example `ssh-ed25519`. + * @property {Uint8Array} blob The public key in the format of SSH. + * @property {(algorithm: string, data: Uint8Array) => Promise} sign + * Signs data, and gives the signature in the format of SSH. + * @property {string[]} algorithms The algorithms of the signatures, in the + * order of preference. + */ + +/** + * Reads a private key of a PEM file: OpenSSH (`BEGIN OPENSSH PRIVATE KEY`), + * PKCS #8 (`BEGIN PRIVATE KEY`), PKCS #1 (`BEGIN RSA PRIVATE KEY`) or SEC 1 + * (`BEGIN EC PRIVATE KEY`). The key must not have a passphrase. + * @param {string} pem + * @returns {Promise} + * @throws {ProtocolError} For a key that rfetch cannot read. + */ +export async function readPrivateKey(pem) { + const match = /-----BEGIN ([A-Z0-9 ]+)-----([\s\S]*?)-----END \1-----/.exec( + pem, + ); + if (!match) throw new ProtocolError("The key is not a PEM file"); + const [, label, body] = match; + if (/Proc-Type:\s*4,ENCRYPTED/.test(body)) { + throw new ProtocolError("rfetch cannot read a key with a passphrase"); + } + const der = fromBase64(body.replace(/^[\w-]+:.*$/gm, "").replace(/\s/g, "")); + if (label === "OPENSSH PRIVATE KEY") return await readOpenSSHKey(der); + if (label === "PRIVATE KEY") return await readPKCS8(der); + if (label === "RSA PRIVATE KEY") return await rsaKey(readPKCS1(der)); + if (label === "EC PRIVATE KEY") return await readSEC1(der); + if (label === "ENCRYPTED PRIVATE KEY") { + throw new ProtocolError("rfetch cannot read a key with a passphrase"); + } + throw new ProtocolError(`rfetch cannot read a key of the type ${label}`); +} + +/** + * Reads a key of the format `openssh-key-v1` of OpenSSH. + * @param {Uint8Array} bytes + */ +async function readOpenSSHKey(bytes) { + const magic = new TextEncoder().encode("openssh-key-v1\0"); + if (!magic.every((byte, i) => bytes[i] === byte)) { + throw new ProtocolError("The key is not an OpenSSH key"); + } + const reader = new Reader(bytes.subarray(magic.length)); + const cipher = reader.string(); + const kdf = reader.string(); + reader.bytes(); + if (cipher !== "none" || kdf !== "none") { + throw new ProtocolError("rfetch cannot read a key with a passphrase"); + } + if (reader.uint32() !== 1) { + throw new ProtocolError("The file must have one key"); + } + reader.bytes(); + const keys = new Reader(reader.bytes()); + if (keys.uint32() !== keys.uint32()) { + throw new ProtocolError("The key is not correct"); + } + const type = keys.string(); + if (type === "ssh-ed25519") { + const pub = keys.bytes(); + const secret = keys.bytes(); + if (pub.length !== 32 || secret.length !== 64) { + throw new ProtocolError("The Ed25519 key is not correct"); + } + return await ed25519Key({ + kty: "OKP", + crv: "Ed25519", + d: base64url(secret.subarray(0, 32)), + x: base64url(pub), + }); + } + if (type.startsWith("ecdsa-sha2-")) { + const curve = CURVES[keys.string()]; + if (!curve) throw new ProtocolError(`The curve of ${type} is not known`); + const point = keys.bytes(); + const d = keys.mpint(); + return await ecdsaKey(curve, pointJWK(curve, point, d)); + } + if (type === "ssh-rsa") { + const n = keys.mpint(); + const e = keys.mpint(); + const d = keys.mpint(); + const qi = keys.mpint(); + const p = keys.mpint(); + const q = keys.mpint(); + return await rsaKey({ n, e, d, p, q, qi }); + } + throw new ProtocolError(`The key type ${type} is not known`); +} + +/** + * The JWK of a private key of ECDSA, from its point and its number. + * @param {typeof CURVES.nistp256} curve + * @param {Uint8Array} point The public point, not compressed. + * @param {bigint} d + */ +function pointJWK(curve, point, d) { + if (point.length !== 1 + 2 * curve.size || point[0] !== 4) { + throw new ProtocolError("The point of the key is not correct"); + } + return { + kty: "EC", + crv: curve.name, + x: base64url(point.subarray(1, 1 + curve.size)), + y: base64url(point.subarray(1 + curve.size)), + d: base64url(fixed(mpintBytes(d), curve.size)), + }; +} + +/** + * Reads a key of PKCS #8, of RSA, ECDSA or Ed25519. + * @param {Uint8Array} bytes + */ +async function readPKCS8(bytes) { + const [version, algorithm, key] = readDER(readOne(bytes, TAG.SEQUENCE)); + if ( + version?.tag !== TAG.INTEGER || algorithm?.tag !== TAG.SEQUENCE || + key?.tag !== TAG.OCTET_STRING + ) { + throw new ProtocolError("The PKCS #8 key is not correct"); + } + const [id, parameter] = readDER(algorithm.value); + if (isOID(id.value, "1.3.101.112")) { + const seed = readOne(key.value, TAG.OCTET_STRING); + const imported = await subtle.importKey("pkcs8", bytes, "Ed25519", true, [ + "sign", + ]); + const jwk = await subtle.exportKey("jwk", imported); + if (seed.length !== 32 || !jwk.x) { + throw new ProtocolError("The Ed25519 key is not correct"); + } + return await ed25519Key(jwk); + } + if (isOID(id.value, "1.2.840.10045.2.1")) { + const curve = Object.values(CURVES).find((curve) => + isOID(parameter?.value ?? new Uint8Array(0), curve.oid) + ); + if (!curve) throw new ProtocolError("The curve of the key is not known"); + const imported = await subtle.importKey( + "pkcs8", + bytes, + { name: "ECDSA", namedCurve: curve.name }, + true, + ["sign"], + ); + return await ecdsaKey(curve, await subtle.exportKey("jwk", imported)); + } + if (isOID(id.value, "1.2.840.113549.1.1.1")) { + // The OCTET STRING of the key has the key of PKCS #1. + return await rsaKey(readPKCS1(key.value)); + } + throw new ProtocolError("The algorithm of the key is not known"); +} + +/** + * Reads the numbers of a key of PKCS #1. + * @param {Uint8Array} bytes + */ +function readPKCS1(bytes) { + const numbers = readDER(readOne(bytes, TAG.SEQUENCE)).map((element) => { + if (element.tag !== TAG.INTEGER) { + throw new ProtocolError("The RSA key is not correct"); + } + let value = 0n; + for (const byte of element.value) value = (value << 8n) | BigInt(byte); + return value; + }); + if (numbers.length < 9) throw new ProtocolError("The RSA key is too short"); + const [, n, e, d, p, q, , , qi] = numbers; + return { n, e, d, p, q, qi }; +} + +/** + * Reads a key of SEC 1, of ECDSA. + * @param {Uint8Array} bytes + */ +async function readSEC1(bytes) { + const elements = readDER(readOne(bytes, TAG.SEQUENCE)); + const secret = elements[1]; + const parameters = elements.find((element) => element.tag === 0xa0); + const publicKey = elements.find((element) => element.tag === 0xa1); + const id = parameters ? readOne(parameters.value, TAG.OID) : null; + const curve = Object.values(CURVES).find((curve) => + id && isOID(id, curve.oid) + ); + if (!curve || secret?.tag !== TAG.OCTET_STRING) { + throw new ProtocolError("The EC key is not correct"); + } + let d = 0n; + for (const byte of secret.value) d = (d << 8n) | BigInt(byte); + if (publicKey) { + // The BIT STRING has a first byte with the number of bits that are not + // used. + const point = readOne(publicKey.value, TAG.BIT_STRING).subarray(1); + return await ecdsaKey(curve, pointJWK(curve, point, d)); + } + // With no public point, PKCS #8 gives the key to WebCrypto, which finds + // the point. + const pkcs8 = writeDER( + TAG.SEQUENCE, + writeDER(TAG.INTEGER, new Uint8Array([0])), + writeDER(TAG.SEQUENCE, oid("1.2.840.10045.2.1"), oid(curve.oid)), + writeDER( + TAG.OCTET_STRING, + writeDER( + TAG.SEQUENCE, + writeDER(TAG.INTEGER, new Uint8Array([1])), + writeDER(TAG.OCTET_STRING, secret.value), + ), + ), + ); + return await readPKCS8(pkcs8); +} + +/** + * A private key of Ed25519. + * @param {JsonWebKey} jwk + * @returns {Promise} + */ +async function ed25519Key(jwk) { + const key = await subtle.importKey("jwk", jwk, "Ed25519", false, ["sign"]); + const type = "ssh-ed25519"; + return { + type, + blob: new Writer().string(type).string(fromBase64url(jwk.x ?? "")).bytes(), + algorithms: [type], + sign: async (_algorithm, data) => + new Writer() + .string(type) + .string(new Uint8Array(await subtle.sign("Ed25519", key, data))) + .bytes(), + }; +} + +/** + * A private key of ECDSA. + * @param {typeof CURVES.nistp256} curve + * @param {JsonWebKey} jwk + * @returns {Promise} + */ +async function ecdsaKey(curve, jwk) { + const key = await subtle.importKey( + "jwk", + { kty: "EC", crv: jwk.crv, x: jwk.x, y: jwk.y, d: jwk.d }, + { name: "ECDSA", namedCurve: curve.name }, + false, + ["sign"], + ); + const name = Object.keys(CURVES).find((name) => CURVES[name] === curve); + const type = `ecdsa-sha2-${name}`; + const point = concat( + new Uint8Array([4]), + fromBase64url(jwk.x ?? ""), + fromBase64url(jwk.y ?? ""), + ); + return { + type, + blob: new Writer().string(type).string(name ?? "").string(point).bytes(), + algorithms: [type], + sign: async (_algorithm, data) => { + const raw = new Uint8Array( + await subtle.sign({ name: "ECDSA", hash: curve.hash }, key, data), + ); + const numbers = new Writer() + .mpint(raw.subarray(0, curve.size)) + .mpint(raw.subarray(curve.size)) + .bytes(); + return new Writer().string(type).string(numbers).bytes(); + }, + }; +} + +/** + * A private key of RSA. + * @param {{ n: bigint, e: bigint, d: bigint, p: bigint, q: bigint, qi: bigint }} numbers + * @returns {Promise} + */ +async function rsaKey({ n, e, d, p, q, qi }) { + const bytes = (value) => + mpintBytes(value).subarray( + mpintBytes(value)[0] === 0 ? 1 : 0, + ); + const bits = bitLength(bytes(n)); + if (bits < MIN_RSA_BITS) { + throw new ProtocolError(`An RSA key of ${bits} bits is too small`); + } + const jwk = { + kty: "RSA", + n: base64url(bytes(n)), + e: base64url(bytes(e)), + d: base64url(bytes(d)), + p: base64url(bytes(p)), + q: base64url(bytes(q)), + dp: base64url(bytes(d % (p - 1n))), + dq: base64url(bytes(d % (q - 1n))), + qi: base64url(bytes(qi)), + }; + /** @type {Map} */ + const keys = new Map(); + const type = "ssh-rsa"; + return { + type, + blob: new Writer().string(type).mpint(e).mpint(n).bytes(), + algorithms: ["rsa-sha2-512", "rsa-sha2-256", "ssh-rsa"], + sign: async (algorithm, data) => { + const hash = RSA_HASHES[algorithm]; + if (!hash) throw new ProtocolError(`${algorithm} is not an RSA hash`); + let key = keys.get(hash); + if (!key) { + key = await subtle.importKey( + "jwk", + jwk, + { name: "RSASSA-PKCS1-v1_5", hash }, + false, + ["sign"], + ); + keys.set(hash, key); + } + const signature = new Uint8Array( + await subtle.sign("RSASSA-PKCS1-v1_5", key, data), + ); + return new Writer().string(algorithm).string(signature).bytes(); + }, + }; +} + +/** + * Reads a line of `authorized_keys` or of the option `host_keys`: the type + * of the key and the key in base64. + * @param {string} line For example `ssh-ed25519 AAAAC3Nza...`. + * @returns {{ type: string, blob: Uint8Array }} + * @throws {ProtocolError} For a line that is not correct. + */ +export function readKeyLine(line) { + const [type, text] = line.trim().split(/\s+/); + if (!type || !text) throw new ProtocolError("The key line is not correct"); + const blob = fromBase64(text); + if (new Reader(blob).string() !== type) { + throw new ProtocolError("The type of the key is not its type in base64"); + } + return { type, blob }; +} diff --git a/backend/sftp/ssh/socket.js b/backend/sftp/ssh/socket.js new file mode 100644 index 0000000..7fdedb2 --- /dev/null +++ b/backend/sftp/ssh/socket.js @@ -0,0 +1,169 @@ +/** + * A TCP connection with `node:net`, with reads of a number of bytes. + */ + +import { connect } from "node:net"; + +/** An error of the connection, for example a timeout. */ +export class ConnectionError extends Error {} + +/** A TCP connection that reads a number of bytes at a time. */ +export class Connection { + /** @type {import("node:net").Socket} */ + #socket; + /** @type {Uint8Array[]} */ + #chunks = []; + #length = 0; + /** @type {Error | null} */ + #error = null; + #ended = false; + /** @type {(() => void) | null} */ + #wake = null; + + /** @param {import("node:net").Socket} socket */ + constructor(socket) { + this.#socket = socket; + socket.on("data", (chunk) => { + this.#chunks.push( + new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.length), + ); + this.#length += chunk.length; + // Too much data that nobody reads stops the reads of the socket. + if (this.#length > 1 << 20) socket.pause(); + this.#notify(); + }); + socket.on("end", () => { + this.#ended = true; + this.#notify(); + }); + socket.on("error", (error) => { + this.#error ??= new ConnectionError(error.message); + this.#notify(); + }); + socket.on("close", () => { + this.#ended = true; + this.#notify(); + }); + } + + #notify() { + const wake = this.#wake; + this.#wake = null; + wake?.(); + } + + /** + * Opens a TCP connection. + * @param {string} host + * @param {number} port + * @param {number} timeout The time to connect, in milliseconds. + * @returns {Promise} + * @throws {ConnectionError} + */ + static open(host, port, timeout) { + return new Promise((resolve, reject) => { + const socket = connect({ host, port, noDelay: true }); + const timer = setTimeout(() => { + socket.destroy(); + reject( + new ConnectionError(`The connection to ${host}:${port} timed out`), + ); + }, timeout); + socket.once("connect", () => { + clearTimeout(timer); + socket.removeAllListeners("error"); + resolve(new Connection(socket)); + }); + socket.once("error", (error) => { + clearTimeout(timer); + reject(new ConnectionError(error.message)); + }); + }); + } + + /** + * Reads a number of bytes. + * @param {number} size + * @returns {Promise} + * @throws {ConnectionError} When the connection closes before. + */ + async read(size) { + while (this.#length < size) { + if (this.#error) throw this.#error; + if (this.#ended) { + throw new ConnectionError("The server closed the connection"); + } + this.#socket.resume(); + await new Promise((resolve) => this.#wake = resolve); + } + const bytes = new Uint8Array(size); + let offset = 0; + while (offset < size) { + const chunk = this.#chunks[0]; + const count = Math.min(chunk.length, size - offset); + bytes.set(chunk.subarray(0, count), offset); + offset += count; + if (count === chunk.length) this.#chunks.shift(); + else this.#chunks[0] = chunk.subarray(count); + } + this.#length -= size; + if (this.#length < 1 << 19) this.#socket.resume(); + return bytes; + } + + /** + * Reads a line that ends with LF, with no CR and no LF. + * @param {number} max The largest number of bytes of the line. + * @returns {Promise} + */ + async readLine(max) { + /** @type {number[]} */ + const line = []; + while (true) { + const [byte] = await this.read(1); + if (byte === 0x0a) break; + line.push(byte); + if (line.length > max) throw new ConnectionError("A line is too long"); + } + if (line.at(-1) === 0x0d) line.pop(); + return Uint8Array.from(line); + } + + /** + * Writes bytes. It waits only when the socket has too many bytes to send. + * @param {Uint8Array} bytes + * @returns {Promise} + */ + async write(bytes) { + if (this.#error) throw this.#error; + if (this.#ended) throw new ConnectionError("The connection is closed"); + if (this.#socket.write(bytes)) return; + await new Promise((resolve) => { + const done = () => { + this.#socket.off("drain", done); + this.#socket.off("close", done); + resolve(undefined); + }; + this.#socket.on("drain", done); + this.#socket.on("close", done); + }); + if (this.#error) throw this.#error; + } + + /** + * Tells if the connection keeps the process of Node.js running. An idle + * connection of a pool does not. + * @param {boolean} active + */ + setActive(active) { + if (active) this.#socket.ref?.(); + else this.#socket.unref?.(); + } + + /** Closes the connection. */ + close() { + this.#ended = true; + this.#socket.destroy(); + this.#notify(); + } +} diff --git a/backend/sftp/ssh/ssh_test.js b/backend/sftp/ssh/ssh_test.js new file mode 100644 index 0000000..98c7b2d --- /dev/null +++ b/backend/sftp/ssh/ssh_test.js @@ -0,0 +1,216 @@ +/** + * The tests of the parts of the client of SSH that need no server: the data + * types, the ciphers, the keys and the file `known_hosts`. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fc from "fast-check"; + +import { CIPHERS, MACS, makeCipher } from "./cipher.js"; +import { findHostKey } from "../knownhosts.js"; +import { + fingerprint, + readKeyLine, + readPrivateKey, + readPublicKey, + toBase64, +} from "./keys.js"; +import { mpintBytes, Reader, Writer } from "./wire.js"; +import { KEY_PARAMS, makeKeys } from "../testing.js"; + +test("Writer and Reader keep each type of SSH", () => { + fc.assert( + fc.property( + fc.integer({ min: 0, max: 255 }), + fc.boolean(), + fc.integer({ min: 0, max: 0xffffffff }), + fc.bigInt({ min: 0n, max: 2n ** 64n - 1n }), + fc.string(), + fc.bigInt({ min: 0n, max: 2n ** 600n }), + fc.array(fc.stringMatching(/^[a-z0-9@.-]+$/)), + (byte, boolean, uint32, uint64, string, mpint, names) => { + const reader = new Reader( + new Writer() + .byte(byte) + .boolean(boolean) + .uint32(uint32) + .uint64(uint64) + .string(string) + .mpint(mpint) + .nameList(names) + .bytes(), + ); + assert.equal(reader.byte(), byte); + assert.equal(reader.boolean(), boolean); + assert.equal(reader.uint32(), uint32); + assert.equal(reader.uint64(), uint64); + assert.equal(reader.string(), string); + assert.equal(reader.mpint(), mpint); + assert.deepEqual(reader.nameList(), names.length ? names : []); + assert.equal(reader.remaining, 0); + }, + ), + ); +}); + +test("mpint is as the examples of RFC 4251", () => { + /** @param {bigint} value */ + const hex = (value) => Buffer.from(mpintBytes(value)).toString("hex"); + assert.equal(hex(0n), ""); + assert.equal(hex(0x9a378f9b2e332a7n), "09a378f9b2e332a7"); + assert.equal(hex(0x80n), "0080"); +}); + +test("each cipher opens the packets that it seals", async () => { + for (const name of Object.keys(CIPHERS)) { + for (const mac of CIPHERS[name].gcm ? [null] : Object.keys(MACS)) { + const keys = { + key: crypto.getRandomValues(new Uint8Array(CIPHERS[name].key)), + iv: crypto.getRandomValues(new Uint8Array(CIPHERS[name].iv)), + mac: crypto.getRandomValues(new Uint8Array(64)), + }; + const out = await makeCipher(name, mac, keys); + const input = await makeCipher(name, mac, keys); + /** @type {Uint8Array[]} */ + const wire = []; + /** @type {Uint8Array[]} */ + const packets = []; + for (let sequence = 0; sequence < 5; sequence++) { + const size = 16 * (1 + sequence * 7) - (out.lengthInClear ? 0 : 4); + const packet = crypto.getRandomValues(new Uint8Array(size)); + packet[0] = 4; + packets.push(packet); + wire.push(await out.seal(sequence, packet)); + } + // The bytes come in one stream. + const stream = Buffer.concat(wire); + let offset = 0; + const source = { + /** @param {number} size */ + read: async (size) => { + const bytes = stream.subarray(offset, offset + size); + offset += size; + return new Uint8Array(bytes); + }, + }; + for (let sequence = 0; sequence < 5; sequence++) { + assert.deepEqual( + await input.open(sequence, source), + packets[sequence], + `${name} ${mac}`, + ); + } + // A changed byte fails the MAC. + const broken = new Uint8Array(wire[0]); + broken[broken.length - 1] ^= 1; + const other = await makeCipher(name, mac, keys); + await assert.rejects( + other.open(0, { read: async (size) => broken.subarray(0, size) }), + /MAC|length/, + ); + } + } +}); + +test("readPrivateKey reads PKCS #8 and OpenSSH, and signs", async () => { + for (const [name, params] of Object.entries(KEY_PARAMS)) { + const { key, openssh } = await makeKeys(params); + const other = await readPrivateKey(openssh); + assert.equal(other.type, key.type, name); + assert.deepEqual(other.blob, key.blob, name); + const publicKey = await readPublicKey(key.blob); + const data = crypto.getRandomValues(new Uint8Array(100)); + for (const algorithm of key.algorithms.slice(0, 2)) { + for (const signer of [key, other]) { + const signature = await signer.sign(algorithm, data); + assert.ok( + await publicKey.verify(algorithm, signature, data), + `${name} ${algorithm}`, + ); + assert.ok( + !await publicKey.verify(algorithm, signature, data.subarray(1)), + `${name} ${algorithm} with other data`, + ); + } + } + } +}); + +test("readPrivateKey refuses a key with a passphrase", async () => { + await assert.rejects( + readPrivateKey( + "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-128-CBC,00\n\nAAAA\n-----END RSA PRIVATE KEY-----", + ), + /passphrase/, + ); + const encrypted = new Writer() + .raw(new TextEncoder().encode("openssh-key-v1\0")) + .string("aes256-ctr") + .string("bcrypt") + .string("") + .bytes(); + await assert.rejects( + readPrivateKey( + `-----BEGIN OPENSSH PRIVATE KEY-----\n${ + toBase64(encrypted) + }\n-----END OPENSSH PRIVATE KEY-----`, + ), + /passphrase/, + ); +}); + +test("fingerprint is the SHA256 fingerprint of OpenSSH", async () => { + // The key of RFC 8709 is not here: the fingerprint of OpenSSH is the + // SHA-256 of the key in base64 with no padding. + const blob = new Writer().string("ssh-ed25519").string(new Uint8Array(32)) + .bytes(); + const digest = Buffer.from(await crypto.subtle.digest("SHA-256", blob)); + assert.equal( + await fingerprint(blob), + `SHA256:${digest.toString("base64").replace(/=+$/, "")}`, + ); + assert.deepEqual(readKeyLine(`ssh-ed25519 ${toBase64(blob)} comment`), { + type: "ssh-ed25519", + blob, + }); + assert.throws(() => readKeyLine(`ssh-rsa ${toBase64(blob)}`), /type/); +}); + +test("findHostKey reads the lines of known_hosts", async () => { + const blob = new Writer().string("ssh-ed25519").string(new Uint8Array(32)) + .bytes(); + const other = new Writer().string("ssh-ed25519").string( + new Uint8Array(32).fill(1), + ).bytes(); + const line = `ssh-ed25519 ${toBase64(blob)}`; + const otherLine = `ssh-ed25519 ${toBase64(other)}`; + for ( + const [text, host, port, expected] of [ + [`example.com ${line}`, "example.com", 22, "known"], + [`EXAMPLE.com ${line}`, "example.com", 22, "known"], + [`a.com,example.com ${line}`, "example.com", 22, "known"], + [`*.com ${line}`, "example.com", 22, "known"], + [`*.com,!example.com ${line}`, "example.com", 22, "unknown"], + [`exampl?.com ${line}`, "example.com", 22, "known"], + [`example.com ${line}`, "example.com", 2222, "unknown"], + [`[example.com]:2222 ${line}`, "example.com", 2222, "known"], + [`example.com ${otherLine}`, "example.com", 22, "changed"], + [ + `example.com ${otherLine}\nexample.com ${line}`, + "example.com", + 22, + "known", + ], + [`@revoked * ${line}\nexample.com ${line}`, "example.com", 22, "revoked"], + [`@cert-authority *.com ${line}`, "example.com", 22, "unknown"], + [`# example.com ${line}\n\n`, "example.com", 22, "unknown"], + [`example.com ssh-ed25519 !!!`, "example.com", 22, "unknown"], + ] + ) { + assert.equal( + await findHostKey(text, host, port, blob), + expected, + `${text} for ${host}:${port}`, + ); + } +}); diff --git a/backend/sftp/ssh/transport.js b/backend/sftp/ssh/transport.js new file mode 100644 index 0000000..ccde680 --- /dev/null +++ b/backend/sftp/ssh/transport.js @@ -0,0 +1,568 @@ +/** + * The transport layer of SSH, as RFC 4253: the exchange of the versions, + * the exchange of the keys, and the packets with their encryption. + * + * The key exchanges: `curve25519-sha256` (RFC 8731) and + * `ecdh-sha2-nistp256`, `ecdh-sha2-nistp384` and `ecdh-sha2-nistp521` + * (RFC 5656), with the strict key exchange of OpenSSH against the Terrapin + * attack. The server can exchange new keys at any time. + */ + +import { CIPHERS, MACS, makeCipher, noCipher } from "./cipher.js"; +import { fingerprint, readPublicKey } from "./keys.js"; +import { concat, ProtocolError, Reader, Writer } from "./wire.js"; + +const { subtle } = globalThis.crypto; + +/** The numbers of the messages of SSH. */ +export const MSG = { + DISCONNECT: 1, + IGNORE: 2, + UNIMPLEMENTED: 3, + DEBUG: 4, + SERVICE_REQUEST: 5, + SERVICE_ACCEPT: 6, + EXT_INFO: 7, + KEXINIT: 20, + NEWKEYS: 21, + KEX_ECDH_INIT: 30, + KEX_ECDH_REPLY: 31, +}; + +/** The version of the client. */ +const VERSION = "SSH-2.0-rfetch"; + +/** The key exchanges, by preference, with their hashes and curves. */ +export const KEX = { + "curve25519-sha256": { hash: "SHA-256", curve: "X25519", bits: 256 }, + "curve25519-sha256@libssh.org": { + hash: "SHA-256", + curve: "X25519", + bits: 256, + }, + "ecdh-sha2-nistp256": { hash: "SHA-256", curve: "P-256", bits: 256 }, + "ecdh-sha2-nistp384": { hash: "SHA-384", curve: "P-384", bits: 384 }, + "ecdh-sha2-nistp521": { hash: "SHA-512", curve: "P-521", bits: 528 }, +}; + +/** The algorithms of the host keys, by preference. */ +export const HOST_KEYS = [ + "ssh-ed25519", + "ecdsa-sha2-nistp256", + "ecdsa-sha2-nistp384", + "ecdsa-sha2-nistp521", + "rsa-sha2-512", + "rsa-sha2-256", +]; + +/** The type of the key of each algorithm of a host key. */ +const KEY_TYPES = { + "rsa-sha2-512": "ssh-rsa", + "rsa-sha2-256": "ssh-rsa", +}; + +/** + * The host key of a server, for the check of the client. + * @typedef {Object} HostKey + * @property {string} type For example `ssh-ed25519`. + * @property {Uint8Array} blob The key in the format of SSH. + * @property {string} fingerprint For example `SHA256:...`. + */ + +/** + * The options of a transport. + * @typedef {Object} TransportOptions + * @property {(key: HostKey) => boolean | Promise} checkHostKey + * Tells if the client accepts the host key of the server. + * @property {string[]} [hostKeyTypes] The types of the keys that the client + * knows for the server: the client asks for them first. + * @property {string[]} [ciphers] The ciphers, by preference. + * @property {string[]} [macs] The MACs, by preference. + * @property {string[]} [kex] The key exchanges, by preference. + * @property {string[]} [hostKeyAlgorithms] The algorithms of the host keys, + * by preference. + */ + +/** An error of the server, with its message `SSH_MSG_DISCONNECT`. */ +export class DisconnectError extends Error { + /** @param {number} code @param {string} message */ + constructor(code, message) { + super(message || `The server closed the connection (code ${code})`); + this.code = code; + } +} + +/** The transport layer of a client of SSH. */ +export class Transport { + /** @type {import("./socket.js").Connection} */ + #socket; + /** @type {TransportOptions} */ + #options; + #clientVersion = VERSION; + #serverVersion = ""; + /** @type {Uint8Array | null} */ + sessionId = null; + /** The algorithms of the signatures of the server, from `EXT_INFO`. */ + /** @type {string[] | null} */ + serverSigAlgs = null; + + #out = { cipher: noCipher(), sequence: 0 }; + #in = { cipher: noCipher(), sequence: 0 }; + #strict = false; + #firstKex = true; + /** The writes of the packets, in the order of their sequence numbers. */ + /** @type {Promise} */ + #writes = Promise.resolve(); + /** A key exchange that runs, or null. */ + /** @type {Promise | null} */ + #kex = null; + /** @type {(() => void) | null} */ + #kexDone = null; + /** Our KEXINIT of the key exchange that runs. */ + /** @type {Uint8Array | null} */ + #ourKexInit = null; + /** True when the next packet of the server is a wrong guess to ignore. */ + #skipGuess = false; + + /** + * @param {import("./socket.js").Connection} socket + * @param {TransportOptions} options + */ + constructor(socket, options) { + this.#socket = socket; + this.#options = options; + } + + /** + * Exchanges the versions and the first keys. + * @returns {Promise} + */ + async start() { + await this.#socket.write( + new TextEncoder().encode(`${this.#clientVersion}\r\n`), + ); + // A server can send other lines before its version. + for (let lines = 0;; lines++) { + if (lines > 50) throw new ProtocolError("The server sent no version"); + const line = new TextDecoder().decode(await this.#socket.readLine(255)); + if (!line.startsWith("SSH-")) continue; + if (!/^SSH-(2\.0|1\.99)-/.test(line)) { + throw new ProtocolError(`The server version ${line} is not SSH 2`); + } + this.#serverVersion = line; + break; + } + this.#beginKex(); + const payload = await this.#readPacket(); + if (payload[0] !== MSG.KEXINIT) { + throw new ProtocolError("The server did not start the key exchange"); + } + await this.#exchange(payload); + } + + /** Starts a key exchange: the writes of other messages wait. */ + #beginKex() { + if (this.#kex) return; + this.#kex = new Promise((resolve) => this.#kexDone = resolve); + this.#ourKexInit = this.#kexInit(); + this.#post(this.#ourKexInit); + } + + /** + * Writes a packet of the key exchange, with no wait. An error of the + * write comes again from the next read. + * @param {Uint8Array} payload + * @param {() => void} [after] + */ + #post(payload, after) { + this.#writePacket(payload, after).catch(() => {}); + } + + /** Our message KEXINIT. */ + #kexInit() { + const { hostKeyTypes = [] } = this.#options; + // The algorithms of the known keys come first, as OpenSSH does. + const algorithms = this.#options.hostKeyAlgorithms ?? HOST_KEYS; + const known = algorithms.filter((name) => + hostKeyTypes.includes(KEY_TYPES[name] ?? name) + ); + const hostKeys = [ + ...known, + ...algorithms.filter((name) => !known.includes(name)), + ]; + const kex = [...(this.#options.kex ?? Object.keys(KEX))]; + if (this.#firstKex) kex.push("ext-info-c", "kex-strict-c-v00@openssh.com"); + const ciphers = this.#options.ciphers ?? Object.keys(CIPHERS); + const macs = this.#options.macs ?? Object.keys(MACS); + return new Writer() + .byte(MSG.KEXINIT) + .raw(crypto.getRandomValues(new Uint8Array(16))) + .nameList(kex) + .nameList(hostKeys) + .nameList(ciphers) + .nameList(ciphers) + .nameList(macs) + .nameList(macs) + .nameList(["none"]) + .nameList(["none"]) + .nameList([]) + .nameList([]) + .boolean(false) + .uint32(0) + .bytes(); + } + + /** + * Runs a key exchange, from the KEXINIT of the server. + * @param {Uint8Array} serverKexInit + */ + async #exchange(serverKexInit) { + this.#beginKex(); + const ours = new Reader(/** @type {Uint8Array} */ (this.#ourKexInit)); + const theirs = new Reader(serverKexInit); + ours.raw(17); + theirs.raw(17); + /** @type {string[][]} */ + const client = []; + /** @type {string[][]} */ + const server = []; + for (let i = 0; i < 10; i++) { + client.push(ours.nameList()); + server.push(theirs.nameList()); + } + const guess = theirs.boolean(); + if (this.#firstKex) { + this.#strict = server[0].includes("kex-strict-s-v00@openssh.com"); + } + /** @param {number} i @param {string} what */ + const choose = (i, what) => { + const found = client[i].find((name) => + server[i].includes(name) && !name.startsWith("ext-info-") && + !name.startsWith("kex-strict-") + ); + if (!found) { + throw new ProtocolError( + `No ${what} in common with the server: ${server[i].join(",")}`, + ); + } + return found; + }; + const kexName = choose(0, "key exchange"); + const hostKeyName = choose(1, "host key algorithm"); + const cipherOut = choose(2, "cipher"); + const cipherIn = choose(3, "cipher"); + const macOut = CIPHERS[cipherOut].gcm ? null : choose(4, "MAC"); + const macIn = CIPHERS[cipherIn].gcm ? null : choose(5, "MAC"); + choose(6, "compression"); + choose(7, "compression"); + // A wrong guess of the server: its next packet is for another exchange. + if (guess && (server[0][0] !== kexName || server[1][0] !== hostKeyName)) { + this.#skipGuess = true; + } + + const kex = KEX[kexName]; + const params = kex.curve === "X25519" + ? { name: "X25519" } + : { name: "ECDH", namedCurve: kex.curve }; + const pair = /** @type {CryptoKeyPair} */ ( + await subtle.generateKey(params, true, ["deriveBits"]) + ); + const clientPublic = new Uint8Array( + await subtle.exportKey("raw", pair.publicKey), + ); + this.#post( + new Writer().byte(MSG.KEX_ECDH_INIT).string(clientPublic).bytes(), + ); + + const reply = new Reader(await this.#readKexPacket(MSG.KEX_ECDH_REPLY)); + reply.byte(); + const hostKeyBlob = reply.bytes(); + const serverPublic = reply.bytes(); + const signature = reply.bytes(); + + let peer; + try { + peer = await subtle.importKey("raw", serverPublic, params, false, []); + } catch { + throw new ProtocolError("The public key of the server is not correct"); + } + const secret = new Uint8Array( + await subtle.deriveBits( + { ...params, public: peer }, + pair.privateKey, + kex.bits, + ), + ); + if (secret.every((byte) => byte === 0)) { + throw new ProtocolError("The shared secret is zero"); + } + const k = new Writer().mpint(secret).bytes(); + const exchange = new Writer() + .string(this.#clientVersion) + .string(this.#serverVersion) + .string(/** @type {Uint8Array} */ (this.#ourKexInit)) + .string(serverKexInit) + .string(hostKeyBlob) + .string(clientPublic) + .string(serverPublic) + .raw(k) + .bytes(); + const h = new Uint8Array(await subtle.digest(kex.hash, exchange)); + + // The host key: its type, the choice of the client, and its signature. + const hostKey = await readPublicKey(hostKeyBlob); + if (hostKey.type !== (KEY_TYPES[hostKeyName] ?? hostKeyName)) { + throw new ProtocolError( + `The host key ${hostKey.type} is not of the algorithm ${hostKeyName}`, + ); + } + if (!await hostKey.verify(hostKeyName, signature, h)) { + throw new ProtocolError("The signature of the host key is not correct"); + } + const accepted = await this.#options.checkHostKey({ + type: hostKey.type, + blob: hostKeyBlob, + fingerprint: await fingerprint(hostKeyBlob), + }); + if (!accepted) { + throw new ProtocolError( + `The host key ${await fingerprint(hostKeyBlob)} is not known`, + ); + } + + this.sessionId ??= h; + /** + * The key of a letter, as RFC 4253 section 7.2. + * @param {string} letter + * @param {number} size + */ + const derive = async (letter, size) => { + let key = new Uint8Array( + await subtle.digest( + kex.hash, + concat( + k, + h, + new TextEncoder().encode(letter), + /** @type {Uint8Array} */ (this.sessionId), + ), + ), + ); + while (key.length < size) { + key = concat( + key, + new Uint8Array(await subtle.digest(kex.hash, concat(k, h, key))), + ); + } + return key.subarray(0, size); + }; + /** @param {string} cipher @param {string | null} mac @param {string[]} letters */ + const keysOf = async (cipher, mac, [iv, key, integrity]) => ({ + iv: await derive(iv, CIPHERS[cipher].iv), + key: await derive(key, CIPHERS[cipher].key), + mac: mac ? await derive(integrity, MACS[mac].key) : new Uint8Array(0), + }); + const out = await makeCipher( + cipherOut, + macOut, + await keysOf(cipherOut, macOut, ["A", "C", "E"]), + ); + const input = await makeCipher( + cipherIn, + macIn, + await keysOf(cipherIn, macIn, ["B", "D", "F"]), + ); + + // NEWKEYS, then the new keys for the next packets. + this.#post(new Uint8Array([MSG.NEWKEYS]), () => { + this.#out.cipher = out; + if (this.#strict) this.#out.sequence = 0; + }); + await this.#readKexPacket(MSG.NEWKEYS); + this.#in.cipher = input; + if (this.#strict) this.#in.sequence = 0; + + this.#firstKex = false; + this.#ourKexInit = null; + const done = this.#kexDone; + this.#kex = null; + this.#kexDone = null; + done?.(); + } + + /** + * Reads a packet of a key exchange. + * @param {number} type The number of the message. + */ + async #readKexPacket(type) { + while (true) { + const payload = await this.#readPacket(); + if (this.#skipGuess) { + this.#skipGuess = false; + continue; + } + if (payload[0] === type) return payload; + if (payload[0] === MSG.DISCONNECT) throw disconnectError(payload); + // The strict key exchange allows no other message. + if ( + !this.#strict && + (payload[0] === MSG.IGNORE || payload[0] === MSG.DEBUG) + ) continue; + throw new ProtocolError( + `The message ${payload[0]} came in the key exchange`, + ); + } + } + + /** + * Writes a packet, after the packets before it. + * @param {Uint8Array} payload + * @param {() => void} [after] Runs after the write of the packet, before + * the next packet. + * @returns {Promise} + */ + #writePacket(payload, after) { + const write = this.#writes.then(async () => { + const { cipher } = this.#out; + const block = Math.max(8, cipher.blockSize); + const counted = cipher.lengthInClear ? 1 : 5; + let padding = block - ((counted + payload.length) % block); + if (padding < 4) padding += block; + const packet = new Uint8Array(1 + payload.length + padding); + packet[0] = padding; + packet.set(payload, 1); + crypto.getRandomValues(packet.subarray(1 + payload.length)); + const sequence = this.#out.sequence; + this.#out.sequence = (sequence + 1) >>> 0; + const bytes = await cipher.seal(sequence, packet); + after?.(); + await this.#socket.write(bytes); + }); + // A failed write fails the writes after it too. + this.#writes = write; + return write; + } + + /** + * Reads the payload of a packet. + * @returns {Promise} + */ + async #readPacket() { + const sequence = this.#in.sequence; + this.#in.sequence = (sequence + 1) >>> 0; + const packet = await this.#in.cipher.open(sequence, this.#socket); + const padding = packet[0]; + if (padding < 4 || padding + 1 > packet.length) { + throw new ProtocolError("The padding of a packet is not correct"); + } + if (packet.length - 1 - padding < 1) { + throw new ProtocolError("A packet has no message"); + } + return packet.subarray(1, packet.length - padding); + } + + /** + * Sends a message. It waits for the end of a key exchange. + * @param {Uint8Array} payload + * @returns {Promise} + */ + async send(payload) { + while (this.#kex) await this.#kex; + return await this.#writePacket(payload); + } + + /** + * Receives the next message of the layers over the transport: the + * transport answers the key exchanges and drops IGNORE and DEBUG. + * @returns {Promise} + * @throws {DisconnectError} When the server closes the connection. + */ + async receive() { + while (true) { + const payload = await this.#readPacket(); + switch (payload[0]) { + case MSG.DISCONNECT: + throw disconnectError(payload); + case MSG.IGNORE: + case MSG.DEBUG: + case MSG.UNIMPLEMENTED: + continue; + case MSG.EXT_INFO: + this.#readExtInfo(payload); + continue; + case MSG.KEXINIT: + await this.#exchange(payload); + continue; + default: + if (payload[0] >= 20 && payload[0] <= 49) { + throw new ProtocolError( + `The message ${payload[0]} came out of a key exchange`, + ); + } + return payload; + } + } + } + + /** @param {Uint8Array} payload */ + #readExtInfo(payload) { + const reader = new Reader(payload); + reader.byte(); + const count = reader.uint32(); + for (let i = 0; i < count; i++) { + const name = reader.string(); + const value = reader.bytes(); + if (name === "server-sig-algs") { + this.serverSigAlgs = new TextDecoder().decode(value).split(","); + } + } + } + + /** + * Tells if the connection keeps the process of Node.js running. + * @param {boolean} active + */ + setActive(active) { + this.#socket.setActive(active); + } + + /** + * Sends DISCONNECT, and closes the connection. + * @param {string} [reason] + */ + async close(reason = "") { + try { + await Promise.race([ + this.#writePacket( + new Writer() + .byte(MSG.DISCONNECT) + .uint32(11) + .string(reason) + .string("") + .bytes(), + ), + new Promise((resolve) => setTimeout(resolve, 1000)), + ]); + } catch { + // The connection is closed. + } + this.#socket.close(); + } +} + +/** + * The error of a message DISCONNECT. + * @param {Uint8Array} payload + */ +function disconnectError(payload) { + const reader = new Reader(payload); + reader.byte(); + const code = reader.uint32(); + let message = ""; + try { + message = reader.string(); + } catch { + // A message that is not UTF-8. + } + return new DisconnectError(code, message); +} diff --git a/backend/sftp/ssh/wire.js b/backend/sftp/ssh/wire.js new file mode 100644 index 0000000..4b0452a --- /dev/null +++ b/backend/sftp/ssh/wire.js @@ -0,0 +1,253 @@ +/** + * The data types of SSH, as RFC 4251 section 5: `byte`, `boolean`, + * `uint32`, `uint64`, `string`, `mpint` and `name-list`. + */ + +const encoder = new TextEncoder(); +const decoder = new TextDecoder("utf-8", { fatal: true }); + +/** An error in the data of the other side. */ +export class ProtocolError extends Error {} + +/** Writes the data of a message. */ +export class Writer { + /** @type {Uint8Array} */ + #bytes; + #length = 0; + + /** @param {number} [size] The number of bytes that the writer can get. */ + constructor(size = 256) { + this.#bytes = new Uint8Array(size); + } + + /** @param {number} size The number of bytes to add. */ + #grow(size) { + if (this.#length + size <= this.#bytes.length) return; + const bytes = new Uint8Array( + Math.max(this.#bytes.length * 2, this.#length + size), + ); + bytes.set(this.#bytes.subarray(0, this.#length)); + this.#bytes = bytes; + } + + /** @param {number} value */ + byte(value) { + this.#grow(1); + this.#bytes[this.#length++] = value; + return this; + } + + /** @param {boolean} value */ + boolean(value) { + return this.byte(value ? 1 : 0); + } + + /** @param {number} value */ + uint32(value) { + this.#grow(4); + new DataView(this.#bytes.buffer).setUint32(this.#length, value); + this.#length += 4; + return this; + } + + /** @param {bigint | number} value */ + uint64(value) { + this.#grow(8); + new DataView(this.#bytes.buffer).setBigUint64(this.#length, BigInt(value)); + this.#length += 8; + return this; + } + + /** + * Adds bytes with no length before them. + * @param {Uint8Array} bytes + */ + raw(bytes) { + this.#grow(bytes.length); + this.#bytes.set(bytes, this.#length); + this.#length += bytes.length; + return this; + } + + /** + * Adds a `string`: its length, then its bytes. A JavaScript string is + * UTF-8. + * @param {Uint8Array | string} value + */ + string(value) { + const bytes = typeof value === "string" ? encoder.encode(value) : value; + return this.uint32(bytes.length).raw(bytes); + } + + /** + * Adds an `mpint`: a number in two's complement, big-endian, with no + * byte that is not necessary. + * @param {bigint | Uint8Array} value A number, or the bytes of a number + * that is not negative, big-endian. + */ + mpint(value) { + return this.string(mpintBytes(value)); + } + + /** @param {string[]} names */ + nameList(names) { + return this.string(names.join(",")); + } + + /** The bytes that this writer has. */ + bytes() { + return this.#bytes.slice(0, this.#length); + } +} + +/** + * The bytes of an `mpint`, with no length. + * @param {bigint | Uint8Array} value + * @returns {Uint8Array} + */ +export function mpintBytes(value) { + let bytes; + if (typeof value === "bigint") { + if (value < 0n) throw new RangeError("A negative mpint is not supported"); + if (value === 0n) return new Uint8Array(0); + let hex = value.toString(16); + if (hex.length % 2) hex = `0${hex}`; + bytes = Uint8Array.from( + { length: hex.length / 2 }, + (_, i) => parseInt(hex.slice(i * 2, i * 2 + 2), 16), + ); + } else { + let start = 0; + while (start < value.length && value[start] === 0) start++; + bytes = value.subarray(start); + } + // A number with its high bit set gets a 0 byte before it: else it is + // negative. + if (bytes.length && bytes[0] & 0x80) { + const padded = new Uint8Array(bytes.length + 1); + padded.set(bytes, 1); + return padded; + } + return bytes; +} + +/** Reads the data of a message. */ +export class Reader { + #bytes; + #view; + #offset = 0; + + /** @param {Uint8Array} bytes */ + constructor(bytes) { + this.#bytes = bytes; + this.#view = new DataView(bytes.buffer, bytes.byteOffset, bytes.length); + } + + /** @param {number} size */ + #need(size) { + if (this.#offset + size > this.#bytes.length) { + throw new ProtocolError("The message is too short"); + } + } + + /** The number of bytes that are not read. */ + get remaining() { + return this.#bytes.length - this.#offset; + } + + byte() { + this.#need(1); + return this.#bytes[this.#offset++]; + } + + boolean() { + return this.byte() !== 0; + } + + uint32() { + this.#need(4); + const value = this.#view.getUint32(this.#offset); + this.#offset += 4; + return value; + } + + uint64() { + this.#need(8); + const value = this.#view.getBigUint64(this.#offset); + this.#offset += 8; + return value; + } + + /** + * Reads bytes with no length before them. + * @param {number} size + */ + raw(size) { + this.#need(size); + const bytes = this.#bytes.subarray(this.#offset, this.#offset + size); + this.#offset += size; + return bytes; + } + + /** Reads a `string` as bytes. */ + bytes() { + return this.raw(this.uint32()); + } + + /** Reads a `string` as UTF-8 text. */ + string() { + try { + return decoder.decode(this.bytes()); + } catch { + throw new ProtocolError("A string is not UTF-8"); + } + } + + /** Reads an `mpint` that is not negative. */ + mpint() { + const bytes = this.bytes(); + if (bytes.length && bytes[0] & 0x80) { + throw new ProtocolError("A negative mpint is not supported"); + } + let value = 0n; + for (const byte of bytes) value = (value << 8n) | BigInt(byte); + return value; + } + + nameList() { + const text = this.string(); + return text === "" ? [] : text.split(","); + } + + /** The bytes that are not read. */ + rest() { + return this.raw(this.remaining); + } +} + +/** + * Tells if two arrays of bytes are equal, in a time that does not depend on + * where they differ. + * @param {Uint8Array} a + * @param {Uint8Array} b + */ +export function equalBytes(a, b) { + if (a.length !== b.length) return false; + let difference = 0; + for (let i = 0; i < a.length; i++) difference |= a[i] ^ b[i]; + return difference === 0; +} + +/** + * Joins arrays of bytes. + * @param {Uint8Array[]} parts + */ +export function concat(...parts) { + const bytes = new Uint8Array(parts.reduce((sum, p) => sum + p.length, 0)); + let offset = 0; + for (const part of parts) { + bytes.set(part, offset); + offset += part.length; + } + return bytes; +} diff --git a/backend/sftp/testing.js b/backend/sftp/testing.js new file mode 100644 index 0000000..0b8033d --- /dev/null +++ b/backend/sftp/testing.js @@ -0,0 +1,205 @@ +/** + * A test server of SFTP: `rclone serve sftp` on a local folder, with a user + * and a password, the host keys of the tests and the authorized keys of + * the tests. The tests run when rclone is in the PATH, or when RCLONE is the + * path of rclone. + */ +import { spawn, spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createConnection, createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import process from "node:process"; + +import { readPrivateKey } from "./ssh/keys.js"; +import { Reader, Writer } from "./ssh/wire.js"; + +const RCLONE = process.env.RCLONE || "rclone"; + +/** Tells if rclone is installed. */ +export const hasRclone = spawnSync(RCLONE, ["version"]).status === 0; + +/** The user and the password of the test server. */ +export const LOGIN = { user: "rfetch-user", pass: "rfetch pass é" }; + +/** + * Gives a free port of 127.0.0.1. + * @returns {Promise} + */ +function freePort() { + return new Promise((resolve) => { + const server = createServer().listen(0, "127.0.0.1", () => { + const { port } = server.address(); + server.close(() => resolve(port)); + }); + }); +} + +/** + * Starts `rclone serve sftp` on a free port. + * @param {Object} [options] + * @param {string[]} [options.hostKeys] The private keys of the server, in + * PEM. + * @param {string[]} [options.authorizedKeys] The public keys of the users, + * as lines of `authorized_keys`. + * @returns {Promise<{ host: string, port: number, folder: string, stop: () => void }>} + * `folder` is the local folder of the server. + */ +export async function serveSFTP({ hostKeys = [], authorizedKeys = [] } = {}) { + const home = mkdtempSync(join(tmpdir(), "rfetch-sftp-")); + const folder = join(home, "data"); + mkdirSync(folder); + const port = await freePort(); + const args = [ + "serve", + "sftp", + "--user", + LOGIN.user, + "--pass", + LOGIN.pass, + "--addr", + `127.0.0.1:${port}`, + ]; + hostKeys.forEach((pem, index) => { + const file = join(home, `host_key_${index}`); + writeFileSync(file, pem, { mode: 0o600 }); + args.push("--key", file); + }); + const authorized = join(home, "authorized_keys"); + writeFileSync(authorized, authorizedKeys.map((line) => `${line}\n`).join("")); + args.push("--authorized-keys", authorized, folder); + const child = spawn(RCLONE, args, { + env: { PATH: process.env.PATH, HOME: home }, + stdio: "ignore", + }); + // Waits until the server takes connections. + for (let i = 0; i < 100; i++) { + const open = await new Promise((resolve) => { + const socket = createConnection({ host: "127.0.0.1", port }); + socket.once("connect", () => { + socket.destroy(); + resolve(true); + }); + socket.once("error", () => resolve(false)); + }); + if (open) break; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + return { + host: "127.0.0.1", + port, + folder, + stop() { + child.kill(); + rmSync(home, { recursive: true, force: true }); + }, + }; +} + +/** + * Makes a private key with openssl, in PEM. + * @param {"ed25519" | "ecdsa" | "rsa"} type + * @param {"pkcs8" | "traditional"} [format] `traditional` is PKCS #1 for + * RSA and SEC 1 for ECDSA. + * @returns {string} + */ +export function makeKey(type, format = "pkcs8") { + const args = { + ed25519: ["genpkey", "-algorithm", "ed25519"], + ecdsa: [ + "genpkey", + "-algorithm", + "EC", + "-pkeyopt", + "ec_paramgen_curve:P-256", + ], + rsa: ["genpkey", "-algorithm", "RSA", "-pkeyopt", "rsa_keygen_bits:2048"], + }[type]; + let pem = spawnSync("openssl", args, { encoding: "utf8" }).stdout; + if (format === "traditional") { + // `openssl ec` writes SEC 1, and `openssl rsa -traditional` PKCS #1. + pem = spawnSync( + "openssl", + type === "rsa" ? ["rsa", "-traditional"] : ["ec"], + { input: pem, encoding: "utf8" }, + ).stdout; + } + return pem; +} + +/** Tells if openssl is installed. */ +export const hasOpenSSL = spawnSync("openssl", ["version"]).status === 0; + +/** + * Writes a private key in the format of OpenSSH, with no passphrase. + * @param {JsonWebKey} jwk The private key. + * @param {Uint8Array} blob The public key in the format of SSH. + */ +export function openSSHKey(jwk, blob) { + /** @param {string | undefined} text */ + const bytes = (text) => new Uint8Array(Buffer.from(text ?? "", "base64url")); + const type = new Reader(blob).string(); + const secret = new Writer().uint32(7).uint32(7).string(type); + if (type === "ssh-ed25519") { + secret.string(bytes(jwk.x)); + secret.string(Buffer.concat([bytes(jwk.d), bytes(jwk.x)])); + } else if (type === "ssh-rsa") { + for (const field of ["n", "e", "d", "qi", "p", "q"]) { + secret.mpint(bytes(jwk[field])); + } + } else { + const reader = new Reader(blob); + reader.string(); + secret.string(reader.string()).string(reader.bytes()).mpint(bytes(jwk.d)); + } + secret.string("comment"); + const padded = secret.bytes(); + /** @type {number[]} */ + const padding = []; + for (let i = 1; (padded.length + padding.length) % 8 !== 0; i++) { + padding.push(i); + } + const body = new Writer() + .raw(new TextEncoder().encode("openssh-key-v1\0")) + .string("none") + .string("none") + .string("") + .uint32(1) + .string(blob) + .string(Buffer.concat([padded, Uint8Array.from(padding)])) + .bytes(); + const base64 = Buffer.from(body).toString("base64").replace(/.{70}/g, "$&\n"); + return `-----BEGIN OPENSSH PRIVATE KEY-----\n${base64}\n-----END OPENSSH PRIVATE KEY-----\n`; +} + +/** + * Makes a key with WebCrypto, as PKCS #8 in PEM and as OpenSSH. + * @param {RsaHashedKeyGenParams | EcKeyGenParams | Algorithm} params + */ +export async function makeKeys(params) { + const pair = /** @type {CryptoKeyPair} */ ( + await crypto.subtle.generateKey(params, true, ["sign", "verify"]) + ); + const pkcs8 = Buffer.from( + await crypto.subtle.exportKey("pkcs8", pair.privateKey), + ).toString("base64").replace(/.{64}/g, "$&\n"); + const pem = + `-----BEGIN PRIVATE KEY-----\n${pkcs8}\n-----END PRIVATE KEY-----\n`; + const key = await readPrivateKey(pem); + const jwk = await crypto.subtle.exportKey("jwk", pair.privateKey); + return { pem, key, openssh: openSSHKey(jwk, key.blob) }; +} + +/** The parameters of WebCrypto of the keys of the tests, by type. */ +export const KEY_PARAMS = { + ed25519: { name: "Ed25519" }, + "ecdsa p256": { name: "ECDSA", namedCurve: "P-256" }, + "ecdsa p384": { name: "ECDSA", namedCurve: "P-384" }, + "ecdsa p521": { name: "ECDSA", namedCurve: "P-521" }, + rsa: { + name: "RSASSA-PKCS1-v1_5", + modulusLength: 2048, + publicExponent: new Uint8Array([1, 0, 1]), + hash: "SHA-256", + }, +}; diff --git a/fetch.js b/fetch.js index 20c67d0..3076766 100644 --- a/fetch.js +++ b/fetch.js @@ -356,6 +356,14 @@ async function resolve(input, headers) { // variable, a header or the query cannot set it. delete optionsEnv.RFETCH_LOCAL_ROOT; if (root !== undefined) optionsEnv.RFETCH_LOCAL_ROOT = root; + // As rclone, the path of a remote of sftp is from the login folder of the + // user, and a path that starts with "/" is from the root of the server. + // The pathname is the same for the two: the root "/" tells the backend. + if (type !== "local") { + const key = `RFETCH_${type.toUpperCase()}_ROOT`; + delete optionsEnv[key]; + if (parsed.path.startsWith("/")) optionsEnv[key] = "/"; + } return { type, name, pathname, params, options: optionsEnv }; } diff --git a/package-lock.json b/package-lock.json index 957c635..d6ff81e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,6 +27,7 @@ "@rfetch/local": "^0.1.0", "@rfetch/memory": "^0.1.0", "@rfetch/s3": "^0.1.0", + "@rfetch/sftp": "^0.1.0", "@rfetch/tiles": "^0.1.0", "@rfetch/union": "^0.1.0", "@rfetch/webdav": "^0.1.0" @@ -200,6 +201,20 @@ "node": ">=22.2.0" } }, + "backend/sftp": { + "name": "@rfetch/sftp", + "version": "0.1.0", + "license": "ISC", + "dependencies": { + "@rfetch/lib": "^0.1.0" + }, + "devDependencies": { + "fast-check": "^3.19.0" + }, + "engines": { + "node": ">=22.2.0" + } + }, "backend/tiles": { "name": "@rfetch/tiles", "version": "0.1.0", @@ -307,6 +322,10 @@ "resolved": "backend/s3", "link": true }, + "node_modules/@rfetch/sftp": { + "resolved": "backend/sftp", + "link": true + }, "node_modules/@rfetch/tiles": { "resolved": "backend/tiles", "link": true diff --git a/package.json b/package.json index 8b2cdd9..14a3584 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "@rfetch/local": "^0.1.0", "@rfetch/memory": "^0.1.0", "@rfetch/s3": "^0.1.0", + "@rfetch/sftp": "^0.1.0", "@rfetch/tiles": "^0.1.0", "@rfetch/union": "^0.1.0", "@rfetch/webdav": "^0.1.0" From af817704192bbbd3fd864e6bf071d7edc1329b9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Mon, 19 Oct 2026 22:58:26 +0000 Subject: [PATCH 3/3] fix(sftp): refuse a server with no known host key With no host_keys and no known_hosts_file, the sftp backend refuses each server, as ssh refuses a host that it does not know. Before, it accepted each host key, as rclone: then a server in the middle could get the password. The answer 502 gives the fingerprint and the line of the host key of the server. Make sure that it is the key of the server, then set host_keys to this line. The check is before the password and before the signature with the key of the user. known_hosts_file=none is not a value with a special meaning now. The tests use all the host keys of the test server for each algorithm of host key, and check that the answer gives a key that then signs in. --- backend/sftp/main.js | 27 +++++++++++++++++---------- backend/sftp/main_test.js | 30 ++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/backend/sftp/main.js b/backend/sftp/main.js index 36c80b3..cfc9968 100644 --- a/backend/sftp/main.js +++ b/backend/sftp/main.js @@ -26,7 +26,7 @@ * `,` between them, as in a file `known_hosts`. The server must have one * of them. * - `known_hosts_file`: the location of a file `known_hosts` of OpenSSH. - * The file must have the host key of the server. `none` is no check. + * The file must have the host key of the server. * - `set_modtime`: `true` by default. Then a `PUT`, a `MOVE` and a `PATCH` * with `X-OC-Mtime` set the time of the file, to the second. * - `skip_links`: with `true`, a list has no symbolic link. Else a link is @@ -44,9 +44,13 @@ * - `encoding`: the names that change on the server, as rclone, by default * `Slash,Ctl,Del,Dot`. * - * Security: with no `host_keys` and no `known_hosts_file`, the backend - * accepts each host key of the server, as rclone does. Then a server in the - * middle can get the password. Set `host_keys` to the key of the server. + * Security: the backend checks the host key of the server before it sends + * the password or signs with the key of the user. With no `host_keys` and + * no `known_hosts_file`, it refuses each server, as ssh refuses a host that + * it does not know: the answer 502 gives the host key of the server. Make + * sure that this key is the key of the server, then set `host_keys` to it. + * rclone accepts each host key in this case: then a server in the middle + * can get the password. * * The algorithms: the key exchanges `curve25519-sha256` and * `ecdh-sha2-nistp256`, `-nistp384` and `-nistp521`, the host keys @@ -436,9 +440,7 @@ async function configOf(values, env) { } } const knownHostsFile = get("known_hosts_file"); - const knownHosts = knownHostsFile && knownHostsFile !== "none" - ? await readLocation(knownHostsFile) - : null; + const knownHosts = knownHostsFile ? await readLocation(knownHostsFile) : null; const algorithms = { ciphers: chooseAlgorithms(get("ciphers"), Object.keys(CIPHERS), "ciphers"), @@ -726,7 +728,8 @@ async function connect(config) { * @param {Config} config * @param {import("./ssh/transport.js").HostKey} key * @returns {Promise} - * @throws {SFTPBackendError} When the server has another key. + * @throws {SFTPBackendError} When the server has another key, or when the + * backend knows no key of the server. */ async function checkHostKey(config, key) { const line = `${key.type} ${toBase64(key.blob)}`; @@ -755,9 +758,13 @@ async function checkHostKey(config, key) { 502, ); } + return true; } - // With no host_keys and no known_hosts_file, each key, as rclone. - return true; + // No key is known: the server can be a server in the middle. + throw new SFTPBackendError( + `The host key ${key.fingerprint} of the server is not known. Make sure that it is the key of the server, then set host_keys to ${line}`, + 502, + ); } /** diff --git a/backend/sftp/main_test.js b/backend/sftp/main_test.js index ff31406..4349e40 100644 --- a/backend/sftp/main_test.js +++ b/backend/sftp/main_test.js @@ -54,8 +54,9 @@ if (hasRclone && hasOpenSSL) { const openSSHKeys = await Promise.all( Object.values(KEY_PARAMS).map((params) => makeKeys(params)), ); + const otherHostKeys = [makeKey("ecdsa"), makeKey("rsa")]; const server = await serveSFTP({ - hostKeys: [hostKey, makeKey("ecdsa"), makeKey("rsa")], + hostKeys: [hostKey, ...otherHostKeys], authorizedKeys: [ ...await Promise.all(Object.values(userKeys).map(keyLine)), ...await Promise.all(openSSHKeys.map(({ pem }) => keyLine(pem))), @@ -63,6 +64,11 @@ if (hasRclone && hasOpenSSL) { }); after(() => server.stop()); const hostKeys = await keyLine(hostKey); + /** All the host keys of the server, for each algorithm of host key. */ + const allHostKeys = [ + hostKeys, + ...await Promise.all(otherHostKeys.map(keyLine)), + ].join(","); const params = { host: server.host, port: `${server.port}`, @@ -132,7 +138,7 @@ if (hasRclone && hasOpenSSL) { ) { const data = new Uint8Array(100000).map((_, i) => i * 13); const other = remote(sftp, { - params: { ...params, host_keys: "", ...options }, + params: { ...params, host_keys: allHostKeys, ...options }, root: "/sftp-test/", }); const put = await other.put("cipher.bin", data); @@ -146,6 +152,26 @@ if (hasRclone && hasOpenSSL) { } }); + test("refuses each server with no host_keys and no known_hosts_file", async () => { + const response = await remote(sftp, { + params: { ...params, host_keys: "" }, + }).get(""); + const text = await response.text(); + assert.equal(response.status, 502); + // The answer gives the key of the server, for host_keys. + const match = + /The host key SHA256:\S+ of the server is not known\. .* set host_keys to (\S+ \S+)$/ + .exec(text); + assert.ok(match, text); + assert.ok(allHostKeys.split(",").includes(match[1]), match[1]); + // That key in host_keys signs in. + const again = await remote(sftp, { + params: { ...params, host_keys: match[1] }, + }).get(""); + await again.body?.cancel(); + assert.equal(again.status, 200); + }); + test("refuses a host key that is not in host_keys", async () => { const response = await remote(sftp, { params: { ...params, host_keys: await keyLine(makeKey("ed25519")) },