SFTP backend, with an SSH client on WebCrypto

.github/workflows/ci.yml

@@ -128,7 +128,7 @@ jobs:
         shell: bash
         run: |
           npm test 2>&1 | tee "$RUNNER_TEMP/test.log"
-          if grep -E "((rclone|7-Zip|Go|go-pmtiles|versatiles|Azurite) is not installed|node:sqlite is not in)" "$RUNNER_TEMP/test.log"; then
+          if grep -E "((rclone|7-Zip|Go|go-pmtiles|versatiles|Azurite|openssl) is not installed|node:sqlite is not in)" "$RUNNER_TEMP/test.log"; then
             echo "::error::A test skipped because a tool is not installed."
             exit 1
           fi

CONTRIBUTING.md

@@ -279,6 +279,13 @@ A backend that uploads a large file in parts, for example s3 and azureblob,
 reads the body with `Parts` of `@rfetch/lib/parts`: each part has the size of
 the option `chunk_size`, and only one part is in memory.
 
+The path of a remote is from its root. For sftp, as in rclone, the root is the
+login folder of the user, and a path that starts with `/` is from the root of
+the server: `fetch.js` gives the same pathname for `sftp:a.txt` and for
+`sftp:/a.txt`, thus it tells a backend with `RFETCH_TYPE_ROOT` of `env`, for
+example `RFETCH_SFTP_ROOT`, which is `/` for a path that starts with `/`. A
+config file, a variable, a header or the query cannot set it.
+
 A backend on another remote, for example alias, gives `fetch` the location
 `encodeLocation(remote) + encodePath(path)`. The option `remote` is a path of
 rclone: `encodeLocation` of `@rfetch/lib/fspath` keeps its `%`, `?` and `#` in
@@ -466,6 +473,12 @@ default). Put the credentials in `RFETCH_S3_ACCESS_KEY_ID` and
 `RCLONE_`. Use a bucket only for the tests: the tests make folders in it and
 delete them.
 
+The tests of `sftp` use `rclone serve sftp`, with host keys and user keys that
+openssl makes for each run: they run when rclone and `openssl` are installed.
+They also compare rfetch with the sftp backend of rclone on the same server, for
+example with keys in the format of OpenSSH that the tests write and rclone
+reads. The tests in `backend/sftp/ssh/ssh_test.js` need no server.
+
 The tests of `azureblob` use Azurite, the emulator of Azure Storage, with its
 data in memory: they run when `azurite-blob` is in the `PATH`, or when the
 variable `AZURITE` is its path. Get it with `npm install azurite@3.37.0`. The
@@ -524,7 +537,8 @@ message of the commit.
 ## Continuous integration
 
 The workflow `.github/workflows/ci.yml` runs for each push and each pull
-request. It installs rclone, 7-Zip, Go, go-pmtiles, versatiles and Azurite, runs
-`npm test` with Node.js 22, 24 and 26, and runs `deno fmt --check`. A test that
-needs one of these tools skips when the tool is not installed: in the workflow,
-such a skip is an error. Before a push, run `npm test` and `deno fmt --check`.
+request. It installs rclone, 7-Zip, Go, go-pmtiles, versatiles and Azurite (the
+runner has openssl), runs `npm test` with Node.js 22, 24 and 26, and runs
+`deno fmt --check`. A test that needs one of these tools skips when the tool is
+not installed: in the workflow, such a skip is an error. Before a push, run
+`npm test` and `deno fmt --check`.

README.md

@@ -18,6 +18,9 @@ A [Rclone wrapper for WebAssembly build](./rclone.js) is also available.
 - Memory [:page_facing_up:](https://rclone.org/memory/)
 - S3: Amazon S3 and the services with the same API, for example Cloudflare R2
   [:page_facing_up:](https://rclone.org/s3/)
+- SFTP: the files of an SSH server, with a client of SSH in JavaScript that runs
+  in Node.js, Deno, Bun and Cloudflare Workers
+  [:page_facing_up:](https://rclone.org/sftp/)
 - WebDAV: Nextcloud, ownCloud, Fastmail Files, `rclone serve webdav` and other
   WebDAV servers [:page_facing_up:](https://rclone.org/webdav/)
 

backend/main.js

@@ -35,6 +35,7 @@ export const backends = {
   local: () => import("@rfetch/local"),
   memory: () => import("@rfetch/memory"),
   s3: () => import("@rfetch/s3"),
+  sftp: () => import("@rfetch/sftp"),
   tiles: () => import("@rfetch/tiles"),
   union: () => import("@rfetch/union"),
   webdav: () => import("@rfetch/webdav"),

backend/sftp/knownhosts.js

@@ -0,0 +1,118 @@
+/**
+ * Reads a file `known_hosts` of OpenSSH, as the option `known_hosts_file`
+ * of rclone, and finds the host key of a server in it.
+ */
+
+import { equalBytes } from "./ssh/wire.js";
+import { fromBase64 } from "./ssh/keys.js";
+
+/**
+ * The name of a host in a file `known_hosts`: the host for the port 22,
+ * else `[host]:port`.
+ * @param {string} host
+ * @param {number} port
+ */
+function nameOf(host, port) {
+  return port === 22 ? host : `[${host}]:${port}`;
+}
+
+/**
+ * Tells if a pattern of OpenSSH matches a name: `*` is any text, and `?`
+ * is one character.
+ * @param {string} pattern
+ * @param {string} name
+ */
+function matches(pattern, name) {
+  const source = pattern
+    .toLowerCase()
+    .replace(/[.+^${}()|[\]\\]/g, "\\$&")
+    .replace(/\*/g, ".*")
+    .replace(/\?/g, ".");
+  return new RegExp(`^${source}$`).test(name.toLowerCase());
+}
+
+/**
+ * Tells if a hashed name, `|1|salt|hash`, is the name: the hash is the
+ * HMAC-SHA1 of the name with the salt as its key.
+ * @param {string} entry
+ * @param {string} name
+ */
+async function matchesHash(entry, name) {
+  const [, , salt, hash] = entry.split("|");
+  if (!salt || !hash) return false;
+  const key = await crypto.subtle.importKey(
+    "raw",
+    fromBase64(salt),
+    { name: "HMAC", hash: "SHA-1" },
+    false,
+    ["sign"],
+  );
+  const mac = new Uint8Array(
+    await crypto.subtle.sign("HMAC", key, new TextEncoder().encode(name)),
+  );
+  return equalBytes(mac, fromBase64(hash));
+}
+
+/**
+ * Tells if the hosts of a line match the name of a server, as OpenSSH: a
+ * pattern with `!` before it excludes the name.
+ * @param {string} hosts The hosts of the line, with `,` between them.
+ * @param {string} name
+ */
+async function hostMatches(hosts, name) {
+  let found = false;
+  for (const pattern of hosts.split(",")) {
+    if (pattern.startsWith("|1|")) {
+      if (await matchesHash(pattern, name)) found = true;
+    } else if (pattern.startsWith("!")) {
+      if (matches(pattern.slice(1), name)) return false;
+    } else if (matches(pattern, name)) {
+      found = true;
+    }
+  }
+  return found;
+}
+
+/**
+ * Finds the host key of a server in a file `known_hosts`.
+ * @param {string} text The content of the file.
+ * @param {string} host
+ * @param {number} port
+ * @param {Uint8Array} blob The host key of the server.
+ * @returns {Promise<"known" | "unknown" | "changed" | "revoked">} `known`
+ *   when the file has the key for the host, `changed` when the file has
+ *   other keys for the host, `revoked` when the file revokes the key.
+ */
+export async function findHostKey(text, host, port, blob) {
+  const name = nameOf(host, port);
+  let other = false;
+  let known = false;
+  for (const raw of text.split(/\r?\n/)) {
+    const line = raw.trim();
+    if (line === "" || line.startsWith("#")) continue;
+    const fields = line.split(/\s+/);
+    let marker = "";
+    if (fields[0].startsWith("@")) marker = fields.shift() ?? "";
+    const [hosts, , key] = fields;
+    if (!hosts || !key) continue;
+    let keyBlob;
+    try {
+      keyBlob = fromBase64(key);
+    } catch {
+      continue;
+    }
+    const same = equalBytes(keyBlob, blob);
+    // A revoked key is refused for each host.
+    if (marker === "@revoked") {
+      if (same) return "revoked";
+      continue;
+    }
+    // rfetch does not know the certificates of hosts.
+    if (marker !== "") continue;
+    if (!await hostMatches(hosts, name)) continue;
+    if (same) known = true;
+    else other = true;
+  }
+  if (known) return "known";
+  return other ? "changed" : "unknown";
+}