From 54efe01ee087da2670b10bd317b57b1d28bb6375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Mon, 19 Oct 2026 20:51:48 +0000 Subject: [PATCH 1/2] feat: COPY, MOVE and the time of a file in the backend contract The contract of a backend gets new methods, as the features Copy, Move, DirMove and SetModTime of rclone: - COPY of a file and MOVE of a file, with the header Destination: the path of the destination in the same root. MOVE of a folder goes to a path that does not exist, else 412. - PATCH of a file with X-OC-Mtime sets its time. - PUT, COPY and MOVE with X-OC-Mtime give that time to the file, and the answer has X-OC-Mtime: accepted. X-OC-Mtime is the header of ownCloud: seconds after 1970, with an optional part of a second. fetch.js gives the backend the path of the destination from its location. The two locations must have the same type, query and options, else the answer is 502 and the backend gets no request: a Destination never takes a backend out of its remote. The backends: - memory and local: COPY, MOVE, PATCH and the time. Node.js sets a time to the microsecond. - s3: COPY with CopyObject, and the time in the metadata mtime, as rclone. PATCH copies the object onto itself. GET and HEAD give the time of the metadata. The list gives the time of the upload. - webdav: COPY and MOVE on the server, with MKCOL of the folders first. The time goes to the vendors of rclone with useOCMtime, and a PROPPATCH of lastmodified sets it for owncloud, nextcloud and infinitescale. The time does not go with a COPY to rclone serve webdav: that server gives it to the source. - crypt, alias, combine (in one upstream), and the folders of chunker and compress give the destination in their remote. The files of an archive or of a tiles file give 405. drive gives 405 to other methods. The commands: - copy and copyto send COPY first, with the time of the source. For 403, 405, 501 and 502, the bytes go through rfetch, with the time. A second copy skips the same files. When the list of the target gives another time than HEAD, as s3 does, HEAD gives the time. - move and moveto send MOVE first. A folder moves with one MOVE when the target does not exist, as DirMove of rclone. - touch sets the time of a file that exists with PATCH, and --timestamp works as rclone: YYMMDD, or a time of 19 characters or more. serve webdav and the Worker send COPY and MOVE to the backend first, give X-OC-Mtime to the destination, and make a PROPPATCH of lastmodified a PATCH. The shared suite tests the new methods with the features of each backend. cmd/rclone_test.js compares move with DirMove, the times of a copy, a second copy and touch with rclone. --- CONTRIBUTING.md | 71 ++++++++++- README.md | 10 +- backend/alias/main.js | 11 ++ backend/alias/main_test.js | 7 + backend/archive/main_test.js | 1 + backend/chunker/main.js | 25 ++-- backend/chunker/main_test.js | 3 + backend/combine/main.js | 15 +++ backend/combine/main_test.js | 14 ++ backend/compress/main.js | 13 +- backend/compress/main_test.js | 2 + backend/crypt/main.js | 38 +++++- backend/crypt/main_test.js | 12 +- backend/drive/main.js | 7 + backend/local/main.js | 112 ++++++++++++++-- backend/local/main_test.js | 10 +- backend/memory/main.js | 130 ++++++++++++++++++- backend/memory/main_test.js | 10 +- backend/s3/main.js | 128 ++++++++++++++++++- backend/s3/main_test.js | 24 ++++ backend/webdav/main.js | 233 +++++++++++++++++++++++++++++++--- backend/webdav/main_test.js | 9 ++ cmd/copy/main.js | 52 ++++++-- cmd/move/main.js | 10 +- cmd/move/main_test.js | 25 +++- cmd/operations.js | 148 ++++++++++++++++++++- cmd/rclone_test.js | 87 +++++++++++-- cmd/serve/webdav/handler.js | 136 +++++++++++++++++--- cmd/serve/webdav/main_test.js | 21 +++ cmd/touch/main.js | 159 ++++++++++++++++++----- cmd/touch/main_test.js | 67 ++++++++-- fetch.js | 152 +++++++++++++++------- lib/overlay.js | 8 ++ lib/testing/suite.js | 196 +++++++++++++++++++++++++++- lib/time.js | 62 +++++++++ lib/time_test.js | 50 +++++++- main_test.js | 51 ++++++++ 37 files changed, 1908 insertions(+), 201 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 916a527..f3236f5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -144,10 +144,17 @@ code of WebDAV is in `cmd/serve/webdav/handler.js`, and the Worker uses it too: - PROPFIND reads the headers of HEAD, and the list of a folder for `Depth: 1`. `Depth: infinity` gets 403, as RFC 4918 permits. -- MKCOL is PUT of a folder. COPY and MOVE read each file from the source and put - it at the destination. MOVE deletes the source after. +- MKCOL is PUT of a folder. COPY and MOVE of a file, and MOVE of a folder to a + path that does not exist, go to the backend. For a backend that cannot do it, + or a destination in another remote, the server reads each file from the source + and puts it at the destination, and MOVE deletes the source after. +- A PUT, a COPY or a MOVE of a file with `X-OC-Mtime` gives that time to the + file, with `X-OC-Mtime: accepted` in the answer when the backend keeps it. + `rclone serve webdav` gives the time of a COPY to the source: rfetch gives it + to the destination, as ownCloud. A PROPPATCH of `lastmodified` with the + seconds after 1970 is a PATCH of the backend, as ownCloud. - The server keeps no data between requests: LOCK gives a token and locks - nothing, and PROPPATCH keeps no property, as `rclone serve webdav`. The + nothing, and PROPPATCH keeps no other property, as `rclone serve webdav`. The clients of Windows and macOS need LOCK to write. - A client writes, unless `--read-only`. With `--user` and `--pass`, a client must give them with the Basic authentication. The default address is @@ -198,9 +205,47 @@ code that only one backend uses is in the folder of that backend, for example - `DELETE /folder/`: deletes folder, and all the files and folders in it. - `DELETE /file`: deletes file. +A backend can also know these methods, as the features `Copy`, `Move`, `DirMove` +and `SetModTime` of rclone: + +- `COPY /file` with `Destination: /path`: copies the file in the backend, and + the bytes do not go through rfetch. The destination is a path in the same + root, with percent-encoding, as the path of a request. A file at the + destination is replaced, and the folders of the destination are made, as `PUT` + does. +- `MOVE /file` with `Destination: /path`: moves the file, with the same rules. +- `MOVE /folder/` with `Destination: /path/`: moves the folder with all that is + in it, to a path that does not exist. A destination that exists gives 412. +- `PATCH /file` with `X-OC-Mtime`: sets the time of the file. + +The header `X-OC-Mtime` of ownCloud is the time of a file, in seconds after +1970, with an optional part of a second, for example `1354040105.123456789`. A +`PUT`, a `COPY` or a `MOVE` of a file with it gives that time to the file, and +the answer of a backend that keeps it has `X-OC-Mtime: accepted`. Then `GET`, +`HEAD` and the list give that time. Read it with `modTimeOf` of +`@rfetch/lib/time`. s3 keeps the time in the metadata of the file, as rclone: +its list gives the time of the upload, because a list of S3 has no metadata. + +A backend that does not know a method gives 405. A backend that cannot do a +`COPY` or a `MOVE`, for example a `COPY` of a folder, gives 501. Then a command +reads the source and puts it at the destination. + +`fetch.js` gives the backend the path of the destination in its root, from the +location of the destination, for example `remote:path/b.txt`. The two locations +must have the same type, the same query and the same options: else `fetch.js` +gives 502, as WebDAV gives for a destination on another server, and the backend +gets no request. Thus a `Destination` never takes a backend out of its remote. A +backend on another remote, for example alias and crypt, gives the location of +the destination in that remote to `fetch`, as it does for the path of the +request. A backend that cannot give that location, for example the files in an +archive, gives 405 to a `COPY` and a `MOVE`: it must not send the `Destination` +of its client to another remote. + The answers: - `PUT` gives 201 with the path in `Content-Location`. `DELETE` gives 204. +- `COPY` and `MOVE` give 201 for a new file, or 204 when the destination was a + file. `PATCH` gives 204. - A file or a folder that does not exist gives 404. - `GET` of a file gives the headers `Content-Type`, `Content-Length`, `Last-Modified` and `Accept-Ranges: bytes`. @@ -289,10 +334,13 @@ the local time zone, and the size and the count with `sizeField` and The other commands use the operations of `cmd/operations.js`, as the package `operations` of rclone: `stat` gives the information of a path, `walk` gives the -files and the folders in a folder, and `copyFile`, `remove` and `mkdir` change -the files. An operation that fails throws an `OperationError` with the status of -the answer, and `failure` gives the answer of the command for it. `log` writes -the errors and the notices of rclone on the standard error, for example +files and the folders in a folder, and `copyFile`, `moveFolder`, `setModTime`, +`remove` and `mkdir` change the files. `copyFile` sends a `COPY` or a `MOVE` to +the backend first, with the time of the source in `X-OC-Mtime`. For 403, 405, +501 and 502, it reads the source and puts it at the destination, with the same +time. An operation that fails throws an `OperationError` with the status of the +answer, and `failure` gives the answer of the command for it. `log` writes the +errors and the notices of rclone on the standard error, for example `2006/01/02 15:04:05 ERROR : a.txt: sizes differ`, and `--quiet` keeps only the errors. A command that has an output and an error, for example `check --combined -`, gives the output in the body and the error in @@ -341,6 +389,9 @@ and deletes it at the end. It tests: 416, and ranges that the backend does not read. - `PUT` over a file, `DELETE` of a file, and `DELETE` of a folder with all that is in it. +- `COPY`, `MOVE`, `PATCH` and `X-OC-Mtime`, as its `features` tell. A backend + with no such feature must give 405, 403, 501 or 502, and must not change a + file. Other options are for some backends only: @@ -358,6 +409,12 @@ Other options are for some backends only: suite also gives the `params` in the headers `RFETCH_TYPE_NAME` and in the variables `RCLONE_TYPE_NAME` of `env`, and makes sure that no link of a list has a query. +- `features`: what the backend can do other than `GET`, `HEAD`, `PUT` and + `DELETE`: `copy`, `move` and `dirMove` for `COPY` of a file, `MOVE` of a file + and `MOVE` of a folder, `modTime` for `X-OC-Mtime` on `PUT`, `copyModTime` for + `X-OC-Mtime` on `COPY` and `MOVE` (by default the value of `modTime`), + `setModTime` for `PATCH`, and `listModTime` for the time of `X-OC-Mtime` in + the list (by default the value of `modTime`). Put the tests for what only one backend does in the same file, after the suite. For example, the tests of the local backend make sure that a path cannot go out diff --git a/README.md b/README.md index cf2fe10..971a35f 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,9 @@ does not exist, 4 for a file that does not exist, and 1 for another error. and with `--download` the bytes. - `config` - `copy` and `copyto`: copy the files that the target does not have, or that - differ by size or by time. + differ by size or by time. A copy gets the time of its source, when the + backend can keep it. In the same remote, the backend copies each file, and the + bytes do not go through rfetch. - `copyurl` - `cryptdecode` - `delete`, `deletefile`, `purge`, `rmdir` and `rmdirs` @@ -86,7 +88,9 @@ does not exist, 4 for a file that does not exist, and 1 for another error. - `lsjson` - `lsl` - `mkdir` -- `move` and `moveto`: copy, then delete each source file. +- `move` and `moveto`: in the same remote, the backend moves each file, or the + folder when the target does not exist. Else copy, then delete each source + file. - `obscure` - `rcat` - `reveal` @@ -96,7 +100,7 @@ does not exist, 4 for a file that does not exist, and 1 for another error. reads and writes, unless `--read-only`. With `--user` and `--pass`, a client must log in with them. - `size` -- `touch`: make an empty file. The backends cannot set the time of a file yet. +- `touch`: make an empty file, or set the time of a file. - `tree` - `version` diff --git a/backend/alias/main.js b/backend/alias/main.js index 7cd4f7d..a382e93 100644 --- a/backend/alias/main.js +++ b/backend/alias/main.js @@ -59,6 +59,17 @@ async function alias(request, env = {}) { method, headers: remoteHeaders(request.headers, "alias", remote), }; + // The destination of a COPY or a MOVE is in the remote too. + if (method === "COPY" || method === "MOVE") { + const target = decodePath(request.headers.get("Destination") ?? ""); + if (target === null) { + return new Response("Destination is not a path", { status: 400 }); + } + init.headers.set( + "Destination", + encodeLocation(remote) + encodePath(target), + ); + } if (request.body) { init.body = request.body; init.duplex = "half"; diff --git a/backend/alias/main_test.js b/backend/alias/main_test.js index 14e6e18..ab81466 100644 --- a/backend/alias/main_test.js +++ b/backend/alias/main_test.js @@ -38,6 +38,13 @@ test("local path", async () => { backendSuite("alias", backend, { type: "alias", params: { remote: ":memory:/alias" }, + features: { + copy: true, + move: true, + dirMove: true, + modTime: true, + setModTime: true, + }, }); test("the links of a list are in the alias", async () => { diff --git a/backend/archive/main_test.js b/backend/archive/main_test.js index a9b822b..334115a 100644 --- a/backend/archive/main_test.js +++ b/backend/archive/main_test.js @@ -18,6 +18,7 @@ const decoder = new TextDecoder(); backendSuite("archive", backend, { type: "archive", params: { remote: ":memory:/archive" }, + features: { modTime: true, setModTime: true }, }); /** Text that compresses, of a size. @param {number} size */ diff --git a/backend/chunker/main.js b/backend/chunker/main.js index 2c424e8..dc1b8fe 100644 --- a/backend/chunker/main.js +++ b/backend/chunker/main.js @@ -207,17 +207,26 @@ async function chunker(request, env = {}) { if (method === "GET" || method === "HEAD") { return await listFolder(config, path, method, `${searchParams}`); } - // A folder is a folder of the remote. - const response = await send(config, path, { - method, - headers: remoteHeaders(request.headers, "chunker", config.remote), - }); + // A folder is a folder of the remote. The destination of a MOVE is in + // the remote too. + const headers = remoteHeaders(request.headers, "chunker", config.remote); + if (method === "COPY" || method === "MOVE") { + const target = decodePath(request.headers.get("Destination") ?? ""); + if (target === null) { + return new Response("Destination is not a path", { status: 400 }); + } + headers.set( + "Destination", + encodeLocation(config.remote) + encodePath(target), + ); + } + const response = await send(config, path, { method, headers }); await response.body?.cancel(); - const headers = new Headers(); + const answer = new Headers(); if (method === "PUT" && response.ok) { - headers.set("Content-Location", pathname); + answer.set("Content-Location", pathname); } - return new Response(null, { status: response.status, headers }); + return new Response(null, { status: response.status, headers: answer }); } try { diff --git a/backend/chunker/main_test.js b/backend/chunker/main_test.js index 8fed704..cddbdc0 100644 --- a/backend/chunker/main_test.js +++ b/backend/chunker/main_test.js @@ -13,6 +13,7 @@ const CHUNK = "64k"; backendSuite("chunker", backend, { type: "chunker", params: { remote: ":memory:/chunker", chunk_size: CHUNK }, + features: { dirMove: true }, }); // Without metadata, a name is always in chunks. rclone does not read the // name of a chunk with "\n": "." of Go does not match it. @@ -25,6 +26,7 @@ backendSuite("chunker meta_format=none", backend, { hash_type: "none", }, skipName: /\n/, + features: { dirMove: true }, }); backendSuite("chunker transactions=norename", backend, { type: "chunker", @@ -33,6 +35,7 @@ backendSuite("chunker transactions=norename", backend, { chunk_size: CHUNK, transactions: "norename", }, + features: { dirMove: true }, }); /** @param {number} size */ diff --git a/backend/combine/main.js b/backend/combine/main.js index 5a0d249..f29b604 100644 --- a/backend/combine/main.js +++ b/backend/combine/main.js @@ -138,6 +138,21 @@ async function combine(request, env = {}) { method, headers: remoteHeaders(request.headers, "combine", remote), }; + // A COPY or a MOVE goes to the upstream when the destination is in the + // same upstream. Else, the answer is 502, as for another remote. + if (method === "COPY" || method === "MOVE") { + const target = decodePath(request.headers.get("Destination") ?? ""); + if (target === null) { + return new Response("Destination is not a path", { status: 400 }); + } + const end = target.indexOf("/", 1); + if (end === -1 || target.slice(0, end) !== path.slice(0, slash)) { + return new Response("The destination is in another upstream.", { + status: 502, + }); + } + init.headers.set("Destination", locationOf(remote, target.slice(end))); + } if (request.body) { init.body = request.body; init.duplex = "half"; diff --git a/backend/combine/main_test.js b/backend/combine/main_test.js index 3f3d776..32913ac 100644 --- a/backend/combine/main_test.js +++ b/backend/combine/main_test.js @@ -14,11 +14,25 @@ backendSuite("combine", backend, { type: "combine", params: { upstreams: UPSTREAMS }, parent: "/a/", + features: { + copy: true, + move: true, + dirMove: true, + modTime: true, + setModTime: true, + }, }); backendSuite("combine", backend, { type: "combine", params: { upstreams: UPSTREAMS }, parent: "/b c/", + features: { + copy: true, + move: true, + dirMove: true, + modTime: true, + setModTime: true, + }, }); test("upstreamsOf reads the upstreams as rclone", () => { diff --git a/backend/compress/main.js b/backend/compress/main.js index 02ba79d..c4d166f 100644 --- a/backend/compress/main.js +++ b/backend/compress/main.js @@ -303,7 +303,18 @@ async function compress(request, env = {}) { if (path.endsWith("/")) { if (method !== "GET") { /** @type {RequestInit} */ - const init = { method, headers: request.headers }; + const init = { method, headers: new Headers(request.headers) }; + // The destination of a MOVE is in the remote too. + if (method === "COPY" || method === "MOVE") { + const target = decodePath(request.headers.get("Destination") ?? ""); + if (target === null) { + return new Response("Destination is not a path", { status: 400 }); + } + init.headers.set( + "Destination", + encodeLocation(remote) + encodePath(target), + ); + } if (request.body) { init.body = request.body; init.duplex = "half"; diff --git a/backend/compress/main_test.js b/backend/compress/main_test.js index db1ce69..425533e 100644 --- a/backend/compress/main_test.js +++ b/backend/compress/main_test.js @@ -13,10 +13,12 @@ const encoder = new TextEncoder(); backendSuite("compress", backend, { type: "compress", params: { remote: ":memory:/compress" }, + features: { dirMove: true }, }); // The data in gzip are compressed. backendSuite("compress", backend, { params: { remote: ":memory:/compress-6", level: "6" }, + features: { dirMove: true }, }); /** Text that compresses, of a size. @param {number} size */ diff --git a/backend/crypt/main.js b/backend/crypt/main.js index 8d9d798..791607b 100644 --- a/backend/crypt/main.js +++ b/backend/crypt/main.js @@ -271,7 +271,7 @@ async function crypt(request, env = {}) { const response = await fetch(target, init); await response.body?.cancel(); - const result = new Headers(); + const result = timeHeaders(response); if (response.ok) { result.set("Content-Location", pathname); } @@ -284,12 +284,46 @@ async function crypt(request, env = {}) { return new Response(null, { status: response.status }); } + // A COPY, a MOVE or a PATCH goes to the remote, with the encrypted name of + // the destination. The encrypted data does not change. + if (method === "COPY" || method === "MOVE" || method === "PATCH") { + if (method !== "PATCH") { + const destination = decodePath(request.headers.get("Destination") ?? ""); + if (destination === null) { + return new Response("Destination is not a path", { status: 400 }); + } + headers.set( + "Destination", + encodeLocation(remote) + cipher.names.encrypt(destination), + ); + } + const response = await fetch(target, { method, headers }); + await response.body?.cancel(); + return new Response(null, { + status: response.status, + headers: timeHeaders(response), + }); + } + return new Response(null, { status: 405, - headers: { Allow: "GET, HEAD, PUT, DELETE" }, + headers: { Allow: "GET, HEAD, PUT, DELETE, COPY, MOVE, PATCH" }, }); } +/** + * The headers of an answer of the remote that go to the client: the answer + * `X-OC-Mtime: accepted` when the remote set the time of the file. + * @param {Response} response + * @returns {Headers} + */ +function timeHeaders(response) { + const headers = new Headers(); + const accepted = response.headers.get("X-OC-Mtime"); + if (accepted) headers.set("X-OC-Mtime", accepted); + return headers; +} + /** * Gets a range of a file, as the seek of rclone: the header of the * encrypted file gives its nonce and its size, then only the blocks of the diff --git a/backend/crypt/main_test.js b/backend/crypt/main_test.js index 5a5eacd..231f871 100644 --- a/backend/crypt/main_test.js +++ b/backend/crypt/main_test.js @@ -147,7 +147,17 @@ const params = { password2: SALT, }; -backendSuite("crypt", crypt, { type: "crypt", params }); +backendSuite("crypt", crypt, { + type: "crypt", + params, + features: { + copy: true, + move: true, + dirMove: true, + modTime: true, + setModTime: true, + }, +}); describe("crypt", () => { const fs = remote(crypt, { params }); diff --git a/backend/drive/main.js b/backend/drive/main.js index 80a9b7d..bdf17ea 100644 --- a/backend/drive/main.js +++ b/backend/drive/main.js @@ -96,6 +96,13 @@ async function drive(request, env = {}) { pathname = decodeURIComponent(pathname); const isDirectory = pathname.endsWith("/"); + if (!["GET", "HEAD", "PUT", "DELETE"].includes(method)) { + return new Response(null, { + status: 405, + headers: { Allow: "GET, HEAD, PUT, DELETE" }, + }); + } + if (method === "PUT") { // TODO: Nested folder creation. if (isDirectory) { diff --git a/backend/local/main.js b/backend/local/main.js index a3497a7..548cdba 100644 --- a/backend/local/main.js +++ b/backend/local/main.js @@ -5,6 +5,7 @@ import { escapeHTML } from "@rfetch/lib/html"; import { mimeType } from "@rfetch/lib/mime"; import { decodePath } from "@rfetch/lib/path"; import { notSatisfiable, parseRange, partialHeaders } from "@rfetch/lib/range"; +import { isoStringOf, modTimeOf } from "@rfetch/lib/time"; /** * Serves a local remote: the files and the folders in its root. The root is @@ -14,6 +15,18 @@ import { notSatisfiable, parseRange, partialHeaders } from "@rfetch/lib/range"; * line or of the code, for example `/tmp/file.txt`: an option, a header or * the query cannot set it. Thus a request from a client of `rfetch serve`, * which has no `env`, stays in the current folder. + * + * Other than GET, HEAD, PUT and DELETE, it knows: + * + * - `COPY` of a file, and `MOVE` of a file or a folder, to the path of the + * header `Destination` in the same root. A folder moves only to a path + * that does not exist. + * - `PATCH` of a file or a folder with the header `X-OC-Mtime`: it sets the + * time. + * + * A `PUT`, a `COPY` or a `MOVE` of a file with `X-OC-Mtime` gives that time + * to the file, and its answer has `X-OC-Mtime: accepted`. The system sets a + * time to the microsecond. * @param {Request} request * @param {Record} [env] * @returns {Promise} @@ -38,6 +51,11 @@ async function local(request, env = {}) { // Worker environments. const { mkdir, open, readdir, rm, stat } = await import("node:fs/promises"); + const modTime = modTimeOf(request); + if (modTime === undefined) { + return new Response("X-OC-Mtime is not a time", { status: 400 }); + } + /** @type {import("node:fs").Stats | null} */ const stats = await stat(absolutePath).catch(() => null); @@ -212,6 +230,10 @@ async function local(request, env = {}) { } finally { await file.close(); } + if (modTime !== null) { + await setTime(absolutePath, modTime); + headers.set("X-OC-Mtime", "accepted"); + } } status = 201; @@ -227,9 +249,74 @@ async function local(request, env = {}) { await rm(absolutePath, { recursive: true }); status = 204; + } else if (method === "COPY" || method === "MOVE") { + if (!stats) return new Response(null, { status: 404 }); + const target = decodePath(request.headers.get("Destination") ?? ""); + const absoluteTarget = target === null ? null : join(root, target); + if (absoluteTarget === null || isOutside(root, absoluteTarget)) { + return new Response("Destination is not a path", { status: 400 }); + } + /** @type {import("node:fs").Stats | null} */ + const existing = await stat(absoluteTarget.replace(/[\\/]+$/, "")) + .catch(() => null); + if (stats.isDirectory()) { + // A folder moves to a new path, out of the folder. + if (method === "COPY") { + return new Response("COPY of a folder is not implemented", { + status: 501, + }); + } + if ( + !path.endsWith("/") || path === "/" || + resolve(absolutePath) === resolve(cwd) + ) { + return new Response(null, { status: 403 }); + } + if ( + !target.endsWith("/") || !isOutside(absolutePath, absoluteTarget) + ) { + return new Response("Destination is not a new folder", { + status: 400, + }); + } + if (existing) { + return new Response("Destination exists", { status: 412 }); + } + } else if (target.endsWith("/") || existing?.isDirectory()) { + return new Response("Destination is a folder", { status: 409 }); + } + if (resolve(absoluteTarget) !== resolve(absolutePath)) { + const { copyFile, rename } = await import("node:fs/promises"); + await mkdir(dirname(absoluteTarget), { recursive: true }); + try { + if (method === "COPY") await copyFile(absolutePath, absoluteTarget); + else await rename(absolutePath, absoluteTarget); + } catch (error) { + // A move to another file system: the caller copies the bytes. + if (error?.code === "EXDEV") { + return new Response("Destination is on another device", { + status: 501, + }); + } + throw error; + } + } + if (modTime !== null && !stats.isDirectory()) { + await setTime(absoluteTarget, modTime); + headers.set("X-OC-Mtime", "accepted"); + } + status = existing ? 204 : 201; + } else if (method === "PATCH") { + if (!stats) return new Response(null, { status: 404 }); + if (modTime === null) { + return new Response("X-OC-Mtime is missing", { status: 400 }); + } + await setTime(absolutePath, modTime); + headers.set("X-OC-Mtime", "accepted"); + status = 204; } else { status = 405; - headers.set("Allow", "GET, HEAD, PUT, DELETE"); + headers.set("Allow", "GET, HEAD, PUT, DELETE, COPY, MOVE, PATCH"); } return new Response(body, { @@ -250,19 +337,18 @@ function isOutside(root, path) { } /** - * The time of a file in ISO 8601, with 9 digits after the second: the - * format of rclone for a remote with a precision of 1 ns. - * @param {bigint} mtimeNs The nanoseconds after 1970. - * @returns {string} + * Sets the time of the last change of a file, and of its last read, as + * rclone. The system sets a time to the microsecond. + * @param {string} path + * @param {bigint} nanoseconds The nanoseconds after 1970. */ -function isoStringOf(mtimeNs) { - const second = 1000000000n; - // The floor, also for a time before 1970. - let nanoseconds = mtimeNs % second; - if (nanoseconds < 0n) nanoseconds += second; - const seconds = (mtimeNs - nanoseconds) / second; - const date = new Date(Number(seconds) * 1000).toISOString().slice(0, 19); - return `${date}.${String(nanoseconds).padStart(9, "0")}Z`; +async function setTime(path, nanoseconds) { + const { utimes } = await import("node:fs/promises"); + // The seconds go in a double, which can be some nanoseconds less than the + // time, and the system cuts the time to the microsecond: half of a + // microsecond more gives the correct microsecond. + const seconds = (Number(nanoseconds / 1000n) + 0.5) / 1e6; + await utimes(path, seconds, seconds); } /** diff --git a/backend/local/main_test.js b/backend/local/main_test.js index dd5056f..7c44d84 100644 --- a/backend/local/main_test.js +++ b/backend/local/main_test.js @@ -24,7 +24,15 @@ await mkdir(root); await writeFile(join(folder, "secret.txt"), "secret"); process.chdir(root); -backendSuite("local", backend); +backendSuite("local", backend, { + features: { + copy: true, + move: true, + dirMove: true, + modTime: true, + setModTime: true, + }, +}); describe("local", () => { after(() => rm(folder, { recursive: true, force: true })); diff --git a/backend/memory/main.js b/backend/memory/main.js index e477628..be2a849 100644 --- a/backend/memory/main.js +++ b/backend/memory/main.js @@ -11,6 +11,7 @@ import { renderListing } from "@rfetch/lib/listing"; import { mimeType } from "@rfetch/lib/mime"; import { decodePath } from "@rfetch/lib/path"; import { notSatisfiable, parseRange, partialHeaders } from "@rfetch/lib/range"; +import { modTimeOf } from "@rfetch/lib/time"; /** * A file: its bytes, its media type, and the time of its last change. @@ -35,7 +36,18 @@ const files = new Map(); const folders = new Map([["/", new Set()]]); /** - * Serves a memory remote + * Serves a memory remote. + * + * Other than GET, HEAD, PUT and DELETE, it knows: + * + * - `COPY` of a file, and `MOVE` of a file or a folder, to the path of the + * header `Destination` in this remote. A folder moves only to a path that + * does not exist. + * - `PATCH` of a file with the header `X-OC-Mtime`: it sets the time of the + * file. + * + * A `PUT`, a `COPY` or a `MOVE` with `X-OC-Mtime` gives that time to the + * file, and its answer has `X-OC-Mtime: accepted`. * @param {Request} request * @returns {Promise} */ @@ -48,6 +60,13 @@ async function memory(request) { return new Response(null, { status: 400 }); } + const modTime = modTimeOf(request); + if (modTime === undefined) { + return new Response("X-OC-Mtime is not a time", { status: 400 }); + } + /** The time of the request, in milliseconds. */ + const modified = modTime === null ? undefined : Number(modTime / 1000000n); + const headers = new Headers(); let status = 200, body = null; @@ -98,18 +117,17 @@ async function memory(request) { files.set(path, { bytes: new Uint8Array(await request.arrayBuffer()), type: request.headers.get("Content-Type") || mimeType(name), - modified: Date.now(), + modified: modified ?? Date.now(), }); folders.get(path.slice(0, slash))?.add(name); + if (modified !== undefined) headers.set("X-OC-Mtime", "accepted"); } status = 201; headers.append("Content-Location", pathname); } else if (method === "DELETE") { if (files.has(path)) { - files.delete(path); - const slash = path.lastIndexOf("/") + 1; - folders.get(path.slice(0, slash))?.delete(path.slice(slash)); + removeFile(path); } else if (folders.has(path)) { // Deletes the folder and all in it. The root folder stays. removeFolder(path); @@ -126,9 +144,24 @@ async function memory(request) { return new Response(null, { status: 404 }); } status = 204; + } else if (method === "COPY" || method === "MOVE") { + const target = decodePath(request.headers.get("Destination") ?? ""); + if (target === null) { + return new Response("Destination is not a path", { status: 400 }); + } + return transfer(method, path, target, modified, headers); + } else if (method === "PATCH") { + const file = files.get(path); + if (!file) return new Response(null, { status: 404 }); + if (modified === undefined) { + return new Response("X-OC-Mtime is missing", { status: 400 }); + } + file.modified = modified; + headers.set("X-OC-Mtime", "accepted"); + status = 204; } else { status = 405; - headers.set("Allow", "GET, HEAD, PUT, DELETE"); + headers.set("Allow", ALLOW); } return new Response(body, { @@ -137,6 +170,91 @@ async function memory(request) { }); } +/** The methods of the backend. */ +const ALLOW = "GET, HEAD, PUT, DELETE, COPY, MOVE, PATCH"; + +/** + * Copies or moves a file, or moves a folder. + * @param {"COPY" | "MOVE"} method + * @param {string} path The source, with no percent-encoding. + * @param {string} target The destination, with no percent-encoding. + * @param {number | undefined} modified The time of the copy, in + * milliseconds, or the time of the source. + * @param {Headers} headers The headers of the answer. + * @returns {Response} + */ +function transfer(method, path, target, modified, headers) { + const file = files.get(path); + if (file) { + if (target.endsWith("/") || folders.has(`${target}/`)) { + return new Response("Destination is a folder", { status: 409 }); + } + const existed = files.has(target); + if (target !== path) { + const slash = target.lastIndexOf("/") + 1; + makeFolder(target.slice(0, slash)); + files.set(target, { + // A body of a GET is the bytes of the file: the copy has its own + // bytes. + bytes: method === "COPY" ? file.bytes.slice() : file.bytes, + type: file.type, + modified: modified ?? file.modified, + }); + folders.get(target.slice(0, slash))?.add(target.slice(slash)); + if (method === "MOVE") removeFile(path); + } else if (modified !== undefined) { + file.modified = modified; + } + if (modified !== undefined) headers.set("X-OC-Mtime", "accepted"); + return new Response(null, { status: existed ? 204 : 201, headers }); + } + if (!path.endsWith("/") || !folders.has(path)) { + return new Response(null, { status: 404 }); + } + // A folder moves to a new path, out of the folder. + if (method === "COPY") { + return new Response("COPY of a folder is not implemented", { + status: 501, + }); + } + if (path === "/") return new Response(null, { status: 403 }); + if (!target.endsWith("/") || target.startsWith(path)) { + return new Response("Destination is not a new folder", { status: 400 }); + } + if (folders.has(target) || files.has(target.slice(0, -1))) { + return new Response("Destination exists", { status: 412 }); + } + const parent = target.slice( + 0, + target.lastIndexOf("/", target.length - 2) + 1, + ); + makeFolder(parent); + for (const [key, value] of [...folders]) { + if (!key.startsWith(path)) continue; + folders.delete(key); + folders.set(`${target}${key.slice(path.length)}`, value); + } + for (const [key, value] of [...files]) { + if (!key.startsWith(path)) continue; + files.delete(key); + files.set(`${target}${key.slice(path.length)}`, value); + } + const from = path.slice(0, path.lastIndexOf("/", path.length - 2) + 1); + folders.get(from)?.delete(path.slice(from.length)); + folders.get(parent)?.add(target.slice(parent.length)); + return new Response(null, { status: 201, headers }); +} + +/** + * Removes a file from its folder. + * @param {string} path + */ +function removeFile(path) { + files.delete(path); + const slash = path.lastIndexOf("/") + 1; + folders.get(path.slice(0, slash))?.delete(path.slice(slash)); +} + /** * Creates a folder and its parent folders. * @param {string} path The path of the folder, with `/` at the end. diff --git a/backend/memory/main_test.js b/backend/memory/main_test.js index db5019f..8109150 100644 --- a/backend/memory/main_test.js +++ b/backend/memory/main_test.js @@ -1,4 +1,12 @@ import { backendSuite } from "@rfetch/lib/testing/suite"; import backend from "./main.js"; -backendSuite("memory", backend); +backendSuite("memory", backend, { + features: { + copy: true, + move: true, + dirMove: true, + modTime: true, + setModTime: true, + }, +}); diff --git a/backend/s3/main.js b/backend/s3/main.js index b12771a..b7c6890 100644 --- a/backend/s3/main.js +++ b/backend/s3/main.js @@ -36,6 +36,19 @@ * the variable of rfetch, for example `RFETCH_S3_SECRET_ACCESS_KEY`, or of * rclone, `RCLONE_S3_SECRET_ACCESS_KEY`. Put the credentials in a header or * in `env`, not in a URL. + * + * As rclone, the backend keeps the time of a file in the metadata `mtime` + * of the object, from the header `X-OC-Mtime` of a `PUT` or a `COPY`. The + * `Last-Modified` of `GET` and `HEAD` is that time. The list of a folder + * gives the time of the upload: a list of S3 does not give the metadata. + * + * Other than GET, HEAD, PUT and DELETE, the backend knows: + * + * - `COPY` of a file to the path of the header `Destination`, with + * CopyObject: the bytes do not go through the backend. A file of more than + * 4768 MiB gives 501, as CopyObject cannot copy it in one request. + * - `PATCH` of a file with `X-OC-Mtime`: it sets the time of the file, with + * a CopyObject of the object onto itself, as rclone. */ import { createHash } from "node:crypto"; import { renderListing } from "@rfetch/lib/listing"; @@ -57,8 +70,18 @@ import { parseXML, textOf, } from "@rfetch/lib/xml"; +import { formatUnixTime, modTimeOf, parseUnixTime } from "@rfetch/lib/time"; import { EMPTY_PAYLOAD, sign, UNSIGNED_PAYLOAD, uriEncode } from "./sigv4.js"; +/** The name of the metadata of the time of a file, as rclone. */ +const MTIME = "x-amz-meta-mtime"; + +/** + * The size of the largest file that CopyObject copies, as rclone: less than + * 5 GiB for Backblaze B2. + */ +const MAX_COPY_SIZE = 4768 * 1024 * 1024; + export const options = { string: [ "access_key_id", @@ -144,6 +167,12 @@ async function s3(request, env = {}) { // A file is in a bucket. return new Response(null, { status: method === "PUT" ? 400 : 404 }); } + const modTime = modTimeOf(request); + if (modTime === undefined) { + return new Response("X-OC-Mtime is not a time", { status: 400 }); + } + /** The headers of an answer that sets the time of a file. */ + const accepted = modTime === null ? {} : { "X-OC-Mtime": "accepted" }; try { if (method === "GET" || method === "HEAD") { @@ -158,12 +187,37 @@ async function s3(request, env = {}) { } await (isFolder ? putFolder(config, bucket, key) - : putFile(config, bucket, key, request)); + : putFile(config, bucket, key, request, modTime)); return new Response(null, { status: 201, - headers: { "Content-Location": pathname }, + headers: { + "Content-Location": pathname, + ...(isFolder ? {} : accepted), + }, }); } + if (method === "COPY" && !isFolder) { + const target = decodePath(request.headers.get("Destination") ?? ""); + const end = target?.indexOf("/", 1) ?? -1; + if (target === null || end === -1 || end === target.length - 1) { + return new Response("Destination is not a file", { status: 400 }); + } + if (target.endsWith("/")) { + return new Response("Destination is a folder", { status: 409 }); + } + await copyFile(config, { bucket, key }, { + bucket: target.slice(1, end), + key: target.slice(end + 1), + }, modTime); + return new Response(null, { status: 201, headers: accepted }); + } + if (method === "PATCH" && !isFolder) { + if (modTime === null) { + return new Response("X-OC-Mtime is missing", { status: 400 }); + } + await copyFile(config, { bucket, key }, { bucket, key }, modTime); + return new Response(null, { status: 204, headers: accepted }); + } if (method === "DELETE") { if (bucket === "") { return new Response(null, { status: 405, headers: { Allow: "GET" } }); @@ -179,9 +233,10 @@ async function s3(request, env = {}) { } throw error; } + await request.body?.cancel(); return new Response(null, { status: 405, - headers: { Allow: "GET, HEAD, PUT, DELETE" }, + headers: { Allow: "GET, HEAD, PUT, DELETE, COPY, PATCH" }, }); } @@ -463,6 +518,15 @@ function fileHeaders(response, key) { const value = response.headers.get(name); if (value) headers.set(name, value); } + // The time of the file, as rclone: the metadata, else the time of the + // upload. + const modTime = parseUnixTime(response.headers.get(MTIME)); + if (modTime !== undefined) { + headers.set( + "Last-Modified", + new Date(Number(modTime / 1000000n)).toUTCString(), + ); + } return headers; } @@ -632,8 +696,10 @@ async function putMarkers(config, bucket, prefix) { * @param {string} bucket * @param {string} key * @param {Request} request + * @param {bigint | null} modTime The time of the file, in the metadata + * `mtime`, or null. */ -async function putFile(config, bucket, key, request) { +async function putFile(config, bucket, key, request, modTime) { if (config.markers) { await putMarkers(config, bucket, key.slice(0, key.lastIndexOf("/") + 1)); } @@ -641,9 +707,11 @@ async function putFile(config, bucket, key, request) { const length = header !== null && /^\d+$/.test(header) ? Number(header) : null; + /** @type {Record} */ const headers = { "Content-Type": request.headers.get("Content-Type") || mimeType(key), }; + if (modTime !== null) headers[MTIME] = formatUnixTime(modTime); if (length !== null && length <= config.uploadCutoff) { const body = sized(request.body ?? new Blob([]).stream(), length); @@ -732,6 +800,58 @@ async function putFile(config, bucket, key, request) { } } +/** + * Copies a file with CopyObject, as `Copy` of rclone. With a time, the copy + * gets the metadata of the source, with that time: a copy onto the same + * object sets the time of the file, as `SetModTime` of rclone. With no + * time, the copy keeps the metadata of the source. + * @param {Config} config + * @param {{ bucket: string, key: string }} source + * @param {{ bucket: string, key: string }} target + * @param {bigint | null} modTime + * @throws {S3Error} With 404 for a source that does not exist, and with 501 + * for a file that is too large for CopyObject. + */ +async function copyFile(config, source, target, modTime) { + const head = await send(config, "HEAD", source.bucket, source.key); + await head.body?.cancel(); + if (head.status === 404) throw new S3Error("Not Found", 404); + await check(head); + if (Number(head.headers.get("Content-Length")) > MAX_COPY_SIZE) { + throw new S3Error("The file is too large for CopyObject", 501); + } + if (config.markers) { + await putMarkers( + config, + target.bucket, + target.key.slice(0, target.key.lastIndexOf("/") + 1), + ); + } + /** @type {Record} */ + const headers = { + "x-amz-copy-source": `/${uriEncode(source.bucket)}/${ + uriEncode(source.key, true) + }`, + }; + if (modTime !== null) { + headers["x-amz-metadata-directive"] = "REPLACE"; + headers["Content-Type"] = head.headers.get("Content-Type") || + mimeType(target.key); + for (const [name, value] of head.headers) { + if (name.startsWith("x-amz-meta-")) headers[name] = value; + } + headers[MTIME] = formatUnixTime(modTime); + } + const response = await check( + await send(config, "PUT", target.bucket, target.key, { headers }), + ); + // S3 can give an error in an answer 200. + const text = await response.text(); + if (//.test(text)) { + throw new S3Error(textOf(parseXML(text), "Code"), 502); + } +} + /** * Deletes a file. As the other backends, a file that does not exist gives * 404: S3 gives no error for it. diff --git a/backend/s3/main_test.js b/backend/s3/main_test.js index 0938389..b48e43d 100644 --- a/backend/s3/main_test.js +++ b/backend/s3/main_test.js @@ -28,6 +28,12 @@ if (hasRclone) { params, parent: "/rfetch/", emptyFolders: false, + features: { + copy: true, + modTime: true, + setModTime: true, + listModTime: false, + }, }); // Files in parts: upload_cutoff of 100 KiB, and parts of 5 MiB. backendSuite("s3 in parts", s3, { @@ -35,6 +41,12 @@ if (hasRclone) { params: { ...params, upload_cutoff: "100k" }, parent: "/rfetch/", emptyFolders: false, + features: { + copy: true, + modTime: true, + setModTime: true, + listModTime: false, + }, }); describe("s3", () => { @@ -198,11 +210,23 @@ if (S3_TEST_ENDPOINT && S3_TEST_BUCKET) { env, parent, emptyFolders: false, + features: { + copy: true, + modTime: true, + setModTime: true, + listModTime: false, + }, }); backendSuite("s3 service with folder markers", s3, { type: "s3", params: { ...params, directory_markers: "true" }, env, parent, + features: { + copy: true, + modTime: true, + setModTime: true, + listModTime: false, + }, }); } diff --git a/backend/webdav/main.js b/backend/webdav/main.js index 36e665c..3fe8c55 100644 --- a/backend/webdav/main.js +++ b/backend/webdav/main.js @@ -44,6 +44,20 @@ * not exist, as rclone. PUT of a file makes its folder first, as rclone. * - DELETE goes to the server. The server deletes a folder with all that is * in it. + * - COPY of a file, and MOVE of a file or a folder, go to the server, with + * the URL of the destination and `Overwrite: T`, after the MKCOL of the + * folder of the destination, as rclone. A folder moves only to a path that + * does not exist. + * + * The time of a file, as rclone: a PUT with `X-OC-Mtime` sends that header, + * in whole seconds, to the vendors that know it: `fastmail`, `owncloud`, + * `nextcloud`, `infinitescale` and `rclone`. For `owncloud`, `nextcloud` + * and `infinitescale`, a COPY and a MOVE send it too, and when the server + * does not answer `X-OC-Mtime: accepted`, a PROPPATCH of `lastmodified` + * sets the time. A PATCH with `X-OC-Mtime` is that PROPPATCH. The other + * vendors cannot set the time of a file that exists: their PATCH gives 405. + * `rclone serve webdav` gives the time of a COPY to the source file, not to + * the copy: thus the backend does not send it there. * * The list has the times of the server, to the second. The type of a file * comes from its name, as in rclone. @@ -56,6 +70,7 @@ import { isTrue, optionsOf } from "@rfetch/lib/options"; import { decodePath, encodePath } from "@rfetch/lib/path"; import { isOneRange } from "@rfetch/lib/range"; import { sized } from "@rfetch/lib/sized"; +import { modTimeOf } from "@rfetch/lib/time"; import { childrenOf, parseXML, textOf } from "@rfetch/lib/xml"; export const options = { @@ -91,20 +106,43 @@ const SECRETS = [ ]; /** - * The quirks of each server, as `setQuirks` of rclone. `props` is the body - * of a PROPFIND, and `stream` tells if the server takes a body of unknown - * size. - * @type {Record} + * The quirks of a server, as `setQuirks` of rclone. + * @typedef {Object} Vendor + * @property {string} props The body of a PROPFIND. + * @property {boolean} stream The server takes a body of unknown size. + * @property {boolean} ocMtime The server sets the time of `X-OC-Mtime` on + * a PUT, as `useOCMtime` of rclone. + * @property {boolean} propset A PROPPATCH of `lastmodified` sets the time + * of a file, as `propsetMtime` of rclone. */ + +/** @type {Record} */ const VENDORS = { - fastmail: { props: "", stream: true }, - owncloud: { props: "owncloud", stream: true }, - infinitescale: { props: "owncloud", stream: false }, - nextcloud: { props: "owncloud", stream: false }, - rclone: { props: "", stream: true }, - other: { props: "standard", stream: false }, + fastmail: { props: "", stream: true, ocMtime: true, propset: false }, + owncloud: { props: "owncloud", stream: true, ocMtime: true, propset: true }, + infinitescale: { + props: "owncloud", + stream: false, + ocMtime: true, + propset: true, + }, + nextcloud: { props: "owncloud", stream: false, ocMtime: true, propset: true }, + rclone: { props: "", stream: true, ocMtime: true, propset: false }, + other: { props: "standard", stream: false, ocMtime: false, propset: false }, }; +/** The body of a PROPPATCH that sets the time of a file, as rclone. */ +const PROPSET = (seconds) => + ` + + + + ${seconds} + + + +`; + /** The bodies of PROPFIND, as rclone. With no body, the server gives all. */ const PROPS = { owncloud: ` @@ -166,7 +204,7 @@ class WebDAVError extends Error { * authentication, the option `headers` and `Referer`. * @property {string[]} secrets The names of the headers that do not go to * a server on a different host. - * @property {{ props: string, stream: boolean }} vendor + * @property {Vendor} vendor * @property {boolean} authRedirect * @property {boolean} excludeShares * @property {boolean} excludeMounts @@ -203,6 +241,11 @@ async function webdav(request, env = {}) { } const url = new URL(encodePath(path).slice(1), config.base); const isFolder = path.endsWith("/"); + const modTime = modTimeOf(request); + if (modTime === undefined) { + await request.body?.cancel(); + return new Response("X-OC-Mtime is not a time", { status: 400 }); + } try { if (method === "GET" || method === "HEAD") { @@ -212,25 +255,45 @@ async function webdav(request, env = {}) { return await getFolder(config, url, method, `${query}`); } if (method === "PUT") { + /** @type {Record} */ + const headers = { "Content-Location": pathname }; if (isFolder) { await request.body?.cancel(); await mkdir(config, url); - } else { - await putFile(config, url, path, request); + } else if (await putFile(config, url, path, request, modTime)) { + headers["X-OC-Mtime"] = "accepted"; } - return new Response(null, { - status: 201, - headers: { "Content-Location": pathname }, - }); + return new Response(null, { status: 201, headers }); } if (method === "DELETE") { await remove(config, url); return new Response(null, { status: 204 }); } + if (method === "COPY" || method === "MOVE") { + await request.body?.cancel(); + const target = decodePath(request.headers.get("Destination") ?? ""); + if (target === null) { + return new Response("Destination is not a path", { status: 400 }); + } + return await transfer(config, method, url, target, modTime); + } + if (method === "PATCH" && !isFolder && config.vendor.propset) { + await request.body?.cancel(); + if (modTime === null) { + return new Response("X-OC-Mtime is missing", { status: 400 }); + } + if (!await proppatch(config, url, modTime)) { + return new Response("can't set modified time", { status: 501 }); + } + return new Response(null, { + status: 204, + headers: { "X-OC-Mtime": "accepted" }, + }); + } await request.body?.cancel(); return new Response(null, { status: 405, - headers: { Allow: "GET, HEAD, PUT, DELETE" }, + headers: { Allow: "GET, HEAD, PUT, DELETE, COPY, MOVE" }, }); } catch (error) { await request.body?.cancel().catch(() => {}); @@ -646,8 +709,10 @@ async function mkdir(config, url) { * @param {URL} url * @param {string} path * @param {Request} request + * @param {bigint | null} modTime The time of the file, or null. + * @returns {Promise} True when the file has the time. */ -async function putFile(config, url, path, request) { +async function putFile(config, url, path, request, modTime) { await mkdir(config, new URL(".", url)); const header = request.headers.get("Content-Length"); /** @type {Record} */ @@ -655,6 +720,8 @@ async function putFile(config, url, path, request) { "Content-Type": request.headers.get("Content-Type") || mimeType(path.slice(path.lastIndexOf("/") + 1)), }; + const timed = modTime !== null && config.vendor.ocMtime; + if (timed) headers["X-OC-Mtime"] = `${secondsOf(modTime)}`; /** @type {BodyInit} */ let body; if (header !== null && /^\d+$/.test(header)) { @@ -669,6 +736,134 @@ async function putFile(config, url, path, request) { const { response } = await send(config, "PUT", url, { headers, body }); if (!response.ok) await failed(response); await response.body?.cancel(); + if (!timed) return false; + // As rclone: a server of a vendor with no PROPPATCH sets the time. + if ( + response.headers.get("X-OC-Mtime") === "accepted" || !config.vendor.propset + ) { + return true; + } + return await proppatch(config, url, modTime); +} + +/** + * Copies a file, or moves a file or a folder, on the server, as `Copy`, + * `Move` and `DirMove` of rclone. + * @param {Config} config + * @param {"COPY" | "MOVE"} method + * @param {URL} url The URL of the source. + * @param {string} target The path of the destination. + * @param {bigint | null} modTime The time of the file, or null. + * @returns {Promise} + */ +async function transfer(config, method, url, target, modTime) { + const destination = new URL(encodePath(target).slice(1), config.base); + const isFolder = url.pathname.endsWith("/"); + if (isFolder) { + if (method === "COPY") { + return new Response("COPY of a folder is not implemented", { + status: 501, + }); + } + if (!target.endsWith("/") || destination.href.startsWith(url.href)) { + return new Response("Destination is not a new folder", { status: 400 }); + } + // As rclone: a folder does not move over a folder. + if (await exists(config, destination)) { + return new Response("Destination exists", { status: 412 }); + } + await mkdir(config, new URL("..", destination)); + } else { + if (target.endsWith("/")) { + return new Response("Destination is a folder", { status: 409 }); + } + await mkdir(config, new URL(".", destination)); + } + /** @type {Record} */ + const headers = { Destination: destination.href, Overwrite: "T" }; + const timed = modTime !== null && !isFolder && config.vendor.propset; + if (timed) headers["X-OC-Mtime"] = `${secondsOf(modTime)}`; + const { response } = await send(config, method, url, { headers }); + if (!response.ok) { + // A server can give another error for a source that does not exist: + // `rclone serve webdav` gives 403 to a MOVE. + if (response.status !== 404 && !await exists(config, url)) { + await response.body?.cancel(); + throw new WebDAVError("Not Found", 404); + } + await failed(response); + } + await response.body?.cancel(); + const status = isFolder ? 201 : response.status; + if ( + timed && (response.headers.get("X-OC-Mtime") === "accepted" || + await proppatch(config, destination, modTime)) + ) { + return new Response(null, { + status, + headers: { "X-OC-Mtime": "accepted" }, + }); + } + return new Response(null, { status }); +} + +/** + * Tells if a file or a folder exists, with a PROPFIND with `Depth: 0`. + * @param {Config} config + * @param {URL} url + * @returns {Promise} + */ +async function exists(config, url) { + return await propfind(config, url, "0").then( + (items) => items.length > 0, + (error) => { + if (error instanceof WebDAVError && error.status === 404) return false; + throw error; + }, + ); +} + +/** + * Sets the time of a file with a PROPPATCH of `lastmodified`, as + * `SetModTime` of rclone for ownCloud and Nextcloud. + * @param {Config} config + * @param {URL} url + * @param {bigint} modTime + * @returns {Promise} True when the server set the time. + * @throws {WebDAVError} With 404 for a file that does not exist. + */ +async function proppatch(config, url, modTime) { + const { response } = await send(config, "PROPPATCH", url, { + headers: { "Content-Type": "application/xml; charset=utf-8" }, + body: PROPSET(secondsOf(modTime)), + }); + if (response.status === 404) await failed(response); + if (response.status !== 207) { + await response.body?.cancel(); + return false; + } + try { + const root = parseXML(await response.text()); + const statuses = childrenOf(root, "response") + .flatMap((item) => childrenOf(item, "propstat")) + .map((propstat) => textOf(propstat, "status")); + return statuses.length > 0 && + statuses.every((status) => /^HTTP\/[\d.]+\s+2\d\d/.test(status)); + } catch { + return false; + } +} + +/** + * The whole seconds of a time, as `Unix` of Go. + * @param {bigint} nanoseconds + */ +function secondsOf(nanoseconds) { + const second = 1_000_000_000n; + const seconds = nanoseconds / second; + return nanoseconds < 0n && nanoseconds % second !== 0n + ? seconds - 1n + : seconds; } /** diff --git a/backend/webdav/main_test.js b/backend/webdav/main_test.js index b98cb27..87335a6 100644 --- a/backend/webdav/main_test.js +++ b/backend/webdav/main_test.js @@ -29,11 +29,20 @@ if (hasRclone) { backendSuite("webdav", webdav, { type: "webdav", params: { url: server.url, vendor: "rclone", user: LOGIN.user, pass }, + // `rclone serve webdav` gives the time of a COPY to the source file. + features: { + copy: true, + move: true, + dirMove: true, + modTime: true, + copyModTime: false, + }, }); // The vendor `other` sends the standard properties in each PROPFIND, and // a body of unknown size in one piece. backendSuite("webdav other", webdav, { params: { url: server.url, user: LOGIN.user, pass }, + features: { copy: true, move: true, dirMove: true }, }); describe("webdav on rclone serve webdav", () => { diff --git a/cmd/copy/main.js b/cmd/copy/main.js index 79d53bd..40354ae 100644 --- a/cmd/copy/main.js +++ b/cmd/copy/main.js @@ -13,9 +13,11 @@ * - `--dry-run`: copy nothing, and tell what it would copy. * - `--transfers N`: copy N files at the same time, 4 by default. * - * The backends do not keep the time of a source file yet: the time of a - * copy is the time of the copy. Thus a second copy sends the files again, - * unless `--size-only`. + * A copy gets the time of its source, with `X-OC-Mtime`, when the backend + * of the target can keep it. In the same remote, the backend copies or + * moves each file with COPY or MOVE, and moves a folder with MOVE to a path + * that does not exist, as `DirMove` of rclone. Else, the bytes go through + * rfetch. */ import { isTrue } from "@rfetch/lib/options"; @@ -25,8 +27,10 @@ import { failure, folderOf, join, + lastModified, log, mkdir, + moveFolder, nameOf, numberOf, OperationError, @@ -119,6 +123,14 @@ export async function transfer(source, target, flags, mode = {}) { return; } + // A folder moves with one MOVE, when the target does not exist. + if ( + mode.move && !isTrue(flags.dry_run) && !await stat(target, flags) && + await moveFolder(source, target, flags) + ) { + return; + } + /** @type {File[]} */ const files = []; /** @type {string[]} */ @@ -200,8 +212,21 @@ export async function transfer(source, target, flags, mode = {}) { */ async function transferFile(file, target, existing, flags, mode) { const verb = mode.move ? "move" : "copy"; - const same = existing && (isTrue(flags.ignore_existing) || + let same = existing && (isTrue(flags.ignore_existing) || (!isTrue(flags.ignore_times) && isSame(file, existing, flags))); + if ( + existing && !same && !isTrue(flags.ignore_times) && + file.size === existing.size + ) { + // A backend can keep the time of a file out of its list, as s3 in the + // metadata of the file: then HEAD gives another time than the list, to + // the second. + const time = await lastModified(target, flags); + const second = (value) => Math.floor(Date.parse(value) / 1000); + if (time && second(time) !== second(existing.modTime)) { + same = isSame(file, { size: existing.size, modTime: time }, flags); + } + } if (isTrue(flags.dry_run)) { if (!same || mode.move) { log( @@ -213,17 +238,24 @@ async function transferFile(file, target, existing, flags, mode) { } return; } - if (!same) await copyFile(file.location, target, flags); - // As rclone: a move deletes the source, also when the target has the same - // file. - if (mode.move) await remove(file.location, flags); + if (!same) { + await copyFile(file.location, target, flags, { + modTime: file.modTime, + move: mode.move, + }); + } else if (mode.move) { + // As rclone: a move deletes the source, also when the target has the + // same file. + await remove(file.location, flags); + } } /** * Tells if two files are the same, as `Equal` of rclone: the same size, and * the same time in the precision of the two remotes. A time with no part of - * a second comes from a remote with a precision of 1 s, as WebDAV: then the - * times can differ by less than 1 s. Else they can differ by less than 1 ms. + * a second comes from a remote with a precision of 1 s, as WebDAV and + * `Last-Modified`: then the times can differ by less than 1 s. Else they can + * differ by less than 1 ms. * A time that a backend does not give is not compared. * @param {{ size: number, modTime: string }} a * @param {{ size: number, modTime: string }} b diff --git a/cmd/move/main.js b/cmd/move/main.js index 5fc0cdb..0a19b0c 100644 --- a/cmd/move/main.js +++ b/cmd/move/main.js @@ -2,11 +2,11 @@ import { failure } from "../operations.js"; import { transfer } from "../copy/main.js"; /** - * Move files from source to dest, as `rclone move`: the copy of `copy`, - * then the delete of each source file. The folders of the source stay, - * unless `--delete-empty-src-dirs`. - * - * The backends cannot move a file yet: each file goes through rfetch. + * Move files from source to dest, as `rclone move`. In the same remote, a + * folder moves with one MOVE when dest does not exist, and else each file + * moves with MOVE, as rclone. Else each file goes as `copy`, then the source + * file goes away. The folders of the source stay, unless + * `--delete-empty-src-dirs`. * * ```js * import { move } from "./main.js"; diff --git a/cmd/move/main_test.js b/cmd/move/main_test.js index df09ca7..3f82bbf 100644 --- a/cmd/move/main_test.js +++ b/cmd/move/main_test.js @@ -14,21 +14,44 @@ describe("move and moveto", () => { await putFiles(SOURCE, { "f.txt": "hello", "a/g.txt": "world!" }); }); - test("move copies the files, and deletes them in the source", async () => { + test("move moves the folder when the target does not exist", async () => { assert.equal((await move(SOURCE, TARGET)).status, 200); assert.deepEqual((await contents(TARGET)).files, { "f.txt": "hello", "a/g.txt": "world!", }); + // As DirMove of rclone: the folder of the source goes too. + assert.equal(await contents(SOURCE), null); + }); + + test("move moves each file into a target that exists", async () => { + await putFiles(TARGET, { "old.txt": "old" }); + assert.equal((await move(SOURCE, TARGET)).status, 200); + assert.deepEqual((await contents(TARGET)).files, { + "old.txt": "old", + "f.txt": "hello", + "a/g.txt": "world!", + }); // The folders of the source stay, as rclone. assert.deepEqual(await contents(SOURCE), { files: {}, folders: ["a"] }); }); test("move --delete-empty-src-dirs deletes the folders too", async () => { + await putFiles(TARGET, { "old.txt": "old" }); await move(SOURCE, TARGET, { delete_empty_src_dirs: true }); assert.deepEqual(await contents(SOURCE), { files: {}, folders: [] }); }); + test("move copies the bytes to another remote", async () => { + const flags = { alias_remote: ":memory:/move-other" }; + assert.equal((await move(SOURCE, ":alias:target/", flags)).status, 200); + assert.deepEqual((await contents(":memory:/move-other/target/")).files, { + "f.txt": "hello", + "a/g.txt": "world!", + }); + assert.deepEqual(await contents(SOURCE), { files: {}, folders: ["a"] }); + }); + test("moveto renames a file", async () => { assert.equal( (await moveto(`${SOURCE}f.txt`, `${SOURCE}renamed.txt`)).status, diff --git a/cmd/operations.js b/cmd/operations.js index f4670bc..67dd131 100644 --- a/cmd/operations.js +++ b/cmd/operations.js @@ -10,6 +10,7 @@ import process from "node:process"; import { isRemoteRoot, parse } from "@rfetch/lib/fspath"; import { readListing } from "@rfetch/lib/listing"; import { encodePath } from "@rfetch/lib/path"; +import { formatUnixTime, nanosecondsOf } from "@rfetch/lib/time"; import { fetch } from "../fetch.js"; import { listItems } from "./lsjson/main.js"; @@ -288,13 +289,92 @@ export function numberOf(value, fallback) { } /** - * Copies a file: it reads the source, and puts it at the target. + * The statuses of a COPY or a MOVE that the backend cannot do: then the + * bytes go through rfetch. 502 is for a destination in another remote. + */ +const CANNOT = new Set([403, 405, 501, 502]); + +/** + * Tells if a location is a URL, which goes to a server and not to a backend. + * @param {string} location + */ +function isURL(location) { + return /^[a-z][a-z0-9+.-]*:\/\//i.test(location); +} + +/** + * The value of the header `X-OC-Mtime` for a time. + * @param {string | undefined} modTime A time in ISO 8601. + * @returns {Record} The header, or no header for no time. + */ +function timeHeader(modTime) { + const nanoseconds = nanosecondsOf(modTime); + return nanoseconds === undefined + ? {} + : { "X-OC-Mtime": formatUnixTime(nanoseconds) }; +} + +/** + * Copies or moves a file, and gives it the time of the source, as `Copy` + * and `Move` of the operations of rclone. When the source and the target + * are in the same remote, the backend copies or moves the file with COPY or + * MOVE, and the bytes do not go through rfetch. Else, and for a backend that + * cannot do it, rfetch reads the source and puts it at the target, then + * deletes the source for a move. * @param {string} source * @param {string} target * @param {Object} [flags] + * @param {{ modTime?: string, move?: boolean }} [options] `modTime` is the + * time of the source, in ISO 8601. + * @throws {OperationError} When the copy or the move fails. + */ +export async function copyFile( + source, + target, + flags = {}, + { modTime, move = false } = {}, +) { + const time = timeHeader(modTime); + if (!isURL(source) && !isURL(target)) { + for (const method of move ? ["MOVE", "COPY"] : ["COPY"]) { + const response = await fetch(withFlags(source, flags), { + method, + headers: { Destination: withFlags(target, flags), ...time }, + }); + if (response.ok) { + await response.body?.cancel(); + if (method === "COPY" && move) await remove(source, flags); + return; + } + if (!CANNOT.has(response.status)) { + const text = (await response.text()).trim(); + throw new OperationError( + response.status === 404 + ? `${source}: not found` + : `${target}: ${ + text || `${response.status} ${response.statusText}`.trim() + }`, + response.status, + ); + } + await response.body?.cancel(); + // Another remote: a COPY gives the same answer. + if (response.status === 502) break; + } + } + await putFile(source, target, flags, time); + if (move) await remove(source, flags); +} + +/** + * Reads a file, and puts it at the target. + * @param {string} source + * @param {string} target + * @param {Object} flags + * @param {Record} time The header of the time of the file. * @throws {OperationError} When the read or the write fails. */ -export async function copyFile(source, target, flags = {}) { +async function putFile(source, target, flags, time) { const response = await fetch(withFlags(source, flags)); if (!response.ok) { await response.body?.cancel(); @@ -304,7 +384,7 @@ export async function copyFile(source, target, flags = {}) { ); } /** @type {Record} */ - const headers = {}; + const headers = { ...time }; for (const name of ["Content-Type", "Content-Length"]) { const value = response.headers.get(name); if (value) headers[name] = value; @@ -324,6 +404,68 @@ export async function copyFile(source, target, flags = {}) { await put.body?.cancel(); } +/** + * Moves a folder with all that is in it to a path that does not exist, with + * a MOVE of the folder in the backend, as `DirMove` of rclone. + * @param {string} source + * @param {string} target + * @param {Object} [flags] + * @returns {Promise} False when the backend cannot move the folder: + * then the caller moves each file. + * @throws {OperationError} When the move fails. + */ +export async function moveFolder(source, target, flags = {}) { + if (isURL(source) || isURL(target)) return false; + const response = await fetch(withFlags(folderOf(source), flags), { + method: "MOVE", + headers: { Destination: withFlags(folderOf(target), flags) }, + }); + await response.body?.cancel(); + if (response.ok) return true; + // 412: the target exists. + if (CANNOT.has(response.status) || response.status === 412) return false; + throw new OperationError( + `${source}: ${response.status} ${response.statusText}`.trim(), + response.status, + ); +} + +/** + * Sets the time of a file with PATCH, as `SetModTime` of rclone. + * @param {string} location + * @param {bigint} nanoseconds The time, in nanoseconds after 1970. + * @param {Object} [flags] + * @returns {Promise} False when the backend cannot set the time. + * @throws {OperationError} When the remote gives another error. + */ +export async function setModTime(location, nanoseconds, flags = {}) { + const response = await fetch(withFlags(location, flags), { + method: "PATCH", + headers: { "X-OC-Mtime": formatUnixTime(nanoseconds) }, + }); + await response.body?.cancel(); + if (response.ok) return true; + if (CANNOT.has(response.status)) return false; + throw new OperationError( + response.status === 404 + ? `${location}: not found` + : `${location}: ${response.status} ${response.statusText}`.trim(), + response.status, + ); +} + +/** + * Gives the `Last-Modified` of a file, from HEAD. + * @param {string} location + * @param {Object} [flags] + * @returns {Promise} + */ +export async function lastModified(location, flags = {}) { + const response = await fetch(withFlags(location, flags), { method: "HEAD" }); + await response.body?.cancel(); + return response.ok ? response.headers.get("Last-Modified") : null; +} + /** * Deletes a file, or a folder with all that is in it. * @param {string} location For a folder, with `/` at the end. diff --git a/cmd/rclone_test.js b/cmd/rclone_test.js index 57ac262..5294753 100644 --- a/cmd/rclone_test.js +++ b/cmd/rclone_test.js @@ -15,6 +15,7 @@ import { readFileSync, rmSync, statSync, + utimesSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; @@ -211,7 +212,10 @@ describe("the commands and the commands of rclone", { ) { const folder = join(home, name); rmSync(folder, { recursive: true, force: true }); - cpSync(source, join(folder, "source"), { recursive: true }); + cpSync(source, join(folder, "source"), { + recursive: true, + preserveTimestamps: true, + }); const { status } = tool(...args(name)); results.push({ status, tree: treeOf(folder) }); } @@ -236,24 +240,81 @@ describe("the commands and the commands of rclone", { sameEffect((f) => ["copy", `${f}/source`, `${f}/target`]); sameEffect((f) => ["copy", `${f}/source/f.txt`, `${f}/target`]); sameEffect((f) => ["copyto", `${f}/source/f.txt`, `${f}/target/new.txt`]); - // rclone moves a folder of a local remote with one rename. rfetch moves - // each file, as rclone does with no DirMove. - const noDirMove = ["--disable", "DirMove"]; - sameEffect((f) => ["move", `${f}/source`, `${f}/target`], noDirMove); - sameEffect((f) => [ - "move", - "--delete-empty-src-dirs", - `${f}/source`, - `${f}/target`, - ], noDirMove); + // A folder moves with one rename when the target does not exist, as + // DirMove of rclone. Else each file moves. + sameEffect((f) => ["move", `${f}/source`, `${f}/target`]); + sameEffect((f) => ["move", `${f}/source/a`, `${f}/source/empty`]); sameEffect((f) => [ "move", "--delete-empty-src-dirs", `${f}/source/a`, - `${f}/target`, - ], noDirMove); + `${f}/source/empty`, + ]); sameEffect((f) => ["moveto", `${f}/source/f.txt`, `${f}/source/g.txt`]); sameEffect((f) => ["mkdir", `${f}/source/x/y`]); sameEffect((f) => ["touch", `${f}/source/new.txt`]); }); + + test("a copy keeps the time of each file, and a second copy skips them", () => { + for (const [name, tool] of [["by-rclone", rclone], ["by-rfetch", rfetch]]) { + const folder = join(home, name); + rmSync(folder, { recursive: true, force: true }); + cpSync(source, join(folder, "source"), { + recursive: true, + preserveTimestamps: true, + }); + assert.equal(tool("copy", `${name}/source`, `${name}/target`).status, 0); + } + // The times of the copies of rfetch are the times of the copies of + // rclone, to the microsecond: Node.js sets a time to the microsecond. + const times = (name) => + JSON.parse(rclone("lsjson", "-R", `${name}/target`).stdout) + .filter((item) => !item.IsDir) + .map(({ Path, ModTime }) => [Path, ModTime.slice(0, 26)]) + .sort(); + assert.deepEqual(times("by-rfetch"), times("by-rclone")); + // rclone sees no difference, and a second copy of rfetch writes no file. + assert.equal( + rclone("check", "by-rclone/target", "by-rfetch/target").status, + 0, + ); + // A file of the same size and time, with other bytes: a copy that + // compares the sizes and the times does not write it. + const marker = join(home, "by-rfetch", "target", "f.txt"); + const { mtimeNs } = statSync(marker, { bigint: true }); + writeFileSync(marker, "HELLO"); + const seconds = (Number(mtimeNs / 1000n) + 0.5) / 1e6; + utimesSync(marker, seconds, seconds); + assert.equal( + rfetch("copy", "by-rfetch/source", "by-rfetch/target").status, + 0, + ); + assert.equal( + readFileSync(marker, "utf8"), + "HELLO", + "the second copy wrote f.txt", + ); + }); + + test("touch sets the time of a file", () => { + sameEffect(( + f, + ) => ["touch", "--timestamp", "2020-01-02T03:04:05", `${f}/source/f.txt`]); + const times = (name) => + JSON.parse(rclone("lsjson", "-R", `${name}/source`).stdout) + .filter((item) => !item.IsDir) + .map(({ Path, ModTime }) => [Path, ModTime]) + .sort(); + assert.deepEqual(times("by-rfetch"), times("by-rclone")); + // A short time is YYMMDD, as rclone. + sameEffect((f) => ["touch", "--timestamp", "200102", `${f}/source/a`]); + assert.deepEqual(times("by-rfetch"), times("by-rclone")); + sameEffect(( + f, + ) => ["touch", "-R", "--timestamp", "210102", `${f}/source`]); + assert.deepEqual(times("by-rfetch"), times("by-rclone")); + sameEffect(( + f, + ) => ["touch", "--timestamp", "2020-01-02", `${f}/source/f.txt`]); + }); }); diff --git a/cmd/serve/webdav/handler.js b/cmd/serve/webdav/handler.js index de564c9..bf4a415 100644 --- a/cmd/serve/webdav/handler.js +++ b/cmd/serve/webdav/handler.js @@ -6,10 +6,17 @@ * - PROPFIND reads the headers of HEAD, and the list of a folder for * `Depth: 1`. `Depth: infinity` gets 403, as RFC 4918 permits: a list of * all the files can be too large. - * - MKCOL is PUT of a folder. COPY and MOVE read each file and put it at the - * destination, and MOVE deletes the source after. + * - MKCOL is PUT of a folder. COPY and MOVE of a file, and MOVE of a folder + * to a path that does not exist, go to the backend. For a backend that + * cannot do it, or a destination in another remote, the server reads each + * file and puts it at the destination, and MOVE deletes the source after. + * - A PUT, a COPY or a MOVE of a file with `X-OC-Mtime` gives that time to + * the file, and the answer has `X-OC-Mtime: accepted` when the backend + * keeps it. A PROPPATCH of `lastmodified`, with the seconds after 1970, is + * a PATCH of the backend, as ownCloud: rclone sets the time so for the + * vendors `owncloud` and `nextcloud`. * - The server keeps no data between requests. Thus a LOCK gives a token - * and locks nothing, and PROPPATCH keeps no property, as + * and locks nothing, and PROPPATCH keeps no other property, as * `rclone serve webdav`. The clients of Windows and macOS need LOCK to * write. */ @@ -42,6 +49,13 @@ const MAX_XML = 1024 * 1024; /** The status codes of the backends for a path that is not there. */ const MISSING = [404, 405, 409, 410]; +/** + * The status codes of a COPY or a MOVE that the backend cannot do: then the + * bytes go through the server. 502 is for a destination in another remote, + * and 412 for a folder that exists. + */ +const CANNOT = [403, 405, 412, 501, 502]; + /** The headers of one connection, which do not go to the remote. */ const HOP = [ "connection", @@ -229,7 +243,10 @@ async function pass(request, server, target) { }); if (method === "PUT") { await response.body?.cancel(); - return new Response(null, { status: response.ok ? 201 : response.status }); + return new Response(null, { + status: response.ok ? 201 : response.status, + headers: accepted(response), + }); } // The links of a list have the options of the backend in their query. const type = response.headers.get("Content-Type") || ""; @@ -242,6 +259,18 @@ async function pass(request, server, target) { return response; } +/** + * The header `X-OC-Mtime: accepted` of an answer of the backend, for the + * answer to the client. + * @param {Response} response + * @returns {Record} + */ +function accepted(response) { + return response.ok && response.headers.get("X-OC-Mtime") === "accepted" + ? { "X-OC-Mtime": "accepted" } + : {}; +} + /** * Deletes a file, or a folder with all that is in it. A path with no `/` at * the end can be a folder: a client does not always put the `/`. @@ -551,10 +580,41 @@ async function proppatch(request, server, target) { if (!info) throw new DAVError(404); const properties = [...childrenOf(body, "set"), ...childrenOf(body, "remove")] .flatMap((change) => childrenOf(change, "prop").flatMap(namesOf)); + // The time of a file, as ownCloud: `` with the + // seconds after 1970. + const time = childrenOf(body, "set") + .flatMap((change) => childrenOf(change, "prop")) + .flatMap((prop) => childrenOf(prop, "lastmodified")) + .find((element) => element.namespace === "DAV:"); + /** @type {typeof properties} */ + let failed = []; + if (time) { + const seconds = (time.text ?? "").trim(); + let ok = false; + if (!info.isDir && /^-?\d{1,15}$/.test(seconds)) { + const response = await send(server, "PATCH", info.href, { + headers: { "X-OC-Mtime": seconds }, + }); + await response.body?.cancel(); + ok = response.ok; + } + if (!ok) { + failed = properties.filter(({ name, namespace }) => + name === "lastmodified" && namespace === "DAV:" + ); + } + } + const done = properties.filter((property) => !failed.includes(property)); + /** @param {typeof properties} list @param {string} status */ + const propstat = (list, status) => + list.length + ? `${list.map(emptyElement).join("")}` + + `HTTP/1.1 ${status}` + : ""; return multistatus( `${escapeXML(info.href)}` + - `${properties.map(emptyElement).join("")}` + - `HTTP/1.1 200 OK`, + propstat(done, "200 OK") + propstat(failed, "403 Forbidden") + + ``, ); } @@ -633,8 +693,10 @@ async function hasBody(request) { } /** - * Answers COPY and MOVE. Each file goes through the server: the server - * reads it from the source, and puts it at the destination. + * Answers COPY and MOVE. The backend copies or moves a file, and moves a + * folder to a path that does not exist. Else each file goes through the + * server: the server reads it from the source, and puts it at the + * destination. * @param {Request} request * @param {Server} server * @param {string} target @@ -694,13 +756,43 @@ async function copy(request, server, target) { throw new DAVError(409, "The parent folder does not exist"); } const existing = await stat(server, to.replace(/\/$/, "")); - if (existing) { - if (overwrite === "F") throw new DAVError(412, "The destination exists"); - await check(send(server, "DELETE", existing.href)); - } - await copyTree(server, source.href, to, source.isDir, depth === "0"); + if (existing && overwrite === "F") { + throw new DAVError(412, "The destination exists"); + } + // The time of the file at the destination, as ownCloud. rclone gives it + // to the source: that is not correct. + const time = request.headers.get("X-OC-Mtime"); + /** @type {Record} */ + const timed = time && !source.isDir ? { "X-OC-Mtime": time } : {}; + if ( + (source.isDir ? method === "MOVE" && !existing : !existing?.isDir) + ) { + const response = await send(server, method, source.href, { + headers: { Destination: server.locate(to), ...timed }, + }); + await response.body?.cancel(); + if (response.ok) { + return new Response(null, { + status: existing ? 204 : 201, + headers: accepted(response), + }); + } + if (!CANNOT.includes(response.status)) throw new DAVError(response.status); + } + if (existing) await check(send(server, "DELETE", existing.href)); + const put = await copyTree( + server, + source.href, + to, + source.isDir, + depth === "0", + timed, + ); if (method === "MOVE") await check(send(server, "DELETE", source.href)); - return new Response(null, { status: existing ? 204 : 201 }); + return new Response(null, { + status: existing ? 204 : 201, + headers: put ? accepted(put) : {}, + }); } /** @@ -710,8 +802,11 @@ async function copy(request, server, target) { * @param {string} to * @param {boolean} isDir * @param {boolean} [only] For a folder, copy only the folder. + * @param {Record} [timed] For a file, the header + * `X-OC-Mtime` of its PUT. + * @returns {Promise} For a file, the answer of its PUT. */ -async function copyTree(server, from, to, isDir, only = false) { +async function copyTree(server, from, to, isDir, only = false, timed = {}) { if (!isDir) { const response = await send(server, "GET", from); if (!response.ok) { @@ -719,13 +814,18 @@ async function copyTree(server, from, to, isDir, only = false) { throw new DAVError(response.status); } /** @type {Record} */ - const headers = {}; + const headers = { ...timed }; for (const name of ["Content-Type", "Content-Length"]) { const value = response.headers.get(name); if (value) headers[name] = value; } - await check(send(server, "PUT", to, { headers, body: response.body })); - return; + const put = await send(server, "PUT", to, { + headers, + body: response.body, + }); + await put.body?.cancel(); + if (!put.ok) throw new DAVError(put.status); + return put; } await check(send(server, "PUT", to)); if (only) return; diff --git a/cmd/serve/webdav/main_test.js b/cmd/serve/webdav/main_test.js index 6124494..b9778a8 100644 --- a/cmd/serve/webdav/main_test.js +++ b/cmd/serve/webdav/main_test.js @@ -79,9 +79,30 @@ after(() => server.stop()); backendSuite("serve webdav", webdav, { type: "webdav", params: { url: server.url, vendor: "rclone" }, + // The vendor rclone sends the time on PUT only. + features: { + copy: true, + move: true, + dirMove: true, + modTime: true, + copyModTime: false, + }, }); backendSuite("serve webdav, vendor other", webdav, { params: { url: server.url }, + features: { copy: true, move: true, dirMove: true }, +}); +// The vendor owncloud sends the time on PUT, COPY and MOVE, and sets it with +// PROPPATCH. +backendSuite("serve webdav, vendor owncloud", webdav, { + params: { url: server.url, vendor: "owncloud" }, + features: { + copy: true, + move: true, + dirMove: true, + modTime: true, + setModTime: true, + }, }); describe("serve webdav", () => { diff --git a/cmd/touch/main.js b/cmd/touch/main.js index 7c55088..c203934 100644 --- a/cmd/touch/main.js +++ b/cmd/touch/main.js @@ -1,38 +1,43 @@ import { isTrue } from "@rfetch/lib/options"; +import { formatUnixTime } from "@rfetch/lib/time"; import { fetch } from "../../fetch.js"; import { failure, log, OperationError, + setModTime, stat, walk, withFlags, } from "../operations.js"; -/** The formats of `--timestamp`, as rclone. */ -const TIMESTAMP = /^\d{4}-\d\d-\d\d(T\d\d:\d\d:\d\d(\.\d{1,9})?)?$/; +/** + * The formats of `--timestamp`, as rclone: the format comes from the length + * of the text. + */ +const SHORT = /^(\d\d)(\d\d)(\d\d)$/; +const LONG = /^(\d{4})-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)(?:\.(\d{1,9}))?$/; /** * Create new file or change file modification time, as `rclone touch`. * - * A file that does not exist gets 0 bytes, unless `--no-create` or - * `--recursive`. The backends cannot set the time of a file yet: thus a file - * that exists, and `--timestamp`, give the error of rclone for a remote that - * cannot set the time. + * The time is now, or the time of `--timestamp`: `YYMMDD`, for example + * `060102`, `2006-01-02T15:04:05` or `2006-01-02T15:04:05.123456789`, in + * UTC, or in the local time with `--localtime`. + * + * A file that does not exist gets 0 bytes and the time, unless `--no-create` + * or `--recursive`. A file that exists gets the time with PATCH. For a + * folder, each file in it gets the time, and with `--recursive`, each file + * in its folders too. A backend that cannot set the time of a file gives the + * error of rclone. * @param {string} location * @param {Object} [flags] * @returns {Promise} */ export async function touch(location, flags = {}) { try { - const timestamp = flags.timestamp; - if (timestamp !== undefined && !TIMESTAMP.test(`${timestamp}`)) { - throw new OperationError( - `failed to parse timestamp argument: ${timestamp}`, - 400, - ); - } + const time = timeOf(flags); const info = await stat(location, flags); if (!info) { if (isTrue(flags.no_create)) { @@ -53,12 +58,6 @@ export async function touch(location, flags = {}) { ); return new Response(null); } - if (timestamp !== undefined) { - throw new OperationError( - "failed to touch (create): can't set modified time", - 501, - ); - } if (isTrue(flags.dry_run)) { log( "NOTICE", @@ -68,9 +67,13 @@ export async function touch(location, flags = {}) { ); return new Response(null); } + // As rclone, a backend that cannot keep the time keeps the file. const response = await fetch(withFlags(location, flags), { method: "PUT", - headers: { "Content-Length": "0" }, + headers: { + "Content-Length": "0", + "X-OC-Mtime": formatUnixTime(time), + }, body: new Uint8Array(0), }); await response.body?.cancel(); @@ -83,22 +86,120 @@ export async function touch(location, flags = {}) { } return new Response(null); } + if (!info.isDir) { + if (isTrue(flags.dry_run)) { + log("NOTICE", location, "Skipped touch as --dry-run is set", flags); + } else if (!await setModTime(location, time, flags)) { + throw new OperationError( + "failed to touch: can't set modified time", + 501, + ); + } + return new Response(null); + } // The files of a folder, as rclone: with `--recursive`, also the files // of its folders. - let files = info.isDir ? 0 : 1; - if (info.isDir) { - const depth = isTrue(flags.recursive) ? Infinity : 1; - for await ( - const item of await walk(location, flags, { maxDepth: depth }) - ) { - if (!item.IsDir) files++; + const depth = isTrue(flags.recursive) ? Infinity : 1; + let errors = 0; + for await ( + const item of await walk(location, flags, { maxDepth: depth }) + ) { + if (item.IsDir) continue; + if (isTrue(flags.dry_run)) { + log("NOTICE", item.Path, "Skipped touch as --dry-run is set", flags); + } else if (!await setModTime(item.location, time, flags)) { + errors++; + log( + "ERROR", + item.Path, + "failed to touch: can't set modified time", + flags, + ); } } - if (files > 0 && !isTrue(flags.dry_run)) { - throw new OperationError("failed to touch: can't set modified time", 501); + if (errors) { + throw new OperationError( + `failed to touch ${errors} files: can't set modified time`, + 501, + ); } return new Response(null); } catch (error) { return failure(error); } } + +/** + * The time of the touch: now, or the time of `--timestamp`. + * @param {Object} flags + * @returns {bigint} The nanoseconds after 1970. + * @throws {OperationError} For a timestamp that is not correct. + */ +function timeOf(flags) { + const timestamp = flags.timestamp; + if (timestamp === undefined || timestamp === "") { + return BigInt(Date.now()) * 1000000n; + } + const text = `${timestamp}`; + const fail = () => + new OperationError(`failed to parse timestamp argument: ${text}`, 400); + /** @type {number[]} */ + let fields; + let fraction = ""; + if (text.length < 19) { + // `060102` of Go: a year of two digits from 1969 to 2068. + const match = SHORT.exec(text); + if (!match) throw fail(); + const year = Number(match[1]); + fields = [ + year >= 69 ? 1900 + year : 2000 + year, + ...[2, 3].map((i) => Number(match[i])), + 0, + 0, + 0, + ]; + } else { + const match = LONG.exec(text); + // A text of 19 characters has no part of a second, and a longer text + // has one. + if (!match || (text.length > 19) !== (match[7] !== undefined)) { + throw fail(); + } + fields = match.slice(1, 7).map(Number); + fraction = match[7] ?? ""; + } + const milliseconds = isTrue(flags.localtime) + ? new Date( + fields[0], + fields[1] - 1, + fields[2], + fields[3], + fields[4], + fields[5], + ).getTime() + : Date.UTC( + fields[0], + fields[1] - 1, + fields[2], + fields[3], + fields[4], + fields[5], + ); + // As Go, a field out of its range is an error. + const date = new Date(milliseconds); + const check = isTrue(flags.localtime) + ? [date.getFullYear(), date.getMonth() + 1, date.getDate(), date.getHours()] + : [ + date.getUTCFullYear(), + date.getUTCMonth() + 1, + date.getUTCDate(), + date.getUTCHours(), + ]; + if ( + isNaN(milliseconds) || check.some((value, i) => value !== fields[i]) || + fields[4] > 59 || fields[5] > 59 + ) { + throw fail(); + } + return BigInt(milliseconds) * 1000000n + BigInt(fraction.padEnd(9, "0")); +} diff --git a/cmd/touch/main_test.js b/cmd/touch/main_test.js index e052b55..17b544b 100644 --- a/cmd/touch/main_test.js +++ b/cmd/touch/main_test.js @@ -2,6 +2,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { contents } from "../testing.js"; +import { lsjson } from "../lsjson/main.js"; import { touch } from "./main.js"; test("touch makes an empty file, as rclone", async () => { @@ -18,19 +19,65 @@ test("touch makes an empty file, as rclone", async () => { ]); }); -test("touch tells that the backends cannot set the time yet", async () => { - const response = await touch(":memory:/touch/new.txt"); +test("touch sets the time of a file, as rclone", async () => { + /** @param {string} name */ + const timeOf = async (name) => + (await (await lsjson(":memory:/touch/")).json()) + .find((item) => item.Name === name)?.ModTime; + // The time of a new file, in UTC. + assert.equal( + (await touch(":memory:/touch/t.txt", { + timestamp: "2020-01-02T03:04:05", + })).status, + 200, + ); + assert.equal(await timeOf("t.txt"), "2020-01-02T03:04:05.000Z"); + // The time of a file that exists. + assert.equal( + (await touch(":memory:/touch/t.txt", { timestamp: "210203" })).status, + 200, + ); + assert.equal(await timeOf("t.txt"), "2021-02-03T00:00:00.000Z"); + // Each file of a folder. + assert.equal( + (await touch(":memory:/touch/", { + timestamp: "2022-03-04T05:06:07.5", + })).status, + 200, + ); + assert.equal(await timeOf("t.txt"), "2022-03-04T05:06:07.500Z"); + assert.equal(await timeOf("new.txt"), "2022-03-04T05:06:07.500Z"); + // As rclone: a year of two digits, and no part of a second in a text of + // 19 characters. + assert.equal( + (await touch(":memory:/touch/t.txt", { timestamp: "991231" })).status, + 200, + ); + assert.equal(await timeOf("t.txt"), "1999-12-31T00:00:00.000Z"); + for ( + const timestamp of [ + "yesterday", + "2020-01-02", + "201301", + "2020-01-02T25:00:00", + "2020-01-02T03:04:05.", + ] + ) { + assert.equal( + (await touch(":memory:/touch/t.txt", { timestamp })).status, + 400, + timestamp, + ); + } +}); + +test("touch gives the error of rclone for a backend that cannot set the time", async () => { + const location = ":chunker,remote=':memory:/touch-chunker':a.txt"; + assert.equal((await touch(location)).status, 200); + const response = await touch(location); assert.equal(response.status, 501); assert.equal( await response.text(), "failed to touch: can't set modified time", ); - const timestamp = await touch(":memory:/touch/t.txt", { - timestamp: "2020-01-02T03:04:05", - }); - assert.equal(timestamp.status, 501); - assert.equal( - (await touch(":memory:/touch/t.txt", { timestamp: "yesterday" })).status, - 400, - ); }); diff --git a/fetch.js b/fetch.js index b3c3994..20c67d0 100644 --- a/fetch.js +++ b/fetch.js @@ -118,11 +118,106 @@ export async function fetch(input, init = {}) { return globalFetch(...withoutOptions(input, init)); } + const headers = new Headers(init.headers); + const { type, name, pathname, params, options } = await resolve( + input, + headers, + ); + + // COPY and MOVE: the backend gets the path of the destination in its own + // root. The destination must be in the same remote, with the same + // options: else, the answer is 502, as WebDAV gives for a destination on + // another server, and the caller copies the bytes. + const method = `${init.method ?? "GET"}`.toUpperCase(); + const destination = headers.get("Destination"); + if ((method === "COPY" || method === "MOVE") && destination !== null) { + // A URL is on a server, not in a remote. + const target = /^[a-z][a-z0-9+.-]*:\/\//i.test(destination) + ? null + : await resolve(destination, headers); + if ( + !target || target.type !== type || !sameEntries(target.params, params) || + !sameEntries(Object.entries(target.options), Object.entries(options)) + ) { + await init.body?.cancel?.(); + return new Response("The destination is not in the same remote.", { + status: 502, + }); + } + headers.set("Destination", target.pathname); + } + + // The headers of this backend are in its options now. + const own = PREFIXES.map((start) => `${start}${type.toLowerCase()}_`); + for (const key of [...headers.keys()]) { + if (own.some((start) => key.startsWith(start))) headers.delete(key); + } + + if (init.method === "TRACE") { + let body = `TRACE ${pathname}?${params} HTTP/1.1\r`; + // Reflects the request as response body, then the options in `env`. + for (const [name, value] of headers) { + body += `${name}: ${value}\r`; + } + for (const [name, value] of Object.entries(options)) { + body += `${name}: ${value}\r`; + } + return new Response(body, { + status: 200, + headers: { + "Content-Type": "message/http", + "Via": `${type}/1.1 ${name}`.trim(), + }, + }); + } + + if (init.body instanceof ReadableStream) { + // @ts-ignore: Must have `duplex` for streaming body + init.duplex = "half"; // Must set this for stream body. + } + + const backend = await load(type); + if (!backend) { + throw new Error(`Backend ${type} not found.`); + } + + // The empty host keeps a path that starts with "//" as a path. + const url = new URL(`file://${pathname}?${params}`); + // Creates a new request with the initial init. + const request = new Request(url, init); + // Clones that request and updates the headers. + return backend.fetch( + new Request(request, { + headers, + }), + options, + ); +} + +/** + * The backend of a location, and the request that it gets: its type, the + * name of the remote, the path in the remote, the query and the options. + * @typedef {Object} Target + * @property {string} type + * @property {string} name + * @property {string} pathname + * @property {URLSearchParams} params + * @property {Record} options The options in `env`, with + * the names of the variables of rfetch. + */ + +/** + * Reads a location of rclone, with the options of the config file, of the + * variables and of the headers. + * @param {string} input + * @param {Headers} headers + * @returns {Promise} + */ +async function resolve(input, headers) { // The parts of the path, as rclone reads them. It throws a SyntaxError // for a path that is not correct. const parsed = parse(input); - const headers = new Headers(init.headers); // The variables, by their names with no prefix, for example // `CONFIG_R2_TYPE`: the environment of the process, then the headers of // the request, which take their place. In each place, `RFETCH_` takes the @@ -261,51 +356,20 @@ export async function fetch(input, init = {}) { // variable, a header or the query cannot set it. delete optionsEnv.RFETCH_LOCAL_ROOT; if (root !== undefined) optionsEnv.RFETCH_LOCAL_ROOT = root; - // The headers of this backend are in its options now. - const own = PREFIXES.map((start) => `${start}${type.toLowerCase()}_`); - for (const key of [...headers.keys()]) { - if (own.some((start) => key.startsWith(start))) headers.delete(key); - } - - if (init.method === "TRACE") { - let body = `TRACE ${pathname}?${params} HTTP/1.1\r`; - // Reflects the request as response body, then the options in `env`. - for (const [name, value] of headers) { - body += `${name}: ${value}\r`; - } - for (const [name, value] of Object.entries(optionsEnv)) { - body += `${name}: ${value}\r`; - } - return new Response(body, { - status: 200, - headers: { - "Content-Type": "message/http", - "Via": `${type}/1.1 ${name}`.trim(), - }, - }); - } - if (init.body instanceof ReadableStream) { - // @ts-ignore: Must have `duplex` for streaming body - init.duplex = "half"; // Must set this for stream body. - } - - const backend = await load(type); - if (!backend) { - throw new Error(`Backend ${type} not found.`); - } + return { type, name, pathname, params, options: optionsEnv }; +} - // The empty host keeps a path that starts with "//" as a path. - const url = new URL(`file://${pathname}?${params}`); - // Creates a new request with the initial init. - const request = new Request(url, init); - // Clones that request and updates the headers. - return backend.fetch( - new Request(request, { - headers, - }), - optionsEnv, - ); +/** + * Tells if two lists of names and values have the same entries, in any + * order. + * @param {Iterable<[string, string]>} a + * @param {Iterable<[string, string]>} b + */ +function sameEntries(a, b) { + const sorted = (entries) => + JSON.stringify([...entries].sort(([x], [y]) => x < y ? -1 : x > y ? 1 : 0)); + return sorted(a) === sorted(b); } /** diff --git a/lib/overlay.js b/lib/overlay.js index 06b6658..6f10210 100644 --- a/lib/overlay.js +++ b/lib/overlay.js @@ -87,6 +87,14 @@ export async function passThrough( ) { const { method } = request; const { pathname, searchParams } = new URL(request.url); + // The path of a destination is not a location of the remote here. + if (method === "COPY" || method === "MOVE") { + await request.body?.cancel(); + return new Response(null, { + status: 405, + headers: { Allow: "GET, HEAD, PUT, DELETE" }, + }); + } /** @type {RequestInit} */ const init = { method, headers }; if (request.body) { diff --git a/lib/testing/suite.js b/lib/testing/suite.js index 96d779b..faf51b7 100644 --- a/lib/testing/suite.js +++ b/lib/testing/suite.js @@ -44,6 +44,30 @@ const NAMES = [ const FILE1 = "file name.txt"; const FILE2 = `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`; +/** + * What a backend can do other than GET, HEAD, PUT and DELETE, as the + * features of a backend in rclone. + * @typedef {Object} Features + * @property {boolean} [copy] `COPY` of a file, as `Copy` of rclone. + * @property {boolean} [move] `MOVE` of a file, as `Move` of rclone. + * @property {boolean} [dirMove] `MOVE` of a folder to a new path, as + * `DirMove` of rclone. + * @property {boolean} [modTime] A `PUT` with `X-OC-Mtime` gives that time + * to the file. + * @property {boolean} [copyModTime] A `COPY` or a `MOVE` with `X-OC-Mtime` + * gives that time to the file. By default, the value of `modTime`. + * @property {boolean} [setModTime] `PATCH` with `X-OC-Mtime` sets the time + * of a file, as `SetModTime` of rclone. + * @property {boolean} [listModTime] The list gives the time of + * `X-OC-Mtime`, as `HEAD` does. By default, the value of `modTime`. + * s3 keeps the time in the metadata of a file, thus only `HEAD` and `GET` + * give it. + */ + +/** A time of `X-OC-Mtime`, and its second for `Last-Modified`. */ +const MTIME = "1354040105.123456789"; +const MTIME_SECOND = "Tue, 27 Nov 2012 18:15:05 GMT"; + /** * Runs the tests of a backend. * @param {string} name The name of the backend, for the report. @@ -56,6 +80,7 @@ const FILE2 = `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`; * parent?: string, * emptyFolders?: boolean, * type?: string, + * features?: Features, * }} [options] `params` go in the query of each request, for example the * options of the backend. `env` is the env of the backend, for example * the credentials of s3. The tests skip the names that match `skipName`, @@ -69,7 +94,8 @@ const FILE2 = `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`; * gives 404, as a folder that does not exist. `type` is the type of the * backend in rclone, for example `s3`: with it, the tests also give the * `params` in the headers `RFETCH_TYPE_NAME` and in the variables - * `RCLONE_TYPE_NAME` of `env`. + * `RCLONE_TYPE_NAME` of `env`. `features` tells what the backend can do + * other than GET, HEAD, PUT and DELETE. */ export function backendSuite( name, @@ -82,6 +108,7 @@ export function backendSuite( parent = "/", emptyFolders = true, type, + features = {}, } = {}, ) { const root = `${parent}rfetch-test-${randomBytes(6).toString("hex")}/`; @@ -325,6 +352,173 @@ export function backendSuite( }); }); + describe("COPY, MOVE and the time of a file", () => { + const { + copy = false, + move = false, + dirMove = false, + modTime = false, + copyModTime = modTime, + setModTime = false, + listModTime = modTime, + } = features; + const content = random(300); + /** The headers of a request with the time of `MTIME`. */ + const timed = modTime ? { "X-OC-Mtime": MTIME } : {}; + const copyTimed = copyModTime ? { "X-OC-Mtime": MTIME } : {}; + /** + * Sends a COPY or a MOVE, to a path in the folder of the tests. + * @param {"COPY" | "MOVE"} method + * @param {string} path + * @param {string} target + */ + const send = (method, path, target) => + fs.send(path, { + method, + headers: { Destination: fs.url(target).pathname, ...copyTimed }, + }); + /** + * Makes sure that a file has the content, and the time of `MTIME`. + * @param {string} path + * @param {boolean} [time] False for no check of the time. + */ + const assertCopy = async (path, time = copyModTime) => { + const response = await fs.get(path); + assert.equal(response.status, 200, `GET ${path}`); + const bytes = new Uint8Array(await response.arrayBuffer()); + assert.ok(Buffer.from(bytes).equals(content), `the bytes of ${path}`); + if (!time) return; + assert.equal(response.headers.get("Last-Modified"), MTIME_SECOND); + if (!listModTime) return; + const slash = path.lastIndexOf("/") + 1; + const entry = (await fs.list(path.slice(0, slash))) + .find((entry) => entry.Name === path.slice(slash)); + assert.equal( + Math.floor(Date.parse(entry?.ModTime ?? "") / 1000), + 1354040105, + `the time of ${path} in the list is ${entry?.ModTime}`, + ); + }; + + it("puts a file with the time of X-OC-Mtime", async () => { + const response = await fs.put("time/a.txt", content, timed); + await response.body?.cancel(); + assert.equal(response.status, 201); + if (modTime) { + assert.equal(response.headers.get("X-OC-Mtime"), "accepted"); + } + await assertCopy("time/a.txt", modTime); + }); + + it("sets the time of a file with PATCH", async () => { + const response = await fs.send("time/a.txt", { + method: "PATCH", + headers: { "X-OC-Mtime": "1354040106" }, + }); + await response.body?.cancel(); + if (!setModTime) { + assert.ok( + response.status === 405 || response.status === 403, + `PATCH gave ${response.status}, not 405`, + ); + return; + } + assert.equal(response.status, 204); + assert.equal(response.headers.get("X-OC-Mtime"), "accepted"); + const head = await fs.head("time/a.txt"); + assert.equal( + head.headers.get("Last-Modified"), + "Tue, 27 Nov 2012 18:15:06 GMT", + ); + await expect( + fs.send("time/missing.txt", { + method: "PATCH", + headers: { "X-OC-Mtime": "1354040106" }, + }), + 404, + ); + }); + + it("copies a file with COPY", async () => { + const response = await send("COPY", "time/a.txt", "copy/b.txt"); + await response.body?.cancel(); + if (!copy) { + assert.ok( + [403, 405, 501, 502].includes(response.status), + `COPY gave ${response.status}, not 405`, + ); + return; + } + assert.equal(response.status, 201, "COPY to a new path"); + if (copyModTime) { + assert.equal(response.headers.get("X-OC-Mtime"), "accepted"); + } + await assertCopy("copy/b.txt"); + await expect(fs.head("time/a.txt"), 200); + // Over a file. + const again = await send("COPY", "time/a.txt", "copy/b.txt"); + await again.body?.cancel(); + assert.ok( + again.status === 201 || again.status === 204, + `COPY over a file gave ${again.status}`, + ); + await expect(send("COPY", "time/missing.txt", "copy/c.txt"), 404); + }); + + it("moves a file with MOVE", async () => { + if (!copy) await put("copy/b.txt", content); + const response = await send("MOVE", "copy/b.txt", "move/c.txt"); + await response.body?.cancel(); + if (!move) { + assert.ok( + [403, 405, 501, 502].includes(response.status), + `MOVE gave ${response.status}, not 405`, + ); + await expect(fs.delete("copy/"), 204); + return; + } + assert.ok( + response.status === 201 || response.status === 204, + `MOVE gave ${response.status}`, + ); + await expect(fs.get("copy/b.txt"), 404); + await assertCopy("move/c.txt"); + await expect(send("MOVE", "copy/b.txt", "move/d.txt"), 404); + // The folder of the file stays. + await expect(fs.delete("copy/"), 204); + }); + + it("moves a folder with MOVE to a new path", async () => { + if (!move) await put("move/c.txt", content); + const response = await send("MOVE", "move/", "moved/folder/"); + await response.body?.cancel(); + if (!dirMove) { + assert.ok( + [403, 405, 501, 502].includes(response.status), + `MOVE of a folder gave ${response.status}, not 405`, + ); + await expect(fs.delete("move/"), 204); + return; + } + assert.equal(response.status, 201, "MOVE of a folder"); + await expect(fs.get("move/c.txt"), 404); + const moved = await fs.get("moved/folder/c.txt"); + const bytes = new Uint8Array(await moved.arrayBuffer()); + assert.ok(Buffer.from(bytes).equals(content), "the moved file"); + // A folder does not move over a folder. + await put("other/x.txt", content); + await expect(send("MOVE", "other/", "moved/"), 412); + await expect(fs.get("other/x.txt"), 200); + await expect(fs.delete("other/"), 204); + await expect(fs.delete("moved/"), 204); + }); + + it("removes the files of these tests", async () => { + await expect(fs.delete("time/"), 204); + assertEntries(await fs.list(""), [file(FILE1, 200)]); + }); + }); + it("reads the options from the headers and from env", { skip: type && Object.keys(params).length ? false : "no options", }, async () => { diff --git a/lib/time.js b/lib/time.js index 70a6ee4..90f5c68 100644 --- a/lib/time.js +++ b/lib/time.js @@ -83,3 +83,65 @@ export function toListTime(value, digits = 0) { if (!digits) return `${day} ${time}`; return `${day} ${time}.${fraction.padEnd(digits, "0").slice(0, digits)}`; } + +/** + * Reads a time in seconds after 1970, with an optional part of a second, for + * example `1354040105` or `1354040105.123456789`. This is the format of the + * header `X-OC-Mtime` of ownCloud, which rclone sends to a WebDAV server, and + * of the metadata `mtime` that rclone keeps on S3 and Swift. The digits after + * the ninth digit of the part of a second are cut. + * @param {string | null | undefined} value + * @returns {bigint | undefined} The nanoseconds after 1970, or no number for + * a value that is not correct. + */ +export function parseUnixTime(value) { + const match = /^(-?)(\d{1,15})(?:\.(\d{1,9})\d*)?$/.exec(value?.trim() ?? ""); + if (!match) return undefined; + const nanoseconds = BigInt(match[2]) * 1_000_000_000n + + BigInt((match[3] ?? "").padEnd(9, "0")); + return match[1] ? -nanoseconds : nanoseconds; +} + +/** + * Writes a time in seconds after 1970, as `TimeToFloatString` of rclone: the + * part of a second has no zeros at its end, and a whole second has no point, + * for example `1354040105.12345` or `1354040105`. + * @param {bigint} nanoseconds The nanoseconds after 1970. + * @returns {string} + */ +export function formatUnixTime(nanoseconds) { + if (nanoseconds < 0n) return `-${formatUnixTime(-nanoseconds)}`; + const second = 1_000_000_000n; + const fraction = String(nanoseconds % second).padStart(9, "0") + .replace(/0+$/, ""); + return `${nanoseconds / second}${fraction ? `.${fraction}` : ""}`; +} + +/** + * Writes a time in ISO 8601 in UTC, with 9 digits after the second: the + * format of rclone for a remote with a precision of 1 ns. + * @param {bigint} nanoseconds The nanoseconds after 1970. + * @returns {string} + */ +export function isoStringOf(nanoseconds) { + const second = 1_000_000_000n; + // The floor, also for a time before 1970. + let fraction = nanoseconds % second; + if (fraction < 0n) fraction += second; + const seconds = (nanoseconds - fraction) / second; + const date = new Date(Number(seconds) * 1000).toISOString().slice(0, 19); + return `${date}.${String(fraction).padStart(9, "0")}Z`; +} + +/** + * The time of the header `X-OC-Mtime` of a request: the time that a `PUT`, + * a `COPY`, a `MOVE` or a `PATCH` gives to the file. + * @param {Request} request + * @returns {bigint | null | undefined} The nanoseconds after 1970, null + * when the request has no such header, or no value for a value that is + * not correct. + */ +export function modTimeOf(request) { + const value = request.headers.get("X-OC-Mtime"); + return value === null ? null : parseUnixTime(value); +} diff --git a/lib/time_test.js b/lib/time_test.js index 8dbbc7c..405aa8e 100644 --- a/lib/time_test.js +++ b/lib/time_test.js @@ -1,7 +1,15 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { nanosecondsOf, toListTime, toLocaleISOString } from "./time.js"; +import { + formatUnixTime, + isoStringOf, + modTimeOf, + nanosecondsOf, + parseUnixTime, + toListTime, + toLocaleISOString, +} from "./time.js"; test("toLocaleISOString writes the times of rclone lsjson", () => { const time = "2026-10-17T03:56:05.819Z"; @@ -96,3 +104,43 @@ test("nanosecondsOf keeps all the digits of a time", () => { assert.equal(nanosecondsOf(value), undefined, `${value}`); } }); + +test("parseUnixTime and formatUnixTime read and write the times of rclone", () => { + for ( + const [text, nanoseconds] of [ + ["1354040105", 1354040105000000000n], + ["1354040105.123456789", 1354040105123456789n], + ["1354040105.5", 1354040105500000000n], + ["0", 0n], + ["-1.25", -1250000000n], + ] + ) { + assert.equal(parseUnixTime(text), nanoseconds, text); + assert.equal(formatUnixTime(nanoseconds), text); + } + // The digits after the ninth digit are cut. + assert.equal(parseUnixTime("1.0000000019"), 1000000001n); + for (const text of ["", "x", "1e9", "1.", ".5", "1 2", null, undefined]) { + assert.equal(parseUnixTime(text), undefined, `${text}`); + } +}); + +test("isoStringOf writes 9 digits after the second", () => { + assert.equal( + isoStringOf(1354040105123456789n), + "2012-11-27T18:15:05.123456789Z", + ); + assert.equal(isoStringOf(-1n), "1969-12-31T23:59:59.999999999Z"); + assert.equal( + nanosecondsOf(isoStringOf(1354040105123456789n)), + 1354040105123456789n, + ); +}); + +test("modTimeOf reads the header X-OC-Mtime", () => { + const request = (value) => + new Request("file:///a", value ? { headers: { "X-OC-Mtime": value } } : {}); + assert.equal(modTimeOf(request()), null); + assert.equal(modTimeOf(request("1354040105")), 1354040105000000000n); + assert.equal(modTimeOf(request("soon")), undefined); +}); diff --git a/main_test.js b/main_test.js index d845d58..d2f111f 100644 --- a/main_test.js +++ b/main_test.js @@ -481,6 +481,57 @@ test("a path keeps the # of its names, and a // in it is not a host", async () = assert.deepEqual(await names(":memory:/paths/f%23g%25h/"), ["d?e.txt"]); }); +test("COPY and MOVE go to the backend when the destination is in the same remote", async () => { + const put = await fetch(":memory:/same/a.txt", { method: "PUT", body: "a" }); + assert.equal(put.status, 201); + // The backend gets the path of the destination in its root. + const copy = await fetch(":memory:/same/a.txt", { + method: "COPY", + headers: { Destination: ":memory:/same/b%20c.txt" }, + }); + assert.equal(copy.status, 201); + assert.equal(await (await fetch(":memory:/same/b%20c.txt")).text(), "a"); + // Through an alias, to the remote under it. + const alias = await fetch(":alias:a.txt", { + method: "MOVE", + headers: { + RFETCH_ALIAS_REMOTE: ":memory:/same", + Destination: ":alias:d.txt", + }, + }); + assert.equal(alias.status, 201); + assert.equal(await (await fetch(":memory:/same/d.txt")).text(), "a"); + assert.equal((await fetch(":memory:/same/a.txt")).status, 404); + + // Another remote, or the same type with other options, gives 502: the + // caller copies the bytes. + for ( + const [destination, headers] of [ + [`${await mkdtemp(join(tmpdir(), "rfetch-same-"))}/d.txt`, {}], + [":alias:e.txt", { RFETCH_ALIAS_REMOTE: ":memory:/other" }], + [":memory:/same/e.txt?memory_x=1", {}], + ["https://example.com/e.txt", {}], + ] + ) { + const response = await fetch(":memory:/same/d.txt", { + method: "COPY", + headers: { Destination: destination, ...headers }, + }); + await response.body?.cancel(); + assert.equal(response.status, 502, destination); + } + // The options of a header apply to the source and to the destination. + const other = await fetch(":alias:d.txt", { + method: "COPY", + headers: { + RFETCH_ALIAS_REMOTE: ":memory:/same", + Destination: ":alias:f.txt", + }, + }); + assert.equal(other.status, 201); + assert.equal(await (await fetch(":memory:/same/f.txt")).text(), "a"); +}); + test("the command line", async (t) => { const folder = await mkdtemp(join(tmpdir(), "rfetch-cli-")); await writeFile(join(folder, "a.txt"), "a"); From f64a1a173313c9aa0ae3e27692351d492e169ec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Mon, 19 Oct 2026 21:00:04 +0000 Subject: [PATCH 2/2] fix(local): the time of a file to the microsecond on each Node.js utimes takes the seconds as a double. Node.js 22 cuts the part of a second to the microsecond, and Node.js 24 and later keep the nanoseconds of the double. The half of a microsecond that the last commit added gave 476 ns to a whole second on Node.js 24 and 26. Now the value is the double nearest to the time. When the system would give the microsecond before or after, the next double in the correct microsecond is the value. A test of 20000 random times on Node.js 22 and 26 gives each time to the correct microsecond, and a whole second with no nanoseconds. --- backend/local/main.js | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/backend/local/main.js b/backend/local/main.js index 548cdba..0b8ab3f 100644 --- a/backend/local/main.js +++ b/backend/local/main.js @@ -344,13 +344,40 @@ function isOutside(root, path) { */ async function setTime(path, nanoseconds) { const { utimes } = await import("node:fs/promises"); - // The seconds go in a double, which can be some nanoseconds less than the - // time, and the system cuts the time to the microsecond: half of a - // microsecond more gives the correct microsecond. - const seconds = (Number(nanoseconds / 1000n) + 0.5) / 1e6; + const seconds = secondsOf(nanoseconds); await utimes(path, seconds, seconds); } +/** + * The seconds of a time, as a double for `utimes`. The system cuts the part + * of a second of the double: Node.js 22 to the microsecond, and Node.js 24 + * and later to the nanosecond. The double nearest to the time can be in the + * microsecond before or after: then the double next to it, in the correct + * microsecond, is the value. Each microsecond has two doubles or more, up to + * the year 2106. + * @param {bigint} nanoseconds The nanoseconds after 1970. + * @returns {number} + */ +export function secondsOf(nanoseconds) { + if (nanoseconds < 0n) return Number(nanoseconds / 1000n) / 1e6; + const whole = Number(nanoseconds / 1_000_000_000n); + const fraction = Number(nanoseconds % 1_000_000_000n); + const microseconds = Math.floor(fraction / 1000); + let seconds = whole + fraction / 1e9; + // The step between two doubles near the value. + const step = 2 ** (Math.floor(Math.log2(Math.max(seconds, 1))) - 52); + /** The microsecond that the system gives for a double, as libuv. */ + const microsecondOf = (value) => + Math.floor(Math.trunc((value - whole) * 1e9) / 1000); + for (let i = 0; i < 8 && microsecondOf(seconds) < microseconds; i++) { + seconds += step; + } + for (let i = 0; i < 8 && microsecondOf(seconds) > microseconds; i++) { + seconds -= step; + } + return seconds; +} + /** * The time of a file to the millisecond. As in Go, the time is cut, not * rounded: the `mtime` of Node.js is rounded to the nearest millisecond.