From a05d04b0de31824243bfc2a27c5fe1cc0cac17f5 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sat, 31 Oct 2026 22:06:24 +0100 Subject: [PATCH 1/5] fix: resolve distributed cloud performance, security, and robustness concerns - Fix POST creation-with-upload lock acquisition and watchdog leak: ensure distributed lease lock is acquired for newly created uploadId and released cleanly on stream completion or abortion across Tus and RUFH. - Enforce stream safety and rollback integrity: rollback uncommitted bytes on client stream interruptions and reset tentative length for RUFH deferred length uploads. - Prevent parent upload truncation on child error rollback in S3 and Azure Blob storage services. - Increase S3 server-side compose HTTP timeouts to 60s for large uploads and preserve fallback behavior. - Harden against HTTP request smuggling and DoS: enforce strict CRLF and chunk size validation in chunked transfer encoding stream and bound metadata length. - Optimize pattern compilation using ConcurrentHashMap caching. - Add preliminary request validation in getRawInterimResponse to prevent phantom upload creation on invalid requests. - Add integration test for chunked streaming with trailing checksum verification across Disk, S3, and Azure backends. --- AGENTS.md | 1 + .../tus/server/TusFileUploadService.java | 32 ++++++- .../server/core/CorePatchRequestHandler.java | 4 + .../validation/UploadMetadataValidator.java | 24 +++++- .../CreationWithUploadPostRequestHandler.java | 79 ++++++++++------- ...HttpDigestsPostPutPatchRequestHandler.java | 69 ++++++++++++--- .../RufhAppendPatchRequestHandler.java | 27 ++++-- .../RufhCreationPostRequestHandler.java | 70 ++++++++++----- .../upload/AbstractLeaseLockingService.java | 12 +++ .../server/upload/UploadLockingService.java | 11 +++ .../upload/azure/AzureBlobLockingService.java | 10 +++ .../upload/azure/AzureBlobStorageService.java | 67 ++++++++++++--- .../tus/server/upload/azure/AzureUtils.java | 17 ++++ ...adLocalCachedStorageAndLockingService.java | 5 ++ .../upload/s3/S3ServerSideComposeHelper.java | 10 ++- .../server/upload/s3/S3StorageService.java | 38 ++++++++- .../desair/tus/server/upload/s3/S3Utils.java | 17 ++++ .../util/HttpChunkedEncodingInputStream.java | 41 ++++++++- .../tus/server/util/TusServletRequest.java | 27 +++--- .../java/me/desair/tus/server/util/Utils.java | 28 ++++-- .../tus/server/AbstractITRufhProtocol.java | 58 +++++++++++++ .../tus/server/TusFileUploadServiceTest.java | 47 +++++++++- .../core/CorePatchRequestHandlerTest.java | 21 +++-- .../UploadMetadataValidatorTest.java | 35 ++++++++ ...ationWithUploadPostRequestHandlerTest.java | 40 +++++++++ ...DigestsPostPutPatchRequestHandlerTest.java | 80 +++++++++++++++++ .../RufhAppendPatchRequestHandlerTest.java | 44 ++++++++++ .../RufhCreationPostRequestHandlerTest.java | 75 ++++++++++++++++ .../azure/AzureBlobLockingServiceTest.java | 16 ++++ .../azure/AzureBlobStorageServiceTest.java | 13 +++ .../azure/ITAzureBlobStorageService.java | 22 +++++ ...calCachedStorageAndLockingServiceTest.java | 3 + .../upload/s3/S3StorageServiceTest.java | 35 ++++++++ .../HttpChunkedEncodingInputStreamTest.java | 85 +++++++++++++++++++ .../server/util/TusServletRequestTest.java | 32 +++++++ .../me/desair/tus/server/util/UtilsTest.java | 7 ++ 36 files changed, 1080 insertions(+), 122 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8243311f..9bd1c924 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -129,6 +129,7 @@ To avoid duplicate test code and ensure all protocol integration tests run consi ### 17. The "2-Year Maintainability Rule" for Inline Documentation & Code Readability - **The "2-Year Maintainability Rule"**: Every single fix, complicated or non-trivial code block, boundary condition, and `if`-test MUST include thorough inline comments explaining *why* that specific call, calculation, or branch condition was chosen. The code must be documented so clearly that after 2 years, any maintainer can immediately understand the reasoning, edge-case rationale, or protocol specification behind it without having to reverse-engineer the commit history. - Write natural, explanatory comments describing the reasoning and intent directly. Do NOT prefix comments with literal meta-labels such as `The 2-Year Maintainability Rule:` or review audit section identifiers (e.g., `H5:`). +- **No Agent File References**: Never mention `AGENTS.md` or reference AGENTS.md rules in source code comments. Comments must describe the engineering intent and domain logic directly without referencing agent prompt files, meta-rules, or instruction documents. - Keep comment line breaks balanced and clean within the 100-character column limit enforced by Google Java Style. - Always write and preserve thorough inline comments across all main and test Java source files to explain non-obvious algorithms, multi-step operations, and complex logic. - Ensure all function implementations remain short, clean, well-documented, and stick to the same level of abstraction. Use helper methods to break down complex logic into smaller, readable pieces. diff --git a/src/main/java/me/desair/tus/server/TusFileUploadService.java b/src/main/java/me/desair/tus/server/TusFileUploadService.java index 2c67b072..f226e655 100644 --- a/src/main/java/me/desair/tus/server/TusFileUploadService.java +++ b/src/main/java/me/desair/tus/server/TusFileUploadService.java @@ -154,13 +154,41 @@ public TusFileUploadService withSupportedProtocolVersions( * * @param servletRequest The incoming {@link HttpServletRequest} * @param ownerKey The owner key identifier for the upload - * @return The raw HTTP 104 interim response string if applicable, or null if not applicable + * @return The raw HTTP 104 interim response string if applicable, or null if not applicable or if + * preliminary validation fails */ public String getRawInterimResponse(HttpServletRequest servletRequest, String ownerKey) { - if (detectProtocolVersion(servletRequest) == ProtocolVersion.RUFH) { + if (servletRequest == null) { + return null; + } + HttpMethod method = + HttpMethod.getMethodIfSupported(servletRequest, EnumSet.allOf(HttpMethod.class)); + if (method == null) { + return null; + } + + ProtocolVersion version = detectProtocolVersion(servletRequest); + try { + validateRequest(method, servletRequest, ownerKey, version); + } catch (Exception e) { + log.debug("Validation failed before generating interim response: {}", e.getMessage()); + // If preliminary validation fails (e.g. invalid metadata, path traversal, or upload length + // exceeding maxUploadSize), return null so that no 104 interim response is emitted and no + // phantom upload is pre-created on storage. The request will proceed down the normal servlet + // container pipeline to process(), where the main validation pipeline will catch the error + // and generate the formal error response (e.g. RFC 7807 problem details or Tus error + // response). + return null; + } + + if (version == ProtocolVersion.RUFH) { return RufhInterimResponseUtil.getRawInterimResponse( servletRequest, uploadStorageService, ownerKey); } + + // Protocol versions other than RUFH (such as Tus 1.0.0) do not define HTTP 104 interim + // responses. + // Return null so the request proceeds to normal servlet processing where process() handles it. return null; } diff --git a/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java b/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java index cb875aed..f5bd10c9 100644 --- a/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java @@ -109,6 +109,10 @@ public HttpProblemDetails process( } else { throw e; } + } finally { + if (lockingService != null) { + lockingService.unregisterInputStream(servletRequest.getRequestURI()); + } } } diff --git a/src/main/java/me/desair/tus/server/creation/validation/UploadMetadataValidator.java b/src/main/java/me/desair/tus/server/creation/validation/UploadMetadataValidator.java index b0e3b605..9714a60e 100644 --- a/src/main/java/me/desair/tus/server/creation/validation/UploadMetadataValidator.java +++ b/src/main/java/me/desair/tus/server/creation/validation/UploadMetadataValidator.java @@ -17,6 +17,8 @@ */ public class UploadMetadataValidator implements RequestValidator { + public static final int MAX_METADATA_HEADER_LENGTH = 65_536; + @Override public void validate( HttpMethod method, @@ -27,6 +29,12 @@ public void validate( String metadata = Utils.getHeader(request, HttpHeader.UPLOAD_METADATA); if (StringUtils.isNotBlank(metadata)) { + if (metadata.length() > MAX_METADATA_HEADER_LENGTH) { + throw new InvalidUploadMetadataException( + "Upload-Metadata header exceeds maximum allowed length of " + + MAX_METADATA_HEADER_LENGTH + + " characters"); + } String[] pairs = metadata.split(","); for (String pair : pairs) { pair = pair.trim(); @@ -41,8 +49,9 @@ public void validate( } String key = keyValue[0]; - if (StringUtils.isBlank(key)) { - throw new InvalidUploadMetadataException("Upload-Metadata key cannot be empty"); + if (!isValidKey(key)) { + throw new InvalidUploadMetadataException( + "Upload-Metadata key must not be empty and must not contain spaces or commas"); } if (keyValue.length == 2) { @@ -56,6 +65,17 @@ public void validate( } } + private boolean isValidKey(String key) { + for (char c : key.toCharArray()) { + // The key MUST NOT contain spaces and commas and MUST NOT be empty (Tus 1.0.0 spec). + // Commas and single spaces are already separated during parsing; reject any other whitespace. + if (Character.isWhitespace(c)) { + return false; + } + } + return true; + } + @Override public boolean supports(HttpMethod method) { return HttpMethod.POST.equals(method); diff --git a/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java b/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java index 3c1e0695..6ad842b7 100644 --- a/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java @@ -7,6 +7,7 @@ import me.desair.tus.server.HttpProblemDetails; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadLock; import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.util.AbstractRequestHandler; @@ -46,40 +47,58 @@ public HttpProblemDetails process( if (location != null) { UploadInfo uploadInfo = uploadStorageService.getUploadInfo(location, ownerKey); if (uploadInfo != null && uploadInfo.isUploadInProgress()) { - InputStream stream = servletRequest.getContentInputStream(); - InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(stream); - uploadLockingService.registerInputStream(location, interruptibleStream); - stream = interruptibleStream; + // A POST creation request targets the base creation URI (e.g. /files), which has no + // UploadId when TusFileUploadService.process() initially acquires the request lock. + // Once CreationPostRequestHandler creates the upload and assigns an UploadId, we must + // acquire the distributed UploadLock for the new location before streaming payload + // bytes so concurrent HEAD/PATCH/DELETE requests across cluster nodes detect the active + // lock and trigger clean contention resolution, and so lock.close() cleans up the + // registered stream from activeInputStreams upon completion. + boolean needsLock = + uploadLockingService != null + && uploadInfo.getId() != null + && !uploadLockingService.isLocked(uploadInfo.getId()); + try (UploadLock ignored = + needsLock ? uploadLockingService.lockUploadByUri(location) : null) { + InputStream stream = servletRequest.getContentInputStream(); + InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(stream); + if (uploadLockingService != null) { + uploadLockingService.registerInputStream(location, interruptibleStream); + } + stream = interruptibleStream; - try { - uploadInfo = uploadStorageService.append(uploadInfo, stream); - } catch (IOException e) { - // When an upload stream is interrupted by the locking service watchdog or a concurrent - // lock contention release request (e.g. from a concurrent HEAD or DELETE), the storage - // backend (Disk, S3, Azure) commits all bytes received up to the interruption and - // updates - // the offset in storage. We reload the updated UploadInfo and acknowledge the partial - // upload rather than propagating an unhandled error to the servlet container. - if (interruptibleStream != null && interruptibleStream.isInterrupted()) { - // Refresh UploadInfo first so the log statement and response reflect the true - // committed - // byte offset persisted by the storage backend up to the interruption point. - UploadInfo refreshed = uploadStorageService.getUploadInfo(location, ownerKey); - if (refreshed != null) { - uploadInfo = refreshed; + try { + uploadInfo = uploadStorageService.append(uploadInfo, stream); + } catch (IOException e) { + // When an upload stream is interrupted by the locking service watchdog or a + // concurrent + // lock contention release request (e.g. from a concurrent HEAD or DELETE), the + // storage + // backend (Disk, S3, Azure) commits all bytes received up to the interruption and + // updates the offset in storage. We reload the updated UploadInfo and acknowledge the + // partial upload rather than propagating an unhandled error to the servlet container. + if (interruptibleStream.isInterrupted()) { + UploadInfo refreshed = uploadStorageService.getUploadInfo(location, ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } + log.info( + "Upload creation-with-upload POST request for URI {} was interrupted by locking" + + " service contention; saved partial upload up to offset {}", + location, + uploadInfo.getOffset()); + } else { + throw e; + } + } finally { + if (uploadLockingService != null) { + uploadLockingService.unregisterInputStream(location); } - log.info( - "Upload creation-with-upload POST request for URI {} was interrupted by locking" - + " service contention; saved partial upload up to offset {}", - location, - uploadInfo.getOffset()); - } else { - throw e; } - } - servletResponse.setHeader( - HttpHeader.UPLOAD_OFFSET, String.valueOf(uploadInfo.getOffset())); + servletResponse.setHeader( + HttpHeader.UPLOAD_OFFSET, String.valueOf(uploadInfo.getOffset())); + } } } } diff --git a/src/main/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandler.java b/src/main/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandler.java index 1f214b83..c2cefa98 100644 --- a/src/main/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandler.java @@ -4,6 +4,7 @@ import java.io.InputStream; import java.security.DigestInputStream; import java.security.MessageDigest; +import java.util.EnumMap; import java.util.Map; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; @@ -73,16 +74,20 @@ public HttpProblemDetails process( } if (uploadInfo != null) { + // Cache full-file digests within this request so having both Repr-Digest and + // Want-Repr-Digest for the same algorithm only streams the file from cloud storage once. + Map fileDigestCache = new EnumMap<>(ChecksumAlgorithm.class); + // 2. Capture client Repr-Digest and Want-Repr-Digest captureDigestPreferences(servletRequest, uploadInfo, uploadStorageService); // 3. Verify Repr-Digest on upload completion verifyReprDigestOnCompletion( - servletRequest, uploadInfo, uploadUri, ownerKey, uploadStorageService); + servletRequest, uploadInfo, uploadUri, ownerKey, uploadStorageService, fileDigestCache); // 4. Provide Repr-Digest response header if requested addReprDigestResponseHeader( - servletResponse, uploadInfo, uploadUri, ownerKey, uploadStorageService); + servletResponse, uploadInfo, uploadUri, ownerKey, uploadStorageService, fileDigestCache); } return null; @@ -148,7 +153,8 @@ private void verifyReprDigestOnCompletion( UploadInfo uploadInfo, String uploadUri, String ownerKey, - UploadStorageService uploadStorageService) + UploadStorageService uploadStorageService, + Map fileDigestCache) throws IOException, TusException { if (!uploadInfo.isUploadInProgress()) { @@ -160,11 +166,15 @@ private void verifyReprDigestOnCompletion( Map expectedDigests = ChecksumAlgorithm.parseDigestHeader(reprDigest); + ChecksumAlgorithm verifiedAlg = null; + String verifiedValue = null; + for (Map.Entry entry : expectedDigests.entrySet()) { ChecksumAlgorithm alg = entry.getKey(); String expectedValue = entry.getValue(); String calculatedValue = - calculateEntireFileDigest(uploadUri, ownerKey, alg, uploadStorageService); + getOrCalculateEntireFileDigest( + uploadInfo, uploadUri, ownerKey, alg, uploadStorageService, fileDigestCache); if (calculatedValue != null) { if (!Strings.CS.equals(expectedValue, calculatedValue)) { @@ -176,13 +186,19 @@ private void verifyReprDigestOnCompletion( + " but was: " + calculatedValue); } else { - // Always save verified checksum on the uploadInfo - uploadInfo.setChecksum(expectedValue); - uploadInfo.setChecksumAlgorithm(alg); - uploadStorageService.update(uploadInfo); + verifiedAlg = alg; + verifiedValue = expectedValue; } } } + + // Only persist the verified checksum (and trigger storage deduplication) after ALL + // provided digest algorithms in Repr-Digest have been verified to match. + if (verifiedAlg != null && verifiedValue != null) { + uploadInfo.setChecksum(verifiedValue); + uploadInfo.setChecksumAlgorithm(verifiedAlg); + uploadStorageService.update(uploadInfo); + } } } } @@ -192,7 +208,8 @@ private void addReprDigestResponseHeader( UploadInfo uploadInfo, String uploadUri, String ownerKey, - UploadStorageService uploadStorageService) + UploadStorageService uploadStorageService, + Map fileDigestCache) throws IOException, TusException { String requestedDigests = uploadInfo.getRequestedRepresentationDigests(); @@ -200,7 +217,13 @@ private void addReprDigestResponseHeader( ChecksumAlgorithm preferredAlg = ChecksumAlgorithm.selectBestAlgorithm(requestedDigests); if (preferredAlg != null) { String calculatedValue = - calculateEntireFileDigest(uploadUri, ownerKey, preferredAlg, uploadStorageService); + getOrCalculateEntireFileDigest( + uploadInfo, + uploadUri, + ownerKey, + preferredAlg, + uploadStorageService, + fileDigestCache); if (calculatedValue != null) { servletResponse.setHeader( HttpHeader.REPR_DIGEST, @@ -210,6 +233,32 @@ private void addReprDigestResponseHeader( } } + private String getOrCalculateEntireFileDigest( + UploadInfo uploadInfo, + String uploadUri, + String ownerKey, + ChecksumAlgorithm alg, + UploadStorageService uploadStorageService, + Map fileDigestCache) + throws IOException, TusException { + if (fileDigestCache != null && fileDigestCache.containsKey(alg)) { + return fileDigestCache.get(alg); + } + if (uploadInfo != null + && alg == uploadInfo.getChecksumAlgorithm() + && StringUtils.isNotBlank(uploadInfo.getChecksum())) { + if (fileDigestCache != null) { + fileDigestCache.put(alg, uploadInfo.getChecksum()); + } + return uploadInfo.getChecksum(); + } + String digest = calculateEntireFileDigest(uploadUri, ownerKey, alg, uploadStorageService); + if (digest != null && fileDigestCache != null) { + fileDigestCache.put(alg, digest); + } + return digest; + } + private String calculateEntireFileDigest( String uploadUri, String ownerKey, diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java index 086e11d4..d0e04627 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java @@ -71,7 +71,9 @@ public HttpProblemDetails process( InterruptibleInputStream interruptibleStream = null; if (is != null) { interruptibleStream = new InterruptibleInputStream(is); - uploadLockingService.registerInputStream(requestUri, interruptibleStream); + if (uploadLockingService != null) { + uploadLockingService.registerInputStream(requestUri, interruptibleStream); + } is = interruptibleStream; try { UploadInfo appended = uploadStorageService.append(uploadInfo, is); @@ -83,21 +85,30 @@ public HttpProblemDetails process( // release request, the storage backend commits all bytes received so far and updates // the offset in storage. We reload the updated UploadInfo and acknowledge the partial // append rather than propagating an unhandled error. - if (interruptibleStream != null && interruptibleStream.isInterrupted()) { - // Refresh UploadInfo first so the log statement and response reflect the true committed - // byte offset persisted by the storage backend up to the interruption point. - UploadInfo refreshed = uploadStorageService.getUploadInfo(requestUri, ownerKey); - if (refreshed != null) { - uploadInfo = refreshed; - } + UploadInfo refreshed = uploadStorageService.getUploadInfo(requestUri, ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } + if (interruptibleStream.isInterrupted()) { log.info( "RUFH append request for URI {} was interrupted by locking service contention; " + "saved partial upload up to offset {}", requestUri, uploadInfo.getOffset()); } else { + // If an un-interrupted network error aborted a deferred-length append that temporarily + // set length from Content-Length + Upload-Complete: ?1, restore length to null before + // rethrowing so the upload remains open with deferred length. + if (!hadLengthInitially && uploadInfo != null) { + uploadInfo.setLength(null); + uploadStorageService.update(uploadInfo); + } throw e; } + } finally { + if (uploadLockingService != null) { + uploadLockingService.unregisterInputStream(requestUri); + } } } diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java index 9ca0442c..28661af5 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java @@ -8,6 +8,7 @@ import me.desair.tus.server.exception.TusException; import me.desair.tus.server.rufh.util.RufhInterimResponseUtil; import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadLock; import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.util.AbstractRequestHandler; @@ -112,34 +113,57 @@ public HttpProblemDetails process( if (hasContent) { InputStream is = servletRequest.getContentInputStream(); if (is != null) { - interruptibleStream = new InterruptibleInputStream(is); - uploadLockingService.registerInputStream(uploadUri, interruptibleStream); - is = interruptibleStream; - try { - UploadInfo appended = uploadStorageService.append(uploadInfo, is); - if (appended != null) { - uploadInfo = appended; + // Acquire the distributed lock for the newly created upload resource if not already locked + // by the outer request so concurrent requests across replicas coordinate via lock + // contention and the registered stream is guaranteed to be cleaned up when the lock closes. + boolean needsLock = + uploadLockingService != null + && uploadInfo != null + && uploadInfo.getId() != null + && !uploadLockingService.isLocked(uploadInfo.getId()); + try (UploadLock ignored = + needsLock ? uploadLockingService.lockUploadByUri(uploadUri) : null) { + interruptibleStream = new InterruptibleInputStream(is); + if (uploadLockingService != null) { + uploadLockingService.registerInputStream(uploadUri, interruptibleStream); } - } catch (IOException e) { - // When an upload stream is interrupted by the locking service watchdog or a concurrent - // lock contention release request (e.g. from a concurrent HEAD or DELETE), the storage - // backend (Disk, S3, Azure) commits all bytes received up to the interruption and updates - // the offset in storage. We reload the updated UploadInfo and acknowledge the partial - // upload rather than propagating an unhandled error to the servlet container. - if (interruptibleStream != null && interruptibleStream.isInterrupted()) { - // Refresh UploadInfo first so the log statement and response reflect the true committed - // byte offset persisted by the storage backend up to the interruption point. + is = interruptibleStream; + try { + UploadInfo appended = uploadStorageService.append(uploadInfo, is); + if (appended != null) { + uploadInfo = appended; + } + } catch (IOException e) { + // When an upload stream is interrupted by the locking service watchdog or a concurrent + // lock contention release request (e.g. from a concurrent HEAD or DELETE), the storage + // backend (Disk, S3, Azure) commits all bytes received up to the interruption and + // updates the offset in storage. We reload the updated UploadInfo and acknowledge the + // partial upload rather than propagating an unhandled error to the servlet container. UploadInfo refreshed = uploadStorageService.getUploadInfo(uploadUri, ownerKey); if (refreshed != null) { uploadInfo = refreshed; } - log.info( - "RUFH creation request with body for URI {} was interrupted by locking service" - + " contention; saved partial upload up to offset {}", - uploadUri, - uploadInfo != null ? uploadInfo.getOffset() : "unknown"); - } else { - throw e; + if (interruptibleStream.isInterrupted()) { + log.info( + "RUFH creation request with body for URI {} was interrupted by locking service" + + " contention; saved partial upload up to offset {}", + uploadUri, + uploadInfo != null ? uploadInfo.getOffset() : "unknown"); + } else { + // If the client socket dropped mid-stream on a deferred-length upload (where length + // was temporarily derived from Content-Length + Upload-Complete: ?1), reset length + // to null in storage before rethrowing so the upload remains resumable with + // deferred length on subsequent PATCH requests. + if (uploadLength == null && uploadInfo != null) { + uploadInfo.setLength(null); + uploadStorageService.update(uploadInfo); + } + throw e; + } + } finally { + if (uploadLockingService != null) { + uploadLockingService.unregisterInputStream(uploadUri); + } } } } diff --git a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java index 8f4e3108..ad79dd28 100644 --- a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java @@ -174,6 +174,18 @@ public void registerInputStream(String requestUri, InputStream inputStream) { } } + @Override + public void unregisterInputStream(String requestUri) { + if (requestUri != null) { + UploadId uploadId = idFactory.readUploadId(requestUri); + if (uploadId != null) { + // Remove stream registration once transfer completes or fails so the background watchdog + // stops polling storage for stop signals on this upload ID. + activeInputStreams.remove(uploadId); + } + } + } + @Override public void requestLockRelease(String requestUri) { if (requestUri == null) { diff --git a/src/main/java/me/desair/tus/server/upload/UploadLockingService.java b/src/main/java/me/desair/tus/server/upload/UploadLockingService.java index f730126e..3675753e 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/UploadLockingService.java @@ -68,6 +68,17 @@ default void registerInputStream(String requestUri, InputStream inputStream) { // No-op by default for backwards compatibility } + /** + * Unregisters the active input stream associated with the given request URI once streaming has + * completed or failed, preventing stale stream references from lingering in memory or being + * repeatedly polled by background watchdog daemon threads across long service lifecycles. + * + * @param requestUri The request URI of the active request + */ + default void unregisterInputStream(String requestUri) { + // No-op by default for backwards compatibility + } + /** * Request that the lock for the given request URI be released. This interrupts the active * request's input stream locally or signals other cluster replicas to interrupt their active diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java index 43d6abab..8378353f 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java @@ -362,6 +362,16 @@ public void registerInputStream(String requestUri, InputStream inputStream) { } } + @Override + public void unregisterInputStream(String requestUri) { + UploadId uploadId = idFactory.readUploadId(requestUri); + if (uploadId != null) { + // Remove stream registration once transfer completes or fails so the background watchdog + // stops polling storage for stop signals on this upload ID. + activeStreams.remove(uploadId.toString()); + } + } + @Override public void requestLockRelease(String requestUri) { UploadId uploadId = idFactory.readUploadId(requestUri); diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java index a3b546d9..40ba0de7 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java @@ -22,6 +22,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; +import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Base64; import java.util.Collections; @@ -207,19 +208,25 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) upload.updateExpiration(uploadExpirationPeriod); } - // 1. Obtain Azure Block Blob client and fetch pre-existing committed block list + // 1. Obtain Azure Block Blob client and fetch pre-existing committed block list. + // Write and modify operations must strictly target the upload's own physical data blob, + // never resolving duplicates to parent blobs, to protect completed parent files. BlockBlobClient blockBlobClient = - containerClient.getBlobClient(getAzureBlobName(upload)).getBlockBlobClient(); - List blockIds = getCommittedBlockIds(blockBlobClient); - - // If metadata records committed uploadPartKeys, ensure the blob's committed blocks do not - // exceed the authoritative metadata offset (e.g. if a prior commitBlockList succeeded - // but updating UploadInfo metadata failed or was interrupted). + containerClient.getBlobClient(getOwnAzureBlobName(upload)).getBlockBlobClient(); List recordedPartKeys = upload.getUploadPartKeys() != null ? upload.getUploadPartKeys() : Collections.emptyList(); - if (blockIds.size() > recordedPartKeys.size()) { + List blockIds; + if (!recordedPartKeys.isEmpty()) { + // Reuse the authoritative committed block IDs recorded in UploadInfo metadata. This avoids + // an extra synchronous listBlocks() network call on every PATCH request and guarantees that + // a transient Azure error cannot return an empty list and overwrite prior blocks. blockIds = new ArrayList<>(recordedPartKeys); - blockBlobClient.commitBlockList(blockIds, true); + } else if (upload.getOffset() != null && upload.getOffset() > 0L) { + // Recovery path: offset > 0 but metadata has no recorded part keys. Fetch committed blocks + // from Azure, propagating non-404 errors so we never commit an empty baseline over data. + blockIds = getCommittedBlockIds(blockBlobClient); + } else { + blockIds = new ArrayList<>(); } // 2. Validate block budget: Azure Block Blobs support up to 50,000 blocks @@ -256,7 +263,9 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) } chunkFile = Files.createTempFile(tempBufferDir, "tus-azure-chunk-", ".tmp").toFile(); - ReadChunkResult chunkResult = readChunk(inputStream, chunkFile, maxBytesToRead); + // Pass interruptibleStream so AsyncChunkUploader's drain timeout or lock contention can + // interrupt blocked reads cleanly even when the caller passed a raw InputStream. + ReadChunkResult chunkResult = readChunk(interruptibleStream, chunkFile, maxBytesToRead); long chunkSize = chunkResult.bytesRead; if (chunkResult.exception != null) { streamException = chunkResult.exception; @@ -420,6 +429,22 @@ public String getAzureBlobName(UploadInfo uploadInfo) { return uploadPrefix + uploadInfo.getId().toString(); } + /** + * Returns the Azure Block Blob name for the upload's own physical data blob without dynamically + * resolving deduplication links. Write and truncate operations must strictly target the upload's + * own physical data blob rather than resolving duplicates to parent blobs, preventing accidental + * truncation or modification of parent files. + * + * @param uploadInfo The upload metadata + * @return The physical blob name for this upload + */ + public String getOwnAzureBlobName(UploadInfo uploadInfo) { + if (uploadInfo == null || uploadInfo.getId() == null) { + return null; + } + return uploadPrefix + uploadInfo.getId().toString(); + } + public String getAzureBlobName(String requestUri, String ownerKey) throws IOException { UploadInfo info = getUploadInfo(requestUri, ownerKey); return info != null ? getAzureBlobName(info) : null; @@ -491,12 +516,13 @@ public void removeLastNumberOfBytes(UploadInfo uploadInfo, long byteCount) return; } - // Write/modify operations MUST NOT resolve duplicates to parent. + // Write and modify operations must strictly target the upload's own physical data blob, + // never resolving duplicates to parent blobs, to protect completed parent files. long currentOffset = uploadInfo.getOffset(); long targetOffset = Math.max(0L, currentOffset - byteCount); BlockBlobClient blockBlobClient = - containerClient.getBlobClient(getAzureBlobName(uploadInfo)).getBlockBlobClient(); + containerClient.getBlobClient(getOwnAzureBlobName(uploadInfo)).getBlockBlobClient(); if (targetOffset == 0) { AzureUtils.deleteBlobQuietly(blockBlobClient); @@ -569,8 +595,17 @@ private void deleteMetadata(UploadId id) { public void cleanupExpiredUploads() throws IOException { Utils.cleanupTempFiles(this.tempBufferDir, "tus-azure-chunk-*.tmp", 24L * 3600_000L); ListBlobsOptions options = new ListBlobsOptions().setPrefix(metadataPrefix); + OffsetDateTime expirationCutoff = AzureUtils.calculateExpirationCutoff(uploadExpirationPeriod); for (BlobItem item : containerClient.listBlobs(options, null)) { if (item.getName().endsWith(".info")) { + // Pre-filter using LIST Last-Modified metadata to avoid an O(N) HTTP GET storm for + // recently updated or created uploads that cannot yet have expired. + if (expirationCutoff != null && item.getProperties() != null) { + OffsetDateTime lastModified = item.getProperties().getLastModified(); + if (lastModified != null && lastModified.isAfter(expirationCutoff)) { + continue; + } + } String infoName = item.getName(); String idStr = infoName.substring(metadataPrefix.length(), infoName.length() - ".info".length()); @@ -713,7 +748,7 @@ private ReadChunkResult readChunk(InputStream is, File targetFile, long maxBytes } /** Retrieves committed block IDs from Azure Block Blob. */ - private List getCommittedBlockIds(BlockBlobClient blockBlobClient) { + private List getCommittedBlockIds(BlockBlobClient blockBlobClient) throws IOException { List blockIds = new ArrayList<>(); try { BlockList blockList = blockBlobClient.listBlocks(BlockListType.COMMITTED); @@ -722,8 +757,12 @@ private List getCommittedBlockIds(BlockBlobClient blockBlobClient) { blockIds.add(block.getName()); } } + } catch (BlobStorageException e) { + if (AzureUtils.parseErrorResponse(e) != AzureErrorType.BLOB_NOT_FOUND) { + throw new IOException("Failed to retrieve committed block list from Azure Blob", e); + } } catch (Exception e) { - log.debug("Could not retrieve committed block list: {}", e.getMessage()); + throw new IOException("Failed to retrieve committed block list from Azure Blob", e); } return blockIds; } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java b/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java index 0bb8d240..1ebc72c9 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java @@ -4,6 +4,8 @@ import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.specialized.BlockBlobClient; +import java.time.Duration; +import java.time.OffsetDateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,6 +27,21 @@ private AzureUtils() { * @return {@code true} if the blob exists; {@code false} if it does not exist, argument is null, * or an error occurs */ + /** + * Calculates the cutoff timestamp before which listed blobs can be considered candidates for + * expiration, subtracting a safety margin to account for clock skew and in-flight writes. + * + * @param uploadExpirationPeriod The upload expiration period in milliseconds + * @return The expiration cutoff timestamp, or null if expiration is disabled + */ + public static OffsetDateTime calculateExpirationCutoff(Long uploadExpirationPeriod) { + if (uploadExpirationPeriod == null || uploadExpirationPeriod <= 0) { + return null; + } + return OffsetDateTime.now() + .minus(Duration.ofMillis(Math.max(0L, uploadExpirationPeriod - 2000L))); + } + public static boolean blobExists(BlobClient blobClient) { if (blobClient == null) { return false; diff --git a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java index 94830da0..c815217e 100644 --- a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java @@ -321,6 +321,11 @@ public void registerInputStream(String requestUri, InputStream inputStream) { lockingServiceDelegate.registerInputStream(requestUri, inputStream); } + @Override + public void unregisterInputStream(String requestUri) { + lockingServiceDelegate.unregisterInputStream(requestUri); + } + @Override public void requestLockRelease(String requestUri) { lockingServiceDelegate.requestLockRelease(requestUri); diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java b/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java index 9b0adf04..6fc75fd5 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java @@ -22,6 +22,7 @@ import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import me.desair.tus.server.upload.UploadInfo; @@ -111,7 +112,14 @@ public S3ServerSideComposeHelper( base.setRegion(region); } prov = new StaticProvider(accessKey, secretKey, null); - client = new OkHttpClient(); + // Server-side UploadPartCopy on multi-hundred-MB or multi-GB S3 parts regularly takes + // longer than OkHttp's default 10-second timeout. Configure generous cloud copy timeouts. + client = + new OkHttpClient.Builder() + .connectTimeout(15, TimeUnit.SECONDS) + .writeTimeout(60, TimeUnit.SECONDS) + .readTimeout(120, TimeUnit.SECONDS) + .build(); MinioAsyncClient.Builder asyncBuilder = MinioAsyncClient.builder().endpoint(endpoint).credentials(accessKey, secretKey); diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index 39e1f337..c2eeba35 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -18,6 +18,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -288,6 +289,25 @@ public String getS3ObjectKey(UploadInfo uploadInfo) { } } + /** + * Returns the S3 object key for this upload's own physical data object without dynamically + * resolving deduplication links. Write and truncate operations must strictly target the upload's + * own physical data object rather than resolving duplicates to parent objects, preventing + * accidental truncation or modification of parent files. + * + * @param uploadInfo The upload metadata + * @return The physical object key for this upload + */ + public String getOwnS3ObjectKey(UploadInfo uploadInfo) { + if (uploadInfo == null || uploadInfo.getId() == null) { + return null; + } + if (uploadInfo.getStorageUploadId() != null) { + return uploadInfo.getStorageUploadId(); + } + return buildObjectKey(uploadInfo.getId()); + } + /** * Return the S3 object key where the uploaded bytes are stored for the given upload URI. * @@ -402,7 +422,9 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) if (upload != null && upload.getLength() != null && info.getLength() == null) { info.setLength(upload.getLength()); } - String objectKey = getS3ObjectKey(info); + // Write and modify operations must strictly target the upload's own physical data object, + // never resolving duplicates to parent objects, to protect completed parent files. + String objectKey = getOwnS3ObjectKey(info); InterruptibleInputStream interruptibleStream = Utils.toInterruptibleStream(inputStream); // Pre-calculate expiration if configured so updated metadata already reflects the renewed @@ -606,9 +628,19 @@ public void cleanupExpiredUploads() throws IOException { minioClient.listObjects( ListObjectsArgs.builder().bucket(bucket).prefix(metadataPrefix).build()); + ZonedDateTime expirationCutoff = S3Utils.calculateExpirationCutoff(uploadExpirationPeriod); + for (Result result : results) { Item item = result.get(); if (item.objectName().endsWith(".info")) { + // Pre-filter using LIST Last-Modified metadata to avoid an O(N) HTTP GET storm for + // recently updated or created uploads that cannot yet have expired. + if (expirationCutoff != null) { + ZonedDateTime lastModified = item.lastModified(); + if (lastModified != null && lastModified.isAfter(expirationCutoff)) { + continue; + } + } String idStr = item.objectName() .substring( @@ -653,7 +685,9 @@ public void removeLastNumberOfBytes(UploadInfo uploadInfo, long byteCount) if (uploadInfo == null || byteCount <= 0) { return; } - String objectKey = getS3ObjectKey(uploadInfo); + // Write and modify operations must strictly target the upload's own physical data object, + // never resolving duplicates to parent objects, to protect completed parent files. + String objectKey = getOwnS3ObjectKey(uploadInfo); long newOffset = Math.max(0L, uploadInfo.getOffset() - byteCount); uploadInfo.setOffset(newOffset); diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java b/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java index 8ebbd8e2..56afbbfb 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java @@ -14,6 +14,8 @@ import io.minio.messages.Item; import java.io.IOException; import java.io.InputStream; +import java.time.Duration; +import java.time.ZonedDateTime; import java.util.Collection; import java.util.List; import java.util.Objects; @@ -49,6 +51,21 @@ public static MinioClient buildMinioClient( .build(); } + /** + * Calculates the cutoff timestamp before which listed objects can be considered candidates for + * expiration, subtracting a safety margin to account for clock skew and in-flight writes. + * + * @param uploadExpirationPeriod The upload expiration period in milliseconds + * @return The expiration cutoff timestamp, or null if expiration is disabled + */ + public static ZonedDateTime calculateExpirationCutoff(Long uploadExpirationPeriod) { + if (uploadExpirationPeriod == null || uploadExpirationPeriod <= 0) { + return null; + } + return ZonedDateTime.now() + .minus(Duration.ofMillis(Math.max(0L, uploadExpirationPeriod - 2000L))); + } + /** * Retrieves an {@link InputStream} for reading the contents of an S3 object. * diff --git a/src/main/java/me/desair/tus/server/util/HttpChunkedEncodingInputStream.java b/src/main/java/me/desair/tus/server/util/HttpChunkedEncodingInputStream.java index a12f56f1..7e7cb679 100644 --- a/src/main/java/me/desair/tus/server/util/HttpChunkedEncodingInputStream.java +++ b/src/main/java/me/desair/tus/server/util/HttpChunkedEncodingInputStream.java @@ -1,6 +1,7 @@ package me.desair.tus.server.util; import java.io.ByteArrayOutputStream; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; @@ -8,6 +9,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,6 +24,9 @@ public class HttpChunkedEncodingInputStream extends InputStream { private static final Logger log = LoggerFactory.getLogger(HttpChunkedEncodingInputStream.class); + public static final int MAX_LINE_LENGTH = 8192; + public static final int MAX_TRAILER_HEADERS = 64; + /** The input stream that we're wrapping. */ private InputStream in; @@ -89,8 +94,17 @@ public int read() throws IOException { return -1; } } + int b = in.read(); + if (b == -1) { + eof = true; + throw new EOFException( + "Unexpected EOF while reading chunked stream: expected " + + chunkSize + + " bytes, got " + + pos); + } pos++; - return in.read(); + return b; } /** @@ -121,6 +135,14 @@ public int read(byte[] b, int off, int len) throws IOException { } int minLen = Math.min(len, chunkSize - pos); int count = in.read(b, off, minLen); + if (count == -1) { + eof = true; + throw new EOFException( + "Unexpected EOF while reading chunked stream: expected " + + chunkSize + + " bytes, got " + + pos); + } pos += count; return count; } @@ -207,6 +229,10 @@ private String readChunkSizeInformation() throws IOException { throw new IOException("Chunked stream ended unexpectedly"); } state = state.process(in, baos, b); + if (baos.size() > MAX_LINE_LENGTH) { + throw new IOException( + "Chunk size line exceeds maximum allowed length of " + MAX_LINE_LENGTH + " bytes"); + } } // parse data @@ -261,7 +287,14 @@ private List> parseHeaders(InputStream is, Charset charset) String name = null; StringBuilder value = null; String line = readLine(is, charset); - while (org.apache.commons.lang3.StringUtils.isNotBlank(line)) { + int headerLines = 0; + while (StringUtils.isNotBlank(line)) { + headerLines++; + if (headerLines > MAX_TRAILER_HEADERS) { + throw new IOException( + "Chunked stream exceeds maximum allowed trailer header count of " + + MAX_TRAILER_HEADERS); + } // Parse the header name and value // Check for folded headers first // Detect LWS-char see HTTP/1.0 or HTTP/1.1 Section 2.2 @@ -329,6 +362,10 @@ private byte[] readRawLine(InputStream inputStream) throws IOException { int ch; while ((ch = inputStream.read()) >= 0) { buf.write(ch); + if (buf.size() > MAX_LINE_LENGTH) { + throw new IOException( + "Trailer header line exceeds maximum allowed length of " + MAX_LINE_LENGTH + " bytes"); + } if (ch == '\n') { // be tolerant (RFC-2616 Section 19.3) break; } diff --git a/src/main/java/me/desair/tus/server/util/TusServletRequest.java b/src/main/java/me/desair/tus/server/util/TusServletRequest.java index 39267949..53dc4db5 100644 --- a/src/main/java/me/desair/tus/server/util/TusServletRequest.java +++ b/src/main/java/me/desair/tus/server/util/TusServletRequest.java @@ -9,6 +9,7 @@ import java.security.MessageDigest; import java.util.Arrays; import java.util.EnumMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -79,20 +80,20 @@ public InputStream getContentInputStream() throws IOException { ChecksumAlgorithm checksumAlgorithm = ChecksumAlgorithm.forUploadChecksumHeader(getHeader(HttpHeader.UPLOAD_CHECKSUM)); - java.util.Set algorithms = new java.util.LinkedHashSet<>(); - - if (isChunked) { - // Since the Checksum header can still come at the end, keep track of all checksums + Set algorithms = new LinkedHashSet<>(); + if (checksumAlgorithm != null) { + algorithms.add(checksumAlgorithm); + } + algorithms.addAll( + ChecksumAlgorithm.parseDigestHeader(getHeader(HttpHeader.CONTENT_DIGEST)).keySet()); + algorithms.addAll( + ChecksumAlgorithm.parseDigestHeader(getHeader(HttpHeader.WANT_CONTENT_DIGEST)).keySet()); + + // Only compute all 5 digest algorithms when Transfer-Encoding is chunked AND no explicit + // checksum or digest header was already provided in the initial request headers (meaning a + // trailing Upload-Checksum or Content-Digest header may arrive at the end of the stream). + if (isChunked && algorithms.isEmpty()) { algorithms.addAll(Arrays.asList(ChecksumAlgorithm.values())); - } else { - if (checksumAlgorithm != null) { - algorithms.add(checksumAlgorithm); - } - algorithms.addAll( - ChecksumAlgorithm.parseDigestHeader(getHeader(HttpHeader.CONTENT_DIGEST)).keySet()); - algorithms.addAll( - ChecksumAlgorithm.parseDigestHeader(getHeader(HttpHeader.WANT_CONTENT_DIGEST)) - .keySet()); } for (ChecksumAlgorithm algorithm : algorithms) { diff --git a/src/main/java/me/desair/tus/server/util/Utils.java b/src/main/java/me/desair/tus/server/util/Utils.java index ef6a7c67..b65a0864 100644 --- a/src/main/java/me/desair/tus/server/util/Utils.java +++ b/src/main/java/me/desair/tus/server/util/Utils.java @@ -13,7 +13,9 @@ import java.util.EnumSet; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; @@ -43,6 +45,8 @@ public class Utils { private static final Logger log = LoggerFactory.getLogger(Utils.class); private static final Pattern CHECKSUM_VALUE_PATTERN = Pattern.compile("^[a-zA-Z0-9+/=\\-_]+$"); + private static final Map CREATION_URI_PATTERN_CACHE = new ConcurrentHashMap<>(); + private static final int MAX_PATTERN_CACHE_SIZE = 64; private Utils() { // This is a utility class that only holds static utility methods @@ -375,16 +379,28 @@ public static boolean isCreationEndpoint( String normalizedBase = Strings.CS.endsWith(baseUri, "/") ? baseUri.substring(0, baseUri.length() - 1) : baseUri; String patternStr = "^" + normalizedBase + "/?$"; - try { - return Pattern.compile(patternStr).matcher(requestUri).matches(); - } catch (Exception e) { - log.debug("Invalid regex in upload URI pattern: {}", baseUri, e); - return false; - } + Pattern pattern = getPatternFromCache(patternStr); + return pattern != null && pattern.matcher(requestUri).matches(); } return false; } + private static Pattern getPatternFromCache(String patternStr) { + try { + Pattern pattern = CREATION_URI_PATTERN_CACHE.get(patternStr); + if (pattern == null) { + pattern = Pattern.compile(patternStr); + if (CREATION_URI_PATTERN_CACHE.size() < MAX_PATTERN_CACHE_SIZE) { + CREATION_URI_PATTERN_CACHE.put(patternStr, pattern); + } + } + return pattern; + } catch (Exception e) { + log.debug("Invalid regex in upload URI pattern: {}", patternStr, e); + return null; + } + } + /** * Determine if the given HTTP servlet request target URI represents an existing upload resource. * diff --git a/src/test/java/me/desair/tus/server/AbstractITRufhProtocol.java b/src/test/java/me/desair/tus/server/AbstractITRufhProtocol.java index 5fa5b978..21181e08 100644 --- a/src/test/java/me/desair/tus/server/AbstractITRufhProtocol.java +++ b/src/test/java/me/desair/tus/server/AbstractITRufhProtocol.java @@ -669,6 +669,64 @@ public void testContentDigestValidation() throws Exception { assertResponseHeader(HttpHeader.UPLOAD_OFFSET, "12"); } + /** + * Section 3 of RFC 9530 (Content-Digest): "The Content-Digest HTTP header field associates one or + * more digests with a message content." When using chunked transfer encoding, digests can be sent + * in trailing header fields. + * + *

Use Case: Send a chunked PATCH payload without initial Content-Digest header, supplying the + * digest only in the chunked trailer. Verify the server calculates digests across algorithms and + * validates the trailer header successfully, returning HTTP 204 No Content for a match and HTTP + * 400 Bad Request for a mismatch. + */ + @Test + public void testChunkedPatchWithTrailingContentDigestValidation() throws Exception { + tusFileUploadService.withChunkedTransferDecoding(true); + + // Step 1: Create upload resource + servletRequest.setMethod("POST"); + servletRequest.setRequestURI(UPLOAD_URI); + servletRequest.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + servletRequest.addHeader(HttpHeader.UPLOAD_LENGTH, "100"); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_CREATED); + String uploadLocation = servletResponse.getHeader(HttpHeader.LOCATION); + + // Step 2: Send chunked PATCH with valid trailing Content-Digest for "hello digest" + // (12 bytes -> 0xC hex chunk size) + String chunkedBody = + "C\r\nhello digest\r\n0\r\nContent-Digest: sha-256=:yV9g7MInOPrtlLDWsplfHK0LaH22Uz70R1ZXbHIjzjU=:\r\n\r\n"; + + resetChunked(); + servletRequest.setMethod("PATCH"); + servletRequest.setRequestURI(uploadLocation); + servletRequest.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + servletRequest.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + servletRequest.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + servletRequest.setContent(chunkedBody.getBytes(StandardCharsets.UTF_8)); + + UploadInfo updated = tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_NO_CONTENT); + assertNotNull(updated); + assertEquals(Long.valueOf(12L), updated.getOffset()); + + // Step 3: Send chunked PATCH with invalid trailing Content-Digest -> 400 Bad Request + String invalidChunkedBody = + "C\r\nhello again!\r\n0\r\nContent-Digest: sha-256=:invalid-digest-here=:\r\n\r\n"; + + resetChunked(); + servletRequest.setMethod("PATCH"); + servletRequest.setRequestURI(uploadLocation); + servletRequest.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + servletRequest.addHeader(HttpHeader.UPLOAD_OFFSET, "12"); + servletRequest.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + servletRequest.setContent(invalidChunkedBody.getBytes(StandardCharsets.UTF_8)); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_BAD_REQUEST); + } + @Test public void testUploadWithAbsoluteUploadUri() throws Exception { String absoluteBaseUri = "https://uploads.example.com"; diff --git a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java index e4588abd..53df7adc 100644 --- a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java +++ b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java @@ -558,7 +558,8 @@ public void testGetRawInterimResponse() throws Exception { request.setMethod("GET"); org.junit.Assert.assertNull(service.getRawInterimResponse(request, "owner-123")); - // Test AUTO protocol service with TUS 1.0.0 request returns null for RUFH 104 interim response + // Test AUTO protocol service with valid TUS 1.0.0 request returns null (no 104 interim response + // for Tus 1.0.0) TusFileUploadService autoService = new TusFileUploadService() .withSupportedProtocolVersions(ProtocolVersion.AUTO) @@ -568,11 +569,55 @@ public void testGetRawInterimResponse() throws Exception { tus10Request.setMethod("POST"); tus10Request.setRequestURI("/files"); tus10Request.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + tus10Request.addHeader(HttpHeader.UPLOAD_LENGTH, "1000"); org.junit.Assert.assertNull(autoService.getRawInterimResponse(tus10Request, "owner-123")); // Test null input org.junit.Assert.assertNull( service.getRawInterimResponse((jakarta.servlet.http.HttpServletRequest) null, "owner-123")); + + // Test invalid creation request where upload length exceeds maxUploadSize: validation fails + // and returns null before any storage creation takes place (error response handled by main + // pipeline) + TusFileUploadService sizeLimitedService = + new TusFileUploadService() + .withSupportedProtocolVersions(ProtocolVersion.RUFH) + .withUploadUri("/files") + .withMaxUploadSize(100L); + org.springframework.mock.web.MockHttpServletRequest oversizedRequest = + new org.springframework.mock.web.MockHttpServletRequest(); + oversizedRequest.setMethod("POST"); + oversizedRequest.setRequestURI("/files"); + oversizedRequest.addHeader(HttpHeader.UPLOAD_LENGTH, "500"); + oversizedRequest.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + org.junit.Assert.assertNull( + sizeLimitedService.getRawInterimResponse(oversizedRequest, "owner-123")); + + // Test path traversal attempt: validation fails and returns null + org.springframework.mock.web.MockHttpServletRequest traversalRequest = + new org.springframework.mock.web.MockHttpServletRequest(); + traversalRequest.setMethod("POST"); + traversalRequest.setRequestURI("/files/../secret"); + traversalRequest.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + org.junit.Assert.assertNull(service.getRawInterimResponse(traversalRequest, "owner-123")); + + // Test Tus 1.0.0 request with invalid metadata fails validation and returns null + org.springframework.mock.web.MockHttpServletRequest invalidTus10Request = + new org.springframework.mock.web.MockHttpServletRequest(); + invalidTus10Request.setMethod("POST"); + invalidTus10Request.setRequestURI("/files"); + invalidTus10Request.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + invalidTus10Request.addHeader(HttpHeader.UPLOAD_METADATA, "file\tname d29ybGQ="); + org.junit.Assert.assertNull( + autoService.getRawInterimResponse(invalidTus10Request, "owner-123")); + + org.springframework.mock.web.MockHttpServletRequest unsupportedMethodRequest = + new org.springframework.mock.web.MockHttpServletRequest(); + unsupportedMethodRequest.setMethod("CUSTOM"); + unsupportedMethodRequest.setRequestURI("/files"); + unsupportedMethodRequest.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + org.junit.Assert.assertNull( + service.getRawInterimResponse(unsupportedMethodRequest, "owner-123")); } @Test diff --git a/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java b/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java index 9ee60c20..1526722b 100644 --- a/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java @@ -257,6 +257,7 @@ public void testProcessInterruptedByLockingServiceContentionReturns204WithUpdate assertThat(servletResponse.getStatus(), is(HttpServletResponse.SC_NO_CONTENT)); assertThat(servletResponse.getHeader(HttpHeader.UPLOAD_OFFSET), is("50")); + verify(mockLocking).unregisterInputStream("/test/upload/123"); } @Test(expected = IOException.class) @@ -276,13 +277,17 @@ public void testProcessUninterruptedIoExceptionRethrown() throws Exception { when(uploadStorageService.append(any(UploadInfo.class), any(InputStream.class))) .thenThrow(new IOException("Disk failure")); - handler.process( - HttpMethod.PATCH, - new TusServletRequest(servletRequest), - new TusServletResponse(servletResponse), - uploadStorageService, - mockLocking, - "owner", - null); + try { + handler.process( + HttpMethod.PATCH, + new TusServletRequest(servletRequest), + new TusServletResponse(servletResponse), + uploadStorageService, + mockLocking, + "owner", + null); + } finally { + verify(mockLocking).unregisterInputStream("/test/upload/123"); + } } } diff --git a/src/test/java/me/desair/tus/server/creation/validation/UploadMetadataValidatorTest.java b/src/test/java/me/desair/tus/server/creation/validation/UploadMetadataValidatorTest.java index 58b69ab6..fb5796d9 100644 --- a/src/test/java/me/desair/tus/server/creation/validation/UploadMetadataValidatorTest.java +++ b/src/test/java/me/desair/tus/server/creation/validation/UploadMetadataValidatorTest.java @@ -5,6 +5,7 @@ import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; +import me.desair.tus.server.exception.InvalidUploadMetadataException; import me.desair.tus.server.exception.TusException; import org.junit.Before; import org.junit.Test; @@ -57,4 +58,38 @@ public void validateInvalidBase64() throws Exception { servletRequest.addHeader(HttpHeader.UPLOAD_METADATA, "filename invalid%base64"); validator.validate(HttpMethod.POST, servletRequest, null, null); } + + @Test(expected = InvalidUploadMetadataException.class) + public void validateOversizedHeaderThrowsException() throws Exception { + // Generate an Upload-Metadata header larger than 65536 bytes to prevent DoS memory spikes + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 7000; i++) { + sb.append("k").append(i).append(" dmFsdWU=,"); + } + servletRequest.addHeader(HttpHeader.UPLOAD_METADATA, sb.toString()); + validator.validate(HttpMethod.POST, servletRequest, null, null); + } + + @Test + public void validateNonAsciiKeyIsAllowed() throws Exception { + // Non-ASCII characters in key (e.g. ñ) are permitted for backward compatibility + servletRequest.addHeader(HttpHeader.UPLOAD_METADATA, "fileñame d29ybGQ="); + validator.validate(HttpMethod.POST, servletRequest, null, null); + // KISS: verifying method executes cleanly without throwing an exception + org.junit.Assert.assertTrue(true); + } + + @Test(expected = InvalidUploadMetadataException.class) + public void validateKeyWithWhitespaceThrowsException() throws Exception { + // Key containing spaces or whitespace is invalid + servletRequest.addHeader(HttpHeader.UPLOAD_METADATA, "file name d29ybGQ="); + validator.validate(HttpMethod.POST, servletRequest, null, null); + } + + @Test(expected = InvalidUploadMetadataException.class) + public void validateKeyWithTabWhitespaceThrowsException() throws Exception { + // Key containing tab whitespace fails isValidKey check + servletRequest.addHeader(HttpHeader.UPLOAD_METADATA, "file\tname d29ybGQ="); + validator.validate(HttpMethod.POST, servletRequest, null, null); + } } diff --git a/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java b/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java index dac31b03..57e99d9e 100644 --- a/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java @@ -69,6 +69,46 @@ public void testProcessWithLockingService() throws Exception { ArgumentCaptor.forClass(InterruptibleInputStream.class); verify(lockingService).registerInputStream(eq("/files/123"), captor.capture()); assertNotNull(captor.getValue()); + verify(lockingService).unregisterInputStream(eq("/files/123")); + verify(response).setHeader(HttpHeader.UPLOAD_OFFSET, "5"); + } + + @Test + public void testProcessWithLockingServiceAcquiresLockAndCleansUp() throws Exception { + TusServletRequest request = mock(TusServletRequest.class); + TusServletResponse response = mock(TusServletResponse.class); + UploadStorageService storageService = mock(UploadStorageService.class); + UploadLockingService lockingService = mock(UploadLockingService.class); + me.desair.tus.server.upload.UploadLock mockLock = + mock(me.desair.tus.server.upload.UploadLock.class); + + when(request.getHeader(HttpHeader.CONTENT_LENGTH)).thenReturn("5"); + when(request.hasContent()).thenReturn(true); + when(response.getHeader(HttpHeader.LOCATION)).thenReturn("/files/123"); + when(request.getContentInputStream()).thenReturn(new ByteArrayInputStream("hello".getBytes())); + + me.desair.tus.server.upload.UploadId id = new me.desair.tus.server.upload.UploadId("123"); + UploadInfo uploadInfo = new UploadInfo(); + uploadInfo.setId(id); + uploadInfo.setLength(10L); + uploadInfo.setOffset(0L); + + UploadInfo updatedInfo = new UploadInfo(); + updatedInfo.setId(id); + updatedInfo.setLength(10L); + updatedInfo.setOffset(5L); + + when(lockingService.isLocked(id)).thenReturn(false); + when(lockingService.lockUploadByUri("/files/123")).thenReturn(mockLock); + when(storageService.getUploadInfo("/files/123", "owner")).thenReturn(uploadInfo); + when(storageService.append(eq(uploadInfo), any(InputStream.class))).thenReturn(updatedInfo); + + handler.process( + HttpMethod.POST, request, response, storageService, lockingService, "owner", null); + + verify(lockingService).lockUploadByUri("/files/123"); + verify(mockLock).close(); + verify(lockingService).unregisterInputStream("/files/123"); verify(response).setHeader(HttpHeader.UPLOAD_OFFSET, "5"); } diff --git a/src/test/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandlerTest.java b/src/test/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandlerTest.java index 928e5555..e2b95927 100644 --- a/src/test/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandlerTest.java @@ -6,6 +6,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -243,4 +244,83 @@ public void testProcessWithWantContentDigestNoBodyRead() throws Exception { assertThat( mockResp.getHeader(HttpHeader.CONTENT_DIGEST), org.hamcrest.CoreMatchers.nullValue()); } + + @Test + public void testProcessMultiAlgorithmReprDigestWhereOneFailsDoesNotCallUpdateAndThrows() + throws Exception { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.setMethod("PATCH"); + request.setRequestURI("/files/123"); + // sha-256 matches "hello", but sha-512 has an invalid digest value + request.addHeader( + HttpHeader.REPR_DIGEST, + "sha-256=:LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ=:, sha-512=:invalid-512-digest=:"); + + TusServletRequest servletRequest = new TusServletRequest(request); + TusServletResponse servletResponse = + new TusServletResponse(new org.springframework.mock.web.MockHttpServletResponse()); + + UploadInfo info = new UploadInfo(); + info.setLength(5L); + info.setOffset(5L); // Complete upload + info.setRepresentationDigest( + "sha-256=:LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ=:, sha-512=:invalid-512-digest=:"); + when(uploadStorageService.getUploadInfo("/files/123", "owner")).thenReturn(info); + when(uploadStorageService.getUploadedBytes("/files/123", "owner")) + .thenReturn(new ByteArrayInputStream("hello".getBytes(StandardCharsets.UTF_8))); + + assertThrows( + UploadDigestMismatchException.class, + () -> + handler.process( + HttpMethod.PATCH, + servletRequest, + servletResponse, + uploadStorageService, + uploadLockingService, + "owner", + null)); + + // Because validation is atomic across all algorithms, storageService.update must NEVER be + // called + verify(uploadStorageService, never()).update(any()); + } + + @Test + public void testProcessReprDigestAndWantReprDigestDownloadsFileOnlyOnceDueToCache() + throws Exception { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.setMethod("PATCH"); + request.setRequestURI("/files/123"); + request.addHeader( + HttpHeader.REPR_DIGEST, "sha-256=:LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ=:"); + request.addHeader(HttpHeader.WANT_REPR_DIGEST, "sha-256"); + + TusServletRequest servletRequest = new TusServletRequest(request); + MockHttpServletResponse mockResp = new MockHttpServletResponse(); + TusServletResponse servletResponse = new TusServletResponse(mockResp); + + UploadInfo info = new UploadInfo(); + info.setLength(5L); + info.setOffset(5L); // Complete upload + when(uploadStorageService.getUploadInfo("/files/123", "owner")).thenReturn(info); + when(uploadStorageService.getUploadedBytes("/files/123", "owner")) + .thenReturn(new ByteArrayInputStream("hello".getBytes(StandardCharsets.UTF_8))); + + handler.process( + HttpMethod.PATCH, + servletRequest, + servletResponse, + uploadStorageService, + uploadLockingService, + "owner", + null); + + // Verify storageService.getUploadedBytes was called only once and reused from request cache + verify(uploadStorageService, times(1)).getUploadedBytes("/files/123", "owner"); + verify(uploadStorageService, times(2)).update(info); + assertThat( + mockResp.getHeader(HttpHeader.REPR_DIGEST), + is("sha-256=:LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ=:")); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java index 547239ce..e46b2d00 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java @@ -482,5 +482,49 @@ public void testProcessInterruptedByLockContentionWhenDeferredLength() throws Ex verify(storageService, org.mockito.Mockito.atLeastOnce()).update(captor.capture()); // The last update before returning must reset length back to null for deferred upload resume org.junit.Assert.assertNull(captor.getValue().getLength()); + + // Stream must be unregistered from locking service to clean up active input stream tracking + verify(lockingService).unregisterInputStream("/files/append-id"); + } + + @Test(expected = IOException.class) + public void testProcessUninterruptedIoExceptionWithDeferredLengthResetsLengthAndRethrows() + throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/append-uninterrupted-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "1000"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + request.setContent("hello world".getBytes()); + + UploadInfo initial = new UploadInfo(); + initial.setId(new UploadId("append-uninterrupted-id")); + initial.setOffset(1000L); + initial.setLength(null); // Deferred length initially + + when(storageService.getUploadInfo("/files/append-uninterrupted-id", "owner")) + .thenReturn(initial); + + when(storageService.append(eq(initial), any(InputStream.class))) + .thenThrow(new IOException("Simulated network drop / connection reset")); + + try { + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + } finally { + // Must reset length back to null so future append requests can resume + ArgumentCaptor captor = ArgumentCaptor.forClass(UploadInfo.class); + verify(storageService, org.mockito.Mockito.atLeastOnce()).update(captor.capture()); + org.junit.Assert.assertNull(captor.getValue().getLength()); + + // Input stream must be cleanly unregistered even on failure + verify(lockingService).unregisterInputStream("/files/append-uninterrupted-id"); + } } } diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java index 2dacb2db..e902fb76 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java @@ -18,6 +18,7 @@ import me.desair.tus.server.upload.NoOpUploadLockingService; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadLock; import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.util.InterruptibleInputStream; @@ -783,5 +784,79 @@ public void testProcessCreationInterruptedByLockContentionWhenDeferredLength() t verify(storageService, org.mockito.Mockito.atLeastOnce()).update(captor.capture()); // The update on interruption must reset length to null so deferred upload can be resumed org.junit.Assert.assertNull(captor.getValue().getLength()); + + // Verify stream was unregistered upon exit to avoid leaking in active input streams map + verify(lockingService).unregisterInputStream("/files/creation-id"); + } + + @Test(expected = IOException.class) + public void testProcessUninterruptedIoExceptionWithDeferredLengthResetsLengthAndRethrows() + throws Exception { + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + request.setContent("creation body".getBytes()); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("creation-uninterrupted-id")); + info.setLength(13L); + info.setOffset(0L); + + when(storageService.create(any(UploadInfo.class), nullable(String.class))).thenReturn(info); + when(storageService.append(eq(info), any(InputStream.class))) + .thenThrow(new IOException("Simulated network drop / client reset")); + + try { + handler.process( + HttpMethod.POST, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + } finally { + // Must reset length to null so subsequent append requests can finish the deferred upload + ArgumentCaptor captor = ArgumentCaptor.forClass(UploadInfo.class); + verify(storageService, org.mockito.Mockito.atLeastOnce()).update(captor.capture()); + org.junit.Assert.assertNull(captor.getValue().getLength()); + + // Lock input stream registration must be cleaned up even when exception is rethrown + verify(lockingService).unregisterInputStream("/files/creation-uninterrupted-id"); + } + } + + @Test + public void testProcessCreationAcquiresLockWhenNullLockProvided() throws Exception { + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_LENGTH, "100"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + request.setContent("data".getBytes()); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("locked-id")); + info.setLength(100L); + info.setOffset(0L); + + when(storageService.create(any(UploadInfo.class), nullable(String.class))).thenReturn(info); + when(storageService.append(any(UploadInfo.class), any())).thenReturn(info); + + UploadLock mockLock = org.mockito.Mockito.mock(UploadLock.class); + when(lockingService.lockUploadByUri("/files/locked-id")).thenReturn(mockLock); + + handler.process( + HttpMethod.POST, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + // Verify lock was acquired for newly created location and closed when finished + verify(lockingService).lockUploadByUri("/files/locked-id"); + verify(mockLock).close(); + verify(lockingService).unregisterInputStream("/files/locked-id"); } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java index 35c2d804..c5f77ad0 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java @@ -98,6 +98,22 @@ public void registerInputStreamShouldDoNothingOnInvalidUriOrStandardStream() { lockingService.registerInputStream(null, bais); } + @Test + public void testUnregisterInputStreamRemovesFromActiveStreams() { + ByteArrayInputStream bais = new ByteArrayInputStream("data".getBytes()); + InterruptibleInputStream stream = new InterruptibleInputStream(bais); + + lockingService.registerInputStream("/test/upload/88888", stream); + assertTrue(lockingService.activeStreams.containsKey("88888")); + + lockingService.unregisterInputStream("/test/upload/88888"); + assertFalse(lockingService.activeStreams.containsKey("88888")); + + // Null or invalid URI handles gracefully without throwing + lockingService.unregisterInputStream(null); + lockingService.unregisterInputStream("invalid-uri"); + } + @Test public void requestLockReleaseShouldDoNothingOnInvalidUri() { // KISS: verifying request release on invalid URI is no-op and does not throw diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java index 22ca8623..21d3f266 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java @@ -133,6 +133,19 @@ public void getAzureBlobNameShouldReturnBlobName() throws Exception { assertEquals("uploads/parent-999", storageService.getAzureBlobName(info)); } + @Test + public void getOwnAzureBlobNameShouldReturnChildBlobName() { + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("12345")); + assertEquals("uploads/12345", storageService.getOwnAzureBlobName(info)); + + // For deduplicated child uploads, getOwnAzureBlobName MUST return the child's own blob name + // (not parent's) to protect parent blobs from write and truncation operations + info.setDuplicatesUploadId(new UploadId("parent-999")); + assertEquals("uploads/12345", storageService.getOwnAzureBlobName(info)); + assertNull(storageService.getOwnAzureBlobName(null)); + } + @Test public void getAzureBlobNameNullInfoShouldReturnNull() throws Exception { assertNull(storageService.getAzureBlobName((UploadInfo) null)); diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java index 7303fec2..f0c04c8b 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java @@ -1060,6 +1060,28 @@ public void testAppendTruncatesCommittedBlocksWhenExceedingAuthoritativeMetadata assertEquals(2, appended.getUploadPartKeys().size()); } + @Test + public void testAppendRecoversCommittedBlocksFromAzureWhenUploadPartKeysNull() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(200L); + UploadInfo created = storageService.create(info, "owner1"); + + // First append writes 100 bytes (creating block in Azure) + UploadInfo first = storageService.append(created, new ByteArrayInputStream(new byte[100])); + assertEquals(Long.valueOf(100L), first.getOffset()); + assertEquals(1, first.getUploadPartKeys().size()); + + // Clear uploadPartKeys in metadata to simulate pre-existing upload or lost metadata keys + first.setUploadPartKeys(null); + storageService.update(first); + + // Second append writes another 50 bytes: offset > 0 but uploadPartKeys is empty. + // This exercises the getCommittedBlockIds recovery path in AzureBlobStorageService. + UploadInfo second = storageService.append(first, new ByteArrayInputStream(new byte[50])); + assertEquals(Long.valueOf(150L), second.getOffset()); + assertEquals(2, second.getUploadPartKeys().size()); + } + @Test public void testTerminateUploadWithUnleasedLockBlob() throws Exception { UploadInfo info = new UploadInfo(); diff --git a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java index bbb3ff8c..c147c4f5 100644 --- a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java @@ -252,6 +252,9 @@ public void testDelegateMethods() throws Exception { service.registerInputStream("/files/1", is); verify(mockLocking, times(1)).registerInputStream("/files/1", is); + service.unregisterInputStream("/files/1"); + verify(mockLocking, times(1)).unregisterInputStream("/files/1"); + service.requestLockRelease("/files/1"); verify(mockLocking, times(1)).requestLockRelease("/files/1"); diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index be126532..9334b187 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -3120,6 +3120,41 @@ public void testPruneOrphanedChecksumIndicesMalformedPathIgnored() throws Except assertTrue(true); } + @Test + public void testGetOwnS3ObjectKeyShouldReturnChildKey() { + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("child-s3-123")); + assertEquals("uploads/child-s3-123", storageService.getOwnS3ObjectKey(info)); + + // For deduplicated child uploads, getOwnS3ObjectKey MUST return the child's own object key + // (not parent's) to protect parent data from write and truncation operations + info.setDuplicatesUploadId(new UploadId("parent-s3-999")); + assertEquals("uploads/child-s3-123", storageService.getOwnS3ObjectKey(info)); + assertNull(storageService.getOwnS3ObjectKey(null)); + } + + @Test + public void testCleanupExpiredUploadsSkipsRecentlyModifiedObjects() throws Exception { + storageService.setUploadExpirationPeriod(3600000L); // 1 hour + + Item freshItem = mock(Item.class); + when(freshItem.objectName()).thenReturn("uploads/fresh-upload.info"); + // Modified 5 minutes ago (newer than 1 hour expiration threshold) + when(freshItem.lastModified()).thenReturn(java.time.ZonedDateTime.now().minusMinutes(5)); + + Result result = new Result<>(freshItem); + when(minioClient.listObjects(any(ListObjectsArgs.class))) + .thenReturn(Collections.singletonList(result)); + + storageService.cleanupExpiredUploads(null); + + // Should skip fetching the .info object because it was recently modified + verify(minioClient, never()) + .getObject( + argThat( + (GetObjectArgs a) -> a != null && a.object().equals("uploads/fresh-upload.info"))); + } + @Test public void testIsJsonSerializationEnabled() { assertTrue(storageService.isJsonSerializationEnabled()); diff --git a/src/test/java/me/desair/tus/server/util/HttpChunkedEncodingInputStreamTest.java b/src/test/java/me/desair/tus/server/util/HttpChunkedEncodingInputStreamTest.java index 85a6bf73..f93894ee 100644 --- a/src/test/java/me/desair/tus/server/util/HttpChunkedEncodingInputStreamTest.java +++ b/src/test/java/me/desair/tus/server/util/HttpChunkedEncodingInputStreamTest.java @@ -4,6 +4,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.io.StringWriter; @@ -283,4 +284,88 @@ public void testNegativeChunkSize() throws IOException { byte[] byteArray = new byte[10]; in.read(byteArray); } + + @Test(expected = EOFException.class) + public void testMidChunkEofSingleByteRead() throws IOException { + // Declares chunk size 10 (0xA), but stream only delivers 4 bytes and ends prematurely + String truncatedInput = "A\r\n1234"; + try (HttpChunkedEncodingInputStream in = + new HttpChunkedEncodingInputStream( + IOUtils.toInputStream(truncatedInput, StandardCharsets.UTF_8), trailerHeaders)) { + for (int i = 0; i < 10; i++) { + in.read(); + } + } + } + + @Test(expected = EOFException.class) + public void testMidChunkEofBlockRead() throws IOException { + // Declares chunk size 10 (0xA), but stream only delivers 4 bytes and ends prematurely + String truncatedInput = "A\r\n1234"; + try (HttpChunkedEncodingInputStream in = + new HttpChunkedEncodingInputStream( + IOUtils.toInputStream(truncatedInput, StandardCharsets.UTF_8), trailerHeaders)) { + byte[] buffer = new byte[10]; + int totalRead = 0; + while (totalRead < 10) { + int read = in.read(buffer, totalRead, 10 - totalRead); + if (read == -1) { + break; + } + totalRead += read; + } + } + } + + @Test(expected = IOException.class) + public void testChunkSizeLineExceedingMaxLineLengthThrowsIOException() throws IOException { + // Build a chunk size line exceeding 8192 characters (e.g. 8200 chars) + StringBuilder sb = new StringBuilder(); + sb.append("5;param="); + for (int i = 0; i < 8200; i++) { + sb.append("a"); + } + sb.append("\r\nhello\r\n0\r\n\r\n"); + + try (HttpChunkedEncodingInputStream in = + new HttpChunkedEncodingInputStream( + IOUtils.toInputStream(sb.toString(), StandardCharsets.UTF_8), trailerHeaders)) { + in.read(); + } + } + + @Test(expected = IOException.class) + public void testTrailerHeaderLineExceedingMaxLineLengthThrowsIOException() throws IOException { + StringBuilder sb = new StringBuilder(); + sb.append("0\r\n"); + sb.append("X-Very-Long-Header: "); + for (int i = 0; i < 8200; i++) { + sb.append("x"); + } + sb.append("\r\n\r\n"); + + try (HttpChunkedEncodingInputStream in = + new HttpChunkedEncodingInputStream( + IOUtils.toInputStream(sb.toString(), StandardCharsets.UTF_8), trailerHeaders)) { + in.read(); + } + } + + @Test(expected = IOException.class) + public void testTrailerHeadersCountExceedingMaxTrailerHeadersThrowsIOException() + throws IOException { + StringBuilder sb = new StringBuilder(); + sb.append("0\r\n"); + // Add 65 trailer headers (max is 64) + for (int i = 0; i < 65; i++) { + sb.append("Trailer-").append(i).append(": value\r\n"); + } + sb.append("\r\n"); + + try (HttpChunkedEncodingInputStream in = + new HttpChunkedEncodingInputStream( + IOUtils.toInputStream(sb.toString(), StandardCharsets.UTF_8), trailerHeaders)) { + in.read(); + } + } } diff --git a/src/test/java/me/desair/tus/server/util/TusServletRequestTest.java b/src/test/java/me/desair/tus/server/util/TusServletRequestTest.java index cb3f80be..a5b3a2e3 100644 --- a/src/test/java/me/desair/tus/server/util/TusServletRequestTest.java +++ b/src/test/java/me/desair/tus/server/util/TusServletRequestTest.java @@ -7,6 +7,7 @@ import static org.hamcrest.Matchers.nullValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -136,6 +137,37 @@ public void testGetContentInputStreamChunkedWithChecksum() throws Exception { is("9I3YU4IIYIFsddVND1hNyGMyenw=")); } + @Test + public void testGetContentInputStreamChunkedWithExplicitInitialChecksumEnablesOnlyThatAlgorithm() + throws Exception { + HttpServletRequest mockReq = mock(HttpServletRequest.class); + byte[] data = "5\r\ntest \r\n4\r\ndata\r\n0\r\n\r\n".getBytes(); + when(mockReq.getInputStream()) + .thenReturn(new MockServletInputStream(new ByteArrayInputStream(data))); + when(mockReq.getHeader(HttpHeader.TRANSFER_ENCODING)).thenReturn("chunked"); + when(mockReq.getHeader(HttpHeader.UPLOAD_CHECKSUM)) + .thenReturn("sha1 9I3YU4IIYIFsddVND1hNyGMyenw="); + + TusServletRequest chunkedRequest = new TusServletRequest(mockReq, true); + InputStream is = chunkedRequest.getContentInputStream(); + assertThat(is, notNullValue()); + + byte[] buffer = new byte[1024]; + while (is.read(buffer) != -1) { + // Consume stream completely + } + + assertThat(chunkedRequest.hasCalculatedChecksum(), is(true)); + Set algorithms = chunkedRequest.getEnabledChecksums(); + // Because explicit checksum algorithm was supplied in the initial headers, only SHA1 should be + // enabled, saving the CPU cost of calculating SHA-256, SHA-384, SHA-512, and MD5 in parallel + assertEquals(1, algorithms.size()); + assertTrue(algorithms.contains(ChecksumAlgorithm.SHA1)); + assertThat( + chunkedRequest.getCalculatedChecksum(ChecksumAlgorithm.SHA1), + is("9I3YU4IIYIFsddVND1hNyGMyenw=")); + } + @Test public void testIsProcessedBy() { TusExtension extension = mock(TusExtension.class); diff --git a/src/test/java/me/desair/tus/server/util/UtilsTest.java b/src/test/java/me/desair/tus/server/util/UtilsTest.java index 471f651a..c8e7edbe 100644 --- a/src/test/java/me/desair/tus/server/util/UtilsTest.java +++ b/src/test/java/me/desair/tus/server/util/UtilsTest.java @@ -630,6 +630,13 @@ public void testIsCreationEndpoint() throws Exception { when(storageService.getUploadUri()).thenReturn("/users/[0-9+/files"); when(request.getRequestURI()).thenReturn("/users/42/files"); assertThat(Utils.isCreationEndpoint(request, storageService), is(false)); + + // Test cached pattern execution across repeated calls and diverse URI templates + for (int i = 0; i < 300; i++) { + when(storageService.getUploadUri()).thenReturn("/tenant-" + (i % 20) + "/files"); + when(request.getRequestURI()).thenReturn("/tenant-" + (i % 20) + "/files"); + assertThat(Utils.isCreationEndpoint(request, storageService), is(true)); + } } @Test From f8dd504cf4e25abda3562573dbcbd0f1949401b4 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sun, 8 Nov 2026 21:32:26 +0100 Subject: [PATCH 2/5] fix(security): prevent path traversal in Content-Disposition and add nosniff header in download extension --- .jules/sentinel.md | 5 ++ CHANGELOG.md | 1 + .../java/me/desair/tus/server/HttpHeader.java | 3 + .../download/DownloadGetRequestHandler.java | 30 +++++++- ...DownloadGetRequestHandlerSecurityTest.java | 77 +++++++++++++++++++ 5 files changed, 114 insertions(+), 2 deletions(-) diff --git a/.jules/sentinel.md b/.jules/sentinel.md index e161a663..c42c07ed 100644 --- a/.jules/sentinel.md +++ b/.jules/sentinel.md @@ -32,3 +32,8 @@ **Vulnerability:** In `DiskUtils.readSerializable`, `ValidatingObjectInputStream.accept()` used wide wildcards (`java.lang.*`, `java.util.*`, `me.desair.tus.server.*`) which permitted deserialization of any class within those packages, exposing potential gadget chains for insecure deserialization attacks. **Learning:** Broad package wildcards in `ValidatingObjectInputStream` reduce the protection against insecure deserialization, allowing arbitrary classes within allowed packages to be instantiated during deserialization. **Prevention:** Always construct explicit, concrete class allow-lists when using `ValidatingObjectInputStream.accept(...)` by supplying expected class literal references rather than wide package-level wildcards. + +## 2026-11-08 - Prevent Path Traversal and MIME Sniffing in Download Extension +**Vulnerability:** In `DownloadGetRequestHandler.java`, `info.getFileName()` derived from user-supplied `Upload-Metadata` was embedded directly into the `Content-Disposition` header without stripping path traversal sequences (`../`, `..\`) or directory paths (`/`, `\`). Downstream clients saving files using the suggested filename could write files outside target directories. Additionally, `X-Content-Type-Options: nosniff` was missing when serving completed downloads, leaving browsers susceptible to MIME sniffing attacks. +**Learning:** File download endpoints that reflect user-supplied filenames in `Content-Disposition` headers can cause path traversal vulnerabilities on consuming clients. Serving user-uploaded content without explicit MIME-sniffing protection allows potential content sniffing / script execution. +**Prevention:** Sanitize filenames before constructing `Content-Disposition` by stripping directory paths (Unix `/` and Windows `\`), removing directory traversal sequences (`..`), and falling back to a safe identifier if the sanitized name is empty. Always include `X-Content-Type-Options: nosniff` when serving user-uploaded files. diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ba77433..628e5554 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ See [docs/UPGRADING.md](docs/UPGRADING.md) for details and code examples. - **`UploadStorageService.cleanupExpiredUploads(UploadLockingService)`**: Deprecated in favor of the zero-argument `cleanupExpiredUploads()`, as storage services coordinate locking through `setUploadLockingService(UploadLockingService)`. ### Security +- **Download Security**: sanitized filenames in the `Content-Disposition` header to prevent directory traversal attacks on clients and added `X-Content-Type-Options: nosniff` header to protect against MIME type sniffing. - **Path traversal protection**: hardened `LeaseFileLockingService` against directory traversal via crafted upload identifiers ([#141](https://github.com/tomdesair/tus-java-server/pull/141)). - **CORS origin validation**: prevented arbitrary `Origin` header reflection in CORS preflight responses ([#132](https://github.com/tomdesair/tus-java-server/pull/132)). - **Secure temporary files**: replaced legacy `File.createTempFile` with `Files.createTempFile` enforcing owner-only POSIX permissions (`0600`, CWE-378 / SonarQube java:S5443) ([#131](https://github.com/tomdesair/tus-java-server/pull/131)). diff --git a/src/main/java/me/desair/tus/server/HttpHeader.java b/src/main/java/me/desair/tus/server/HttpHeader.java index cb1d3ceb..1c60e3ae 100644 --- a/src/main/java/me/desair/tus/server/HttpHeader.java +++ b/src/main/java/me/desair/tus/server/HttpHeader.java @@ -101,6 +101,9 @@ public class HttpHeader { */ public static final String X_FORWARDED_FOR = "X-Forwarded-For"; + /** The X-Content-Type-Options response header prevents MIME-type sniffing by browsers. */ + public static final String X_CONTENT_TYPE_OPTIONS = "X-Content-Type-Options"; + /** * The Upload-Complete request and response header field conveys the completeness state in IETF * Resumable Uploads. Its value is a Structured Header Boolean (?1 or ?0). diff --git a/src/main/java/me/desair/tus/server/download/DownloadGetRequestHandler.java b/src/main/java/me/desair/tus/server/download/DownloadGetRequestHandler.java index 352f4057..309348e0 100644 --- a/src/main/java/me/desair/tus/server/download/DownloadGetRequestHandler.java +++ b/src/main/java/me/desair/tus/server/download/DownloadGetRequestHandler.java @@ -9,6 +9,7 @@ import me.desair.tus.server.HttpMethod; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.exception.UploadNotFoundException; +import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.util.AbstractRequestHandler; @@ -50,17 +51,42 @@ public void process( servletResponse.setHeader(HttpHeader.CONTENT_LENGTH, Objects.toString(info.getLength())); + String sanitizedFileName = sanitizeFileName(info.getFileName(), info.getId()); servletResponse.setHeader( HttpHeader.CONTENT_DISPOSITION, String.format( CONTENT_DISPOSITION_FORMAT, - info.getFileName().replace("\"", ""), - URLEncoder.encode(info.getFileName(), StandardCharsets.UTF_8.toString()) + sanitizedFileName.replace("\"", ""), + URLEncoder.encode(sanitizedFileName, StandardCharsets.UTF_8.toString()) .replace("+", "%20"))); servletResponse.setHeader(HttpHeader.CONTENT_TYPE, info.getFileMimeType()); + // Prevent MIME-sniffing attacks by browsers when serving uploaded user content + servletResponse.setHeader(HttpHeader.X_CONTENT_TYPE_OPTIONS, "nosniff"); servletResponse.setStatus(HttpServletResponse.SC_OK); uploadStorageService.copyUploadTo(info, servletResponse.getOutputStream()); } + + /** + * Sanitizes the given filename to prevent directory traversal and header corruption attacks on + * clients consuming the Content-Disposition header. + */ + private String sanitizeFileName(String fileName, UploadId uploadId) { + if (fileName == null || fileName.trim().isEmpty()) { + return uploadId != null ? uploadId.toString() : "download"; + } + // Prevent client-side path traversal: strip directory paths across Unix and Windows separators + String sanitized = fileName.trim(); + int lastSlash = Math.max(sanitized.lastIndexOf('/'), sanitized.lastIndexOf('\\')); + if (lastSlash >= 0) { + sanitized = sanitized.substring(lastSlash + 1).trim(); + } + // Remove directory traversal sequences (..) + sanitized = sanitized.replace("..", "").trim(); + if (sanitized.isEmpty()) { + return uploadId != null ? uploadId.toString() : "download"; + } + return sanitized; + } } diff --git a/src/test/java/me/desair/tus/server/download/DownloadGetRequestHandlerSecurityTest.java b/src/test/java/me/desair/tus/server/download/DownloadGetRequestHandlerSecurityTest.java index 76ac0766..1e641c94 100644 --- a/src/test/java/me/desair/tus/server/download/DownloadGetRequestHandlerSecurityTest.java +++ b/src/test/java/me/desair/tus/server/download/DownloadGetRequestHandlerSecurityTest.java @@ -64,4 +64,81 @@ public void testCrlfInjectionInFileName() throws Exception { "attachment; filename=\"testInjected: true.txt\";" + " filename*=UTF-8''test%0D%0AInjected%3A%20true.txt%22")); } + + @Test + public void testPathTraversalInFileName() throws Exception { + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("upload-id-123")); + info.setLength(10L); + info.setOffset(10L); + + // 1. Unix directory traversal path + info.setEncodedMetadata( + "filename " + Base64.getEncoder().encodeToString("../../../../etc/passwd".getBytes())); + when(uploadStorageService.getUploadInfo(anyString(), any())).thenReturn(info); + handler.process(HttpMethod.GET, tusRequest, tusResponse, uploadStorageService, null); + assertThat( + servletResponse.getHeader(HttpHeader.CONTENT_DISPOSITION), + is("attachment; filename=\"passwd\"; filename*=UTF-8''passwd")); + + // 2. Windows directory traversal path + servletResponse = new MockHttpServletResponse(); + tusResponse = new TusServletResponse(servletResponse); + info.setEncodedMetadata( + "filename " + Base64.getEncoder().encodeToString("..\\..\\malicious.bat".getBytes())); + handler.process(HttpMethod.GET, tusRequest, tusResponse, uploadStorageService, null); + assertThat( + servletResponse.getHeader(HttpHeader.CONTENT_DISPOSITION), + is("attachment; filename=\"malicious.bat\"; filename*=UTF-8''malicious.bat")); + + // 3. Traversal sequence only falls back to upload ID + servletResponse = new MockHttpServletResponse(); + tusResponse = new TusServletResponse(servletResponse); + info.setEncodedMetadata("filename " + Base64.getEncoder().encodeToString("../..".getBytes())); + handler.process(HttpMethod.GET, tusRequest, tusResponse, uploadStorageService, null); + assertThat( + servletResponse.getHeader(HttpHeader.CONTENT_DISPOSITION), + is("attachment; filename=\"upload-id-123\";" + " filename*=UTF-8''upload-id-123")); + + // 4. Blank filename falls back to upload ID + servletResponse = new MockHttpServletResponse(); + tusResponse = new TusServletResponse(servletResponse); + info.setEncodedMetadata("filename " + Base64.getEncoder().encodeToString(" ".getBytes())); + handler.process(HttpMethod.GET, tusRequest, tusResponse, uploadStorageService, null); + assertThat( + servletResponse.getHeader(HttpHeader.CONTENT_DISPOSITION), + is("attachment; filename=\"upload-id-123\";" + " filename*=UTF-8''upload-id-123")); + + // 5. Null or blank filename falls back to upload ID or default "download" + servletResponse = new MockHttpServletResponse(); + tusResponse = new TusServletResponse(servletResponse); + info.setId(new UploadId("fallback-id")); + info.setEncodedMetadata("filename"); + handler.process(HttpMethod.GET, tusRequest, tusResponse, uploadStorageService, null); + assertThat( + servletResponse.getHeader(HttpHeader.CONTENT_DISPOSITION), + is("attachment; filename=\"fallback-id\"; filename*=UTF-8''fallback-id")); + + servletResponse = new MockHttpServletResponse(); + tusResponse = new TusServletResponse(servletResponse); + info.setId(null); + info.setEncodedMetadata("filename"); + handler.process(HttpMethod.GET, tusRequest, tusResponse, uploadStorageService, null); + assertThat( + servletResponse.getHeader(HttpHeader.CONTENT_DISPOSITION), + is("attachment; filename=\"download\"; filename*=UTF-8''download")); + } + + @Test + public void testXContentTypeOptionsHeader() throws Exception { + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("123")); + info.setLength(10L); + info.setOffset(10L); + when(uploadStorageService.getUploadInfo(anyString(), any())).thenReturn(info); + + handler.process(HttpMethod.GET, tusRequest, tusResponse, uploadStorageService, null); + + assertThat(servletResponse.getHeader(HttpHeader.X_CONTENT_TYPE_OPTIONS), is("nosniff")); + } } From 0171156dc53d7e1d6fa07cf05e91cfdfd4357c47 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sat, 14 Nov 2026 10:17:04 +0100 Subject: [PATCH 3/5] fix: harden 2.0.0 concurrency, cache eviction, error handling, and docs - Evict pre-lock ThreadLocal cache entries on lock acquisition in ThreadLocalCachedStorageAndLockingService - Harden protocol handlers, validators, stream/header parsing, and Disk/S3/Azure storage, locking, and deduplication - Add unit tests covering new edge cases and error paths - Clean up CHANGELOG.md for 2.0.0 and update AGENTS.md and storage/architecture docs --- AGENTS.md | 15 +- CHANGELOG.md | 66 ++++---- docs/ARCHITECTURE.md | 9 +- docs/AZURE_BLOB_STORAGE.md | 8 +- docs/DISK_BASED_LOCKING.md | 5 +- docs/LOCKING.md | 32 ++-- docs/S3_STORAGE.md | 12 +- docs/TESTING.md | 4 +- .../desair/tus/server/HttpProblemDetails.java | 5 +- .../tus/server/TusFileUploadService.java | 134 ++++++++++++---- .../server/checksum/ChecksumAlgorithm.java | 5 +- .../checksum/ChecksumRequestHandler.java | 6 + .../CoreDefaultResponseHeadersHandler.java | 4 +- .../server/core/CorePatchRequestHandler.java | 3 + .../desair/tus/server/cors/CorsExtension.java | 3 +- .../PatchUploadLengthValidator.java | 17 ++ .../creation/validation/PostUriValidator.java | 27 +++- .../CreationWithUploadPostRequestHandler.java | 3 + ...HttpDigestsPostPutPatchRequestHandler.java | 29 ++-- .../RufhAppendPatchRequestHandler.java | 40 ++++- .../RufhCreationPostRequestHandler.java | 13 +- .../handler/RufhDeleteRequestHandler.java | 3 +- .../server/rufh/handler/RufhErrorHandler.java | 32 +++- .../handler/RufhHeadGetRequestHandler.java | 3 +- .../rufh/validation/RufhAppendValidator.java | 105 ++++++++++-- .../validation/RufhCreationValidator.java | 13 +- .../tus/server/upload/AbstractLeaseLock.java | 100 ++++++++---- .../server/upload/UploadStorageService.java | 28 ++-- .../server/upload/UuidUploadIdFactory.java | 2 +- .../upload/azure/AzureBlobLockingService.java | 19 ++- .../upload/azure/AzureBlobStorageService.java | 61 ++++--- ...adLocalCachedStorageAndLockingService.java | 85 ++++++++-- .../upload/checksum/DeduplicationService.java | 19 ++- .../upload/disk/DiskStorageService.java | 31 +++- .../upload/s3/S3ConcatenationService.java | 6 +- .../server/upload/s3/S3LockingService.java | 8 +- .../server/upload/s3/S3StorageService.java | 134 ++++++++++------ .../desair/tus/server/upload/s3/S3Utils.java | 19 +-- .../tus/server/util/StructuredHeaderUtil.java | 16 +- .../tus/server/util/TusServletRequest.java | 21 +++ .../java/me/desair/tus/server/util/Utils.java | 19 ++- .../tus/server/TusFileUploadServiceTest.java | 107 +++++++++++++ .../checksum/ChecksumRequestHandlerTest.java | 15 ++ .../core/CorePatchRequestHandlerTest.java | 4 +- .../PatchUploadLengthValidatorTest.java | 26 +++ ...ationWithUploadPostRequestHandlerTest.java | 1 + ...DigestsPostPutPatchRequestHandlerTest.java | 28 ++++ .../server/rufh/RufhProtocolAppendTest.java | 10 +- .../RufhAppendPatchRequestHandlerTest.java | 91 +++++++++++ .../RufhCreationPostRequestHandlerTest.java | 52 ++++++ .../rufh/handler/RufhErrorHandlerTest.java | 42 +++++ .../validation/RufhAppendValidatorTest.java | 150 ++++++++++++++---- .../validation/RufhCreationValidatorTest.java | 24 +++ .../server/upload/AbstractLeaseLockTest.java | 26 +++ .../azure/AzureBlobStorageServiceTest.java | 22 +++ ...calCachedStorageAndLockingServiceTest.java | 73 ++++++++- .../checksum/DeduplicationServiceTest.java | 60 +++++++ .../upload/disk/DiskStorageServiceTest.java | 27 ++++ .../upload/s3/S3StorageServiceTest.java | 49 +++++- .../me/desair/tus/server/util/UtilsTest.java | 4 + 60 files changed, 1595 insertions(+), 380 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9bd1c924..d8410387 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -136,7 +136,7 @@ To avoid duplicate test code and ensure all protocol integration tests run consi ### 18. Efficient Batch Test & Code Coverage Verification Strategy To maximize developer velocity and minimize test execution overhead when increasing code coverage: -- **Batch Test Updates**: When addressing missing line/branch coverage reported by JaCoCo, batch multiple test additions across all relevant test classes (`S3StorageServiceTest`, `S3LockingServiceTest`, `S3UploadLockTest`, `S3ConcatenationServiceTest`, `UploadInfoSerializerTest`) at once rather than running test-by-test iterations. +- **Batch Test Updates**: When addressing missing line/branch coverage reported by JaCoCo, batch multiple test additions across all relevant test classes (`S3StorageServiceTest`, `S3LockingServiceTest`, `S3UploadLockTest`, `S3ConcatenationServiceTest`, `UploadInfoJsonSerializerTest`) at once rather than running test-by-test iterations. - **Fast Unit Test Execution**: Verify all local unit tests rapidly using target wildcard patterns (e.g. `mvn test -Dtest="S3*" -q` or `mvn test -Dtest="*Test" -q`). Unit tests run in under 2 seconds without launching test containers. - **Single Verification Gate**: Only run the python coverage script (`python3 scripts/check-coverage.py --per-file-limit 90`) after all batched unit test updates have been applied and locally validated. @@ -184,12 +184,13 @@ To maintain a clear separation between fast, offline unit tests and containerize - Do NOT use legacy `java.io.File.createTempFile(...)`. On POSIX systems, it creates files with overly permissive default umask permissions (often world-readable or group-readable, CWE-378 / SonarQube java:S5443). - Always use `java.nio.file.Files.createTempFile(...)` (or `Utils.createTempSiblingPath(...)` / `Utils.createTempSibling(...)`), which creates temporary files with restricted owner-only permissions (`rw-------` / `0600`) by default. -### 23. Thread Safety: Prevent Volatile Class Variables & Use ReentrantReadWriteLock +### 23. Thread Safety: Prevent Volatile Class Variables & Use ReentrantReadWriteLock or Synchronized - Do NOT use `volatile` class variables for mutable state, lazy compilation, or feature flags. `volatile` references on complex objects or mutable fields can trigger static analysis warnings (e.g. SonarQube java:S3077) and cannot guard multi-step compound actions, non-atomic transitions, or lazy initializations. -- Guard concurrent mutable state using a `ReadWriteLock` (`ReentrantReadWriteLock`). -- Consistently name the lock field `private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock();` in every class for pattern consistency across the codebase. -- Use `thisObjectLock.readLock()` for fast, concurrent reads, and `thisObjectLock.writeLock()` for state mutations, lazy pattern compilation, or feature capability degradation. -- Always acquire and release locks using `try ... finally` blocks (e.g. `thisObjectLock.readLock().lock(); try { return field; } finally { thisObjectLock.readLock().unlock(); }`). +- When a class requires both concurrent reads and exclusive writes, guard mutable state using a `ReadWriteLock` (`ReentrantReadWriteLock`): + - Consistently name the lock field `private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock();` in every class for pattern consistency across the codebase. + - Use `thisObjectLock.readLock()` for fast, concurrent reads, and `thisObjectLock.writeLock()` for state mutations, lazy pattern compilation, or feature capability degradation. + - Always acquire and release locks using `try ... finally` blocks (e.g. `thisObjectLock.readLock().lock(); try { return field; } finally { thisObjectLock.readLock().unlock(); }`). +- If a class only needs exclusive mutual exclusion (i.e. only `writeLock` semantics are used and no `readLock` is needed in the class), prefer the simpler `synchronized` keyword on the method or block rather than introducing a `ReadWriteLock`. ### 24. UploadLock AutoCloseable Lifecycle & Try-With-Resources - `UploadLock` implements `java.lang.AutoCloseable`. @@ -211,7 +212,7 @@ When a new draft revision of the IETF Resumable Uploads specification (`draft-ie - Identify any changed header names, structured field syntax changes, response status codes, or problem details schemas. ### 2. Spec-Driven Compliance Test Maintenance -Compliance unit tests located in `src/test/java/me/desair/tus.server.rufh/` (`RufhProtocolCreationTest`, `RufhProtocolAppendTest`, `RufhProtocolHeadTest`, `RufhProtocolCancellationTest`, `HttpProblemDetailsTest`) contain verbatim quotes from the specification in their method Javadocs. +Compliance unit tests located in `src/test/java/me/desair/tus/server/rufh/` (`RufhProtocolCreationTest`, `RufhProtocolAppendTest`, `RufhProtocolHeadTest`, `RufhProtocolCancellationTest`, `HttpProblemDetailsTest`) contain verbatim quotes from the specification in their method Javadocs. - **Workflow**: 1. Update the verbatim spec quotes in test method Javadocs to reflect the new draft revision text. 2. Update test assertions and expected header/status formats. diff --git a/CHANGELOG.md b/CHANGELOG.md index 628e5554..261bf510 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,55 +7,49 @@ All notable changes to this project will be documented in this file. Major release adding the IETF Resumable Uploads for HTTP (RUFH) protocol and cloud storage backends. **Upgrading from `1.0.0-3.x`? Read the [Upgrade Guide](docs/UPGRADING.md).** ### Highlights -- IETF Resumable Uploads for HTTP (`draft-ietf-httpbis-resumable-upload-12`) served next to Tus 1.0.0 on the same endpoint. +- IETF Resumable Uploads for HTTP (`draft-ietf-httpbis-resumable-upload-12`) served alongside Tus 1.0.0 on the same endpoint. - S3-compatible and Azure Blob storage backends with distributed, database-free locking. -- NFS/SMB-safe lease locking is now the default for disk storage. -- Server-side `UploadCompletionListener` and a `process()` method returning the `UploadInfo`. +- NFS/SMB-safe lease locking as the new default for disk storage. +- Server-side `UploadCompletionListener` callbacks and `process()` returning `UploadInfo`. ### Upgrade notes (breaking & behavioural changes) See [docs/UPGRADING.md](docs/UPGRADING.md) for details and code examples. - **Java 17 and Jakarta EE**: requires Java 17+ and `jakarta.servlet.*` (Tomcat 10+, Jetty 11+, Spring Boot 3+). `javax.servlet` is no longer supported. -- **Default locking**: `withStoragePath(String)` now uses `LeaseFileLockingService` instead of `DiskLockingService`. Opt out with `withUploadLockingService(new DiskLockingService(path))`, see [docs/DISK_BASED_LOCKING.md](docs/DISK_BASED_LOCKING.md). -- **Lock-holding streams**: `getUploadedBytes(...)` keeps the upload locked until the returned stream is closed. Always use try-with-resources. -- **`process(...)` return value**: now returns the created/updated `UploadInfo` on successful `POST`, `PATCH`, and `HEAD` requests (or `null` on errors, `OPTIONS`, or `DELETE` termination). It also returns a value for progress requests, so check the HTTP method when recording a newly created upload. -- **Download extension (breaking)**: no longer returns `204` for uploads still in progress and no longer sets the `Tus-Resumable` header (needed for dual-protocol support). The `UploadInProgressException` class was removed. -- **Jackson bundled**: `jackson-databind`, `jackson-annotations` and `jackson-core` are now compile-scope dependencies; no separate declaration needed. -- **Protocol detection**: `ProtocolVersion.AUTO` is the default. Restrict with `withSupportedProtocolVersions(...)`, see [docs/PROTOCOLS.md](docs/PROTOCOLS.md). -- **Checksum Deduplication Index**: The checksum index format has been updated to support reference tracking, parent tombstoning, and full cleanup upon upload deletion across Disk, S3, and Azure Blob storage. See [docs/DEDUPLICATION.md](docs/DEDUPLICATION.md) for migration instructions (clearing the index directory or running the migration script). +- **Default locking**: `withStoragePath(String)` now uses `LeaseFileLockingService` instead of `DiskLockingService`. Opt out with `withUploadLockingService(new DiskLockingService(path))`; see [docs/DISK_BASED_LOCKING.md](docs/DISK_BASED_LOCKING.md). +- **Lock-holding streams**: `getUploadedBytes(...)` keeps the upload locked until the returned stream is closed. Always use `try-with-resources`. +- **`process(...)` return value**: returns the created or updated `UploadInfo` on successful `POST`, `PATCH`, and `HEAD` requests (or `null` on errors, `OPTIONS`, or `DELETE`). +- **Download extension (breaking)**: no longer returns `204` for in-progress uploads or sets the `Tus-Resumable` header. `UploadInProgressException` was removed. +- **Jackson bundled**: `jackson-databind`, `jackson-annotations`, and `jackson-core` are now compile-scope dependencies. +- **Protocol detection**: `ProtocolVersion.AUTO` is the default. Restrict with `withSupportedProtocolVersions(...)`; see [docs/PROTOCOLS.md](docs/PROTOCOLS.md). +- **Checksum deduplication index**: updated index layout for reference tracking and cleanup across Disk, S3, and Azure Blob storage; see [docs/DEDUPLICATION.md](docs/DEDUPLICATION.md) for migration steps. ### New -- **IETF Resumable Uploads for HTTP (RUFH)**: full draft-12 support with automatic protocol detection, RFC 9651 structured headers (`Upload-Offset`, `Upload-Complete`, `Upload-Length`, `Upload-Limit`), RFC 7807 `application/problem+json` errors, RFC 9530 HTTP digests, `104` interim responses and a spec-quoted conformity suite (`scripts/rufh_conformity_test.py`). See [docs/PROTOCOLS.md](docs/PROTOCOLS.md), [docs/MIGRATION.md](docs/MIGRATION.md), [docs/INTERIM_RESPONSES.md](docs/INTERIM_RESPONSES.md) and [docs/CONFORMITY_TESTING.md](docs/CONFORMITY_TESTING.md). -- **NFS- & SMB-safe lease locking (`LeaseFileLockingService`)**: atomic sibling mutex directories, TTL lease files with heartbeat renewal and ownership fencing; safe on NFS v3/v4, AWS EFS, Azure Files, SMB/CIFS and local disks, with no Redis, ZooKeeper or `FileLock` daemons. See [docs/DISK_BASED_LOCKING.md](docs/DISK_BASED_LOCKING.md). -- **S3-compatible storage**: `S3StorageService`, `S3LockingService` (conditional writes, TTL leases, automatic non-CAS fallback for e.g. Backblaze B2 and Ceph RGW, configurable jitter via `withJitter(minMs, maxMs)` and `withS3ConditionalWritesSupported(boolean)`) and `S3ConcatenationService`, including server-side multipart composition that works on AWS S3 and MinIO/Ceph. See [docs/S3_STORAGE.md](docs/S3_STORAGE.md). -- **Azure Blob Storage**: `AzureBlobStorageService`, `AzureBlobLockingService` (Blob Leases with auto-renewal and cross-replica stop signals) and zero-copy `AzureBlobConcatenationService`. See [docs/AZURE_BLOB_STORAGE.md](docs/AZURE_BLOB_STORAGE.md). -- **`UploadCompletionListener`**: register via `withUploadCompletionListener(...)` / `addUploadCompletionListener(...)`. Listeners run after the lock is released, so they can stream and delete the bytes immediately. Added `getUploadedBytes(UploadInfo)` and `deleteUpload(UploadInfo)` overloads. -- **`withLockWaitTimeout(Duration)`** (default 60 s): maximum time a request waits for a contended upload lock; also derives the cloud chunk drain timeout (`lockWaitTimeout - 5 s`) and retry budget. See [docs/LOCKING.md](docs/LOCKING.md). -- **Automatic lock lifecycle cleanup** on release, termination (`DELETE`), and expiration across all locking services, deleting lock artifacts immediately without leaving lingering locks, plus a fallback sweeper for abandoned locks. -- **JSON serialization**: store `UploadInfo` as JSON with `withJsonSerialization(true)`. -- **Absolute base URLs**: `withUploadUri(String)` accepts e.g. `https://upload.example.com/files`, returning full `Location` URLs for Tus and RUFH; relative paths keep working. -- **Multi-Backend Upload Deduplication & Lifecycle Cleanup**: Content-based upload deduplication with `withUploadDeduplication(true)` across Disk, S3, and Azure Blob storage. Features child reference markers, parent tombstoning, and complete index directory cleanup when uploads expire or are deleted. See [docs/DEDUPLICATION.md](docs/DEDUPLICATION.md). -- **`NoOpUploadLockingService`**: Zero-overhead Null Object implementation of `UploadLockingService` for unit tests and custom environments, making locking service usage mandatory and eliminating defensive null checks across all storage backends, request handlers, and deduplication logic. +- **IETF Resumable Uploads for HTTP (RUFH)**: full `draft-12` support with automatic protocol detection, RFC 9651 structured headers (`Upload-Offset`, `Upload-Complete`, `Upload-Length`, `Upload-Limit`), RFC 7807 `application/problem+json` errors, RFC 9530 HTTP digests, `104` interim responses, and a conformity test suite (`scripts/rufh_conformity_test.py`). See [docs/PROTOCOLS.md](docs/PROTOCOLS.md), [docs/MIGRATION.md](docs/MIGRATION.md), and [docs/INTERIM_RESPONSES.md](docs/INTERIM_RESPONSES.md). +- **NFS- & SMB-safe lease locking (`LeaseFileLockingService`)**: distributed TTL lease files with atomic mutex arbitration, heartbeat renewal, and ownership fencing; safe on NFS v3/v4, AWS EFS, Azure Files, SMB/CIFS, and local disks without external databases. See [docs/DISK_BASED_LOCKING.md](docs/DISK_BASED_LOCKING.md). +- **S3-compatible storage**: `S3StorageService`, `S3LockingService`, and `S3ConcatenationService` with conditional writes, TTL leases, automatic non-CAS fallback, and server-side multipart composition. See [docs/S3_STORAGE.md](docs/S3_STORAGE.md). +- **Azure Blob Storage**: `AzureBlobStorageService`, `AzureBlobLockingService`, and zero-copy `AzureBlobConcatenationService` using native Blob Leases. See [docs/AZURE_BLOB_STORAGE.md](docs/AZURE_BLOB_STORAGE.md). +- **`UploadCompletionListener`**: register via `withUploadCompletionListener(...)` / `addUploadCompletionListener(...)` to process completed uploads after lock release. Added `getUploadedBytes(UploadInfo)` and `deleteUpload(UploadInfo)` overloads. +- **`withLockWaitTimeout(Duration)`** (default 60 s): configurable wait budget for contended upload locks and cloud chunk draining. See [docs/LOCKING.md](docs/LOCKING.md). +- **JSON serialization**: opt-in JSON storage for `UploadInfo` metadata via `withJsonSerialization(true)`. +- **Absolute base URLs**: `withUploadUri(String)` accepts full URLs (e.g. `https://upload.example.com/files`) for `Location` headers. +- **Multi-backend upload deduplication**: extended `withUploadDeduplication(true)` to S3 and Azure Blob storage with automatic reference tracking and index cleanup. See [docs/DEDUPLICATION.md](docs/DEDUPLICATION.md). +- **`NoOpUploadLockingService`**: zero-overhead `UploadLockingService` implementation for custom or single-threaded setups. ### Fixed -- **Azure Lock Cleanup**: eliminated lingering `.lock` blobs in Azure Blob Storage by having `AzureBlobUploadLock.release()` delete the actively leased blob directly using lease conditions upon lock release, and cleaning up orphaned lock artifacts in `cleanupExpiredUploads()`. -- **Storage Backend Utilities**: introduced dedicated `DiskUtils` and expanded `AzureUtils` alongside `S3Utils`, centralizing safe path resolution, storage directory initialization, path component validation, checksum normalization, blob existence checks, and quiet resource deletion across storage, checksum, and locking backends. -- **S3 Upload Performance**: reduced S3 upload latency by ~70% (~3x speedup in conformity benchmarks) via optimistic conditional writes (`If-None-Match: *`) during lock acquisition, caching part sizes to avoid repeated `statObject` roundtrips, and stream-based byte truncation. -- **Lock Lifecycle & Expired Upload Cleanup**: unified `cleanupExpiredUploads()` across Disk, S3, and Azure backends to clean up orphaned locks and prune stale checksum index entries, while ensuring `terminateUpload()` relies on the active `UploadLock` lifecycle. -- **Async Chunk Uploading**: hardened `AsyncChunkUploader` against servlet thread pool starvation under `CallerRunsPolicy` and fixed race conditions during temporary chunk file deletions. -- **Network Filesystem Locking**: prevented `FileChannel.lock()` `ENOLCK` failures on distributed filesystems (NFS, AWS EFS, Kubernetes PVs) by restricting kernel file locks exclusively to legacy `DiskLockingService`. -- **Deduplication Resilience**: protected deduplication traversal against infinite cyclic recursion (`StackOverflowError`) and handled missing concatenation parts safely. -- **Content-Type matching**: media types are matched per RFC 9110 and tolerate parameters such as `;charset=UTF-8` (added by Spring `CharacterEncodingFilter`, proxies or clients) instead of returning `406`. -- **Undertow & other containers**: `Content-Length` is cleared before `sendError(...)` ([#40](https://github.com/tomdesair/tus-java-server/issues/40)). -- **Disk**: truncate underflow guard in `DiskStorageService`; `FileChannel` leak on lock acquisition errors fixed in `FileBasedLock`. +- **Interrupted upload streams**: skip checksum verification and preserve declared `Upload-Length` when an in-flight stream is interrupted during lock contention resolution. +- **Thread-local cache consistency**: evict cached `UploadInfo` entries on lock acquisition and after storage write errors so `ThreadLocalCachedStorageAndLockingService` never serves stale offsets or metadata. +- **Deduplication**: guard against cyclic or chained duplicate references and in-progress parent uploads, and handle missing concatenation parts safely. +- **Content-Type matching**: match media types per RFC 9110 and tolerate parameters such as `;charset=UTF-8` instead of returning `406`. +- **Servlet container error responses**: clear `Content-Length` before `sendError(...)` for compatibility with Undertow and other strict containers ([#40](https://github.com/tomdesair/tus-java-server/issues/40)). +- **Disk storage & locking**: guard against truncate underflow in `DiskStorageService`, fix `FileChannel` leaks on lock acquisition failure in `FileBasedLock`, and restrict kernel `FileChannel.lock()` calls to `DiskLockingService`. ### Deprecated -- **`UploadStorageService.cleanupExpiredUploads(UploadLockingService)`**: Deprecated in favor of the zero-argument `cleanupExpiredUploads()`, as storage services coordinate locking through `setUploadLockingService(UploadLockingService)`. +- **`UploadStorageService.cleanupExpiredUploads(UploadLockingService)`**: deprecated in favor of the zero-argument `cleanupExpiredUploads()`. ### Security -- **Download Security**: sanitized filenames in the `Content-Disposition` header to prevent directory traversal attacks on clients and added `X-Content-Type-Options: nosniff` header to protect against MIME type sniffing. -- **Path traversal protection**: hardened `LeaseFileLockingService` against directory traversal via crafted upload identifiers ([#141](https://github.com/tomdesair/tus-java-server/pull/141)). -- **CORS origin validation**: prevented arbitrary `Origin` header reflection in CORS preflight responses ([#132](https://github.com/tomdesair/tus-java-server/pull/132)). -- **Secure temporary files**: replaced legacy `File.createTempFile` with `Files.createTempFile` enforcing owner-only POSIX permissions (`0600`, CWE-378 / SonarQube java:S5443) ([#131](https://github.com/tomdesair/tus-java-server/pull/131)). +- **Download extension**: sanitize filenames in `Content-Disposition` to prevent client directory traversal and add `X-Content-Type-Options: nosniff`. +- **CORS origin validation**: prevent arbitrary `Origin` header reflection in CORS responses ([#132](https://github.com/tomdesair/tus-java-server/pull/132)). +- **Temporary file permissions**: use `Files.createTempFile` instead of `File.createTempFile` to enforce owner-only permissions (`0600`, CWE-378) ([#131](https://github.com/tomdesair/tus-java-server/pull/131)). ## [1.0.0-3.3] diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e3a25c26..095cfa4b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -131,8 +131,7 @@ sequenceDiagram Handler-->>Service: Write RFC 7807 problem details / error status end - opt Upload Finished - Service->>Locking: cleanupLock(uploadId) + opt Upload Finished (FINAL or COMPLETE, not PARTIAL) Service->>Service: notifyUploadCompletionListeners(uploadInfo) end @@ -169,7 +168,7 @@ For local filesystems and shared network volumes, `tus-java-server` provides two ### Cloud Storage Implementations - **`S3StorageService`**: Stores chunks as S3 objects and concatenates them using multipart upload copy or compose helpers (`S3ConcatenationService`). -- **`AzureBlobStorageService`**: Stages chunks as uncommitted block blobs using `.part` temporary buffers and commits block lists atomically upon completion. Zero-copy concatenation is supported via `stageBlockFromUrl`. +- **`AzureBlobStorageService`**: Stages chunks directly as uncommitted Azure Block Blob blocks and commits block lists atomically upon completion. Zero-copy concatenation is supported via `stageBlockFromUrl`. ### Storage Extensions & Helpers @@ -252,7 +251,7 @@ classDiagram #isLockExpired(id)* boolean #evictExpiredLock(id)* boolean #writeStopSignal(id)* void - #checkStopSignals()* void + #checkStopSignalForUpload(id, inputStream)* void } class LeaseFileLockingService { @@ -339,4 +338,4 @@ For detailed testing instructions, see [`docs/TESTING.md`](TESTING.md) and [`doc | **Add a Request Processing Action** | Create a class implementing `RequestHandler` (or extending `AbstractRequestHandler`). | Wrap streaming payload inputs in `InterruptibleInputStream`. Set headers on `TusServletResponse`. Register in `extension.initRequestHandlers()`. | | **Add a New Protocol Extension** | Create a package under `me.desair.tus.server.` and extend `AbstractTusExtension`. | Implement `getName()`, `isApplicable(method, version)`, `initValidators()`, and `initRequestHandlers()`. Register in `TusFileUploadService.initFeatures()`. | | **Add a Storage Backend** | Implement `UploadStorageService` (and optionally `UploadConcatenationService`). | Implement `terminateUpload` (free data, metadata `.info`, and delegate `cleanupLock`). If supporting deduplication, implement `deduplicate`, `dataExists`, and `deleteData`, follow read/write safety rules (`duplicatesUploadId`), and implement a `ChecksumIndex`. Implement offline unit tests (`*Test.java`) and Testcontainers integration tests (`IT*`). | -| **Add a Distributed Locking Provider** | Extend `AbstractLeaseLockingService` and `AbstractLeaseLock`. | Implement the template hooks (`tryAcquireLock`, `isLockExpired`, `evictExpiredLock`, `writeStopSignal`, `checkStopSignals`). Inherit NTP safety margins and stream interruption automatically. | +| **Add a Distributed Locking Provider** | Extend `AbstractLeaseLockingService` and `AbstractLeaseLock`. | Implement the template hooks (`tryAcquireLock`, `isLockExpired`, `evictExpiredLock`, `writeStopSignal`, `checkStopSignalForUpload`). Inherit NTP safety margins and stream interruption automatically. | diff --git a/docs/AZURE_BLOB_STORAGE.md b/docs/AZURE_BLOB_STORAGE.md index 83c242ec..fc6a9e5d 100644 --- a/docs/AZURE_BLOB_STORAGE.md +++ b/docs/AZURE_BLOB_STORAGE.md @@ -111,8 +111,8 @@ TusFileUploadService tusService = new TusFileUploadService() During an in-progress upload, `AzureBlobLockingService` creates a target lock blob (`locks/.lock`) and holds a 30-second native Azure Blob Lease with automatic background renewal. -1. **Automatic Cleanup on Completion**: Once an upload completes (`offset == length`), `TusFileUploadService` calls `cleanupLock(UploadId)` after releasing the lease, deleting the `.lock` blob and any `.stop` signal blob. -2. **Automatic Cleanup on Termination & Expiration**: When an upload is canceled (`DELETE`) or purged via `cleanupExpiredUploads()`, the corresponding `.lock` and `.stop` blobs are deleted. +1. **Cleanup on Termination & Expiration**: When an upload is canceled (`DELETE`) or purged via `cleanupExpiredUploads()`, the storage service calls `cleanupLock(UploadId)` so the corresponding `.lock` and `.stop` blobs are deleted. +2. **Periodic Stale Lock Sweeping**: Any remaining `.stop` blobs older than 30s or un-leased `.lock` blobs from finished/abandoned uploads older than the stale lock threshold (default 120s) are cleaned up via `cleanupStaleLocks()`. 3. **Lease Safety Guard**: Deleting an actively leased blob without providing the lease ID triggers HTTP 412 (`LeaseIdMissing`). `AzureBlobLockingService` inspects the blob's lease status and skips deleting blobs that are actively leased by concurrent requests. 4. **Fallback Stale Lock Sweeper (`cleanupStaleLocks`)**: To prevent abandoned lock blobs after ungraceful pod termination or network partitions, `cleanupStaleLocks()` runs as a scheduled fallback. It sweeps unleased `.lock` blobs older than 120 seconds and `.stop` signal blobs older than 30 seconds. Actively leased blobs are never deleted. @@ -120,7 +120,7 @@ During an in-progress upload, `AzureBlobLockingService` creates a target lock bl ## 4. Post-Upload Processing (`getAzureBlobName`) -After an upload completes, downstream services can obtain the direct Azure blob name of the final object using `getAzureBlobName(uploadUri, ownerKey)`: +After an upload completes, downstream services can obtain the direct Azure blob name of the final object using `getAzureBlobName(uploadUri)` or `getAzureBlobName(uploadUri, ownerKey)`: ```java import com.azure.storage.blob.BlobClient; @@ -228,7 +228,7 @@ When a client resumes an interrupted transfer by sending a `HEAD` request (or ca | `AzureBlobLockingServiceTest` | Unit Test | Offline | < 1s | Pure offline unit tests for Azure locking service parameters, prefix normalization, URI parsing, and defensive handling. | | `AzureBlobUploadLockTest` | Unit Test | Offline | < 1s | Pure offline unit tests for Azure upload lock parameters, getters, and daemon executor lifecycle. | | `AzureBlobConcatenationServiceTest` | Unit Test | Offline | < 1s | Pure offline unit tests for Azure concatenation service parameters, prefix sanitization, guard clauses, and partial upload handling. | -| `ITAzureBlobStorageService` | Integration | Azurite (Testcontainers / Podman or Docker) | ~ 3s | Live end-to-end storage integration test against Azurite emulator (Block Blob staging, `.part` buffer commits, truncations, deduplication, and expiration). | +| `ITAzureBlobStorageService` | Integration | Azurite (Testcontainers / Podman or Docker) | ~ 3s | Live end-to-end storage integration test against Azurite emulator (Block Blob staging, block list commits, truncations, deduplication, and expiration). | | `ITAzureBlobLockingService` | Integration | Azurite (Testcontainers / Podman or Docker) | ~ 2s | Distributed locking, Azure Blob Leases, lease renewals, lock contention (409 Conflict), stream interruption, & stop signal integration tests. | | `ITAzureBlobConcatenationService` | Integration | Azurite (Testcontainers / Podman or Docker) | ~ 2s | Server-side zero-copy block concatenation (`stageBlockFromUrl`) integration tests. | | `ITAzureBlobRufhProtocol` | Integration | Azurite (Testcontainers / Podman or Docker) | ~ 3s | IETF RUFH protocol integration suite for Azure backend. | diff --git a/docs/DISK_BASED_LOCKING.md b/docs/DISK_BASED_LOCKING.md index d7072478..4372bfd4 100644 --- a/docs/DISK_BASED_LOCKING.md +++ b/docs/DISK_BASED_LOCKING.md @@ -38,10 +38,11 @@ Locks are structured as a dedicated directory containing a JSON lease metadata f { "holderId": "pod-tus-backend-7d9c6-thread-14", "requestUri": "/files/upload/018f3b2a-7140-7e1d-8f92-5cb640d28362", - "storagePath": "/mnt/uploads/locks/018f3b2a-7140-7e1d-8f92-5cb640d28362.lock", "leaseDurationMs": 30000, + "acquiredAt": 1723401204567, "expiresAt": 1723401234567, - "acquiredAt": 1723401204567 + "lockPath": "/mnt/uploads/locks/018f3b2a-7140-7e1d-8f92-5cb640d28362.lock", + "stopPath": "/mnt/uploads/locks/018f3b2a-7140-7e1d-8f92-5cb640d28362.stop" } ``` diff --git a/docs/LOCKING.md b/docs/LOCKING.md index 348bbea5..9d8bced5 100644 --- a/docs/LOCKING.md +++ b/docs/LOCKING.md @@ -31,17 +31,14 @@ public interface UploadLockingService { // Acquires an exclusive lock on an upload resource UploadLock lockUploadByUri(String requestUri) throws TusException, IOException; - // Checks if an upload is currently locked - boolean isLocked(UploadId id); + // Acquires an exclusive lock on an upload resource by UploadId + default UploadLock lockUpload(UploadId id) throws TusException, IOException { ... } // Cleans up stale or expired locks (fallback sweeper) void cleanupStaleLocks() throws IOException; - // Cleans up the lock file or object for a completed, terminated, or expired upload - default void cleanupLock(UploadId id) throws IOException {} - - // Cleans up the lock file or object for a completed, terminated, or expired upload by URI - default void cleanupLock(String uploadUri) throws IOException {} + // Checks if an upload is currently locked + boolean isLocked(UploadId id); // Injects the UploadIdFactory instance used to parse upload IDs from request URIs void setIdFactory(UploadIdFactory idFactory); @@ -49,36 +46,41 @@ public interface UploadLockingService { // Registers the active request input stream so it can be interrupted cleanly default void registerInputStream(String requestUri, InputStream inputStream) {} + // Unregisters the active request input stream after streaming completes or fails + default void unregisterInputStream(String requestUri) {} + // Requests that any active lock for the URI be released default void requestLockRelease(String requestUri) {} // Closes resources, interrupts in-flight streams, and shuts down background daemon threads default void close() throws IOException {} + + // Cleans up the lock file or object for a terminated or expired upload + default void cleanupLock(UploadId id) throws IOException {} } ``` ### `UploadLock` Interface ```java -public interface UploadLock extends Closeable { +public interface UploadLock extends AutoCloseable { // Gets the request URI associated with this lock String getUploadUri(); // Explicitly releases the lock - default void release() { - try { - close(); - } catch (IOException ignored) {} - } + void release(); + + @Override + void close() throws IOException; } ``` ### Request Flow, Lock Cleanup & Contention Resolution - **Stream Registration**: When a request starts streaming payload bytes, its input stream is wrapped in an `InterruptibleInputStream` and registered via `lockingService.registerInputStream(requestUri, inputStream)`. -- **Automatic Lock Cleanup on Completion**: When an upload finishes (`offset == length`), `TusFileUploadService` automatically invokes `lockingService.cleanupLock(uploadId)` to remove the lock file/blob (`.lock`) and any stop signal file/blob (`.stop`). -- **Automatic Lock Cleanup on Termination & Expiration**: When an upload is canceled via `DELETE` (204 No Content) or expired via `cleanupExpiredUploads()`, the associated lock file or blob is also automatically deleted via `lockingService.cleanupLock(uploadId)`. Actively held locks are defended and never deleted mid-stream. +- **Lock Release on Request End**: Every request acquires an `UploadLock` via `try-with-resources` (`lock.close()`), automatically releasing the active lease or lock at the end of the request. +- **Lock Cleanup on Termination & Expiration**: When an upload is canceled via `DELETE` (204 No Content) or expired via `cleanupExpiredUploads()`, the storage service invokes `lockingService.cleanupLock(uploadId)` to remove the associated `.lock` and `.stop` files/objects. Actively held locks are defended and never deleted mid-stream. - **Fallback Stale Lock Sweeper**: As a safety net, `cleanupStaleLocks()` periodically sweeps orphaned lock records (e.g. unleased Azure blobs older than 120s or stop signal markers older than 30s) left behind after hard node crashes or power outages. - **Release Request**: When a concurrent `HEAD` or `DELETE` request encounters an active lock, `TusFileUploadService` catches `UploadAlreadyLockedException` and calls `lockingService.requestLockRelease(requestUri)`. - **Stream Interruption**: diff --git a/docs/S3_STORAGE.md b/docs/S3_STORAGE.md index 539f6e55..d768bfe0 100644 --- a/docs/S3_STORAGE.md +++ b/docs/S3_STORAGE.md @@ -83,7 +83,7 @@ TusFileUploadService tusService = new TusFileUploadService() ``` / # Final completed data object /.info # JSON-serialized UploadInfo metadata -/.part # Incomplete sub-5MB part buffer +/.part. # Incomplete sub-5MB part buffers ///parent # Deduplication parent pointer object ///parent-deleted # Tombstone marker if parent deleted ///children/ # Active duplicate child reference @@ -96,7 +96,7 @@ TusFileUploadService tusService = new TusFileUploadService() | Setting | Default Value | Description | |---------|---------------|-------------| | `objectPrefix` | `"uploads/"` | Key prefix for final completed file objects | -| `metadataPrefix` | `"metadata/"` | Key prefix for `.info` JSON and `.part` buffers | +| `metadataPrefix` | `"metadata/"` | Key prefix for `.info` JSON and `.part.` buffers | | `checksumsPrefix` | `"checksums/"` | Key prefix for deduplication index objects | | `locksPrefix` | `"locks/"` | Key prefix for distributed lock lease objects | @@ -201,7 +201,7 @@ S3 requires every part chunk of a multipart upload to be at least 5 MB (except t - **Slot 3 (Uploading)**: Actively uploading a chunk to S3 on a background daemon worker thread. - When the background thread pool is fully utilized, work seamlessly falls back to the client thread without blocking (`ThreadPoolExecutor.CallerRunsPolicy` on a `SynchronousQueue`), ensuring zero queuing overhead and immediate backpressure. - **Thread Pool Sizing**: Worker threads are managed via a shared executor, default 10 threads, configurable on `TusFileUploadService` via `.withCloudUploadThreadPoolSize(int)` or directly on `S3StorageService.setCloudUploadThreadPoolSize(int)`. -- **Incomplete Parts & Stale Buffer Protection**: If a client upload stream ends before reaching 5 MB and the upload is not complete, the sub-5MB chunk is saved as a `/.part` object in S3. On subsequent requests, an arithmetic budget guard ($\text{existingPartsTotalSize} + \text{size}(.part) == \text{length}$) verifies whether the `.part` represents a legitimate final part or an obsolete buffer from a previous attempt, preventing file corruption and size inflation. +- **Incomplete Parts & Stale Buffer Protection**: If a client upload stream ends before reaching 5 MB and the upload is not complete, the sub-5MB chunk is saved as an indexed `/.part.` object in S3. On subsequent requests, an arithmetic budget guard ($\text{existingPartsTotalSize} + \text{size}(.part) == \text{length}$) verifies whether the `.part.*` buffers represent a legitimate continuation or obsolete buffers from a previous attempt, preventing file corruption and size inflation. - **Configurable Temp Directory**: The temporary buffer directory can be configured in the constructor or builder: ```java @@ -427,7 +427,11 @@ When the test suite executes: | `LeaseDataJsonSerializerTest` | Unit test for lease data JSON serialization (`me.desair.tus.server.util`) | Mocked / JVM | | `S3StorageServiceTest` | Fast unit test for S3 storage logic | Mocked `MinioClient` | | `S3LockingServiceTest` | Fast unit test for S3 distributed locking | Mocked `MinioClient` | +| `S3UploadLockTest` | Fast unit test for S3 upload lock lease renewal & release | Mocked `MinioClient` | | `S3ConcatenationServiceTest` | Fast unit test for S3 concatenation logic | Mocked `MinioClient` | +| `S3ChecksumIndexTest` | Fast unit test for S3 checksum deduplication index | Mocked `MinioClient` | +| `S3ServerSideComposeHelperTest` | Fast unit test for S3 server-side multipart composition | Mocked `MinioClient` | +| `S3UtilsTest` | Fast unit test for S3 error classification & utilities | JVM | | `ITS3StorageService` | Integration test for S3 storage | Live RustFS Testcontainer | | `ITS3LockingService` | Integration test for S3 distributed locking & contention | Live RustFS Testcontainer | | `ITS3RufhProtocol` | IETF RUFH protocol integration suite for S3 backend | Live RustFS Testcontainer | @@ -457,7 +461,7 @@ When the test suite executes: To automatically clean up abandoned partial uploads or lock files in case of sudden server crashes, configure S3 Lifecycle Rules on your bucket: - **Expire Incomplete Multipart Uploads**: Set rule to abort incomplete multipart uploads after 1–7 days. -- **Expire `metadata/*.part` Objects**: Configure lifecycle expiration for objects under `metadata/` prefix matching `*.part` older than 7 days. +- **Expire `metadata/*.part.*` Objects**: Configure lifecycle expiration for objects under `metadata/` prefix matching `*.part.*` older than 7 days. - **Expire `locks/*` Objects**: Configure lifecycle expiration for objects under `locks/` prefix older than 1 day. ### IAM Data Action Summary diff --git a/docs/TESTING.md b/docs/TESTING.md index bf55f415..8bc1e020 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -139,9 +139,9 @@ curl -X HEAD -H "Tus-Resumable: 1.0.0" -I ${UPLOAD_URL} ## Automated Unit Test Coverage Verification -To locally verify unit test coverage against a target percentage (default: 95%) and report which lines/files are not covered, you can run the Maven build with the `check-coverage` profile. +To locally verify unit test coverage against a target percentage (default: 93%) and report which lines/files are not covered, you can run the Maven build with the `check-coverage` profile. -### Run with default threshold (95%) +### Run with default threshold (93%) ```bash mvn verify -Pcheck-coverage ``` diff --git a/src/main/java/me/desair/tus/server/HttpProblemDetails.java b/src/main/java/me/desair/tus/server/HttpProblemDetails.java index ad7187bf..72237108 100644 --- a/src/main/java/me/desair/tus/server/HttpProblemDetails.java +++ b/src/main/java/me/desair/tus/server/HttpProblemDetails.java @@ -2,6 +2,7 @@ import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; @@ -149,11 +150,11 @@ public String toJson() { */ public void writeTo(HttpServletResponse response) throws IOException { Objects.requireNonNull(response, "Response cannot be null"); - byte[] jsonBytes = toJson().getBytes(java.nio.charset.StandardCharsets.UTF_8); + byte[] jsonBytes = toJson().getBytes(StandardCharsets.UTF_8); response.setStatus(status); response.setHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PROBLEM_JSON); response.setContentLength(jsonBytes.length); - response.getWriter().write(new String(jsonBytes, java.nio.charset.StandardCharsets.UTF_8)); + response.getWriter().write(new String(jsonBytes, StandardCharsets.UTF_8)); response.getWriter().flush(); } diff --git a/src/main/java/me/desair/tus/server/TusFileUploadService.java b/src/main/java/me/desair/tus/server/TusFileUploadService.java index f226e655..ea469b43 100644 --- a/src/main/java/me/desair/tus/server/TusFileUploadService.java +++ b/src/main/java/me/desair/tus/server/TusFileUploadService.java @@ -35,6 +35,7 @@ import me.desair.tus.server.upload.UploadLock; import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; +import me.desair.tus.server.upload.UploadType; import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.upload.cache.ThreadLocalCachedStorageAndLockingService; import me.desair.tus.server.upload.disk.DiskStorageService; @@ -70,6 +71,7 @@ public class TusFileUploadService implements Closeable { private final LinkedHashMap enabledFeatures = new LinkedHashMap<>(); private final Set supportedHttpMethods = EnumSet.noneOf(HttpMethod.class); private boolean isThreadLocalCacheEnabled = false; + private Long configuredMaxAppendSize = null; private boolean isChunkedTransferDecodingEnabled = false; private ProtocolVersion supportedProtocolVersion = ProtocolVersion.AUTO; private Duration lockWaitTimeout = DEFAULT_LOCK_WAIT_TIMEOUT; @@ -244,6 +246,7 @@ public TusFileUploadService withMaxAppendSize(Long maxAppendSize) { Validate.exclusiveBetween( 0, Long.MAX_VALUE, maxAppendSize, "The max append size must be bigger than 0"); } + this.configuredMaxAppendSize = maxAppendSize; this.uploadStorageService.setMaxAppendSize(maxAppendSize); return this; } @@ -321,24 +324,37 @@ public TusFileUploadService withUploadIdFactory(UploadIdFactory uploadIdFactory) */ public TusFileUploadService withUploadStorageService(UploadStorageService uploadStorageService) { Objects.requireNonNull(uploadStorageService, "The UploadStorageService cannot be null"); - // Copy over any previous configuration - uploadStorageService.setMaxUploadSize(this.uploadStorageService.getMaxUploadSize()); - uploadStorageService.setMaxAppendSize(this.uploadStorageService.getMaxAppendSize()); - uploadStorageService.setMinAppendSize(this.uploadStorageService.getMinAppendSize()); - uploadStorageService.setMinSize(this.uploadStorageService.getMinSize()); - uploadStorageService.setCloudUploadThreadPoolSize( - this.uploadStorageService.getCloudUploadThreadPoolSize()); - uploadStorageService.setUploadExpirationPeriod( - this.uploadStorageService.getUploadExpirationPeriod()); - uploadStorageService.setUploadDeduplicationEnabled( - this.uploadStorageService.isUploadDeduplicationEnabled()); - uploadStorageService.setJsonSerializationEnabled( - this.uploadStorageService.isJsonSerializationEnabled()); - uploadStorageService.setDrainTimeout(this.uploadStorageService.getDrainTimeout()); + // Unwrap any active ThreadLocal cache wrapper so configuration copies between concrete + // delegates and double-wrapping is prevented. + UploadStorageService previousStorage = unwrapStorageService(this.uploadStorageService); + UploadLockingService rawLocking = unwrapLockingService(this.uploadLockingService); + UploadStorageService targetStorage = unwrapStorageService(uploadStorageService); + + // Copy over previous configuration, preserving backend-specific maxAppendSize defaults + // (such as Azure's 400 GiB limit) unless withMaxAppendSize() was explicitly configured. + targetStorage.setMaxUploadSize(previousStorage.getMaxUploadSize()); + if (this.configuredMaxAppendSize != null) { + targetStorage.setMaxAppendSize(this.configuredMaxAppendSize); + } + targetStorage.setMinAppendSize(previousStorage.getMinAppendSize()); + targetStorage.setMinSize(previousStorage.getMinSize()); + targetStorage.setCloudUploadThreadPoolSize(previousStorage.getCloudUploadThreadPoolSize()); + targetStorage.setUploadExpirationPeriod(previousStorage.getUploadExpirationPeriod()); + targetStorage.setUploadDeduplicationEnabled(previousStorage.isUploadDeduplicationEnabled()); + targetStorage.setJsonSerializationEnabled(previousStorage.isJsonSerializationEnabled()); + targetStorage.setDrainTimeout(previousStorage.getDrainTimeout()); + // Call setIdFactory on the caller-provided reference so external cache wrappers wired elsewhere + // (such as VirtualConcatenationService) also receive idFactory before unwrapping. uploadStorageService.setIdFactory(this.idFactory); - uploadStorageService.setUploadLockingService(this.uploadLockingService); - // Update the upload storage service - this.uploadStorageService = uploadStorageService; + targetStorage.setUploadLockingService(rawLocking); + + if (previousStorage != targetStorage) { + closeQuietly(previousStorage); + } + + // Update the upload storage service and re-wrap cache if enabled + this.uploadStorageService = targetStorage; + this.uploadLockingService = rawLocking; prepareCacheIfEnabled(); return this; } @@ -383,12 +399,22 @@ public UploadStorageService getUploadStorageService() { */ public TusFileUploadService withUploadLockingService(UploadLockingService uploadLockingService) { Objects.requireNonNull(uploadLockingService, "The UploadLockingService cannot be null"); + UploadLockingService previousLocking = unwrapLockingService(this.uploadLockingService); + UploadStorageService rawStorage = unwrapStorageService(this.uploadStorageService); + UploadLockingService targetLocking = unwrapLockingService(uploadLockingService); + + // Preserve idFactory on both any caller-provided cache wrapper and the underlying delegate. uploadLockingService.setIdFactory(this.idFactory); - // Update the upload locking service - this.uploadLockingService = uploadLockingService; - if (this.uploadStorageService != null) { - this.uploadStorageService.setUploadLockingService(uploadLockingService); + if (rawStorage != null) { + rawStorage.setUploadLockingService(targetLocking); + } + if (previousLocking != null && previousLocking != targetLocking) { + closeQuietly(previousLocking); } + + // Update the upload locking service and re-wrap cache if enabled + this.uploadStorageService = rawStorage; + this.uploadLockingService = targetLocking; prepareCacheIfEnabled(); return this; } @@ -453,7 +479,6 @@ public TusFileUploadService withStoragePath(String storagePath) { Validate.notBlank(storagePath, "The storage path cannot be blank"); withUploadStorageService(new DiskStorageService(storagePath)); withUploadLockingService(new LeaseFileLockingService(storagePath)); - prepareCacheIfEnabled(); return this; } @@ -635,7 +660,12 @@ public UploadInfo process( response.sendError(e.getStatus(), e.getMessage()); } - if (wasInProgress && processedUploadInfo != null && !processedUploadInfo.isUploadInProgress()) { + // Only notify completion listeners for standalone/final uploads, not concatenated PARTIAL + // segments whose data will still be merged into a separate final resource. + if (wasInProgress + && processedUploadInfo != null + && !processedUploadInfo.isUploadInProgress() + && !UploadType.PARTIAL.equals(processedUploadInfo.getUploadType())) { notifyUploadCompletionListeners(processedUploadInfo); } @@ -1226,12 +1256,52 @@ private void updateSupportedHttpMethods() { } private void prepareCacheIfEnabled() { - if (isThreadLocalCacheEnabled && uploadStorageService != null) { + UploadStorageService rawStorage = unwrapStorageService(this.uploadStorageService); + UploadLockingService rawLocking = unwrapLockingService(this.uploadLockingService); + + if (isThreadLocalCacheEnabled && rawStorage != null) { ThreadLocalCachedStorageAndLockingService service = - new ThreadLocalCachedStorageAndLockingService(uploadStorageService, uploadLockingService); + new ThreadLocalCachedStorageAndLockingService(rawStorage, rawLocking); service.setIdFactory(this.idFactory); this.uploadStorageService = service; this.uploadLockingService = service; + } else { + this.uploadStorageService = rawStorage; + this.uploadLockingService = rawLocking; + } + } + + private UploadStorageService unwrapStorageService(UploadStorageService service) { + if (service instanceof ThreadLocalCachedStorageAndLockingService cached) { + return cached.getStorageServiceDelegate(); + } + return service; + } + + private UploadLockingService unwrapLockingService(UploadLockingService service) { + if (service instanceof ThreadLocalCachedStorageAndLockingService cached) { + return cached.getLockingServiceDelegate(); + } + return service; + } + + private void closeQuietly(UploadStorageService storageService) { + if (storageService != null) { + try { + storageService.close(); + } catch (Exception e) { + log.warn("Error closing replaced storage service instance: {}", e.getMessage()); + } + } + } + + private void closeQuietly(UploadLockingService lockingService) { + if (lockingService != null) { + try { + lockingService.close(); + } catch (Exception e) { + log.warn("Error closing replaced locking service instance: {}", e.getMessage()); + } } } @@ -1242,9 +1312,19 @@ private void prepareCacheIfEnabled() { */ @Override public void close() throws IOException { - uploadLockingService.close(); - if (uploadStorageService != null) { + if (uploadStorageService instanceof ThreadLocalCachedStorageAndLockingService) { + // Cache wrapper owns and closes both the locking and storage delegates once. uploadStorageService.close(); + return; + } + try { + if (uploadLockingService != null) { + uploadLockingService.close(); + } + } finally { + if (uploadStorageService != null) { + uploadStorageService.close(); + } } } diff --git a/src/main/java/me/desair/tus/server/checksum/ChecksumAlgorithm.java b/src/main/java/me/desair/tus/server/checksum/ChecksumAlgorithm.java index 2ea54fa6..635cef23 100644 --- a/src/main/java/me/desair/tus/server/checksum/ChecksumAlgorithm.java +++ b/src/main/java/me/desair/tus/server/checksum/ChecksumAlgorithm.java @@ -2,6 +2,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; @@ -112,7 +113,7 @@ public static ChecksumAlgorithm forHttpDigestName(String name) { * @return The comma-separated list of digest algorithms */ public static String getSupportedHttpDigestAlgorithmsHeaderValue() { - List sorted = new java.util.ArrayList<>(Arrays.asList(values())); + List sorted = new ArrayList<>(Arrays.asList(values())); sorted.sort((a, b) -> Integer.compare(b.getPriority(), a.getPriority())); StringBuilder sb = new StringBuilder(); for (int i = 0; i < sorted.size(); i++) { @@ -156,7 +157,7 @@ public static ChecksumAlgorithm selectBestAlgorithm(String wantReprDigest) { return null; } - java.util.List items = StructuredHeaderUtil.parseList(wantReprDigest); + List items = StructuredHeaderUtil.parseList(wantReprDigest); double bestQ = -1.0; ChecksumAlgorithm bestAlg = null; diff --git a/src/main/java/me/desair/tus/server/checksum/ChecksumRequestHandler.java b/src/main/java/me/desair/tus/server/checksum/ChecksumRequestHandler.java index 0ff26a8b..fafc74d4 100644 --- a/src/main/java/me/desair/tus/server/checksum/ChecksumRequestHandler.java +++ b/src/main/java/me/desair/tus/server/checksum/ChecksumRequestHandler.java @@ -32,6 +32,12 @@ public void process( String ownerKey) throws IOException, TusException { + // Skip per-chunk checksum validation when the stream was intentionally interrupted by lock + // contention so partial bytes saved before interruption are not rolled back. + if (servletRequest.isStreamInterrupted()) { + return; + } + String uploadChecksumHeader = servletRequest.getHeader(HttpHeader.UPLOAD_CHECKSUM); if (servletRequest.hasCalculatedChecksum() && StringUtils.isNotBlank(uploadChecksumHeader)) { diff --git a/src/main/java/me/desair/tus/server/core/CoreDefaultResponseHeadersHandler.java b/src/main/java/me/desair/tus/server/core/CoreDefaultResponseHeadersHandler.java index 108713be..a4c5f077 100644 --- a/src/main/java/me/desair/tus/server/core/CoreDefaultResponseHeadersHandler.java +++ b/src/main/java/me/desair/tus/server/core/CoreDefaultResponseHeadersHandler.java @@ -7,6 +7,7 @@ import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; +import org.apache.commons.lang3.Strings; /** * The Tus-Resumable header MUST be included in every request and response except for OPTIONS @@ -34,8 +35,7 @@ public void process( String requestVersion = servletRequest.getHeader(HttpHeader.TUS_RESUMABLE); if (requestVersion != null - && !org.apache.commons.lang3.Strings.CS.equals( - requestVersion, TusFileUploadService.TUS_API_VERSION)) { + && !Strings.CS.equals(requestVersion, TusFileUploadService.TUS_API_VERSION)) { servletResponse.setHeader(HttpHeader.TUS_VERSION, TusFileUploadService.TUS_API_VERSION); } } diff --git a/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java b/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java index f5bd10c9..7da24591 100644 --- a/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java @@ -94,6 +94,9 @@ public HttpProblemDetails process( // the offset in storage. We reload the updated UploadInfo and acknowledge the partial // PATCH with 204 No Content and the new Upload-Offset rather than throwing a 500 error. if (interruptibleStream != null && interruptibleStream.isInterrupted()) { + // Mark the request as interrupted so downstream checksum/digest handlers preserve the + // partial progress saved so far instead of rejecting the truncated stream. + servletRequest.setStreamInterrupted(true); // Refresh UploadInfo first so the log statement and response reflect the true committed // byte offset persisted by the storage backend up to the interruption point. UploadInfo refreshed = diff --git a/src/main/java/me/desair/tus/server/cors/CorsExtension.java b/src/main/java/me/desair/tus/server/cors/CorsExtension.java index 96206f2f..535f503f 100644 --- a/src/main/java/me/desair/tus/server/cors/CorsExtension.java +++ b/src/main/java/me/desair/tus/server/cors/CorsExtension.java @@ -1,6 +1,7 @@ package me.desair.tus.server.cors; import java.util.Collection; +import java.util.Collections; import java.util.List; import me.desair.tus.server.HttpMethod; import me.desair.tus.server.RequestHandler; @@ -17,7 +18,7 @@ public String getName() { @Override public Collection getMinimalSupportedHttpMethods() { - return java.util.Collections.emptyList(); + return Collections.emptyList(); } @Override diff --git a/src/main/java/me/desair/tus/server/creation/validation/PatchUploadLengthValidator.java b/src/main/java/me/desair/tus/server/creation/validation/PatchUploadLengthValidator.java index d373c499..b99e6dc7 100644 --- a/src/main/java/me/desair/tus/server/creation/validation/PatchUploadLengthValidator.java +++ b/src/main/java/me/desair/tus/server/creation/validation/PatchUploadLengthValidator.java @@ -7,6 +7,7 @@ import me.desair.tus.server.RequestValidator; import me.desair.tus.server.exception.InvalidUploadLengthException; import me.desair.tus.server.exception.MaxUploadLengthExceededException; +import me.desair.tus.server.exception.MinUploadLengthNotReachedException; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.upload.UploadInfo; import me.desair.tus.server.upload.UploadStorageService; @@ -41,6 +42,13 @@ public void validate( "The Upload-Length cannot be modified once set."); } } else { + if (uploadInfo.getOffset() != null && uploadLength < uploadInfo.getOffset()) { + throw new InvalidUploadLengthException( + "The Upload-Length " + + uploadLength + + " is smaller than the current offset " + + uploadInfo.getOffset()); + } // Verify max upload size if (uploadStorageService.getMaxUploadSize() > 0 && uploadLength > uploadStorageService.getMaxUploadSize()) { @@ -50,6 +58,15 @@ public void validate( + " exceeds the maximum allowed size of " + uploadStorageService.getMaxUploadSize()); } + // Verify min upload size + Long minSize = uploadStorageService.getMinSize(); + if (minSize != null && minSize > 0 && uploadLength < minSize) { + throw new MinUploadLengthNotReachedException( + "The Upload-Length " + + uploadLength + + " is smaller than the minimum allowed size of " + + minSize); + } } } } diff --git a/src/main/java/me/desair/tus/server/creation/validation/PostUriValidator.java b/src/main/java/me/desair/tus/server/creation/validation/PostUriValidator.java index 67265ac6..371cd0e4 100644 --- a/src/main/java/me/desair/tus/server/creation/validation/PostUriValidator.java +++ b/src/main/java/me/desair/tus/server/creation/validation/PostUriValidator.java @@ -1,6 +1,8 @@ package me.desair.tus.server.creation.validation; import jakarta.servlet.http.HttpServletRequest; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.regex.Matcher; import java.util.regex.Pattern; import me.desair.tus.server.HttpMethod; @@ -16,6 +18,7 @@ */ public class PostUriValidator implements RequestValidator { + private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); private Pattern uploadUriPattern = null; @Override @@ -41,11 +44,25 @@ public boolean supports(HttpMethod method) { } private Pattern getUploadUriPattern(UploadStorageService uploadStorageService) { - if (uploadUriPattern == null) { - // A POST request should match the full URI path - String path = Utils.extractUriPath(uploadStorageService.getUploadUri()); - uploadUriPattern = Pattern.compile("^" + path + "$"); + thisObjectLock.readLock().lock(); + try { + if (uploadUriPattern != null) { + return uploadUriPattern; + } + } finally { + thisObjectLock.readLock().unlock(); + } + + thisObjectLock.writeLock().lock(); + try { + if (uploadUriPattern == null) { + // A POST request should match the full URI path + String path = Utils.extractUriPath(uploadStorageService.getUploadUri()); + uploadUriPattern = Pattern.compile("^" + path + "$"); + } + return uploadUriPattern; + } finally { + thisObjectLock.writeLock().unlock(); } - return uploadUriPattern; } } diff --git a/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java b/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java index 6ad842b7..ccbfadb4 100644 --- a/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java @@ -78,6 +78,9 @@ public HttpProblemDetails process( // updates the offset in storage. We reload the updated UploadInfo and acknowledge the // partial upload rather than propagating an unhandled error to the servlet container. if (interruptibleStream.isInterrupted()) { + // Mark the request as interrupted so downstream checksum handlers keep the + // partial bytes persisted before contention interrupted the upload stream. + servletRequest.setStreamInterrupted(true); UploadInfo refreshed = uploadStorageService.getUploadInfo(location, ownerKey); if (refreshed != null) { uploadInfo = refreshed; diff --git a/src/main/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandler.java b/src/main/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandler.java index c2cefa98..44f3c9c2 100644 --- a/src/main/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandler.java @@ -45,8 +45,13 @@ public HttpProblemDetails process( TusException exception) throws IOException, TusException { - // 1. Verify Content-Digest for the request chunk - verifyContentDigest(method, servletRequest); + boolean streamInterrupted = servletRequest != null && servletRequest.isStreamInterrupted(); + + // 1. Verify Content-Digest for the request chunk (skip when stream was interrupted early by + // lock contention so saved partial progress is preserved for resumption) + if (!streamInterrupted) { + verifyContentDigest(method, servletRequest); + } // Resolve UploadInfo using centralized helpers String uploadUri = Utils.getUploadUri(servletRequest, servletResponse); @@ -60,15 +65,17 @@ public HttpProblemDetails process( } // 4. Provide Repr-Digest / Content-Digest response header if requested - String wantContentDigest = servletRequest.getHeader(HttpHeader.WANT_CONTENT_DIGEST); - if (StringUtils.isNotBlank(wantContentDigest)) { - ChecksumAlgorithm preferredAlg = ChecksumAlgorithm.selectBestAlgorithm(wantContentDigest); - if (preferredAlg != null) { - String calculatedVal = servletRequest.getCalculatedChecksum(preferredAlg); - if (calculatedVal != null) { - servletResponse.setHeader( - HttpHeader.CONTENT_DIGEST, - preferredAlg.getHttpDigestNames().get(0) + "=:" + calculatedVal + ":"); + if (!streamInterrupted) { + String wantContentDigest = servletRequest.getHeader(HttpHeader.WANT_CONTENT_DIGEST); + if (StringUtils.isNotBlank(wantContentDigest)) { + ChecksumAlgorithm preferredAlg = ChecksumAlgorithm.selectBestAlgorithm(wantContentDigest); + if (preferredAlg != null) { + String calculatedVal = servletRequest.getCalculatedChecksum(preferredAlg); + if (calculatedVal != null) { + servletResponse.setHeader( + HttpHeader.CONTENT_DIGEST, + preferredAlg.getHttpDigestNames().get(0) + "=:" + calculatedVal + ":"); + } } } } diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java index d0e04627..bd798357 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java @@ -1,10 +1,12 @@ package me.desair.tus.server.rufh.handler; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.InputStream; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; import me.desair.tus.server.HttpProblemDetails; +import me.desair.tus.server.exception.MinUploadLengthNotReachedException; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.upload.UploadInfo; import me.desair.tus.server.upload.UploadLockingService; @@ -62,9 +64,19 @@ public HttpProblemDetails process( boolean hadLengthInitially = uploadInfo.hasLength(); long initialOffset = uploadInfo.getOffset() != null ? uploadInfo.getOffset() : 0L; long contentLength = servletRequest.getContentLengthLong(); - if (Boolean.TRUE.equals(uploadComplete) && !hadLengthInitially && contentLength >= 0) { + Long providedLength = + StructuredHeaderUtil.parseInteger(servletRequest.getHeader(HttpHeader.UPLOAD_LENGTH)); + // Persist an explicitly supplied Upload-Length on a deferred-length upload before appending + // so progress and completion checks see the declared length; only fall back to Content-Length + // when Upload-Complete: ?1 completes a deferred upload without an Upload-Length header. + boolean derivedLengthFromComplete = false; + if (!hadLengthInitially && providedLength != null && providedLength >= 0) { + uploadInfo.setLength(providedLength); + uploadStorageService.update(uploadInfo); + } else if (Boolean.TRUE.equals(uploadComplete) && !hadLengthInitially && contentLength >= 0) { uploadInfo.setLength(initialOffset + contentLength); uploadStorageService.update(uploadInfo); + derivedLengthFromComplete = true; } InputStream is = servletRequest.getContentInputStream(); @@ -99,7 +111,7 @@ public HttpProblemDetails process( // If an un-interrupted network error aborted a deferred-length append that temporarily // set length from Content-Length + Upload-Complete: ?1, restore length to null before // rethrowing so the upload remains open with deferred length. - if (!hadLengthInitially && uploadInfo != null) { + if (derivedLengthFromComplete && uploadInfo != null) { uploadInfo.setLength(null); uploadStorageService.update(uploadInfo); } @@ -117,6 +129,11 @@ public HttpProblemDetails process( boolean streamInterrupted = (interruptibleStream != null && interruptibleStream.isInterrupted()) || (contentLength >= 0 && (uploadInfo.getOffset() - initialOffset) < contentLength); + if (streamInterrupted) { + // Mark the request as interrupted so Content-Digest validation preserves saved partial bytes + // instead of rejecting the truncated chunk. + servletRequest.setStreamInterrupted(true); + } // Per RUFH §4.2: An upload must only be marked complete if the stream completed uninterrupted. // If the stream was interrupted, we acknowledge partial progress and allow subsequent PATCH @@ -125,11 +142,20 @@ public HttpProblemDetails process( !streamInterrupted && (Boolean.TRUE.equals(uploadComplete) || isUploadCompleted(uploadInfo)); if (isFinished) { + Long minSize = uploadStorageService.getMinSize(); + if (minSize != null && minSize > 0 && uploadInfo.getOffset() < minSize) { + throw new MinUploadLengthNotReachedException( + "The upload length (" + + uploadInfo.getOffset() + + ") is smaller than the minimum allowed size (" + + minSize + + ")"); + } uploadInfo.setLength(uploadInfo.getOffset()); uploadStorageService.update(uploadInfo); - } else if (streamInterrupted && !hadLengthInitially) { - // If the stream was interrupted on an upload whose length was deferred, - // reset length to null so that it remains open for subsequent resumes. + } else if (streamInterrupted && derivedLengthFromComplete) { + // If the stream was interrupted on a deferred-length upload whose length was only tentatively + // inferred from Content-Length, reset length to null so it remains open for resume. uploadInfo.setLength(null); uploadStorageService.update(uploadInfo); } @@ -139,9 +165,9 @@ public HttpProblemDetails process( HttpHeader.UPLOAD_COMPLETE, StructuredHeaderUtil.formatBoolean(isFinished)); if (isFinished) { - servletResponse.setStatus(200); + servletResponse.setStatus(HttpServletResponse.SC_OK); } else { - servletResponse.setStatus(204); + servletResponse.setStatus(HttpServletResponse.SC_NO_CONTENT); } return null; } diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java index 28661af5..594effcd 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java @@ -1,5 +1,6 @@ package me.desair.tus.server.rufh.handler; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.InputStream; import me.desair.tus.server.HttpHeader; @@ -97,6 +98,9 @@ public HttpProblemDetails process( String uploadUri = Utils.getUploadUriOnCreation(uploadInfo, servletRequest, uploadStorageService); + // Set Location early so error handling and downstream digest processing can resolve the created + // resource URI even if payload streaming is interrupted or throws an exception. + servletResponse.setHeader(HttpHeader.LOCATION, uploadUri); // Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no content, // or to requests completing the upload by including the Upload-Complete: ?1 header field." @@ -174,6 +178,11 @@ public HttpProblemDetails process( boolean streamInterrupted = (interruptibleStream != null && interruptibleStream.isInterrupted()) || (contentLength >= 0 && uploadInfo != null && uploadInfo.getOffset() < contentLength); + if (streamInterrupted && servletRequest != null) { + // Mark the request as interrupted so Content-Digest validation does not discard partial bytes + // saved before contention or early truncation. + servletRequest.setStreamInterrupted(true); + } // Per RUFH §4.2: An upload must only be marked complete if the stream completed uninterrupted. // If the stream was interrupted, we acknowledge partial progress and allow subsequent PATCH @@ -186,7 +195,7 @@ public HttpProblemDetails process( if (isFinished && uploadInfo != null) { uploadInfo.setLength(uploadInfo.getOffset()); uploadStorageService.update(uploadInfo); - servletResponse.setStatus(200); + servletResponse.setStatus(HttpServletResponse.SC_OK); servletResponse.setHeader(HttpHeader.LOCATION, uploadUri); servletResponse.setHeader( HttpHeader.UPLOAD_COMPLETE, StructuredHeaderUtil.formatBoolean(true)); @@ -198,7 +207,7 @@ public HttpProblemDetails process( uploadInfo.setLength(null); uploadStorageService.update(uploadInfo); } - servletResponse.setStatus(201); + servletResponse.setStatus(HttpServletResponse.SC_CREATED); servletResponse.setHeader(HttpHeader.LOCATION, uploadUri); servletResponse.setHeader( HttpHeader.UPLOAD_COMPLETE, StructuredHeaderUtil.formatBoolean(false)); diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandler.java index 35b9f2cc..c24e7e9e 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandler.java @@ -1,5 +1,6 @@ package me.desair.tus.server.rufh.handler; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import me.desair.tus.server.HttpMethod; import me.desair.tus.server.HttpProblemDetails; @@ -46,7 +47,7 @@ public HttpProblemDetails process( } uploadStorageService.terminateUpload(uploadInfo); - servletResponse.setStatus(204); + servletResponse.setStatus(HttpServletResponse.SC_NO_CONTENT); return null; } } diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhErrorHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhErrorHandler.java index 141c05fb..9346511e 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhErrorHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhErrorHandler.java @@ -1,5 +1,6 @@ package me.desair.tus.server.rufh.handler; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; @@ -8,6 +9,7 @@ import me.desair.tus.server.exception.TusException; import me.desair.tus.server.exception.UploadAlreadyCompletedException; import me.desair.tus.server.exception.UploadDigestMismatchException; +import me.desair.tus.server.exception.UploadLengthExceededException; import me.desair.tus.server.exception.UploadOffsetMismatchException; import me.desair.tus.server.upload.UploadInfo; import me.desair.tus.server.upload.UploadLockingService; @@ -78,8 +80,16 @@ public HttpProblemDetails process( return HttpProblemDetails.forOffsetMismatch(expectedOffset, provided); } else if (exception instanceof UploadAlreadyCompletedException) { - // Section 7.2: Completed Upload - return HttpProblemDetails.forCompletedUpload(400); + // Section 4.4.2 & 7.2: Deactivate completed upload resource under the request lock before + // returning the 400 Problem Details error so pre-flight checks stay side-effect free. + terminateUploadQuietly(servletRequest, uploadStorageService, ownerKey); + return HttpProblemDetails.forCompletedUpload(HttpServletResponse.SC_BAD_REQUEST); + + } else if (exception instanceof UploadLengthExceededException) { + // Section 4.4.2: Invalidate the upload resource when an append pushes offset past length + // so subsequent interactions fail cleanly while this request still returns 409 Conflict. + terminateUploadQuietly(servletRequest, uploadStorageService, ownerKey); + return null; } else if (exception instanceof InconsistentUploadLengthException) { // Section 7.2: Inconsistent Length @@ -92,4 +102,22 @@ public HttpProblemDetails process( } return null; } + + private void terminateUploadQuietly( + TusServletRequest servletRequest, + UploadStorageService uploadStorageService, + String ownerKey) { + if (uploadStorageService == null || servletRequest == null) { + return; + } + try { + UploadInfo uploadInfo = + uploadStorageService.getUploadInfo(servletRequest.getRequestURI(), ownerKey); + if (uploadInfo != null) { + uploadStorageService.terminateUpload(uploadInfo); + } + } catch (Exception e) { + // Ignore cleanup failure so the protocol error response is still returned + } + } } diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhHeadGetRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhHeadGetRequestHandler.java index 85d4ee9c..71e3e39e 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhHeadGetRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhHeadGetRequestHandler.java @@ -1,5 +1,6 @@ package me.desair.tus.server.rufh.handler; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; @@ -63,7 +64,7 @@ public HttpProblemDetails process( return null; } - servletResponse.setStatus(204); + servletResponse.setStatus(HttpServletResponse.SC_NO_CONTENT); servletResponse.setHeader(HttpHeader.UPLOAD_OFFSET, String.valueOf(uploadInfo.getOffset())); servletResponse.setHeader( HttpHeader.UPLOAD_COMPLETE, diff --git a/src/main/java/me/desair/tus/server/rufh/validation/RufhAppendValidator.java b/src/main/java/me/desair/tus/server/rufh/validation/RufhAppendValidator.java index 165788f0..0cdd28fa 100644 --- a/src/main/java/me/desair/tus/server/rufh/validation/RufhAppendValidator.java +++ b/src/main/java/me/desair/tus/server/rufh/validation/RufhAppendValidator.java @@ -7,9 +7,12 @@ import me.desair.tus.server.RequestValidator; import me.desair.tus.server.exception.InconsistentUploadLengthException; import me.desair.tus.server.exception.InvalidUploadCompleteHeaderException; +import me.desair.tus.server.exception.InvalidUploadLengthException; import me.desair.tus.server.exception.InvalidUploadOffsetHeaderException; import me.desair.tus.server.exception.MaxAppendSizeExceededException; +import me.desair.tus.server.exception.MaxUploadLengthExceededException; import me.desair.tus.server.exception.MinAppendSizeNotMetException; +import me.desair.tus.server.exception.MinUploadLengthNotReachedException; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.exception.UnsupportedMediaTypeException; import me.desair.tus.server.exception.UploadAlreadyCompletedException; @@ -80,14 +83,9 @@ public void validate( } if (!uploadInfo.isUploadInProgress()) { - try { - // Section 4.4.2: Deactivate upload resource when append is attempted on a completed upload - // Terminate/deactivate upload resource per §4.4.2 when append is attempted past - // declared length - uploadStorageService.terminateUpload(uploadInfo); - } catch (Exception e) { - // Log or ignore cleanup failure - } + // Section 4.4.2: Reject append attempts on a completed upload; resource deactivation is + // performed under the request lock in RufhErrorHandler so read-only pre-checks (like 104 + // interim generation) never mutate storage before process() executes. throw new UploadAlreadyCompletedException("Upload resource is already completed"); } @@ -137,11 +135,6 @@ public void validate( if (uploadInfo.hasLength()) { if (currentOffset >= uploadInfo.getLength() || (contentLength > 0 && currentOffset + contentLength > uploadInfo.getLength())) { - try { - uploadStorageService.terminateUpload(uploadInfo); - } catch (Exception e) { - // Log or ignore cleanup failure - } throw new UploadLengthExceededException( "Appended content length (" + contentLength @@ -172,11 +165,29 @@ public void validate( } } - // Section 4.1.3: Validate consistency of Upload-Length if provided in append request + // Section 4.1.3: Validate consistency of Upload-Length if provided in append request, or + // enforce declared/derived final length bounds when length was previously deferred String uploadLengthHeader = request.getHeader(HttpHeader.UPLOAD_LENGTH); Long providedLength = StructuredHeaderUtil.parseInteger(uploadLengthHeader); - if (providedLength != null && uploadInfo.hasLength()) { - if (!providedLength.equals(uploadInfo.getLength())) { + validateAppendUploadLength( + uploadInfo, + uploadStorageService, + providedLength, + uploadComplete, + contentLength, + currentOffset); + } + + private void validateAppendUploadLength( + UploadInfo uploadInfo, + UploadStorageService uploadStorageService, + Long providedLength, + Boolean uploadComplete, + long contentLength, + long currentOffset) + throws TusException { + if (uploadInfo.hasLength()) { + if (providedLength != null && !providedLength.equals(uploadInfo.getLength())) { throw new InconsistentUploadLengthException( "Provided Upload-Length (" + providedLength @@ -184,6 +195,68 @@ public void validate( + uploadInfo.getLength() + ")"); } + return; + } + + if (providedLength != null) { + if (providedLength < currentOffset) { + throw new InvalidUploadLengthException( + "Provided Upload-Length (" + + providedLength + + ") is smaller than current offset (" + + currentOffset + + ")"); + } + if (Boolean.TRUE.equals(uploadComplete) + && contentLength >= 0 + && providedLength != currentOffset + contentLength) { + throw new InconsistentUploadLengthException( + "Provided Upload-Length (" + + providedLength + + ") does not match final offset (" + + (currentOffset + contentLength) + + ")"); + } + if (contentLength > 0 && currentOffset + contentLength > providedLength) { + throw new UploadLengthExceededException( + "Appended content length (" + + contentLength + + ") pushes total offset past declared upload length (" + + providedLength + + ")"); + } + } + + // Enforce global min/max size constraints once an append either declares Upload-Length or + // finalizes a deferred upload via Upload-Complete: ?1 with known Content-Length + Long effectiveLength = + providedLength != null + ? providedLength + : (Boolean.TRUE.equals(uploadComplete) && contentLength >= 0 + ? currentOffset + contentLength + : null); + if (effectiveLength == null) { + return; + } + + long maxUploadSize = uploadStorageService.getMaxUploadSize(); + if (maxUploadSize > 0 && effectiveLength > maxUploadSize) { + throw new MaxUploadLengthExceededException( + "The requested upload length (" + + effectiveLength + + ") exceeds the maximum allowed size (" + + maxUploadSize + + ")"); + } + + Long minSize = uploadStorageService.getMinSize(); + if (minSize != null && minSize > 0 && effectiveLength < minSize) { + throw new MinUploadLengthNotReachedException( + "The requested upload length (" + + effectiveLength + + ") is smaller than the minimum allowed size (" + + minSize + + ")"); } } } diff --git a/src/main/java/me/desair/tus/server/rufh/validation/RufhCreationValidator.java b/src/main/java/me/desair/tus/server/rufh/validation/RufhCreationValidator.java index 7a9e401f..10a7f731 100644 --- a/src/main/java/me/desair/tus/server/rufh/validation/RufhCreationValidator.java +++ b/src/main/java/me/desair/tus/server/rufh/validation/RufhCreationValidator.java @@ -65,17 +65,24 @@ public void validate( } } + // Derive effective length from Upload-Length or from Content-Length on single-request + // Upload-Complete: ?1 creations so size bounds apply uniformly to both styles. + Long effectiveLength = + uploadLength != null + ? uploadLength + : (Boolean.TRUE.equals(uploadComplete) && contentLength >= 0 ? contentLength : null); + long maxUploadSize = uploadStorageService.getMaxUploadSize(); - if (maxUploadSize > 0 && uploadLength != null && uploadLength > maxUploadSize) { + if (maxUploadSize > 0 && effectiveLength != null && effectiveLength > maxUploadSize) { throw new MaxUploadLengthExceededException( "The requested upload length exceeds the maximum allowed size"); } Long minSize = uploadStorageService.getMinSize(); - if (minSize != null && minSize > 0 && uploadLength != null && uploadLength < minSize) { + if (minSize != null && minSize > 0 && effectiveLength != null && effectiveLength < minSize) { throw new MinUploadLengthNotReachedException( "The requested upload length (" - + uploadLength + + effectiveLength + ") is smaller than the minimum allowed size (" + minSize + ")"); diff --git a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLock.java b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLock.java index 89b38c08..c089876d 100644 --- a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLock.java +++ b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLock.java @@ -6,6 +6,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import me.desair.tus.server.util.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,6 +30,8 @@ public abstract class AbstractLeaseLock implements UploadLock { private final UploadId uploadId; private final Map activeInputStreams; private final ScheduledFuture heartbeatTask; + private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); + private boolean released = false; /** * Constructs an active lock and schedules the background heartbeat renewal task using the shared @@ -79,11 +83,21 @@ public long getLeaseDurationMs() { } public long getExpiresAt() { - return leaseData.getExpiresAt(); + thisObjectLock.readLock().lock(); + try { + return leaseData.getExpiresAt(); + } finally { + thisObjectLock.readLock().unlock(); + } } public void setExpiresAt(long expiresAt) { - leaseData.setExpiresAt(expiresAt); + thisObjectLock.writeLock().lock(); + try { + leaseData.setExpiresAt(expiresAt); + } finally { + thisObjectLock.writeLock().unlock(); + } } public long getAcquiredAt() { @@ -102,52 +116,72 @@ public void release() { @Override public void close() { - // 1. Cancel heartbeat task cleanly + // 1. Cancel heartbeat task before waiting on the write lock so any blocked renewal wakes up + // immediately instead of holding up lock release. if (heartbeatTask != null) { heartbeatTask.cancel(true); } - // 2. Remove active stream registration from JVM heap - if (activeInputStreams != null && uploadId != null) { - activeInputStreams.remove(uploadId); - } + thisObjectLock.writeLock().lock(); + try { + if (released) { + return; + } + released = true; - // 3. Release backend storage resources - releaseLockResource(); + // 2. Remove active stream registration from JVM heap + if (activeInputStreams != null && uploadId != null) { + activeInputStreams.remove(uploadId); + } + + // 3. Release backend storage resources only after any in-flight renewLease() has finished + releaseLockResource(); + } finally { + thisObjectLock.writeLock().unlock(); + } } /** * Renew the lock lease by advancing the expiration timestamp and persisting the updated metadata. */ public void renewLease() { - long oldExpiresAt = leaseData.getExpiresAt(); - long candidateExpiresAt = System.currentTimeMillis() + leaseData.getLeaseDurationMs(); + thisObjectLock.writeLock().lock(); try { - leaseData.setExpiresAt(candidateExpiresAt); - doRenewLease(); - } catch (IllegalStateException e) { - leaseData.setExpiresAt(oldExpiresAt); - // Ownership lost (e.g. lease taken over by another node): abort immediately - log.info( - "Aborting active input stream for upload ID {} as lock was taken over: {}", - uploadId, - e.getMessage()); - abortActiveStreamAndHeartbeat(); - } catch (Exception e) { - leaseData.setExpiresAt(oldExpiresAt); - // Transient failure (e.g. network timeout). Only abort if previous lease actually expired. - if (System.currentTimeMillis() >= oldExpiresAt) { - log.warn( - "Lease for upload ID {} expired after renewal failure; aborting active stream", - uploadId, - e); - abortActiveStreamAndHeartbeat(); - } else { - log.warn( - "Transient failure renewing lease for upload ID {} (will retry on next tick): {}", + // Do not overwrite or recreate lock objects once close() has released the lock. + if (released) { + return; + } + long oldExpiresAt = leaseData.getExpiresAt(); + long candidateExpiresAt = System.currentTimeMillis() + leaseData.getLeaseDurationMs(); + try { + leaseData.setExpiresAt(candidateExpiresAt); + doRenewLease(); + } catch (IllegalStateException e) { + leaseData.setExpiresAt(oldExpiresAt); + // Ownership lost (e.g. lease taken over by another node): abort immediately + log.info( + "Aborting active input stream for upload ID {} as lock was taken over: {}", uploadId, e.getMessage()); + abortActiveStreamAndHeartbeat(); + } catch (Exception e) { + leaseData.setExpiresAt(oldExpiresAt); + // Transient failure (e.g. network timeout). Only abort if previous lease actually expired. + if (System.currentTimeMillis() >= oldExpiresAt) { + log.warn( + "Lease for upload ID {} expired after renewal failure; aborting active stream", + uploadId, + e); + abortActiveStreamAndHeartbeat(); + } else { + log.warn( + "Transient failure renewing lease for upload ID {} (will retry on next tick): {}", + uploadId, + e.getMessage()); + } } + } finally { + thisObjectLock.writeLock().unlock(); } } diff --git a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java index 57ad0f48..25e7fbee 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java @@ -9,6 +9,7 @@ import me.desair.tus.server.checksum.ChecksumKey; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.exception.UploadNotFoundException; +import me.desair.tus.server.upload.checksum.DeduplicationService; import me.desair.tus.server.upload.concatenation.UploadConcatenationService; import me.desair.tus.server.util.Utils; import org.apache.commons.io.IOUtils; @@ -35,8 +36,8 @@ * String)} retrieve metadata. {@link #getUploadedBytes(UploadId)} and {@link * #copyUploadTo(UploadInfo, OutputStream)} stream committed data bytes. *

  • Termination: {@link #terminateUpload(UploadInfo)} is the authoritative method to - * free physical payload bytes, remove metadata, clean up locks, and coordinate deduplication - * index cleanup. + * free physical payload bytes, remove metadata, and coordinate deduplication index cleanup. + * Active locks are released automatically when the request's {@link UploadLock} closes. * * *

    2. Mandatory Locking Service Integration

    @@ -44,8 +45,9 @@ *

    Concurrency control is mandatory across all storage backends. Every storage service receives * an {@link UploadLockingService} instance via {@link * #setUploadLockingService(UploadLockingService)} and exposes it via {@link - * #getUploadLockingService()}. Implementations MUST retain this reference and delegate lock cleanup - * during {@link #terminateUpload(UploadInfo)} and {@link #cleanupExpiredUploads()}. + * #getUploadLockingService()}. Implementations MUST retain this reference and clean up orphaned + * locks via {@link UploadLockingService#cleanupLock(UploadId)} during {@link + * #cleanupExpiredUploads()}. * *

    3. Checksum Deduplication Support

    * @@ -54,11 +56,10 @@ * *
      *
    • {@link #deduplicate(UploadInfo, ChecksumKey)}: Coordinates cluster-safe deduplication when - * an upload completes, typically delegating to {@link - * me.desair.tus.server.upload.checksum.DeduplicationService}. + * an upload completes, typically delegating to {@link DeduplicationService}. *
    • {@link #dataExists(UploadId)}: Checks whether committed physical data exists in storage. - * Used by {@link me.desair.tus.server.upload.checksum.DeduplicationService} to self-heal - * stale checksum index entries when parent data has been deleted out-of-band. + * Used by {@link DeduplicationService} to self-heal stale checksum index entries when parent + * data has been deleted out-of-band. *
    • {@link #deleteData(UploadId)}: Deletes physical committed data without deleting the * upload's {@link UploadInfo} metadata record. Used to free storage for child duplicates * linked to a parent, and to purge retained parent data when the last active child @@ -392,13 +393,12 @@ default UploadInfo deduplicate(UploadInfo uploadInfo, ChecksumKey checksumKey) * *

      Mandatory Locking Integration: An {@link UploadLockingService} is mandatory across * all storage backends. Implementations MUST retain this reference and invoke {@link - * UploadLockingService#cleanupLock(UploadId)} during {@link #terminateUpload(UploadInfo)} and - * coordinate lock state during {@link #cleanupExpiredUploads()}. + * UploadLockingService#cleanupLock(UploadId)} during {@link #cleanupExpiredUploads()}. * * @param uploadLockingService the locking service instance (must not be {@code null}) */ default void setUploadLockingService(UploadLockingService uploadLockingService) { - java.util.Objects.requireNonNull(uploadLockingService, "uploadLockingService must not be null"); + Objects.requireNonNull(uploadLockingService, "uploadLockingService must not be null"); } /** @@ -415,9 +415,9 @@ default UploadLockingService getUploadLockingService() { * *

      Critical for Deduplication: Implementations supporting upload deduplication MUST * override this method to accurately verify that physical data bytes exist in storage. The {@link - * me.desair.tus.server.upload.checksum.DeduplicationService} invokes this check during parent - * lookup to self-heal stale checksum index entries when parent data has been deleted out-of-band - * (e.g., via storage lifecycle policies or manual deletion). + * DeduplicationService} invokes this check during parent lookup to self-heal stale checksum index + * entries when parent data has been deleted out-of-band (e.g., via storage lifecycle policies or + * manual deletion). * *

      The default implementation returns {@code false} for backward compatibility. If not * overridden in a deduplication-enabled backend, parent uploads will be treated as missing, diff --git a/src/main/java/me/desair/tus/server/upload/UuidUploadIdFactory.java b/src/main/java/me/desair/tus/server/upload/UuidUploadIdFactory.java index 645592f4..e57c0730 100644 --- a/src/main/java/me/desair/tus/server/upload/UuidUploadIdFactory.java +++ b/src/main/java/me/desair/tus/server/upload/UuidUploadIdFactory.java @@ -22,7 +22,7 @@ protected Serializable getIdValueIfValid(String extractedUrlId) { } @Override - public synchronized UploadId createId() { + public UploadId createId() { return new UploadId(UUID.randomUUID()); } } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java index 8378353f..6e2b75bb 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java @@ -252,16 +252,15 @@ boolean deleteLockBlobIfUnleased(BlobClient lockBlob) { return false; } try { - if (Boolean.TRUE.equals(lockBlob.exists())) { - BlobProperties props = lockBlob.getProperties(); - // Check lease state before deletion: if actively held by another thread or node, - // do not attempt deletion to avoid lease conflict errors. - if (props.getLeaseState() != null - && !Strings.CS.equals(props.getLeaseState().toString(), "leased")) { - lockBlob.deleteIfExists(); - log.debug("Cleaned up Azure lock blob {}", lockBlob.getBlobName()); - return true; - } + // Skip a separate exists() call: getProperties() throws BlobStorageException (404) if absent. + BlobProperties props = lockBlob.getProperties(); + // Check lease state before deletion: if actively held by another thread or node, + // do not attempt deletion to avoid lease conflict errors. + if (props.getLeaseState() != null + && !Strings.CS.equals(props.getLeaseState().toString(), "leased")) { + lockBlob.deleteIfExists(); + log.debug("Cleaned up Azure lock blob {}", lockBlob.getBlobName()); + return true; } } catch (Exception e) { log.debug( diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java index 40ba0de7..0d84b597 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java @@ -10,6 +10,7 @@ import com.azure.storage.blob.models.BlockListType; import com.azure.storage.blob.models.ListBlobsOptions; import com.azure.storage.blob.specialized.BlockBlobClient; +import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; @@ -328,12 +329,26 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) } } - // Calculate actual confirmed bytes appended + // Calculate actual confirmed bytes appended and validate minAppendSize BEFORE committing + // staged blocks so a rejected undersized request does not advance committed state. long confirmedAppended = 0L; for (int i = 0; i < blockIds.size() - initialBlockCount; i++) { confirmedAppended += plannedChunkSizes.get(i); } + long candidateOffset = initialOffset + confirmedAppended; + boolean isCompletingOrEmpty = + !upload.isUploadInProgress() + || (upload.getLength() != null && candidateOffset >= upload.getLength()); + if (drainException == null + && streamException == null + && pendingTusException == null + && !isCompletingOrEmpty) { + // Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no content, + // or to requests completing the upload by including the Upload-Complete: ?1 header field." + validateMinAppendSize(confirmedAppended); + } + // 5. Commit any newly staged blocks in batch (1 single commit call for entire request) // Batching block commits into a single call at the end eliminates redundant network // round-trips for every chunk, drastically improving performance. In addition, committing @@ -342,21 +357,20 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) if (blockIds.size() > initialBlockCount) { try { blockBlobClient.commitBlockList(blockIds, true); - upload.setOffset(initialOffset + confirmedAppended); + upload.setOffset(candidateOffset); upload.setUploadPartKeys(new ArrayList<>(blockIds)); if (uploadExpirationPeriod != null && uploadExpirationPeriod > 0) { upload.setExpirationTimestamp(System.currentTimeMillis() + uploadExpirationPeriod); } + // Persist updated offset and block IDs before attempting deduplication so durable + // metadata is always saved regardless of whether deduplication applies. + saveUploadInfo(upload); + boolean finalComplete = upload.getLength() != null && upload.getOffset().equals(upload.getLength()); - boolean deduplicated = false; if (finalComplete) { - deduplicated = checkAndApplyDeduplication(upload); - } - - if (!deduplicated) { - saveUploadInfo(upload); + checkAndApplyDeduplication(upload); } } catch (Exception e) { log.error( @@ -380,18 +394,6 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) throw pendingTusException; } - // 6. Validate minimum append size constraints if configured - // Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no content, - // or to requests completing the upload by including the Upload-Complete: ?1 header field." - boolean isCompletingOrEmpty = - !upload.isUploadInProgress() - || (upload.getLength() != null - && upload.getOffset() != null - && upload.getOffset() >= upload.getLength()); - if (!isCompletingOrEmpty) { - validateMinAppendSize(confirmedAppended); - } - return upload; } @@ -445,6 +447,17 @@ public String getOwnAzureBlobName(UploadInfo uploadInfo) { return uploadPrefix + uploadInfo.getId().toString(); } + /** + * Resolves the effective Azure blob name for an upload identified by its request URI. + * + * @param requestUri The upload request URI + * @return The Azure blob name, or {@code null} if the upload is not found + * @throws IOException If reading upload metadata fails + */ + public String getAzureBlobName(String requestUri) throws IOException { + return getAzureBlobName(requestUri, null); + } + public String getAzureBlobName(String requestUri, String ownerKey) throws IOException { UploadInfo info = getUploadInfo(requestUri, ownerKey); return info != null ? getAzureBlobName(info) : null; @@ -539,7 +552,7 @@ public void removeLastNumberOfBytes(UploadInfo uploadInfo, long byteCount) IOUtils.copyLarge(is, os); } String newBlockId = generateBlockId(0); - try (InputStream is = new java.io.BufferedInputStream(new FileInputStream(tempFile))) { + try (InputStream is = new BufferedInputStream(new FileInputStream(tempFile))) { blockBlobClient.stageBlock(newBlockId, is, tempFile.length()); } blockBlobClient.commitBlockList(List.of(newBlockId), true); @@ -663,6 +676,9 @@ public void close() throws IOException { if (uploadLockingService != null) { uploadLockingService.close(); } + if (concatenationService != null) { + concatenationService.close(); + } } @Override @@ -795,7 +811,7 @@ private void stageBlock( BlockBlobClient blockBlobClient, String blockId, File chunkFile, long chunkSize) throws IOException { if (chunkSize > 0) { - try (InputStream chunkIs = new java.io.BufferedInputStream(new FileInputStream(chunkFile))) { + try (InputStream chunkIs = new BufferedInputStream(new FileInputStream(chunkFile))) { blockBlobClient.stageBlock(blockId, chunkIs, chunkSize); } } @@ -813,6 +829,7 @@ private void commitEmptyDataBlob(UploadId id) { * *

      Incorporates an 8-character random UUID suffix to guarantee that late-landing blocks from * previous aborted or timed-out worker threads never collide with or overwrite blocks in + * subsequent retry attempts. * * @param index The block index * @return The Base64-encoded block ID diff --git a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java index c815217e..0caa1a6a 100644 --- a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java @@ -51,6 +51,24 @@ public ThreadLocalCachedStorageAndLockingService( } } + /** + * Returns the underlying wrapped {@link UploadStorageService} delegate. + * + * @return the storage service delegate + */ + public UploadStorageService getStorageServiceDelegate() { + return storageServiceDelegate; + } + + /** + * Returns the underlying wrapped {@link UploadLockingService} delegate. + * + * @return the locking service delegate + */ + public UploadLockingService getLockingServiceDelegate() { + return lockingServiceDelegate; + } + @Override public UploadInfo getUploadInfo(UploadId id) throws IOException { if (id == null) { @@ -67,7 +85,10 @@ public UploadInfo getUploadInfo(UploadId id) throws IOException { @Override public UploadInfo getUploadInfo(String uploadUrl, String ownerKey) throws IOException { - UploadInfo uploadInfo = getUploadInfo(idFactory.readUploadId(uploadUrl)); + // Skip cache lookup when idFactory has not been wired yet (e.g., an external cache wrapper + // passed into VirtualConcatenationService before setIdFactory runs). + UploadInfo uploadInfo = + idFactory != null ? getUploadInfo(idFactory.readUploadId(uploadUrl)) : null; if (uploadInfo == null || !Objects.equals(uploadInfo.getOwnerKey(), ownerKey)) { uploadInfo = storageServiceDelegate.getUploadInfo(uploadUrl, ownerKey); uploadInfoCache.set(new WeakReference<>(uploadInfo)); @@ -77,8 +98,14 @@ public UploadInfo getUploadInfo(String uploadUrl, String ownerKey) throws IOExce @Override public void update(UploadInfo uploadInfo) throws IOException, UploadNotFoundException { - storageServiceDelegate.update(uploadInfo); - uploadInfoCache.set(new WeakReference<>(uploadInfo)); + try { + storageServiceDelegate.update(uploadInfo); + uploadInfoCache.set(new WeakReference<>(uploadInfo)); + } catch (IOException | UploadNotFoundException | RuntimeException e) { + // Clear cached state so failure recovery reads the true persisted metadata from storage. + cleanupCache(); + throw e; + } } @Override @@ -96,9 +123,16 @@ public String getUploadUri() { @Override public UploadInfo append(UploadInfo upload, InputStream inputStream) throws IOException, TusException { - UploadInfo info = storageServiceDelegate.append(upload, inputStream); - uploadInfoCache.set(new WeakReference<>(info)); - return info; + try { + UploadInfo info = storageServiceDelegate.append(upload, inputStream); + uploadInfoCache.set(new WeakReference<>(info)); + return info; + } catch (IOException | TusException | RuntimeException e) { + // Evict stale pre-append UploadInfo so error handling calculates rolled-back bytes from + // the actual persisted offset rather than the stale cached offset. + cleanupCache(); + throw e; + } } @Override @@ -186,8 +220,13 @@ public void cleanupExpiredUploads(UploadLockingService uploadLockingService) thr @Override public void removeLastNumberOfBytes(UploadInfo uploadInfo, long byteCount) throws UploadNotFoundException, IOException { - storageServiceDelegate.removeLastNumberOfBytes(uploadInfo, byteCount); - uploadInfoCache.set(new WeakReference<>(uploadInfo)); + try { + storageServiceDelegate.removeLastNumberOfBytes(uploadInfo, byteCount); + uploadInfoCache.set(new WeakReference<>(uploadInfo)); + } catch (IOException | UploadNotFoundException | RuntimeException e) { + cleanupCache(); + throw e; + } } @Override @@ -246,11 +285,16 @@ public UploadInfo getUploadInfoByChecksum(String checksum, ChecksumAlgorithm alg @Override public UploadInfo deduplicate(UploadInfo uploadInfo, ChecksumKey checksumKey) throws IOException { - UploadInfo result = storageServiceDelegate.deduplicate(uploadInfo, checksumKey); - if (result != null) { - uploadInfoCache.set(new WeakReference<>(result)); + try { + UploadInfo result = storageServiceDelegate.deduplicate(uploadInfo, checksumKey); + if (result != null) { + uploadInfoCache.set(new WeakReference<>(result)); + } + return result; + } catch (IOException | RuntimeException e) { + cleanupCache(); + throw e; } - return result; } @Override @@ -295,12 +339,17 @@ public Duration getDrainTimeout() { @Override public UploadLock lockUploadByUri(String requestUri) throws TusException, IOException { + // Evict any unlocked pre-lock cached state (e.g. populated during getRawInterimResponse) + // so the first read inside the exclusive lock always fetches fresh state from storage. + cleanupCache(); UploadLock uploadLock = lockingServiceDelegate.lockUploadByUri(requestUri); return new CachedLock(uploadLock); } @Override public UploadLock lockUpload(UploadId id) throws TusException, IOException { + // Evict any unlocked pre-lock cached state before entering the exclusive lock scope. + cleanupCache(); UploadLock uploadLock = lockingServiceDelegate.lockUpload(id); return new CachedLock(uploadLock); } @@ -339,9 +388,15 @@ public void cleanupLock(UploadId id) throws IOException { @Override public void close() throws IOException { - lockingServiceDelegate.close(); - storageServiceDelegate.close(); - cleanupCache(); + try { + lockingServiceDelegate.close(); + } finally { + try { + storageServiceDelegate.close(); + } finally { + cleanupCache(); + } + } } private void cleanupCache() { diff --git a/src/main/java/me/desair/tus/server/upload/checksum/DeduplicationService.java b/src/main/java/me/desair/tus/server/upload/checksum/DeduplicationService.java index 77661fae..b3cb3914 100644 --- a/src/main/java/me/desair/tus/server/upload/checksum/DeduplicationService.java +++ b/src/main/java/me/desair/tus/server/upload/checksum/DeduplicationService.java @@ -77,7 +77,7 @@ public void setUploadLockingService(UploadLockingService lockingService) { * @throws IOException on storage I/O error */ public UploadInfo deduplicate(UploadInfo uploadInfo, ChecksumKey key) throws IOException { - if (uploadInfo == null || key == null) { + if (uploadInfo == null || key == null || uploadInfo.getDuplicatesUploadId() != null) { return uploadInfo; } @@ -273,7 +273,7 @@ private UploadInfo joinParentUnderLock( private UploadInfo doJoinParentUnderLock( UploadInfo uploadInfo, ChecksumKey key, UploadId expectedParentId, boolean allowRetry) throws IOException { - // Under parent lock: B2 Check - re-read current parent from index + // Under parent lock: re-read current parent from index UploadId currentParent = checksumIndex.findParent(key); if (!Objects.equals(expectedParentId, currentParent)) { if (allowRetry && currentParent != null) { @@ -285,9 +285,14 @@ private UploadInfo doJoinParentUnderLock( UploadInfo parentInfo = storageService.getUploadInfo(expectedParentId); boolean parentTombstoned = (parentInfo == null) && checksumIndex.isParentDeleted(key); - // B3 Self-healing check: if parent is missing and not legitimately tombstoned, - // or if parent physical bytes are missing - if ((parentInfo == null && !parentTombstoned) || !storageService.dataExists(expectedParentId)) { + // Self-healing check: reject missing non-tombstoned parents, parents that are still in + // progress or themselves deduplicated children, or parents whose physical bytes are missing. + boolean invalidParentState = + (parentInfo == null && !parentTombstoned) + || (parentInfo != null + && (parentInfo.isUploadInProgress() || parentInfo.getDuplicatesUploadId() != null)) + || !storageService.dataExists(expectedParentId); + if (invalidParentState) { log.warn( "Parent {} for checksum {} is invalid (exists={}, tombstoned={}, bytesExist={}). Purging index.", expectedParentId, @@ -463,7 +468,7 @@ public void pruneOrphanedChecksumIndices() { /** Handles removal of a parent upload from the index. */ private boolean removeParentFromIndex(UploadId parentId, ChecksumKey key) throws IOException { - // B4 Check: Verify whether parentId is the registered parent in the checksum index + // Verify whether parentId is the registered parent in the checksum index UploadId indexedParent = checksumIndex.findParent(key); if (!Objects.equals(parentId, indexedParent)) { // Unindexed standalone upload (e.g. deduplication skipped due to lock contention) @@ -494,7 +499,7 @@ private boolean removeParentFromIndex(UploadId parentId, ChecksumKey key) throws /** Handles removal of a child duplicate upload from the index. */ private boolean removeChildFromIndex(UploadId childId, UploadId parentId, ChecksumKey key) throws IOException { - // Step 1: B4 Check - if parentId is the registered parent in index, remove child marker + // Step 1: if parentId is the registered parent in index, remove child marker UploadId indexedParent = checksumIndex.findParent(key); if (Objects.equals(parentId, indexedParent)) { checksumIndex.removeChild(key, childId); diff --git a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java index 18e5c609..5f3b43a1 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java @@ -63,6 +63,7 @@ public class DiskStorageService extends AbstractDiskBasedService implements Uplo private final DiskChecksumIndex checksumIndex; private DeduplicationService deduplicationService; private UploadLockingService uploadLockingService; + private boolean ownsUploadLockingService = true; public DiskStorageService(String storagePath) { super(storagePath + File.separator + UPLOAD_SUB_DIRECTORY); @@ -164,10 +165,36 @@ public boolean isUploadDeduplicationEnabled() { @Override public void setUploadLockingService(UploadLockingService uploadLockingService) { - this.uploadLockingService = + UploadLockingService target = Objects.requireNonNull(uploadLockingService, "uploadLockingService must not be null"); + // Only close the default internal DiskLockingService when replaced; externally managed + // locking services remain owned by the caller (e.g., TusFileUploadService). + if (this.ownsUploadLockingService + && this.uploadLockingService != null + && this.uploadLockingService != target) { + try { + this.uploadLockingService.close(); + } catch (IOException e) { + log.warn("Error closing replaced upload locking service: {}", e.getMessage()); + } + } + this.uploadLockingService = target; + this.ownsUploadLockingService = false; if (this.deduplicationService != null) { - this.deduplicationService.setUploadLockingService(uploadLockingService); + this.deduplicationService.setUploadLockingService(target); + } + } + + @Override + protected void cleanupOnClose() throws IOException { + try { + if (this.ownsUploadLockingService && this.uploadLockingService != null) { + this.uploadLockingService.close(); + } + } finally { + if (this.uploadConcatenationService != null) { + this.uploadConcatenationService.close(); + } } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java b/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java index 2ce22f64..57228976 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java @@ -2,10 +2,12 @@ import io.minio.MinioClient; import io.minio.PutObjectArgs; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.SequenceInputStream; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -169,7 +171,7 @@ public S3ConcatenationService( this.temporaryDirectory = temporaryDirectory != null ? temporaryDirectory - : java.nio.file.Paths.get(System.getProperty("java.io.tmpdir")); + : Paths.get(System.getProperty("java.io.tmpdir")); this.minPartSize = minPartSize; this.s3ComposeHelper = s3ComposeHelper != null ? s3ComposeHelper : new S3ServerSideComposeHelper(this.minioClient); @@ -238,7 +240,7 @@ public InputStream getConcatenatedBytes(UploadInfo uploadInfo) return uploadStorageService.getUploadedBytes(uploadInfo.getId()); } - return new java.io.ByteArrayInputStream(new byte[0]); + return new ByteArrayInputStream(new byte[0]); } private void mergeUsingServerSideCopy( diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index 86fe975b..8341b9b3 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -27,6 +27,7 @@ import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.util.LeaseDataJsonSerializer; import me.desair.tus.server.util.Utils; +import org.apache.commons.io.input.BoundedInputStream; import org.apache.commons.lang3.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -701,7 +702,12 @@ private void handleConditionalWriteNotImplemented(String mutexKey) { } private boolean isMutexStale(String mutexKey) { - try (InputStream stream = S3Utils.getObjectInputStream(minioClient, bucket, mutexKey)) { + // Bound mutex reads to 64 KiB so a corrupted or oversized object never exhausts heap memory. + try (InputStream stream = + BoundedInputStream.builder() + .setInputStream(S3Utils.getObjectInputStream(minioClient, bucket, mutexKey)) + .setMaxCount(64 * 1024L) + .get()) { byte[] bytes = stream.readAllBytes(); long createdTime = Long.parseLong(new String(bytes, StandardCharsets.UTF_8).trim()); return (System.currentTimeMillis() - createdTime) >= MUTEX_GRACE_PERIOD_MS; diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index c2eeba35..ef5603d0 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -456,22 +456,56 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) // Step 3: Process payload stream in optimal chunk parts and upload to S3. AppendResult appendResult = processPayloadChunks(info, preparedStream, info.getId()); - // Step 4: Persist any newly confirmed chunk parts and update the byte offset. - // Drained chunks and trailing parts are committed to the manifest before rethrowing any - // stream interruption exception, ensuring maximum byte persistence. + // Step 4: Validate minimum append size BEFORE persisting metadata or deleting stale parts on + // non-error requests, so rejected undersized appends never advance the durable offset. boolean hasNewParts = appendResult.allPartKeys.size() > preparedStream.remainingPartKeys.size() || appendResult.confirmedBytesAppended > 0; + long currentOffsetBeforeStream = + Math.max( + info.getOffset() != null ? info.getOffset() : 0L, + preparedStream.existingPartsTotalSize + preparedStream.prependedBytes); + long baseOffset = currentOffsetBeforeStream - preparedStream.prependedBytes; + long candidateOffset = + hasNewParts + ? baseOffset + appendResult.confirmedBytesAppended + : (info.getOffset() != null ? info.getOffset() : 0L); + boolean isCompletingOrEmpty = + !info.isUploadInProgress() + || (info.getLength() != null && candidateOffset >= info.getLength()); + long requestPayloadAppended = + Math.max(0L, appendResult.confirmedBytesAppended - preparedStream.prependedBytes); + + // Subtract prependedBytes so minAppendSize accurately measures payload transferred in THIS + // request. Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no + // content, or to requests completing the upload by including the Upload-Complete: ?1 header + // field." + if (appendResult.readException == null + && appendResult.maxAppendSizeException == null + && minAppendSize != null + && !isCompletingOrEmpty + && requestPayloadAppended < minAppendSize) { + // Clean up only the newly uploaded parts from this rejected request so prior state stays + // intact + for (int i = preparedStream.remainingPartKeys.size(); + i < appendResult.allPartKeys.size(); + i++) { + deleteObjectQuietly(appendResult.allPartKeys.get(i)); + } + throw new MinAppendSizeNotMetException( + "Append payload size " + + requestPayloadAppended + + " is below minimum limit " + + minAppendSize); + } + + // Step 5: Persist any newly confirmed chunk parts and update the byte offset. + // Drained chunks and trailing parts are committed to the manifest before rethrowing any + // stream interruption exception, ensuring maximum byte persistence. if (hasNewParts) { - long currentOffsetBeforeStream = - Math.max( - info.getOffset() != null ? info.getOffset() : 0L, - preparedStream.existingPartsTotalSize + preparedStream.prependedBytes); - long baseOffset = currentOffsetBeforeStream - preparedStream.prependedBytes; - long newOffset = baseOffset + appendResult.confirmedBytesAppended; - info.setOffset(newOffset); - upload.setOffset(newOffset); + info.setOffset(candidateOffset); + upload.setOffset(candidateOffset); info.setUploadPartKeys(new ArrayList<>(appendResult.allPartKeys)); upload.setUploadPartKeys(new ArrayList<>(appendResult.allPartKeys)); @@ -497,26 +531,8 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) throw appendResult.maxAppendSizeException; } - // Step 5: Validate minimum append size constraints if configured. - // Subtract prependedBytes so minAppendSize accurately measures payload transferred in THIS - // request. Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no - // content, or to requests completing the upload by including the Upload-Complete: ?1 header - // field." - long currentOffset = info.getOffset() != null ? info.getOffset() : 0L; - boolean isCompletingOrEmpty = - !info.isUploadInProgress() - || (info.getLength() != null && currentOffset >= info.getLength()); - long requestPayloadAppended = - Math.max(0L, appendResult.confirmedBytesAppended - preparedStream.prependedBytes); - if (minAppendSize != null && !isCompletingOrEmpty && requestPayloadAppended < minAppendSize) { - throw new MinAppendSizeNotMetException( - "Append payload size " - + requestPayloadAppended - + " is below minimum limit " - + minAppendSize); - } - // Step 6: If all expected bytes are uploaded, compose all part chunks into final S3 object + long currentOffset = info.getOffset() != null ? info.getOffset() : 0L; if (isCompletingOrEmpty) { finalizeCompletedUploadIfFinished(info, objectKey, info.getId(), currentOffset); update(info); @@ -689,11 +705,11 @@ public void removeLastNumberOfBytes(UploadInfo uploadInfo, long byteCount) // never resolving duplicates to parent objects, to protect completed parent files. String objectKey = getOwnS3ObjectKey(uploadInfo); long newOffset = Math.max(0L, uploadInfo.getOffset() - byteCount); - uploadInfo.setOffset(newOffset); - // If final completed object exists in S3, truncate it + // If final completed object exists in S3, truncate it before updating upload offset if (objectExists(objectKey)) { truncateCompletedObject(uploadInfo, objectKey, newOffset); + uploadInfo.setOffset(newOffset); update(uploadInfo); return; } @@ -701,10 +717,12 @@ public void removeLastNumberOfBytes(UploadInfo uploadInfo, long byteCount) // If upload has manifest parts in uploadPartKeys, truncate backwards from parts if (uploadInfo.getUploadPartKeys() != null && !uploadInfo.getUploadPartKeys().isEmpty()) { truncateManifestParts(uploadInfo, byteCount); + uploadInfo.setOffset(newOffset); update(uploadInfo); return; } + uploadInfo.setOffset(newOffset); update(uploadInfo); } @@ -769,6 +787,9 @@ public void setS3ConditionalWritesSupported(boolean supported) { if (this.checksumIndex != null) { this.checksumIndex.setS3ConditionalWritesSupported(supported); } + if (this.uploadLockingService instanceof S3LockingService s3Locking) { + s3Locking.withS3ConditionalWritesSupported(supported); + } } /** @@ -925,7 +946,6 @@ private PreparedStream prepareStreamWithExistingIncompletePart( IOUtils.copyLarge(lastPartStream, fos); } long tempPrepLength = tempPrepFile.length(); - prependedBytes += tempPrepLength; prependedStream = new FileInputStream(tempPrepFile) { @Override @@ -938,6 +958,9 @@ public void close() throws IOException { } }; combinedStream = new SequenceInputStream(prependedStream, combinedStream); + // Only credit prependedBytes after stream wrapping succeeds so exceptions do not skew + // offset + prependedBytes += tempPrepLength; staleTailKeys.add(lastPartKey); existingPartsTotalSize -= tempPrepLength; existingPartKeys.remove(existingPartKeys.size() - 1); @@ -1369,33 +1392,40 @@ private InputStream fetchS3ByteStream(UploadId id, UploadInfo info) * * @param uploadInfo The upload metadata containing the list of part keys * @param bytesToRemove The total number of bytes to strip from the tail + * @throws IOException If inspecting or truncating a manifest part fails */ - private void truncateManifestParts(UploadInfo uploadInfo, long bytesToRemove) { + private void truncateManifestParts(UploadInfo uploadInfo, long bytesToRemove) throws IOException { List partKeys = new ArrayList<>(uploadInfo.getUploadPartKeys()); long bytesToRemoveRemaining = bytesToRemove; - while (!partKeys.isEmpty() && bytesToRemoveRemaining > 0) { - String lastKey = partKeys.get(partKeys.size() - 1); - try { - long partSize = getPartSize(lastKey); + try { + while (!partKeys.isEmpty() && bytesToRemoveRemaining > 0) { + String lastKey = partKeys.get(partKeys.size() - 1); + try { + long partSize = getPartSize(lastKey); - if (bytesToRemoveRemaining >= partSize) { - // The entire part is subsumed by the truncation; delete it and adjust remaining budget - deleteObjectQuietly(lastKey); - partKeys.remove(partKeys.size() - 1); - bytesToRemoveRemaining -= partSize; - } else { - // Only a fraction of this part is being removed; rewrite the remaining prefix - String newKey = truncateSinglePart(lastKey, partSize, bytesToRemoveRemaining); - partKeys.set(partKeys.size() - 1, newKey); - bytesToRemoveRemaining = 0; + if (bytesToRemoveRemaining >= partSize) { + // The entire part is subsumed by the truncation; delete it and adjust remaining budget + deleteObjectQuietly(lastKey); + partKeys.remove(partKeys.size() - 1); + bytesToRemoveRemaining -= partSize; + } else { + // Only a fraction of this part is being removed; rewrite the remaining prefix + String newKey = truncateSinglePart(lastKey, partSize, bytesToRemoveRemaining); + partKeys.set(partKeys.size() - 1, newKey); + bytesToRemoveRemaining = 0; + } + } catch (IOException e) { + log.debug("Error truncating manifest part {}: {}", lastKey, e.getMessage()); + throw e; + } catch (Exception e) { + log.debug("Error truncating manifest part {}", lastKey, e); + throw new IOException("Error truncating manifest part " + lastKey, e); } - } catch (Exception e) { - log.debug("Error truncating manifest part {}", lastKey, e); - break; } + } finally { + uploadInfo.setUploadPartKeys(partKeys); } - uploadInfo.setUploadPartKeys(partKeys); } /** diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java b/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java index 56afbbfb..f33ca672 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import org.apache.commons.lang3.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -275,27 +276,27 @@ public static S3ErrorType parseErrorResponse(ErrorResponseException exception) { ErrorResponse response = exception.errorResponse(); String code = response != null && response.code() != null ? response.code() : ""; - if ("NoSuchKey".equalsIgnoreCase(code) - || "NoSuchBucket".equalsIgnoreCase(code) - || "NoSuchUpload".equalsIgnoreCase(code)) { + if (Strings.CI.equals("NoSuchKey", code) + || Strings.CI.equals("NoSuchBucket", code) + || Strings.CI.equals("NoSuchUpload", code)) { return S3ErrorType.NO_SUCH_KEY; } - if ("PreconditionFailed".equalsIgnoreCase(code)) { + if (Strings.CI.equals("PreconditionFailed", code)) { return S3ErrorType.PRECONDITION_FAILED; } - if ("ObjectAlreadyExists".equalsIgnoreCase(code) - || "BucketAlreadyExists".equalsIgnoreCase(code)) { + if (Strings.CI.equals("ObjectAlreadyExists", code) + || Strings.CI.equals("BucketAlreadyExists", code)) { return S3ErrorType.CONFLICT; } - if ("AccessDenied".equalsIgnoreCase(code)) { + if (Strings.CI.equals("AccessDenied", code)) { return S3ErrorType.ACCESS_DENIED; } - if ("APINotImplemented".equalsIgnoreCase(code) - || "NotImplemented".equalsIgnoreCase(code) + if (Strings.CI.equals("APINotImplemented", code) + || Strings.CI.equals("NotImplemented", code) || (exception.response() != null && exception.response().code() == 501)) { return S3ErrorType.API_NOT_IMPLEMENTED; } diff --git a/src/main/java/me/desair/tus/server/util/StructuredHeaderUtil.java b/src/main/java/me/desair/tus/server/util/StructuredHeaderUtil.java index 7fa51762..e6f8d509 100644 --- a/src/main/java/me/desair/tus/server/util/StructuredHeaderUtil.java +++ b/src/main/java/me/desair/tus/server/util/StructuredHeaderUtil.java @@ -1,6 +1,8 @@ package me.desair.tus.server.util; +import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import org.apache.commons.lang3.Strings; @@ -89,7 +91,7 @@ public static Map parseDictionary(String headerValue) { } // Split dictionary members by comma respecting RFC 9651 quoted strings - java.util.List members = splitMembers(headerValue); + List members = splitMembers(headerValue); for (String member : members) { int eqIdx = member.indexOf('='); if (eqIdx > 0) { @@ -182,14 +184,14 @@ public static String sanitizeHeaderValue(String value) { * Parse a Structured Header List according to RFC 9651 Section 3.1. * * @param headerValue The list header value string - * @return Ordered {@link java.util.List} of member item strings + * @return Ordered {@link List} of member item strings */ - public static java.util.List parseList(String headerValue) { - java.util.List list = new java.util.ArrayList<>(); + public static List parseList(String headerValue) { + List list = new ArrayList<>(); if (headerValue == null || headerValue.isBlank()) { return list; } - java.util.List members = splitMembers(headerValue); + List members = splitMembers(headerValue); for (String member : members) { list.add(sanitizeHeaderValue(unquoteString(member))); } @@ -203,8 +205,8 @@ public static java.util.List parseList(String headerValue) { * @param headerValue The raw header string * @return List of trimmed member strings */ - private static java.util.List splitMembers(String headerValue) { - java.util.List members = new java.util.ArrayList<>(); + private static List splitMembers(String headerValue) { + List members = new ArrayList<>(); StringBuilder current = new StringBuilder(); boolean inQuotes = false; boolean escaped = false; diff --git a/src/main/java/me/desair/tus/server/util/TusServletRequest.java b/src/main/java/me/desair/tus/server/util/TusServletRequest.java index 53dc4db5..217c319c 100644 --- a/src/main/java/me/desair/tus/server/util/TusServletRequest.java +++ b/src/main/java/me/desair/tus/server/util/TusServletRequest.java @@ -35,6 +35,7 @@ public class TusServletRequest extends HttpServletRequestWrapper { private Map> trailerHeaders = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); private Set processedBySet = new TreeSet<>(); private Boolean hasContent = null; + private boolean streamInterrupted = false; /** * Constructs a request object wrapping the given request. @@ -214,6 +215,26 @@ public void addProcessor(TusExtension processor) { processedBySet.add(processor.getName()); } + /** + * Returns true if the request payload stream was prematurely interrupted (e.g. due to lock + * contention or mid-stream truncation) after partial bytes were saved. + * + * @return true if the upload stream was interrupted before completion + */ + public boolean isStreamInterrupted() { + return streamInterrupted; + } + + /** + * Marks whether the request payload stream was prematurely interrupted after partial bytes were + * saved. + * + * @param streamInterrupted true if the upload stream was interrupted + */ + public void setStreamInterrupted(boolean streamInterrupted) { + this.streamInterrupted = streamInterrupted; + } + private boolean hasChunkedTransferEncoding() { return Strings.CI.equals("chunked", getHeader(HttpHeader.TRANSFER_ENCODING)); } diff --git a/src/main/java/me/desair/tus/server/util/Utils.java b/src/main/java/me/desair/tus/server/util/Utils.java index b65a0864..2dac2dbf 100644 --- a/src/main/java/me/desair/tus/server/util/Utils.java +++ b/src/main/java/me/desair/tus/server/util/Utils.java @@ -232,7 +232,6 @@ public static String extractMediaType(String contentTypeHeader) { /** * Resolves the upload URI from the HTTP request and response context. * - * @param method The HttpMethod of the request * @param request The TusServletRequest * @param response The TusServletResponse * @return The upload URI string, or null if it cannot be determined @@ -243,8 +242,16 @@ public static String getUploadUri(TusServletRequest request, TusServletResponse ? HttpMethod.getMethodIfSupported(request, EnumSet.allOf(HttpMethod.class)) : null; + // Creation methods (POST, PUT, and RUFH creation via PATCH) allocate a new upload resource + // and publish its URI via the Location response header. if (HttpMethod.POST.equals(method) || HttpMethod.PUT.equals(method)) { - return response != null ? response.getHeader(HttpHeader.LOCATION) : null; + return response != null && StringUtils.isNotBlank(response.getHeader(HttpHeader.LOCATION)) + ? response.getHeader(HttpHeader.LOCATION) + : null; + } else if (HttpMethod.PATCH.equals(method) + && response != null + && StringUtils.isNotBlank(response.getHeader(HttpHeader.LOCATION))) { + return response.getHeader(HttpHeader.LOCATION); } else if (request != null) { return request.getRequestURI(); } @@ -284,9 +291,9 @@ public static ProtocolVersion detectProtocolVersion( return ProtocolVersion.RUFH; } String method = request.getMethod(); - if (HttpMethod.HEAD.name().equalsIgnoreCase(method) - || HttpMethod.GET.name().equalsIgnoreCase(method) - || HttpMethod.DELETE.name().equalsIgnoreCase(method)) { + if (Strings.CI.equals(HttpMethod.HEAD.name(), method) + || Strings.CI.equals(HttpMethod.GET.name(), method) + || Strings.CI.equals(HttpMethod.DELETE.name(), method)) { return ProtocolVersion.RUFH; } } @@ -584,7 +591,7 @@ public static InterruptibleInputStream toInterruptibleStream(InputStream inputSt * * @param inputStream The InputStream to interrupt or close */ - public static void interruptStream(java.io.InputStream inputStream) { + public static void interruptStream(InputStream inputStream) { if (inputStream == null) { return; } diff --git a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java index 53df7adc..ccc437c4 100644 --- a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java +++ b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java @@ -22,6 +22,8 @@ import me.desair.tus.server.upload.UploadLock; import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; +import me.desair.tus.server.upload.UploadType; +import me.desair.tus.server.upload.cache.ThreadLocalCachedStorageAndLockingService; import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; import org.junit.Test; @@ -1116,4 +1118,109 @@ public void testProcessTusExceptionOverloadDelegates() throws Exception { assertEquals(HttpServletResponse.SC_BAD_REQUEST, servletResponse.getStatus()); } + + @Test + public void testWithUploadStorageServicePreservesBackendMaxAppendSizeUnlessConfigured() + throws Exception { + UploadStorageService storage1 = mock(UploadStorageService.class); + UploadStorageService storage2 = mock(UploadStorageService.class); + UploadLockingService locking1 = mock(UploadLockingService.class); + UploadLockingService locking2 = mock(UploadLockingService.class); + + // Without withMaxAppendSize(), replacing storage must preserve the new backend default + TusFileUploadService service = + new TusFileUploadService() + .withThreadLocalCache(true) + .withUploadStorageService(storage1) + .withUploadLockingService(locking1); + verify(storage1, never()).setMaxAppendSize(any()); + + // Replacing wrapped services closes the old unwrapped delegates cleanly + service.withUploadStorageService(storage2).withUploadLockingService(locking2); + verify(storage1).close(); + verify(locking1).close(); + verify(storage2, never()).setMaxAppendSize(any()); + + // Once explicitly configured, subsequent storage replacements inherit the configured value + service.withMaxAppendSize(4096L); + UploadStorageService storage3 = mock(UploadStorageService.class); + service.withUploadStorageService(storage3); + verify(storage3).setMaxAppendSize(4096L); + verify(storage2).close(); + org.junit.Assert.assertTrue( + service.getUploadStorageService() instanceof ThreadLocalCachedStorageAndLockingService); + } + + @Test + public void testPartialUploadDoesNotTriggerCompletionListeners() throws Exception { + UploadLockingService mockLockingService = mock(UploadLockingService.class); + UploadLock mockLock = mock(UploadLock.class); + when(mockLockingService.lockUploadByUri(anyString())).thenReturn(mockLock); + + UploadStorageService mockStorage = mock(UploadStorageService.class); + UUID uuid = UUID.randomUUID(); + String uploadUri = "/files/upload/" + uuid; + + UploadInfo partialInProgress = new UploadInfo(); + partialInProgress.setId(new UploadId(uuid)); + partialInProgress.setUploadType(UploadType.PARTIAL); + partialInProgress.setOffset(0L); + partialInProgress.setLength(5L); + + UploadInfo partialCompleted = new UploadInfo(); + partialCompleted.setId(new UploadId(uuid)); + partialCompleted.setUploadType(UploadType.PARTIAL); + partialCompleted.setOffset(5L); + partialCompleted.setLength(5L); + + when(mockStorage.getUploadInfo(uploadUri, "owner")) + .thenReturn(partialInProgress) + .thenReturn(partialInProgress) + .thenReturn(partialInProgress) + .thenReturn(partialInProgress) + .thenReturn(partialCompleted); + when(mockStorage.append(any(), any())).thenReturn(partialCompleted); + + MockHttpServletRequest mockReq = new MockHttpServletRequest(); + mockReq.setMethod("PATCH"); + mockReq.setRequestURI(uploadUri); + mockReq.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + mockReq.addHeader(HttpHeader.CONTENT_TYPE, "application/offset+octet-stream"); + mockReq.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + mockReq.setContent(new byte[5]); + + MockHttpServletResponse mockResp = new MockHttpServletResponse(); + UploadCompletionListener listener = mock(UploadCompletionListener.class); + + TusFileUploadService service = + new TusFileUploadService() + .withUploadUri("/files/upload") + .withUploadLockingService(mockLockingService) + .withUploadStorageService(mockStorage) + .withUploadCompletionListener(listener); + + service.process(mockReq, mockResp, "owner"); + + verify(listener, never()).onUploadComplete(any(), any()); + } + + @Test + public void testCloseStillClosesStorageWhenLockingCloseThrows() throws Exception { + UploadLockingService mockLocking = mock(UploadLockingService.class); + UploadStorageService mockStorage = mock(UploadStorageService.class); + doThrow(new IOException("locking failure")).when(mockLocking).close(); + + TusFileUploadService service = + new TusFileUploadService() + .withUploadLockingService(mockLocking) + .withUploadStorageService(mockStorage); + + try { + service.close(); + fail("Expected IOException from locking close"); + } catch (IOException expected) { + assertEquals("locking failure", expected.getMessage()); + } + verify(mockStorage).close(); + } } diff --git a/src/test/java/me/desair/tus/server/checksum/ChecksumRequestHandlerTest.java b/src/test/java/me/desair/tus/server/checksum/ChecksumRequestHandlerTest.java index 462439e5..1aff5574 100644 --- a/src/test/java/me/desair/tus/server/checksum/ChecksumRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/checksum/ChecksumRequestHandlerTest.java @@ -93,4 +93,19 @@ public void testInvalidHeader() throws Exception { handler.process(HttpMethod.PATCH, servletRequest, null, uploadStorageService, null); } + + @Test + public void testSkipsChecksumVerificationWhenStreamInterrupted() throws Exception { + // Interrupted streams must not trigger trailing-checksum mismatch errors on saved partial bytes + when(servletRequest.isStreamInterrupted()).thenReturn(true); + when(servletRequest.getHeader(HttpHeader.UPLOAD_CHECKSUM)).thenReturn("sha1 1234567890"); + when(servletRequest.getCalculatedChecksum(ArgumentMatchers.any(ChecksumAlgorithm.class))) + .thenReturn("0000000000"); + when(servletRequest.hasCalculatedChecksum()).thenReturn(true); + + handler.process(HttpMethod.PATCH, servletRequest, null, uploadStorageService, null); + + verify(servletRequest, times(1)).isStreamInterrupted(); + verify(servletRequest, never()).getCalculatedChecksum(any(ChecksumAlgorithm.class)); + } } diff --git a/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java b/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java index 1526722b..0747016f 100644 --- a/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java @@ -246,9 +246,10 @@ public void testProcessInterruptedByLockingServiceContentionReturns204WithUpdate "Stream was interrupted by the upload locking service watchdog"); }); + TusServletRequest tusRequest = new TusServletRequest(servletRequest); handler.process( HttpMethod.PATCH, - new TusServletRequest(servletRequest), + tusRequest, new TusServletResponse(servletResponse), uploadStorageService, mockLocking, @@ -257,6 +258,7 @@ public void testProcessInterruptedByLockingServiceContentionReturns204WithUpdate assertThat(servletResponse.getStatus(), is(HttpServletResponse.SC_NO_CONTENT)); assertThat(servletResponse.getHeader(HttpHeader.UPLOAD_OFFSET), is("50")); + assertThat(tusRequest.isStreamInterrupted(), is(true)); verify(mockLocking).unregisterInputStream("/test/upload/123"); } diff --git a/src/test/java/me/desair/tus/server/creation/validation/PatchUploadLengthValidatorTest.java b/src/test/java/me/desair/tus/server/creation/validation/PatchUploadLengthValidatorTest.java index 4844ad97..1851b90f 100644 --- a/src/test/java/me/desair/tus/server/creation/validation/PatchUploadLengthValidatorTest.java +++ b/src/test/java/me/desair/tus/server/creation/validation/PatchUploadLengthValidatorTest.java @@ -9,6 +9,7 @@ import me.desair.tus.server.HttpMethod; import me.desair.tus.server.exception.InvalidUploadLengthException; import me.desair.tus.server.exception.MaxUploadLengthExceededException; +import me.desair.tus.server.exception.MinUploadLengthNotReachedException; import me.desair.tus.server.upload.UploadInfo; import me.desair.tus.server.upload.UploadStorageService; import org.junit.Before; @@ -94,4 +95,29 @@ public void validateModifiedLength() throws Exception { validator.validate(HttpMethod.PATCH, servletRequest, uploadStorageService, "owner"); } + + @Test(expected = InvalidUploadLengthException.class) + public void validateDeferredLengthSmallerThanOffsetThrows() throws Exception { + servletRequest.addHeader(HttpHeader.UPLOAD_LENGTH, 400L); + servletRequest.setRequestURI("/files/123"); + + UploadInfo info = new UploadInfo(); + info.setOffset(500L); + when(uploadStorageService.getUploadInfo("/files/123", "owner")).thenReturn(info); + + validator.validate(HttpMethod.PATCH, servletRequest, uploadStorageService, "owner"); + } + + @Test(expected = MinUploadLengthNotReachedException.class) + public void validateDeferredLengthBelowMinSizeThrows() throws Exception { + servletRequest.addHeader(HttpHeader.UPLOAD_LENGTH, 400L); + servletRequest.setRequestURI("/files/123"); + + UploadInfo info = new UploadInfo(); + info.setOffset(100L); + when(uploadStorageService.getUploadInfo("/files/123", "owner")).thenReturn(info); + when(uploadStorageService.getMinSize()).thenReturn(500L); + + validator.validate(HttpMethod.PATCH, servletRequest, uploadStorageService, "owner"); + } } diff --git a/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java b/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java index 57e99d9e..e6db72ce 100644 --- a/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java @@ -184,6 +184,7 @@ public void testProcessInterruptedByLockingServiceContention() throws Exception handler.process( HttpMethod.POST, request, response, storageService, lockingService, "owner", null); + verify(request).setStreamInterrupted(true); verify(response).setHeader(HttpHeader.UPLOAD_OFFSET, "50"); } diff --git a/src/test/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandlerTest.java b/src/test/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandlerTest.java index e2b95927..7d61d986 100644 --- a/src/test/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandlerTest.java @@ -2,6 +2,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -323,4 +324,31 @@ public void testProcessReprDigestAndWantReprDigestDownloadsFileOnlyOnceDueToCach mockResp.getHeader(HttpHeader.REPR_DIGEST), is("sha-256=:LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ=:")); } + + @Test + public void testSkipsDigestVerificationAndEmissionWhenStreamInterrupted() throws Exception { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.setMethod("PATCH"); + request.setRequestURI("/files/123"); + request.addHeader(HttpHeader.CONTENT_DIGEST, "sha-256=:invalid-digest=:"); + request.addHeader(HttpHeader.WANT_CONTENT_DIGEST, "sha-256"); + + TusServletRequest servletRequest = new TusServletRequest(request); + servletRequest.setStreamInterrupted(true); + MockHttpServletResponse mockResp = new MockHttpServletResponse(); + TusServletResponse servletResponse = new TusServletResponse(mockResp); + + // Must not throw UploadDigestMismatchException or emit Content-Digest on interrupted streams + handler.process( + HttpMethod.PATCH, + servletRequest, + servletResponse, + uploadStorageService, + uploadLockingService, + "owner", + null); + + assertNull(mockResp.getHeader(HttpHeader.CONTENT_DIGEST)); + verify(uploadStorageService, never()).update(any()); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/RufhProtocolAppendTest.java b/src/test/java/me/desair/tus/server/rufh/RufhProtocolAppendTest.java index 9b105976..f6e098cb 100644 --- a/src/test/java/me/desair/tus/server/rufh/RufhProtocolAppendTest.java +++ b/src/test/java/me/desair/tus/server/rufh/RufhProtocolAppendTest.java @@ -144,7 +144,15 @@ public void testAppendExceedingLengthInvalidatesResource() throws Exception { protocol.validate( HttpMethod.PATCH, request, storageService, lockingService, null, ProtocolVersion.RUFH); } catch (TusException expected) { - // Expected exception + protocol.handleError( + HttpMethod.PATCH, + new TusServletRequest(request, true), + new TusServletResponse(response), + storageService, + lockingService, + null, + ProtocolVersion.RUFH, + expected); } verify(storageService).terminateUpload(info); diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java index e46b2d00..1c5a838b 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java @@ -4,6 +4,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -14,6 +15,7 @@ import java.io.InputStream; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; +import me.desair.tus.server.exception.MinUploadLengthNotReachedException; import me.desair.tus.server.upload.NoOpUploadLockingService; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; @@ -527,4 +529,93 @@ public void testProcessUninterruptedIoExceptionWithDeferredLengthResetsLengthAnd verify(lockingService).unregisterInputStream("/files/append-uninterrupted-id"); } } + + @Test + public void testProcessCompletingBelowMinSizeThrows() throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/min-size-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + request.setContent("hello".getBytes()); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("min-size-id")); + info.setOffset(0L); + info.setLength(null); + + UploadInfo appended = new UploadInfo(); + appended.setId(new UploadId("min-size-id")); + appended.setOffset(5L); + appended.setLength(5L); + + when(storageService.getUploadInfo("/files/min-size-id", "owner")).thenReturn(info); + when(storageService.getMinSize()).thenReturn(50L); + when(storageService.append(eq(info), any(InputStream.class))).thenReturn(appended); + + assertThrows( + MinUploadLengthNotReachedException.class, + () -> + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null)); + } + + @Test + public void testProcessInterruptedPreservesExplicitUploadLengthAndMarksRequestInterrupted() + throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/explicit-len-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "10"); + request.addHeader(HttpHeader.UPLOAD_LENGTH, "100"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + request.setContent("partial".getBytes()); + + UploadInfo initial = new UploadInfo(); + initial.setId(new UploadId("explicit-len-id")); + initial.setOffset(10L); + initial.setLength(null); // Initially deferred, explicitly supplied on this PATCH + + UploadInfo refreshed = new UploadInfo(); + refreshed.setId(new UploadId("explicit-len-id")); + refreshed.setOffset(14L); + refreshed.setLength(100L); + + when(storageService.getUploadInfo("/files/explicit-len-id", "owner")) + .thenReturn(initial) + .thenReturn(refreshed); + when(storageService.append(eq(initial), any(InputStream.class))) + .thenAnswer( + inv -> { + Object s = inv.getArgument(1); + if (s instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) s).interrupt(); + } + throw new IOException("Interrupted by contention"); + }); + + TusServletRequest tusRequest = new TusServletRequest(request); + handler.process( + HttpMethod.PATCH, + tusRequest, + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertTrue(tusRequest.isStreamInterrupted()); + assertThat(response.getStatus(), is(204)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("14")); + // Explicitly provided Upload-Length must remain 100 and not be cleared back to null + ArgumentCaptor captor = ArgumentCaptor.forClass(UploadInfo.class); + verify(storageService, org.mockito.Mockito.times(1)).update(captor.capture()); + assertThat(captor.getValue().getLength(), is(100L)); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java index e902fb76..704120ab 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java @@ -859,4 +859,56 @@ public void testProcessCreationAcquiresLockWhenNullLockProvided() throws Excepti verify(mockLock).close(); verify(lockingService).unregisterInputStream("/files/locked-id"); } + + @Test + public void testProcessCreationSetsLocationEarlyAndMarksStreamInterruptedOnContention() + throws Exception { + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_LENGTH, "100"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + request.setContent("partial payload".getBytes()); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("early-loc-id")); + info.setLength(100L); + info.setOffset(0L); + + UploadInfo refreshed = new UploadInfo(); + refreshed.setId(new UploadId("early-loc-id")); + refreshed.setLength(100L); + refreshed.setOffset(5L); + + when(storageService.create(any(UploadInfo.class), nullable(String.class))).thenReturn(info); + when(storageService.getUploadInfo("/files/early-loc-id", "owner")).thenReturn(refreshed); + + final String[] locationSeenDuringAppend = new String[1]; + when(storageService.append(eq(info), any(InputStream.class))) + .thenAnswer( + inv -> { + // Verify Location header is already published before streaming begins + locationSeenDuringAppend[0] = response.getHeader(HttpHeader.LOCATION); + Object s = inv.getArgument(1); + if (s instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) s).interrupt(); + } + throw new IOException("Stream interrupted by contention"); + }); + + TusServletRequest tusRequest = new TusServletRequest(request); + handler.process( + HttpMethod.POST, + tusRequest, + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertThat(locationSeenDuringAppend[0], is("/files/early-loc-id")); + assertThat(response.getHeader(HttpHeader.LOCATION), is("/files/early-loc-id")); + assertTrue(tusRequest.isStreamInterrupted()); + assertThat(response.getStatus(), is(201)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("5")); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhErrorHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhErrorHandlerTest.java index 66362a5e..ee51b488 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhErrorHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhErrorHandlerTest.java @@ -3,7 +3,10 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import me.desair.tus.server.HttpHeader; @@ -11,6 +14,7 @@ import me.desair.tus.server.HttpProblemDetails; import me.desair.tus.server.exception.InconsistentUploadLengthException; import me.desair.tus.server.exception.UploadAlreadyCompletedException; +import me.desair.tus.server.exception.UploadLengthExceededException; import me.desair.tus.server.exception.UploadOffsetMismatchException; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; @@ -141,6 +145,7 @@ public void testProcessErrorHandler400CompletedUpload() throws Exception { assertThat(response.getStatus(), is(400)); assertThat(response.getContentAsString(), containsString("completed-upload")); + verify(storageService).terminateUpload(info); } @Test @@ -356,4 +361,41 @@ public void testProcessWithNonErrorStatus() throws Exception { assertThat(problem, org.hamcrest.CoreMatchers.nullValue()); assertThat(response.getContentAsString(), is("")); } + + @Test + public void testProcessLengthExceededTerminatesUploadAndIgnoresCleanupException() + throws Exception { + request.setRequestURI("/files/exceeded-id"); + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("exceeded-id")); + info.setOffset(50L); + info.setLength(50L); + when(storageService.getUploadInfo("/files/exceeded-id", "owner")).thenReturn(info); + doThrow(new RuntimeException("Terminate failed")).when(storageService).terminateUpload(info); + + HttpProblemDetails problem = + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + null, + "owner", + new UploadLengthExceededException("Exceeded")); + + assertNull(problem); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?0")); + verify(storageService).terminateUpload(info); + + // Null storageService guard + assertNull( + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + null, + null, + "owner", + new UploadLengthExceededException("Exceeded"))); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/validation/RufhAppendValidatorTest.java b/src/test/java/me/desair/tus/server/rufh/validation/RufhAppendValidatorTest.java index 0a1e6d44..8c62f7c6 100644 --- a/src/test/java/me/desair/tus/server/rufh/validation/RufhAppendValidatorTest.java +++ b/src/test/java/me/desair/tus/server/rufh/validation/RufhAppendValidatorTest.java @@ -1,13 +1,22 @@ package me.desair.tus.server.rufh.validation; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; +import me.desair.tus.server.exception.InconsistentUploadLengthException; +import me.desair.tus.server.exception.InvalidUploadLengthException; +import me.desair.tus.server.exception.MaxUploadLengthExceededException; import me.desair.tus.server.exception.MinAppendSizeNotMetException; +import me.desair.tus.server.exception.MinUploadLengthNotReachedException; import me.desair.tus.server.exception.TusException; +import me.desair.tus.server.exception.UploadLengthExceededException; import me.desair.tus.server.upload.UploadInfo; import me.desair.tus.server.upload.UploadStorageService; import org.junit.Before; @@ -239,8 +248,9 @@ public void testValidateMissingUploadCompleteHeader() throws Exception { * from exceeding the representation's length by rejecting the request once the offset exceeds the * length, marking the upload resource invalid and rejecting any further interaction with it." */ - @Test(expected = TusException.class) - public void testValidateExceedingUploadLengthInvalidatesResource() throws Exception { + @Test + public void testValidateExceedingUploadLengthDoesNotMutateStorageDuringValidation() + throws Exception { request.setRequestURI("/files/exists"); request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); request.addHeader(HttpHeader.UPLOAD_OFFSET, "1000"); @@ -251,44 +261,31 @@ public void testValidateExceedingUploadLengthInvalidatesResource() throws Except info.setOffset(1000L); when(storageService.getUploadInfo("/files/exists", "owner")).thenReturn(info); - try { - validator.validate(HttpMethod.PATCH, request, storageService, "owner"); - } finally { - org.mockito.Mockito.verify(storageService).terminateUpload(info); - } + assertThrows( + UploadLengthExceededException.class, + () -> validator.validate(HttpMethod.PATCH, request, storageService, "owner")); + // Validation must remain side-effect free because pre-flight interim checks run before + // process(). + verify(storageService, never()).terminateUpload(any()); } - @Test(expected = TusException.class) - public void testValidateCompletedUploadTerminateThrowsException() throws Exception { - request.setRequestURI("/files/exists"); + @Test(expected = UploadLengthExceededException.class) + public void testValidateOffsetAlreadyAtDeclaredLengthThrows() throws Exception { + request.setRequestURI("/files/at-length"); request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); request.addHeader(HttpHeader.UPLOAD_OFFSET, "5000"); + request.setContent(new byte[10]); - UploadInfo info = new UploadInfo(); - info.setLength(5000L); - info.setOffset(5000L); // Completed - when(storageService.getUploadInfo("/files/exists", "owner")).thenReturn(info); - org.mockito.Mockito.doThrow(new RuntimeException("Terminate failed")) - .when(storageService) - .terminateUpload(info); - - validator.validate(HttpMethod.PATCH, request, storageService, "owner"); - } - - @Test(expected = TusException.class) - public void testValidateExceedingLengthTerminateThrowsException() throws Exception { - request.setRequestURI("/files/exists"); - request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); - request.addHeader(HttpHeader.UPLOAD_OFFSET, "1000"); - request.setContent(new byte[5000]); - - UploadInfo info = new UploadInfo(); + UploadInfo info = + new UploadInfo() { + @Override + public boolean isUploadInProgress() { + return true; + } + }; info.setLength(5000L); - info.setOffset(1000L); - when(storageService.getUploadInfo("/files/exists", "owner")).thenReturn(info); - org.mockito.Mockito.doThrow(new RuntimeException("Terminate failed")) - .when(storageService) - .terminateUpload(info); + info.setOffset(5000L); + when(storageService.getUploadInfo("/files/at-length", "owner")).thenReturn(info); validator.validate(HttpMethod.PATCH, request, storageService, "owner"); } @@ -330,7 +327,7 @@ public long getContentLengthLong() { validator.validate(HttpMethod.PATCH, chunkedReq, storageService, "owner"); // KISS: verifying method executes cleanly without throwing MinAppendSizeNotMetException - org.junit.Assert.assertTrue(true); + assertTrue(true); } @Test(expected = MinAppendSizeNotMetException.class) @@ -407,4 +404,87 @@ public void testValidateMinAppendSizeZeroOrNegativeIgnored() throws Exception { // KISS: verifying method executes cleanly without throwing an exception assertTrue(true); } + + @Test(expected = InvalidUploadLengthException.class) + public void testDeferredUploadLengthSmallerThanOffsetThrows() throws Exception { + request.setRequestURI("/files/deferred"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "500"); + request.addHeader(HttpHeader.UPLOAD_LENGTH, "400"); + + UploadInfo info = new UploadInfo(); + info.setOffset(500L); + when(storageService.getUploadInfo("/files/deferred", "owner")).thenReturn(info); + + validator.validate(HttpMethod.PATCH, request, storageService, "owner"); + } + + @Test(expected = InconsistentUploadLengthException.class) + public void testDeferredUploadLengthInconsistentWithCompleteContentLengthThrows() + throws Exception { + MockHttpServletRequest completeReq = new MockHttpServletRequest(); + completeReq.setRequestURI("/files/deferred"); + completeReq.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + completeReq.addHeader(HttpHeader.UPLOAD_OFFSET, "10"); + completeReq.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + completeReq.addHeader(HttpHeader.UPLOAD_LENGTH, "20"); + completeReq.setContent(new byte[5]); + + UploadInfo info = new UploadInfo(); + info.setOffset(10L); + when(storageService.getUploadInfo("/files/deferred", "owner")).thenReturn(info); + + validator.validate(HttpMethod.PATCH, completeReq, storageService, "owner"); + } + + @Test + public void testDeferredUploadLengthExceededThrowsWithoutMutatingStorage() throws Exception { + request.setRequestURI("/files/deferred"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "10"); + request.addHeader(HttpHeader.UPLOAD_LENGTH, "15"); + request.setContent(new byte[10]); + + UploadInfo info = new UploadInfo(); + info.setOffset(10L); + when(storageService.getUploadInfo("/files/deferred", "owner")).thenReturn(info); + + assertThrows( + UploadLengthExceededException.class, + () -> validator.validate(HttpMethod.PATCH, request, storageService, "owner")); + verify(storageService, never()).terminateUpload(any()); + } + + @Test(expected = MaxUploadLengthExceededException.class) + public void testDeferredUploadExceedsMaxUploadSizeThrows() throws Exception { + request.setRequestURI("/files/deferred"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "10"); + request.addHeader(HttpHeader.UPLOAD_LENGTH, "200"); + request.setContent(new byte[10]); + + UploadInfo info = new UploadInfo(); + info.setOffset(10L); + when(storageService.getUploadInfo("/files/deferred", "owner")).thenReturn(info); + when(storageService.getMaxUploadSize()).thenReturn(100L); + + validator.validate(HttpMethod.PATCH, request, storageService, "owner"); + } + + @Test(expected = MinUploadLengthNotReachedException.class) + public void testDeferredUploadCompleteBelowMinSizeThrows() throws Exception { + MockHttpServletRequest completeReq = new MockHttpServletRequest(); + completeReq.setRequestURI("/files/deferred"); + completeReq.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + completeReq.addHeader(HttpHeader.UPLOAD_OFFSET, "10"); + completeReq.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + completeReq.setContent(new byte[10]); + + UploadInfo info = new UploadInfo(); + info.setOffset(10L); + when(storageService.getUploadInfo("/files/deferred", "owner")).thenReturn(info); + when(storageService.getMinSize()).thenReturn(50L); + + validator.validate(HttpMethod.PATCH, completeReq, storageService, "owner"); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/validation/RufhCreationValidatorTest.java b/src/test/java/me/desair/tus/server/rufh/validation/RufhCreationValidatorTest.java index a9104580..6d00b379 100644 --- a/src/test/java/me/desair/tus/server/rufh/validation/RufhCreationValidatorTest.java +++ b/src/test/java/me/desair/tus/server/rufh/validation/RufhCreationValidatorTest.java @@ -8,6 +8,8 @@ import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; +import me.desair.tus.server.exception.MaxUploadLengthExceededException; +import me.desair.tus.server.exception.MinUploadLengthNotReachedException; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.upload.UploadStorageService; import org.junit.Before; @@ -157,4 +159,26 @@ public void testValidatePatchOnExistingResourceSkipped() throws Exception { validator.validate(HttpMethod.PATCH, request, storageService, "owner"); } + + @Test(expected = MaxUploadLengthExceededException.class) + public void testValidateDerivedCompleteLengthExceedsMaxUploadSize() throws Exception { + // Single-request creation with Upload-Complete: ?1 and no Upload-Length must enforce + // maxUploadSize against Content-Length. + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + request.setContent(new byte[200]); + when(storageService.getMaxUploadSize()).thenReturn(100L); + + validator.validate(HttpMethod.POST, request, storageService, null); + } + + @Test(expected = MinUploadLengthNotReachedException.class) + public void testValidateDerivedCompleteLengthBelowMinSize() throws Exception { + // Single-request creation with Upload-Complete: ?1 and no Upload-Length must enforce + // minSize against Content-Length. + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + request.setContent(new byte[20]); + when(storageService.getMinSize()).thenReturn(100L); + + validator.validate(HttpMethod.POST, request, storageService, null); + } } diff --git a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockTest.java b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockTest.java index ee35579b..0ecc9628 100644 --- a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockTest.java +++ b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockTest.java @@ -8,6 +8,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -26,6 +27,7 @@ public class AbstractLeaseLockTest { private static class TestLeaseLock extends AbstractLeaseLock { private final AtomicInteger renewCount = new AtomicInteger(0); + private final AtomicInteger releaseCount = new AtomicInteger(0); private final AtomicBoolean released = new AtomicBoolean(false); public TestLeaseLock( @@ -48,6 +50,7 @@ protected void doRenewLease() { @Override protected void releaseLockResource() { + releaseCount.incrementAndGet(); released.set(true); } } @@ -233,4 +236,27 @@ protected void doRenewLease() { assertTrue(stream.isInterrupted()); verify(mockFuture).cancel(true); } + + @Test + public void testRenewLeaseNoOpAfterClose() { + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + ScheduledFuture mockFuture = mock(ScheduledFuture.class); + when(mockExecutor.scheduleWithFixedDelay(any(), anyLong(), anyLong(), any())) + .thenReturn(mockFuture); + UploadId uploadId = new UploadId("upload-closed"); + LeaseData leaseData = + new LeaseData( + "holder-closed", "/files/upload-closed", 10000L, System.currentTimeMillis() + 10000L); + + TestLeaseLock lock = new TestLeaseLock(leaseData, uploadId, null, mockExecutor); + lock.close(); + // Second close and subsequent heartbeat renewal must both be harmless no-ops + lock.close(); + lock.renewLease(); + + assertTrue(lock.released.get()); + assertThat(lock.releaseCount.get(), is(1)); + assertThat(lock.renewCount.get(), is(0)); + verify(mockFuture, times(2)).cancel(true); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java index 21d3f266..078b5227 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java @@ -7,8 +7,11 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.azure.storage.blob.BlobContainerClient; @@ -32,6 +35,7 @@ import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; import me.desair.tus.server.upload.UploadLockingService; +import me.desair.tus.server.upload.concatenation.UploadConcatenationService; import org.junit.Before; import org.junit.Test; @@ -454,4 +458,22 @@ public void testUploadExecutorRejectedExecutionHandler() throws Exception { assertTrue(Thread.interrupted()); // clears interrupted flag } } + + @Test + public void testGetAzureBlobNameSingleArgAndCloseClosesConcatenationService() throws Exception { + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("az-single-arg")); + info.setOffset(5L); + info.setLength(5L); + + AzureBlobStorageService spyService = spy(storageService); + UploadConcatenationService mockConcat = mock(UploadConcatenationService.class); + spyService.setUploadConcatenationService(mockConcat); + doReturn(info).when(spyService).getUploadInfo("/files/az-single-arg", null); + + assertEquals("uploads/az-single-arg", spyService.getAzureBlobName("/files/az-single-arg")); + + spyService.close(); + verify(mockConcat).close(); + } } diff --git a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java index c147c4f5..a455c2e7 100644 --- a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java @@ -2,7 +2,10 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -133,21 +136,35 @@ public void testLockUploadAndClearCache() throws Exception { when(mockLocking.lockUploadByUri("/files/1")).thenReturn(mockLock); when(mockIdFactory.readUploadId("/files/1")).thenReturn(id); - service.getUploadInfo(id); + // Pre-lock read populates the ThreadLocal cache + assertSame(info, service.getUploadInfo(id)); UploadLock lock = service.lockUploadByUri("/files/1"); assertNotNull(lock); lock.getUploadUri(); verify(mockLock).getUploadUri(); + // First read inside the lock must re-fetch from storage because pre-lock cache was evicted; + // second read inside the same lock hits the cache without calling mockStorage again. + assertSame(info, service.getUploadInfo(id)); + assertSame(info, service.getUploadInfo(id)); + lock.release(); verify(mockLock, times(1)).release(); lock.close(); verify(mockLock, times(1)).close(); - service.getUploadInfo(id); - verify(mockStorage, times(2)).getUploadInfo(id); + assertSame(info, service.getUploadInfo(id)); + verify(mockStorage, times(3)).getUploadInfo(id); + + // Acquiring a lock by UploadId must also evict any prior ThreadLocal cached entry + when(mockLocking.lockUpload(id)).thenReturn(mockLock); + try (UploadLock idLock = service.lockUpload(id)) { + assertNotNull(idLock); + assertSame(info, service.getUploadInfo(id)); + } + verify(mockStorage, times(4)).getUploadInfo(id); } @Test @@ -283,4 +300,54 @@ public void testCleanupLockDelegates() throws IOException { service.cleanupLock(id); verify(mockLocking, times(1)).cleanupLock(id); } + + @Test + public void testDelegateGettersReturnRawDelegates() throws IOException { + assertSame(mockStorage, service.getStorageServiceDelegate()); + assertSame(mockLocking, service.getLockingServiceDelegate()); + + ThreadLocalCachedStorageAndLockingService noFactoryService = + new ThreadLocalCachedStorageAndLockingService(mockStorage, mockLocking); + UploadInfo info = new UploadInfo(); + when(mockStorage.getUploadInfo("/files/no-factory", "owner")).thenReturn(info); + assertSame(info, noFactoryService.getUploadInfo("/files/no-factory", "owner")); + } + + @Test + public void testCacheEvictedOnWriteExceptionsAndCloseGuaranteesBothDelegatesClosed() + throws Exception { + UploadId id = new UploadId(UUID.randomUUID().toString()); + UploadInfo info = new UploadInfo(); + info.setId(id); + info.setOffset(0L); + info.setLength(100L); + + when(mockStorage.getUploadInfo(id)).thenReturn(info); + // Warm the cache once; subsequent write errors must evict it so the delegate is queried again + service.getUploadInfo(id); + + doThrow(new IOException("update failed")).when(mockStorage).update(info); + assertThrows(IOException.class, () -> service.update(info)); + service.getUploadInfo(id); + + when(mockStorage.append(any(), any())).thenThrow(new IOException("append failed")); + assertThrows( + IOException.class, () -> service.append(info, new ByteArrayInputStream(new byte[1]))); + service.getUploadInfo(id); + + doThrow(new IOException("truncate failed")).when(mockStorage).removeLastNumberOfBytes(info, 1L); + assertThrows(IOException.class, () -> service.removeLastNumberOfBytes(info, 1L)); + service.getUploadInfo(id); + + when(mockStorage.deduplicate(any(), any())).thenThrow(new IOException("dedup failed")); + assertThrows(IOException.class, () -> service.deduplicate(info, null)); + service.getUploadInfo(id); + + // 1 initial load + 4 cache misses after each exception = 5 delegate reads + verify(mockStorage, times(5)).getUploadInfo(id); + + doThrow(new IOException("locking close failed")).when(mockLocking).close(); + assertThrows(IOException.class, () -> service.close()); + verify(mockStorage, times(1)).close(); + } } diff --git a/src/test/java/me/desair/tus/server/upload/checksum/DeduplicationServiceTest.java b/src/test/java/me/desair/tus/server/upload/checksum/DeduplicationServiceTest.java index 63a3f6ba..f1bdb050 100644 --- a/src/test/java/me/desair/tus/server/upload/checksum/DeduplicationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/checksum/DeduplicationServiceTest.java @@ -887,6 +887,66 @@ public void testPruneOrphanedIndicesRepairsMissingTombstoneWithActiveChildren() assertEquals(parentId, index.findParent(key)); } + @Test + public void testDeduplicateSkipsWhenAlreadyDeduplicatedChild() throws Exception { + UploadId existingParent = new UploadId("existing-parent"); + UploadInfo child = createUploadInfo(new UploadId("child-already"), checksumKey); + child.setDuplicatesUploadId(existingParent); + + UploadInfo result = deduplicationService.deduplicate(child, checksumKey); + + assertEquals(existingParent, result.getDuplicatesUploadId()); + assertNull(index.findParent(checksumKey)); + } + + @Test + public void testDeduplicateRejectsInProgressParentAndDuplicateParentAndReplacesIndex() + throws Exception { + UploadLock mockLock = mock(UploadLock.class); + when(lockingService.lockUpload(any())).thenReturn(mockLock); + + // 1. Parent still in progress (offset < length) must be invalidated and replaced + UploadId inProgressParentId = new UploadId("in-progress-parent"); + UploadInfo inProgressParent = createUploadInfo(inProgressParentId, checksumKey); + inProgressParent.setOffset(10L); + adapter.put(inProgressParentId, inProgressParent, true); + index.tryRegisterParent(checksumKey, inProgressParentId); + + UploadId contenderId = new UploadId("completed-contender"); + UploadInfo contender = createUploadInfo(contenderId, checksumKey); + adapter.put(contenderId, contender, true); + + UploadInfo result1 = deduplicationService.deduplicate(contender, checksumKey); + assertNull(result1.getDuplicatesUploadId()); + assertEquals(contenderId, index.findParent(checksumKey)); + + // 2. Parent that is itself a deduplicated child must also be rejected + index.deleteIndex(checksumKey); + UploadId chainedParentId = new UploadId("chained-parent"); + UploadInfo chainedParent = createUploadInfo(chainedParentId, checksumKey); + chainedParent.setDuplicatesUploadId(new UploadId("grandparent")); + adapter.put(chainedParentId, chainedParent, true); + index.tryRegisterParent(checksumKey, chainedParentId); + + UploadId newUploadId = new UploadId("new-standalone"); + UploadInfo newUpload = createUploadInfo(newUploadId, checksumKey); + adapter.put(newUploadId, newUpload, true); + + UploadInfo result2 = deduplicationService.deduplicate(newUpload, checksumKey); + assertNull(result2.getDuplicatesUploadId()); + assertEquals(newUploadId, index.findParent(checksumKey)); + + // 3. Tombstoned parent with retained physical bytes remains joinable by a new child + index.markParentDeleted(checksumKey); + adapter.deleteMetadata(newUploadId); + UploadId leafChildId = new UploadId("leaf-child"); + UploadInfo leafChild = createUploadInfo(leafChildId, checksumKey); + adapter.put(leafChildId, leafChild, true); + + UploadInfo result3 = deduplicationService.deduplicate(leafChild, checksumKey); + assertEquals(newUploadId, result3.getDuplicatesUploadId()); + } + private UploadInfo createUploadInfo(UploadId id, ChecksumKey key) { UploadInfo info = new UploadInfo(); info.setId(id); diff --git a/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java index 65bfbaed..6770d97b 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java @@ -17,6 +17,7 @@ import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; @@ -43,6 +44,7 @@ import me.desair.tus.server.upload.UploadIdFactory; import me.desair.tus.server.upload.UploadInfo; import me.desair.tus.server.upload.UploadLockingService; +import me.desair.tus.server.upload.concatenation.UploadConcatenationService; import me.desair.tus.server.util.Utils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; @@ -1522,4 +1524,29 @@ public void testUpdateWithMalformedChecksumThrowsIOException() throws Exception storageService.update(info); } + + @Test + public void testCloseClosesOwnedLockingAndConcatenationAndRespectsExternalLocking() + throws Exception { + Path tempDir = Files.createTempDirectory("disk-close-test"); + try { + DiskStorageService ownedService = new DiskStorageService(tempDir.toString()); + UploadConcatenationService mockConcat = mock(UploadConcatenationService.class); + ownedService.setUploadConcatenationService(mockConcat); + UploadLockingService internalLocking = ownedService.getUploadLockingService(); + + // Replacing the internally owned locking service must close it immediately + UploadLockingService externalLocking = mock(UploadLockingService.class); + ownedService.setUploadLockingService(externalLocking); + assertEquals(externalLocking, ownedService.getUploadLockingService()); + assertNotSame(internalLocking, ownedService.getUploadLockingService()); + + // Closing DiskStorageService closes concatenation but must not close external locking + ownedService.close(); + verify(mockConcat).close(); + verify(externalLocking, never()).close(); + } finally { + FileUtils.deleteQuietly(tempDir.toFile()); + } + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index 9334b187..134bb6de 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -4,6 +4,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; @@ -52,6 +53,7 @@ import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.checksum.ChecksumKey; import me.desair.tus.server.exception.MaxAppendSizeExceededException; +import me.desair.tus.server.exception.MinAppendSizeNotMetException; import me.desair.tus.server.exception.MinUploadLengthNotReachedException; import me.desair.tus.server.exception.UploadNotFoundException; import me.desair.tus.server.upload.UploadId; @@ -2590,8 +2592,10 @@ public void testRemoveLastNumberOfBytesManifestPartPutObjectException() throws E (PutObjectArgs args) -> args != null && "metadata/trunc-put-err.part.00001".equals(args.object()))); - storageService.removeLastNumberOfBytes(info, 3L); - assertEquals(7L, info.getOffset().longValue()); + IOException thrown = + assertThrows(IOException.class, () -> storageService.removeLastNumberOfBytes(info, 3L)); + assertTrue(thrown.getMessage().contains("metadata/trunc-put-err.part.00001")); + assertEquals(10L, info.getOffset().longValue()); } @Test @@ -3438,6 +3442,47 @@ public void testGetUploadLockingServiceReturnsConfiguredInstance() { assertNotNull(storageService.getUploadLockingService()); } + @Test + public void testAppendMinAppendSizeNotMetDeletesNewPartsWithoutMutatingMetadataOrOffset() + throws Exception { + storageService.setMinAppendSize(100L); + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("s3-min-append")); + info.setOffset(0L); + info.setLength(1000L); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/s3-min-append.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes())); + + assertThrows( + MinAppendSizeNotMetException.class, + () -> storageService.append(info, new ByteArrayInputStream(new byte[10]))); + + assertEquals(0L, info.getOffset().longValue()); + assertTrue(info.getUploadPartKeys() == null || info.getUploadPartKeys().isEmpty()); + // Newly uploaded temporary part object must be cleaned up and .info metadata must stay + // untouched + verify(minioClient) + .removeObject(argThat((RemoveObjectArgs a) -> a != null && a.object().contains(".part."))); + verify(minioClient, never()) + .putObject(argThat((PutObjectArgs a) -> a != null && a.object().endsWith(".info"))); + } + + @Test + public void testSetS3ConditionalWritesSupportedForwardsToS3LockingService() { + S3LockingService mockS3Locking = mock(S3LockingService.class); + storageService.setUploadLockingService(mockS3Locking); + + storageService.setS3ConditionalWritesSupported(false); + verify(mockS3Locking).withS3ConditionalWritesSupported(false); + + storageService.withS3ConditionalWritesSupported(true); + verify(mockS3Locking).withS3ConditionalWritesSupported(true); + } + private GetObjectResponse mockGetObjectResponse(byte[] bytes) { return new GetObjectResponse( null, "test-bucket", "eu-central-1", "object-key", new ByteArrayInputStream(bytes)); diff --git a/src/test/java/me/desair/tus/server/util/UtilsTest.java b/src/test/java/me/desair/tus/server/util/UtilsTest.java index c8e7edbe..2661a3b8 100644 --- a/src/test/java/me/desair/tus/server/util/UtilsTest.java +++ b/src/test/java/me/desair/tus/server/util/UtilsTest.java @@ -370,8 +370,12 @@ public void testGetUploadUri() { assertThat(Utils.getUploadUri(request, response), is("/files/location")); when(request.getMethod()).thenReturn("PUT"); assertThat(Utils.getUploadUri(request, response), is("/files/location")); + // RUFH creation via PATCH sets Location, whereas append PATCH leaves Location unset when(request.getMethod()).thenReturn("PATCH"); + assertThat(Utils.getUploadUri(request, response), is("/files/location")); + when(response.getHeader(HttpHeader.LOCATION)).thenReturn(null); assertThat(Utils.getUploadUri(request, response), is("/files/123")); + when(response.getHeader(HttpHeader.LOCATION)).thenReturn("/files/location"); when(request.getMethod()).thenReturn("GET"); assertThat(Utils.getUploadUri(request, response), is("/files/123")); when(request.getMethod()).thenReturn("HEAD"); From fe8e8218af30ee077b1ef6b343370395e135e8bc Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sat, 14 Nov 2026 13:09:12 +0100 Subject: [PATCH 4/5] Harden containerized CI integration tests, extract InMemoryAzureHttpResponse, and improve coverage --- .github/workflows/build.yml | 4 + pom.xml | 8 +- .../tus/server/TusFileUploadService.java | 18 +-- .../server/upload/UploadLockingService.java | 4 +- .../server/upload/UploadStorageService.java | 4 +- .../upload/disk/DiskStorageService.java | 8 +- .../server/upload/s3/S3StorageService.java | 3 - .../java/me/desair/tus/server/TestUtils.java | 84 +++++++++----- .../tus/server/TusFileUploadServiceTest.java | 28 +++++ .../upload/azure/AzureBlobUploadLockTest.java | 11 ++ .../server/upload/azure/AzureUtilsTest.java | 17 +++ .../ITAzureBlobConcatenationService.java | 108 +++--------------- .../azure/InMemoryAzureHttpResponse.java | 74 ++++++++++++ ...calCachedStorageAndLockingServiceTest.java | 38 ++++++ .../upload/s3/S3StorageServiceTest.java | 27 +++++ 15 files changed, 284 insertions(+), 152 deletions(-) create mode 100644 src/test/java/me/desair/tus/server/upload/azure/InMemoryAzureHttpResponse.java diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b95b4402..2374c27c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,6 +50,10 @@ jobs: - name: Build with Maven run: mvn install + - if: ${{ runner.os == 'Linux' }} + name: Enforce Code Coverage Threshold + run: python3 scripts/check-coverage.py --per-file-limit 90 + # Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive - if: ${{ matrix.upload-dependency-graph && github.event_name != 'pull_request' }} name: Update dependency graph diff --git a/pom.xml b/pom.xml index 6040b024..dbc5e622 100644 --- a/pom.xml +++ b/pom.xml @@ -171,13 +171,7 @@ org.testcontainers testcontainers - 1.20.4 - test - - - org.testcontainers - minio - 1.20.4 + 1.21.4 test diff --git a/src/main/java/me/desair/tus/server/TusFileUploadService.java b/src/main/java/me/desair/tus/server/TusFileUploadService.java index ea469b43..5379d3e3 100644 --- a/src/main/java/me/desair/tus/server/TusFileUploadService.java +++ b/src/main/java/me/desair/tus/server/TusFileUploadService.java @@ -1285,22 +1285,12 @@ private UploadLockingService unwrapLockingService(UploadLockingService service) return service; } - private void closeQuietly(UploadStorageService storageService) { - if (storageService != null) { + private void closeQuietly(Closeable service) { + if (service != null) { try { - storageService.close(); + service.close(); } catch (Exception e) { - log.warn("Error closing replaced storage service instance: {}", e.getMessage()); - } - } - } - - private void closeQuietly(UploadLockingService lockingService) { - if (lockingService != null) { - try { - lockingService.close(); - } catch (Exception e) { - log.warn("Error closing replaced locking service instance: {}", e.getMessage()); + log.warn("Error closing replaced service instance: {}", e.getMessage()); } } } diff --git a/src/main/java/me/desair/tus/server/upload/UploadLockingService.java b/src/main/java/me/desair/tus/server/upload/UploadLockingService.java index 3675753e..1445c5a1 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/UploadLockingService.java @@ -1,5 +1,6 @@ package me.desair.tus.server.upload; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import me.desair.tus.server.exception.TusException; @@ -8,7 +9,7 @@ * Service interface that can lock a specific upload so that it cannot be modified by other * requests/threads. */ -public interface UploadLockingService { +public interface UploadLockingService extends Closeable { /** * If the given URI represents a valid upload, lock that upload for processing. @@ -96,6 +97,7 @@ default void requestLockRelease(String requestUri) { * * @throws IOException If closing fails */ + @Override default void close() throws IOException { // No-op by default for backwards compatibility } diff --git a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java index 25e7fbee..3c8c6067 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java @@ -1,5 +1,6 @@ package me.desair.tus.server.upload; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -72,7 +73,7 @@ * child-owned storage. *

    */ -public interface UploadStorageService { +public interface UploadStorageService extends Closeable { /** * Method to retrieve the upload info by its upload URL. @@ -554,6 +555,7 @@ default Duration getDrainTimeout() { * * @throws IOException If closing fails */ + @Override default void close() throws IOException { // No-op by default for backward compatibility } diff --git a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java index 5f3b43a1..62319b6e 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java @@ -36,6 +36,7 @@ import me.desair.tus.server.upload.concatenation.VirtualConcatenationService; import me.desair.tus.server.util.UploadInfoJsonSerializer; import me.desair.tus.server.util.Utils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.Validate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -172,11 +173,7 @@ public void setUploadLockingService(UploadLockingService uploadLockingService) { if (this.ownsUploadLockingService && this.uploadLockingService != null && this.uploadLockingService != target) { - try { - this.uploadLockingService.close(); - } catch (IOException e) { - log.warn("Error closing replaced upload locking service: {}", e.getMessage()); - } + IOUtils.closeQuietly(this.uploadLockingService); } this.uploadLockingService = target; this.ownsUploadLockingService = false; @@ -188,6 +185,7 @@ public void setUploadLockingService(UploadLockingService uploadLockingService) { @Override protected void cleanupOnClose() throws IOException { try { + super.cleanupOnClose(); if (this.ownsUploadLockingService && this.uploadLockingService != null) { this.uploadLockingService.close(); } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index ef5603d0..f230115e 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -1418,9 +1418,6 @@ private void truncateManifestParts(UploadInfo uploadInfo, long bytesToRemove) th } catch (IOException e) { log.debug("Error truncating manifest part {}: {}", lastKey, e.getMessage()); throw e; - } catch (Exception e) { - log.debug("Error truncating manifest part {}", lastKey, e); - throw new IOException("Error truncating manifest part " + lastKey, e); } } } finally { diff --git a/src/test/java/me/desair/tus/server/TestUtils.java b/src/test/java/me/desair/tus/server/TestUtils.java index a8358eea..a7a952cf 100644 --- a/src/test/java/me/desair/tus/server/TestUtils.java +++ b/src/test/java/me/desair/tus/server/TestUtils.java @@ -3,6 +3,8 @@ import io.minio.BucketExistsArgs; import io.minio.MakeBucketArgs; import io.minio.MinioClient; +import java.io.File; +import java.util.Locale; import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.GenericContainer; @@ -22,49 +24,77 @@ private TestUtils() { * @return True if Docker or Podman is available for Testcontainers, false otherwise */ public static boolean isContainerRuntimeAvailable() { + // Configure Docker API version 1.44 when unspecified so Testcontainers works against + // Docker Engine 29+ on CI runners as well as Podman 4/5/6+. + if (System.getProperty("api.version") == null && System.getenv("DOCKER_API_VERSION") == null) { + System.setProperty("api.version", "1.44"); + } + + String userHome = System.getProperty("user.home", ""); + String[] possiblePodmanSockets = { + "/run/podman/podman.sock", + userHome + "/.local/share/containers/podman/machine/podman-machine-default/podman.sock", + userHome + "/.local/share/containers/podman/machine/qemu/podman.sock", + System.getProperty("java.io.tmpdir", "/tmp") + "/podman/podman-machine-default-api.sock" + }; + + // Detect Podman socket before calling DockerClientFactory so we don't cache a client failure + // when /var/run/docker.sock is absent. + if (System.getProperty("DOCKER_HOST") == null + && System.getenv("DOCKER_HOST") == null + && !new File("/var/run/docker.sock").exists()) { + for (String socketPath : possiblePodmanSockets) { + if (new File(socketPath).exists()) { + System.setProperty("DOCKER_HOST", "unix://" + socketPath); + System.setProperty("TESTCONTAINERS_RYUK_DISABLED", "true"); + break; + } + } + } + + Throwable lastFailure = null; try { if (DockerClientFactory.instance().isDockerAvailable()) { return true; } - } catch (Throwable ignored) { + } catch (Throwable t) { + lastFailure = t; } try { - String userHome = System.getProperty("user.home", ""); - String[] possibleSockets = { - "/var/run/docker.sock", - "/run/podman/podman.sock", - userHome + "/.local/share/containers/podman/machine/podman-machine-default/podman.sock", - userHome + "/.local/share/containers/podman/machine/qemu/podman.sock", - System.getProperty("java.io.tmpdir", "/tmp") + "/podman/podman-machine-default-api.sock" - }; - - for (String socketPath : possibleSockets) { - if (new java.io.File(socketPath).exists()) { - if (System.getProperty("DOCKER_HOST") == null) { - System.setProperty("DOCKER_HOST", "unix://" + socketPath); - } - System.setProperty("TESTCONTAINERS_RYUK_DISABLED", "true"); - try { - if (DockerClientFactory.instance().isDockerAvailable()) { - return true; - } - } catch (Throwable ignored) { - } - } - } - Process process = new ProcessBuilder("podman", "info").start(); if (process.waitFor() == 0) { System.setProperty("TESTCONTAINERS_RYUK_DISABLED", "true"); - return DockerClientFactory.instance().isDockerAvailable(); + if (DockerClientFactory.instance().isDockerAvailable()) { + return true; + } + } + } catch (Throwable t) { + if (lastFailure == null) { + lastFailure = t; } - } catch (Throwable ignored) { + } + + // Fail loudly on Linux CI runners instead of silently skipping containerized integration tests + // if the runner's Docker configuration changes in the future. + if (isLinuxCiEnvironment()) { + throw new IllegalStateException( + "Container runtime (Docker/Podman) is required on Linux CI runners but was not" + + " available.", + lastFailure); } return false; } + private static boolean isLinuxCiEnvironment() { + boolean isCi = + Boolean.parseBoolean(System.getenv("CI")) + || Boolean.parseBoolean(System.getenv("GITHUB_ACTIONS")); + boolean isLinux = System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("linux"); + return isCi && isLinux; + } + /** * Create and configure a GenericContainer running RustFS for S3 integration testing. * diff --git a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java index ccc437c4..b410f2ba 100644 --- a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java +++ b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java @@ -1223,4 +1223,32 @@ public void testCloseStillClosesStorageWhenLockingCloseThrows() throws Exception } verify(mockStorage).close(); } + + @Test + public void testCloseQuietlySwallowsReplacementCloseExceptionsAndClosesCachedService() + throws Exception { + UploadStorageService storage1 = mock(UploadStorageService.class); + UploadStorageService storage2 = mock(UploadStorageService.class); + UploadLockingService locking1 = mock(UploadLockingService.class); + UploadLockingService locking2 = mock(UploadLockingService.class); + + doThrow(new IOException("Simulated storage close failure")).when(storage1).close(); + doThrow(new IOException("Simulated locking close failure")).when(locking1).close(); + + TusFileUploadService service = + new TusFileUploadService() + .withUploadStorageService(storage1) + .withUploadLockingService(locking1); + + service.withUploadStorageService(storage2); + verify(storage1).close(); + + service.withUploadLockingService(locking2); + verify(locking1).close(); + + service.withThreadLocalCache(true); + service.close(); + verify(storage2).close(); + verify(locking2).close(); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java index ff132cb5..1b3397cb 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java @@ -557,4 +557,15 @@ public void testReleaseHandlesStopBlobExceptionQuietly() { lock.release(); assertNotNull(lock); } + + @Test + public void testThreeArgConstructorInitializesAndSkipsReleaseWhenMarkedDeleted() + throws Exception { + try (AzureBlobUploadLock lock = + new AzureBlobUploadLock(leaseClient, lockBlob, "/test/upload/3arg")) { + assertEquals("/test/upload/3arg", lock.getUploadUri()); + // Marking the blob deleted skips Azure SDK delete/release network calls during close(). + lock.markBlobDeleted(); + } + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java index 8a6032df..9207b157 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java @@ -2,6 +2,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; @@ -19,6 +21,7 @@ import com.azure.storage.blob.models.BlobErrorCode; import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.specialized.BlockBlobClient; +import java.time.OffsetDateTime; import org.junit.Test; import reactor.core.publisher.Mono; @@ -190,4 +193,18 @@ public void testParseErrorResponseErrorCodeThrowsExceptionHandled() { BlobStorageException ex = new BlobStorageException("Test exception", response, null); assertEquals(AzureErrorType.UNKNOWN, AzureUtils.parseErrorResponse(ex)); } + + @Test + public void testCalculateExpirationCutoff() { + assertNull(AzureUtils.calculateExpirationCutoff(null)); + assertNull(AzureUtils.calculateExpirationCutoff(0L)); + assertNull(AzureUtils.calculateExpirationCutoff(-100L)); + + OffsetDateTime shortCutoff = AzureUtils.calculateExpirationCutoff(1000L); + assertNotNull(shortCutoff); + + OffsetDateTime longCutoff = AzureUtils.calculateExpirationCutoff(10_000L); + assertNotNull(longCutoff); + assertTrue(longCutoff.isBefore(OffsetDateTime.now())); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobConcatenationService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobConcatenationService.java index 3762c8a4..ab72150e 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobConcatenationService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobConcatenationService.java @@ -5,6 +5,8 @@ import static org.junit.Assert.assertTrue; import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.specialized.BlockBlobClient; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -12,8 +14,10 @@ import java.util.List; import me.desair.tus.server.TestUtils; import me.desair.tus.server.exception.UploadNotFoundException; +import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; import me.desair.tus.server.upload.UploadType; +import org.apache.commons.io.IOUtils; import org.junit.After; import org.junit.AfterClass; import org.junit.Assume; @@ -65,7 +69,7 @@ public void tearDown() throws Exception { @Test(expected = UploadNotFoundException.class) public void mergeShouldThrowWhenPartialUploadNotFound() throws Exception { UploadInfo finalInfo = new UploadInfo(); - finalInfo.setId(new me.desair.tus.server.upload.UploadId("final-id")); + finalInfo.setId(new UploadId("final-id")); finalInfo.setConcatenationPartIds(Arrays.asList("/test/upload/non-existing")); finalInfo.setUploadType(UploadType.CONCATENATED); @@ -116,9 +120,7 @@ public void getConcatenationBytesShouldReturnCombinedStream() throws Exception { InputStream is = concatenationService.getConcatenatedBytes(createdFinal); assertNotNull(is); - assertEquals( - "hello-world", - org.apache.commons.io.IOUtils.toString(is, java.nio.charset.StandardCharsets.UTF_8)); + assertEquals("hello-world", IOUtils.toString(is, StandardCharsets.UTF_8)); } @Test(expected = UploadNotFoundException.class) @@ -170,7 +172,7 @@ public void getConcatenatedBytesShouldReturnBytesForCompletedUpload() throws Exc InputStream is = concatenationService.getConcatenatedBytes(created); assertNotNull(is); - assertEquals("hello", org.apache.commons.io.IOUtils.toString(is, StandardCharsets.UTF_8)); + assertEquals("hello", IOUtils.toString(is, StandardCharsets.UTF_8)); } @Test @@ -188,50 +190,9 @@ public void mergeShouldFallbackToStreamingWhenStageBlockFromUrlFails() throws Ex AzureBlobConcatenationService fallbackService = new AzureBlobConcatenationService(containerClient, storageService) { @Override - void stageBlockFromUrl( - com.azure.storage.blob.specialized.BlockBlobClient client, - String blockId, - String sourceUrl) { - throw new com.azure.storage.blob.models.BlobStorageException( - "Simulated 403", - new com.azure.core.http.HttpResponse(null) { - @Override - public int getStatusCode() { - return 403; - } - - @Override - public String getHeaderValue(String name) { - return null; - } - - @Override - public com.azure.core.http.HttpHeaders getHeaders() { - return new com.azure.core.http.HttpHeaders(); - } - - @Override - public reactor.core.publisher.Flux getBody() { - return reactor.core.publisher.Flux.empty(); - } - - @Override - public reactor.core.publisher.Mono getBodyAsByteArray() { - return reactor.core.publisher.Mono.empty(); - } - - @Override - public reactor.core.publisher.Mono getBodyAsString() { - return reactor.core.publisher.Mono.empty(); - } - - @Override - public reactor.core.publisher.Mono getBodyAsString( - java.nio.charset.Charset charset) { - return reactor.core.publisher.Mono.empty(); - } - }, - null); + void stageBlockFromUrl(BlockBlobClient client, String blockId, String sourceUrl) { + throw new BlobStorageException( + "Simulated 403", new InMemoryAzureHttpResponse(403), null); } }; @@ -241,7 +202,7 @@ public reactor.core.publisher.Mono getBodyAsString( assertEquals(Long.valueOf(10L), createdFinal.getLength()); } - @Test(expected = com.azure.storage.blob.models.BlobStorageException.class) + @Test(expected = BlobStorageException.class) public void mergeShouldRethrowUnexpectedBlobStorageException() throws Exception { UploadInfo part1Info = new UploadInfo(); part1Info.setLength(10L); @@ -256,50 +217,9 @@ public void mergeShouldRethrowUnexpectedBlobStorageException() throws Exception AzureBlobConcatenationService failingService = new AzureBlobConcatenationService(containerClient, storageService) { @Override - void stageBlockFromUrl( - com.azure.storage.blob.specialized.BlockBlobClient client, - String blockId, - String sourceUrl) { - throw new com.azure.storage.blob.models.BlobStorageException( - "Simulated 500 Server Error", - new com.azure.core.http.HttpResponse(null) { - @Override - public int getStatusCode() { - return 500; - } - - @Override - public String getHeaderValue(String name) { - return null; - } - - @Override - public com.azure.core.http.HttpHeaders getHeaders() { - return new com.azure.core.http.HttpHeaders(); - } - - @Override - public reactor.core.publisher.Flux getBody() { - return reactor.core.publisher.Flux.empty(); - } - - @Override - public reactor.core.publisher.Mono getBodyAsByteArray() { - return reactor.core.publisher.Mono.empty(); - } - - @Override - public reactor.core.publisher.Mono getBodyAsString() { - return reactor.core.publisher.Mono.empty(); - } - - @Override - public reactor.core.publisher.Mono getBodyAsString( - java.nio.charset.Charset charset) { - return reactor.core.publisher.Mono.empty(); - } - }, - null); + void stageBlockFromUrl(BlockBlobClient client, String blockId, String sourceUrl) { + throw new BlobStorageException( + "Simulated 500 Server Error", new InMemoryAzureHttpResponse(500), null); } }; diff --git a/src/test/java/me/desair/tus/server/upload/azure/InMemoryAzureHttpResponse.java b/src/test/java/me/desair/tus/server/upload/azure/InMemoryAzureHttpResponse.java new file mode 100644 index 00000000..6d1a0a6a --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/azure/InMemoryAzureHttpResponse.java @@ -0,0 +1,74 @@ +package me.desair.tus.server.upload.azure; + +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpResponse; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +/** + * Lightweight in-memory {@link HttpResponse} stub for simulating specific Azure HTTP error status + * codes and headers in tests without anonymous boilerplate. + */ +public class InMemoryAzureHttpResponse extends HttpResponse { + + private final int statusCode; + private final HttpHeaders headers; + + /** + * Create a response stub with the given HTTP status code and empty headers. + * + * @param statusCode HTTP status code to return + */ + public InMemoryAzureHttpResponse(int statusCode) { + this(statusCode, new HttpHeaders()); + } + + /** + * Create a response stub with the given HTTP status code and headers. + * + * @param statusCode HTTP status code to return + * @param headers HTTP headers to return (defaults to empty headers if {@code null}) + */ + public InMemoryAzureHttpResponse(int statusCode, HttpHeaders headers) { + super(null); + this.statusCode = statusCode; + this.headers = headers != null ? headers : new HttpHeaders(); + } + + @Override + public int getStatusCode() { + return statusCode; + } + + @Override + public String getHeaderValue(String name) { + return headers.getValue(name); + } + + @Override + public HttpHeaders getHeaders() { + return headers; + } + + @Override + public Flux getBody() { + return Flux.empty(); + } + + @Override + public Mono getBodyAsByteArray() { + return Mono.just(new byte[0]); + } + + @Override + public Mono getBodyAsString() { + return Mono.just(""); + } + + @Override + public Mono getBodyAsString(Charset charset) { + return Mono.just(""); + } +} diff --git a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java index a455c2e7..5dc4f9ee 100644 --- a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java @@ -2,11 +2,14 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -18,6 +21,7 @@ import java.time.Duration; import java.util.UUID; import me.desair.tus.server.checksum.ChecksumAlgorithm; +import me.desair.tus.server.checksum.ChecksumKey; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadIdFactory; import me.desair.tus.server.upload.UploadInfo; @@ -350,4 +354,38 @@ public void testCacheEvictedOnWriteExceptionsAndCloseGuaranteesBothDelegatesClos assertThrows(IOException.class, () -> service.close()); verify(mockStorage, times(1)).close(); } + + @Test + public void testJsonSerializationDeduplicateSetLockingAndNullDelegateLock() throws Exception { + service.setJsonSerializationEnabled(true); + verify(mockStorage, times(1)).setJsonSerializationEnabled(true); + + when(mockStorage.isJsonSerializationEnabled()).thenReturn(true); + assertTrue(service.isJsonSerializationEnabled()); + + service.setUploadLockingService(mockLocking); + verify(mockStorage, times(1)).setUploadLockingService(mockLocking); + + UploadId id = new UploadId(UUID.randomUUID().toString()); + UploadInfo input = new UploadInfo(); + input.setId(id); + UploadInfo deduped = new UploadInfo(); + deduped.setId(id); + deduped.setDuplicatesUploadId(new UploadId(UUID.randomUUID().toString())); + ChecksumKey key = ChecksumKey.of("abcdef", ChecksumAlgorithm.SHA256); + + when(mockStorage.deduplicate(input, key)).thenReturn(deduped).thenReturn(null); + assertSame(deduped, service.deduplicate(input, key)); + // Non-null deduplication result must populate the ThreadLocal cache + assertSame(deduped, service.getUploadInfo(id)); + verify(mockStorage, never()).getUploadInfo(id); + + assertNull(service.deduplicate(input, key)); + + ThreadLocalCachedStorageAndLockingService.CachedLock nullDelegateLock = + service.new CachedLock(null); + assertNull(nullDelegateLock.getUploadUri()); + nullDelegateLock.release(); + nullDelegateLock.close(); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index 134bb6de..cc7bf5b0 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -56,6 +56,7 @@ import me.desair.tus.server.exception.MinAppendSizeNotMetException; import me.desair.tus.server.exception.MinUploadLengthNotReachedException; import me.desair.tus.server.exception.UploadNotFoundException; +import me.desair.tus.server.upload.TimeBasedUploadIdFactory; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; import me.desair.tus.server.upload.UploadLockingService; @@ -3483,6 +3484,32 @@ public void testSetS3ConditionalWritesSupportedForwardsToS3LockingService() { verify(mockS3Locking).withS3ConditionalWritesSupported(true); } + @Test + public void testCreateWithExpirationPresetPartsAndGetUploadUri() { + TimeBasedUploadIdFactory customFactory = new TimeBasedUploadIdFactory(); + customFactory.setUploadUri("/custom/s3"); + storageService.setIdFactory(customFactory); + storageService.setUploadExpirationPeriod(3600000L); + + assertEquals("/custom/s3", storageService.getUploadUri()); + + UploadInfo info = new UploadInfo(); + info.setLength(10L); + info.setOffset(2L); + info.setUploadPartKeys(new ArrayList<>(Arrays.asList("part-existing"))); + + UploadInfo created = null; + try { + created = storageService.create(info, "owner-exp"); + } catch (Exception e) { + fail("Unexpected exception: " + e.getMessage()); + } + assertNotNull(created); + assertNotNull(created.getExpirationTimestamp()); + assertEquals(Long.valueOf(2L), created.getOffset()); + assertEquals(1, created.getUploadPartKeys().size()); + } + private GetObjectResponse mockGetObjectResponse(byte[] bytes) { return new GetObjectResponse( null, "test-bucket", "eu-central-1", "object-key", new ByteArrayInputStream(bytes)); From f9f02cf1e772f7b53de5484792a5baff32ba4b4a Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sun, 15 Nov 2026 18:24:38 +0100 Subject: [PATCH 5/5] fix(security): sanitize UploadInfo.getFileName() to prevent path traversal (#153) --- .jules/sentinel.md | 5 ++ CHANGELOG.md | 1 + .../download/DownloadGetRequestHandler.java | 28 ++--------- .../desair/tus/server/upload/UploadInfo.java | 17 +++++-- .../java/me/desair/tus/server/util/Utils.java | 47 +++++++++++++++++++ .../me/desair/tus/server/CoverageGapTest.java | 15 ++++++ .../tus/server/upload/UploadInfoTest.java | 17 +++++++ .../me/desair/tus/server/util/UtilsTest.java | 18 +++++++ 8 files changed, 119 insertions(+), 29 deletions(-) diff --git a/.jules/sentinel.md b/.jules/sentinel.md index c42c07ed..bb4f2aa4 100644 --- a/.jules/sentinel.md +++ b/.jules/sentinel.md @@ -37,3 +37,8 @@ **Vulnerability:** In `DownloadGetRequestHandler.java`, `info.getFileName()` derived from user-supplied `Upload-Metadata` was embedded directly into the `Content-Disposition` header without stripping path traversal sequences (`../`, `..\`) or directory paths (`/`, `\`). Downstream clients saving files using the suggested filename could write files outside target directories. Additionally, `X-Content-Type-Options: nosniff` was missing when serving completed downloads, leaving browsers susceptible to MIME sniffing attacks. **Learning:** File download endpoints that reflect user-supplied filenames in `Content-Disposition` headers can cause path traversal vulnerabilities on consuming clients. Serving user-uploaded content without explicit MIME-sniffing protection allows potential content sniffing / script execution. **Prevention:** Sanitize filenames before constructing `Content-Disposition` by stripping directory paths (Unix `/` and Windows `\`), removing directory traversal sequences (`..`), and falling back to a safe identifier if the sanitized name is empty. Always include `X-Content-Type-Options: nosniff` when serving user-uploaded files. + +## 2026-11-15 - Prevent Path Traversal in UploadInfo.getFileName() +**Vulnerability:** `UploadInfo.getFileName()` returned untrusted client-supplied metadata directly without sanitizing path traversal sequences (`../`, `..\`) or directory delimiters (`/`, `\`). Downstream consumers relying on `uploadInfo.getFileName()` to persist uploaded files were vulnerable to path traversal and arbitrary file overwrite outside the destination directory. Additionally, calling `getFileName()` on an `UploadInfo` with a null ID caused a `NullPointerException`. +**Learning:** Helper methods that extract filenames from untrusted client metadata must enforce path sanitization at the source, preventing downstream consumers from accidentally introducing path traversal vulnerabilities. +**Prevention:** Sanitize filenames at extraction in `UploadInfo.getFileName()` via `Utils.sanitizeFileName(...)`, stripping directory paths, traversal sequences, and null bytes while safely falling back to the upload ID or `"download"` if null. diff --git a/CHANGELOG.md b/CHANGELOG.md index 261bf510..e94f3666 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ See [docs/UPGRADING.md](docs/UPGRADING.md) for details and code examples. - **`UploadStorageService.cleanupExpiredUploads(UploadLockingService)`**: deprecated in favor of the zero-argument `cleanupExpiredUploads()`. ### Security +- **UploadInfo filename sanitization**: sanitize `UploadInfo.getFileName()` to prevent directory traversal and null-byte injection, stripping directory paths and traversal sequences with safe fallback to the upload ID. - **Download extension**: sanitize filenames in `Content-Disposition` to prevent client directory traversal and add `X-Content-Type-Options: nosniff`. - **CORS origin validation**: prevent arbitrary `Origin` header reflection in CORS responses ([#132](https://github.com/tomdesair/tus-java-server/pull/132)). - **Temporary file permissions**: use `Files.createTempFile` instead of `File.createTempFile` to enforce owner-only permissions (`0600`, CWE-378) ([#131](https://github.com/tomdesair/tus-java-server/pull/131)). diff --git a/src/main/java/me/desair/tus/server/download/DownloadGetRequestHandler.java b/src/main/java/me/desair/tus/server/download/DownloadGetRequestHandler.java index 309348e0..1a8b5c90 100644 --- a/src/main/java/me/desair/tus/server/download/DownloadGetRequestHandler.java +++ b/src/main/java/me/desair/tus/server/download/DownloadGetRequestHandler.java @@ -9,12 +9,12 @@ import me.desair.tus.server.HttpMethod; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.exception.UploadNotFoundException; -import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.util.AbstractRequestHandler; import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; +import me.desair.tus.server.util.Utils; /** Send the uploaded bytes of finished uploads. */ public class DownloadGetRequestHandler extends AbstractRequestHandler { @@ -51,7 +51,9 @@ public void process( servletResponse.setHeader(HttpHeader.CONTENT_LENGTH, Objects.toString(info.getLength())); - String sanitizedFileName = sanitizeFileName(info.getFileName(), info.getId()); + // info.getFileName() is pre-sanitized, but passing through Utils.sanitizeFileName ensures + // idempotent directory traversal protection before generating Content-Disposition headers + String sanitizedFileName = Utils.sanitizeFileName(info.getFileName(), info.getId()); servletResponse.setHeader( HttpHeader.CONTENT_DISPOSITION, String.format( @@ -67,26 +69,4 @@ public void process( uploadStorageService.copyUploadTo(info, servletResponse.getOutputStream()); } - - /** - * Sanitizes the given filename to prevent directory traversal and header corruption attacks on - * clients consuming the Content-Disposition header. - */ - private String sanitizeFileName(String fileName, UploadId uploadId) { - if (fileName == null || fileName.trim().isEmpty()) { - return uploadId != null ? uploadId.toString() : "download"; - } - // Prevent client-side path traversal: strip directory paths across Unix and Windows separators - String sanitized = fileName.trim(); - int lastSlash = Math.max(sanitized.lastIndexOf('/'), sanitized.lastIndexOf('\\')); - if (lastSlash >= 0) { - sanitized = sanitized.substring(lastSlash + 1).trim(); - } - // Remove directory traversal sequences (..) - sanitized = sanitized.replace("..", "").trim(); - if (sanitized.isEmpty()) { - return uploadId != null ? uploadId.toString() : "download"; - } - return sanitized; - } } diff --git a/src/main/java/me/desair/tus/server/upload/UploadInfo.java b/src/main/java/me/desair/tus/server/upload/UploadInfo.java index 56f6356f..4691907f 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadInfo.java +++ b/src/main/java/me/desair/tus/server/upload/UploadInfo.java @@ -339,20 +339,27 @@ public String getUploadConcatHeaderValue() { /** * Try to guess the filename of the uploaded data. If we cannot guess the name we fall back to the - * ID.
    - * NOTE: This is only a guess, there are no guarantees that the return value is correct + * ID (or {@code "download"} if the ID is not set). * - * @return A potential file name + *

    The returned filename is sanitized via {@link Utils#sanitizeFileName(String, UploadId)} to + * strip directory paths and path traversal sequences. + * + *

    NOTE: This is only a guess, there are no guarantees that the return value is correct. + * + * @return A sanitized potential file name (never null or blank) */ public String getFileName() { Map metadata = getMetadata(); for (String fileNameKey : fileNameKeys) { if (metadata.containsKey(fileNameKey)) { - return metadata.get(fileNameKey); + // Sanitize client-supplied metadata against directory traversal (../, ..\) and null-bytes + // to protect downstream callers who persist uploaded files using getFileName() + return Utils.sanitizeFileName(metadata.get(fileNameKey), getId()); } } - return getId().toString(); + // Safely fall back to the upload ID (or "download" if ID is null) without throwing NPE + return Utils.sanitizeFileName(null, getId()); } /** diff --git a/src/main/java/me/desair/tus/server/util/Utils.java b/src/main/java/me/desair/tus/server/util/Utils.java index 2dac2dbf..92f96763 100644 --- a/src/main/java/me/desair/tus/server/util/Utils.java +++ b/src/main/java/me/desair/tus/server/util/Utils.java @@ -31,6 +31,7 @@ import me.desair.tus.server.ProtocolVersion; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.upload.NoOpUploadLockingService; +import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; @@ -817,4 +818,50 @@ public static String sanitizePrefix(String prefix) { } return result + "/"; } + + /** + * Sanitizes a client-supplied filename to prevent directory traversal and null-byte injection + * when serving downloads or persisting uploaded files. + * + *

    Strips Unix ({@code /}) and Windows ({@code \}) directory prefixes, removes {@code ..} + * traversal sequences and null bytes ({@code \0}), and falls back to {@code "download"} when the + * resulting name is null or blank. + * + * @param fileName The raw filename string (may be null) + * @return A non-null, non-blank sanitized filename + */ + public static String sanitizeFileName(String fileName) { + return sanitizeFileName(fileName, null); + } + + /** + * Sanitizes a client-supplied filename to prevent directory traversal and null-byte injection + * when serving downloads or persisting uploaded files. + * + *

    Strips Unix ({@code /}) and Windows ({@code \}) directory prefixes, removes {@code ..} + * traversal sequences and null bytes ({@code \0}), and falls back to the provided {@link + * UploadId} (or {@code "download"} if {@code uploadId} is {@code null}) when the resulting name + * is null or blank. + * + * @param fileName The raw filename string (may be null) + * @param uploadId Optional fallback upload ID if the filename is null, blank, or unsafe + * @return A non-null, non-blank sanitized filename + */ + public static String sanitizeFileName(String fileName, UploadId uploadId) { + if (fileName == null || fileName.trim().isEmpty()) { + return uploadId != null ? uploadId.toString() : "download"; + } + // Strip Unix and Windows directory segments so only the terminal filename remains + String sanitized = fileName.trim(); + int lastSlash = Math.max(sanitized.lastIndexOf('/'), sanitized.lastIndexOf('\\')); + if (lastSlash >= 0) { + sanitized = sanitized.substring(lastSlash + 1).trim(); + } + // Remove path traversal sequences and null bytes that could bypass filesystem or header checks + sanitized = sanitized.replace("..", "").replace("\0", "").trim(); + if (sanitized.isEmpty() || ".".equals(sanitized)) { + return uploadId != null ? uploadId.toString() : "download"; + } + return sanitized; + } } diff --git a/src/test/java/me/desair/tus/server/CoverageGapTest.java b/src/test/java/me/desair/tus/server/CoverageGapTest.java index 8ddde36f..e0640b58 100644 --- a/src/test/java/me/desair/tus/server/CoverageGapTest.java +++ b/src/test/java/me/desair/tus/server/CoverageGapTest.java @@ -193,6 +193,21 @@ public void testAbstractTusExtensionHandleError5Args() throws Exception { DummyAbstractExtension ext = new DummyAbstractExtension(); ext.handleError(HttpMethod.POST, null, null, null, null); assertThat(ext.handleError8Called, is(true)); + + // Verify 5-arg validate delegates cleanly + ext.validate( + HttpMethod.POST, new org.springframework.mock.web.MockHttpServletRequest(), null, "owner"); + + // Verify 5-arg process delegates cleanly + org.springframework.mock.web.MockHttpServletResponse response = + new org.springframework.mock.web.MockHttpServletResponse(); + ext.process( + HttpMethod.POST, + new TusServletRequest(new org.springframework.mock.web.MockHttpServletRequest()), + new TusServletResponse(response), + null, + "owner"); + assertThat(ext.getName(), is("dummy")); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/UploadInfoTest.java b/src/test/java/me/desair/tus/server/upload/UploadInfoTest.java index 9d7371f5..af0af642 100644 --- a/src/test/java/me/desair/tus/server/upload/UploadInfoTest.java +++ b/src/test/java/me/desair/tus/server/upload/UploadInfoTest.java @@ -13,7 +13,9 @@ import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; +import java.nio.charset.StandardCharsets; import java.text.ParseException; +import java.util.Base64; import java.util.Stack; import java.util.UUID; import me.desair.tus.server.HttpHeader; @@ -194,11 +196,26 @@ public void testGetNameAndTypeWithMetadata() throws Exception { assertThat(info.getFileName(), is("test.jpg")); assertThat(info.getFileMimeType(), is("image/jpeg")); + + // Untrusted path traversal sequences in filename metadata must be stripped by getFileName() + info.setEncodedMetadata( + "filename " + + Base64.getEncoder() + .encodeToString("../../etc/passwd".getBytes(StandardCharsets.UTF_8))); + assertThat(info.getFileName(), is("passwd")); + + // Traversal-only or directory metadata must safely fall back to the upload ID + info.setId(new UploadId("test-id")); + info.setEncodedMetadata( + "filename " + Base64.getEncoder().encodeToString("../..".getBytes(StandardCharsets.UTF_8))); + assertThat(info.getFileName(), is("test-id")); } @Test public void testGetNameAndTypeWithoutMetadata() throws Exception { UploadInfo info = new UploadInfo(); + assertThat(info.getFileName(), is("download")); + final UploadId id = new UploadId(UUID.randomUUID()); info.setId(id); diff --git a/src/test/java/me/desair/tus/server/util/UtilsTest.java b/src/test/java/me/desair/tus/server/util/UtilsTest.java index 2661a3b8..caa9210b 100644 --- a/src/test/java/me/desair/tus/server/util/UtilsTest.java +++ b/src/test/java/me/desair/tus/server/util/UtilsTest.java @@ -35,6 +35,7 @@ import me.desair.tus.server.HttpHeader; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.upload.NoOpUploadLockingService; +import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; import org.apache.commons.io.FileUtils; @@ -988,6 +989,23 @@ public void testSanitizePrefix() { assertEquals("custom/", Utils.sanitizePrefix("/custom///")); } + @Test + public void testSanitizeFileName() { + UploadId fallbackId = new UploadId("id-1"); + assertEquals("download", Utils.sanitizeFileName(null)); + assertEquals("download", Utils.sanitizeFileName("")); + assertEquals("id-1", Utils.sanitizeFileName(" ", fallbackId)); + assertEquals("passwd", Utils.sanitizeFileName("../../etc/passwd", fallbackId)); + assertEquals("secret.pdf", Utils.sanitizeFileName("C:\\Users\\test\\..\\secret.pdf")); + assertEquals("id-1", Utils.sanitizeFileName("../..", fallbackId)); + assertEquals("download", Utils.sanitizeFileName("../..", null)); + assertEquals("id-1", Utils.sanitizeFileName(".", fallbackId)); + assertEquals("download", Utils.sanitizeFileName(".", null)); + assertEquals("id-1", Utils.sanitizeFileName(".../...", fallbackId)); + assertEquals("filename.txt", Utils.sanitizeFileName("file\0name.txt")); + assertEquals("normal-file.txt", Utils.sanitizeFileName("normal-file.txt", fallbackId)); + } + @Test public void testCreateUploadExecutor() throws Exception { ThreadPoolExecutor executor = Utils.createUploadExecutor(2, "test-upload");