diff --git a/.ci-info.json b/.ci-info.json deleted file mode 100644 index 65650a4d..00000000 --- a/.ci-info.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/fclairamb/ci-info/main/config-schema.json", - "version_input_git_tag": { - "pattern": "^v?([0-9.]+)$" - }, - "templates": [{ - "input_file": "build.go.tpl", - "output_file": "build.go" - }] -} \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..4158e7fd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.git +.github +.vscode +ftpserver +ftpserver.json +ftpserver.log +*.pem diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 78e78924..cdccc4c9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,43 +1,27 @@ name: Build -# Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch on: push: pull_request: -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "build" build: - # The type of runner that the job will run on runs-on: ubuntu-26.04 - - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE - uses: actions/checkout@v7.0.1 - # Running golangci-lint - - name: Linting - if: matrix.lint - uses: golangci/golangci-lint-action@v9 - with: - version: v2.15.0 - - # Install Go - name: Setup go uses: actions/setup-go@v7.0.0 with: go-version: 1.27.3 + - name: Linting + uses: golangci/golangci-lint-action@v9 + with: + version: v2.15.0 + - name: Build run: go build -v ./... - name: Test - # We need GCC because of the "go test -race" - # env: - # CGO_ENABLED: 0 - run: | - apt-get update && apt-get install gcc -y - go test -race -v ./... + run: go test -race -v ./... diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 02776d69..325209d3 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,22 +1,10 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# name: "CodeQL" on: push: - branches: [ master ] + branches: [ main ] pull_request: - # The branches below must be a subset of the branches above - branches: [ master ] + branches: [ main ] schedule: - cron: '44 20 * * 0' @@ -29,39 +17,18 @@ jobs: fail-fast: false matrix: language: [ 'go' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] - # Learn more: - # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed steps: - name: Checkout repository uses: actions/checkout@v7.0.1 - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v4 - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index 22b522f3..92687f40 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -1,43 +1,59 @@ -name: Test the current docker image +name: Test the docker image on: schedule: - # We want to run it every day - - cron: '0 0 * * *' + # We want to check the published image every day + - cron: '0 0 * * *' + pull_request: + paths: + - Dockerfile* + - .dockerignore + - .github/workflows/docker-test.yml + workflow_dispatch: jobs: test-docker-image: + strategy: + fail-fast: false + matrix: + tag: ["latest", "alpine"] runs-on: ubuntu-26.04 - timeout-minutes: 1 - services: - ftpserver: - image: fclairamb/ftpserver - ports: - - 2121-2130:2121-2130 - + timeout-minutes: 15 + env: + SERVER_PORT: 2121 steps: - - name: Run sample - env: - SERVER_PORT: 2121 + - uses: actions/checkout@v7.0.1 + if: github.event_name == 'pull_request' + + - name: Build the image + if: github.event_name == 'pull_request' + run: | + docker build -t "fclairamb/ftpserver:${{ matrix.tag }}" \ + -f "Dockerfile${{ matrix.tag == 'alpine' && '.alpine' || '' }}" . + + - name: Start the server + run: | + docker run -d --name ftpserver -p 2121-2130:2121-2130 "fclairamb/ftpserver:${{ matrix.tag }}" + + - name: Upload and download a file run: | set -xe - # Getting a file - curl https://placekitten.com/2048/2048 -o kitty.jpg + # Generating a file + head -c 2097152 /dev/urandom > local.bin # Waiting for server to be ready - while ! nc -z localhost ${SERVER_PORT} > "$GITHUB_OUTPUT" + - name: Checkout uses: actions/checkout@v7.0.1 with: - fetch-depth: 0 - - name: Setup go - uses: actions/setup-go@v7.0.0 - with: - go-version-file: go.mod - - name: Change version + ref: ${{ steps.tag.outputs.tag || github.sha }} + + - name: Resolve build info + id: build run: | - curl -L https://github.com/fclairamb/ci-info/releases/download/v0.4.4/ci-info_0.4.4_linux_amd64.tar.gz | tar -zx - ./ci-info + echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> "$GITHUB_OUTPUT" + - name: Docker meta id: docker_meta uses: docker/metadata-action@v6 @@ -44,50 +74,43 @@ jobs: suffix=${{ matrix.type != '' && format('-{0}', matrix.type) || '' }} latest=false tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=semver,suffix=,pattern=${{ matrix.type != '' && matrix.type || 'latest' }} - type=ref,event=branch + type=semver,pattern={{version}},value=${{ steps.tag.outputs.tag }} + type=semver,pattern={{major}}.{{minor}},value=${{ steps.tag.outputs.tag }} + type=semver,pattern={{major}},value=${{ steps.tag.outputs.tag }} + type=semver,suffix=,pattern=${{ matrix.type != '' && matrix.type || 'latest' }},value=${{ steps.tag.outputs.tag }} type=ref,event=pr - - name: Set up QEMU - uses: docker/setup-qemu-action@v4 + type=ref,event=branch + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - - name: Cache Docker layers - uses: actions/cache@v6 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ matrix.type }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- + - name: Login to DockerHub - if: github.repository == 'fclairamb/ftpserver' + if: env.PUSH == 'true' uses: docker/login-action@v4 with: username: ${{ github.repository_owner }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to GitHub Container Registry - if: github.repository == 'fclairamb/ftpserver' + if: env.PUSH == 'true' uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push (scratch) + + - name: Build and push uses: docker/build-push-action@v7 with: context: . file: ./Dockerfile${{ matrix.type != '' && format('.{0}', matrix.type) || '' }} platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7 - push: ${{ github.repository == 'fclairamb/ftpserver' }} + push: ${{ env.PUSH == 'true' }} tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - build-and-push-status: - needs: build-and-push - runs-on: ubuntu-26.04 - steps: - - run: echo 'All good' + build-args: | + VERSION=${{ steps.tag.outputs.tag || 'dev' }} + COMMIT=${{ steps.build.outputs.commit }} + BUILD_DATE=${{ steps.build.outputs.date }} + cache-from: type=gha,scope=docker${{ matrix.type }} + cache-to: type=gha,mode=max,scope=docker${{ matrix.type }} diff --git a/.github/workflows/jsonschema.yml b/.github/workflows/jsonschema.yml index c1dfddc3..669e05d0 100644 --- a/.github/workflows/jsonschema.yml +++ b/.github/workflows/jsonschema.yml @@ -7,14 +7,24 @@ on: jobs: json-schema-check: runs-on: ubuntu-26.04 - container: python:3.11-alpine3.16 + container: python:3.14-alpine steps: - uses: actions/checkout@v7.0.1 - name: Test run: | - pip install jsonschema==4.14.0 - for f in $(find config/ -name '*.json') - do - echo "Checking $f" - jsonschema -i $f config-schema.json - done + pip install jsonschema==4.26.0 + python - config-schema.json $(find config/ fs/ -name '*.json') <<'EOF_PY' + import json, sys, jsonschema + + schema = json.load(open(sys.argv[1])) + jsonschema.Draft7Validator.check_schema(schema) + failed = False + for f in sys.argv[2:]: + try: + jsonschema.validate(json.load(open(f)), schema) + print(f"OK {f}") + except Exception as e: + failed = True + print(f"FAIL {f}: {e}") + sys.exit(1 if failed else 0) + EOF_PY diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index b8796bed..c028ac2e 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -21,8 +21,8 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} # The release is created with the default GITHUB_TOKEN, which by design does not - # trigger any further workflow. The binaries therefore have to be built from here - # rather than from a `push: tags` trigger (see #1738). + # trigger any further workflow. The binaries and docker images therefore have to be + # built from here rather than from a `push: tags` trigger (see #1738). release-artifacts: needs: release-please if: needs.release-please.outputs.release_created == 'true' @@ -31,3 +31,14 @@ jobs: tag: ${{ needs.release-please.outputs.tag_name }} permissions: contents: write + + docker: + needs: release-please + if: needs.release-please.outputs.release_created == 'true' + uses: ./.github/workflows/docker.yml + with: + tag: ${{ needs.release-please.outputs.tag_name }} + secrets: inherit + permissions: + contents: read + packages: write diff --git a/.gitignore b/.gitignore index 9cdef23e..4ed65225 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,7 @@ ftpserver ftpserver.json ftpserver.log -ci-info .idea/ *.pem -gdrive_token_gdrive.json +gdrive_token_*.json __* diff --git a/.golangci.yml b/.golangci.yml index c0dbed6d..68eeeb57 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,222 +1,51 @@ -run: - # default concurrency is a available CPU number - concurrency: 4 - - # timeout for analysis, e.g. 30s, 5m, default is 1m - timeout: 1m - - # exit code when at least one issue was found, default is 1 - issues-exit-code: 1 - - # include test files or not, default is true - tests: true - -# all available settings of specific linters -linters-settings: - errcheck: - # report about not checking of errors in type assetions: `a := b.(MyStruct)`; - # default is false: such cases aren't reported by default. - check-type-assertions: false - - # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; - # default is false: such cases aren't reported by default. - check-blank: false - - funlen: - lines: 80 - statements: 40 - - govet: - # report about shadowed variables - check-shadowing: true - - # settings per analyzer - settings: - printf: # analyzer name, run `go tool vet help` to see all analyzers - funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf - - # enable or disable analyzers by name - enable: - - atomicalign - # - fieldalignment - enable-all: false - disable: - - shadow - disable-all: false - gofmt: - # simplify code: gofmt with `-s` option, true by default - simplify: true - goimports: - # put imports beginning with prefix after 3rd-party packages; - # it's a comma-separated list of prefixes - local-prefixes: github.com/fclairamb/ftpserver/ - gocyclo: - # minimal code complexity to report, 30 by default (but we recommend 10-20) - min-complexity: 15 - gocognit: - # minimal code complexity to report, 30 by default (but we recommend 10-20) - min-complexity: 30 - maligned: - # print struct with more effective memory layout or not, false by default - suggest-new: true - dupl: - # tokens count to trigger issue, 150 by default - threshold: 100 - goconst: - # minimal length of string constant, 3 by default - min-len: 3 - # minimal occurrences count to trigger, 3 by default - min-occurrences: 3 - depguard: - list-type: blacklist - include-go-root: false - packages: - - github.com/sirupsen/logrus - packages-with-error-messages: - # specify an error message to output when a blacklisted package is used - github.com/sirupsen/logrus: "logging is allowed only by logutils.Log" - misspell: - # Correct spellings using locale preferences for US or UK. - # Default is to use a neutral variety of English. - # Setting locale to US will correct the British spelling of 'colour' to 'color'. - locale: US - ignore-words: - - someword - lll: - # max line length, lines longer will be reported. Default is 120. - # '\t' is counted as 1 character by default, and can be changed with the tab-width option - line-length: 120 - # tab width in spaces. Default to 1. - tab-width: 1 - unused: - # treat code as a program (not a library) and report unused exported identifiers; default is false. - # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: - # if it's called for subdir of a project it can't find funcs usages. All text editor integrations - # with golangci-lint call it on a directory with the changed file. - check-exported: false - unparam: - # Inspect exported functions, default is false. Set to true if no external program/library imports your code. - # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: - # if it's called for subdir of a project it can't find external interfaces. All text editor integrations - # with golangci-lint call it on a directory with the changed file. - check-exported: false - nakedret: - # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 - max-func-lines: 30 - prealloc: - # XXX: we don't recommend using this linter before doing performance profiling. - # For most programs usage of prealloc will be a premature optimization. - - # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. - # True by default. - simple: true - range-loops: true # Report preallocation suggestions on range loops, true by default - for-loops: false # Report preallocation suggestions on for loops, false by default - gocritic: - # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint run` to see all tags and checks. - # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags". - enabled-tags: - - performance - - settings: # settings passed to gocritic - captLocal: # must be valid enabled check name - paramsOnly: true - rangeValCopy: - sizeThreshold: 32 - godox: - # report any comments starting with keywords, this is useful for TODO or FIXME comments that - # might be left in the code accidentally and should be resolved before merging - keywords: # default keywords are TODO, BUG, and FIXME, these can be overwritten by this setting - - NOTE - - OPTIMIZE # marks code that should be optimized before merging - - HACK # marks hack-arounds that should be removed before merging - dogsled: - # checks assignments with too many blank identifiers; default is 2 - max-blank-identifiers: 2 - - whitespace: - multi-if: false # Enforces newlines (or comments) after every multi-line if statement - multi-func: false # Enforces newlines (or comments) after every multi-line function signature - wsl: - # If true append is only allowed to be cuddled if appending value is - # matching variables, fields or types on line above. Default is true. - strict-append: true - # Allow calls and assignments to be cuddled as long as the lines have any - # matching variables, fields or types. Default is true. - allow-assign-and-call: true - # Allow multiline assignments to be cuddled. Default is true. - allow-multiline-assign: true - # Allow declarations (var) to be cuddled. - allow-cuddle-declarations: true - # Allow trailing comments in ending of blocks - allow-trailing-comment: false - # Force newlines in end of case at this limit (0 = never). - force-case-trailing-whitespace: 0 +version: "2" linters: - disable-all: true + default: standard # errcheck, govet, ineffassign, staticcheck, unused enable: - bodyclose - - deadcode - - depguard - - dogsled - - dupl - - errcheck - - exhaustive - # - exhaustivestruct - - exportloopref - - funlen - # - gci - - gochecknoinits - # - gochecknoglobals - - gocognit - - goconst + - copyloopvar + - errorlint - gocritic - - gocyclo - - godox - - goerr113 - - gofmt - - goimports - # - golint #deprecated - - gomnd - - goprintffuncname - gosec - - gosimple - - govet - - ineffassign - # - interfacer #deprecated - lll - # - maligned - - megacheck - misspell - - nakedret - - nestif - - nlreturn - - prealloc - nolintlint - revive - - rowserrcheck - - exportloopref # replaces scopelint - - staticcheck - - structcheck - - stylecheck - - typecheck - unconvert - - unparam - - unused - - varcheck - whitespace - # - wrapcheck - - wsl - fast: false - -issues: - # Independently from option `exclude` we use default exclude patterns, - # it can be disabled by this option. To list all - # excluded by default patterns execute `golangci-lint run --help`. - # Default value for this option is true. - exclude-use-default: false + settings: + gocritic: + disabled-checks: + - ifElseChain + lll: + line-length: 120 + misspell: + locale: US + revive: + rules: + - name: exported + - name: package-comments + - name: var-naming + - name: error-strings + - name: error-naming + - name: receiver-naming + - name: indent-error-flow + - name: blank-imports + exclusions: + presets: + - std-error-handling + rules: + # Tests can open files from variables and use relaxed permissions + - path: _test\.go + linters: + - gosec + +formatters: + enable: + - gofmt + - goimports + settings: + goimports: + local-prefixes: + - github.com/fclairamb/ftpserver diff --git a/CLAUDE.md b/CLAUDE.md index aa998afc..33ca6c53 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,89 +4,34 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Commands -### Build & Development - `go build -v ./...` - Build all packages -- `go run main.go` - Run the FTP server locally -- `go run main.go -conf ftpserver.json` - Run with specific config file -- `go run main.go -conf-only` - Generate configuration file only - -### Testing - `go test -race -v ./...` - Run all tests with race detection -- Individual test: `go test -v ./fs/utils/` - -### Linting & Code Quality -- `golangci-lint run` - Run comprehensive linting (configured via .golangci.yml) -- `go fmt ./...` - Format code -- `goimports -w ./` - Fix imports +- `golangci-lint run` - Lint (golangci-lint v2, configured by `.golangci.yml`, run by the CI) +- `go run . -conf ftpserver.json` - Run with a specific config file (`ftpserver.json` is created if no `-conf` is given) +- `go run . -conf-only` - Only create/check the configuration file ## Architecture -### Core Structure -This is a Go FTP server that acts as a gateway between FTP clients and various cloud storage backends. It's built on top of [ftpserverlib](https://github.com/fclairamb/ftpserverlib) and uses the [afero](https://github.com/spf13/afero) filesystem abstraction. - -### Key Components - -#### main.go:25 -Entry point that handles configuration loading, server instantiation, and graceful shutdown. Supports SIGTERM and SIGHUP (config reload) signals. - -#### config/ package -- `config/config.go` - Configuration management and user authentication -- `config/confpar/` - Configuration parameter definitions -- Uses JSON configuration files (default: `ftpserver.json`) - -#### fs/ package - Filesystem Backends -Each subdirectory implements a specific storage backend: -- `fs/afos/` - Local OS filesystem (afero wrapper) -- `fs/s3/` - Amazon S3 storage -- `fs/gdrive/` - Google Drive integration -- `fs/dropbox/` - Dropbox storage -- `fs/gcs/` - Google Cloud Storage -- `fs/sftp/` - SFTP backend -- `fs/mail/` - Email storage backend -- `fs/telegram/` - Telegram bot storage -- `fs/keycloak/` - Keycloak authentication integration -- `fs/utils/` - Shared filesystem utilities -- `fs/stripprefix/` - Path manipulation utilities -- `fs/fslog/` - Filesystem logging wrapper - -#### server/ package -Contains the main FTP server driver implementation that bridges ftpserverlib with the various filesystem backends. - -### Configuration System -- JSON-based configuration with schema validation -- Supports multiple user accounts with different backends -- Each user can have different filesystem backends (S3, Dropbox, etc.) -- TLS/SSL support for secure connections -- Configurable passive transfer port ranges - -### Backend Integration Pattern -Each filesystem backend in `fs/` follows a consistent pattern: -1. Implements afero.Fs interface -2. Handles backend-specific authentication/configuration -3. Maps FTP operations to backend API calls -4. Provides error handling and logging - -### Build Information -- `build.go` contains build-time variables (version, date, commit) -- Cross-platform builds supported via GitHub Actions -- Docker images available and built automatically - -## Development Notes - -### Testing Strategy -- Limited test coverage (only `fs/utils/env_test.go` currently) -- Integration testing likely done via Docker containers -- GitHub Actions runs tests with race detection - -### Code Quality -- Comprehensive golangci-lint configuration with 40+ linters enabled -- Strict formatting and import organization rules -- Local package prefix: `github.com/fclairamb/ftpserver/` -- Line length limit: 120 characters -- Function length limits: 80 lines/40 statements - -### Dependencies -- Go 1.24+ required (toolchain 1.25+) -- Key dependencies: ftpserverlib, afero, various cloud SDK packages -- Logging via go-kit/log framework -- Authentication via go-crypt for password hashing \ No newline at end of file +A Go FTP server acting as a gateway between FTP clients and various storage backends. It's built on top of +[ftpserverlib](https://github.com/fclairamb/ftpserverlib) (the FTP protocol) and uses the +[afero](https://github.com/spf13/afero) `Fs` interface for the backends. + +- `main.go` - Entry point: config loading, server instantiation, signals (`SIGHUP` reloads the config, `SIGTERM` + stops gracefully). +- `config/` - Config file loading and user authentication (plain-text and hashed passwords). + `config/confpar/` holds the config structures, which must stay in sync with `config-schema.json`. +- `server/` - The ftpserverlib driver: settings, client counting, authentication (config or webhook), shared file + system cache, TLS. +- `fs/` - `fs.go` instantiates the backend of an access and applies the common options (read-only, sync & delete). + Each subdirectory is a backend exposing a `LoadFs` function: `afos` (local disk), `s3`, `gcs`, `dropbox`, `gdrive`, + `sftp`, `mail`, `telegram`, `keycloak`. `fslog` wraps a file system to log accesses, `utils` has shared helpers. + +## Conventions + +- Backend parameters are strings read from `access.Params`. Document them in the backend's `README.md` and keep the + JSON schema and the main README backend table up to date. +- Imports: standard library, third-party, then `github.com/fclairamb/ftpserver/...` packages. +- Line length limit: 120 characters. +- Pull request titles follow conventional commits; releases are cut by release-please, which also triggers the + binaries (`release-artifacts.yml`) and docker images (`docker.yml`) builds. +- `build.go` holds the version variables, set at build time with `-ldflags -X`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 34039c8e..3b110305 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,27 @@ # Contributing -If you want to contribute, just add an issue or submit a pull request. +Issues and pull requests are welcome. + +## Development + +```sh +go build -v ./... # Build +go test -race -v ./... # Run the tests +golangci-lint run # Lint (golangci-lint v2, configured by .golangci.yml) +go run . -conf ftpserver.json +``` + +If you change the config file format, update the [JSON schema](config-schema.json) and the documentation accordingly. +Sample config files (in `config/samples/` and `fs/*/`) are validated against the schema by the CI. + +## Adding a backend + +A backend is a package in `fs/` exposing a `LoadFs(access *confpar.Access, ...) (afero.Fs, error)` function, that +reads its parameters from `access.Params`. It must then be registered in [fs/fs.go](fs/fs.go), added to the `fs` enum +of the [JSON schema](config-schema.json), and documented in a `README.md` in its directory, linked from the +[main README](README.md#supported-backends). + +## Pull requests + +Pull request titles must follow the [conventional commits](https://www.conventionalcommits.org/) format +(`feat: ...`, `fix: ...`, `chore: ...`), as they are used to generate the [changelog](CHANGELOG.md) and the releases. diff --git a/Dockerfile b/Dockerfile index 1b9c0807..524e38ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,31 @@ # Should be started with: # docker run -ti -p 2121-2130:2121-2130 fclairamb/ftpserver -# Preparing the build environment -FROM golang:1.27-alpine AS builder -ENV GOFLAGS="-mod=readonly" -RUN apk add --update --no-cache bash ca-certificates curl git && update-ca-certificates -RUN mkdir -p /workspace +# Preparing the build environment. The binary is cross-compiled on the build platform, which is much faster than +# building it in an emulated environment. +FROM --platform=$BUILDPLATFORM golang:1.27-alpine AS builder +ENV GOFLAGS="-mod=readonly" CGO_ENABLED=0 +RUN apk add --update --no-cache ca-certificates && update-ca-certificates WORKDIR /workspace +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download + # Building COPY . . -RUN CGO_ENABLED=0 go build -mod=readonly -ldflags='-w -s' -v -o ftpserver +ARG TARGETOS TARGETARCH TARGETVARIANT +ARG VERSION=dev COMMIT="" BUILD_DATE="" +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ + if [ "$TARGETARCH" = "arm" ]; then export GOARM="${TARGETVARIANT#v}"; fi && \ + GOOS=$TARGETOS GOARCH=$TARGETARCH go build -v -o ftpserver \ + -ldflags="-w -s -X main.BuildVersion=${VERSION} -X main.Commit=${COMMIT} -X main.BuildDate=${BUILD_DATE}" # Preparing the final image FROM scratch -WORKDIR /app COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +# The default configuration serves the /tmp directory +COPY --from=builder /tmp /tmp +WORKDIR /app EXPOSE 2121-2130 COPY --from=builder /workspace/ftpserver /bin/ftpserver ENTRYPOINT [ "/bin/ftpserver" ] diff --git a/Dockerfile.alpine b/Dockerfile.alpine index cab710e7..620dc561 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -1,16 +1,24 @@ # Should be started with: -# docker run -ti -p 2121-2130:2121-2130 fclairamb/ftpserver +# docker run -ti -p 2121-2130:2121-2130 fclairamb/ftpserver:alpine -# Preparing the build environment -FROM golang:1.27-alpine AS builder -ENV GOFLAGS="-mod=readonly" -RUN apk add --update --no-cache bash ca-certificates curl git && update-ca-certificates -RUN mkdir -p /workspace +# Preparing the build environment. The binary is cross-compiled on the build platform, which is much faster than +# building it in an emulated environment. +FROM --platform=$BUILDPLATFORM golang:1.27-alpine AS builder +ENV GOFLAGS="-mod=readonly" CGO_ENABLED=0 +RUN apk add --update --no-cache ca-certificates && update-ca-certificates WORKDIR /workspace +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download + # Building COPY . . -RUN go build -v -o ftpserver +ARG TARGETOS TARGETARCH TARGETVARIANT +ARG VERSION=dev COMMIT="" BUILD_DATE="" +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ + if [ "$TARGETARCH" = "arm" ]; then export GOARM="${TARGETVARIANT#v}"; fi && \ + GOOS=$TARGETOS GOARCH=$TARGETARCH go build -v -o ftpserver \ + -ldflags="-w -s -X main.BuildVersion=${VERSION} -X main.Commit=${COMMIT} -X main.BuildDate=${BUILD_DATE}" # Preparing the final image FROM alpine:3.24.3 diff --git a/README.md b/README.md index f708e7cb..d90e0a3f 100644 --- a/README.md +++ b/README.md @@ -1,77 +1,81 @@ # Golang FTP Server -![Build](https://github.com/fclairamb/ftpserver/workflows/Build/badge.svg) -![Cross Build](https://github.com/fclairamb/ftpserver/workflows/Cross%20Build/badge.svg) -![Docker Image](https://github.com/fclairamb/ftpserver/workflows/Docker%20Image/badge.svg) -![Docker test](https://github.com/fclairamb/ftpserver/workflows/Docker%20test/badge.svg) -![Docker downloads](https://img.shields.io/docker/pulls/fclairamb/ftpserver) -[![Go Report Card](https://goreportcard.com/badge/fclairamb/ftpserver)](https://goreportcard.com/report/fclairamb/ftpserver) -[![GoDoc](https://godoc.org/github.com/fclairamb/ftpserver?status.svg)](https://godoc.org/github.com/fclairamb/ftpserver/server) - -This FTP server is a gateway between old-school FTP devices and modern cloud based file systems, using the -[afero](https://github.com/spf13/afero)'s Fs interface and acting as a reference implementation of the [ftpserverlib](https://github.com/fclairamb/ftpserverlib). - -At the current stage, supported backend are: -- Local disk -- [S3](https://aws.amazon.com/s3/) through [afero-s3](https://github.com/fclairamb/afero-s3) -- [Dropbox](https://www.dropbox.com/) (see [doc](https://github.com/fclairamb/ftpserver/tree/master/fs/dropbox)) through [afero-dropbox](https://github.com/fclairamb/afero-dropbox) -- [Google Drive](https://developers.google.com/drive) (see [doc](https://github.com/fclairamb/ftpserver/tree/master/fs/gdrive)) through [afero-gdrive](https://github.com/fclairamb/afero-gdrive) -- [SFTP](https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol) through [afero's sftpfs](https://github.com/spf13/afero/) -- Email through [go-mail](https://github.com/go-mail/mail) thanks to [@x-way](https://github.com/x-way) -- Telegram through [telebot](https://github.com/tucnak/telebot) by [@slayer](https://github.com/slayer), see [doc](fs/telegram/README.md) - -And with those are supported common parameters to switch them to read-only, enable login access, or use a temporary directory file (see [doc](https://github.com/fclairamb/ftpserver/tree/master/fs)). - -## Current status of the project - -### Features - -#### FTP protocol - -These features are brought by [ftpserverlib](https://github.com/fclairamb/ftpserverlib) itself: - - * Uploading and downloading files - * Directory listing (LIST + MLST) - * File and directory deletion and renaming - * TLS support (AUTH + PROT) - * File download/upload resume support (REST) - * Complete driver for all the above features - * Passive socket connections (EPSV and PASV commands) - * Active socket connections (PORT command) - * Small memory footprint - * Only relies on the standard library except for: - * [go-kit log](https://github.com/go-kit/kit/tree/master/log) for logging - * [afero](https://github.com/spf13/afero) for generic file systems handling - * Supported extensions: - * [AUTH](https://tools.ietf.org/html/rfc2228#page-6) - Control session protection - * [AUTH TLS](https://tools.ietf.org/html/rfc4217#section-4.1) - TLS session - * [PROT](https://tools.ietf.org/html/rfc2228#page-8) - Transfer protection - * [EPRT/EPSV](https://tools.ietf.org/html/rfc2428) - IPv6 support - * [MDTM](https://tools.ietf.org/html/rfc3659#page-8) - File Modification Time - * [SIZE](https://tools.ietf.org/html/rfc3659#page-11) - Size of a file - * [REST](https://tools.ietf.org/html/rfc3659#page-13) - Restart of interrupted transfer - * [MLST](https://tools.ietf.org/html/rfc3659#page-23) - Simple file listing for machine processing - * [MLSD](https://tools.ietf.org/html/rfc3659#page-23) - Directory listing for machine processing +[![Build](https://github.com/fclairamb/ftpserver/actions/workflows/build.yml/badge.svg)](https://github.com/fclairamb/ftpserver/actions/workflows/build.yml) +[![Docker images](https://github.com/fclairamb/ftpserver/actions/workflows/docker.yml/badge.svg)](https://github.com/fclairamb/ftpserver/actions/workflows/docker.yml) +[![Docker test](https://github.com/fclairamb/ftpserver/actions/workflows/docker-test.yml/badge.svg)](https://github.com/fclairamb/ftpserver/actions/workflows/docker-test.yml) +[![Docker pulls](https://img.shields.io/docker/pulls/fclairamb/ftpserver)](https://hub.docker.com/r/fclairamb/ftpserver) +[![Go Report Card](https://goreportcard.com/badge/github.com/fclairamb/ftpserver)](https://goreportcard.com/report/github.com/fclairamb/ftpserver) +[![Go Reference](https://pkg.go.dev/badge/github.com/fclairamb/ftpserver.svg)](https://pkg.go.dev/github.com/fclairamb/ftpserver) + +A gateway between old-school FTP devices (cameras, scanners, printers, legacy software...) and modern cloud based +file systems. + +It is built on top of [ftpserverlib](https://github.com/fclairamb/ftpserverlib), for which it is the reference +implementation, and of the [afero](https://github.com/spf13/afero) `Fs` interface. + +## Supported backends + +| Backend | `fs` value | Documentation | +| --- | --- | --- | +| Local disk | `os` | [below](#local-disk) | +| [Amazon S3](https://aws.amazon.com/s3/) and S3-compatible storages (MinIO, Ceph...), through [afero-s3](https://github.com/fclairamb/afero-s3) | `s3` | [fs/s3](fs/s3/README.md) | +| [Google Cloud Storage](https://cloud.google.com/storage), through afero's [gcsfs](https://github.com/spf13/afero/tree/master/gcsfs) | `gcs` | [fs/gcs](fs/gcs/README.md) | +| [Dropbox](https://www.dropbox.com/), through [afero-dropbox](https://github.com/fclairamb/afero-dropbox) | `dropbox` | [fs/dropbox](fs/dropbox/README.md) | +| [Google Drive](https://developers.google.com/drive), through [afero-gdrive](https://github.com/fclairamb/afero-gdrive) | `gdrive` | [fs/gdrive](fs/gdrive/README.md) | +| [SFTP](https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol), through afero's [sftpfs](https://github.com/spf13/afero/tree/master/sftpfs) | `sftp` | [fs/sftp](fs/sftp/README.md) | +| Email (each uploaded file is sent as an attachment), thanks to [@x-way](https://github.com/x-way) | `mail` | [fs/mail](fs/mail/README.md) | +| Telegram (each uploaded file is sent to a chat), thanks to [@slayer](https://github.com/slayer) | `telegram` | [fs/telegram](fs/telegram/README.md) | +| Local disk with users authenticated by [Keycloak](https://www.keycloak.org/) | `keycloak` | [fs/keycloak](fs/keycloak/README.md) | + +On top of any backend, an access can be made read-only, share its backend instance between sessions, write through a +local temporary directory or log every file access (see [fs/README.md](fs/README.md)). + +## Features + +FTP protocol support is brought by [ftpserverlib](https://github.com/fclairamb/ftpserverlib): + +* Uploading and downloading files, with resume support (`REST`) +* Directory listing (`LIST`, `NLST`, [`MLST`/`MLSD`](https://tools.ietf.org/html/rfc3659#page-23)) +* File and directory deletion and renaming, symbolic links (`SITE SYMLINK`, on the local disk backend) +* Passive (`PASV`, `EPSV`) and active (`PORT`, `EPRT`) transfers, IPv6 support ([RFC 2428](https://tools.ietf.org/html/rfc2428)) +* Explicit TLS ([`AUTH TLS`](https://tools.ietf.org/html/rfc4217#section-4.1), [`PROT`](https://tools.ietf.org/html/rfc2228#page-8)) and implicit TLS +* File modification time ([`MDTM`](https://tools.ietf.org/html/rfc3659#page-8)) and size ([`SIZE`](https://tools.ietf.org/html/rfc3659#page-11)) +* File hashing (`HASH`, `XCRC`, `MD5`, `XSHA256`...), when enabled +* Small memory footprint + +On top of it, this server provides: + +* Multiple users, each one with their own backend +* Hashed passwords (bcrypt, md5crypt, sha256crypt, sha512crypt), plain-text passwords can be hashed automatically +* Authentication through an HTTP webhook +* Configuration hot-reload (`SIGHUP`) and graceful shutdown (`SIGTERM`) ## Getting started -### Get it +### Download it -#### Download it Fetch a binary for your platform from the [latest release](https://github.com/fclairamb/ftpserver/releases/latest) and run it. Archives are named `ftpserver---` and are provided for Linux, macOS and Windows on `amd64` and `arm64`, plus `armv6`/`armv7` for Linux (Raspberry Pi and friends). -#### Build & run it locally +### Build it -```bash -go install github.com/fclairamb/ftpserver@main - -ftpserver & +```sh +go install github.com/fclairamb/ftpserver@latest +ftpserver ``` -#### Run it with docker -There's also a containerized version of the server (31MB, based on alpine). +### Run it with docker + +Images are available on [Docker Hub](https://hub.docker.com/r/fclairamb/ftpserver) and on the +[GitHub container registry](https://github.com/fclairamb/ftpserver/pkgs/container/ftpserver) +(`ghcr.io/fclairamb/ftpserver`) for `linux/amd64`, `linux/arm64`, `linux/arm/v6` and `linux/arm/v7`: + +* `latest`, ``: minimal image, containing only the binary +* `alpine`, `-alpine`: image based on alpine, if you need a shell + +The image runs from the `/app` directory, which is where it reads (or creates) its `ftpserver.json` config file. The +default config file serves the `/tmp` directory. ```sh # Creating a directory @@ -81,157 +85,177 @@ mkdir -p files docker run --rm -d -p 2121-2130:2121-2130 -v $(pwd)/files:/tmp -v $(pwd):/app fclairamb/ftpserver ``` -#### Run it with docker compose +### Run it with docker compose + ```yml # docker-compose.yml - -version: '3.3' services: ftpserver: + image: fclairamb/ftpserver ports: - '2121-2130:2121-2130' volumes: - ./files:/tmp - .:/app - image: fclairamb/ftpserver ``` ```sh -docker-compose up -d +docker compose up -d ``` ### Test it -This is a quick way to see if it's working correctly: ```sh -# Download some file -[ -f kitty.jpg ] || (curl -o kitty.jpg.tmp https://placekitten.com/2048/2048 && mv kitty.jpg.tmp kitty.jpg) +# Create some file +head -c 1048576 /dev/urandom > file.bin # Upload it to the server -curl -v -T kitty.jpg ftp://test:test@localhost:2121/ +curl -v -T file.bin ftp://test:test@localhost:2121/ # Download it back -curl ftp://test:test@localhost:2121/kitty.jpg -o kitty2.jpg +curl ftp://test:test@localhost:2121/file.bin -o file2.bin # Compare it -diff kitty.jpg kitty2.jpg +cmp file.bin file2.bin ``` -### Config file -If you don't create a `ftpserver.json` file, one will be created for you. +## Usage -Here is a sample config file: +``` +ftpserver [-conf ] [-conf-only] +``` + +* `-conf `: config file to use. When not specified, `ftpserver.json` is used, and created if it doesn't exist. +* `-conf-only`: only create the config file (if it doesn't exist) and check it can be loaded, then exit. + +Signals: + +* `SIGHUP` reloads the config file. +* `SIGTERM` stops accepting new connections and waits up to one minute for the connected clients to leave. + +## Configuration + +The configuration is a JSON file, described by a [JSON schema](config-schema.json). Referencing it with the +`$schema` property gives you auto-completion and validation in most editors. If you start the server without any +config file, a sample one is created with a single `test`/`test` user serving `/tmp`. ```json { - "version": 1, - "passive_transfer_port_range": { - "start": 2122, - "end": 2130 - }, - "tls": { - "server_cert": { - "cert": "cert.pem", - "key": "key.pem" + "$schema": "https://raw.githubusercontent.com/fclairamb/ftpserver/main/config-schema.json", + "version": 1, + "listen_address": "0.0.0.0:2121", + "passive_transfer_port_range": { + "start": 2122, + "end": 2130 + }, + "accesses": [ + { + "user": "test", + "pass": "test", + "fs": "os", + "params": { + "basePath": "/tmp" } - }, - "accesses": [ - { - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/tmp" - } - }, - { - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/tmp" - } - }, - { - "user": "dropbox", - "pass": "dropbox", - "fs": "dropbox", - "params": { - "token": "..." - } - }, - { - "user": "gdrive", - "pass": "gdrive", - "fs": "gdrive", - "params": { - "google_client_id": "***.apps.googleusercontent.com", - "google_client_secret": "****", - "base_path": "ftp" - } - }, - { - - "user": "s3", - "pass": "s3", - "fs": "s3", - "params": { - "endpoint": "https://s3.amazonaws.com", - "region": "eu-west-1", - "bucket": "my-bucket", - "access_key_id": "AKIA....", - "secret_access_key": "IDxd....", - "disable_ssl": "false", - "path_style": "false" - } - }, - { - "user": "sftp", - "pass": "sftp", - "fs": "sftp", - "params": { - "username": "user", - "password": "password", - "hostname": "192.168.168.11:22", - "known_hosts": "/home/ftpserver/.ssh/known_hosts" - } - }, - { - "user": "telegram", - "pass": "telegram", - "fs": "telegram", - "shared": true, - "params": { - "Token": "", - "ChatID": "", - "MaxPartSize": "51380224", - "TempDir": "/tmp", - "RetryAttempts": "10", - "RetryDelay": "2000", - "PartUploadDelay": "500" - } + }, + { + "user": "s3", + "pass": "$2a$10$jG7tuqIlcUDMl1m1Ytj1TunU7pk.ko8lj3nOGzZvkIeU/BsfPVBra", + "fs": "s3", + "read_only": true, + "params": { + "bucket": "my-bucket", + "region": "eu-west-1" } - ] + } + ] } ``` -You can generate the TLS key pair files with the following command: -```bash +More samples are available in [config/samples](config/samples). + +### General options + +| Property | Description | Default | +| --- | --- | --- | +| `listen_address` | Address to listen on | `0.0.0.0:2121` | +| `public_host` | Public IP address to advertise for passive transfers, when the server is behind a NAT gateway. Can be overridden with the `PUBLIC_HOST` environment variable | | +| `passive_transfer_port_range` | Ports to use for passive transfers (`start`, `end`) | any port | +| `max_clients` | Maximum number of simultaneous clients, `0` means no limit | `0` | +| `idle_timeout` | Duration after which idle connections are closed (e.g. `5m`), a negative value disables it | `15m` | +| `tls` | TLS certificate files: `{"server_cert": {"cert": "cert.pem", "key": "key.pem"}}` | | +| `tls_required` | `ClearOrEncrypted`, `MandatoryEncryption` (explicit TLS required) or `ImplicitEncryption` (implicit FTPS) | `ClearOrEncrypted` | +| `hash_plaintext_passwords` | Replace the plain-text passwords of the config file by their bcrypt hash | `false` | +| `extensions.enable_hash` | Enable the `HASH` command and its variants | `false` | +| `logging.ftp_exchanges` | Log all the FTP commands and responses | `false` | +| `logging.file_accesses` | Log all the file accesses | `false` | +| `logging.file` | Also write the logs to this file | | +| `accesses` | The users and their backend, see below | | +| `accesses_webhook` | Authenticate users with a webhook instead of the `accesses`, see below | | + +### Accesses + +Each access defines a user and the backend serving their files: + +| Property | Description | +| --- | --- | +| `user` | User name | +| `pass` | Password: plain-text or hashed (`$2a$`/`$2b$`/`$2y$` bcrypt, `$1$` md5crypt, `$5$` sha256crypt, `$6$` sha512crypt). The `anonymous` user with a `*` password accepts any password | +| `fs` | Backend to use (see [supported backends](#supported-backends)) | +| `params` | Backend parameters, as strings. See each backend's documentation | +| `read_only`, `shared`, `sync_and_delete`, `logging` | Common options, see [fs/README.md](fs/README.md) | + +Accesses are evaluated in their declaration order. A hashed password can be generated with +`htpasswd -nbBC 10 "" 'my-password' | cut -d: -f2`, or by enabling `hash_plaintext_passwords`. + +#### Local disk + +The `os` backend serves a local directory, defined by the `basePath` parameter. Environment variables (`$NAME`) in +the path are replaced by their value. + +```json +{ + "user": "test", + "pass": "test", + "fs": "os", + "params": { + "basePath": "/srv/ftp/$FTP_DIR" + } +} +``` + +### Webhook authentication + +When `accesses_webhook` is defined, the `accesses` of the config file are ignored: for each login, the user and +password are posted as JSON (`{"user": "...", "pass": "..."}`) to the webhook. It must reply with a `200` status code +and an [access](#accesses) object, any other status code refuses the login. + +```json +{ + "accesses_webhook": { + "url": "https://auth.example.com/ftp", + "headers": { + "Authorization": "Bearer my-secret-token" + }, + "timeout": "5s" + }, + "accesses": [] +} +``` + +The `timeout` defaults to `30s`. + +### TLS + +You can generate a self-signed TLS key pair with the following command: + +```sh openssl req -new -newkey rsa:4096 -x509 -sha256 -days 365 -nodes -out cert.pem -keyout key.pem ``` -### SFTP backend host key verification +## Contributing -The `sftp` backend connects to an upstream SSH/SFTP server and verifies its host key to -protect against man-in-the-middle attacks. You must configure exactly one of the following -`params`: +See [CONTRIBUTING.md](CONTRIBUTING.md). -- `known_hosts`: path to an OpenSSH `known_hosts` file containing the upstream server's key - (e.g. generated with `ssh-keyscan -H host >> known_hosts`). -- `host_key`: the upstream server's public key pinned inline, in `authorized_keys` / - `known_hosts` line format (e.g. `ssh-ed25519 AAAA...`). -- `insecure_ignore_host_key`: set to `"true"` to disable host key verification entirely. - This is **unsafe** (it exposes the connection to man-in-the-middle attacks) and should only - be used in trusted networks or for testing. +## License -If none of these is set, the connection is refused. +[MIT](LICENSE) diff --git a/build.go.tpl b/build.go.tpl deleted file mode 100644 index 265c2da1..00000000 --- a/build.go.tpl +++ /dev/null @@ -1,12 +0,0 @@ -package main - -var ( - // BuildVersion is the current version of the program - BuildVersion = "{{ .Version }}" - - // BuildDate is the time the program was built - BuildDate = "{{ .BuildDate }}" - - // Commit is the git hash of the program - Commit = "{{ .GitCommitHashShort }}" -) diff --git a/config-schema.json b/config-schema.json index d51d81d9..271e19bd 100644 --- a/config-schema.json +++ b/config-schema.json @@ -38,10 +38,11 @@ "public_host": { "type": "string", "default": "", - "title": "The public listening address (when behing a NAT gateway)", + "title": "The public IP address (when behind a NAT gateway)", "examples": [ "1.2.3.4" - ] + ], + "description": "Can be overridden with the PUBLIC_HOST environment variable." }, "max_clients": { "type": "integer", @@ -49,7 +50,8 @@ "title": "The maximum number of concurrent clients to accept", "examples": [ 200 - ] + ], + "description": "0 means no limit." }, "hash_plaintext_passwords": { "type": "boolean", @@ -62,14 +64,13 @@ }, "idle_timeout": { "type": "string", - "default": "0s", + "default": "15m", "title": "Maximum idle time for client connections", - "description": "Duration after which idle client connections will be closed. Use Go duration format (e.g., '5m', '30s', '1h'). Set to 0 or empty to disable idle timeout.", + "description": "Duration after which idle client connections are closed, in Go duration format (e.g. '5m', '30s', '1h'). Defaults to 15 minutes when unset or 0, a negative value (e.g. '-1s') disables it.", "examples": [ "5m", "30s", - "1h", - "0s" + "1h" ] }, "passive_transfer_port_range": { @@ -98,10 +99,12 @@ ] } }, - "examples": [{ - "start": 2122, - "end": 2130 - }] + "examples": [ + { + "start": 2122, + "end": 2130 + } + ] }, "logging": { "type": "object", @@ -126,18 +129,20 @@ }, "file": { "type": "string", - "default": "ftpserver.log", + "default": "", "title": "Perform logging to a file", "examples": [ "ftpserver.log" ] } }, - "examples": [{ - "ftp_exchanges": true, - "file_accesses": true, - "file": "ftpserver.log" - }] + "examples": [ + { + "ftp_exchanges": true, + "file_accesses": true, + "file": "ftpserver.log" + } + ] }, "tls": { "type": "object", @@ -173,18 +178,84 @@ ] } }, - "examples": [{ + "examples": [ + { + "cert": "cert.pem", + "key": "key.pem" + } + ] + } + }, + "examples": [ + { + "server_cert": { "cert": "cert.pem", "key": "key.pem" - }] + } } - }, - "examples": [{ - "server_cert": { - "cert": "cert.pem", - "key": "key.pem" + ] + }, + "tls_required": { + "type": "string", + "default": "ClearOrEncrypted", + "title": "TLS requirement", + "description": "ClearOrEncrypted: TLS is optional, MandatoryEncryption: clients must use explicit TLS (AUTH TLS), ImplicitEncryption: the whole connection is encrypted (implicit FTPS).", + "enum": [ + "ClearOrEncrypted", + "MandatoryEncryption", + "ImplicitEncryption" + ] + }, + "extensions": { + "type": "object", + "default": {}, + "title": "Extended features", + "additionalProperties": false, + "properties": { + "enable_hash": { + "type": "boolean", + "default": false, + "title": "Enable the HASH command (and its XCRC, MD5, XMD5, XSHA, XSHA1, XSHA256, XSHA512 variants)" } - }] + } + }, + "accesses_webhook": { + "type": "object", + "title": "Webhook authenticating users", + "description": "When set, the accesses are not read from the config file: the user and password are POSTed as JSON to the webhook, which must reply with a 200 status code and an access object.", + "required": [ + "url" + ], + "additionalProperties": false, + "properties": { + "url": { + "type": "string", + "title": "URL to call", + "examples": [ + "https://auth.example.com/ftp" + ] + }, + "headers": { + "type": "object", + "title": "HTTP headers to add to the request", + "additionalProperties": { + "type": "string" + }, + "examples": [ + { + "Authorization": "Bearer secret" + } + ] + }, + "timeout": { + "type": "string", + "default": "30s", + "title": "Maximum duration of the request (Go duration format)", + "examples": [ + "5s" + ] + } + } }, "accesses": { "type": "array", @@ -192,12 +263,9 @@ "title": "The accesses", "items": { "type": "object", - "title": "A Schema", + "title": "An access", "required": [ - "user", - "pass", - "fs", - "params" + "fs" ], "properties": { "user": { @@ -205,7 +273,8 @@ "title": "The FTP user", "examples": [ "username" - ] + ], + "description": "Not used by the keycloak backend, which accepts any user." }, "pass": { "type": "string", @@ -213,17 +282,22 @@ "examples": [ "plaintext-password", "$2a$10$jG7tuqIlcUDMl1m1Ytj1TunU7pk.ko8lj3nOGzZvkIeU/BsfPVBra" - ] + ], + "description": "Plain-text, or hashed with md5crypt, bcrypt, sha256crypt or sha512crypt. Use \"*\" with the \"anonymous\" user to accept any password." }, "fs": { "type": "string", "title": "The backend file system to use", - "examples": [ + "enum": [ "os", + "s3", + "gcs", + "sftp", "dropbox", "gdrive", - "s3", - "sftp" + "mail", + "telegram", + "keycloak" ] }, "params": { @@ -233,7 +307,7 @@ "shared": { "type": "boolean", "default": false, - "title": "If the backend shall be shared between session", + "title": "If the backend instance shall be shared between sessions", "examples": [ true ] @@ -251,14 +325,13 @@ "default": {}, "title": "If we should use the sync & delete mechanism", "required": [ - "enable", - "directory" + "enable" ], "properties": { "enable": { "type": "boolean", "default": false, - "title": "The enable Schema", + "title": "Enable the sync & delete mechanism", "examples": [ true ] @@ -266,83 +339,97 @@ "directory": { "type": "string", "default": "", - "title": "The directory Schema", + "title": "Temporary directory (optional, a system one is used otherwise)", "examples": [ "/tmp/snd" ] } }, - "examples": [{ - "enable": true, - "directory": "/tmp/snd" - }] - } - }, - "examples": [{ - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/tmp" - } - }, - { - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/tmp" - } - }, - { - "user": "dropbox", - "pass": "dropbox", - "fs": "dropbox", - "params": { - "token": "..." - } - }, - { - "user": "gdrive", - "pass": "gdrive", - "fs": "gdrive", - "params": { - "google_client_id": "***.apps.googleusercontent.com", - "google_client_secret": "****", - "base_path": "ftp" - } - }, - { - "user": "s3", - "pass": "s3", - "fs": "s3", - "params": { - "endpoint": "https://s3.amazonaws.com", - "region": "eu-west-1", - "bucket": "my-bucket", - "access_key_id": "AKIA....", - "secret_access_key": "IDxd....", - "disable_ssl": "false", - "path_style": "false" + "examples": [ + { + "enable": true, + "directory": "/tmp/snd" + } + ] + }, + "logging": { + "type": "object", + "default": {}, + "title": "Logging options of this access", + "additionalProperties": false, + "properties": { + "ftp_exchanges": { + "type": "boolean", + "default": false, + "title": "Log all FTP exchanges" + }, + "file_accesses": { + "type": "boolean", + "default": false, + "title": "Log all file accesses" + } + } } }, - { - "user": "sftp", - "pass": "sftp", - "fs": "sftp", - "shared": true, - "read_only": true, - "sync_and_delete": { - "enable": true, - "directory": "/tmp/snd" + "examples": [ + { + "user": "test", + "pass": "test", + "fs": "os", + "params": { + "basePath": "/tmp" + } }, - "params": { - "username": "user", - "password": "password", - "hostname": "192.168.168.11:22", - "known_hosts": "/home/ftpserver/.ssh/known_hosts" + { + "user": "dropbox", + "pass": "dropbox", + "fs": "dropbox", + "params": { + "token": "..." + } + }, + { + "user": "gdrive", + "pass": "gdrive", + "fs": "gdrive", + "params": { + "google_client_id": "***.apps.googleusercontent.com", + "google_client_secret": "****", + "base_path": "ftp" + } + }, + { + "user": "s3", + "pass": "s3", + "fs": "s3", + "params": { + "endpoint": "https://s3.amazonaws.com", + "region": "eu-west-1", + "bucket": "my-bucket", + "access_key_id": "AKIA....", + "secret_access_key": "IDxd....", + "disable_ssl": "false", + "path_style": "false" + } + }, + { + "user": "sftp", + "pass": "sftp", + "fs": "sftp", + "shared": true, + "read_only": true, + "sync_and_delete": { + "enable": true, + "directory": "/tmp/snd" + }, + "params": { + "username": "user", + "password": "password", + "hostname": "192.168.168.11:22", + "known_hosts": "/home/ftpserver/.ssh/known_hosts" + } } - }] + ] } } }, diff --git a/config/README.md b/config/README.md index 99e235a5..ca6fe323 100644 --- a/config/README.md +++ b/config/README.md @@ -1,20 +1,12 @@ # Sample config files -## FTP Server behind a NAT gateway -```json -{ - "$schema": "https://raw.githubusercontent.com/fclairamb/ftpserver/main/config-schema.json", - "listen_address": ":2121", - "public_host": "1.2.3.4", - "accesses": [ - { - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/tmp" - } - } - ] -} -``` \ No newline at end of file +The config file format is described in the [main README](../README.md#configuration) and by the +[JSON schema](../config-schema.json). + +- [behind_nat.json](samples/behind_nat.json): FTP server behind a NAT gateway, advertising its public IP address + for passive transfers. +- [hashed_passwords.json](samples/hashed_passwords.json): plain-text, bcrypt and sha512crypt passwords (the + password of each user is its user name). +- [log_file.json](samples/log_file.json): logging to a file. + +Each backend also has its own sample in its [documentation](../fs). diff --git a/config/config.go b/config/config.go index 22076a25..b46d1efe 100644 --- a/config/config.go +++ b/config/config.go @@ -2,24 +2,37 @@ package config import ( - "bytes" + "crypto/subtle" "encoding/json" "errors" "fmt" "log/slog" "os" - - "github.com/fclairamb/ftpserver/config/confpar" - "github.com/fclairamb/ftpserver/fs" + "strings" + "sync" "github.com/go-crypt/crypt" + "github.com/go-crypt/crypt/algorithm" "github.com/go-crypt/crypt/algorithm/bcrypt" "github.com/tidwall/sjson" + + "github.com/fclairamb/ftpserver/config/confpar" + "github.com/fclairamb/ftpserver/fs" ) // ErrUnknownUser is returned when the provided user cannot be identified through our authentication mechanism var ErrUnknownUser = errors.New("unknown user") +// hashedPasswordPrefixes lists the prefixes of the password hash formats we support (md5crypt, bcrypt variants, +// sha256crypt and sha512crypt). A password without any of these prefixes is considered to be plain-text. +var hashedPasswordPrefixes = []string{"$1$", "$2$", "$2a$", "$2b$", "$2x$", "$2y$", "$5$", "$6$"} + +// bcryptCost is the cost used when hashing plain-text passwords +const bcryptCost = 10 + +// passwordDecoder is shared between all the authentication requests, it's safe for concurrent use +var passwordDecoder = sync.OnceValues(crypt.NewDecoderAll) + // Config provides the general server config type Config struct { fileName string @@ -96,71 +109,75 @@ func (c *Config) Load() error { return c.Prepare() } +// isHashedPassword tells if a password is stored in one of the supported hashed formats +func isHashedPassword(pass string) bool { + for _, prefix := range hashedPasswordPrefixes { + if strings.HasPrefix(pass, prefix) { + return true + } + } + + return false +} + +// HashPlaintextPasswords replaces all the plain-text passwords of the config file by their bcrypt equivalent, +// and saves the config file. func (c *Config) HashPlaintextPasswords() error { - json, errReadFile := os.ReadFile(c.fileName) + fileInfo, errStat := os.Stat(c.fileName) + if errStat != nil { + c.logger.Error("Cannot stat config file", "err", errStat) + + return errStat + } + + content, errReadFile := os.ReadFile(c.fileName) if errReadFile != nil { - c.logger.Error("Cannot read config file!", "err", errReadFile) + c.logger.Error("Cannot read config file", "err", errReadFile) + return errReadFile } + var hasher algorithm.Hash + save := false + for i, a := range c.Content.Accesses { - if a.User == "anonymous" && a.Pass == "*" { + if a.Fs == "keycloak" || (a.User == "anonymous" && a.Pass == "*") || isHashedPassword(a.Pass) { continue } - switch true { - case bytes.HasPrefix([]byte(a.Pass), []byte("$1$")): - //This user's password is md5crypt - continue - case bytes.HasPrefix([]byte(a.Pass), []byte("$2$")): - //This user's password is bcrypt - continue - case bytes.HasPrefix([]byte(a.Pass), []byte("$2a$")): - //This user's password is bcrypt-a - continue - case bytes.HasPrefix([]byte(a.Pass), []byte("$2b$")): - //This user's password is bcrypt-b - continue - case bytes.HasPrefix([]byte(a.Pass), []byte("$2x$")): - //This user's password is bcrypt-x - continue - case bytes.HasPrefix([]byte(a.Pass), []byte("$2y$")): - //This user's password is bcrypt-y - continue - case bytes.HasPrefix([]byte(a.Pass), []byte("$5$")): - //This user's password is sha256crypt - continue - case bytes.HasPrefix([]byte(a.Pass), []byte("$6$")): - //This user's password is sha512crypt - continue - default: - //This password is not hashed - hasher, err := bcrypt.New(bcrypt.WithCost(10)) - if err != nil { + if hasher == nil { + var err error + if hasher, err = bcrypt.New(bcrypt.WithCost(bcryptCost)); err != nil { return err } + } - digest, err := hasher.Hash(a.Pass) - if err != nil { - return err - } + digest, err := hasher.Hash(a.Pass) + if err != nil { + return err + } - modified, errJsonSet := sjson.Set(string(json), "accesses."+fmt.Sprint(i)+".pass", digest.Encode()) - c.Content.Accesses[i].Pass = digest.Encode() - if errJsonSet == nil { - save = true - json = []byte(modified) - } + modified, errJSONSet := sjson.Set(string(content), fmt.Sprintf("accesses.%d.pass", i), digest.Encode()) + if errJSONSet != nil { + return fmt.Errorf("could not update the password of access %d: %w", i, errJSONSet) } + + a.Pass = digest.Encode() + content = []byte(modified) + save = true } + if save { - errWriteFile := os.WriteFile(c.fileName, json, 0644) - if errWriteFile != nil { - c.logger.Error("Cannot write config file!", "err", errWriteFile) + // We keep the original file permissions as the file usually contains secrets + //nolint:gosec // the file name comes from the command line + if errWriteFile := os.WriteFile(c.fileName, content, fileInfo.Mode().Perm()); errWriteFile != nil { + c.logger.Error("Cannot write config file", "err", errWriteFile) + return errWriteFile } } + return nil } @@ -192,150 +209,54 @@ func (c *Config) CheckAccesses() error { return nil } -// GetAccess return a file system access given some credentials -func (c *Config) GetAccess(user string, pass string) (*confpar.Access, error) { - decoder, err := crypt.NewDecoderAll() +// checkPassword checks a password against its stored (plain-text or hashed) version +func checkPassword(stored, pass string) (bool, error) { + if !isHashedPassword(stored) { + return subtle.ConstantTimeCompare([]byte(stored), []byte(pass)) == 1, nil + } + + decoder, err := passwordDecoder() if err != nil { - return nil, err + return false, err } + digest, err := decoder.Decode(stored) + if err != nil { + return false, err + } + + return digest.MatchAdvanced(pass) +} + +// GetAccess return a file system access given some credentials. Accesses are evaluated in their declaration order. +func (c *Config) GetAccess(user string, pass string) (*confpar.Access, error) { for _, a := range c.Content.Accesses { if a.Fs == "keycloak" { - a.User = user - a.Pass = pass + // The credentials are checked by keycloak when loading the file system. We work on a copy because + // the access is shared between all the connections. + access := *a + access.User = user + access.Pass = pass + + return &access, nil + } + + if a.User != user { + continue + } + + if a.User == "anonymous" && a.Pass == "*" { return a, nil } - if a.User == user { - switch true { - case bytes.HasPrefix([]byte(a.Pass), []byte("$1$")): - //This user's password is md5crypt - digest, err := decoder.Decode(a.Pass) - if err != nil { - return nil, err - } - - ok, err := digest.MatchAdvanced(pass) - if err != nil { - return nil, err - } - - if ok { - return a, nil - } - case bytes.HasPrefix([]byte(a.Pass), []byte("$2$")): - //This user's password is bcrypt - digest, err := decoder.Decode(a.Pass) - if err != nil { - return nil, err - } - - ok, err := digest.MatchAdvanced(pass) - if err != nil { - return nil, err - } - - if ok { - return a, nil - } - case bytes.HasPrefix([]byte(a.Pass), []byte("$2a$")): - //This user's password is bcrypt-a - digest, err := decoder.Decode(a.Pass) - if err != nil { - return nil, err - } - - ok, err := digest.MatchAdvanced(pass) - if err != nil { - return nil, err - } - - if ok { - return a, nil - } - case bytes.HasPrefix([]byte(a.Pass), []byte("$2b$")): - //This user's password is bcrypt-b - digest, err := decoder.Decode(a.Pass) - if err != nil { - return nil, err - } - - ok, err := digest.MatchAdvanced(pass) - if err != nil { - return nil, err - } - - if ok { - return a, nil - } - case bytes.HasPrefix([]byte(a.Pass), []byte("$2x$")): - //This user's password is bcrypt-x - digest, err := decoder.Decode(a.Pass) - if err != nil { - return nil, err - } - - ok, err := digest.MatchAdvanced(pass) - if err != nil { - return nil, err - } - - if ok { - return a, nil - } - case bytes.HasPrefix([]byte(a.Pass), []byte("$2y$")): - //This user's password is bcrypt-y - digest, err := decoder.Decode(a.Pass) - if err != nil { - return nil, err - } - - ok, err := digest.MatchAdvanced(pass) - if err != nil { - return nil, err - } - - if ok { - return a, nil - } - case bytes.HasPrefix([]byte(a.Pass), []byte("$5$")): - //This user's password is sha256crypt - digest, err := decoder.Decode(a.Pass) - if err != nil { - return nil, err - } - - ok, err := digest.MatchAdvanced(pass) - if err != nil { - return nil, err - } - - if ok { - return a, nil - } - case bytes.HasPrefix([]byte(a.Pass), []byte("$6$")): - //This user's password is sha512crypt - digest, err := decoder.Decode(a.Pass) - if err != nil { - return nil, err - } - - ok, err := digest.MatchAdvanced(pass) - if err != nil { - return nil, err - } - - if ok { - return a, nil - } - default: - //This user's password is plain-text - if a.Pass == pass || (a.User == "anonymous" && a.Pass == "*") { - return a, nil - } - } + ok, err := checkPassword(a.Pass, pass) + if err != nil { + return nil, err } + if ok { + return a, nil + } } return nil, ErrUnknownUser diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 00000000..62f96e8f --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,148 @@ +package config + +import ( + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/fclairamb/ftpserver/config/confpar" +) + +func newTestConfig(t *testing.T, accesses ...*confpar.Access) *Config { + t.Helper() + + conf, err := FromContent(&confpar.Content{Accesses: accesses}, "test.json", slog.Default()) + if err != nil { + t.Fatalf("FromContent: %v", err) + } + + return conf +} + +func TestGetAccess(t *testing.T) { + conf := newTestConfig(t, + &confpar.Access{User: "plain", Pass: "plain", Fs: "os"}, + &confpar.Access{User: "bcrypt", Pass: "$2b$05$aco32n/z66W2Va5bqZHXqOZVYZ0SWW3CvGVg8802jSJWRH15SYWUC", Fs: "os"}, + &confpar.Access{ + User: "sha512crypt", + Pass: "$6$NM1oYAm3MGJY/TZa$srQwlvW1g7isQFL5.ECpxYo6sK4DNQXwlEmmIdQVbAGS2LJ6zb8ACrpMRGzWx4i9r1p886sT7EY" + + "Ncva7Ypsc/1", + Fs: "os", + }, + &confpar.Access{User: "anonymous", Pass: "*", Fs: "os"}, + ) + + for _, tc := range []struct { + user, pass string + ok bool + }{ + {"plain", "plain", true}, + {"plain", "wrong", false}, + {"bcrypt", "bcrypt", true}, + {"bcrypt", "wrong", false}, + {"sha512crypt", "sha512crypt", true}, + {"sha512crypt", "wrong", false}, + {"anonymous", "anything", true}, + {"unknown", "plain", false}, + } { + access, err := conf.GetAccess(tc.user, tc.pass) + + if tc.ok { + if err != nil || access == nil || access.User != tc.user { + t.Errorf("%s/%s: expected access, got %v / %v", tc.user, tc.pass, access, err) + } + } else if !errors.Is(err, ErrUnknownUser) { + t.Errorf("%s/%s: expected ErrUnknownUser, got %v / %v", tc.user, tc.pass, access, err) + } + } +} + +// TestGetAccessKeycloakDoesNotShareState makes sure concurrent keycloak logins don't overwrite each other's +// credentials, which could otherwise make a user end up in another user's directory. +func TestGetAccessKeycloakDoesNotShareState(t *testing.T) { + shared := &confpar.Access{Fs: "keycloak", Params: map[string]string{"base_path": "/ftp"}} + conf := newTestConfig(t, shared) + + var wg sync.WaitGroup + + for _, user := range []string{"alice", "bob", "carol", "dave"} { + wg.Add(1) + + go func() { + defer wg.Done() + + access, err := conf.GetAccess(user, user+"-pass") + if err != nil { + t.Errorf("GetAccess: %v", err) + + return + } + + if access.User != user || access.Pass != user+"-pass" { + t.Errorf("got access for %s/%s instead of %s", access.User, access.Pass, user) + } + }() + } + + wg.Wait() + + if shared.User != "" || shared.Pass != "" { + t.Errorf("the shared access was modified: %s/%s", shared.User, shared.Pass) + } +} + +func TestHashPlaintextPasswords(t *testing.T) { + fileName := filepath.Join(t.TempDir(), "ftpserver.json") + content := `{ + "hash_plaintext_passwords": true, + "accesses": [ + {"user": "plain", "pass": "secret", "fs": "os", "params": {"basePath": "/tmp"}}, + {"user": "anonymous", "pass": "*", "fs": "os", "params": {"basePath": "/tmp"}}, + {"user": "bcrypt", "pass": "$2b$05$aco32n/z66W2Va5bqZHXqOZVYZ0SWW3CvGVg8802jSJWRH15SYWUC", "fs": "os"} + ] +}` + + if err := os.WriteFile(fileName, []byte(content), 0o600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + conf, err := NewConfig(fileName, slog.Default()) + if err != nil { + t.Fatalf("NewConfig: %v", err) + } + + if !strings.HasPrefix(conf.Content.Accesses[0].Pass, "$2") { + t.Errorf("plain-text password wasn't hashed: %s", conf.Content.Accesses[0].Pass) + } + + if conf.Content.Accesses[1].Pass != "*" { + t.Errorf("anonymous password shouldn't be hashed: %s", conf.Content.Accesses[1].Pass) + } + + if _, err := conf.GetAccess("plain", "secret"); err != nil { + t.Errorf("couldn't authenticate with the hashed password: %v", err) + } + + // The saved file must be hashed too, and keep its restrictive permissions + saved, err := os.ReadFile(fileName) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + if strings.Contains(string(saved), `"secret"`) { + t.Error("the saved config file still contains the plain-text password") + } + + info, err := os.Stat(fileName) + if err != nil { + t.Fatalf("Stat: %v", err) + } + + if perm := info.Mode().Perm(); perm != 0o600 { + t.Errorf("config file permissions changed to %o", perm) + } +} diff --git a/config/confpar/confpar.go b/config/confpar/confpar.go index 5a1e085e..4eb4a98d 100644 --- a/config/confpar/confpar.go +++ b/config/confpar/confpar.go @@ -21,11 +21,11 @@ type Access struct { // AccessesWebhook defines an optional webhook to get user's access type AccessesWebhook struct { URL string `json:"url"` // URL to call - Headers map[string]string `json:"headers"` // Token to use in the + Headers map[string]string `json:"headers"` // Headers to add to the request Timeout Duration `json:"timeout"` // Max time request can take } -// SyncAndDelete provides +// SyncAndDelete provides a local temporary directory whose files are moved to the actual file system type SyncAndDelete struct { Enable bool `json:"enable"` // Instant write Directory string `json:"directory"` // Directory @@ -67,15 +67,15 @@ type Content struct { ListenAddress string `json:"listen_address"` // Address to listen on PublicHost string `json:"public_host"` // Public host to listen on MaxClients int `json:"max_clients"` // Maximum clients who can connect - HashPlaintextPasswords bool `json:"hash_plaintext_passwords"` // Overwrite plain-text passwords with hashed equivalents - IdleTimeout Duration `json:"idle_timeout"` // Maximum idle time for client connections + HashPlaintextPasswords bool `json:"hash_plaintext_passwords"` // Hash plain-text passwords in the file + IdleTimeout Duration `json:"idle_timeout"` // Max idle time for client connections Accesses []*Access `json:"accesses"` // Accesses offered to users PassiveTransferPortRange *PortRange `json:"passive_transfer_port_range"` // Listen port range Extensions Extensions `json:"extensions"` // Extended features Logging Logging `json:"logging"` // Logging parameters TLS *TLS `json:"tls"` // TLS Config - TLSRequired string `json:"tls_required"` - AccessesWebhook *AccessesWebhook `json:"accesses_webhook"` // Webhook to call when accesses are updated + TLSRequired string `json:"tls_required"` // TLS requirement level + AccessesWebhook *AccessesWebhook `json:"accesses_webhook"` // Webhook authenticating users } // Duration wraps time.Duration to allow unmarshaling from JSON strings @@ -84,10 +84,12 @@ type Duration struct { time.Duration } +// MarshalJSON encodes the duration as a Go duration string func (d *Duration) MarshalJSON() ([]byte, error) { return json.Marshal(d.String()) } +// UnmarshalJSON decodes a Go duration string (e.g. "5m", "30s", "1h") func (d *Duration) UnmarshalJSON(b []byte) (err error) { var s string if err = json.Unmarshal(b, &s); err == nil { diff --git a/docker-compose.yml b/docker-compose.yml index d7f7fa8d..9e8fa523 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,8 @@ -version: '3.3' services: ftpserver: + image: fclairamb/ftpserver ports: - '2121-2130:2121-2130' volumes: - ./files:/tmp - .:/app - image: fclairamb/ftpserver - diff --git a/fs/README.md b/fs/README.md index 860f9a48..1c7c7412 100644 --- a/fs/README.md +++ b/fs/README.md @@ -1,94 +1,78 @@ -# File system loading +# Common access options -On top of the supported file system and their associated config, -they are few parameters that can be enabled. +On top of the backend (`fs`) and its `params`, every access supports a few options. ## Read-only -This blocks any writing call to the underlying file system. It can be abled by the `read_only` boolean parameter. + +The `read_only` boolean blocks any write to the underlying file system. ```json { - "version": 1, - "accesses": [ - { - "read_only": true, - - // The usual FS config: - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/tmp" - } - } - ] + "user": "test", + "pass": "test", + "fs": "os", + "read_only": true, + "params": { + "basePath": "/tmp" + } } -``` +``` ## Shared -This makes sure the underlying filesystem is loaded once per use. It can be enabled by the `shared` boolean parameter. + +The `shared` boolean makes the backend instance created at the first login of the user reused by all their later +sessions, instead of creating a new one for each session. This is useful for backends with an expensive setup (SFTP +connection, Telegram bot...). Shared instances are dropped when the configuration is reloaded. ```json { - "version": 1, - "accesses": [ - { - "shared": true, - // The usual FS config: - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/tmp" - } - } - ] + "user": "test", + "pass": "test", + "fs": "os", + "shared": true, + "params": { + "basePath": "/tmp" + } } -``` +``` ## Sync & Delete -This creates a temporary local folder that will be replicated to a destination file system. +The `sync_and_delete` option makes uploaded files written to a local temporary directory first, and then moved to the +backend once they are complete (see [afero-snd](https://github.com/fclairamb/afero-snd)). The `directory` is optional, +a system temporary directory is used when it isn't specified. ```json { - "version": 1, - "accesses": [ - { - "sync_and_delete": { - "enable": true, // To enable it - "directory": "/tmp/dir" // Temporary directory (optional) - }, - // The usual FS config: - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/target" - } - } - ] + "user": "test", + "pass": "test", + "fs": "os", + "sync_and_delete": { + "enable": true, + "directory": "/tmp/snd" + }, + "params": { + "basePath": "/target" + } } -``` +``` ## Logging + +The `logging` option enables the logging of the FTP exchanges and/or the file accesses of this access only. The same +options exist at the server level to enable them for all the accesses. + ```json { - "version": 1, - "accesses": [ - { - "logging": { - "file_accesses": true, // This logs every single file access - "ftp_exchanges": true // This logs every single FTP command - }, - // The usual FS config: - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/target" - } - } - ] + "user": "test", + "pass": "test", + "fs": "os", + "logging": { + "ftp_exchanges": true, + "file_accesses": true + }, + "params": { + "basePath": "/target" + } } -``` \ No newline at end of file +``` diff --git a/fs/dropbox/README.md b/fs/dropbox/README.md index 75cd1e70..8c7224cf 100644 --- a/fs/dropbox/README.md +++ b/fs/dropbox/README.md @@ -1,32 +1,31 @@ # Dropbox backend ## Parameters -Possible parameters are: -- `token`: The non-expiring authentication token to use + +- `token`: The Dropbox access token to use. If it isn't set, the `DROPBOX_TOKEN` environment variable is used. ## How to use it + ### Create an app -You can follow -[these instructions](https://medium.com/swlh/google-drive-api-with-python-part-i-set-up-credentials-1f729cb0372b). -### Use the created client credentials -You can either define the `google_client_id` and `google_client_secret` access params or specify the `GOOGLE_CLIENT_ID` -and `GOOGLE_CLIENT_SECRET` environment variables. +1. Create an app in the [Dropbox App Console](https://www.dropbox.com/developers/apps), with the access type you + need ("App folder" restricts the FTP server to a dedicated folder, "Full Dropbox" gives access to everything). +2. In the "Permissions" tab, enable at least `files.metadata.read`, `files.content.read` and `files.content.write`. +3. In the "Settings" tab, generate an access token. + +### Configure the access -You should have a config file looking like this one: ```json { "accesses": [ - { + { "user": "dropbox", "pass": "dropbox", "fs": "dropbox", "params": { - "google_client_id": "*****.apps.googleusercontent.com", - "google_client_secret": "*****" + "token": "sl.*****" } } ] } ``` - diff --git a/fs/fs.go b/fs/fs.go index 73cacf6c..bc9121fe 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -62,7 +62,7 @@ func LoadFs(access *confpar.Access, logger *slog.Logger) (afero.Fs, error) { } // If we're defining a dubious behavior, we can use it - if err == nil && access != nil && access.SyncAndDelete != nil && access.SyncAndDelete.Enable { + if err == nil && access.SyncAndDelete != nil && access.SyncAndDelete.Enable { var temp afero.Fs if access.SyncAndDelete.Directory != "" { diff --git a/fs/gcs/README.md b/fs/gcs/README.md index dc333845..b7c45dd7 100644 --- a/fs/gcs/README.md +++ b/fs/gcs/README.md @@ -1,6 +1,7 @@ # Google Cloud Storage (GCS) filesystem -This directory contains the Google Cloud Storage implementation for the FTP server using the official `github.com/spf13/afero/gcsfs` package. +This backend gives access to a Google Cloud Storage bucket, using afero's +[gcsfs](https://github.com/spf13/afero/tree/master/gcsfs) package. ## Configuration @@ -12,11 +13,10 @@ The GCS filesystem supports the following configuration parameters: "accesses": [ { "user": "test", - "pass": "test", + "pass": "test", "fs": "gcs", "params": { "bucket": "your-gcs-bucket-name", - "project_id": "your-project-id", "key_file": "/path/to/service-account-key.json" } } @@ -26,9 +26,10 @@ The GCS filesystem supports the following configuration parameters: ### Parameters -- `bucket` (required): The GCS bucket name to use -- `project_id` (optional): Google Cloud Project ID. If not provided, it will use the default project from the environment or service account -- `key_file` (optional): Path to a service account key JSON file for authentication. If not provided, it will use default credentials (environment variables, metadata service, etc.) +- `bucket` (required): The GCS bucket name to use. It is the root directory of the FTP access. +- `key_file` (optional): Path to a service account key JSON file for authentication. If not provided, the default + credentials are used (environment variables, metadata service, etc.) +- `project_id`: Not used (the bucket name is enough to access it), it is accepted for backward compatibility. ### Authentication @@ -48,7 +49,6 @@ The GCS filesystem supports multiple authentication methods: "fs": "gcs", "params": { "bucket": "my-ftp-bucket", - "project_id": "my-project-123", "key_file": "/etc/gcs-key.json" } } @@ -57,7 +57,7 @@ The GCS filesystem supports multiple authentication methods: #### With default credentials: ```json { - "fs": "gcs", + "fs": "gcs", "params": { "bucket": "my-ftp-bucket" } @@ -68,4 +68,4 @@ The GCS filesystem supports multiple authentication methods: - GCS doesn't have real directories, so directory operations are emulated - File permissions and ownership operations are no-ops since GCS doesn't support them -- Seeking to end of file and some advanced file operations are not supported due to GCS limitations \ No newline at end of file +- Seeking to end of file and some advanced file operations are not supported due to GCS limitations diff --git a/fs/gcs/gcs.go b/fs/gcs/gcs.go index e3c9adad..8db8a623 100644 --- a/fs/gcs/gcs.go +++ b/fs/gcs/gcs.go @@ -14,36 +14,35 @@ import ( "github.com/fclairamb/ftpserver/config/confpar" ) +// ErrMissingBucket is returned when the bucket parameter isn't specified +var ErrMissingBucket = errors.New("bucket parameter is required for GCS") + // LoadFs loads a GCS file system from an access description func LoadFs(access *confpar.Access) (afero.Fs, error) { bucket := access.Params["bucket"] if bucket == "" { - return nil, errors.New("bucket parameter is required for GCS") + return nil, ErrMissingBucket } - projectID := access.Params["project_id"] - keyFile := access.Params["key_file"] - ctx := context.Background() - var client *storage.Client - var err error - - // Create client with different authentication methods - if keyFile != "" { - // Use service account key file - client, err = storage.NewClient(ctx, option.WithCredentialsFile(keyFile)) - } else if projectID != "" { - // Use default credentials with specific project ID - client, err = storage.NewClient(ctx, option.WithScopes(storage.ScopeFullControl)) - } else { - // Use default credentials - client, err = storage.NewClient(ctx) + + var opts []option.ClientOption + + if keyFile := access.Params["key_file"]; keyFile != "" { + // Use a service account key file. Without it, the application default credentials are used. + opts = append(opts, option.WithAuthCredentialsFile(option.ServiceAccount, keyFile)) } + client, err := storage.NewClient(ctx, opts...) if err != nil { return nil, fmt.Errorf("failed to create GCS client: %w", err) } - return gcsfs.NewGcsFSFromClient(ctx, client) -} + gcsFs, err := gcsfs.NewGcsFSFromClient(ctx, client) + if err != nil { + return nil, err + } + // gcsfs uses the first path component as the bucket name, each access is restricted to its bucket + return afero.NewBasePathFs(gcsFs, "/"+bucket), nil +} diff --git a/fs/gdrive/README.md b/fs/gdrive/README.md index 4816e923..f73d1e26 100644 --- a/fs/gdrive/README.md +++ b/fs/gdrive/README.md @@ -1,36 +1,38 @@ # Google Drive backend ## Parameters -Possible parameters are: -- `google_client_id`: The Google Client ID -- `google_client_secret`: The Google Client Secret, it shall never be shared -- `token_file`: The path of the token file that will be created -- `base_path`: The base path of the Google Drive folder -_Note:_ The token file will be updated regularly (every day). It's important that the server can write the token file -after its initial creation. +- `google_client_id`: The Google client ID. If it isn't set, the `GOOGLE_CLIENT_ID` environment variable is used. +- `google_client_secret`: The Google client secret, it shall never be shared. If it isn't set, the + `GOOGLE_CLIENT_SECRET` environment variable is used. +- `token_file` (optional): The path of the token file that will be created. Defaults to `gdrive_token_.json`. +- `base_path` (optional): The Google Drive folder to use as the root directory. Environment variables (`$NAME`) are + replaced by their value. + +_Note:_ The token file is updated regularly (every day). The server must be able to write the token file after its +initial creation. ## How to use it + ### Create an app -You can follow -[these instructions](https://medium.com/swlh/google-drive-api-with-python-part-i-set-up-credentials-1f729cb0372b). -### Use the created client credentials -You can either define the `google_client_id` and `google_client_secret` access params or specify the `GOOGLE_CLIENT_ID` -and `GOOGLE_CLIENT_SECRET` environment variables. +1. In the [Google Cloud console](https://console.cloud.google.com/), create a project and enable the Google Drive API. +2. Configure the OAuth consent screen. +3. Create an "OAuth client ID" credential of the "Desktop app" type, and note its client ID and client secret. + +### Configure the access -You should have a config file looking like this one: ```json { "accesses": [ - { + { "user": "gdrive", "pass": "gdrive", "fs": "gdrive", "params": { "google_client_id": "*****.apps.googleusercontent.com", "google_client_secret": "*****", - "basePath": "/ftp" + "base_path": "ftp" } } ] @@ -38,8 +40,7 @@ You should have a config file looking like this one: ``` ### Connect for the first time -You should run the ftpserver in interactive mode. -The first time you will login with the "gdrive" user, the server will ask you to open a link, wich -will authorize the app you just created and return a code. This code will allow the server to create -a token file (named `gdrive_token_$username.json` by default). +Run the server in interactive mode. The first time you log in with the "gdrive" user, the server prints a link +that you need to open to authorize the app you just created. It gives you a code that you need to type in the +server's console. The server then creates the token file, and won't ask you again. diff --git a/fs/keycloak/README.md b/fs/keycloak/README.md index c3aacfe9..be2946e9 100644 --- a/fs/keycloak/README.md +++ b/fs/keycloak/README.md @@ -1,16 +1,37 @@ +# Keycloak backend + +This backend serves local directories to users authenticated by a [Keycloak](https://www.keycloak.org/) server. +Each user gets their own `/` directory. + +## Parameters + +- `keycloak_url`: The URL of the Keycloak server. +- `keycloak_realm`: The realm of the users. +- `keycloak_client_id`: The client ID. If it isn't set, the `KEYCLOAK_CLIENT_ID` environment variable is used. +- `keycloak_client_secret`: The client secret. If it isn't set, the `KEYCLOAK_CLIENT_SECRET` environment variable is + used. +- `base_path`: The local directory containing the users' directories. Environment variables (`$NAME`) are replaced by + their value. + +The client must allow the "Direct Access Grants" (resource owner password credentials) flow, which is how the +credentials sent by the FTP clients are checked. + +## Configuration + +A keycloak access doesn't have any `user` or `pass`, it accepts any user that Keycloak authenticates. As accesses are +evaluated in their declaration order, it should be declared last. -You should have a config file looking like this one: ```json { "accesses": [ - { + { "fs": "keycloak", "params": { - "keycloak_client_id": "*****", - "keycloak_client_secret": "*****", "keycloak_url": "http://localhost:8080", "keycloak_realm": "master", - "base_path": "/ftp" + "keycloak_client_id": "ftpserver", + "keycloak_client_secret": "*****", + "base_path": "/srv/ftp" } } ] diff --git a/fs/keycloak/keycloak.go b/fs/keycloak/keycloak.go index 6ecb9e4d..0f2dffe4 100644 --- a/fs/keycloak/keycloak.go +++ b/fs/keycloak/keycloak.go @@ -1,55 +1,80 @@ +// Package keycloak provides a local file system access layer whose users are authenticated by Keycloak package keycloak import ( "context" "errors" + "fmt" "os" "path" + "strings" "github.com/Nerzal/gocloak/v14" + "github.com/spf13/afero" + "github.com/fclairamb/ftpserver/config/confpar" "github.com/fclairamb/ftpserver/fs/utils" - "github.com/spf13/afero" ) -type KeycloakAuthenticator struct { - client gocloak.GoCloak +// Authenticator checks users credentials against a Keycloak server +type Authenticator struct { + client *gocloak.GoCloak clientID string secret string realm string } -// CheckPasswd -func (a *KeycloakAuthenticator) CheckPasswd(user, pass string) (bool, error) { - ctx := context.Background() - _, err := a.client.Login(ctx, a.clientID, a.secret, a.realm, user, pass) - if err != nil { +// KeycloakAuthenticator is the former name of Authenticator +// +// Deprecated: use Authenticator instead. +type KeycloakAuthenticator = Authenticator //nolint:revive // kept for backward compatibility + +// CheckPasswd checks the user's credentials against Keycloak +func (a *Authenticator) CheckPasswd(user, pass string) (bool, error) { + if _, err := a.client.Login(context.Background(), a.clientID, a.secret, a.realm, user, pass); err != nil { return false, err } + return true, nil } -// ErrMissingBasePath is triggered when the basePath property isn't specified -var ErrMissingBasePath = errors.New("basePath must be specified") -var ErrLogin = errors.New("login failed, User or Password error") -var ErrMissingKeycloakClientCredentials = errors.New("missing the keycloak client credentials") +var ( + // ErrMissingBasePath is triggered when the base_path property isn't specified + ErrMissingBasePath = errors.New("base_path must be specified") -// LoadFs loads a file system from an access description -func LoadFs(access *confpar.Access) (afero.Fs, error) { + // ErrLogin is returned when the user couldn't be authenticated by Keycloak + ErrLogin = errors.New("login failed, User or Password error") - keycloakClientID := access.Params["keycloak_client_id"] - keycloakClientSecret := access.Params["keycloak_client_secret"] - keycloakURL := access.Params["keycloak_url"] - keycloakRealm := access.Params["keycloak_realm"] + // ErrMissingKeycloakClientCredentials is returned when the keycloak parameters are incomplete + ErrMissingKeycloakClientCredentials = errors.New("missing the keycloak client credentials") + + // ErrInvalidUserName is returned when the user name can't be used as a directory name + ErrInvalidUserName = errors.New("invalid user name") +) - if keycloakClientID == "" { - keycloakClientID = os.Getenv("KEYCLOAK_CLIENT_ID") +// getParam returns a parameter, or the first non-empty of the fallback environment variables +func getParam(access *confpar.Access, name string, envVars ...string) string { + if value := access.Params[name]; value != "" { + return value } - if keycloakClientSecret == "" { - keycloakClientSecret = os.Getenv("KEYCLAOK_CLIENT_SECRET") + for _, envVar := range envVars { + if value := os.Getenv(envVar); value != "" { + return value + } } + return "" +} + +// LoadFs loads a file system from an access description +func LoadFs(access *confpar.Access) (afero.Fs, error) { + keycloakClientID := getParam(access, "keycloak_client_id", "KEYCLOAK_CLIENT_ID") + // KEYCLAOK_CLIENT_SECRET is a typo that was supported by previous versions, we keep it for backward compatibility + keycloakClientSecret := getParam(access, "keycloak_client_secret", "KEYCLOAK_CLIENT_SECRET", "KEYCLAOK_CLIENT_SECRET") + keycloakURL := access.Params["keycloak_url"] + keycloakRealm := access.Params["keycloak_realm"] + if keycloakClientID == "" || keycloakClientSecret == "" || keycloakURL == "" || keycloakRealm == "" { return nil, ErrMissingKeycloakClientCredentials } @@ -59,20 +84,23 @@ func LoadFs(access *confpar.Access) (afero.Fs, error) { return nil, ErrMissingBasePath } - // Keycloak Authenticator - auth := &KeycloakAuthenticator{ - client: *gocloak.NewClient(keycloakURL), + // The user name is used as a directory name, it must not allow to escape the base path + if access.User == "" || access.User == "." || access.User == ".." || strings.ContainsAny(access.User, `/\`) { + return nil, fmt.Errorf("%w: %q", ErrInvalidUserName, access.User) + } + + auth := &Authenticator{ + client: gocloak.NewClient(keycloakURL), clientID: keycloakClientID, secret: keycloakClientSecret, realm: keycloakRealm, } - valid, _ := auth.CheckPasswd(access.User, access.Pass) - if !valid { + if valid, _ := auth.CheckPasswd(access.User, access.Pass); !valid { return nil, ErrLogin } - basePath = utils.ReplaceEnvVars(path.Join(basePath, access.User)) + basePath = path.Join(utils.ReplaceEnvVars(basePath), access.User) return afero.NewBasePathFs(afero.NewOsFs(), basePath), nil } diff --git a/fs/mail/README.md b/fs/mail/README.md new file mode 100644 index 00000000..47fe67af --- /dev/null +++ b/fs/mail/README.md @@ -0,0 +1,43 @@ +# Mail backend + +This is a write-only backend: each uploaded file is sent as an email attachment. Listing directories always returns +an empty result. + +## Parameters + +- `Host`: The SMTP server. +- `Port` (optional): The SMTP port. Defaults to `25`. +- `SSL` (optional): `true` to connect with SSL/TLS (SMTPS). Defaults to `true` on port `465`, `false` otherwise. +- `StartTLSPolicy` (optional): `MandatoryStartTLS` (default), `OpportunisticStartTLS` or `NoStartTLS`. +- `Username`, `Password` (optional): The SMTP credentials. +- `Localname` (optional): The host name sent in the `HELO` command. +- `From`: The sender address. +- `To`: The recipient address. +- `Subject`: The subject of the emails. +- `Message`: The body of the emails, `%s` is replaced by the file name. + +## Configuration + +See [example.json](example.json): + +```json +{ + "accesses": [ + { + "user": "test", + "pass": "test", + "fs": "mail", + "params": { + "From": "alice@example.com", + "To": "bob@example.com", + "Subject": "New File", + "Message": "A new file has been uploaded: %s", + "Host": "smtp.example.com", + "Port": "465", + "Username": "smtpuser", + "Password": "smtppassword" + } + } + ] +} +``` diff --git a/fs/mail/example.conf b/fs/mail/example.json similarity index 78% rename from fs/mail/example.conf rename to fs/mail/example.json index 7192c953..b16f03c2 100644 --- a/fs/mail/example.conf +++ b/fs/mail/example.json @@ -1,4 +1,5 @@ { + "$schema": "https://raw.githubusercontent.com/fclairamb/ftpserver/main/config-schema.json", "version": 1, "accesses": [ { @@ -13,7 +14,7 @@ "Host": "smtp.example.com", "Port": "465", "Username": "smtpuser", - "Password": "smtppassword", + "Password": "smtppassword" } } ], diff --git a/fs/mail/mail.go b/fs/mail/mail.go index 6211b142..f1a21342 100644 --- a/fs/mail/mail.go +++ b/fs/mail/mail.go @@ -42,33 +42,48 @@ type File struct { At int64 } +// defaultPort is the SMTP port used when none is specified +const defaultPort = 25 + +// sslPort is the SMTPS port, on which SSL is enabled by default +const sslPort = 465 + // LoadFs loads a file system from an access description func LoadFs(access *confpar.Access) (afero.Fs, error) { - port, err := strconv.Atoi(access.Params["Port"]) - if err != nil { - return nil, err - } + port := defaultPort + + if portStr := access.Params["Port"]; portStr != "" { + var err error + if port, err = strconv.Atoi(portStr); err != nil { + return nil, fmt.Errorf("%w: Port: %w", ErrInvalidParameter, err) + } - if port < 1 || port > 65535 { - port = 25 + if port < 1 || port > 65535 { + port = defaultPort + } } - ssl, err2 := strconv.ParseBool(access.Params["SSL"]) - if err2 != nil { - return nil, err2 + // SSL is enabled by default on the SMTPS port + ssl := port == sslPort + + if sslStr := access.Params["SSL"]; sslStr != "" { + var err error + if ssl, err = strconv.ParseBool(sslStr); err != nil { + return nil, fmt.Errorf("%w: SSL: %w", ErrInvalidParameter, err) + } } var starttlspolicy mail.StartTLSPolicy switch access.Params["StartTLSPolicy"] { + case "", "MandatoryStartTLS": + starttlspolicy = mail.MandatoryStartTLS case "OpportunisticStartTLS": starttlspolicy = mail.OpportunisticStartTLS - case "MandatoryStartTLS": - starttlspolicy = mail.MandatoryStartTLS case "NoStartTLS": starttlspolicy = mail.NoStartTLS default: - return nil, ErrInvalidParameter + return nil, fmt.Errorf("%w: StartTLSPolicy: %q", ErrInvalidParameter, access.Params["StartTLSPolicy"]) } f := &Fs{ diff --git a/fs/s3/s3.go b/fs/s3/s3.go index a2ef4350..f88e3975 100644 --- a/fs/s3/s3.go +++ b/fs/s3/s3.go @@ -48,13 +48,7 @@ func LoadFs(access *confpar.Access) (afero.Fs, error) { var s3Opts []func(*s3.Options) if endpoint != "" { - endpointURL := endpoint - if disableSSL { - // Ensure HTTP scheme if SSL is disabled - if len(endpointURL) > 0 && endpointURL[0] != 'h' { - endpointURL = "http://" + endpointURL - } - } + endpointURL := endpointWithScheme(endpoint, disableSSL) s3Opts = append(s3Opts, func(o *s3.Options) { o.BaseEndpoint = aws.String(endpointURL) }) @@ -81,3 +75,17 @@ func LoadFs(access *confpar.Access) (afero.Fs, error) { return fs, nil } + +// endpointWithScheme adds the URL scheme to an endpoint that doesn't have one: "http" if SSL is disabled, +// "https" otherwise. An endpoint that already has a scheme is returned as is. +func endpointWithScheme(endpoint string, disableSSL bool) string { + if strings.Contains(endpoint, "://") { + return endpoint + } + + if disableSSL { + return "http://" + endpoint + } + + return "https://" + endpoint +} diff --git a/fs/s3/s3_test.go b/fs/s3/s3_test.go new file mode 100644 index 00000000..f9d0f39f --- /dev/null +++ b/fs/s3/s3_test.go @@ -0,0 +1,22 @@ +package s3 + +import "testing" + +func TestEndpointWithScheme(t *testing.T) { + for _, tc := range []struct { + endpoint string + disableSSL bool + want string + }{ + {"minio:9000", true, "http://minio:9000"}, + {"minio:9000", false, "https://minio:9000"}, + // Hostnames starting with an "h" used to be considered as already having a scheme + {"host.example.com:9000", true, "http://host.example.com:9000"}, + {"http://minio:9000", false, "http://minio:9000"}, + {"https://s3.amazonaws.com", true, "https://s3.amazonaws.com"}, + } { + if got := endpointWithScheme(tc.endpoint, tc.disableSSL); got != tc.want { + t.Errorf("endpointWithScheme(%q, %v) = %q, want %q", tc.endpoint, tc.disableSSL, got, tc.want) + } + } +} diff --git a/fs/sftp/README.md b/fs/sftp/README.md new file mode 100644 index 00000000..ad0b015b --- /dev/null +++ b/fs/sftp/README.md @@ -0,0 +1,47 @@ +# SFTP backend + +This backend connects to an upstream SSH/SFTP server. + +## Parameters + +- `hostname`: The upstream server address, with its port (e.g. `192.168.1.10:22`). +- `username`: The SSH user. +- `password`: The SSH password. +- One of the host key verification parameters below. + +## Host key verification + +The upstream server's host key is verified to protect against man-in-the-middle attacks. You must configure exactly +one of the following parameters: + +- `known_hosts`: path to an OpenSSH `known_hosts` file containing the upstream server's key + (e.g. generated with `ssh-keyscan -H host >> known_hosts`). +- `host_key`: the upstream server's public key pinned inline, in `authorized_keys` / `known_hosts` line format + (e.g. `ssh-ed25519 AAAA...`). +- `insecure_ignore_host_key`: set to `"true"` to disable host key verification entirely. This is **unsafe** (it + exposes the connection to man-in-the-middle attacks) and should only be used in trusted networks or for testing. + +If none of these is set, the connection is refused. + +## Configuration + +```json +{ + "accesses": [ + { + "user": "sftp", + "pass": "sftp", + "fs": "sftp", + "shared": true, + "params": { + "hostname": "192.168.1.10:22", + "username": "user", + "password": "password", + "known_hosts": "/home/ftpserver/.ssh/known_hosts" + } + } + ] +} +``` + +With `shared` enabled, a single SSH connection is used for all the FTP sessions of this access. diff --git a/fs/telegram/example.conf b/fs/telegram/example.json similarity index 84% rename from fs/telegram/example.conf rename to fs/telegram/example.json index 54c8ddc2..7a07a048 100644 --- a/fs/telegram/example.conf +++ b/fs/telegram/example.json @@ -1,4 +1,5 @@ { + "$schema": "https://raw.githubusercontent.com/fclairamb/ftpserver/main/config-schema.json", "version": 1, "accesses": [ { @@ -15,7 +16,6 @@ "RetryDelay": "2000", "PartUploadDelay": "500" } - } ], "passive_transfer_port_range": { diff --git a/fs/telegram/file_info.go b/fs/telegram/file_info.go index 8b95387f..e80798b2 100644 --- a/fs/telegram/file_info.go +++ b/fs/telegram/file_info.go @@ -16,20 +16,30 @@ type FileData struct { content []byte } +// FileInfo implements the os.FileInfo interface on top of FileData type FileInfo struct { *FileData } -// Implements os.FileInfo +// Name returns the base name of the file func (s *FileInfo) Name() string { _, name := filepath.Split(s.name) return name } -func (s *FileInfo) Mode() os.FileMode { return s.mode } +// Mode returns the file mode bits +func (s *FileInfo) Mode() os.FileMode { return s.mode } + +// ModTime returns the modification time func (s *FileInfo) ModTime() time.Time { return s.modtime } -func (s *FileInfo) IsDir() bool { return s.dir } -func (s *FileInfo) Sys() interface{} { return nil } + +// IsDir tells if the file is a directory +func (s *FileInfo) IsDir() bool { return s.dir } + +// Sys returns the underlying data source (always nil) +func (s *FileInfo) Sys() any { return nil } + +// Size returns the size of the file func (s *FileInfo) Size() int64 { if s.IsDir() { return int64(42) diff --git a/fs/telegram/telegram.go b/fs/telegram/telegram.go index a421ae51..61d622bd 100644 --- a/fs/telegram/telegram.go +++ b/fs/telegram/telegram.go @@ -15,12 +15,11 @@ import ( "sync/atomic" "time" - tele "gopkg.in/telebot.v3" - "github.com/spf13/afero" + tele "gopkg.in/telebot.v3" + "gopkg.in/telebot.v3/middleware" "github.com/fclairamb/ftpserver/config/confpar" - "gopkg.in/telebot.v3/middleware" ) // ErrNotImplemented is returned when something is not implemented @@ -88,7 +87,9 @@ type File struct { var imageExtensions = []string{".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif"} // videoExtensions is the list of supported video extensions -var videoExtensions = []string{".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".mpeg", ".mpg", ".m4v", ".3gp", ".3g2"} +var videoExtensions = []string{ + ".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".mpeg", ".mpg", ".m4v", ".3gp", ".3g2", +} // textExtensions is the list of supported text extensions var textExtensions = []string{".txt", ".md"} @@ -98,7 +99,6 @@ var audioExtensions = []string{".mp3", ".ogg", ".flac", ".wav", ".m4a", ".opus"} // LoadFs loads a file system from an access description func LoadFs(access *confpar.Access, logger *slog.Logger) (afero.Fs, error) { - token := access.Params["Token"] if token == "" { return nil, fmt.Errorf("parameter Token is empty") @@ -106,7 +106,7 @@ func LoadFs(access *confpar.Access, logger *slog.Logger) (afero.Fs, error) { chatID, err := strconv.ParseInt(access.Params["ChatID"], 10, 64) if err != nil { - return nil, fmt.Errorf("invalid ChatID parameter: %v", err) + return nil, fmt.Errorf("invalid ChatID parameter: %w", err) } // Parse MaxPartSize param, default to defaultMaxPartSize if not set @@ -115,7 +115,7 @@ func LoadFs(access *confpar.Access, logger *slog.Logger) (afero.Fs, error) { if partSizeStr != "" { parsed, parseErr := strconv.ParseInt(partSizeStr, 10, 64) if parseErr != nil { - return nil, fmt.Errorf("invalid MaxPartSize parameter: %v", parseErr) + return nil, fmt.Errorf("invalid MaxPartSize parameter: %w", parseErr) } if parsed <= 0 { return nil, fmt.Errorf("invalid MaxPartSize parameter: must be > 0") @@ -239,7 +239,7 @@ func (f *File) Close() error { partFilename := fmt.Sprintf("%s.part%dof%d", basePath, partNum, totalParts) partCaption := fmt.Sprintf("%s (part %d/%d)", basePath, partNum, totalParts) - partData, readErr := os.ReadFile(tempPath) + partData, readErr := os.ReadFile(tempPath) //nolint:gosec // temporary file created by ourselves if readErr != nil { return readErr } @@ -260,7 +260,6 @@ func (f *File) Close() error { time.Sleep(f.Fs.PartUploadDelay) } } - } if f.Fs.fakeFs.stat(f.Path) == nil { @@ -371,7 +370,15 @@ func (f *File) sendWithRetry(filename string, caption string, send func() error) return err } - f.Fs.Logger.Warn("telegram send retry", "attempt", attempt, "maxAttempts", f.Fs.RetryAttempts, "path", f.Path, "filename", filename, "caption", caption, "err", err) + f.Fs.Logger.Warn( + "telegram send retry", + "attempt", attempt, + "maxAttempts", f.Fs.RetryAttempts, + "path", f.Path, + "filename", filename, + "caption", caption, + "err", err, + ) time.Sleep(f.Fs.RetryDelay) } @@ -402,7 +409,10 @@ func (f *File) spillNextPartToDisk(data []byte) error { tempFilePath := filepath.Join(f.PartTempDir, fmt.Sprintf("part-%05d.bin", f.PartNumber)) if err := os.WriteFile(tempFilePath, data, 0o600); err != nil { - f.Fs.Logger.Error("telegram spillNextPartToDisk WriteFile", "err", err, "tempFilePath", tempFilePath, "bytes", len(data)) + f.Fs.Logger.Error( + "telegram spillNextPartToDisk WriteFile", + "err", err, "tempFilePath", tempFilePath, "bytes", len(data), + ) return err } @@ -550,7 +560,7 @@ func (m *Fs) Remove(name string) error { return nil } -// Mkdir +// Mkdir creates a directory func (m *Fs) Mkdir(name string, mode os.FileMode) error { m.fakeFs.mkdir(name, mode) return nil @@ -618,7 +628,7 @@ func (m *Fs) OpenFile(name string, flag int, mode os.FileMode) (afero.File, erro return &File{Path: name, Fs: m, Content: content}, nil } -// Stat() fake implementation +// Stat returns the file information func (m *Fs) Stat(name string) (os.FileInfo, error) { fileInfo, _ := m.statPath(name) diff --git a/fs/utils/env.go b/fs/utils/env.go index 960df97c..cab4b3bd 100644 --- a/fs/utils/env.go +++ b/fs/utils/env.go @@ -1,3 +1,4 @@ +// Package utils provides helpers shared by the file systems package utils import ( @@ -5,10 +6,9 @@ import ( "regexp" ) -// Replace all environment variables in a string by their actual values - var envVar = regexp.MustCompile(`\$[a-zA-Z0-9_]+`) +// ReplaceEnvVars replaces all the environment variables ($NAME) in a string by their actual values func ReplaceEnvVars(s string) string { return envVar.ReplaceAllStringFunc(s, func(s string) string { return os.Getenv(s[1:]) diff --git a/local-docker-test.sh b/local-docker-test.sh index c82ebb10..d042625f 100755 --- a/local-docker-test.sh +++ b/local-docker-test.sh @@ -1,12 +1,15 @@ #!/bin/sh -ex +# Builds the docker image and checks that a file can be uploaded and downloaded back -docker rm -f ftpserver ||: +docker rm -f ftpserver-test || : -GOOS=linux GOARCH=amd64 go build -docker build . -t test -if [ ! -f kitty.jpg ]; then - curl -o kitty.jpg.tmp https://placekitten.com/2048/2048 && mv kitty.jpg.tmp kitty.jpg -fi -docker run --name ftpserver -p 2121-2130:2121-2130 test & +docker build . -t ftpserver-test +docker run -d --rm --name ftpserver-test -p 2121-2130:2121-2130 ftpserver-test +trap 'docker rm -f ftpserver-test' EXIT + +head -c 2097152 /dev/urandom > local.bin.tmp while ! nc -z localhost 2121 0 && int64(s.nbClients) > int64(maxClients) { + s.logger.Warn("Too many clients, refusing connection", "clientId", cc.ID(), "maxClients", maxClients) + + return "Too many clients, try again later", ErrTooManyClients + } + if s.config.Content.Logging.FtpExchanges { cc.SetDebug(true) } @@ -126,7 +154,7 @@ func (s *Server) ClientConnected(cc serverlib.ClientContext) (string, error) { return "ftpserver", nil } -// ClientDisconnected is called when the user disconnects, even if he never authenticated +// ClientDisconnected is called when the user disconnects, even if they never authenticated func (s *Server) ClientDisconnected(cc serverlib.ClientContext) { s.nbClientsSync.Lock() defer s.nbClientsSync.Unlock() @@ -154,10 +182,17 @@ func (s *Server) Stop() { func (s *Server) WaitGracefully(timeout time.Duration) error { s.logger.Info("Waiting for last client to disconnect...") - defer func() { s.zeroClientEvent = nil }() + s.nbClientsSync.Lock() + zeroClientEvent := s.zeroClientEvent + s.nbClientsSync.Unlock() + + if zeroClientEvent == nil { + // Stop wasn't called, there's nothing to wait for + return nil + } select { - case err := <-s.zeroClientEvent: + case err := <-zeroClientEvent: return err case <-time.After(timeout): return ErrTimeout @@ -204,43 +239,44 @@ func (s *Server) getAccessFromWebhook(user, pass string) (*confpar.Access, error return nil, err } + webhook := s.config.Content.AccessesWebhook + + timeout := webhook.Timeout.Duration + if timeout <= 0 { + timeout = defaultWebhookTimeout + } + // Timeout is implemented with context termination - ctx, cancel := context.WithTimeout(context.Background(), s.config.Content.AccessesWebhook.Timeout.Duration) + ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() // Create a new HTTP request - req, err := http.NewRequestWithContext(ctx, "POST", s.config.Content.AccessesWebhook.URL, bytes.NewBuffer(jsonData)) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, webhook.URL, bytes.NewReader(jsonData)) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/json") - for key, value := range s.config.Content.AccessesWebhook.Headers { + + for key, value := range webhook.Headers { req.Header.Set(key, value) } // Execute the request - client := &http.Client{} - resp, err := client.Do(req) + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } + defer func() { _ = resp.Body.Close() }() // Check the response status if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - - // Return the response - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err + return nil, fmt.Errorf("%w: %d", ErrWebhookStatus, resp.StatusCode) } access := new(confpar.Access) - err = json.Unmarshal(body, &access) - if err != nil { + if err := json.NewDecoder(io.LimitReader(resp.Body, maxWebhookResponseSize)).Decode(access); err != nil { return nil, err } @@ -330,7 +366,7 @@ func (s *Server) loadTLSConfig() (*tls.Config, error) { keyBytes, errReadFileKey := os.ReadFile(serverCert.Key) if errReadFileKey != nil { - return nil, fmt.Errorf("could not load key file: %s: %w", serverCert.Cert, errReadFileCert) + return nil, fmt.Errorf("could not load key file: %s: %w", serverCert.Key, errReadFileKey) } keypair, errKeyPair := tls.X509KeyPair(certBytes, keyBytes) diff --git a/server/server_internal_test.go b/server/server_internal_test.go new file mode 100644 index 00000000..c91370af --- /dev/null +++ b/server/server_internal_test.go @@ -0,0 +1,127 @@ +package server + +import ( + "bufio" + "encoding/json" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + serverlib "github.com/fclairamb/ftpserverlib" + + "github.com/fclairamb/ftpserver/config" + "github.com/fclairamb/ftpserver/config/confpar" +) + +func newTestServer(t *testing.T, content *confpar.Content) *Server { + t.Helper() + + conf, err := config.FromContent(content, "test.json", slog.Default()) + if err != nil { + t.Fatalf("FromContent: %v", err) + } + + srv, err := NewServer(conf, slog.Default()) + if err != nil { + t.Fatalf("NewServer: %v", err) + } + + return srv +} + +// TestWebhookWithoutTimeout makes sure a webhook without timeout works (it used to time out instantly) +func TestWebhookWithoutTimeout(t *testing.T) { + webhook := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req map[string]string + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + w.WriteHeader(http.StatusBadRequest) + + return + } + + if r.Header.Get("Authorization") != "Bearer token" || req["pass"] != "secret" { + w.WriteHeader(http.StatusForbidden) + + return + } + + _ = json.NewEncoder(w).Encode(confpar.Access{ + User: req["user"], + Fs: "os", + Params: map[string]string{"basePath": "/tmp"}, + }) + })) + defer webhook.Close() + + srv := newTestServer(t, &confpar.Content{ + AccessesWebhook: &confpar.AccessesWebhook{ + URL: webhook.URL, + Headers: map[string]string{"Authorization": "Bearer token"}, + }, + }) + + access, err := srv.getAccessFromWebhook("alice", "secret") + if err != nil { + t.Fatalf("getAccessFromWebhook: %v", err) + } + + if access.User != "alice" || access.Fs != "os" || access.Params["basePath"] != "/tmp" { + t.Fatalf("unexpected access: %+v", access) + } + + if _, err := srv.getAccessFromWebhook("alice", "wrong"); err == nil { + t.Fatal("expected an error for a refused authentication") + } +} + +func readBanner(t *testing.T, addr string) (net.Conn, string) { + t.Helper() + + conn, err := net.DialTimeout("tcp", addr, 5*time.Second) + if err != nil { + t.Fatalf("Dial: %v", err) + } + + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + + line, err := bufio.NewReader(conn).ReadString('\n') + if err != nil { + t.Fatalf("couldn't read banner: %v", err) + } + + return conn, line +} + +func TestMaxClients(t *testing.T) { + srv := newTestServer(t, &confpar.Content{ + ListenAddress: "127.0.0.1:0", + MaxClients: 1, + }) + + ftpServer := serverlib.NewFtpServer(srv) + if err := ftpServer.Listen(); err != nil { + t.Fatalf("Listen: %v", err) + } + + go func() { _ = ftpServer.Serve() }() + + defer func() { _ = ftpServer.Stop() }() + + first, banner := readBanner(t, ftpServer.Addr()) + defer func() { _ = first.Close() }() + + if !strings.HasPrefix(banner, "220") { + t.Fatalf("first client should be accepted, got: %q", banner) + } + + second, banner := readBanner(t, ftpServer.Addr()) + defer func() { _ = second.Close() }() + + if !strings.Contains(banner, "Too many clients") { + t.Fatalf("second client should be refused, got: %q", banner) + } +}