From 8f6755797c086d410908b2744a614a0a8e3f0efb Mon Sep 17 00:00:00 2001 From: fclairamb Date: Thu, 22 Oct 2026 07:57:16 +0200 Subject: [PATCH 1/4] fix: authentication, backend and server lifecycle bugs - keycloak: work on a copy of the shared access instead of writing each user's credentials into it (concurrent logins could end up in another user's directory), reject user names that would escape base_path, accept the KEYCLOAK_CLIENT_SECRET env var (KEYCLAOK_CLIENT_SECRET still works) - config: deduplicate the password hash handling, compare plain-text passwords in constant time, keep the config file permissions when hashing passwords and don't hash keycloak accesses - server: enforce max_clients, default the accesses webhook timeout to 30s (an unset timeout made every login fail), limit the webhook response size, report the right file when the TLS key can't be read, drop shared file systems on config reload, fix a data race in WaitGracefully - main: exit with a non-zero status on startup errors, don't wait a minute when the server couldn't listen - s3: add the right scheme to endpoints without one (hosts starting with an "h" were considered to already have one) - gcs: restrict the access to the configured bucket, which was ignored - mail: SSL, StartTLSPolicy and Port are now optional Co-Authored-By: Claude Opus 5.5 --- config/config.go | 287 ++++++++++++--------------------- config/config_test.go | 148 +++++++++++++++++ config/confpar/confpar.go | 14 +- fs/fs.go | 2 +- fs/gcs/gcs.go | 37 +++-- fs/keycloak/keycloak.go | 84 ++++++---- fs/mail/mail.go | 39 +++-- fs/s3/s3.go | 22 ++- fs/s3/s3_test.go | 22 +++ fs/telegram/file_info.go | 18 ++- fs/telegram/telegram.go | 36 +++-- fs/utils/env.go | 4 +- main.go | 71 ++++---- server/server.go | 82 +++++++--- server/server_internal_test.go | 127 +++++++++++++++ 15 files changed, 666 insertions(+), 327 deletions(-) create mode 100644 config/config_test.go create mode 100644 fs/s3/s3_test.go create mode 100644 server/server_internal_test.go diff --git a/config/config.go b/config/config.go index 22076a25..b46d1efe 100644 --- a/config/config.go +++ b/config/config.go @@ -2,24 +2,37 @@ package config import ( - "bytes" + "crypto/subtle" "encoding/json" "errors" "fmt" "log/slog" "os" - - "github.com/fclairamb/ftpserver/config/confpar" - "github.com/fclairamb/ftpserver/fs" + "strings" + "sync" "github.com/go-crypt/crypt" + "github.com/go-crypt/crypt/algorithm" "github.com/go-crypt/crypt/algorithm/bcrypt" "github.com/tidwall/sjson" + + "github.com/fclairamb/ftpserver/config/confpar" + "github.com/fclairamb/ftpserver/fs" ) // ErrUnknownUser is returned when the provided user cannot be identified through our authentication mechanism var ErrUnknownUser = errors.New("unknown user") +// hashedPasswordPrefixes lists the prefixes of the password hash formats we support (md5crypt, bcrypt variants, +// sha256crypt and sha512crypt). A password without any of these prefixes is considered to be plain-text. +var hashedPasswordPrefixes = []string{"$1$", "$2$", "$2a$", "$2b$", "$2x$", "$2y$", "$5$", "$6$"} + +// bcryptCost is the cost used when hashing plain-text passwords +const bcryptCost = 10 + +// passwordDecoder is shared between all the authentication requests, it's safe for concurrent use +var passwordDecoder = sync.OnceValues(crypt.NewDecoderAll) + // Config provides the general server config type Config struct { fileName string @@ -96,71 +109,75 @@ func (c *Config) Load() error { return c.Prepare() } +// isHashedPassword tells if a password is stored in one of the supported hashed formats +func isHashedPassword(pass string) bool { + for _, prefix := range hashedPasswordPrefixes { + if strings.HasPrefix(pass, prefix) { + return true + } + } + + return false +} + +// HashPlaintextPasswords replaces all the plain-text passwords of the config file by their bcrypt equivalent, +// and saves the config file. func (c *Config) HashPlaintextPasswords() error { - json, errReadFile := os.ReadFile(c.fileName) + fileInfo, errStat := os.Stat(c.fileName) + if errStat != nil { + c.logger.Error("Cannot stat config file", "err", errStat) + + return errStat + } + + content, errReadFile := os.ReadFile(c.fileName) if errReadFile != nil { - c.logger.Error("Cannot read config file!", "err", errReadFile) + c.logger.Error("Cannot read config file", "err", errReadFile) + return errReadFile } + var hasher algorithm.Hash + save := false + for i, a := range c.Content.Accesses { - if a.User == "anonymous" && a.Pass == "*" { + if a.Fs == "keycloak" || (a.User == "anonymous" && a.Pass == "*") || isHashedPassword(a.Pass) { continue } - switch true { - case bytes.HasPrefix([]byte(a.Pass), []byte("$1$")): - //This user's password is md5crypt - continue - case bytes.HasPrefix([]byte(a.Pass), []byte("$2$")): - //This user's password is bcrypt - continue - case bytes.HasPrefix([]byte(a.Pass), []byte("$2a$")): - //This user's password is bcrypt-a - continue - case bytes.HasPrefix([]byte(a.Pass), []byte("$2b$")): - //This user's password is bcrypt-b - continue - case bytes.HasPrefix([]byte(a.Pass), []byte("$2x$")): - //This user's password is bcrypt-x - continue - case bytes.HasPrefix([]byte(a.Pass), []byte("$2y$")): - //This user's password is bcrypt-y - continue - case bytes.HasPrefix([]byte(a.Pass), []byte("$5$")): - //This user's password is sha256crypt - continue - case bytes.HasPrefix([]byte(a.Pass), []byte("$6$")): - //This user's password is sha512crypt - continue - default: - //This password is not hashed - hasher, err := bcrypt.New(bcrypt.WithCost(10)) - if err != nil { + if hasher == nil { + var err error + if hasher, err = bcrypt.New(bcrypt.WithCost(bcryptCost)); err != nil { return err } + } - digest, err := hasher.Hash(a.Pass) - if err != nil { - return err - } + digest, err := hasher.Hash(a.Pass) + if err != nil { + return err + } - modified, errJsonSet := sjson.Set(string(json), "accesses."+fmt.Sprint(i)+".pass", digest.Encode()) - c.Content.Accesses[i].Pass = digest.Encode() - if errJsonSet == nil { - save = true - json = []byte(modified) - } + modified, errJSONSet := sjson.Set(string(content), fmt.Sprintf("accesses.%d.pass", i), digest.Encode()) + if errJSONSet != nil { + return fmt.Errorf("could not update the password of access %d: %w", i, errJSONSet) } + + a.Pass = digest.Encode() + content = []byte(modified) + save = true } + if save { - errWriteFile := os.WriteFile(c.fileName, json, 0644) - if errWriteFile != nil { - c.logger.Error("Cannot write config file!", "err", errWriteFile) + // We keep the original file permissions as the file usually contains secrets + //nolint:gosec // the file name comes from the command line + if errWriteFile := os.WriteFile(c.fileName, content, fileInfo.Mode().Perm()); errWriteFile != nil { + c.logger.Error("Cannot write config file", "err", errWriteFile) + return errWriteFile } } + return nil } @@ -192,150 +209,54 @@ func (c *Config) CheckAccesses() error { return nil } -// GetAccess return a file system access given some credentials -func (c *Config) GetAccess(user string, pass string) (*confpar.Access, error) { - decoder, err := crypt.NewDecoderAll() +// checkPassword checks a password against its stored (plain-text or hashed) version +func checkPassword(stored, pass string) (bool, error) { + if !isHashedPassword(stored) { + return subtle.ConstantTimeCompare([]byte(stored), []byte(pass)) == 1, nil + } + + decoder, err := passwordDecoder() if err != nil { - return nil, err + return false, err } + digest, err := decoder.Decode(stored) + if err != nil { + return false, err + } + + return digest.MatchAdvanced(pass) +} + +// GetAccess return a file system access given some credentials. Accesses are evaluated in their declaration order. +func (c *Config) GetAccess(user string, pass string) (*confpar.Access, error) { for _, a := range c.Content.Accesses { if a.Fs == "keycloak" { - a.User = user - a.Pass = pass + // The credentials are checked by keycloak when loading the file system. We work on a copy because + // the access is shared between all the connections. + access := *a + access.User = user + access.Pass = pass + + return &access, nil + } + + if a.User != user { + continue + } + + if a.User == "anonymous" && a.Pass == "*" { return a, nil } - if a.User == user { - switch true { - case bytes.HasPrefix([]byte(a.Pass), []byte("$1$")): - //This user's password is md5crypt - digest, err := decoder.Decode(a.Pass) - if err != nil { - return nil, err - } - - ok, err := digest.MatchAdvanced(pass) - if err != nil { - return nil, err - } - - if ok { - return a, nil - } - case bytes.HasPrefix([]byte(a.Pass), []byte("$2$")): - //This user's password is bcrypt - digest, err := decoder.Decode(a.Pass) - if err != nil { - return nil, err - } - - ok, err := digest.MatchAdvanced(pass) - if err != nil { - return nil, err - } - - if ok { - return a, nil - } - case bytes.HasPrefix([]byte(a.Pass), []byte("$2a$")): - //This user's password is bcrypt-a - digest, err := decoder.Decode(a.Pass) - if err != nil { - return nil, err - } - - ok, err := digest.MatchAdvanced(pass) - if err != nil { - return nil, err - } - - if ok { - return a, nil - } - case bytes.HasPrefix([]byte(a.Pass), []byte("$2b$")): - //This user's password is bcrypt-b - digest, err := decoder.Decode(a.Pass) - if err != nil { - return nil, err - } - - ok, err := digest.MatchAdvanced(pass) - if err != nil { - return nil, err - } - - if ok { - return a, nil - } - case bytes.HasPrefix([]byte(a.Pass), []byte("$2x$")): - //This user's password is bcrypt-x - digest, err := decoder.Decode(a.Pass) - if err != nil { - return nil, err - } - - ok, err := digest.MatchAdvanced(pass) - if err != nil { - return nil, err - } - - if ok { - return a, nil - } - case bytes.HasPrefix([]byte(a.Pass), []byte("$2y$")): - //This user's password is bcrypt-y - digest, err := decoder.Decode(a.Pass) - if err != nil { - return nil, err - } - - ok, err := digest.MatchAdvanced(pass) - if err != nil { - return nil, err - } - - if ok { - return a, nil - } - case bytes.HasPrefix([]byte(a.Pass), []byte("$5$")): - //This user's password is sha256crypt - digest, err := decoder.Decode(a.Pass) - if err != nil { - return nil, err - } - - ok, err := digest.MatchAdvanced(pass) - if err != nil { - return nil, err - } - - if ok { - return a, nil - } - case bytes.HasPrefix([]byte(a.Pass), []byte("$6$")): - //This user's password is sha512crypt - digest, err := decoder.Decode(a.Pass) - if err != nil { - return nil, err - } - - ok, err := digest.MatchAdvanced(pass) - if err != nil { - return nil, err - } - - if ok { - return a, nil - } - default: - //This user's password is plain-text - if a.Pass == pass || (a.User == "anonymous" && a.Pass == "*") { - return a, nil - } - } + ok, err := checkPassword(a.Pass, pass) + if err != nil { + return nil, err } + if ok { + return a, nil + } } return nil, ErrUnknownUser diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 00000000..62f96e8f --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,148 @@ +package config + +import ( + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/fclairamb/ftpserver/config/confpar" +) + +func newTestConfig(t *testing.T, accesses ...*confpar.Access) *Config { + t.Helper() + + conf, err := FromContent(&confpar.Content{Accesses: accesses}, "test.json", slog.Default()) + if err != nil { + t.Fatalf("FromContent: %v", err) + } + + return conf +} + +func TestGetAccess(t *testing.T) { + conf := newTestConfig(t, + &confpar.Access{User: "plain", Pass: "plain", Fs: "os"}, + &confpar.Access{User: "bcrypt", Pass: "$2b$05$aco32n/z66W2Va5bqZHXqOZVYZ0SWW3CvGVg8802jSJWRH15SYWUC", Fs: "os"}, + &confpar.Access{ + User: "sha512crypt", + Pass: "$6$NM1oYAm3MGJY/TZa$srQwlvW1g7isQFL5.ECpxYo6sK4DNQXwlEmmIdQVbAGS2LJ6zb8ACrpMRGzWx4i9r1p886sT7EY" + + "Ncva7Ypsc/1", + Fs: "os", + }, + &confpar.Access{User: "anonymous", Pass: "*", Fs: "os"}, + ) + + for _, tc := range []struct { + user, pass string + ok bool + }{ + {"plain", "plain", true}, + {"plain", "wrong", false}, + {"bcrypt", "bcrypt", true}, + {"bcrypt", "wrong", false}, + {"sha512crypt", "sha512crypt", true}, + {"sha512crypt", "wrong", false}, + {"anonymous", "anything", true}, + {"unknown", "plain", false}, + } { + access, err := conf.GetAccess(tc.user, tc.pass) + + if tc.ok { + if err != nil || access == nil || access.User != tc.user { + t.Errorf("%s/%s: expected access, got %v / %v", tc.user, tc.pass, access, err) + } + } else if !errors.Is(err, ErrUnknownUser) { + t.Errorf("%s/%s: expected ErrUnknownUser, got %v / %v", tc.user, tc.pass, access, err) + } + } +} + +// TestGetAccessKeycloakDoesNotShareState makes sure concurrent keycloak logins don't overwrite each other's +// credentials, which could otherwise make a user end up in another user's directory. +func TestGetAccessKeycloakDoesNotShareState(t *testing.T) { + shared := &confpar.Access{Fs: "keycloak", Params: map[string]string{"base_path": "/ftp"}} + conf := newTestConfig(t, shared) + + var wg sync.WaitGroup + + for _, user := range []string{"alice", "bob", "carol", "dave"} { + wg.Add(1) + + go func() { + defer wg.Done() + + access, err := conf.GetAccess(user, user+"-pass") + if err != nil { + t.Errorf("GetAccess: %v", err) + + return + } + + if access.User != user || access.Pass != user+"-pass" { + t.Errorf("got access for %s/%s instead of %s", access.User, access.Pass, user) + } + }() + } + + wg.Wait() + + if shared.User != "" || shared.Pass != "" { + t.Errorf("the shared access was modified: %s/%s", shared.User, shared.Pass) + } +} + +func TestHashPlaintextPasswords(t *testing.T) { + fileName := filepath.Join(t.TempDir(), "ftpserver.json") + content := `{ + "hash_plaintext_passwords": true, + "accesses": [ + {"user": "plain", "pass": "secret", "fs": "os", "params": {"basePath": "/tmp"}}, + {"user": "anonymous", "pass": "*", "fs": "os", "params": {"basePath": "/tmp"}}, + {"user": "bcrypt", "pass": "$2b$05$aco32n/z66W2Va5bqZHXqOZVYZ0SWW3CvGVg8802jSJWRH15SYWUC", "fs": "os"} + ] +}` + + if err := os.WriteFile(fileName, []byte(content), 0o600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + conf, err := NewConfig(fileName, slog.Default()) + if err != nil { + t.Fatalf("NewConfig: %v", err) + } + + if !strings.HasPrefix(conf.Content.Accesses[0].Pass, "$2") { + t.Errorf("plain-text password wasn't hashed: %s", conf.Content.Accesses[0].Pass) + } + + if conf.Content.Accesses[1].Pass != "*" { + t.Errorf("anonymous password shouldn't be hashed: %s", conf.Content.Accesses[1].Pass) + } + + if _, err := conf.GetAccess("plain", "secret"); err != nil { + t.Errorf("couldn't authenticate with the hashed password: %v", err) + } + + // The saved file must be hashed too, and keep its restrictive permissions + saved, err := os.ReadFile(fileName) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + if strings.Contains(string(saved), `"secret"`) { + t.Error("the saved config file still contains the plain-text password") + } + + info, err := os.Stat(fileName) + if err != nil { + t.Fatalf("Stat: %v", err) + } + + if perm := info.Mode().Perm(); perm != 0o600 { + t.Errorf("config file permissions changed to %o", perm) + } +} diff --git a/config/confpar/confpar.go b/config/confpar/confpar.go index 5a1e085e..4eb4a98d 100644 --- a/config/confpar/confpar.go +++ b/config/confpar/confpar.go @@ -21,11 +21,11 @@ type Access struct { // AccessesWebhook defines an optional webhook to get user's access type AccessesWebhook struct { URL string `json:"url"` // URL to call - Headers map[string]string `json:"headers"` // Token to use in the + Headers map[string]string `json:"headers"` // Headers to add to the request Timeout Duration `json:"timeout"` // Max time request can take } -// SyncAndDelete provides +// SyncAndDelete provides a local temporary directory whose files are moved to the actual file system type SyncAndDelete struct { Enable bool `json:"enable"` // Instant write Directory string `json:"directory"` // Directory @@ -67,15 +67,15 @@ type Content struct { ListenAddress string `json:"listen_address"` // Address to listen on PublicHost string `json:"public_host"` // Public host to listen on MaxClients int `json:"max_clients"` // Maximum clients who can connect - HashPlaintextPasswords bool `json:"hash_plaintext_passwords"` // Overwrite plain-text passwords with hashed equivalents - IdleTimeout Duration `json:"idle_timeout"` // Maximum idle time for client connections + HashPlaintextPasswords bool `json:"hash_plaintext_passwords"` // Hash plain-text passwords in the file + IdleTimeout Duration `json:"idle_timeout"` // Max idle time for client connections Accesses []*Access `json:"accesses"` // Accesses offered to users PassiveTransferPortRange *PortRange `json:"passive_transfer_port_range"` // Listen port range Extensions Extensions `json:"extensions"` // Extended features Logging Logging `json:"logging"` // Logging parameters TLS *TLS `json:"tls"` // TLS Config - TLSRequired string `json:"tls_required"` - AccessesWebhook *AccessesWebhook `json:"accesses_webhook"` // Webhook to call when accesses are updated + TLSRequired string `json:"tls_required"` // TLS requirement level + AccessesWebhook *AccessesWebhook `json:"accesses_webhook"` // Webhook authenticating users } // Duration wraps time.Duration to allow unmarshaling from JSON strings @@ -84,10 +84,12 @@ type Duration struct { time.Duration } +// MarshalJSON encodes the duration as a Go duration string func (d *Duration) MarshalJSON() ([]byte, error) { return json.Marshal(d.String()) } +// UnmarshalJSON decodes a Go duration string (e.g. "5m", "30s", "1h") func (d *Duration) UnmarshalJSON(b []byte) (err error) { var s string if err = json.Unmarshal(b, &s); err == nil { diff --git a/fs/fs.go b/fs/fs.go index 73cacf6c..bc9121fe 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -62,7 +62,7 @@ func LoadFs(access *confpar.Access, logger *slog.Logger) (afero.Fs, error) { } // If we're defining a dubious behavior, we can use it - if err == nil && access != nil && access.SyncAndDelete != nil && access.SyncAndDelete.Enable { + if err == nil && access.SyncAndDelete != nil && access.SyncAndDelete.Enable { var temp afero.Fs if access.SyncAndDelete.Directory != "" { diff --git a/fs/gcs/gcs.go b/fs/gcs/gcs.go index e3c9adad..8db8a623 100644 --- a/fs/gcs/gcs.go +++ b/fs/gcs/gcs.go @@ -14,36 +14,35 @@ import ( "github.com/fclairamb/ftpserver/config/confpar" ) +// ErrMissingBucket is returned when the bucket parameter isn't specified +var ErrMissingBucket = errors.New("bucket parameter is required for GCS") + // LoadFs loads a GCS file system from an access description func LoadFs(access *confpar.Access) (afero.Fs, error) { bucket := access.Params["bucket"] if bucket == "" { - return nil, errors.New("bucket parameter is required for GCS") + return nil, ErrMissingBucket } - projectID := access.Params["project_id"] - keyFile := access.Params["key_file"] - ctx := context.Background() - var client *storage.Client - var err error - - // Create client with different authentication methods - if keyFile != "" { - // Use service account key file - client, err = storage.NewClient(ctx, option.WithCredentialsFile(keyFile)) - } else if projectID != "" { - // Use default credentials with specific project ID - client, err = storage.NewClient(ctx, option.WithScopes(storage.ScopeFullControl)) - } else { - // Use default credentials - client, err = storage.NewClient(ctx) + + var opts []option.ClientOption + + if keyFile := access.Params["key_file"]; keyFile != "" { + // Use a service account key file. Without it, the application default credentials are used. + opts = append(opts, option.WithAuthCredentialsFile(option.ServiceAccount, keyFile)) } + client, err := storage.NewClient(ctx, opts...) if err != nil { return nil, fmt.Errorf("failed to create GCS client: %w", err) } - return gcsfs.NewGcsFSFromClient(ctx, client) -} + gcsFs, err := gcsfs.NewGcsFSFromClient(ctx, client) + if err != nil { + return nil, err + } + // gcsfs uses the first path component as the bucket name, each access is restricted to its bucket + return afero.NewBasePathFs(gcsFs, "/"+bucket), nil +} diff --git a/fs/keycloak/keycloak.go b/fs/keycloak/keycloak.go index 6ecb9e4d..0f2dffe4 100644 --- a/fs/keycloak/keycloak.go +++ b/fs/keycloak/keycloak.go @@ -1,55 +1,80 @@ +// Package keycloak provides a local file system access layer whose users are authenticated by Keycloak package keycloak import ( "context" "errors" + "fmt" "os" "path" + "strings" "github.com/Nerzal/gocloak/v14" + "github.com/spf13/afero" + "github.com/fclairamb/ftpserver/config/confpar" "github.com/fclairamb/ftpserver/fs/utils" - "github.com/spf13/afero" ) -type KeycloakAuthenticator struct { - client gocloak.GoCloak +// Authenticator checks users credentials against a Keycloak server +type Authenticator struct { + client *gocloak.GoCloak clientID string secret string realm string } -// CheckPasswd -func (a *KeycloakAuthenticator) CheckPasswd(user, pass string) (bool, error) { - ctx := context.Background() - _, err := a.client.Login(ctx, a.clientID, a.secret, a.realm, user, pass) - if err != nil { +// KeycloakAuthenticator is the former name of Authenticator +// +// Deprecated: use Authenticator instead. +type KeycloakAuthenticator = Authenticator //nolint:revive // kept for backward compatibility + +// CheckPasswd checks the user's credentials against Keycloak +func (a *Authenticator) CheckPasswd(user, pass string) (bool, error) { + if _, err := a.client.Login(context.Background(), a.clientID, a.secret, a.realm, user, pass); err != nil { return false, err } + return true, nil } -// ErrMissingBasePath is triggered when the basePath property isn't specified -var ErrMissingBasePath = errors.New("basePath must be specified") -var ErrLogin = errors.New("login failed, User or Password error") -var ErrMissingKeycloakClientCredentials = errors.New("missing the keycloak client credentials") +var ( + // ErrMissingBasePath is triggered when the base_path property isn't specified + ErrMissingBasePath = errors.New("base_path must be specified") -// LoadFs loads a file system from an access description -func LoadFs(access *confpar.Access) (afero.Fs, error) { + // ErrLogin is returned when the user couldn't be authenticated by Keycloak + ErrLogin = errors.New("login failed, User or Password error") - keycloakClientID := access.Params["keycloak_client_id"] - keycloakClientSecret := access.Params["keycloak_client_secret"] - keycloakURL := access.Params["keycloak_url"] - keycloakRealm := access.Params["keycloak_realm"] + // ErrMissingKeycloakClientCredentials is returned when the keycloak parameters are incomplete + ErrMissingKeycloakClientCredentials = errors.New("missing the keycloak client credentials") + + // ErrInvalidUserName is returned when the user name can't be used as a directory name + ErrInvalidUserName = errors.New("invalid user name") +) - if keycloakClientID == "" { - keycloakClientID = os.Getenv("KEYCLOAK_CLIENT_ID") +// getParam returns a parameter, or the first non-empty of the fallback environment variables +func getParam(access *confpar.Access, name string, envVars ...string) string { + if value := access.Params[name]; value != "" { + return value } - if keycloakClientSecret == "" { - keycloakClientSecret = os.Getenv("KEYCLAOK_CLIENT_SECRET") + for _, envVar := range envVars { + if value := os.Getenv(envVar); value != "" { + return value + } } + return "" +} + +// LoadFs loads a file system from an access description +func LoadFs(access *confpar.Access) (afero.Fs, error) { + keycloakClientID := getParam(access, "keycloak_client_id", "KEYCLOAK_CLIENT_ID") + // KEYCLAOK_CLIENT_SECRET is a typo that was supported by previous versions, we keep it for backward compatibility + keycloakClientSecret := getParam(access, "keycloak_client_secret", "KEYCLOAK_CLIENT_SECRET", "KEYCLAOK_CLIENT_SECRET") + keycloakURL := access.Params["keycloak_url"] + keycloakRealm := access.Params["keycloak_realm"] + if keycloakClientID == "" || keycloakClientSecret == "" || keycloakURL == "" || keycloakRealm == "" { return nil, ErrMissingKeycloakClientCredentials } @@ -59,20 +84,23 @@ func LoadFs(access *confpar.Access) (afero.Fs, error) { return nil, ErrMissingBasePath } - // Keycloak Authenticator - auth := &KeycloakAuthenticator{ - client: *gocloak.NewClient(keycloakURL), + // The user name is used as a directory name, it must not allow to escape the base path + if access.User == "" || access.User == "." || access.User == ".." || strings.ContainsAny(access.User, `/\`) { + return nil, fmt.Errorf("%w: %q", ErrInvalidUserName, access.User) + } + + auth := &Authenticator{ + client: gocloak.NewClient(keycloakURL), clientID: keycloakClientID, secret: keycloakClientSecret, realm: keycloakRealm, } - valid, _ := auth.CheckPasswd(access.User, access.Pass) - if !valid { + if valid, _ := auth.CheckPasswd(access.User, access.Pass); !valid { return nil, ErrLogin } - basePath = utils.ReplaceEnvVars(path.Join(basePath, access.User)) + basePath = path.Join(utils.ReplaceEnvVars(basePath), access.User) return afero.NewBasePathFs(afero.NewOsFs(), basePath), nil } diff --git a/fs/mail/mail.go b/fs/mail/mail.go index 6211b142..f1a21342 100644 --- a/fs/mail/mail.go +++ b/fs/mail/mail.go @@ -42,33 +42,48 @@ type File struct { At int64 } +// defaultPort is the SMTP port used when none is specified +const defaultPort = 25 + +// sslPort is the SMTPS port, on which SSL is enabled by default +const sslPort = 465 + // LoadFs loads a file system from an access description func LoadFs(access *confpar.Access) (afero.Fs, error) { - port, err := strconv.Atoi(access.Params["Port"]) - if err != nil { - return nil, err - } + port := defaultPort + + if portStr := access.Params["Port"]; portStr != "" { + var err error + if port, err = strconv.Atoi(portStr); err != nil { + return nil, fmt.Errorf("%w: Port: %w", ErrInvalidParameter, err) + } - if port < 1 || port > 65535 { - port = 25 + if port < 1 || port > 65535 { + port = defaultPort + } } - ssl, err2 := strconv.ParseBool(access.Params["SSL"]) - if err2 != nil { - return nil, err2 + // SSL is enabled by default on the SMTPS port + ssl := port == sslPort + + if sslStr := access.Params["SSL"]; sslStr != "" { + var err error + if ssl, err = strconv.ParseBool(sslStr); err != nil { + return nil, fmt.Errorf("%w: SSL: %w", ErrInvalidParameter, err) + } } var starttlspolicy mail.StartTLSPolicy switch access.Params["StartTLSPolicy"] { + case "", "MandatoryStartTLS": + starttlspolicy = mail.MandatoryStartTLS case "OpportunisticStartTLS": starttlspolicy = mail.OpportunisticStartTLS - case "MandatoryStartTLS": - starttlspolicy = mail.MandatoryStartTLS case "NoStartTLS": starttlspolicy = mail.NoStartTLS default: - return nil, ErrInvalidParameter + return nil, fmt.Errorf("%w: StartTLSPolicy: %q", ErrInvalidParameter, access.Params["StartTLSPolicy"]) } f := &Fs{ diff --git a/fs/s3/s3.go b/fs/s3/s3.go index a2ef4350..f88e3975 100644 --- a/fs/s3/s3.go +++ b/fs/s3/s3.go @@ -48,13 +48,7 @@ func LoadFs(access *confpar.Access) (afero.Fs, error) { var s3Opts []func(*s3.Options) if endpoint != "" { - endpointURL := endpoint - if disableSSL { - // Ensure HTTP scheme if SSL is disabled - if len(endpointURL) > 0 && endpointURL[0] != 'h' { - endpointURL = "http://" + endpointURL - } - } + endpointURL := endpointWithScheme(endpoint, disableSSL) s3Opts = append(s3Opts, func(o *s3.Options) { o.BaseEndpoint = aws.String(endpointURL) }) @@ -81,3 +75,17 @@ func LoadFs(access *confpar.Access) (afero.Fs, error) { return fs, nil } + +// endpointWithScheme adds the URL scheme to an endpoint that doesn't have one: "http" if SSL is disabled, +// "https" otherwise. An endpoint that already has a scheme is returned as is. +func endpointWithScheme(endpoint string, disableSSL bool) string { + if strings.Contains(endpoint, "://") { + return endpoint + } + + if disableSSL { + return "http://" + endpoint + } + + return "https://" + endpoint +} diff --git a/fs/s3/s3_test.go b/fs/s3/s3_test.go new file mode 100644 index 00000000..f9d0f39f --- /dev/null +++ b/fs/s3/s3_test.go @@ -0,0 +1,22 @@ +package s3 + +import "testing" + +func TestEndpointWithScheme(t *testing.T) { + for _, tc := range []struct { + endpoint string + disableSSL bool + want string + }{ + {"minio:9000", true, "http://minio:9000"}, + {"minio:9000", false, "https://minio:9000"}, + // Hostnames starting with an "h" used to be considered as already having a scheme + {"host.example.com:9000", true, "http://host.example.com:9000"}, + {"http://minio:9000", false, "http://minio:9000"}, + {"https://s3.amazonaws.com", true, "https://s3.amazonaws.com"}, + } { + if got := endpointWithScheme(tc.endpoint, tc.disableSSL); got != tc.want { + t.Errorf("endpointWithScheme(%q, %v) = %q, want %q", tc.endpoint, tc.disableSSL, got, tc.want) + } + } +} diff --git a/fs/telegram/file_info.go b/fs/telegram/file_info.go index 8b95387f..e80798b2 100644 --- a/fs/telegram/file_info.go +++ b/fs/telegram/file_info.go @@ -16,20 +16,30 @@ type FileData struct { content []byte } +// FileInfo implements the os.FileInfo interface on top of FileData type FileInfo struct { *FileData } -// Implements os.FileInfo +// Name returns the base name of the file func (s *FileInfo) Name() string { _, name := filepath.Split(s.name) return name } -func (s *FileInfo) Mode() os.FileMode { return s.mode } +// Mode returns the file mode bits +func (s *FileInfo) Mode() os.FileMode { return s.mode } + +// ModTime returns the modification time func (s *FileInfo) ModTime() time.Time { return s.modtime } -func (s *FileInfo) IsDir() bool { return s.dir } -func (s *FileInfo) Sys() interface{} { return nil } + +// IsDir tells if the file is a directory +func (s *FileInfo) IsDir() bool { return s.dir } + +// Sys returns the underlying data source (always nil) +func (s *FileInfo) Sys() any { return nil } + +// Size returns the size of the file func (s *FileInfo) Size() int64 { if s.IsDir() { return int64(42) diff --git a/fs/telegram/telegram.go b/fs/telegram/telegram.go index a421ae51..61d622bd 100644 --- a/fs/telegram/telegram.go +++ b/fs/telegram/telegram.go @@ -15,12 +15,11 @@ import ( "sync/atomic" "time" - tele "gopkg.in/telebot.v3" - "github.com/spf13/afero" + tele "gopkg.in/telebot.v3" + "gopkg.in/telebot.v3/middleware" "github.com/fclairamb/ftpserver/config/confpar" - "gopkg.in/telebot.v3/middleware" ) // ErrNotImplemented is returned when something is not implemented @@ -88,7 +87,9 @@ type File struct { var imageExtensions = []string{".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif"} // videoExtensions is the list of supported video extensions -var videoExtensions = []string{".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".mpeg", ".mpg", ".m4v", ".3gp", ".3g2"} +var videoExtensions = []string{ + ".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".mpeg", ".mpg", ".m4v", ".3gp", ".3g2", +} // textExtensions is the list of supported text extensions var textExtensions = []string{".txt", ".md"} @@ -98,7 +99,6 @@ var audioExtensions = []string{".mp3", ".ogg", ".flac", ".wav", ".m4a", ".opus"} // LoadFs loads a file system from an access description func LoadFs(access *confpar.Access, logger *slog.Logger) (afero.Fs, error) { - token := access.Params["Token"] if token == "" { return nil, fmt.Errorf("parameter Token is empty") @@ -106,7 +106,7 @@ func LoadFs(access *confpar.Access, logger *slog.Logger) (afero.Fs, error) { chatID, err := strconv.ParseInt(access.Params["ChatID"], 10, 64) if err != nil { - return nil, fmt.Errorf("invalid ChatID parameter: %v", err) + return nil, fmt.Errorf("invalid ChatID parameter: %w", err) } // Parse MaxPartSize param, default to defaultMaxPartSize if not set @@ -115,7 +115,7 @@ func LoadFs(access *confpar.Access, logger *slog.Logger) (afero.Fs, error) { if partSizeStr != "" { parsed, parseErr := strconv.ParseInt(partSizeStr, 10, 64) if parseErr != nil { - return nil, fmt.Errorf("invalid MaxPartSize parameter: %v", parseErr) + return nil, fmt.Errorf("invalid MaxPartSize parameter: %w", parseErr) } if parsed <= 0 { return nil, fmt.Errorf("invalid MaxPartSize parameter: must be > 0") @@ -239,7 +239,7 @@ func (f *File) Close() error { partFilename := fmt.Sprintf("%s.part%dof%d", basePath, partNum, totalParts) partCaption := fmt.Sprintf("%s (part %d/%d)", basePath, partNum, totalParts) - partData, readErr := os.ReadFile(tempPath) + partData, readErr := os.ReadFile(tempPath) //nolint:gosec // temporary file created by ourselves if readErr != nil { return readErr } @@ -260,7 +260,6 @@ func (f *File) Close() error { time.Sleep(f.Fs.PartUploadDelay) } } - } if f.Fs.fakeFs.stat(f.Path) == nil { @@ -371,7 +370,15 @@ func (f *File) sendWithRetry(filename string, caption string, send func() error) return err } - f.Fs.Logger.Warn("telegram send retry", "attempt", attempt, "maxAttempts", f.Fs.RetryAttempts, "path", f.Path, "filename", filename, "caption", caption, "err", err) + f.Fs.Logger.Warn( + "telegram send retry", + "attempt", attempt, + "maxAttempts", f.Fs.RetryAttempts, + "path", f.Path, + "filename", filename, + "caption", caption, + "err", err, + ) time.Sleep(f.Fs.RetryDelay) } @@ -402,7 +409,10 @@ func (f *File) spillNextPartToDisk(data []byte) error { tempFilePath := filepath.Join(f.PartTempDir, fmt.Sprintf("part-%05d.bin", f.PartNumber)) if err := os.WriteFile(tempFilePath, data, 0o600); err != nil { - f.Fs.Logger.Error("telegram spillNextPartToDisk WriteFile", "err", err, "tempFilePath", tempFilePath, "bytes", len(data)) + f.Fs.Logger.Error( + "telegram spillNextPartToDisk WriteFile", + "err", err, "tempFilePath", tempFilePath, "bytes", len(data), + ) return err } @@ -550,7 +560,7 @@ func (m *Fs) Remove(name string) error { return nil } -// Mkdir +// Mkdir creates a directory func (m *Fs) Mkdir(name string, mode os.FileMode) error { m.fakeFs.mkdir(name, mode) return nil @@ -618,7 +628,7 @@ func (m *Fs) OpenFile(name string, flag int, mode os.FileMode) (afero.File, erro return &File{Path: name, Fs: m, Content: content}, nil } -// Stat() fake implementation +// Stat returns the file information func (m *Fs) Stat(name string) (os.FileInfo, error) { fileInfo, _ := m.statPath(name) diff --git a/fs/utils/env.go b/fs/utils/env.go index 960df97c..cab4b3bd 100644 --- a/fs/utils/env.go +++ b/fs/utils/env.go @@ -1,3 +1,4 @@ +// Package utils provides helpers shared by the file systems package utils import ( @@ -5,10 +6,9 @@ import ( "regexp" ) -// Replace all environment variables in a string by their actual values - var envVar = regexp.MustCompile(`\$[a-zA-Z0-9_]+`) +// ReplaceEnvVars replaces all the environment variables ($NAME) in a string by their actual values func ReplaceEnvVars(s string) string { return envVar.ReplaceAllStringFunc(s, func(s string) string { return os.Getenv(s[1:]) diff --git a/main.go b/main.go index ecccd04a..4ae9b753 100644 --- a/main.go +++ b/main.go @@ -22,6 +22,12 @@ var ( ) func main() { + if err := run(); err != nil { + os.Exit(1) + } +} + +func run() error { // Arguments vars var confFile string var onlyConf bool @@ -46,30 +52,29 @@ func main() { } if autoCreate { - if _, err := os.Stat(confFile); err != nil && os.IsNotExist(err) { - logger.Warn("No conf file, creating one", "confFile", confFile) - - if err := os.WriteFile(confFile, confFileContent(), 0600); err != nil { //nolint: gomnd - logger.Warn("Couldn't create conf file", "confFile", confFile) - } - } + createConfFile(confFile, logger) } conf, errConfig := config.NewConfig(confFile, logger) if errConfig != nil { logger.Error("Can't load conf", "err", errConfig) - return + return errConfig + } + + if onlyConf { + logger.Warn("Only creating conf") + + return nil } // Now is a good time to open a logging file if conf.Content.Logging.File != "" { - writer, err := os.OpenFile(conf.Content.Logging.File, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600) //nolint:gomnd - + writer, err := os.OpenFile(conf.Content.Logging.File, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o600) if err != nil { logger.Error("Can't open log file", "err", err) - return + return err } // Create a slog handler that writes to both file and stdout @@ -87,7 +92,7 @@ func main() { if errNewServer != nil { logger.Error("Could not load the driver", "err", errNewServer) - return + return errNewServer } // Instantiating the server by passing our driver implementation @@ -100,20 +105,31 @@ func main() { go signalHandler() // Blocking call, behaving similarly to the http.ListenAndServe - if onlyConf { - logger.Warn("Only creating conf") - - return - } - if err := ftpServer.ListenAndServe(); err != nil { logger.Error("Problem listening", "err", err) + + return err } // We wait at most 1 minutes for all clients to disconnect if err := driver.WaitGracefully(time.Minute); err != nil { ftpServer.Logger.Warn("Problem stopping server", "err", err) } + + return nil +} + +// createConfFile creates a sample config file if it doesn't exist yet +func createConfFile(confFile string, logger *slog.Logger) { + if _, err := os.Stat(confFile); err == nil || !os.IsNotExist(err) { + return + } + + logger.Warn("No conf file, creating one", "confFile", confFile) + + if err := os.WriteFile(confFile, confFileContent(), 0o600); err != nil { + logger.Warn("Couldn't create conf file", "confFile", confFile, "err", err) + } } func stop() { @@ -126,23 +142,20 @@ func stop() { func signalHandler() { ch := make(chan os.Signal, 1) - signal.Notify(ch, syscall.SIGTERM) - signal.Notify(ch, syscall.SIGHUP) - - for { - sig := <-ch - if sig == syscall.SIGHUP { - err := driver.ReloadConfig() - if err != nil { + signal.Notify(ch, syscall.SIGTERM, syscall.SIGHUP) + + for sig := range ch { + switch sig { + case syscall.SIGHUP: + if err := driver.ReloadConfig(); err != nil { ftpServer.Logger.Warn("Error reloading config", "err", err) } else { ftpServer.Logger.Info("Successfully reloaded config") } - } - if sig == syscall.SIGTERM { + case syscall.SIGTERM: stop() - break + return } } } diff --git a/server/server.go b/server/server.go index 2e2a0642..cac49c88 100644 --- a/server/server.go +++ b/server/server.go @@ -26,7 +26,7 @@ import ( ) // Server structure -type Server struct { // nolint: maligned +type Server struct { config *config.Config logger *slog.Logger nbClients uint32 @@ -52,12 +52,21 @@ func newFsCache() *fsCache { // ErrTimeout is returned when an operation timeouts var ErrTimeout = errors.New("timeout") -// ErrNotImplemented is returned when we're using something that has not been implemented yet -// var ErrNotImplemented = errors.New("not implemented") - // ErrNotEnabled is returned when a feature hasn't been enabled var ErrNotEnabled = errors.New("not enabled") +// ErrTooManyClients is returned when the max_clients limit has been reached +var ErrTooManyClients = errors.New("too many clients") + +// ErrWebhookStatus is returned when the accesses webhook doesn't return a 200 status code +var ErrWebhookStatus = errors.New("unexpected webhook status code") + +// defaultWebhookTimeout is used when the accesses webhook timeout isn't specified +const defaultWebhookTimeout = 30 * time.Second + +// maxWebhookResponseSize is the maximum size of an accesses webhook response +const maxWebhookResponseSize = 1 << 20 + // NewServer creates a server instance func NewServer(config *config.Config, logger *slog.Logger) (*Server, error) { return &Server{ @@ -103,8 +112,20 @@ func (s *Server) GetSettings() (*serverlib.Settings, error) { DefaultTransferType: serverlib.TransferTypeBinary, }, nil } + +// ReloadConfig reloads the configuration file. Shared file systems are dropped so that they get re-created with +// the new configuration. func (s *Server) ReloadConfig() error { - return s.config.Load() + if err := s.config.Load(); err != nil { + return err + } + + s.accesses.Lock() + defer s.accesses.Unlock() + + s.accesses.accesses = make(map[string]afero.Fs) + + return nil } // ClientConnected is called to send the very first welcome message @@ -119,6 +140,13 @@ func (s *Server) ClientConnected(cc serverlib.ClientContext) (string, error) { "nbClients", s.nbClients, ) + // The client is counted until it disconnects, which also happens when we refuse it + if maxClients := s.config.Content.MaxClients; maxClients > 0 && int64(s.nbClients) > int64(maxClients) { + s.logger.Warn("Too many clients, refusing connection", "clientId", cc.ID(), "maxClients", maxClients) + + return "Too many clients, try again later", ErrTooManyClients + } + if s.config.Content.Logging.FtpExchanges { cc.SetDebug(true) } @@ -126,7 +154,7 @@ func (s *Server) ClientConnected(cc serverlib.ClientContext) (string, error) { return "ftpserver", nil } -// ClientDisconnected is called when the user disconnects, even if he never authenticated +// ClientDisconnected is called when the user disconnects, even if they never authenticated func (s *Server) ClientDisconnected(cc serverlib.ClientContext) { s.nbClientsSync.Lock() defer s.nbClientsSync.Unlock() @@ -154,10 +182,17 @@ func (s *Server) Stop() { func (s *Server) WaitGracefully(timeout time.Duration) error { s.logger.Info("Waiting for last client to disconnect...") - defer func() { s.zeroClientEvent = nil }() + s.nbClientsSync.Lock() + zeroClientEvent := s.zeroClientEvent + s.nbClientsSync.Unlock() + + if zeroClientEvent == nil { + // Stop wasn't called, there's nothing to wait for + return nil + } select { - case err := <-s.zeroClientEvent: + case err := <-zeroClientEvent: return err case <-time.After(timeout): return ErrTimeout @@ -204,43 +239,44 @@ func (s *Server) getAccessFromWebhook(user, pass string) (*confpar.Access, error return nil, err } + webhook := s.config.Content.AccessesWebhook + + timeout := webhook.Timeout.Duration + if timeout <= 0 { + timeout = defaultWebhookTimeout + } + // Timeout is implemented with context termination - ctx, cancel := context.WithTimeout(context.Background(), s.config.Content.AccessesWebhook.Timeout.Duration) + ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() // Create a new HTTP request - req, err := http.NewRequestWithContext(ctx, "POST", s.config.Content.AccessesWebhook.URL, bytes.NewBuffer(jsonData)) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, webhook.URL, bytes.NewReader(jsonData)) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/json") - for key, value := range s.config.Content.AccessesWebhook.Headers { + + for key, value := range webhook.Headers { req.Header.Set(key, value) } // Execute the request - client := &http.Client{} - resp, err := client.Do(req) + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } + defer func() { _ = resp.Body.Close() }() // Check the response status if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - - // Return the response - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err + return nil, fmt.Errorf("%w: %d", ErrWebhookStatus, resp.StatusCode) } access := new(confpar.Access) - err = json.Unmarshal(body, &access) - if err != nil { + if err := json.NewDecoder(io.LimitReader(resp.Body, maxWebhookResponseSize)).Decode(access); err != nil { return nil, err } @@ -330,7 +366,7 @@ func (s *Server) loadTLSConfig() (*tls.Config, error) { keyBytes, errReadFileKey := os.ReadFile(serverCert.Key) if errReadFileKey != nil { - return nil, fmt.Errorf("could not load key file: %s: %w", serverCert.Cert, errReadFileCert) + return nil, fmt.Errorf("could not load key file: %s: %w", serverCert.Key, errReadFileKey) } keypair, errKeyPair := tls.X509KeyPair(certBytes, keyBytes) diff --git a/server/server_internal_test.go b/server/server_internal_test.go new file mode 100644 index 00000000..c91370af --- /dev/null +++ b/server/server_internal_test.go @@ -0,0 +1,127 @@ +package server + +import ( + "bufio" + "encoding/json" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + serverlib "github.com/fclairamb/ftpserverlib" + + "github.com/fclairamb/ftpserver/config" + "github.com/fclairamb/ftpserver/config/confpar" +) + +func newTestServer(t *testing.T, content *confpar.Content) *Server { + t.Helper() + + conf, err := config.FromContent(content, "test.json", slog.Default()) + if err != nil { + t.Fatalf("FromContent: %v", err) + } + + srv, err := NewServer(conf, slog.Default()) + if err != nil { + t.Fatalf("NewServer: %v", err) + } + + return srv +} + +// TestWebhookWithoutTimeout makes sure a webhook without timeout works (it used to time out instantly) +func TestWebhookWithoutTimeout(t *testing.T) { + webhook := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req map[string]string + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + w.WriteHeader(http.StatusBadRequest) + + return + } + + if r.Header.Get("Authorization") != "Bearer token" || req["pass"] != "secret" { + w.WriteHeader(http.StatusForbidden) + + return + } + + _ = json.NewEncoder(w).Encode(confpar.Access{ + User: req["user"], + Fs: "os", + Params: map[string]string{"basePath": "/tmp"}, + }) + })) + defer webhook.Close() + + srv := newTestServer(t, &confpar.Content{ + AccessesWebhook: &confpar.AccessesWebhook{ + URL: webhook.URL, + Headers: map[string]string{"Authorization": "Bearer token"}, + }, + }) + + access, err := srv.getAccessFromWebhook("alice", "secret") + if err != nil { + t.Fatalf("getAccessFromWebhook: %v", err) + } + + if access.User != "alice" || access.Fs != "os" || access.Params["basePath"] != "/tmp" { + t.Fatalf("unexpected access: %+v", access) + } + + if _, err := srv.getAccessFromWebhook("alice", "wrong"); err == nil { + t.Fatal("expected an error for a refused authentication") + } +} + +func readBanner(t *testing.T, addr string) (net.Conn, string) { + t.Helper() + + conn, err := net.DialTimeout("tcp", addr, 5*time.Second) + if err != nil { + t.Fatalf("Dial: %v", err) + } + + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + + line, err := bufio.NewReader(conn).ReadString('\n') + if err != nil { + t.Fatalf("couldn't read banner: %v", err) + } + + return conn, line +} + +func TestMaxClients(t *testing.T) { + srv := newTestServer(t, &confpar.Content{ + ListenAddress: "127.0.0.1:0", + MaxClients: 1, + }) + + ftpServer := serverlib.NewFtpServer(srv) + if err := ftpServer.Listen(); err != nil { + t.Fatalf("Listen: %v", err) + } + + go func() { _ = ftpServer.Serve() }() + + defer func() { _ = ftpServer.Stop() }() + + first, banner := readBanner(t, ftpServer.Addr()) + defer func() { _ = first.Close() }() + + if !strings.HasPrefix(banner, "220") { + t.Fatalf("first client should be accepted, got: %q", banner) + } + + second, banner := readBanner(t, ftpServer.Addr()) + defer func() { _ = second.Close() }() + + if !strings.Contains(banner, "Too many clients") { + t.Fatalf("second client should be refused, got: %q", banner) + } +} From ae62c8819b4f3ef6c24aad68e303ea5f79970ab4 Mon Sep 17 00:00:00 2001 From: fclairamb Date: Thu, 22 Oct 2026 08:00:39 +0200 Subject: [PATCH 2/4] ci: fix linting, CodeQL, docker images and docker test - build: golangci-lint never ran (it depended on an undefined matrix value) and its config was in the v1 format rejected by golangci-lint v2: migrate it to a v2 config, run it after setting up go, and drop the apt-get call that always failed (gcc is already installed on the runners) - codeql: analyze pushes and pull requests on main instead of master - docker: cross-compile the binary instead of building it under QEMU, publish the images from release-please (tags pushed with the default GITHUB_TOKEN don't trigger workflows, images weren't published since v0.15.1), pass the version info as build args instead of templating build.go with ci-info, and build the images on pull requests touching them - docker test: placekitten.com is gone, use a random file, test both images, and build them locally on pull requests touching them - scratch image: provide the /tmp directory served by the default config - schema: document tls_required, extensions, accesses_webhook and the access logging options, which were rejected by the schema, and validate the backend examples too (the mail one wasn't valid JSON) Co-Authored-By: Claude Opus 5.5 --- .ci-info.json | 10 - .dockerignore | 7 + .github/workflows/build.yml | 28 +- .github/workflows/codeql-analysis.yml | 37 +-- .github/workflows/docker-test.yml | 66 +++-- .github/workflows/docker.yml | 97 ++++--- .github/workflows/jsonschema.yml | 24 +- .github/workflows/release-please.yml | 15 +- .gitignore | 3 +- .golangci.yml | 249 +++-------------- Dockerfile | 24 +- Dockerfile.alpine | 22 +- build.go.tpl | 12 - config-schema.json | 294 +++++++++++++-------- docker-compose.yml | 4 +- fs/mail/{example.conf => example.json} | 3 +- fs/telegram/{example.conf => example.json} | 2 +- local-docker-test.sh | 19 +- 18 files changed, 422 insertions(+), 494 deletions(-) delete mode 100644 .ci-info.json create mode 100644 .dockerignore delete mode 100644 build.go.tpl rename fs/mail/{example.conf => example.json} (78%) rename fs/telegram/{example.conf => example.json} (84%) diff --git a/.ci-info.json b/.ci-info.json deleted file mode 100644 index 65650a4d..00000000 --- a/.ci-info.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/fclairamb/ci-info/main/config-schema.json", - "version_input_git_tag": { - "pattern": "^v?([0-9.]+)$" - }, - "templates": [{ - "input_file": "build.go.tpl", - "output_file": "build.go" - }] -} \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..4158e7fd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.git +.github +.vscode +ftpserver +ftpserver.json +ftpserver.log +*.pem diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 78e78924..cdccc4c9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,43 +1,27 @@ name: Build -# Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch on: push: pull_request: -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "build" build: - # The type of runner that the job will run on runs-on: ubuntu-26.04 - - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE - uses: actions/checkout@v7.0.1 - # Running golangci-lint - - name: Linting - if: matrix.lint - uses: golangci/golangci-lint-action@v9 - with: - version: v2.15.0 - - # Install Go - name: Setup go uses: actions/setup-go@v7.0.0 with: go-version: 1.27.3 + - name: Linting + uses: golangci/golangci-lint-action@v9 + with: + version: v2.15.0 + - name: Build run: go build -v ./... - name: Test - # We need GCC because of the "go test -race" - # env: - # CGO_ENABLED: 0 - run: | - apt-get update && apt-get install gcc -y - go test -race -v ./... + run: go test -race -v ./... diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 02776d69..325209d3 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,22 +1,10 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# name: "CodeQL" on: push: - branches: [ master ] + branches: [ main ] pull_request: - # The branches below must be a subset of the branches above - branches: [ master ] + branches: [ main ] schedule: - cron: '44 20 * * 0' @@ -29,39 +17,18 @@ jobs: fail-fast: false matrix: language: [ 'go' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] - # Learn more: - # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed steps: - name: Checkout repository uses: actions/checkout@v7.0.1 - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v4 - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index 22b522f3..92687f40 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -1,43 +1,59 @@ -name: Test the current docker image +name: Test the docker image on: schedule: - # We want to run it every day - - cron: '0 0 * * *' + # We want to check the published image every day + - cron: '0 0 * * *' + pull_request: + paths: + - Dockerfile* + - .dockerignore + - .github/workflows/docker-test.yml + workflow_dispatch: jobs: test-docker-image: + strategy: + fail-fast: false + matrix: + tag: ["latest", "alpine"] runs-on: ubuntu-26.04 - timeout-minutes: 1 - services: - ftpserver: - image: fclairamb/ftpserver - ports: - - 2121-2130:2121-2130 - + timeout-minutes: 15 + env: + SERVER_PORT: 2121 steps: - - name: Run sample - env: - SERVER_PORT: 2121 + - uses: actions/checkout@v7.0.1 + if: github.event_name == 'pull_request' + + - name: Build the image + if: github.event_name == 'pull_request' + run: | + docker build -t "fclairamb/ftpserver:${{ matrix.tag }}" \ + -f "Dockerfile${{ matrix.tag == 'alpine' && '.alpine' || '' }}" . + + - name: Start the server + run: | + docker run -d --name ftpserver -p 2121-2130:2121-2130 "fclairamb/ftpserver:${{ matrix.tag }}" + + - name: Upload and download a file run: | set -xe - # Getting a file - curl https://placekitten.com/2048/2048 -o kitty.jpg + # Generating a file + head -c 2097152 /dev/urandom > local.bin # Waiting for server to be ready - while ! nc -z localhost ${SERVER_PORT} > "$GITHUB_OUTPUT" + - name: Checkout uses: actions/checkout@v7.0.1 with: - fetch-depth: 0 - - name: Setup go - uses: actions/setup-go@v7.0.0 - with: - go-version-file: go.mod - - name: Change version + ref: ${{ steps.tag.outputs.tag || github.sha }} + + - name: Resolve build info + id: build run: | - curl -L https://github.com/fclairamb/ci-info/releases/download/v0.4.4/ci-info_0.4.4_linux_amd64.tar.gz | tar -zx - ./ci-info + echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> "$GITHUB_OUTPUT" + - name: Docker meta id: docker_meta uses: docker/metadata-action@v6 @@ -44,50 +71,42 @@ jobs: suffix=${{ matrix.type != '' && format('-{0}', matrix.type) || '' }} latest=false tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=semver,suffix=,pattern=${{ matrix.type != '' && matrix.type || 'latest' }} - type=ref,event=branch + type=semver,pattern={{version}},value=${{ steps.tag.outputs.tag }} + type=semver,pattern={{major}}.{{minor}},value=${{ steps.tag.outputs.tag }} + type=semver,pattern={{major}},value=${{ steps.tag.outputs.tag }} + type=semver,suffix=,pattern=${{ matrix.type != '' && matrix.type || 'latest' }},value=${{ steps.tag.outputs.tag }} type=ref,event=pr - - name: Set up QEMU - uses: docker/setup-qemu-action@v4 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - - name: Cache Docker layers - uses: actions/cache@v6 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ matrix.type }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- + - name: Login to DockerHub - if: github.repository == 'fclairamb/ftpserver' + if: env.PUSH == 'true' uses: docker/login-action@v4 with: username: ${{ github.repository_owner }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to GitHub Container Registry - if: github.repository == 'fclairamb/ftpserver' + if: env.PUSH == 'true' uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push (scratch) + + - name: Build and push uses: docker/build-push-action@v7 with: context: . file: ./Dockerfile${{ matrix.type != '' && format('.{0}', matrix.type) || '' }} platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7 - push: ${{ github.repository == 'fclairamb/ftpserver' }} + push: ${{ env.PUSH == 'true' }} tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - build-and-push-status: - needs: build-and-push - runs-on: ubuntu-26.04 - steps: - - run: echo 'All good' + build-args: | + VERSION=${{ steps.tag.outputs.tag || 'dev' }} + COMMIT=${{ steps.build.outputs.commit }} + BUILD_DATE=${{ steps.build.outputs.date }} + cache-from: type=gha,scope=docker${{ matrix.type }} + cache-to: type=gha,mode=max,scope=docker${{ matrix.type }} diff --git a/.github/workflows/jsonschema.yml b/.github/workflows/jsonschema.yml index c1dfddc3..669e05d0 100644 --- a/.github/workflows/jsonschema.yml +++ b/.github/workflows/jsonschema.yml @@ -7,14 +7,24 @@ on: jobs: json-schema-check: runs-on: ubuntu-26.04 - container: python:3.11-alpine3.16 + container: python:3.14-alpine steps: - uses: actions/checkout@v7.0.1 - name: Test run: | - pip install jsonschema==4.14.0 - for f in $(find config/ -name '*.json') - do - echo "Checking $f" - jsonschema -i $f config-schema.json - done + pip install jsonschema==4.26.0 + python - config-schema.json $(find config/ fs/ -name '*.json') <<'EOF_PY' + import json, sys, jsonschema + + schema = json.load(open(sys.argv[1])) + jsonschema.Draft7Validator.check_schema(schema) + failed = False + for f in sys.argv[2:]: + try: + jsonschema.validate(json.load(open(f)), schema) + print(f"OK {f}") + except Exception as e: + failed = True + print(f"FAIL {f}: {e}") + sys.exit(1 if failed else 0) + EOF_PY diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index b8796bed..c028ac2e 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -21,8 +21,8 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} # The release is created with the default GITHUB_TOKEN, which by design does not - # trigger any further workflow. The binaries therefore have to be built from here - # rather than from a `push: tags` trigger (see #1738). + # trigger any further workflow. The binaries and docker images therefore have to be + # built from here rather than from a `push: tags` trigger (see #1738). release-artifacts: needs: release-please if: needs.release-please.outputs.release_created == 'true' @@ -31,3 +31,14 @@ jobs: tag: ${{ needs.release-please.outputs.tag_name }} permissions: contents: write + + docker: + needs: release-please + if: needs.release-please.outputs.release_created == 'true' + uses: ./.github/workflows/docker.yml + with: + tag: ${{ needs.release-please.outputs.tag_name }} + secrets: inherit + permissions: + contents: read + packages: write diff --git a/.gitignore b/.gitignore index 9cdef23e..4ed65225 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,7 @@ ftpserver ftpserver.json ftpserver.log -ci-info .idea/ *.pem -gdrive_token_gdrive.json +gdrive_token_*.json __* diff --git a/.golangci.yml b/.golangci.yml index c0dbed6d..68eeeb57 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,222 +1,51 @@ -run: - # default concurrency is a available CPU number - concurrency: 4 - - # timeout for analysis, e.g. 30s, 5m, default is 1m - timeout: 1m - - # exit code when at least one issue was found, default is 1 - issues-exit-code: 1 - - # include test files or not, default is true - tests: true - -# all available settings of specific linters -linters-settings: - errcheck: - # report about not checking of errors in type assetions: `a := b.(MyStruct)`; - # default is false: such cases aren't reported by default. - check-type-assertions: false - - # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; - # default is false: such cases aren't reported by default. - check-blank: false - - funlen: - lines: 80 - statements: 40 - - govet: - # report about shadowed variables - check-shadowing: true - - # settings per analyzer - settings: - printf: # analyzer name, run `go tool vet help` to see all analyzers - funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf - - # enable or disable analyzers by name - enable: - - atomicalign - # - fieldalignment - enable-all: false - disable: - - shadow - disable-all: false - gofmt: - # simplify code: gofmt with `-s` option, true by default - simplify: true - goimports: - # put imports beginning with prefix after 3rd-party packages; - # it's a comma-separated list of prefixes - local-prefixes: github.com/fclairamb/ftpserver/ - gocyclo: - # minimal code complexity to report, 30 by default (but we recommend 10-20) - min-complexity: 15 - gocognit: - # minimal code complexity to report, 30 by default (but we recommend 10-20) - min-complexity: 30 - maligned: - # print struct with more effective memory layout or not, false by default - suggest-new: true - dupl: - # tokens count to trigger issue, 150 by default - threshold: 100 - goconst: - # minimal length of string constant, 3 by default - min-len: 3 - # minimal occurrences count to trigger, 3 by default - min-occurrences: 3 - depguard: - list-type: blacklist - include-go-root: false - packages: - - github.com/sirupsen/logrus - packages-with-error-messages: - # specify an error message to output when a blacklisted package is used - github.com/sirupsen/logrus: "logging is allowed only by logutils.Log" - misspell: - # Correct spellings using locale preferences for US or UK. - # Default is to use a neutral variety of English. - # Setting locale to US will correct the British spelling of 'colour' to 'color'. - locale: US - ignore-words: - - someword - lll: - # max line length, lines longer will be reported. Default is 120. - # '\t' is counted as 1 character by default, and can be changed with the tab-width option - line-length: 120 - # tab width in spaces. Default to 1. - tab-width: 1 - unused: - # treat code as a program (not a library) and report unused exported identifiers; default is false. - # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: - # if it's called for subdir of a project it can't find funcs usages. All text editor integrations - # with golangci-lint call it on a directory with the changed file. - check-exported: false - unparam: - # Inspect exported functions, default is false. Set to true if no external program/library imports your code. - # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: - # if it's called for subdir of a project it can't find external interfaces. All text editor integrations - # with golangci-lint call it on a directory with the changed file. - check-exported: false - nakedret: - # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 - max-func-lines: 30 - prealloc: - # XXX: we don't recommend using this linter before doing performance profiling. - # For most programs usage of prealloc will be a premature optimization. - - # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. - # True by default. - simple: true - range-loops: true # Report preallocation suggestions on range loops, true by default - for-loops: false # Report preallocation suggestions on for loops, false by default - gocritic: - # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint run` to see all tags and checks. - # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags". - enabled-tags: - - performance - - settings: # settings passed to gocritic - captLocal: # must be valid enabled check name - paramsOnly: true - rangeValCopy: - sizeThreshold: 32 - godox: - # report any comments starting with keywords, this is useful for TODO or FIXME comments that - # might be left in the code accidentally and should be resolved before merging - keywords: # default keywords are TODO, BUG, and FIXME, these can be overwritten by this setting - - NOTE - - OPTIMIZE # marks code that should be optimized before merging - - HACK # marks hack-arounds that should be removed before merging - dogsled: - # checks assignments with too many blank identifiers; default is 2 - max-blank-identifiers: 2 - - whitespace: - multi-if: false # Enforces newlines (or comments) after every multi-line if statement - multi-func: false # Enforces newlines (or comments) after every multi-line function signature - wsl: - # If true append is only allowed to be cuddled if appending value is - # matching variables, fields or types on line above. Default is true. - strict-append: true - # Allow calls and assignments to be cuddled as long as the lines have any - # matching variables, fields or types. Default is true. - allow-assign-and-call: true - # Allow multiline assignments to be cuddled. Default is true. - allow-multiline-assign: true - # Allow declarations (var) to be cuddled. - allow-cuddle-declarations: true - # Allow trailing comments in ending of blocks - allow-trailing-comment: false - # Force newlines in end of case at this limit (0 = never). - force-case-trailing-whitespace: 0 +version: "2" linters: - disable-all: true + default: standard # errcheck, govet, ineffassign, staticcheck, unused enable: - bodyclose - - deadcode - - depguard - - dogsled - - dupl - - errcheck - - exhaustive - # - exhaustivestruct - - exportloopref - - funlen - # - gci - - gochecknoinits - # - gochecknoglobals - - gocognit - - goconst + - copyloopvar + - errorlint - gocritic - - gocyclo - - godox - - goerr113 - - gofmt - - goimports - # - golint #deprecated - - gomnd - - goprintffuncname - gosec - - gosimple - - govet - - ineffassign - # - interfacer #deprecated - lll - # - maligned - - megacheck - misspell - - nakedret - - nestif - - nlreturn - - prealloc - nolintlint - revive - - rowserrcheck - - exportloopref # replaces scopelint - - staticcheck - - structcheck - - stylecheck - - typecheck - unconvert - - unparam - - unused - - varcheck - whitespace - # - wrapcheck - - wsl - fast: false - -issues: - # Independently from option `exclude` we use default exclude patterns, - # it can be disabled by this option. To list all - # excluded by default patterns execute `golangci-lint run --help`. - # Default value for this option is true. - exclude-use-default: false + settings: + gocritic: + disabled-checks: + - ifElseChain + lll: + line-length: 120 + misspell: + locale: US + revive: + rules: + - name: exported + - name: package-comments + - name: var-naming + - name: error-strings + - name: error-naming + - name: receiver-naming + - name: indent-error-flow + - name: blank-imports + exclusions: + presets: + - std-error-handling + rules: + # Tests can open files from variables and use relaxed permissions + - path: _test\.go + linters: + - gosec + +formatters: + enable: + - gofmt + - goimports + settings: + goimports: + local-prefixes: + - github.com/fclairamb/ftpserver diff --git a/Dockerfile b/Dockerfile index 1b9c0807..524e38ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,31 @@ # Should be started with: # docker run -ti -p 2121-2130:2121-2130 fclairamb/ftpserver -# Preparing the build environment -FROM golang:1.27-alpine AS builder -ENV GOFLAGS="-mod=readonly" -RUN apk add --update --no-cache bash ca-certificates curl git && update-ca-certificates -RUN mkdir -p /workspace +# Preparing the build environment. The binary is cross-compiled on the build platform, which is much faster than +# building it in an emulated environment. +FROM --platform=$BUILDPLATFORM golang:1.27-alpine AS builder +ENV GOFLAGS="-mod=readonly" CGO_ENABLED=0 +RUN apk add --update --no-cache ca-certificates && update-ca-certificates WORKDIR /workspace +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download + # Building COPY . . -RUN CGO_ENABLED=0 go build -mod=readonly -ldflags='-w -s' -v -o ftpserver +ARG TARGETOS TARGETARCH TARGETVARIANT +ARG VERSION=dev COMMIT="" BUILD_DATE="" +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ + if [ "$TARGETARCH" = "arm" ]; then export GOARM="${TARGETVARIANT#v}"; fi && \ + GOOS=$TARGETOS GOARCH=$TARGETARCH go build -v -o ftpserver \ + -ldflags="-w -s -X main.BuildVersion=${VERSION} -X main.Commit=${COMMIT} -X main.BuildDate=${BUILD_DATE}" # Preparing the final image FROM scratch -WORKDIR /app COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +# The default configuration serves the /tmp directory +COPY --from=builder /tmp /tmp +WORKDIR /app EXPOSE 2121-2130 COPY --from=builder /workspace/ftpserver /bin/ftpserver ENTRYPOINT [ "/bin/ftpserver" ] diff --git a/Dockerfile.alpine b/Dockerfile.alpine index cab710e7..620dc561 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -1,16 +1,24 @@ # Should be started with: -# docker run -ti -p 2121-2130:2121-2130 fclairamb/ftpserver +# docker run -ti -p 2121-2130:2121-2130 fclairamb/ftpserver:alpine -# Preparing the build environment -FROM golang:1.27-alpine AS builder -ENV GOFLAGS="-mod=readonly" -RUN apk add --update --no-cache bash ca-certificates curl git && update-ca-certificates -RUN mkdir -p /workspace +# Preparing the build environment. The binary is cross-compiled on the build platform, which is much faster than +# building it in an emulated environment. +FROM --platform=$BUILDPLATFORM golang:1.27-alpine AS builder +ENV GOFLAGS="-mod=readonly" CGO_ENABLED=0 +RUN apk add --update --no-cache ca-certificates && update-ca-certificates WORKDIR /workspace +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download + # Building COPY . . -RUN go build -v -o ftpserver +ARG TARGETOS TARGETARCH TARGETVARIANT +ARG VERSION=dev COMMIT="" BUILD_DATE="" +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ + if [ "$TARGETARCH" = "arm" ]; then export GOARM="${TARGETVARIANT#v}"; fi && \ + GOOS=$TARGETOS GOARCH=$TARGETARCH go build -v -o ftpserver \ + -ldflags="-w -s -X main.BuildVersion=${VERSION} -X main.Commit=${COMMIT} -X main.BuildDate=${BUILD_DATE}" # Preparing the final image FROM alpine:3.24.3 diff --git a/build.go.tpl b/build.go.tpl deleted file mode 100644 index 265c2da1..00000000 --- a/build.go.tpl +++ /dev/null @@ -1,12 +0,0 @@ -package main - -var ( - // BuildVersion is the current version of the program - BuildVersion = "{{ .Version }}" - - // BuildDate is the time the program was built - BuildDate = "{{ .BuildDate }}" - - // Commit is the git hash of the program - Commit = "{{ .GitCommitHashShort }}" -) diff --git a/config-schema.json b/config-schema.json index d51d81d9..5ad33a09 100644 --- a/config-schema.json +++ b/config-schema.json @@ -38,10 +38,11 @@ "public_host": { "type": "string", "default": "", - "title": "The public listening address (when behing a NAT gateway)", + "title": "The public IP address (when behind a NAT gateway)", "examples": [ "1.2.3.4" - ] + ], + "description": "Can be overridden with the PUBLIC_HOST environment variable." }, "max_clients": { "type": "integer", @@ -49,7 +50,8 @@ "title": "The maximum number of concurrent clients to accept", "examples": [ 200 - ] + ], + "description": "0 means no limit." }, "hash_plaintext_passwords": { "type": "boolean", @@ -98,10 +100,12 @@ ] } }, - "examples": [{ - "start": 2122, - "end": 2130 - }] + "examples": [ + { + "start": 2122, + "end": 2130 + } + ] }, "logging": { "type": "object", @@ -126,18 +130,20 @@ }, "file": { "type": "string", - "default": "ftpserver.log", + "default": "", "title": "Perform logging to a file", "examples": [ "ftpserver.log" ] } }, - "examples": [{ - "ftp_exchanges": true, - "file_accesses": true, - "file": "ftpserver.log" - }] + "examples": [ + { + "ftp_exchanges": true, + "file_accesses": true, + "file": "ftpserver.log" + } + ] }, "tls": { "type": "object", @@ -173,18 +179,84 @@ ] } }, - "examples": [{ + "examples": [ + { + "cert": "cert.pem", + "key": "key.pem" + } + ] + } + }, + "examples": [ + { + "server_cert": { "cert": "cert.pem", "key": "key.pem" - }] + } } - }, - "examples": [{ - "server_cert": { - "cert": "cert.pem", - "key": "key.pem" + ] + }, + "tls_required": { + "type": "string", + "default": "ClearOrEncrypted", + "title": "TLS requirement", + "description": "ClearOrEncrypted: TLS is optional, MandatoryEncryption: clients must use explicit TLS (AUTH TLS), ImplicitEncryption: the whole connection is encrypted (implicit FTPS).", + "enum": [ + "ClearOrEncrypted", + "MandatoryEncryption", + "ImplicitEncryption" + ] + }, + "extensions": { + "type": "object", + "default": {}, + "title": "Extended features", + "additionalProperties": false, + "properties": { + "enable_hash": { + "type": "boolean", + "default": false, + "title": "Enable the HASH command (and its XCRC, MD5, XMD5, XSHA, XSHA1, XSHA256, XSHA512 variants)" } - }] + } + }, + "accesses_webhook": { + "type": "object", + "title": "Webhook authenticating users", + "description": "When set, the accesses are not read from the config file: the user and password are POSTed as JSON to the webhook, which must reply with a 200 status code and an access object.", + "required": [ + "url" + ], + "additionalProperties": false, + "properties": { + "url": { + "type": "string", + "title": "URL to call", + "examples": [ + "https://auth.example.com/ftp" + ] + }, + "headers": { + "type": "object", + "title": "HTTP headers to add to the request", + "additionalProperties": { + "type": "string" + }, + "examples": [ + { + "Authorization": "Bearer secret" + } + ] + }, + "timeout": { + "type": "string", + "default": "30s", + "title": "Maximum duration of the request (Go duration format)", + "examples": [ + "5s" + ] + } + } }, "accesses": { "type": "array", @@ -192,12 +264,9 @@ "title": "The accesses", "items": { "type": "object", - "title": "A Schema", + "title": "An access", "required": [ - "user", - "pass", - "fs", - "params" + "fs" ], "properties": { "user": { @@ -205,7 +274,8 @@ "title": "The FTP user", "examples": [ "username" - ] + ], + "description": "Not used by the keycloak backend, which accepts any user." }, "pass": { "type": "string", @@ -213,17 +283,22 @@ "examples": [ "plaintext-password", "$2a$10$jG7tuqIlcUDMl1m1Ytj1TunU7pk.ko8lj3nOGzZvkIeU/BsfPVBra" - ] + ], + "description": "Plain-text, or hashed with md5crypt, bcrypt, sha256crypt or sha512crypt. Use \"*\" with the \"anonymous\" user to accept any password." }, "fs": { "type": "string", "title": "The backend file system to use", - "examples": [ + "enum": [ "os", + "s3", + "gcs", + "sftp", "dropbox", "gdrive", - "s3", - "sftp" + "mail", + "telegram", + "keycloak" ] }, "params": { @@ -233,7 +308,7 @@ "shared": { "type": "boolean", "default": false, - "title": "If the backend shall be shared between session", + "title": "If the backend instance shall be shared between sessions", "examples": [ true ] @@ -251,14 +326,13 @@ "default": {}, "title": "If we should use the sync & delete mechanism", "required": [ - "enable", - "directory" + "enable" ], "properties": { "enable": { "type": "boolean", "default": false, - "title": "The enable Schema", + "title": "Enable the sync & delete mechanism", "examples": [ true ] @@ -266,83 +340,97 @@ "directory": { "type": "string", "default": "", - "title": "The directory Schema", + "title": "Temporary directory (optional, a system one is used otherwise)", "examples": [ "/tmp/snd" ] } }, - "examples": [{ - "enable": true, - "directory": "/tmp/snd" - }] - } - }, - "examples": [{ - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/tmp" - } - }, - { - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/tmp" - } - }, - { - "user": "dropbox", - "pass": "dropbox", - "fs": "dropbox", - "params": { - "token": "..." - } - }, - { - "user": "gdrive", - "pass": "gdrive", - "fs": "gdrive", - "params": { - "google_client_id": "***.apps.googleusercontent.com", - "google_client_secret": "****", - "base_path": "ftp" - } - }, - { - "user": "s3", - "pass": "s3", - "fs": "s3", - "params": { - "endpoint": "https://s3.amazonaws.com", - "region": "eu-west-1", - "bucket": "my-bucket", - "access_key_id": "AKIA....", - "secret_access_key": "IDxd....", - "disable_ssl": "false", - "path_style": "false" + "examples": [ + { + "enable": true, + "directory": "/tmp/snd" + } + ] + }, + "logging": { + "type": "object", + "default": {}, + "title": "Logging options of this access", + "additionalProperties": false, + "properties": { + "ftp_exchanges": { + "type": "boolean", + "default": false, + "title": "Log all FTP exchanges" + }, + "file_accesses": { + "type": "boolean", + "default": false, + "title": "Log all file accesses" + } + } } }, - { - "user": "sftp", - "pass": "sftp", - "fs": "sftp", - "shared": true, - "read_only": true, - "sync_and_delete": { - "enable": true, - "directory": "/tmp/snd" + "examples": [ + { + "user": "test", + "pass": "test", + "fs": "os", + "params": { + "basePath": "/tmp" + } }, - "params": { - "username": "user", - "password": "password", - "hostname": "192.168.168.11:22", - "known_hosts": "/home/ftpserver/.ssh/known_hosts" + { + "user": "dropbox", + "pass": "dropbox", + "fs": "dropbox", + "params": { + "token": "..." + } + }, + { + "user": "gdrive", + "pass": "gdrive", + "fs": "gdrive", + "params": { + "google_client_id": "***.apps.googleusercontent.com", + "google_client_secret": "****", + "base_path": "ftp" + } + }, + { + "user": "s3", + "pass": "s3", + "fs": "s3", + "params": { + "endpoint": "https://s3.amazonaws.com", + "region": "eu-west-1", + "bucket": "my-bucket", + "access_key_id": "AKIA....", + "secret_access_key": "IDxd....", + "disable_ssl": "false", + "path_style": "false" + } + }, + { + "user": "sftp", + "pass": "sftp", + "fs": "sftp", + "shared": true, + "read_only": true, + "sync_and_delete": { + "enable": true, + "directory": "/tmp/snd" + }, + "params": { + "username": "user", + "password": "password", + "hostname": "192.168.168.11:22", + "known_hosts": "/home/ftpserver/.ssh/known_hosts" + } } - }] + ] } } }, diff --git a/docker-compose.yml b/docker-compose.yml index d7f7fa8d..9e8fa523 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,8 @@ -version: '3.3' services: ftpserver: + image: fclairamb/ftpserver ports: - '2121-2130:2121-2130' volumes: - ./files:/tmp - .:/app - image: fclairamb/ftpserver - diff --git a/fs/mail/example.conf b/fs/mail/example.json similarity index 78% rename from fs/mail/example.conf rename to fs/mail/example.json index 7192c953..b16f03c2 100644 --- a/fs/mail/example.conf +++ b/fs/mail/example.json @@ -1,4 +1,5 @@ { + "$schema": "https://raw.githubusercontent.com/fclairamb/ftpserver/main/config-schema.json", "version": 1, "accesses": [ { @@ -13,7 +14,7 @@ "Host": "smtp.example.com", "Port": "465", "Username": "smtpuser", - "Password": "smtppassword", + "Password": "smtppassword" } } ], diff --git a/fs/telegram/example.conf b/fs/telegram/example.json similarity index 84% rename from fs/telegram/example.conf rename to fs/telegram/example.json index 54c8ddc2..7a07a048 100644 --- a/fs/telegram/example.conf +++ b/fs/telegram/example.json @@ -1,4 +1,5 @@ { + "$schema": "https://raw.githubusercontent.com/fclairamb/ftpserver/main/config-schema.json", "version": 1, "accesses": [ { @@ -15,7 +16,6 @@ "RetryDelay": "2000", "PartUploadDelay": "500" } - } ], "passive_transfer_port_range": { diff --git a/local-docker-test.sh b/local-docker-test.sh index c82ebb10..d042625f 100755 --- a/local-docker-test.sh +++ b/local-docker-test.sh @@ -1,12 +1,15 @@ #!/bin/sh -ex +# Builds the docker image and checks that a file can be uploaded and downloaded back -docker rm -f ftpserver ||: +docker rm -f ftpserver-test || : -GOOS=linux GOARCH=amd64 go build -docker build . -t test -if [ ! -f kitty.jpg ]; then - curl -o kitty.jpg.tmp https://placekitten.com/2048/2048 && mv kitty.jpg.tmp kitty.jpg -fi -docker run --name ftpserver -p 2121-2130:2121-2130 test & +docker build . -t ftpserver-test +docker run -d --rm --name ftpserver-test -p 2121-2130:2121-2130 ftpserver-test +trap 'docker rm -f ftpserver-test' EXIT + +head -c 2097152 /dev/urandom > local.bin.tmp while ! nc -z localhost 2121 Date: Thu, 22 Oct 2026 08:03:12 +0200 Subject: [PATCH 3/4] docs: rewrite the README and the backends documentation - README: fix the badges (some workflows didn't exist anymore), list every backend with its documentation, document the command line, the signals, all the config options, password hashing, webhook authentication and the docker image tags, use a valid sample config, and replace the dead placekitten.com test file - dropbox: the documentation was a copy of the Google Drive one - gdrive: the base path parameter is base_path, not basePath - keycloak, sftp, mail: write a proper documentation - gcs: document the bucket as the root directory, project_id is unused - fs: the samples weren't valid JSON, explain what each option does - CONTRIBUTING, CLAUDE.md: development workflow and conventions Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 107 +++--------- CONTRIBUTING.md | 26 ++- README.md | 374 ++++++++++++++++++++++-------------------- config-schema.json | 7 +- config/README.md | 28 ++-- fs/README.md | 122 ++++++-------- fs/dropbox/README.md | 23 ++- fs/gcs/README.md | 18 +- fs/gdrive/README.md | 39 ++--- fs/keycloak/README.md | 31 +++- fs/mail/README.md | 43 +++++ fs/sftp/README.md | 47 ++++++ 12 files changed, 472 insertions(+), 393 deletions(-) create mode 100644 fs/mail/README.md create mode 100644 fs/sftp/README.md diff --git a/CLAUDE.md b/CLAUDE.md index aa998afc..33ca6c53 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,89 +4,34 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Commands -### Build & Development - `go build -v ./...` - Build all packages -- `go run main.go` - Run the FTP server locally -- `go run main.go -conf ftpserver.json` - Run with specific config file -- `go run main.go -conf-only` - Generate configuration file only - -### Testing - `go test -race -v ./...` - Run all tests with race detection -- Individual test: `go test -v ./fs/utils/` - -### Linting & Code Quality -- `golangci-lint run` - Run comprehensive linting (configured via .golangci.yml) -- `go fmt ./...` - Format code -- `goimports -w ./` - Fix imports +- `golangci-lint run` - Lint (golangci-lint v2, configured by `.golangci.yml`, run by the CI) +- `go run . -conf ftpserver.json` - Run with a specific config file (`ftpserver.json` is created if no `-conf` is given) +- `go run . -conf-only` - Only create/check the configuration file ## Architecture -### Core Structure -This is a Go FTP server that acts as a gateway between FTP clients and various cloud storage backends. It's built on top of [ftpserverlib](https://github.com/fclairamb/ftpserverlib) and uses the [afero](https://github.com/spf13/afero) filesystem abstraction. - -### Key Components - -#### main.go:25 -Entry point that handles configuration loading, server instantiation, and graceful shutdown. Supports SIGTERM and SIGHUP (config reload) signals. - -#### config/ package -- `config/config.go` - Configuration management and user authentication -- `config/confpar/` - Configuration parameter definitions -- Uses JSON configuration files (default: `ftpserver.json`) - -#### fs/ package - Filesystem Backends -Each subdirectory implements a specific storage backend: -- `fs/afos/` - Local OS filesystem (afero wrapper) -- `fs/s3/` - Amazon S3 storage -- `fs/gdrive/` - Google Drive integration -- `fs/dropbox/` - Dropbox storage -- `fs/gcs/` - Google Cloud Storage -- `fs/sftp/` - SFTP backend -- `fs/mail/` - Email storage backend -- `fs/telegram/` - Telegram bot storage -- `fs/keycloak/` - Keycloak authentication integration -- `fs/utils/` - Shared filesystem utilities -- `fs/stripprefix/` - Path manipulation utilities -- `fs/fslog/` - Filesystem logging wrapper - -#### server/ package -Contains the main FTP server driver implementation that bridges ftpserverlib with the various filesystem backends. - -### Configuration System -- JSON-based configuration with schema validation -- Supports multiple user accounts with different backends -- Each user can have different filesystem backends (S3, Dropbox, etc.) -- TLS/SSL support for secure connections -- Configurable passive transfer port ranges - -### Backend Integration Pattern -Each filesystem backend in `fs/` follows a consistent pattern: -1. Implements afero.Fs interface -2. Handles backend-specific authentication/configuration -3. Maps FTP operations to backend API calls -4. Provides error handling and logging - -### Build Information -- `build.go` contains build-time variables (version, date, commit) -- Cross-platform builds supported via GitHub Actions -- Docker images available and built automatically - -## Development Notes - -### Testing Strategy -- Limited test coverage (only `fs/utils/env_test.go` currently) -- Integration testing likely done via Docker containers -- GitHub Actions runs tests with race detection - -### Code Quality -- Comprehensive golangci-lint configuration with 40+ linters enabled -- Strict formatting and import organization rules -- Local package prefix: `github.com/fclairamb/ftpserver/` -- Line length limit: 120 characters -- Function length limits: 80 lines/40 statements - -### Dependencies -- Go 1.24+ required (toolchain 1.25+) -- Key dependencies: ftpserverlib, afero, various cloud SDK packages -- Logging via go-kit/log framework -- Authentication via go-crypt for password hashing \ No newline at end of file +A Go FTP server acting as a gateway between FTP clients and various storage backends. It's built on top of +[ftpserverlib](https://github.com/fclairamb/ftpserverlib) (the FTP protocol) and uses the +[afero](https://github.com/spf13/afero) `Fs` interface for the backends. + +- `main.go` - Entry point: config loading, server instantiation, signals (`SIGHUP` reloads the config, `SIGTERM` + stops gracefully). +- `config/` - Config file loading and user authentication (plain-text and hashed passwords). + `config/confpar/` holds the config structures, which must stay in sync with `config-schema.json`. +- `server/` - The ftpserverlib driver: settings, client counting, authentication (config or webhook), shared file + system cache, TLS. +- `fs/` - `fs.go` instantiates the backend of an access and applies the common options (read-only, sync & delete). + Each subdirectory is a backend exposing a `LoadFs` function: `afos` (local disk), `s3`, `gcs`, `dropbox`, `gdrive`, + `sftp`, `mail`, `telegram`, `keycloak`. `fslog` wraps a file system to log accesses, `utils` has shared helpers. + +## Conventions + +- Backend parameters are strings read from `access.Params`. Document them in the backend's `README.md` and keep the + JSON schema and the main README backend table up to date. +- Imports: standard library, third-party, then `github.com/fclairamb/ftpserver/...` packages. +- Line length limit: 120 characters. +- Pull request titles follow conventional commits; releases are cut by release-please, which also triggers the + binaries (`release-artifacts.yml`) and docker images (`docker.yml`) builds. +- `build.go` holds the version variables, set at build time with `-ldflags -X`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 34039c8e..3b110305 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,27 @@ # Contributing -If you want to contribute, just add an issue or submit a pull request. +Issues and pull requests are welcome. + +## Development + +```sh +go build -v ./... # Build +go test -race -v ./... # Run the tests +golangci-lint run # Lint (golangci-lint v2, configured by .golangci.yml) +go run . -conf ftpserver.json +``` + +If you change the config file format, update the [JSON schema](config-schema.json) and the documentation accordingly. +Sample config files (in `config/samples/` and `fs/*/`) are validated against the schema by the CI. + +## Adding a backend + +A backend is a package in `fs/` exposing a `LoadFs(access *confpar.Access, ...) (afero.Fs, error)` function, that +reads its parameters from `access.Params`. It must then be registered in [fs/fs.go](fs/fs.go), added to the `fs` enum +of the [JSON schema](config-schema.json), and documented in a `README.md` in its directory, linked from the +[main README](README.md#supported-backends). + +## Pull requests + +Pull request titles must follow the [conventional commits](https://www.conventionalcommits.org/) format +(`feat: ...`, `fix: ...`, `chore: ...`), as they are used to generate the [changelog](CHANGELOG.md) and the releases. diff --git a/README.md b/README.md index f708e7cb..d90e0a3f 100644 --- a/README.md +++ b/README.md @@ -1,77 +1,81 @@ # Golang FTP Server -![Build](https://github.com/fclairamb/ftpserver/workflows/Build/badge.svg) -![Cross Build](https://github.com/fclairamb/ftpserver/workflows/Cross%20Build/badge.svg) -![Docker Image](https://github.com/fclairamb/ftpserver/workflows/Docker%20Image/badge.svg) -![Docker test](https://github.com/fclairamb/ftpserver/workflows/Docker%20test/badge.svg) -![Docker downloads](https://img.shields.io/docker/pulls/fclairamb/ftpserver) -[![Go Report Card](https://goreportcard.com/badge/fclairamb/ftpserver)](https://goreportcard.com/report/fclairamb/ftpserver) -[![GoDoc](https://godoc.org/github.com/fclairamb/ftpserver?status.svg)](https://godoc.org/github.com/fclairamb/ftpserver/server) - -This FTP server is a gateway between old-school FTP devices and modern cloud based file systems, using the -[afero](https://github.com/spf13/afero)'s Fs interface and acting as a reference implementation of the [ftpserverlib](https://github.com/fclairamb/ftpserverlib). - -At the current stage, supported backend are: -- Local disk -- [S3](https://aws.amazon.com/s3/) through [afero-s3](https://github.com/fclairamb/afero-s3) -- [Dropbox](https://www.dropbox.com/) (see [doc](https://github.com/fclairamb/ftpserver/tree/master/fs/dropbox)) through [afero-dropbox](https://github.com/fclairamb/afero-dropbox) -- [Google Drive](https://developers.google.com/drive) (see [doc](https://github.com/fclairamb/ftpserver/tree/master/fs/gdrive)) through [afero-gdrive](https://github.com/fclairamb/afero-gdrive) -- [SFTP](https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol) through [afero's sftpfs](https://github.com/spf13/afero/) -- Email through [go-mail](https://github.com/go-mail/mail) thanks to [@x-way](https://github.com/x-way) -- Telegram through [telebot](https://github.com/tucnak/telebot) by [@slayer](https://github.com/slayer), see [doc](fs/telegram/README.md) - -And with those are supported common parameters to switch them to read-only, enable login access, or use a temporary directory file (see [doc](https://github.com/fclairamb/ftpserver/tree/master/fs)). - -## Current status of the project - -### Features - -#### FTP protocol - -These features are brought by [ftpserverlib](https://github.com/fclairamb/ftpserverlib) itself: - - * Uploading and downloading files - * Directory listing (LIST + MLST) - * File and directory deletion and renaming - * TLS support (AUTH + PROT) - * File download/upload resume support (REST) - * Complete driver for all the above features - * Passive socket connections (EPSV and PASV commands) - * Active socket connections (PORT command) - * Small memory footprint - * Only relies on the standard library except for: - * [go-kit log](https://github.com/go-kit/kit/tree/master/log) for logging - * [afero](https://github.com/spf13/afero) for generic file systems handling - * Supported extensions: - * [AUTH](https://tools.ietf.org/html/rfc2228#page-6) - Control session protection - * [AUTH TLS](https://tools.ietf.org/html/rfc4217#section-4.1) - TLS session - * [PROT](https://tools.ietf.org/html/rfc2228#page-8) - Transfer protection - * [EPRT/EPSV](https://tools.ietf.org/html/rfc2428) - IPv6 support - * [MDTM](https://tools.ietf.org/html/rfc3659#page-8) - File Modification Time - * [SIZE](https://tools.ietf.org/html/rfc3659#page-11) - Size of a file - * [REST](https://tools.ietf.org/html/rfc3659#page-13) - Restart of interrupted transfer - * [MLST](https://tools.ietf.org/html/rfc3659#page-23) - Simple file listing for machine processing - * [MLSD](https://tools.ietf.org/html/rfc3659#page-23) - Directory listing for machine processing +[![Build](https://github.com/fclairamb/ftpserver/actions/workflows/build.yml/badge.svg)](https://github.com/fclairamb/ftpserver/actions/workflows/build.yml) +[![Docker images](https://github.com/fclairamb/ftpserver/actions/workflows/docker.yml/badge.svg)](https://github.com/fclairamb/ftpserver/actions/workflows/docker.yml) +[![Docker test](https://github.com/fclairamb/ftpserver/actions/workflows/docker-test.yml/badge.svg)](https://github.com/fclairamb/ftpserver/actions/workflows/docker-test.yml) +[![Docker pulls](https://img.shields.io/docker/pulls/fclairamb/ftpserver)](https://hub.docker.com/r/fclairamb/ftpserver) +[![Go Report Card](https://goreportcard.com/badge/github.com/fclairamb/ftpserver)](https://goreportcard.com/report/github.com/fclairamb/ftpserver) +[![Go Reference](https://pkg.go.dev/badge/github.com/fclairamb/ftpserver.svg)](https://pkg.go.dev/github.com/fclairamb/ftpserver) + +A gateway between old-school FTP devices (cameras, scanners, printers, legacy software...) and modern cloud based +file systems. + +It is built on top of [ftpserverlib](https://github.com/fclairamb/ftpserverlib), for which it is the reference +implementation, and of the [afero](https://github.com/spf13/afero) `Fs` interface. + +## Supported backends + +| Backend | `fs` value | Documentation | +| --- | --- | --- | +| Local disk | `os` | [below](#local-disk) | +| [Amazon S3](https://aws.amazon.com/s3/) and S3-compatible storages (MinIO, Ceph...), through [afero-s3](https://github.com/fclairamb/afero-s3) | `s3` | [fs/s3](fs/s3/README.md) | +| [Google Cloud Storage](https://cloud.google.com/storage), through afero's [gcsfs](https://github.com/spf13/afero/tree/master/gcsfs) | `gcs` | [fs/gcs](fs/gcs/README.md) | +| [Dropbox](https://www.dropbox.com/), through [afero-dropbox](https://github.com/fclairamb/afero-dropbox) | `dropbox` | [fs/dropbox](fs/dropbox/README.md) | +| [Google Drive](https://developers.google.com/drive), through [afero-gdrive](https://github.com/fclairamb/afero-gdrive) | `gdrive` | [fs/gdrive](fs/gdrive/README.md) | +| [SFTP](https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol), through afero's [sftpfs](https://github.com/spf13/afero/tree/master/sftpfs) | `sftp` | [fs/sftp](fs/sftp/README.md) | +| Email (each uploaded file is sent as an attachment), thanks to [@x-way](https://github.com/x-way) | `mail` | [fs/mail](fs/mail/README.md) | +| Telegram (each uploaded file is sent to a chat), thanks to [@slayer](https://github.com/slayer) | `telegram` | [fs/telegram](fs/telegram/README.md) | +| Local disk with users authenticated by [Keycloak](https://www.keycloak.org/) | `keycloak` | [fs/keycloak](fs/keycloak/README.md) | + +On top of any backend, an access can be made read-only, share its backend instance between sessions, write through a +local temporary directory or log every file access (see [fs/README.md](fs/README.md)). + +## Features + +FTP protocol support is brought by [ftpserverlib](https://github.com/fclairamb/ftpserverlib): + +* Uploading and downloading files, with resume support (`REST`) +* Directory listing (`LIST`, `NLST`, [`MLST`/`MLSD`](https://tools.ietf.org/html/rfc3659#page-23)) +* File and directory deletion and renaming, symbolic links (`SITE SYMLINK`, on the local disk backend) +* Passive (`PASV`, `EPSV`) and active (`PORT`, `EPRT`) transfers, IPv6 support ([RFC 2428](https://tools.ietf.org/html/rfc2428)) +* Explicit TLS ([`AUTH TLS`](https://tools.ietf.org/html/rfc4217#section-4.1), [`PROT`](https://tools.ietf.org/html/rfc2228#page-8)) and implicit TLS +* File modification time ([`MDTM`](https://tools.ietf.org/html/rfc3659#page-8)) and size ([`SIZE`](https://tools.ietf.org/html/rfc3659#page-11)) +* File hashing (`HASH`, `XCRC`, `MD5`, `XSHA256`...), when enabled +* Small memory footprint + +On top of it, this server provides: + +* Multiple users, each one with their own backend +* Hashed passwords (bcrypt, md5crypt, sha256crypt, sha512crypt), plain-text passwords can be hashed automatically +* Authentication through an HTTP webhook +* Configuration hot-reload (`SIGHUP`) and graceful shutdown (`SIGTERM`) ## Getting started -### Get it +### Download it -#### Download it Fetch a binary for your platform from the [latest release](https://github.com/fclairamb/ftpserver/releases/latest) and run it. Archives are named `ftpserver---` and are provided for Linux, macOS and Windows on `amd64` and `arm64`, plus `armv6`/`armv7` for Linux (Raspberry Pi and friends). -#### Build & run it locally +### Build it -```bash -go install github.com/fclairamb/ftpserver@main - -ftpserver & +```sh +go install github.com/fclairamb/ftpserver@latest +ftpserver ``` -#### Run it with docker -There's also a containerized version of the server (31MB, based on alpine). +### Run it with docker + +Images are available on [Docker Hub](https://hub.docker.com/r/fclairamb/ftpserver) and on the +[GitHub container registry](https://github.com/fclairamb/ftpserver/pkgs/container/ftpserver) +(`ghcr.io/fclairamb/ftpserver`) for `linux/amd64`, `linux/arm64`, `linux/arm/v6` and `linux/arm/v7`: + +* `latest`, ``: minimal image, containing only the binary +* `alpine`, `-alpine`: image based on alpine, if you need a shell + +The image runs from the `/app` directory, which is where it reads (or creates) its `ftpserver.json` config file. The +default config file serves the `/tmp` directory. ```sh # Creating a directory @@ -81,157 +85,177 @@ mkdir -p files docker run --rm -d -p 2121-2130:2121-2130 -v $(pwd)/files:/tmp -v $(pwd):/app fclairamb/ftpserver ``` -#### Run it with docker compose +### Run it with docker compose + ```yml # docker-compose.yml - -version: '3.3' services: ftpserver: + image: fclairamb/ftpserver ports: - '2121-2130:2121-2130' volumes: - ./files:/tmp - .:/app - image: fclairamb/ftpserver ``` ```sh -docker-compose up -d +docker compose up -d ``` ### Test it -This is a quick way to see if it's working correctly: ```sh -# Download some file -[ -f kitty.jpg ] || (curl -o kitty.jpg.tmp https://placekitten.com/2048/2048 && mv kitty.jpg.tmp kitty.jpg) +# Create some file +head -c 1048576 /dev/urandom > file.bin # Upload it to the server -curl -v -T kitty.jpg ftp://test:test@localhost:2121/ +curl -v -T file.bin ftp://test:test@localhost:2121/ # Download it back -curl ftp://test:test@localhost:2121/kitty.jpg -o kitty2.jpg +curl ftp://test:test@localhost:2121/file.bin -o file2.bin # Compare it -diff kitty.jpg kitty2.jpg +cmp file.bin file2.bin ``` -### Config file -If you don't create a `ftpserver.json` file, one will be created for you. +## Usage -Here is a sample config file: +``` +ftpserver [-conf ] [-conf-only] +``` + +* `-conf `: config file to use. When not specified, `ftpserver.json` is used, and created if it doesn't exist. +* `-conf-only`: only create the config file (if it doesn't exist) and check it can be loaded, then exit. + +Signals: + +* `SIGHUP` reloads the config file. +* `SIGTERM` stops accepting new connections and waits up to one minute for the connected clients to leave. + +## Configuration + +The configuration is a JSON file, described by a [JSON schema](config-schema.json). Referencing it with the +`$schema` property gives you auto-completion and validation in most editors. If you start the server without any +config file, a sample one is created with a single `test`/`test` user serving `/tmp`. ```json { - "version": 1, - "passive_transfer_port_range": { - "start": 2122, - "end": 2130 - }, - "tls": { - "server_cert": { - "cert": "cert.pem", - "key": "key.pem" + "$schema": "https://raw.githubusercontent.com/fclairamb/ftpserver/main/config-schema.json", + "version": 1, + "listen_address": "0.0.0.0:2121", + "passive_transfer_port_range": { + "start": 2122, + "end": 2130 + }, + "accesses": [ + { + "user": "test", + "pass": "test", + "fs": "os", + "params": { + "basePath": "/tmp" } - }, - "accesses": [ - { - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/tmp" - } - }, - { - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/tmp" - } - }, - { - "user": "dropbox", - "pass": "dropbox", - "fs": "dropbox", - "params": { - "token": "..." - } - }, - { - "user": "gdrive", - "pass": "gdrive", - "fs": "gdrive", - "params": { - "google_client_id": "***.apps.googleusercontent.com", - "google_client_secret": "****", - "base_path": "ftp" - } - }, - { - - "user": "s3", - "pass": "s3", - "fs": "s3", - "params": { - "endpoint": "https://s3.amazonaws.com", - "region": "eu-west-1", - "bucket": "my-bucket", - "access_key_id": "AKIA....", - "secret_access_key": "IDxd....", - "disable_ssl": "false", - "path_style": "false" - } - }, - { - "user": "sftp", - "pass": "sftp", - "fs": "sftp", - "params": { - "username": "user", - "password": "password", - "hostname": "192.168.168.11:22", - "known_hosts": "/home/ftpserver/.ssh/known_hosts" - } - }, - { - "user": "telegram", - "pass": "telegram", - "fs": "telegram", - "shared": true, - "params": { - "Token": "", - "ChatID": "", - "MaxPartSize": "51380224", - "TempDir": "/tmp", - "RetryAttempts": "10", - "RetryDelay": "2000", - "PartUploadDelay": "500" - } + }, + { + "user": "s3", + "pass": "$2a$10$jG7tuqIlcUDMl1m1Ytj1TunU7pk.ko8lj3nOGzZvkIeU/BsfPVBra", + "fs": "s3", + "read_only": true, + "params": { + "bucket": "my-bucket", + "region": "eu-west-1" } - ] + } + ] } ``` -You can generate the TLS key pair files with the following command: -```bash +More samples are available in [config/samples](config/samples). + +### General options + +| Property | Description | Default | +| --- | --- | --- | +| `listen_address` | Address to listen on | `0.0.0.0:2121` | +| `public_host` | Public IP address to advertise for passive transfers, when the server is behind a NAT gateway. Can be overridden with the `PUBLIC_HOST` environment variable | | +| `passive_transfer_port_range` | Ports to use for passive transfers (`start`, `end`) | any port | +| `max_clients` | Maximum number of simultaneous clients, `0` means no limit | `0` | +| `idle_timeout` | Duration after which idle connections are closed (e.g. `5m`), a negative value disables it | `15m` | +| `tls` | TLS certificate files: `{"server_cert": {"cert": "cert.pem", "key": "key.pem"}}` | | +| `tls_required` | `ClearOrEncrypted`, `MandatoryEncryption` (explicit TLS required) or `ImplicitEncryption` (implicit FTPS) | `ClearOrEncrypted` | +| `hash_plaintext_passwords` | Replace the plain-text passwords of the config file by their bcrypt hash | `false` | +| `extensions.enable_hash` | Enable the `HASH` command and its variants | `false` | +| `logging.ftp_exchanges` | Log all the FTP commands and responses | `false` | +| `logging.file_accesses` | Log all the file accesses | `false` | +| `logging.file` | Also write the logs to this file | | +| `accesses` | The users and their backend, see below | | +| `accesses_webhook` | Authenticate users with a webhook instead of the `accesses`, see below | | + +### Accesses + +Each access defines a user and the backend serving their files: + +| Property | Description | +| --- | --- | +| `user` | User name | +| `pass` | Password: plain-text or hashed (`$2a$`/`$2b$`/`$2y$` bcrypt, `$1$` md5crypt, `$5$` sha256crypt, `$6$` sha512crypt). The `anonymous` user with a `*` password accepts any password | +| `fs` | Backend to use (see [supported backends](#supported-backends)) | +| `params` | Backend parameters, as strings. See each backend's documentation | +| `read_only`, `shared`, `sync_and_delete`, `logging` | Common options, see [fs/README.md](fs/README.md) | + +Accesses are evaluated in their declaration order. A hashed password can be generated with +`htpasswd -nbBC 10 "" 'my-password' | cut -d: -f2`, or by enabling `hash_plaintext_passwords`. + +#### Local disk + +The `os` backend serves a local directory, defined by the `basePath` parameter. Environment variables (`$NAME`) in +the path are replaced by their value. + +```json +{ + "user": "test", + "pass": "test", + "fs": "os", + "params": { + "basePath": "/srv/ftp/$FTP_DIR" + } +} +``` + +### Webhook authentication + +When `accesses_webhook` is defined, the `accesses` of the config file are ignored: for each login, the user and +password are posted as JSON (`{"user": "...", "pass": "..."}`) to the webhook. It must reply with a `200` status code +and an [access](#accesses) object, any other status code refuses the login. + +```json +{ + "accesses_webhook": { + "url": "https://auth.example.com/ftp", + "headers": { + "Authorization": "Bearer my-secret-token" + }, + "timeout": "5s" + }, + "accesses": [] +} +``` + +The `timeout` defaults to `30s`. + +### TLS + +You can generate a self-signed TLS key pair with the following command: + +```sh openssl req -new -newkey rsa:4096 -x509 -sha256 -days 365 -nodes -out cert.pem -keyout key.pem ``` -### SFTP backend host key verification +## Contributing -The `sftp` backend connects to an upstream SSH/SFTP server and verifies its host key to -protect against man-in-the-middle attacks. You must configure exactly one of the following -`params`: +See [CONTRIBUTING.md](CONTRIBUTING.md). -- `known_hosts`: path to an OpenSSH `known_hosts` file containing the upstream server's key - (e.g. generated with `ssh-keyscan -H host >> known_hosts`). -- `host_key`: the upstream server's public key pinned inline, in `authorized_keys` / - `known_hosts` line format (e.g. `ssh-ed25519 AAAA...`). -- `insecure_ignore_host_key`: set to `"true"` to disable host key verification entirely. - This is **unsafe** (it exposes the connection to man-in-the-middle attacks) and should only - be used in trusted networks or for testing. +## License -If none of these is set, the connection is refused. +[MIT](LICENSE) diff --git a/config-schema.json b/config-schema.json index 5ad33a09..271e19bd 100644 --- a/config-schema.json +++ b/config-schema.json @@ -64,14 +64,13 @@ }, "idle_timeout": { "type": "string", - "default": "0s", + "default": "15m", "title": "Maximum idle time for client connections", - "description": "Duration after which idle client connections will be closed. Use Go duration format (e.g., '5m', '30s', '1h'). Set to 0 or empty to disable idle timeout.", + "description": "Duration after which idle client connections are closed, in Go duration format (e.g. '5m', '30s', '1h'). Defaults to 15 minutes when unset or 0, a negative value (e.g. '-1s') disables it.", "examples": [ "5m", "30s", - "1h", - "0s" + "1h" ] }, "passive_transfer_port_range": { diff --git a/config/README.md b/config/README.md index 99e235a5..ca6fe323 100644 --- a/config/README.md +++ b/config/README.md @@ -1,20 +1,12 @@ # Sample config files -## FTP Server behind a NAT gateway -```json -{ - "$schema": "https://raw.githubusercontent.com/fclairamb/ftpserver/main/config-schema.json", - "listen_address": ":2121", - "public_host": "1.2.3.4", - "accesses": [ - { - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/tmp" - } - } - ] -} -``` \ No newline at end of file +The config file format is described in the [main README](../README.md#configuration) and by the +[JSON schema](../config-schema.json). + +- [behind_nat.json](samples/behind_nat.json): FTP server behind a NAT gateway, advertising its public IP address + for passive transfers. +- [hashed_passwords.json](samples/hashed_passwords.json): plain-text, bcrypt and sha512crypt passwords (the + password of each user is its user name). +- [log_file.json](samples/log_file.json): logging to a file. + +Each backend also has its own sample in its [documentation](../fs). diff --git a/fs/README.md b/fs/README.md index 860f9a48..1c7c7412 100644 --- a/fs/README.md +++ b/fs/README.md @@ -1,94 +1,78 @@ -# File system loading +# Common access options -On top of the supported file system and their associated config, -they are few parameters that can be enabled. +On top of the backend (`fs`) and its `params`, every access supports a few options. ## Read-only -This blocks any writing call to the underlying file system. It can be abled by the `read_only` boolean parameter. + +The `read_only` boolean blocks any write to the underlying file system. ```json { - "version": 1, - "accesses": [ - { - "read_only": true, - - // The usual FS config: - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/tmp" - } - } - ] + "user": "test", + "pass": "test", + "fs": "os", + "read_only": true, + "params": { + "basePath": "/tmp" + } } -``` +``` ## Shared -This makes sure the underlying filesystem is loaded once per use. It can be enabled by the `shared` boolean parameter. + +The `shared` boolean makes the backend instance created at the first login of the user reused by all their later +sessions, instead of creating a new one for each session. This is useful for backends with an expensive setup (SFTP +connection, Telegram bot...). Shared instances are dropped when the configuration is reloaded. ```json { - "version": 1, - "accesses": [ - { - "shared": true, - // The usual FS config: - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/tmp" - } - } - ] + "user": "test", + "pass": "test", + "fs": "os", + "shared": true, + "params": { + "basePath": "/tmp" + } } -``` +``` ## Sync & Delete -This creates a temporary local folder that will be replicated to a destination file system. +The `sync_and_delete` option makes uploaded files written to a local temporary directory first, and then moved to the +backend once they are complete (see [afero-snd](https://github.com/fclairamb/afero-snd)). The `directory` is optional, +a system temporary directory is used when it isn't specified. ```json { - "version": 1, - "accesses": [ - { - "sync_and_delete": { - "enable": true, // To enable it - "directory": "/tmp/dir" // Temporary directory (optional) - }, - // The usual FS config: - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/target" - } - } - ] + "user": "test", + "pass": "test", + "fs": "os", + "sync_and_delete": { + "enable": true, + "directory": "/tmp/snd" + }, + "params": { + "basePath": "/target" + } } -``` +``` ## Logging + +The `logging` option enables the logging of the FTP exchanges and/or the file accesses of this access only. The same +options exist at the server level to enable them for all the accesses. + ```json { - "version": 1, - "accesses": [ - { - "logging": { - "file_accesses": true, // This logs every single file access - "ftp_exchanges": true // This logs every single FTP command - }, - // The usual FS config: - "user": "test", - "pass": "test", - "fs": "os", - "params": { - "basePath": "/target" - } - } - ] + "user": "test", + "pass": "test", + "fs": "os", + "logging": { + "ftp_exchanges": true, + "file_accesses": true + }, + "params": { + "basePath": "/target" + } } -``` \ No newline at end of file +``` diff --git a/fs/dropbox/README.md b/fs/dropbox/README.md index 75cd1e70..8c7224cf 100644 --- a/fs/dropbox/README.md +++ b/fs/dropbox/README.md @@ -1,32 +1,31 @@ # Dropbox backend ## Parameters -Possible parameters are: -- `token`: The non-expiring authentication token to use + +- `token`: The Dropbox access token to use. If it isn't set, the `DROPBOX_TOKEN` environment variable is used. ## How to use it + ### Create an app -You can follow -[these instructions](https://medium.com/swlh/google-drive-api-with-python-part-i-set-up-credentials-1f729cb0372b). -### Use the created client credentials -You can either define the `google_client_id` and `google_client_secret` access params or specify the `GOOGLE_CLIENT_ID` -and `GOOGLE_CLIENT_SECRET` environment variables. +1. Create an app in the [Dropbox App Console](https://www.dropbox.com/developers/apps), with the access type you + need ("App folder" restricts the FTP server to a dedicated folder, "Full Dropbox" gives access to everything). +2. In the "Permissions" tab, enable at least `files.metadata.read`, `files.content.read` and `files.content.write`. +3. In the "Settings" tab, generate an access token. + +### Configure the access -You should have a config file looking like this one: ```json { "accesses": [ - { + { "user": "dropbox", "pass": "dropbox", "fs": "dropbox", "params": { - "google_client_id": "*****.apps.googleusercontent.com", - "google_client_secret": "*****" + "token": "sl.*****" } } ] } ``` - diff --git a/fs/gcs/README.md b/fs/gcs/README.md index dc333845..b7c45dd7 100644 --- a/fs/gcs/README.md +++ b/fs/gcs/README.md @@ -1,6 +1,7 @@ # Google Cloud Storage (GCS) filesystem -This directory contains the Google Cloud Storage implementation for the FTP server using the official `github.com/spf13/afero/gcsfs` package. +This backend gives access to a Google Cloud Storage bucket, using afero's +[gcsfs](https://github.com/spf13/afero/tree/master/gcsfs) package. ## Configuration @@ -12,11 +13,10 @@ The GCS filesystem supports the following configuration parameters: "accesses": [ { "user": "test", - "pass": "test", + "pass": "test", "fs": "gcs", "params": { "bucket": "your-gcs-bucket-name", - "project_id": "your-project-id", "key_file": "/path/to/service-account-key.json" } } @@ -26,9 +26,10 @@ The GCS filesystem supports the following configuration parameters: ### Parameters -- `bucket` (required): The GCS bucket name to use -- `project_id` (optional): Google Cloud Project ID. If not provided, it will use the default project from the environment or service account -- `key_file` (optional): Path to a service account key JSON file for authentication. If not provided, it will use default credentials (environment variables, metadata service, etc.) +- `bucket` (required): The GCS bucket name to use. It is the root directory of the FTP access. +- `key_file` (optional): Path to a service account key JSON file for authentication. If not provided, the default + credentials are used (environment variables, metadata service, etc.) +- `project_id`: Not used (the bucket name is enough to access it), it is accepted for backward compatibility. ### Authentication @@ -48,7 +49,6 @@ The GCS filesystem supports multiple authentication methods: "fs": "gcs", "params": { "bucket": "my-ftp-bucket", - "project_id": "my-project-123", "key_file": "/etc/gcs-key.json" } } @@ -57,7 +57,7 @@ The GCS filesystem supports multiple authentication methods: #### With default credentials: ```json { - "fs": "gcs", + "fs": "gcs", "params": { "bucket": "my-ftp-bucket" } @@ -68,4 +68,4 @@ The GCS filesystem supports multiple authentication methods: - GCS doesn't have real directories, so directory operations are emulated - File permissions and ownership operations are no-ops since GCS doesn't support them -- Seeking to end of file and some advanced file operations are not supported due to GCS limitations \ No newline at end of file +- Seeking to end of file and some advanced file operations are not supported due to GCS limitations diff --git a/fs/gdrive/README.md b/fs/gdrive/README.md index 4816e923..f73d1e26 100644 --- a/fs/gdrive/README.md +++ b/fs/gdrive/README.md @@ -1,36 +1,38 @@ # Google Drive backend ## Parameters -Possible parameters are: -- `google_client_id`: The Google Client ID -- `google_client_secret`: The Google Client Secret, it shall never be shared -- `token_file`: The path of the token file that will be created -- `base_path`: The base path of the Google Drive folder -_Note:_ The token file will be updated regularly (every day). It's important that the server can write the token file -after its initial creation. +- `google_client_id`: The Google client ID. If it isn't set, the `GOOGLE_CLIENT_ID` environment variable is used. +- `google_client_secret`: The Google client secret, it shall never be shared. If it isn't set, the + `GOOGLE_CLIENT_SECRET` environment variable is used. +- `token_file` (optional): The path of the token file that will be created. Defaults to `gdrive_token_.json`. +- `base_path` (optional): The Google Drive folder to use as the root directory. Environment variables (`$NAME`) are + replaced by their value. + +_Note:_ The token file is updated regularly (every day). The server must be able to write the token file after its +initial creation. ## How to use it + ### Create an app -You can follow -[these instructions](https://medium.com/swlh/google-drive-api-with-python-part-i-set-up-credentials-1f729cb0372b). -### Use the created client credentials -You can either define the `google_client_id` and `google_client_secret` access params or specify the `GOOGLE_CLIENT_ID` -and `GOOGLE_CLIENT_SECRET` environment variables. +1. In the [Google Cloud console](https://console.cloud.google.com/), create a project and enable the Google Drive API. +2. Configure the OAuth consent screen. +3. Create an "OAuth client ID" credential of the "Desktop app" type, and note its client ID and client secret. + +### Configure the access -You should have a config file looking like this one: ```json { "accesses": [ - { + { "user": "gdrive", "pass": "gdrive", "fs": "gdrive", "params": { "google_client_id": "*****.apps.googleusercontent.com", "google_client_secret": "*****", - "basePath": "/ftp" + "base_path": "ftp" } } ] @@ -38,8 +40,7 @@ You should have a config file looking like this one: ``` ### Connect for the first time -You should run the ftpserver in interactive mode. -The first time you will login with the "gdrive" user, the server will ask you to open a link, wich -will authorize the app you just created and return a code. This code will allow the server to create -a token file (named `gdrive_token_$username.json` by default). +Run the server in interactive mode. The first time you log in with the "gdrive" user, the server prints a link +that you need to open to authorize the app you just created. It gives you a code that you need to type in the +server's console. The server then creates the token file, and won't ask you again. diff --git a/fs/keycloak/README.md b/fs/keycloak/README.md index c3aacfe9..be2946e9 100644 --- a/fs/keycloak/README.md +++ b/fs/keycloak/README.md @@ -1,16 +1,37 @@ +# Keycloak backend + +This backend serves local directories to users authenticated by a [Keycloak](https://www.keycloak.org/) server. +Each user gets their own `/` directory. + +## Parameters + +- `keycloak_url`: The URL of the Keycloak server. +- `keycloak_realm`: The realm of the users. +- `keycloak_client_id`: The client ID. If it isn't set, the `KEYCLOAK_CLIENT_ID` environment variable is used. +- `keycloak_client_secret`: The client secret. If it isn't set, the `KEYCLOAK_CLIENT_SECRET` environment variable is + used. +- `base_path`: The local directory containing the users' directories. Environment variables (`$NAME`) are replaced by + their value. + +The client must allow the "Direct Access Grants" (resource owner password credentials) flow, which is how the +credentials sent by the FTP clients are checked. + +## Configuration + +A keycloak access doesn't have any `user` or `pass`, it accepts any user that Keycloak authenticates. As accesses are +evaluated in their declaration order, it should be declared last. -You should have a config file looking like this one: ```json { "accesses": [ - { + { "fs": "keycloak", "params": { - "keycloak_client_id": "*****", - "keycloak_client_secret": "*****", "keycloak_url": "http://localhost:8080", "keycloak_realm": "master", - "base_path": "/ftp" + "keycloak_client_id": "ftpserver", + "keycloak_client_secret": "*****", + "base_path": "/srv/ftp" } } ] diff --git a/fs/mail/README.md b/fs/mail/README.md new file mode 100644 index 00000000..47fe67af --- /dev/null +++ b/fs/mail/README.md @@ -0,0 +1,43 @@ +# Mail backend + +This is a write-only backend: each uploaded file is sent as an email attachment. Listing directories always returns +an empty result. + +## Parameters + +- `Host`: The SMTP server. +- `Port` (optional): The SMTP port. Defaults to `25`. +- `SSL` (optional): `true` to connect with SSL/TLS (SMTPS). Defaults to `true` on port `465`, `false` otherwise. +- `StartTLSPolicy` (optional): `MandatoryStartTLS` (default), `OpportunisticStartTLS` or `NoStartTLS`. +- `Username`, `Password` (optional): The SMTP credentials. +- `Localname` (optional): The host name sent in the `HELO` command. +- `From`: The sender address. +- `To`: The recipient address. +- `Subject`: The subject of the emails. +- `Message`: The body of the emails, `%s` is replaced by the file name. + +## Configuration + +See [example.json](example.json): + +```json +{ + "accesses": [ + { + "user": "test", + "pass": "test", + "fs": "mail", + "params": { + "From": "alice@example.com", + "To": "bob@example.com", + "Subject": "New File", + "Message": "A new file has been uploaded: %s", + "Host": "smtp.example.com", + "Port": "465", + "Username": "smtpuser", + "Password": "smtppassword" + } + } + ] +} +``` diff --git a/fs/sftp/README.md b/fs/sftp/README.md new file mode 100644 index 00000000..ad0b015b --- /dev/null +++ b/fs/sftp/README.md @@ -0,0 +1,47 @@ +# SFTP backend + +This backend connects to an upstream SSH/SFTP server. + +## Parameters + +- `hostname`: The upstream server address, with its port (e.g. `192.168.1.10:22`). +- `username`: The SSH user. +- `password`: The SSH password. +- One of the host key verification parameters below. + +## Host key verification + +The upstream server's host key is verified to protect against man-in-the-middle attacks. You must configure exactly +one of the following parameters: + +- `known_hosts`: path to an OpenSSH `known_hosts` file containing the upstream server's key + (e.g. generated with `ssh-keyscan -H host >> known_hosts`). +- `host_key`: the upstream server's public key pinned inline, in `authorized_keys` / `known_hosts` line format + (e.g. `ssh-ed25519 AAAA...`). +- `insecure_ignore_host_key`: set to `"true"` to disable host key verification entirely. This is **unsafe** (it + exposes the connection to man-in-the-middle attacks) and should only be used in trusted networks or for testing. + +If none of these is set, the connection is refused. + +## Configuration + +```json +{ + "accesses": [ + { + "user": "sftp", + "pass": "sftp", + "fs": "sftp", + "shared": true, + "params": { + "hostname": "192.168.1.10:22", + "username": "user", + "password": "password", + "known_hosts": "/home/ftpserver/.ssh/known_hosts" + } + } + ] +} +``` + +With `shared` enabled, a single SSH connection is used for all the FTP sessions of this access. From 0071ee5b1eaba77eee762c277fa4a4aea7a8d6c6 Mon Sep 17 00:00:00 2001 From: fclairamb Date: Thu, 22 Oct 2026 08:19:19 +0200 Subject: [PATCH 4/4] ci: test the docker images upload from the chore/cleanup branch To be removed before merging. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c4df94aa..70552b9e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,6 +19,9 @@ on: push: tags: - "v*.*.*" + # TODO: remove before merging, only used to test the Docker Hub / GHCR upload from this branch + branches: + - chore/cleanup pull_request: paths: - Dockerfile* @@ -76,6 +79,7 @@ jobs: type=semver,pattern={{major}},value=${{ steps.tag.outputs.tag }} type=semver,suffix=,pattern=${{ matrix.type != '' && matrix.type || 'latest' }},value=${{ steps.tag.outputs.tag }} type=ref,event=pr + type=ref,event=branch - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4