docker-compose.yml

name: marketos

x-logging: &default-logging
  driver: json-file
  options:
    max-size: "10m"
    max-file: "5"

services:
  # Self-hosted PgBouncer in front of Aiven Postgres — Aiven's Free plan is
  # capped at 20 connections and doesn't include managed pooling (that's
  # gated to paid plans). Transaction-pooling mode here means every
  # worker/service below shares a small number of real Aiven connections
  # instead of each holding its own, without which this stack's worker count
  # alone can exhaust the cap. Two paths deliberately bypass this and talk to
  # Aiven directly via DIRECT_DATABASE_URL: live_updates.py's LISTEN/NOTIFY
  # connection (session-scoped, incompatible with transaction pooling) and
  # Prisma Migrate (its schema engine doesn't support running through a
  # pooler).
  # DEFAULT_POOL_SIZE=8, not higher: production (docker-compose.prod.yml) runs
  # its own separate PgBouncer against this same Aiven instance (see
  # deploy/RUNBOOK.md — dev and prod share one database), so the two
  # PgBouncers' pools are additive against the 20-connection cap. 8+8=16
  # leaves headroom for both sides' direct-bypass connections.
  pgbouncer:
    image: edoburu/pgbouncer:latest
    container_name: marketos_pgbouncer
    environment:
      - DATABASE_URL=${PGBOUNCER_UPSTREAM_URL}
      - LISTEN_PORT=6432
      - POOL_MODE=transaction
      - AUTH_TYPE=scram-sha-256
      - SERVER_TLS_SSLMODE=require
      - MAX_CLIENT_CONN=100
      - DEFAULT_POOL_SIZE=8
    healthcheck:
      test: ["CMD", "pg_isready", "-h", "localhost", "-p", "6432"]
      interval: 5s
      timeout: 3s
      retries: 5

  redis:
    image: redis:7-alpine
    container_name: marketos_redis
    command: ["redis-server", "--appendonly", "yes"]
    ports:  
      - "6379:6379"
    volumes:
      - redis_data:/data
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 3s
      retries: 5

  # Merged scraper + extraction + scheduler + matcher into one process.
  # All four ran at --concurrency=1 under Celery's default prefork pool,
  # which forks a child even at concurrency=1 (parent + 1 child = ~2x a
  # single process's memory, per worker container). None of these tasks
  # are CPU-bound (they wait on Firecrawl/Groq/DB calls), so --pool=solo
  # (no fork, single-threaded) drops that multiplier entirely, and
  # merging 4 containers into 1 removes 3x the fixed per-process import
  # overhead (Prisma client, Celery, Google Cloud SDK). Rate-limit
  # budgets stay separate — each task still reads its own GROQ_API_KEY*
  # env var by name, merging processes doesn't merge quotas.
  core-worker:
    build:
      context: .
      dockerfile: Dockerfile.python
    container_name: core_worker
    logging: *default-logging
    environment:
      - DATABASE_URL=${PGBOUNCER_DATABASE_URL}
      - REDIS_URL=redis://redis:6379/0
      - FIRECRAWL_API_KEY=${FIRECRAWL_API_KEY}
      - GROQ_API_KEY=${GROQ_API_KEY}
      - GROQ_API_KEY_BACKUP=${GROQ_API_KEY_BACKUP}
      - GROQ_API_KEY_CANDIDATE=${GROQ_API_KEY_CANDIDATE}
      - GCS_IMAGE_BUCKET=${GCS_IMAGE_BUCKET}
      - GCS_MARKDOWN_BUCKET=${GCS_MARKDOWN_BUCKET}
      - GOOGLE_CLOUD_PROJECT=${VERTEX_PROJECT}
      - GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/application_default_credentials.json
    depends_on:
      redis:
        condition: service_healthy
      pgbouncer:
        condition: service_healthy
    volumes:
      - ./services:/app/services
      - ~/.config/gcloud:/root/.config/gcloud:ro
    command: ["uv", "run", "celery", "-A", "services.common.celery_app", "worker",
              "-Q", "scraping_queue,extraction_queue,scheduler_queue,match_queue",
              "-n", "core-worker@%h", "--pool=solo", "--loglevel=info"]

  # Merged worker for discovery, competitor/shopify semantic generation, and
  # embedding — none of these are individually high-throughput enough to
  # justify their own process. Each still uses its own env-scoped Groq/Vertex
  # key inside the task code, so merging processes doesn't merge rate-limit
  # budgets. Was 4 separate processes (1+1+1+2 = 5 DB connections); merging
  # cuts that to 2 (--concurrency=2, matching embedding's prior concurrency).
  # --pool=threads (not the prefork default) since these tasks are I/O-bound
  # (Groq/Vertex/Serper calls) — avoids paying prefork's per-child-process
  # memory duplication for concurrency that doesn't need real process isolation.
  content-worker:
    build:
      context: .
      dockerfile: Dockerfile.python
    container_name: content_worker
    logging: *default-logging
    environment:
      - DATABASE_URL=${PGBOUNCER_DATABASE_URL}
      - REDIS_URL=redis://redis:6379/0
      - SERPER_API_KEY=${SERPER_API_KEY}
      - GROQ_API_KEY=${GROQ_API_KEY}
      - GROQ_API_KEY_BACKUP=${GROQ_API_KEY_BACKUP}
      - GROQ_API_KEY_SEMANTIC=${GROQ_API_KEY_SEMANTIC}
      - GROQ_API_KEY_SHOPIFY=${GROQ_API_KEY_SHOPIFY}
      - VERTEX_PROJECT=${VERTEX_PROJECT}
      - VERTEX_LOCATION=${VERTEX_LOCATION}
      - GOOGLE_CLOUD_PROJECT=${VERTEX_PROJECT}
      - GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/application_default_credentials.json
    depends_on:
      redis:
        condition: service_healthy
      pgbouncer:
        condition: service_healthy
    volumes:
      - ./services:/app/services
      - ~/.config/gcloud:/root/.config/gcloud:ro
    command: ["uv", "run", "celery", "-A", "services.common.celery_app", "worker",
              "-Q", "discovery_queue,semantic_queue,shopify_semantic_queue,embedding_queue",
              "-n", "content-worker@%h", "--pool=threads", "--concurrency=2", "--loglevel=info"]

  # Single worker handles all dynamic-pricing queues — stats, pricing decisions,
  # Shopify sync (orders/inventory pulls), and Shopify write-back. Combined
  # (both here and in production) to keep memory footprint low — none of
  # these queues have enough volume yet to need independent-process scaling.
  # --pool=threads: I/O-bound tasks, avoids prefork's per-child memory cost.
  pricing-worker:
    build:
      context: .
      dockerfile: Dockerfile.python
    container_name: pricing_worker
    logging: *default-logging
    env_file: .env
    environment:
      - DATABASE_URL=${PGBOUNCER_DATABASE_URL}
      - REDIS_URL=redis://redis:6379/0
      - INTERNAL_API_TOKEN=${INTERNAL_API_TOKEN}
      - SHOPIFY_API_KEY=${SHOPIFY_API_KEY}
      - SHOPIFY_API_SECRET=${SHOPIFY_API_SECRET}
      # Overrides the root .env's APP_URL=http://localhost:3000 (only valid
      # inside the `web` container itself) — this container must reach `web`
      # over the Docker network by service name, on its internal port 3000.
      - APP_URL=http://web:3000
    extra_hosts:
      - "host.docker.internal:host-gateway"
    depends_on:
      redis:
        condition: service_healthy
      pgbouncer:
        condition: service_healthy
    volumes:
      - ./services:/app/services
    command: ["uv", "run", "celery", "-A", "services.common.celery_app", "worker",
              "-Q", "stats_queue,pricing_queue,writer_queue,shopify_sync_queue",
              "-n", "pricing-worker@%h", "--pool=threads", "--concurrency=2", "--loglevel=info"]

  celery-beat:
    build:
      context: .
      dockerfile: Dockerfile.python
    container_name: celery_beat
    logging: *default-logging
    environment:
      - DATABASE_URL=${PGBOUNCER_DATABASE_URL}
      - REDIS_URL=redis://redis:6379/0
    depends_on:
      redis:
        condition: service_healthy
      pgbouncer:
        condition: service_healthy
    volumes:
      - ./services:/app/services
    command: ["uv", "run", "celery", "-A", "services.common.celery_app", "beat", "--loglevel=info"]

  api-gateway:
    build:
      context: .
      dockerfile: Dockerfile.python
    container_name: api_gateway
    environment:
      - DATABASE_URL=${PGBOUNCER_DATABASE_URL}
      # Bypasses PgBouncer — live_updates.py holds one dedicated LISTEN
      # connection, which requires a session-stable connection, incompatible
      # with transaction pooling.
      - DIRECT_DATABASE_URL=${DIRECT_DATABASE_URL}
      - REDIS_URL=redis://redis:6379/0
      - INTERNAL_API_TOKEN=${INTERNAL_API_TOKEN}
      - SHOPIFY_API_KEY=${SHOPIFY_API_KEY}
      - SHOPIFY_API_SECRET=${SHOPIFY_API_SECRET}
    ports:
      - "8000:8000"
    depends_on:
      redis:
        condition: service_healthy
      pgbouncer:
        condition: service_healthy
    volumes:
      - ./services:/app/services
    command: ["uv", "run", "uvicorn", "services.api_gateway.main:app", "--host", "0.0.0.0", "--port", "8000"]

  # Production build of the React Router app (embedded UI + internal apply
  # routes). Browser-based dev keeps using `npm run dev` on host port 3000;
  # this instance serves the in-stack consumers (chatbot apply tools, evals)
  # and is published on host port 3300 to avoid clashing with the dev server.
  web:
    build:
      context: ./shopify_ui
    container_name: marketos_web
    environment:
      - DATABASE_URL=${PGBOUNCER_DATABASE_URL}
      # Prisma Migrate's schema engine doesn't support running through a
      # pooler — schema.prisma's datasource uses this as its directUrl.
      - DIRECT_DATABASE_URL=${DIRECT_DATABASE_URL}
      - SHOPIFY_API_KEY=${SHOPIFY_API_KEY}
      - SHOPIFY_API_SECRET=${SHOPIFY_API_SECRET}
      # Must be a valid URL for shopifyApp(); OAuth through this instance is
      # not used (browser dev flow stays on the Shopify CLI tunnel).
      - SHOPIFY_APP_URL=${SHOPIFY_APP_URL:-http://localhost:3300}
      - SCOPES=read_analytics,read_inventory,read_orders,read_product_listings,read_products,write_inventory,write_product_listings,write_products
      # Base URL app.apply-preview uses to call its own /internal/* routes.
      - APP_URL=http://localhost:3000
      - INTERNAL_API_TOKEN=${INTERNAL_API_TOKEN}
      - CHATBOT_SVC_URL=http://chatbot-svc:8088
      - PYTHON_API_URL=http://api-gateway:8000
      - MARKETOS_DEMO_TENANT_ID=${MARKETOS_DEMO_TENANT_ID}
      - GROQ_API_KEY=${GROQ_API_KEY}
      - GROQ_REFINER_MODEL=${GROQ_REFINER_MODEL:-}
      - RESEND_API_KEY=${RESEND_API_KEY}
      - RESEND_FROM_EMAIL=${RESEND_FROM_EMAIL}
    ports:
      - "3300:3000"
    depends_on:
      redis:
        condition: service_healthy
      pgbouncer:
        condition: service_healthy

  chatbot-svc:
    build:
      context: .
      dockerfile: Dockerfile.python
    container_name: chatbot_svc
    environment:
      - DATABASE_URL=${PGBOUNCER_DATABASE_URL}
      - REDIS_URL=redis://redis:6379/0
      - GROQ_API_KEY=${GROQ_API_KEY}
      - GROQ_API_KEY_CHATBOT=${GROQ_API_KEY_CHATBOT}
      - CHATBOT_MODEL=${CHATBOT_MODEL}
      - CHATBOT_FALLBACK_MODEL=${CHATBOT_FALLBACK_MODEL:-}
      # Apply actions POST to the React Router app's internal routes. Defaults
      # to the in-stack web service; set REACT_ROUTER_INTERNAL_URL to
      # http://host.docker.internal:3000 to target `npm run dev` instead
      # (e.g. when live-editing RR internal routes).
      - CHATBOT_RR_URL=${REACT_ROUTER_INTERNAL_URL:-http://web:3000}
      - INTERNAL_API_TOKEN=${INTERNAL_API_TOKEN}
      # Query Studio grounding fetches real competitor brands via Serper when the
      # product has no already-scraped competitors.
      - SERPER_API_KEY=${SERPER_API_KEY}
      - VERTEX_PROJECT=${VERTEX_PROJECT}
      - VERTEX_LOCATION=${VERTEX_LOCATION}
      - GOOGLE_CLOUD_PROJECT=${VERTEX_PROJECT}
      - GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/application_default_credentials.json
      # Uploads agent traces to Logfire; unset token = console-only (if-token-present).
      - LOGFIRE_TOKEN=${LOGFIRE_TOKEN}
    ports:
      - "8088:8088"
    extra_hosts:
      - "host.docker.internal:host-gateway"
    depends_on:
      redis:
        condition: service_healthy
      pgbouncer:
        condition: service_healthy
    volumes:
      - ./services:/app/services
      - ~/.config/gcloud:/root/.config/gcloud:ro
    command: ["uv", "run", "uvicorn", "services.chatbot_svc.app:app", "--host", "0.0.0.0", "--port", "8088"]

  dozzle:
    image: amir20/dozzle:latest
    container_name: dozzle
    ports:
      - "8888:8080"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro

volumes:
  redis_data: