diff --git a/.circleci/config.yml b/.circleci/config.yml index 712804a..cf20c95 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,107 +1,143 @@ version: 2.1 - -jobs: - quality: +executors: + reviewed: docker: - - image: node:26.8.1-bookworm - - image: postgres:16-alpine - environment: - POSTGRES_PASSWORD: forms_local - POSTGRES_DB: forms_processor_test - environment: - TEST_DATABASE_URL: postgresql://postgres:forms_local@localhost:5432/forms_processor_test + - image: cimg/aws@sha256:4cf3768db00ddf9bc439b8c84e8a60c11a14edf81c8baf76f74132144f8340d5 +commands: + checkout-source: steps: - - checkout - - run: npm ci - - run: - name: Lint, type-check, test routing and retries, and compile - command: | - npm run lint - npm run typecheck - npm test - npm run test:integration - npm run build - deployment-checks: - docker: - - image: cimg/python:3.13.2 + - run: + name: Check out the exact public source commit over HTTPS + command: | + set -euo pipefail + [[ "$CIRCLE_SHA1" =~ ^[0-9a-f]{40}$ ]] + export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 + git init --quiet . + git remote add origin https://github.com/topcoder-platform/forms-processor-v6.git + git -c credential.helper= fetch --quiet --depth 1 origin "$CIRCLE_SHA1" + git checkout --quiet --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$CIRCLE_SHA1" + checkout-suite: + steps: + - run: + name: Verify deployment suite v2.0.5 + command: | + set -euo pipefail + git init --quiet /tmp/tc-deploy-scripts + git -C /tmp/tc-deploy-scripts remote add origin https://github.com/topcoder-platform/tc-deploy-scripts.git + git -C /tmp/tc-deploy-scripts fetch --quiet --depth 1 origin refs/tags/v2.0.5 + git -C /tmp/tc-deploy-scripts checkout --quiet --detach FETCH_HEAD + test "$(git -C /tmp/tc-deploy-scripts rev-parse HEAD)" = bff574ab5c0b46773e25d2af1eff76675fba2ba5 +jobs: + build: + executor: reviewed steps: - - checkout - - run: pip install boto3 - - run: python3 -m unittest discover -s deploy/tests + - checkout-source + - setup_remote_docker + - checkout-suite + - run: + name: Build committed source without AWS or broker credentials + command: | + set -euo pipefail + python3 -I /tmp/tc-deploy-scripts/build_image.py --dockerfile Dockerfile --image forms-processor-v6:candidate + mkdir -p /tmp/release + docker save --output /tmp/release/image.tar forms-processor-v6:candidate + no_output_timeout: 30m + - persist_to_workspace: + root: /tmp/release + paths: + - image.tar deploy: - docker: - - image: cimg/python:3.13.2-browsers + executor: reviewed parameters: - deploy_env: - type: enum - enum: [DEV, PROD] environment: type: enum - enum: [dev, production] + enum: + - dev + - prod environment: - AWS_REGION: us-east-1 - AWS_DEFAULT_REGION: us-east-1 - AWS_STS_REGIONAL_ENDPOINTS: regional - AWS_RETRY_MODE: standard - AWS_MAX_ATTEMPTS: "10" - APPNAME: forms-processor-v6 - DEPLOY_ENV: << parameters.deploy_env >> - DEPLOYMENT_ENVIRONMENT: << parameters.environment >> + DEPLOY_ENV: << parameters.environment >> + steps: + - checkout-source + - setup_remote_docker + - checkout-suite + - attach_workspace: + at: /tmp/release + - run: + name: Promote a scanned digest with repository-bound OIDC + command: | + set +x + set -euo pipefail + test -f "deploy/${DEPLOY_ENV}.json" + docker load --input /tmp/release/image.tar + python3 -I deploy/release.py "deploy/${DEPLOY_ENV}.json" + no_output_timeout: 45m + quality: + docker: + - image: node@sha256:9f94d34c787165dca03b74e5bf9c3bf90e8de79b19aa3d87fe1fa1694bf75c89 + - image: postgres@sha256:0cefaabd882e42bcb2a2f3dc56d3ce4770560ba62fd7f197f1a2e008fd5731ab + environment: + POSTGRES_PASSWORD: forms_local + POSTGRES_DB: forms_processor_test + environment: + TEST_DATABASE_URL: postgresql://postgres:forms_local@localhost:5432/forms_processor_test + steps: + - checkout-source + - run: npm ci --install-links=true + - run: + name: Lint, type-check, test routing and retries, and compile + command: | + npm run lint + npm run typecheck + npm test + npm run test:integration + npm run build + deployment-checks: + executor: reviewed steps: - - checkout - - setup_remote_docker - - run: - name: Install deployment dependencies - command: | - sudo apt-get update -qq - sudo apt-get install -y -qq jq python3-pip - pip install boto3 awscli - - run: - name: Install the pinned Topcoder AWS credential helper - command: | - git -c advice.detachedHead=false clone --quiet --depth 1 --branch v1.4.20 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript - test "$(git -C ../buildscript rev-parse HEAD)" = 5f3745c35463ce0e2475c12269aa7a2b446456e5 - cp ../buildscript/awsconfiguration.sh . - - run: - name: Build tested runtime image - command: docker buildx build --load --tag "${APPNAME}:candidate" . - - run: - name: Deploy a new task revision to the existing ECS service - no_output_timeout: 35m - command: | - set -euo pipefail - set +x - trap 'rm -f awsenvconf awsconfiguration.sh' EXIT - ./awsconfiguration.sh "$DEPLOY_ENV" "$AWS_REGION" - source awsenvconf - python3 -u deploy/release.py "$DEPLOYMENT_ENVIRONMENT" "${DEPLOYMENT_ENVIRONMENT}-${CIRCLE_SHA1}-${CIRCLE_BUILD_NUM}" - - store_artifacts: - path: deploy/release-<< parameters.environment >>.json - destination: release + - checkout-suite + - run: + name: Test the pinned deployment security and rollout checks + command: python3 -m unittest discover -s /tmp/tc-deploy-scripts/tests -q workflows: - build-and-deploy: + release: jobs: - - quality - - deployment-checks - - deploy: - name: deploy-dev - context: org-global - requires: [quality, deployment-checks] - deploy_env: DEV - environment: dev - serial-group: << pipeline.project.slug >>/deploy-dev - filters: - branches: - only: develop - tags: - only: /^dev-.*/ - - deploy: - name: deploy-production - context: org-global - requires: [quality, deployment-checks] - deploy_env: PROD - environment: production - serial-group: << pipeline.project.slug >>/deploy-production - filters: - branches: - only: master + - quality + - deployment-checks + - build: + requires: + - quality + - deployment-checks + - deploy: + name: deploy-dev + environment: dev + requires: + - build + context: forms-processor-v6-dev-deploy + serial-group: << pipeline.project.slug >>/deploy-dev + filters: + branches: + only: develop + tags: + ignore: /.*/ + - approve-prod: + type: approval + requires: + - build + filters: + branches: + only: master + tags: + ignore: /.*/ + - deploy: + name: deploy-prod + environment: prod + requires: + - approve-prod + context: forms-processor-v6-prod-deploy + serial-group: << pipeline.project.slug >>/deploy-prod + filters: + branches: + only: master + tags: + ignore: /.*/ diff --git a/.gitignore b/.gitignore index ded37c1..3532471 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ awsenvconf awsconfiguration.sh __pycache__/ deploy/release-*.json + +# Infrastructure definitions and detailed AWS inventories are maintained privately. +/deploy/iam/ +/deploy/*-baseline.json diff --git a/.npmrc b/.npmrc deleted file mode 100644 index c7d3517..0000000 --- a/.npmrc +++ /dev/null @@ -1 +0,0 @@ -install-links=true diff --git a/Dockerfile b/Dockerfile index 339e971..4bea4b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,14 @@ ARG NODE_VERSION=26.12.1 FROM node:${NODE_VERSION}-alpine AS build WORKDIR /app -COPY package.json package-lock.json .npmrc ./ +COPY package.json package-lock.json ./ COPY tools ./tools -RUN npm ci +RUN npm ci --install-links=true COPY . . RUN npm run lint && npm run typecheck && npm test && npm run test:dependency-compat && npm run build FROM build AS production-dependencies -RUN npm prune --omit=dev +RUN npm prune --install-links=true --omit=dev FROM alpine:3.24 AS runtime ARG NODE_VERSION diff --git a/README.md b/README.md index 4a07de6..3d134d1 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Submissions that do not opt in stay in the API and do not reach this processor. ```sh nvm use -npm ci +npm ci --install-links=true cp .env.example .env # Apply forms-api-v6 migrations to your local forms database first. npm run dev diff --git a/deploy/MIGRATION.md b/deploy/MIGRATION.md new file mode 100644 index 0000000..08cffb1 --- /dev/null +++ b/deploy/MIGRATION.md @@ -0,0 +1,42 @@ +# OIDC deployment migration + +This change replaces the Auth0 credential broker and SSM-to-shell deployment path with [tc-deploy-scripts v2.0.5](https://github.com/topcoder-platform/tc-deploy-scripts/releases/tag/v2.0.5), fetched by tag and verified against commit `bff574ab5c0b46773e25d2af1eff76675fba2ba5`. Build and deployment run in separate jobs. The deployer cannot read Parameter Store, Secrets Manager or decrypt KMS data, including through another assumed role. ECS continues resolving the existing runtime secret references using the execution role. + +## Production target + +The read-only inventory on 2026-11-03 verified account `409275337247`, cluster `topcoder-infrastructure`, service `forms-processor-v6`, family `forms-processor-v6`, and container `forms-processor-v6`. `prod.json` preserves the observed task/execution roles and names the proposed service-specific deployment role. [prod-baseline.json](https://github.com/topcoder-platform/v6-services-iac/blob/main/services/forms-processor-v6/deploy/prod-baseline.json) records references only; it contains no parameter values. Do not recreate the service or replace its complete task definition from this baseline. + +## Infrastructure prerequisites, applied separately by an administrator + +1. In each account, register the CircleCI OIDC issuer `https://oidc.circleci.com/org/947bef5e-1669-415d-b83c-4fc85155b7df` with client ID `947bef5e-1669-415d-b83c-4fc85155b7df`. The current production account has only the legacy Auth0 provider. Do not remove that provider until all its consumers have migrated. +2. Review and install [iam/prod-role.cloudformation.json](https://github.com/topcoder-platform/v6-services-iac/blob/main/services/forms-processor-v6/deploy/iam/prod-role.cloudformation.json) with an administrator/IaC identity. It creates the proposed deployment role and applies the same least-privilege policy as a permissions boundary. The trust binds this CircleCI project, repository, checkout branch and configuration branch. It does not grant infrastructure creation to CircleCI. The role and matching boundary include `inspector2:ListFindings` and `inspector2:ListCoverage` restricted to the deployment region; these scan-metadata APIs require `Resource: "*"`. Secret-access denies remain explicit. +3. Configure ECR repository `forms-processor-v6` with unconditional `IMMUTABLE` tags and supported scanning. Preserve existing enhanced scanning where enabled. Existing tags are not retagged. The release fails for MEDIUM/HIGH/CRITICAL/UNDEFINED findings, failed scans, or a mismatched digest. +4. In the infrastructure that owns the task definition, add the reviewed container check from `prod-healthcheck.json`, keeping all roles, runtime secret references, resources, mounts and sidecars. Verify the health endpoint against the running container in dev first. Preserve circuit-breaker rollback and positive desired capacity. Remove any `RESET_DB` secret binding and plaintext secret environment entries before adoption. Keep the execution role's runtime read permissions scoped to the references that service needs. +5. Create restricted `forms-processor-v6-dev-deploy` and `forms-processor-v6-prod-deploy` CircleCI contexts without Auth0/static AWS credentials. Restrict production to the reviewed `master` checkout and configuration ref, authorized operators, and approval job. Remove legacy credentials from project settings too: removing `org-global` alone does not remove project-level variables. Confirm `CIRCLE_OIDC_TOKEN_V3` is issued. Disable secret access for forks and SSH debugging in deployment jobs. + +## Development acceptance before merge/release + +Read-only inventory on 2026-11-04 verified `dev.json` against the actual development service, task/execution roles and image repositories. This PR targets `develop`. Matching IAM templates and detailed baselines remain private. **Keep this PR in draft until the service-specific prerequisites below are complete.** No AWS or CircleCI resources were provisioned by this refresh. + +- Install the service deployment role and matching permissions boundary. +- Create the restricted empty CircleCI deployment context. + +Run the shared offline suite, validate the CircleCI config and IAM policies, and test a dev release plus an unhealthy-image rollback. Verify CloudTrail contains no SSM/Secrets Manager calls from the deployment principal. Remove its old broker grants only after migration. Check the runtime secret references and roles stayed unchanged. Production requires the manual approval job and a passing digest/scan/health verification. + +The pinned executor is `cimg/aws@sha256:4cf3768db00ddf9bc439b8c84e8a60c11a14edf81c8baf76f74132144f8340d5`. Python/AWS CLI/Buildx/Git compatibility was tested with the shared offline suite. No package installation or service build runs in the deployment job. The Dockerfile retains the existing in-image quality gates. This PR does not deploy, change IAM, rotate credentials, or claim a successful live rollout. + +## Processor release behavior + +Application lint/type checks, routing/retry unit tests, PostgreSQL integration tests and compilation remain prerequisites. Deployment tests now run the pinned shared security/rollout suite. The old SSM-discovery release implementation and its discovery-specific tests are replaced. Existing ECS runtime references are preserved byte-for-byte instead of discovering optional settings during a release; additions/removals belong in the infrastructure that owns the task. Forms API schema migration must still precede a processor release. Zero-capacity activation/bootstrap is an administrator operation: the deployer refuses it. + +## Infrastructure source + +IAM templates and detailed AWS baselines are maintained in the private [v6-services-iac service directory](https://github.com/topcoder-platform/v6-services-iac/tree/main/services/forms-processor-v6). Provision infrastructure separately with an administrator identity; application CI does not fetch that repository or install IAM. Required application manifests and health-check proposals remain here. + +## Dev branch refresh + +This PR targets `develop` and preserves its current application/dependency fixes. The shared suite is v2.0.5 (82 offline tests); service-specific adapters and their tests remain part of review. Keep this PR in draft until its documented infrastructure and integration prerequisites are met. Updating these files does not provision AWS or CircleCI resources. + +Source checkout uses the public HTTPS repository and verifies the exact CircleCI commit, avoiding dependency on failing SSH checkout keys. No GitHub token is added to build or deployment contexts. + +The credential-free builder rejects committed `.npmrc` files. Preserve the existing local compatibility dependencies with explicit `--install-links=true` npm options instead; do not reintroduce credential-capable npm configuration files. diff --git a/deploy/dev-healthcheck.json b/deploy/dev-healthcheck.json new file mode 100644 index 0000000..d23559b --- /dev/null +++ b/deploy/dev-healthcheck.json @@ -0,0 +1,14 @@ +{ + "forms-processor-v6": { + "command": [ + "CMD", + "node", + "-e", + "fetch('http://127.0.0.1:'+(process.env.PORT||3000)+'/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" + ], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 60 + } +} diff --git a/deploy/dev.json b/deploy/dev.json new file mode 100644 index 0000000..a36697a --- /dev/null +++ b/deploy/dev.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": 2, + "environment": "dev", + "accountId": "811668436784", + "region": "us-east-1", + "deployRoleArn": "arn:aws:iam::811668436784:role/forms-processor-v6-dev-deploy", + "taskRoleArn": "arn:aws:iam::811668436784:role/forms-processor-v6-dev-TaskRole-zeTo8YcPLgMU", + "executionRoleArn": "arn:aws:iam::811668436784:role/forms-processor-v6-dev-ExecutionRole-jvv8jQ05uWJh", + "cluster": "topcoder-infrastructure", + "service": "forms-processor-v6", + "taskFamily": "forms-processor-v6", + "circleciOrgId": "947bef5e-1669-415d-b83c-4fc85155b7df", + "circleciProjectId": "9abc16b0-0d52-490f-a4c0-6e3db3ede656", + "repository": "github.com/topcoder-platform/forms-processor-v6", + "branch": "develop", + "images": { + "forms-processor-v6": "forms-processor-v6" + } +} diff --git a/deploy/prod-healthcheck.json b/deploy/prod-healthcheck.json new file mode 100644 index 0000000..d23559b --- /dev/null +++ b/deploy/prod-healthcheck.json @@ -0,0 +1,14 @@ +{ + "forms-processor-v6": { + "command": [ + "CMD", + "node", + "-e", + "fetch('http://127.0.0.1:'+(process.env.PORT||3000)+'/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" + ], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 60 + } +} diff --git a/deploy/prod.json b/deploy/prod.json new file mode 100644 index 0000000..6fcfac1 --- /dev/null +++ b/deploy/prod.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": 2, + "environment": "prod", + "accountId": "409275337247", + "region": "us-east-1", + "deployRoleArn": "arn:aws:iam::409275337247:role/forms-processor-v6-prod-deploy", + "taskRoleArn": "arn:aws:iam::409275337247:role/forms-processor-v6-production-TaskRole-F7Uvj6UKRNRm", + "executionRoleArn": "arn:aws:iam::409275337247:role/forms-processor-v6-production-ExecutionRole-M3KAOLfyCIxx", + "cluster": "topcoder-infrastructure", + "service": "forms-processor-v6", + "taskFamily": "forms-processor-v6", + "circleciOrgId": "947bef5e-1669-415d-b83c-4fc85155b7df", + "circleciProjectId": "9abc16b0-0d52-490f-a4c0-6e3db3ede656", + "repository": "github.com/topcoder-platform/forms-processor-v6", + "branch": "master", + "images": { + "forms-processor-v6": "forms-processor-v6" + } +} diff --git a/deploy/release.py b/deploy/release.py index ade1a8d..de70d94 100644 --- a/deploy/release.py +++ b/deploy/release.py @@ -1,210 +1,13 @@ #!/usr/bin/env python3 -"""Release a prebuilt image to an existing ECS service, without provisioning infrastructure. - -Usage: release.py dev|production IMAGE_TAG [LOCAL_IMAGE]. Uses inherited temporary -AWS credentials, boto3 and Docker. API migrations must precede this release. -""" -import base64 -from copy import deepcopy +"""Pass the reviewed manifest and this job's prebuilt images to the pinned suite.""" import json -import os from pathlib import Path -import re import subprocess import sys -import time - -import boto3 -from botocore.config import Config - -APP = 'forms-processor-v6' -CLUSTER = 'topcoder-infrastructure' -ACCOUNTS = {'dev': '811668436784', 'production': '409275337247'} -SSM_CLIENT_CONFIG = Config(retries={'mode': 'standard', 'total_max_attempts': 10}) - - -def read_service(ecs): - """Read the fixed ECS service using ecs; return active service or raise on missing/invalid state.""" - result = ecs.describe_services(cluster=CLUSTER, services=[APP]) - if result.get('failures') or len(result.get('services', [])) != 1: - raise RuntimeError('Existing forms-processor-v6 ECS service is required') - service = result['services'][0] - if service['status'] != 'ACTIVE': - raise RuntimeError('Existing ECS service must be active') - return service - - -# Optional processor-owned settings for AI classification and label actions (docs/adr/0001), -# and for the SpaceTech challenge notification. Absent values defer the matching flows; -# the code defaults apply where documented. -LABEL_ACTION_SETTINGS = [ - 'AI_API_URL', 'AI_AUTH0_AUDIENCE', - 'LETS_TALK_CLASSIFY_MAX_ATTEMPTS', 'LETS_TALK_CLASSIFY_MAX_AGE_MINUTES', - 'LETS_TALK_OPPORTUNITY_TEMPLATE_ID', 'LETS_TALK_SKILLS_TEMPLATE_ID', - 'LETS_TALK_GENERAL_TEMPLATE_ID', 'LETS_TALK_WIPRO_TEMPLATE_ID', - 'LETS_TALK_SALES_REPLY_TEMPLATE_ID', 'LETS_TALK_SUPPORT_TEMPLATE_ID', - 'LETS_TALK_SUPPORT_RECIPIENTS', 'LETS_TALK_SALES_REPLY_TO', 'LETS_TALK_REPLY_CAP_PER_DAY', - 'SPACE_TECH_CHALLENGE_RECIPIENTS', 'SPACE_TECH_CHALLENGE_TEMPLATE_ID', -] - - -def secret_references(ssm): - """Resolve named SSM references in batches of ten, without decrypting SecureStrings. - - Takes an SSM client; returns name/valueFrom pairs. API Auth0 settings take - precedence over shared defaults. Kafka uses globals. Missing optional sales - settings permit a disabled-flow deployment. Raises for absent required values - or sales/Bus parameters that are not SecureStrings; AWS errors propagate after - client retries. GetParameters values are discarded; only names/types are used. - Avoids the low-throughput DescribeParameters scan of whole parameter prefixes. - """ - prefixes = {'api': '/config/forms-api-v6/appvar', - 'app': f'/config/{APP}/appvar', - 'common': '/config/common/global-appvar'} - sources = {'DATABASE_URL': ['api'], 'BUSAPI_URL': ['app'], - 'AUTH0_CLIENT_ID': ['api'], 'AUTH0_CLIENT_SECRET': ['api'], - 'AUTH0_URL': ['api', 'common'], 'AUTH0_AUDIENCE': ['api', 'common'], - 'AUTH0_PROXY_SERVER_URL': ['api', 'common'], - 'KAFKA_URL': ['common'], 'KAFKA_SSL_ENABLED': ['common'], - 'LETS_TALK_RECIPIENTS': ['app'], 'LETS_TALK_SENDGRID_TEMPLATE_ID': ['app'], - **{name: ['app'] for name in LABEL_ACTION_SETTINGS}} - names = [prefixes[source] + '/' + name - for name, candidates in sources.items() for source in candidates] - available = {} - for start in range(0, len(names), 10): - response = ssm.get_parameters(Names=names[start:start + 10], WithDecryption=False) - available.update({p['Name']: p['Type'] for p in response['Parameters']}) - optional = {'AUTH0_PROXY_SERVER_URL', 'KAFKA_SSL_ENABLED', - 'LETS_TALK_RECIPIENTS', 'LETS_TALK_SENDGRID_TEMPLATE_ID', *LABEL_ACTION_SETTINGS} - references = [] - for name, candidates in sources.items(): - path = next((prefixes[source] + '/' + name for source in candidates - if prefixes[source] + '/' + name in available), None) - if path is None: - if name not in optional: - raise RuntimeError('Missing required SSM parameter: ' + name) - continue - # Every processor-owned parameter (Bus URL, recipients, templates, AI settings) is a SecureString. - if path.startswith(prefixes['app'] + '/'): - if available[path] != 'SecureString': - raise RuntimeError('SecureString required: ' + path) - references.append({'name': name, 'valueFrom': path}) - return references - - -def task_request(ecs, current, image, references): - """Clone the live task's registerable fields and replace the processor image/managed secrets. - - Inputs are the ECS client, described task, ECR image URI and SSM references. - Returns a register-task-definition request, preserving roles/resources/sidecars. - Raises if family or application container identity differs from the fixed service. - """ - if current['family'] != APP: - raise RuntimeError('Unexpected task family') - allowed = ecs.meta.service_model.operation_model('RegisterTaskDefinition').input_shape.members - request = {key: deepcopy(value) for key, value in current.items() if key in allowed} - containers = [c for c in request['containerDefinitions'] if c['name'] == APP] - if len(containers) != 1: - raise RuntimeError('Expected one forms-processor-v6 container') - container = containers[0] - managed = {'DATABASE_URL', 'BUSAPI_URL', 'AUTH0_CLIENT_ID', 'AUTH0_CLIENT_SECRET', - 'AUTH0_URL', 'AUTH0_AUDIENCE', 'AUTH0_PROXY_SERVER_URL', 'KAFKA_URL', - 'LETS_TALK_RECIPIENTS', 'LETS_TALK_SENDGRID_TEMPLATE_ID', *LABEL_ACTION_SETTINGS} - # Preserve an existing TLS setting if the global store does not define it. - managed.update(ref['name'] for ref in references) - container['environment'] = [e for e in container.get('environment', []) if e['name'] not in managed] - container['secrets'] = [s for s in container.get('secrets', []) if s['name'] not in managed] + references - container['image'] = image - container['healthCheck'] = { - 'command': ['CMD', 'node', '-e', - "fetch('http://127.0.0.1:'+(process.env.PORT||3000)+'/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"], - 'interval': 30, 'timeout': 5, 'retries': 3, 'startPeriod': 60, - } - return request - - -def verify_service(ecs, definition_arn): - """Observe one rollout using ecs and the requested ARN; return service when tasks are healthy. - - Returns None while converging; raises on rollback/replacement/failed rollout or - AWS errors. Used by the bounded release waiter, including activation from zero. - """ - service = read_service(ecs) - if service['taskDefinition'] != definition_arn: - raise RuntimeError('ECS rolled back or replaced the requested task definition') - deployments = service['deployments'] - if any(d.get('rolloutState') == 'FAILED' for d in deployments): - raise RuntimeError('ECS deployment failed') - if (len(deployments) != 1 or deployments[0].get('rolloutState') != 'COMPLETED' - or service['runningCount'] != service['desiredCount'] or service['desiredCount'] < 1 - or service['pendingCount']): - return None - arns = [] - for page in ecs.get_paginator('list_tasks').paginate( - cluster=CLUSTER, serviceName=APP, desiredStatus='RUNNING'): - arns.extend(page['taskArns']) - tasks = [] - for start in range(0, len(arns), 100): - tasks.extend(ecs.describe_tasks(cluster=CLUSTER, tasks=arns[start:start + 100])['tasks']) - healthy = [t for t in tasks if t['taskDefinitionArn'] == definition_arn - and t.get('healthStatus') == 'HEALTHY' and t['lastStatus'] == 'RUNNING'] - return service if len(healthy) >= service['desiredCount'] else None - - -def main(): - """Read CLI arguments, verify account/service, push image, register a revision and update ECS. - - Returns None after verified deployment and a non-secret release artifact. Raises - on invalid arguments/account, missing existing infrastructure, AWS/Docker errors, - concurrent service change, rollback or a 30-minute timeout. Never updates stacks, - creates services, modifies networking/IAM, runs migrations, or enables email flows. - """ - if len(sys.argv) not in (3, 4) or sys.argv[1] not in ACCOUNTS: - raise RuntimeError(__doc__) - environment, tag = sys.argv[1:3] - if not re.fullmatch(r'[A-Za-z0-9_][A-Za-z0-9_.-]{0,110}', tag): - raise RuntimeError('Invalid image tag') - session = boto3.Session(region_name=os.environ.get('AWS_REGION', 'us-east-1')) - account = session.client('sts').get_caller_identity()['Account'] - if account != ACCOUNTS[environment]: - raise RuntimeError('AWS account does not match requested environment') - ecs, ecr = [session.client(name) for name in ('ecs', 'ecr')] - ssm = session.client('ssm', config=SSM_CLIENT_CONFIG) - service = read_service(ecs) - if len(service['deployments']) != 1 or service['deployments'][0].get('rolloutState') != 'COMPLETED': - raise RuntimeError('Existing ECS deployment must be stable before releasing') - current = ecs.describe_task_definition(taskDefinition=service['taskDefinition'])['taskDefinition'] - references = secret_references(ssm) - repository = ecr.describe_repositories(repositoryNames=[APP])['repositories'][0]['repositoryUri'] - image = f'{repository}:{tag}' - request = task_request(ecs, current, image, references) - authorization = ecr.get_authorization_token()['authorizationData'][0] - user, password = base64.b64decode(authorization['authorizationToken']).decode().split(':', 1) - subprocess.run(['docker', 'login', '--username', user, '--password-stdin', authorization['proxyEndpoint']], - input=password, text=True, check=True, stdout=subprocess.DEVNULL) - local_image = sys.argv[3] if len(sys.argv) == 4 else APP + ':candidate' - subprocess.run(['docker', 'tag', local_image, image], check=True) - subprocess.run(['docker', 'push', image], check=True) - if read_service(ecs)['taskDefinition'] != service['taskDefinition']: - raise RuntimeError('Service changed during image push; retry after inspecting concurrent release') - definition_arn = ecs.register_task_definition(**request)['taskDefinition']['taskDefinitionArn'] - ecs.update_service(cluster=CLUSTER, service=APP, taskDefinition=definition_arn, - desiredCount=max(1, service['desiredCount'])) - deadline = time.monotonic() + 1800 - while time.monotonic() < deadline: - retained = verify_service(ecs, definition_arn) - if retained: - break - print('Waiting for healthy requested ECS revision', flush=True) - time.sleep(15) - else: - raise RuntimeError('ECS deployment timed out; inspect before retrying') - evidence = {'environment': environment, 'cluster': CLUSTER, 'service': APP, 'image': image, - 'taskDefinition': definition_arn, 'previousTaskDefinition': service['taskDefinition'], - 'runningCount': retained['runningCount']} - Path(f'deploy/release-{environment}.json').write_text(json.dumps(evidence, indent=2) + '\n') - print(json.dumps(evidence, indent=2)) - -if __name__ == '__main__': - main() +manifest = Path(sys.argv[1]) +data = json.loads(manifest.read_text()) +command = [sys.executable, '-I', '/tmp/tc-deploy-scripts/tcdeploy.py', 'deploy', '--manifest', str(manifest)] +for container in data['images']: + command.extend(['--image', container + '=forms-processor-v6:candidate']) +subprocess.run(command, check=True) diff --git a/deploy/tests/test_release.py b/deploy/tests/test_release.py deleted file mode 100644 index c83e9a1..0000000 --- a/deploy/tests/test_release.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Exercise rollout guards and secret-reference selection without contacting AWS.""" -import importlib.util -import json -from pathlib import Path -from types import SimpleNamespace -import unittest -from unittest.mock import Mock, patch - -import boto3 -from botocore.awsrequest import AWSResponse -from botocore.exceptions import ClientError - -spec = importlib.util.spec_from_file_location('release', Path(__file__).parents[1] / 'release.py') -release = importlib.util.module_from_spec(spec) -spec.loader.exec_module(release) - - -class ReleaseTests(unittest.TestCase): - """Verify that releases update existing ECS tasks and cannot cross environment boundaries.""" - - def setUp(self): - """Build named-parameter fixtures for each test; takes no arguments and returns nothing.""" - entries = { - '/config/forms-api-v6/appvar': ['DATABASE_URL', 'AUTH0_CLIENT_ID', 'AUTH0_CLIENT_SECRET', 'AUTH0_URL'], - '/config/forms-processor-v6/appvar': ['BUSAPI_URL', 'LETS_TALK_RECIPIENTS', 'LETS_TALK_SENDGRID_TEMPLATE_ID', - *release.LABEL_ACTION_SETTINGS], - '/config/common/global-appvar': ['AUTH0_URL', 'AUTH0_AUDIENCE', 'AUTH0_PROXY_SERVER_URL', 'KAFKA_URL', 'KAFKA_SSL_ENABLED'], - } - self.parameters = { - path + '/' + name: {'Name': path + '/' + name, 'Type': 'SecureString', 'Value': 'unused-ciphertext'} - for path, names in entries.items() for name in names - } - self.ssm = Mock() - self.ssm.get_parameters.side_effect = self.parameter_response - - def parameter_response(self, *, Names, WithDecryption): - """Return the requested fixture parameters and absent names; rejects oversized/decrypting reads. - - Called by the SSM mock with AWS request arguments. Reverses result order to - verify lookup by name, and raises AssertionError for unsafe request options. - """ - self.assertFalse(WithDecryption) - self.assertGreater(len(Names), 0) - self.assertLessEqual(len(Names), 10) - return {'Parameters': [self.parameters[name] for name in reversed(Names) if name in self.parameters], - 'InvalidParameters': [name for name in Names if name not in self.parameters]} - - def test_missing_service_is_rejected(self): - """A missing service raises before release can provision anything; no inputs or return value.""" - ecs = Mock() - ecs.describe_services.return_value = {'services': [], 'failures': [{'reason': 'MISSING'}]} - with self.assertRaisesRegex(RuntimeError, 'Existing'): - release.read_service(ecs) - ecs.update_service.assert_not_called() - - def test_secret_sources_and_types(self): - """Two bounded non-decrypting reads return only named references with the intended precedence.""" - refs = {r['name']: r['valueFrom'] for r in release.secret_references(self.ssm)} - self.assertEqual(refs['AUTH0_URL'], '/config/forms-api-v6/appvar/AUTH0_URL') - self.assertEqual(refs['AUTH0_AUDIENCE'], '/config/common/global-appvar/AUTH0_AUDIENCE') - self.assertEqual(refs['DATABASE_URL'], '/config/forms-api-v6/appvar/DATABASE_URL') - self.assertEqual(refs['KAFKA_URL'], '/config/common/global-appvar/KAFKA_URL') - self.assertEqual(refs['BUSAPI_URL'], '/config/forms-processor-v6/appvar/BUSAPI_URL') - self.assertEqual(len(refs), 11 + len(release.LABEL_ACTION_SETTINGS)) - self.assertNotIn('unused-ciphertext', json.dumps(refs)) - requested = [name for call in self.ssm.get_parameters.call_args_list for name in call.kwargs['Names']] - expected = 14 + len(release.LABEL_ACTION_SETTINGS) - self.assertEqual(self.ssm.get_parameters.call_count, -(-expected // 10)) - self.assertEqual(len(requested), expected) - self.assertEqual(len(set(requested)), expected) - self.assertEqual(refs['AI_API_URL'], '/config/forms-processor-v6/appvar/AI_API_URL') - self.assertNotIn('/config/forms-api-v6/appvar/MIGRATION_DATABASE_URL', requested) - self.ssm.get_paginator.assert_not_called() - self.ssm.describe_parameters.assert_not_called() - - def test_optional_parameters_and_auth_fallback(self): - """Missing optional settings are omitted and absent API Auth0 settings use shared defaults.""" - del self.parameters['/config/forms-api-v6/appvar/AUTH0_URL'] - for name in ['LETS_TALK_RECIPIENTS', 'LETS_TALK_SENDGRID_TEMPLATE_ID', *release.LABEL_ACTION_SETTINGS]: - del self.parameters['/config/forms-processor-v6/appvar/' + name] - for name in ['KAFKA_SSL_ENABLED', 'AUTH0_PROXY_SERVER_URL']: - del self.parameters['/config/common/global-appvar/' + name] - refs = {r['name']: r['valueFrom'] for r in release.secret_references(self.ssm)} - self.assertEqual(refs['AUTH0_URL'], '/config/common/global-appvar/AUTH0_URL') - self.assertEqual(len(refs), 7) - - def test_required_parameter_missing_fails(self): - """A missing required database reference raises instead of publishing a broken task.""" - del self.parameters['/config/forms-api-v6/appvar/DATABASE_URL'] - with self.assertRaisesRegex(RuntimeError, 'Missing required'): - release.secret_references(self.ssm) - - def test_sales_parameters_must_remain_secure_strings(self): - """Wrong types for each required Bus or optional sales value raise without exposing its value.""" - for name in ['BUSAPI_URL', 'LETS_TALK_RECIPIENTS', 'LETS_TALK_SENDGRID_TEMPLATE_ID', - 'AI_API_URL', 'LETS_TALK_SALES_REPLY_TO']: - parameter = self.parameters['/config/forms-processor-v6/appvar/' + name] - with self.subTest(name=name): - parameter['Type'] = 'String' - with self.assertRaisesRegex(RuntimeError, 'SecureString') as raised: - release.secret_references(self.ssm) - self.assertNotIn('unused-ciphertext', str(raised.exception)) - parameter['Type'] = 'SecureString' - - def test_ssm_failures_stop_before_deployment(self): - """Access denial and exhausted throttling propagate before any ECR, Docker or ECS mutation.""" - for code in ['AccessDeniedException', 'ThrottlingException']: - with self.subTest(code=code): - session, ecs, ecr = Mock(), Mock(), Mock() - sts = Mock() - sts.get_caller_identity.return_value = {'Account': release.ACCOUNTS['production']} - session.client.side_effect = lambda name, **kwargs: { - 'sts': sts, 'ecs': ecs, 'ecr': ecr, 'ssm': self.ssm}[name] - ecs.describe_services.return_value = {'services': [{ - 'status': 'ACTIVE', 'taskDefinition': 'current', - 'deployments': [{'rolloutState': 'COMPLETED'}]}]} - ecs.describe_task_definition.return_value = {'taskDefinition': {'family': release.APP}} - self.ssm.get_parameters.side_effect = ClientError({'Error': { - 'Code': code, 'Message': 'Parameter read failed'}}, 'GetParameters') - with patch.object(release.boto3, 'Session', return_value=session), \ - patch.object(release.sys, 'argv', ['release.py', 'production', 'candidate']), \ - patch.object(release.subprocess, 'run') as run: - with self.assertRaises(ClientError): - release.main() - session.client.assert_any_call('ssm', config=release.SSM_CLIENT_CONFIG) - ecr.describe_repositories.assert_not_called() - ecs.register_task_definition.assert_not_called() - ecs.update_service.assert_not_called() - run.assert_not_called() - - def test_sdk_retries_throttling_and_has_a_finite_budget(self): - """Exercise real SDK retry handling offline: recover after five throttles and stop after ten attempts.""" - # GetParameters reads ten names per request. - batches = -(-(14 + len(release.LABEL_ACTION_SETTINGS)) // 10) - for throttle_count, expected_requests in [(5, 5 + batches), (100, 10)]: - with self.subTest(throttle_count=throttle_count): - ssm = boto3.client('ssm', region_name='us-east-1', aws_access_key_id='test', - aws_secret_access_key='test', config=release.SSM_CLIENT_CONFIG) - requests = [] - - def respond(request): - """Return an AWSResponse for a prepared HTTP request; fake throttles precede fixture reads.""" - requests.append(json.loads(request.body)) - self.assertEqual(request.headers['X-Amz-Target'], b'AmazonSSM.GetParameters') - throttled = len(requests) <= throttle_count - body = {'__type': 'ThrottlingException', 'message': 'Rate exceeded'} if throttled \ - else self.parameter_response(**requests[-1]) - raw = SimpleNamespace(stream=lambda: iter([json.dumps(body).encode()])) - return AWSResponse(request.url, 400 if throttled else 200, - {'content-type': 'application/x-amz-json-1.1'}, raw) - - with patch.object(ssm._endpoint.http_session, 'send', side_effect=respond), \ - patch('botocore.endpoint.time.sleep'): - if throttle_count == 5: - self.assertEqual(len(release.secret_references(ssm)), 11 + len(release.LABEL_ACTION_SETTINGS)) - else: - with self.assertRaises(ClientError): - release.secret_references(ssm) - self.assertEqual(len(requests), expected_requests) - self.assertTrue(all(not request['WithDecryption'] for request in requests)) - ssm.close() - - def test_clone_preserves_roles_resources_and_removes_stale_literals(self): - """Cloning returns a registerable copy with exact secret overrides and unchanged sidecars/roles.""" - ecs = boto3.client('ecs', region_name='us-east-1', aws_access_key_id='test', aws_secret_access_key='test') - current = {'family': release.APP, 'taskDefinitionArn': 'old', 'revision': 7, - 'executionRoleArn': 'execution', 'taskRoleArn': 'task', 'cpu': '256', 'memory': '512', - 'networkMode': 'awsvpc', 'requiresCompatibilities': ['FARGATE'], - 'containerDefinitions': [{'name': release.APP, 'image': 'old', 'readonlyRootFilesystem': True, - 'environment': [{'name': 'BUSAPI_URL', 'value': 'old'}, {'name': 'KAFKA_SSL_ENABLED', 'value': 'true'}], - 'secrets': [{'name': 'LETS_TALK_RECIPIENTS', 'valueFrom': 'old'}, {'name': 'OTHER', 'valueFrom': '/other'}]}, - {'name': 'sidecar', 'image': 'sidecar'}]} - refs = [{'name': 'BUSAPI_URL', 'valueFrom': '/config/forms-processor-v6/appvar/BUSAPI_URL'}] - result = release.task_request(ecs, current, 'new', refs) - self.assertNotIn('revision', result) - self.assertNotIn('taskDefinitionArn', result) - for key in ['executionRoleArn', 'taskRoleArn', 'cpu', 'memory', 'networkMode', 'requiresCompatibilities']: - self.assertEqual(result[key], current[key]) - container = result['containerDefinitions'][0] - self.assertEqual(container['environment'], [{'name': 'KAFKA_SSL_ENABLED', 'value': 'true'}]) - self.assertEqual(container['secrets'], [{'name': 'OTHER', 'valueFrom': '/other'}] + refs) - self.assertEqual(container['image'], 'new') - self.assertEqual(current['containerDefinitions'][0]['image'], 'old') - self.assertEqual(result['containerDefinitions'][1], current['containerDefinitions'][1]) - self.assertIn('healthCheck', container) - - def test_rollback_and_unhealthy_tasks_are_not_success(self): - """Only healthy tasks of the requested revision finish verification; rollback raises.""" - ecs = Mock() - service = {'status': 'ACTIVE', 'taskDefinition': 'new', 'desiredCount': 1, 'runningCount': 1, - 'pendingCount': 0, 'deployments': [{'rolloutState': 'COMPLETED'}]} - ecs.describe_services.return_value = {'services': [service]} - ecs.get_paginator.return_value.paginate.return_value = [{'taskArns': ['task']}] - task = {'taskDefinitionArn': 'new', 'healthStatus': 'UNHEALTHY', 'lastStatus': 'RUNNING'} - ecs.describe_tasks.return_value = {'tasks': [task]} - self.assertIsNone(release.verify_service(ecs, 'new')) - task['healthStatus'] = 'HEALTHY' - self.assertEqual(release.verify_service(ecs, 'new'), service) - service['deployments'][0]['rolloutState'] = 'FAILED' - with self.assertRaisesRegex(RuntimeError, 'failed'): - release.verify_service(ecs, 'new') - service['taskDefinition'] = 'old' - with self.assertRaisesRegex(RuntimeError, 'rolled back'): - release.verify_service(ecs, 'new') - - def test_wrong_account_stops_before_mutation(self): - """Production with dev credentials raises before ECR, ECS, SSM or Docker are called.""" - session = Mock(region_name='us-east-1') - session.client.return_value.get_caller_identity.return_value = {'Account': release.ACCOUNTS['dev']} - with patch.object(release.boto3, 'Session', return_value=session), \ - patch.object(release.sys, 'argv', ['release.py', 'production', 'candidate']), \ - patch.object(release.subprocess, 'run') as run: - with self.assertRaisesRegex(RuntimeError, 'account'): - release.main() - session.client.assert_called_once_with('sts') - run.assert_not_called() - - -if __name__ == '__main__': - unittest.main()