From 388bbc4713dd35d230013efe9969747ac1d1d7d2 Mon Sep 17 00:00:00 2001 From: EmmaLien Date: Thu, 29 Oct 2026 06:51:31 +0000 Subject: [PATCH] ci: schedule the nightly and RC pipelines for the E2E TPU quota window The MaxText E2E TPU tests are moving to the ubench-regression-tests queue on bodaborg-v6e-nap, which only has TPU quota from 20:00 to 24:00 Pacific time each day (b/568477549). With the current schedules, the nightly pipeline (00:00 UTC) submits its E2E TPU jobs at about 01:00-02:00 UTC and the RC pipeline (Mondays 06:00 UTC) at about 07:00-08:00 UTC, both outside the window. - Run the TPU Docker Images Pipeline daily, and the Release Candidate pipeline every Sunday, at 19:00 Pacific time. The first E2E TPU job is submitted about an hour after the scheduled time, when the window opens. The schedules use the America/Los_Angeles timezone, so they follow daylight saving time like the window does. - Bump actionlint to v1.7.12.25. v1.7.7 rejects the schedule timezone key, and v1.7.12 reports no findings on the existing workflows. - Update the TPU pipeline schedule in ci_docker_image_pipelines.md. --- .github/workflows/rc_release.yml | 11 +++++++---- .github/workflows/tpu_docker_images_pipeline.yml | 9 ++++++--- .pre-commit-config.yaml | 2 +- docs/development/ci_docker_image_pipelines.md | 2 +- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rc_release.yml b/.github/workflows/rc_release.yml index 14adfe8a6d..34119bccd3 100644 --- a/.github/workflows/rc_release.yml +++ b/.github/workflows/rc_release.yml @@ -15,15 +15,18 @@ # This workflow builds MaxText package, builds and tags TPU Release Candidate # Docker images (:maxtext_${SHA_RC}_${YYYYMMDD}, :${SHA_RC}, and :latest-rc), # and triggers Airflow E2E functional tests. -# The workflow is designed to run on a schedule (every Monday at 06:00 UTC) -# or on-demand via workflow_dispatch. +# The workflow is designed to run on a schedule (every Sunday at 19:00 Pacific +# time) or on-demand via workflow_dispatch. name: MaxText Release Candidate Pipeline on: schedule: - # Runs every Monday at 06:00 UTC - - cron: '0 6 * * 1' + # Runs every Sunday at 19:00 Pacific time (Monday 02:00 or 03:00 UTC), one + # hour before the daily 20:00-24:00 Pacific TPU quota window of the queue + # that runs the E2E tests. + - cron: '0 19 * * 0' + timezone: 'America/Los_Angeles' workflow_dispatch: inputs: maxtext_sha: diff --git a/.github/workflows/tpu_docker_images_pipeline.yml b/.github/workflows/tpu_docker_images_pipeline.yml index ecd368a722..40adb5ec5f 100644 --- a/.github/workflows/tpu_docker_images_pipeline.yml +++ b/.github/workflows/tpu_docker_images_pipeline.yml @@ -13,14 +13,17 @@ # limitations under the License. # This workflow builds and pushes TPU MaxText images. -# It runs automatically daily at 12am UTC, or manually via Workflow Dispatch. +# It runs automatically daily at 19:00 Pacific time, or manually via Workflow Dispatch. name: TPU Docker Images Pipeline on: schedule: - # Run the job daily at 12AM UTC - - cron: '0 0 * * *' + # Run daily at 19:00 Pacific time, one hour before the daily 20:00-24:00 + # Pacific TPU quota window of the queue that runs the E2E tests, so the TPU + # jobs are submitted when the window opens. + - cron: '0 19 * * *' + timezone: 'America/Los_Angeles' workflow_dispatch: inputs: repository_name: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5dd4eb45e0..b1c4a52c34 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -76,7 +76,7 @@ repos: types: [yaml] - repo: https://github.com/Mateusz-Grzelinski/actionlint-py - rev: v1.7.7.23 + rev: v1.7.12.25 hooks: - id: actionlint args: diff --git a/docs/development/ci_docker_image_pipelines.md b/docs/development/ci_docker_image_pipelines.md index d4f518e274..3e314da8db 100644 --- a/docs/development/ci_docker_image_pipelines.md +++ b/docs/development/ci_docker_image_pipelines.md @@ -30,7 +30,7 @@ workflows call it, and these are the only events that build an image: | Caller | Trigger | Images | | ------------------------------------------------------------- | ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | -| TPU Docker Images Pipeline (`tpu_docker_images_pipeline.yml`) | Nightly at 00:00 UTC, or manual **Run workflow** | `maxtext_jax_stable`, `maxtext_jax_nightly`, `maxtext_post_training_stable`, `maxtext_post_training_nightly` | +| TPU Docker Images Pipeline (`tpu_docker_images_pipeline.yml`) | Daily at 19:00 Pacific, or manual **Run workflow** | `maxtext_jax_stable`, `maxtext_jax_nightly`, `maxtext_post_training_stable`, `maxtext_post_training_nightly` | | GPU Docker Images Pipeline (`gpu_docker_images_pipeline.yml`) | Nightly at 00:00 UTC, or manual **Run workflow** | `maxtext_gpu_jax_stable`, `maxtext_gpu_jax_nightly` | | Release Pipeline (`release_pipeline.yml`) | GitHub release published, or manual **Run workflow** | Release images. Always a full build. |