From 8f95fe26df623d1a25fb75c535c740037f75d204 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Thu, 12 Nov 2026 10:42:57 -0800 Subject: [PATCH 1/3] fix(labels): add and remove deploy labels without replacing the PR's label set The label job replaced every label on the pull request with a snapshot taken from the webhook payload. When autoDeploy queued the deploy label right after a PR opened, labels applied in the seconds after open (bots like Dependabot add them in a separate call) were missing from that snapshot and got wiped. Use GitHub's add-labels and remove-label endpoints instead, so Lifecycle only touches the labels it owns. Removing a label that is already gone is treated as success. TTL cleanup gets the same treatment when it swaps the deploy label for the disabled label. --- src/server/lib/github/__tests__/index.test.ts | 38 +++++++++++--- src/server/lib/github/index.ts | 38 ++++++++++++-- src/server/services/__tests__/label.test.ts | 49 ++++++++++++------- .../services/__tests__/ttlCleanup.test.ts | 43 +++++++++++----- .../__tests__/ttlCleanupApiDrift.test.ts | 6 +-- src/server/services/label.ts | 24 +++++---- src/server/services/ttlCleanup.ts | 14 ++++-- 7 files changed, 149 insertions(+), 63 deletions(-) diff --git a/src/server/lib/github/__tests__/index.test.ts b/src/server/lib/github/__tests__/index.test.ts index 4c820c19..cba9696f 100644 --- a/src/server/lib/github/__tests__/index.test.ts +++ b/src/server/lib/github/__tests__/index.test.ts @@ -30,6 +30,8 @@ import { getPullRequest, getPullRequestByRepositoryFullName, getPullRequestLabels, + addPullRequestLabels, + removePullRequestLabel, getRepositoryByFullName, getSHAForBranch, getShaForDeploy, @@ -38,7 +40,6 @@ import { getYamlFileContentFromPullRequest, listBranchesForRepo, listInstallationRepositories, - updatePullRequestLabels, verifyWebhookSignature, } from 'server/lib/github'; import * as client from 'server/lib/github/client'; @@ -357,24 +358,24 @@ describe('repository and pull-request API wrappers', () => { ).rejects.toThrow('Unable to create or update pull request comment'); }); - test('replaces pull-request labels and preserves provider errors', async () => { - const request = mockClient(jest.fn().mockResolvedValue({ data: [{ name: 'ready' }] })); + test('adds pull-request labels without replacing existing ones and preserves provider errors', async () => { + const request = mockClient(jest.fn().mockResolvedValue({ data: [{ name: 'existing' }, { name: 'ready' }] })); await expect( - updatePullRequestLabels({ + addPullRequestLabels({ installationId: 19, pullRequestNumber: 7, fullName: 'org/repo', labels: ['ready'], }) - ).resolves.toEqual({ data: [{ name: 'ready' }] }); - expect(request).toHaveBeenCalledWith('PUT /repos/org/repo/issues/7/labels', { + ).resolves.toEqual({ data: [{ name: 'existing' }, { name: 'ready' }] }); + expect(request).toHaveBeenCalledWith('POST /repos/org/repo/issues/7/labels', { data: { labels: ['ready'] }, }); - const failure = new Error('label update failed'); + const failure = new Error('label add failed'); mockClient(jest.fn().mockRejectedValue(failure)); await expect( - updatePullRequestLabels({ + addPullRequestLabels({ installationId: 19, pullRequestNumber: 7, fullName: 'org/repo', @@ -383,6 +384,27 @@ describe('repository and pull-request API wrappers', () => { ).rejects.toBe(failure); }); + test('removes a single URL-encoded pull-request label', async () => { + const request = mockClient(jest.fn().mockResolvedValue({ data: [] })); + await expect( + removePullRequestLabel({ installationId: 19, pullRequestNumber: 7, fullName: 'org/repo', label: 'deploy now!' }) + ).resolves.toEqual({ data: [] }); + expect(request).toHaveBeenCalledWith('DELETE /repos/org/repo/issues/7/labels/deploy%20now!'); + }); + + test('treats a missing label as already removed but preserves other provider errors', async () => { + mockClient(jest.fn().mockRejectedValue(Object.assign(new Error('Label does not exist'), { status: 404 }))); + await expect( + removePullRequestLabel({ installationId: 19, pullRequestNumber: 7, fullName: 'org/repo', label: 'deploy' }) + ).resolves.toBeNull(); + + const failure = Object.assign(new Error('forbidden'), { status: 403 }); + mockClient(jest.fn().mockRejectedValue(failure)); + await expect( + removePullRequestLabel({ installationId: 19, pullRequestNumber: 7, fullName: 'org/repo', label: 'deploy' }) + ).rejects.toBe(failure); + }); + test('normalizes pull-request fetch failures', async () => { (cacheRequest as jest.Mock).mockRejectedValue({}); await expect(getPullRequest('org', 'repo', 7, 19)).rejects.toThrow('Unable to retrieve pull request'); diff --git a/src/server/lib/github/index.ts b/src/server/lib/github/index.ts index 83ffb846..485fc2bc 100644 --- a/src/server/lib/github/index.ts +++ b/src/server/lib/github/index.ts @@ -134,7 +134,7 @@ export async function createOrUpdatePullRequestComment({ } } -export async function updatePullRequestLabels({ +export async function addPullRequestLabels({ installationId, pullRequestNumber, fullName, @@ -146,9 +146,8 @@ export async function updatePullRequestLabels({ labels: string[]; }) { try { - const client = await createOctokitClient({ installationId, caller: 'updatePullRequestLabels' }); - const requestUrl = `PUT /repos/${fullName}/issues/${pullRequestNumber}/labels`; - return await client.request(requestUrl, { + const client = await createOctokitClient({ installationId, caller: 'addPullRequestLabels' }); + return await client.request(`POST /repos/${fullName}/issues/${pullRequestNumber}/labels`, { data: { labels }, }); } catch (error) { @@ -157,7 +156,36 @@ export async function updatePullRequestLabels({ repo: fullName, pr: pullRequestNumber, labels: labels.toString(), - }).error('GitHub: labels update failed'); + }).error('GitHub: labels add failed'); + throw error; + } +} + +export async function removePullRequestLabel({ + installationId, + pullRequestNumber, + fullName, + label, +}: { + installationId: number; + pullRequestNumber: number; + fullName: string; + label: string; +}) { + try { + const client = await createOctokitClient({ installationId, caller: 'removePullRequestLabel' }); + return await client.request( + `DELETE /repos/${fullName}/issues/${pullRequestNumber}/labels/${encodeURIComponent(label)}` + ); + } catch (error) { + // GitHub answers 404 when the label is already gone, which is the state we wanted. + if (error?.status === 404) return null; + getLogger({ + error, + repo: fullName, + pr: pullRequestNumber, + label, + }).error('GitHub: label remove failed'); throw error; } } diff --git a/src/server/services/__tests__/label.test.ts b/src/server/services/__tests__/label.test.ts index 5c8b1a82..c7aae261 100644 --- a/src/server/services/__tests__/label.test.ts +++ b/src/server/services/__tests__/label.test.ts @@ -23,7 +23,8 @@ const mockLogger = { const mockWithLogContext = jest.fn((_context, callback) => callback()); const mockUpdateLogContext = jest.fn(); const mockWaitForColumnValue = jest.fn(); -const mockUpdatePullRequestLabels = jest.fn(); +const mockAddPullRequestLabels = jest.fn(); +const mockRemovePullRequestLabel = jest.fn(); const mockGetDeployLabel = jest.fn(); const mockRegisterQueue = jest.fn(); @@ -52,7 +53,8 @@ jest.mock('shared/utils', () => ({ })); jest.mock('server/lib/github', () => ({ - updatePullRequestLabels: (input: unknown) => mockUpdatePullRequestLabels(input), + addPullRequestLabels: (input: unknown) => mockAddPullRequestLabels(input), + removePullRequestLabel: (input: unknown) => mockRemovePullRequestLabel(input), })); jest.mock('server/lib/utils', () => ({ @@ -121,10 +123,11 @@ describe('LabelService.processLabelQueue', () => { jest.clearAllMocks(); mockGetDeployLabel.mockResolvedValue('deploy'); mockWaitForColumnValue.mockResolvedValue({ commentId: 101 }); - mockUpdatePullRequestLabels.mockResolvedValue(undefined); + mockAddPullRequestLabels.mockResolvedValue(undefined); + mockRemovePullRequestLabel.mockResolvedValue(undefined); }); - it('adds the deploy label and propagates the queued logging context', async () => { + it('adds only the deploy label and propagates the queued logging context', async () => { const { service, findById, withGraphFetched } = createService(); await service.processLabelQueue(job() as any); @@ -136,12 +139,13 @@ describe('LabelService.processLabelQueue', () => { expect(findById).toHaveBeenCalledWith(8); expect(withGraphFetched).toHaveBeenCalledWith('[repository, build]'); expect(mockUpdateLogContext).toHaveBeenCalledWith({ buildUuid: 'build-uuid' }); - expect(mockUpdatePullRequestLabels).toHaveBeenCalledWith({ + expect(mockAddPullRequestLabels).toHaveBeenCalledWith({ installationId: 22, pullRequestNumber: 17, fullName: 'goodrx/lifecycle', - labels: ['existing', 'deploy'], + labels: ['deploy'], }); + expect(mockRemovePullRequestLabel).not.toHaveBeenCalled(); expect(mockLogger.info).toHaveBeenCalledWith('Label: added label=deploy'); }); @@ -150,11 +154,11 @@ describe('LabelService.processLabelQueue', () => { await service.processLabelQueue(job({ labels: ['existing', 'deploy'] }) as any); - expect(mockUpdatePullRequestLabels).not.toHaveBeenCalled(); + expect(mockAddPullRequestLabels).not.toHaveBeenCalled(); expect(mockLogger.debug).toHaveBeenCalledWith('Deploy label "deploy" already exists on PR, skipping update'); }); - it('removes every configured deploy label while preserving unrelated labels', async () => { + it('removes each configured deploy label individually without touching other labels', async () => { const { service, getLabels } = createService(); await service.processLabelQueue( @@ -162,18 +166,29 @@ describe('LabelService.processLabelQueue', () => { ); expect(getLabels).toHaveBeenCalledTimes(1); - expect(mockUpdatePullRequestLabels).toHaveBeenCalledWith( - expect.objectContaining({ labels: ['keep', 'also-keep'] }) - ); + expect(mockRemovePullRequestLabel.mock.calls.map(([input]) => input)).toEqual([ + { installationId: 22, pullRequestNumber: 17, fullName: 'goodrx/lifecycle', label: 'deploy' }, + { installationId: 22, pullRequestNumber: 17, fullName: 'goodrx/lifecycle', label: 'deploy-stg' }, + ]); + expect(mockAddPullRequestLabels).not.toHaveBeenCalled(); expect(mockLogger.info).toHaveBeenCalledWith('Label: removed label=deploy'); }); - it('leaves labels unchanged when no deploy labels are configured', async () => { + it('makes no GitHub calls on disable when no deploy labels are configured', async () => { const { service } = createService({ labelsConfig: { deploy: undefined } as any }); await service.processLabelQueue(job({ action: 'disable', labels: ['keep'] }) as any); - expect(mockUpdatePullRequestLabels).toHaveBeenCalledWith(expect.objectContaining({ labels: ['keep'] })); + expect(mockRemovePullRequestLabel).not.toHaveBeenCalled(); + expect(mockAddPullRequestLabels).not.toHaveBeenCalled(); + }); + + it('makes no GitHub calls on disable when the PR carries no deploy label', async () => { + const { service } = createService(); + + await service.processLabelQueue(job({ action: 'disable', labels: ['keep', 'dependencies'] }) as any); + + expect(mockRemovePullRequestLabel).not.toHaveBeenCalled(); }); it('waits for a Mission Control comment before updating labels when requested', async () => { @@ -184,7 +199,7 @@ describe('LabelService.processLabelQueue', () => { expect(mockWaitForColumnValue).toHaveBeenCalledWith(pullRequest, 'commentId', 60, 5000); expect(mockLogger.warn).not.toHaveBeenCalled(); - expect(mockUpdatePullRequestLabels).toHaveBeenCalledTimes(1); + expect(mockAddPullRequestLabels).toHaveBeenCalledTimes(1); }); it('continues the label update but records a timeout when the comment never appears', async () => { @@ -197,7 +212,7 @@ describe('LabelService.processLabelQueue', () => { expect(mockLogger.warn).toHaveBeenCalledWith( 'Timeout waiting for comment_id while updating labels after 5 minutes' ); - expect(mockUpdatePullRequestLabels).toHaveBeenCalledTimes(1); + expect(mockAddPullRequestLabels).toHaveBeenCalledTimes(1); }); it('does not wait when the comment is already persisted', async () => { @@ -212,7 +227,7 @@ describe('LabelService.processLabelQueue', () => { const { service } = createService({ pullRequest: null as any }); await expect(service.processLabelQueue(job() as any)).rejects.toThrow('Pull request with id 8 not found'); - expect(mockUpdatePullRequestLabels).not.toHaveBeenCalled(); + expect(mockAddPullRequestLabels).not.toHaveBeenCalled(); expect(mockLogger.error).toHaveBeenCalledWith(expect.any(Object), 'Failed to process label job for PR 8'); }); @@ -225,7 +240,7 @@ describe('LabelService.processLabelQueue', () => { it('propagates GitHub update failures after recording the failed job', async () => { const failure = new Error('github unavailable'); - mockUpdatePullRequestLabels.mockRejectedValue(failure); + mockAddPullRequestLabels.mockRejectedValue(failure); const { service } = createService(); await expect(service.processLabelQueue(job() as any)).rejects.toBe(failure); diff --git a/src/server/services/__tests__/ttlCleanup.test.ts b/src/server/services/__tests__/ttlCleanup.test.ts index 480c6a28..93771fa6 100644 --- a/src/server/services/__tests__/ttlCleanup.test.ts +++ b/src/server/services/__tests__/ttlCleanup.test.ts @@ -17,7 +17,8 @@ const mockListNamespace = jest.fn(); const mockGetAllConfigs = jest.fn(); const mockGetPullRequestLabels = jest.fn(); -const mockUpdatePullRequestLabels = jest.fn(); +const mockAddPullRequestLabels = jest.fn(); +const mockRemovePullRequestLabel = jest.fn(); const mockCreateOrUpdatePullRequestComment = jest.fn(); const mockEnqueueBuildDeletion = jest.fn(); const mockBuildQuery = jest.fn(); @@ -59,7 +60,8 @@ jest.mock('server/lib/logger', () => ({ jest.mock('server/lib/github', () => ({ getPullRequestLabels: (...args: any[]) => mockGetPullRequestLabels(...args), - updatePullRequestLabels: (...args: any[]) => mockUpdatePullRequestLabels(...args), + addPullRequestLabels: (...args: any[]) => mockAddPullRequestLabels(...args), + removePullRequestLabel: (...args: any[]) => mockRemovePullRequestLabel(...args), createOrUpdatePullRequestComment: (...args: any[]) => mockCreateOrUpdatePullRequestComment(...args), })); @@ -199,7 +201,8 @@ describe('TTLCleanupService', () => { expect(mockEnqueueBuildDeletion).toHaveBeenCalledWith(build, 'ttl_closed_pull_request'); expect(mockGetPullRequestLabels).not.toHaveBeenCalled(); - expect(mockUpdatePullRequestLabels).not.toHaveBeenCalled(); + expect(mockRemovePullRequestLabel).not.toHaveBeenCalled(); + expect(mockAddPullRequestLabels).not.toHaveBeenCalled(); expect(mockCreateOrUpdatePullRequestComment).not.toHaveBeenCalled(); }); @@ -225,13 +228,20 @@ describe('TTLCleanupService', () => { }, }); mockGetPullRequestLabels.mockResolvedValue(['sample-deploy']); - mockUpdatePullRequestLabels.mockResolvedValue(undefined); + mockRemovePullRequestLabel.mockResolvedValue(undefined); + mockAddPullRequestLabels.mockResolvedValue(undefined); mockCreateOrUpdatePullRequestComment.mockResolvedValue(undefined); await buildService().processTTLCleanupQueue({ data: {} } as any); expect(mockEnqueueBuildDeletion).not.toHaveBeenCalled(); - expect(mockUpdatePullRequestLabels).toHaveBeenCalledWith({ + expect(mockRemovePullRequestLabel).toHaveBeenCalledWith({ + installationId: 2002, + pullRequestNumber: 77, + fullName: 'ExampleOrg/open-service', + label: 'sample-deploy', + }); + expect(mockAddPullRequestLabels).toHaveBeenCalledWith({ installationId: 2002, pullRequestNumber: 77, fullName: 'ExampleOrg/open-service', @@ -280,7 +290,8 @@ describe('TTLCleanupService', () => { await buildService().processTTLCleanupQueue({ data: { dryRun: true } } as any); expect(mockGetPullRequestLabels).toHaveBeenCalledTimes(1); - expect(mockUpdatePullRequestLabels).not.toHaveBeenCalled(); + expect(mockRemovePullRequestLabel).not.toHaveBeenCalled(); + expect(mockAddPullRequestLabels).not.toHaveBeenCalled(); expect(mockCreateOrUpdatePullRequestComment).not.toHaveBeenCalled(); expect(mockEnqueueBuildDeletion).not.toHaveBeenCalled(); expect(patch).not.toHaveBeenCalled(); @@ -362,7 +373,8 @@ describe('TTLCleanupService', () => { expect(mockBuildQuery).not.toHaveBeenCalled(); expect(mockEnqueueBuildDeletion).not.toHaveBeenCalled(); - expect(mockUpdatePullRequestLabels).not.toHaveBeenCalled(); + expect(mockRemovePullRequestLabel).not.toHaveBeenCalled(); + expect(mockAddPullRequestLabels).not.toHaveBeenCalled(); }); it('derives a missing build UUID from the namespace name', async () => { @@ -408,7 +420,8 @@ describe('TTLCleanupService', () => { expect(mockGetPullRequestLabels).not.toHaveBeenCalled(); expect(mockEnqueueBuildDeletion).not.toHaveBeenCalled(); - expect(mockUpdatePullRequestLabels).not.toHaveBeenCalled(); + expect(mockRemovePullRequestLabel).not.toHaveBeenCalled(); + expect(mockAddPullRequestLabels).not.toHaveBeenCalled(); }); it('skips repositories excluded by configuration before fetching GitHub labels', async () => { @@ -437,7 +450,8 @@ describe('TTLCleanupService', () => { await buildService().processTTLCleanupQueue({ data: {} } as any); expect(mockGetPullRequestLabels).not.toHaveBeenCalled(); - expect(mockUpdatePullRequestLabels).not.toHaveBeenCalled(); + expect(mockRemovePullRequestLabel).not.toHaveBeenCalled(); + expect(mockAddPullRequestLabels).not.toHaveBeenCalled(); }); it('synchronizes GitHub label drift to the database and honors the current keep label', async () => { @@ -462,7 +476,8 @@ describe('TTLCleanupService', () => { await buildService().processTTLCleanupQueue({ data: {} } as any); expect(patch).toHaveBeenCalledWith({ labels: JSON.stringify(['current', 'sample-keep']) }); - expect(mockUpdatePullRequestLabels).not.toHaveBeenCalled(); + expect(mockRemovePullRequestLabel).not.toHaveBeenCalled(); + expect(mockAddPullRequestLabels).not.toHaveBeenCalled(); expect(mockCreateOrUpdatePullRequestComment).not.toHaveBeenCalled(); }); @@ -488,7 +503,8 @@ describe('TTLCleanupService', () => { await buildService().processTTLCleanupQueue({ data: {} } as any); expect(patch).not.toHaveBeenCalled(); - expect(mockUpdatePullRequestLabels).not.toHaveBeenCalled(); + expect(mockRemovePullRequestLabel).not.toHaveBeenCalled(); + expect(mockAddPullRequestLabels).not.toHaveBeenCalled(); expect(mockCreateOrUpdatePullRequestComment).not.toHaveBeenCalled(); }); @@ -510,7 +526,7 @@ describe('TTLCleanupService', () => { }, }); mockGetPullRequestLabels.mockResolvedValue(['sample-deploy']); - mockUpdatePullRequestLabels.mockRejectedValue(new Error('GitHub update failed')); + mockRemovePullRequestLabel.mockRejectedValue(new Error('GitHub update failed')); await expect(buildService().processTTLCleanupQueue({ data: {} } as any)).resolves.toBeUndefined(); @@ -545,7 +561,8 @@ describe('TTLCleanupService', () => { }, }); mockGetPullRequestLabels.mockResolvedValue(['sample-deploy']); - mockUpdatePullRequestLabels.mockResolvedValue(undefined); + mockRemovePullRequestLabel.mockResolvedValue(undefined); + mockAddPullRequestLabels.mockResolvedValue(undefined); mockCreateOrUpdatePullRequestComment.mockResolvedValue(undefined); await buildService().processTTLCleanupQueue({ data: {} } as any); diff --git a/src/server/services/__tests__/ttlCleanupApiDrift.test.ts b/src/server/services/__tests__/ttlCleanupApiDrift.test.ts index 9f48b000..765b4f2a 100644 --- a/src/server/services/__tests__/ttlCleanupApiDrift.test.ts +++ b/src/server/services/__tests__/ttlCleanupApiDrift.test.ts @@ -45,7 +45,8 @@ jest.mock('server/lib/logger', () => ({ })); jest.mock('server/lib/github', () => ({ getPullRequestLabels: (...args: any[]) => mockGetPullRequestLabels(...args), - updatePullRequestLabels: jest.fn(), + addPullRequestLabels: jest.fn(), + removePullRequestLabel: jest.fn(), createOrUpdatePullRequestComment: jest.fn(), })); jest.mock('server/lib/utils', () => ({ @@ -104,8 +105,7 @@ const buildLookup = (build: any) => { }); }; -const runCleanup = () => - service().processTTLCleanupQueue({ data: {} } as any); +const runCleanup = () => service().processTTLCleanupQueue({ data: {} } as any); beforeEach(() => { jest.clearAllMocks(); diff --git a/src/server/services/label.ts b/src/server/services/label.ts index 212b45ca..cec00157 100644 --- a/src/server/services/label.ts +++ b/src/server/services/label.ts @@ -20,7 +20,7 @@ import { QUEUE_NAMES } from 'shared/config'; import { redisClient } from 'server/lib/dependencies'; import { withLogContext, getLogger, LogStage, updateLogContext } from 'server/lib/logger'; import { waitForColumnValue } from 'shared/utils'; -import { updatePullRequestLabels } from 'server/lib/github'; +import { addPullRequestLabels, removePullRequestLabel } from 'server/lib/github'; import { getDeployLabel } from 'server/lib/utils'; interface LabelJob { @@ -96,31 +96,29 @@ export default class LabelService extends Service { } } - let updatedLabels: string[]; + const target = { + installationId: repository.githubInstallationId, + pullRequestNumber: pullRequest.pullRequestNumber, + fullName: pullRequest.fullName, + }; const deployLabel = await getDeployLabel(); if (action === 'enable') { - if (!currentLabels.includes(deployLabel)) { - updatedLabels = [...currentLabels, deployLabel]; - } else { + if (currentLabels.includes(deployLabel)) { getLogger({ stage: LogStage.LABEL_COMPLETE }).debug( `Deploy label "${deployLabel}" already exists on PR, skipping update` ); return; } + await addPullRequestLabels({ ...target, labels: [deployLabel] }); } else { const labelsConfig = await this.db.services.GlobalConfig.getLabels(); const deployLabels = labelsConfig.deploy || []; - updatedLabels = currentLabels.filter((label) => !deployLabels.includes(label)); + for (const label of currentLabels.filter((name) => deployLabels.includes(name))) { + await removePullRequestLabel({ ...target, label }); + } } - await updatePullRequestLabels({ - installationId: repository.githubInstallationId, - pullRequestNumber: pullRequest.pullRequestNumber, - fullName: pullRequest.fullName, - labels: updatedLabels, - }); - getLogger({ stage: LogStage.LABEL_COMPLETE }).info( `Label: ${action === 'enable' ? 'added' : 'removed'} label=${deployLabel}` ); diff --git a/src/server/services/ttlCleanup.ts b/src/server/services/ttlCleanup.ts index 1ecdae72..d173cc9d 100644 --- a/src/server/services/ttlCleanup.ts +++ b/src/server/services/ttlCleanup.ts @@ -20,7 +20,12 @@ import { QUEUE_NAMES } from 'shared/config'; import { redisClient } from 'server/lib/dependencies'; import { withLogContext, updateLogContext, getLogger, LogStage } from 'server/lib/logger'; import * as k8s from '@kubernetes/client-node'; -import { updatePullRequestLabels, createOrUpdatePullRequestComment, getPullRequestLabels } from 'server/lib/github'; +import { + addPullRequestLabels, + removePullRequestLabel, + createOrUpdatePullRequestComment, + getPullRequestLabels, +} from 'server/lib/github'; import { getKeepLabel, getDisabledLabel, getDeployLabel, parsePullRequestLabels } from 'server/lib/utils'; import { Build, PullRequest } from 'server/models'; import Metrics from 'server/lib/metrics'; @@ -356,12 +361,13 @@ export default class TTLCleanupService extends Service { const updatedLabels = currentLabels.filter((label) => label !== deployLabel).concat(disabledLabel); try { - await updatePullRequestLabels({ + const target = { installationId: repository.githubInstallationId, pullRequestNumber: pullRequest.pullRequestNumber, fullName: pullRequest.fullName, - labels: updatedLabels, - }); + }; + await removePullRequestLabel({ ...target, label: deployLabel }); + await addPullRequestLabels({ ...target, labels: [disabledLabel] }); getLogger().debug(`TTL: labels updated PR#${pullRequest.pullRequestNumber}`); From f6008ac8a01283be3835e3c738af625642d97cd3 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Thu, 12 Nov 2026 10:58:00 -0800 Subject: [PATCH 2/3] fix(labels): surface label-remove 404s and add the disabled label before removing deploy --- src/server/lib/github/__tests__/index.test.ts | 18 ++++++++++ src/server/lib/github/index.ts | 10 ++++-- src/server/services/__tests__/label.test.ts | 11 ++++++ .../services/__tests__/ttlCleanup.test.ts | 35 ++++++++++++++++++- src/server/services/ttlCleanup.ts | 2 +- 5 files changed, 72 insertions(+), 4 deletions(-) diff --git a/src/server/lib/github/__tests__/index.test.ts b/src/server/lib/github/__tests__/index.test.ts index cba9696f..b443e6f3 100644 --- a/src/server/lib/github/__tests__/index.test.ts +++ b/src/server/lib/github/__tests__/index.test.ts @@ -390,6 +390,14 @@ describe('repository and pull-request API wrappers', () => { removePullRequestLabel({ installationId: 19, pullRequestNumber: 7, fullName: 'org/repo', label: 'deploy now!' }) ).resolves.toEqual({ data: [] }); expect(request).toHaveBeenCalledWith('DELETE /repos/org/repo/issues/7/labels/deploy%20now!'); + + await removePullRequestLabel({ + installationId: 19, + pullRequestNumber: 7, + fullName: 'org/repo', + label: 'team/infra', + }); + expect(request).toHaveBeenCalledWith('DELETE /repos/org/repo/issues/7/labels/team%2Finfra'); }); test('treats a missing label as already removed but preserves other provider errors', async () => { @@ -397,6 +405,16 @@ describe('repository and pull-request API wrappers', () => { await expect( removePullRequestLabel({ installationId: 19, pullRequestNumber: 7, fullName: 'org/repo', label: 'deploy' }) ).resolves.toBeNull(); + expect(getLogger).toHaveBeenCalledWith({ + repo: 'org/repo', + pr: 7, + label: 'deploy', + message: 'Label does not exist', + }); + expect((getLogger as jest.Mock)().warn).toHaveBeenCalledWith( + 'GitHub: label remove returned 404, treating as already removed' + ); + expect((getLogger as jest.Mock)().error).not.toHaveBeenCalledWith('GitHub: label remove failed'); const failure = Object.assign(new Error('forbidden'), { status: 403 }); mockClient(jest.fn().mockRejectedValue(failure)); diff --git a/src/server/lib/github/index.ts b/src/server/lib/github/index.ts index 485fc2bc..61c37e75 100644 --- a/src/server/lib/github/index.ts +++ b/src/server/lib/github/index.ts @@ -178,8 +178,14 @@ export async function removePullRequestLabel({ `DELETE /repos/${fullName}/issues/${pullRequestNumber}/labels/${encodeURIComponent(label)}` ); } catch (error) { - // GitHub answers 404 when the label is already gone, which is the state we wanted. - if (error?.status === 404) return null; + // GitHub answers 404 when the label is already gone, which is the state we wanted. A missing + // issue or lost app access also 404s, so keep it visible. + if (error?.status === 404) { + getLogger({ repo: fullName, pr: pullRequestNumber, label, message: error?.message }).warn( + 'GitHub: label remove returned 404, treating as already removed' + ); + return null; + } getLogger({ error, repo: fullName, diff --git a/src/server/services/__tests__/label.test.ts b/src/server/services/__tests__/label.test.ts index c7aae261..848b466e 100644 --- a/src/server/services/__tests__/label.test.ts +++ b/src/server/services/__tests__/label.test.ts @@ -191,6 +191,17 @@ describe('LabelService.processLabelQueue', () => { expect(mockRemovePullRequestLabel).not.toHaveBeenCalled(); }); + it('stops removing deploy labels at the first GitHub failure so the job retries', async () => { + const failure = new Error('github unavailable'); + mockRemovePullRequestLabel.mockRejectedValueOnce(failure); + const { service } = createService(); + + await expect( + service.processLabelQueue(job({ action: 'disable', labels: ['deploy', 'deploy-stg'] }) as any) + ).rejects.toBe(failure); + expect(mockRemovePullRequestLabel).toHaveBeenCalledTimes(1); + }); + it('waits for a Mission Control comment before updating labels when requested', async () => { const pullRequest = pullRequestRecord({ commentId: null }); const { service } = createService({ pullRequest }); diff --git a/src/server/services/__tests__/ttlCleanup.test.ts b/src/server/services/__tests__/ttlCleanup.test.ts index 93771fa6..29259d8c 100644 --- a/src/server/services/__tests__/ttlCleanup.test.ts +++ b/src/server/services/__tests__/ttlCleanup.test.ts @@ -247,6 +247,9 @@ describe('TTLCleanupService', () => { fullName: 'ExampleOrg/open-service', labels: ['sample-disabled'], }); + expect(mockAddPullRequestLabels.mock.invocationCallOrder[0]).toBeLessThan( + mockRemovePullRequestLabel.mock.invocationCallOrder[0] + ); expect(mockCreateOrUpdatePullRequestComment).toHaveBeenCalledWith({ installationId: 2002, pullRequestNumber: 77, @@ -508,7 +511,35 @@ describe('TTLCleanupService', () => { expect(mockCreateOrUpdatePullRequestComment).not.toHaveBeenCalled(); }); - it('contains GitHub cleanup failures without persisting or reporting success', async () => { + it('keeps the deploy label and skips reporting when adding the disabled label fails', async () => { + const patch = jest.fn(); + mockExpiredNamespace(); + mockBuildLookup({ + id: 1, + uuid: 'sample-123456', + status: 'deployed', + isStatic: false, + pullRequest: { + status: 'open', + pullRequestNumber: 1, + fullName: 'ExampleOrg/failing', + labels: JSON.stringify(['sample-deploy']), + repository: { githubInstallationId: 1 }, + $query: jest.fn(() => ({ patch })), + }, + }); + mockGetPullRequestLabels.mockResolvedValue(['sample-deploy']); + mockAddPullRequestLabels.mockRejectedValue(new Error('GitHub update failed')); + + await expect(buildService().processTTLCleanupQueue({ data: {} } as any)).resolves.toBeUndefined(); + + expect(mockRemovePullRequestLabel).not.toHaveBeenCalled(); + expect(mockCreateOrUpdatePullRequestComment).not.toHaveBeenCalled(); + expect(patch).not.toHaveBeenCalled(); + expect(mockMetricsIncrement).not.toHaveBeenCalled(); + }); + + it('skips reporting when removing the deploy label fails after the disabled label was added', async () => { const patch = jest.fn(); mockExpiredNamespace(); mockBuildLookup({ @@ -526,10 +557,12 @@ describe('TTLCleanupService', () => { }, }); mockGetPullRequestLabels.mockResolvedValue(['sample-deploy']); + mockAddPullRequestLabels.mockResolvedValue(undefined); mockRemovePullRequestLabel.mockRejectedValue(new Error('GitHub update failed')); await expect(buildService().processTTLCleanupQueue({ data: {} } as any)).resolves.toBeUndefined(); + expect(mockAddPullRequestLabels).toHaveBeenCalledWith(expect.objectContaining({ labels: ['sample-disabled'] })); expect(mockCreateOrUpdatePullRequestComment).not.toHaveBeenCalled(); expect(patch).not.toHaveBeenCalled(); expect(mockMetricsIncrement).not.toHaveBeenCalled(); diff --git a/src/server/services/ttlCleanup.ts b/src/server/services/ttlCleanup.ts index d173cc9d..c42596b9 100644 --- a/src/server/services/ttlCleanup.ts +++ b/src/server/services/ttlCleanup.ts @@ -366,8 +366,8 @@ export default class TTLCleanupService extends Service { pullRequestNumber: pullRequest.pullRequestNumber, fullName: pullRequest.fullName, }; - await removePullRequestLabel({ ...target, label: deployLabel }); await addPullRequestLabels({ ...target, labels: [disabledLabel] }); + await removePullRequestLabel({ ...target, label: deployLabel }); getLogger().debug(`TTL: labels updated PR#${pullRequest.pullRequestNumber}`); From d227c13af1ef6a0aee48c0d03cb5ba6630d8e9c1 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Thu, 12 Nov 2026 12:15:20 -0800 Subject: [PATCH 3/3] fix(labels): finish half-done TTL swaps and remove deploy labels from live PR labels on close --- src/server/services/__tests__/label.test.ts | 28 ++++++++++--- .../services/__tests__/ttlCleanup.test.ts | 39 ++++++++++++++++--- src/server/services/label.ts | 10 +++-- src/server/services/ttlCleanup.ts | 11 +++++- 4 files changed, 72 insertions(+), 16 deletions(-) diff --git a/src/server/services/__tests__/label.test.ts b/src/server/services/__tests__/label.test.ts index 848b466e..3fb6f263 100644 --- a/src/server/services/__tests__/label.test.ts +++ b/src/server/services/__tests__/label.test.ts @@ -25,6 +25,7 @@ const mockUpdateLogContext = jest.fn(); const mockWaitForColumnValue = jest.fn(); const mockAddPullRequestLabels = jest.fn(); const mockRemovePullRequestLabel = jest.fn(); +const mockGetPullRequestLabels = jest.fn(); const mockGetDeployLabel = jest.fn(); const mockRegisterQueue = jest.fn(); @@ -55,6 +56,7 @@ jest.mock('shared/utils', () => ({ jest.mock('server/lib/github', () => ({ addPullRequestLabels: (input: unknown) => mockAddPullRequestLabels(input), removePullRequestLabel: (input: unknown) => mockRemovePullRequestLabel(input), + getPullRequestLabels: (input: unknown) => mockGetPullRequestLabels(input), })); jest.mock('server/lib/utils', () => ({ @@ -125,6 +127,7 @@ describe('LabelService.processLabelQueue', () => { mockWaitForColumnValue.mockResolvedValue({ commentId: 101 }); mockAddPullRequestLabels.mockResolvedValue(undefined); mockRemovePullRequestLabel.mockResolvedValue(undefined); + mockGetPullRequestLabels.mockResolvedValue([]); }); it('adds only the deploy label and propagates the queued logging context', async () => { @@ -158,12 +161,11 @@ describe('LabelService.processLabelQueue', () => { expect(mockLogger.debug).toHaveBeenCalledWith('Deploy label "deploy" already exists on PR, skipping update'); }); - it('removes each configured deploy label individually without touching other labels', async () => { + it('removes each configured deploy label currently on the PR without touching other labels', async () => { + mockGetPullRequestLabels.mockResolvedValue(['deploy', 'keep', 'deploy-stg', 'also-keep']); const { service, getLabels } = createService(); - await service.processLabelQueue( - job({ action: 'disable', labels: ['deploy', 'keep', 'deploy-stg', 'also-keep'] }) as any - ); + await service.processLabelQueue(job({ action: 'disable', labels: [] }) as any); expect(getLabels).toHaveBeenCalledTimes(1); expect(mockRemovePullRequestLabel.mock.calls.map(([input]) => input)).toEqual([ @@ -179,21 +181,35 @@ describe('LabelService.processLabelQueue', () => { await service.processLabelQueue(job({ action: 'disable', labels: ['keep'] }) as any); + expect(mockGetPullRequestLabels).not.toHaveBeenCalled(); expect(mockRemovePullRequestLabel).not.toHaveBeenCalled(); expect(mockAddPullRequestLabels).not.toHaveBeenCalled(); }); - it('makes no GitHub calls on disable when the PR carries no deploy label', async () => { + it('removes nothing on disable when the PR carries no deploy label', async () => { + mockGetPullRequestLabels.mockResolvedValue(['keep', 'dependencies']); const { service } = createService(); - await service.processLabelQueue(job({ action: 'disable', labels: ['keep', 'dependencies'] }) as any); + await service.processLabelQueue(job({ action: 'disable', labels: ['keep', 'dependencies', 'deploy'] }) as any); + + expect(mockRemovePullRequestLabel).not.toHaveBeenCalled(); + }); + it('propagates a failed label lookup on disable so the job retries', async () => { + const failure = new Error('labels unavailable'); + mockGetPullRequestLabels.mockRejectedValue(failure); + const { service } = createService(); + + await expect(service.processLabelQueue(job({ action: 'disable', labels: ['deploy'] }) as any)).rejects.toBe( + failure + ); expect(mockRemovePullRequestLabel).not.toHaveBeenCalled(); }); it('stops removing deploy labels at the first GitHub failure so the job retries', async () => { const failure = new Error('github unavailable'); mockRemovePullRequestLabel.mockRejectedValueOnce(failure); + mockGetPullRequestLabels.mockResolvedValue(['deploy', 'deploy-stg']); const { service } = createService(); await expect( diff --git a/src/server/services/__tests__/ttlCleanup.test.ts b/src/server/services/__tests__/ttlCleanup.test.ts index 29259d8c..d8f45fbb 100644 --- a/src/server/services/__tests__/ttlCleanup.test.ts +++ b/src/server/services/__tests__/ttlCleanup.test.ts @@ -539,8 +539,8 @@ describe('TTLCleanupService', () => { expect(mockMetricsIncrement).not.toHaveBeenCalled(); }); - it('skips reporting when removing the deploy label fails after the disabled label was added', async () => { - const patch = jest.fn(); + it('still posts the cleanup comment when removing the deploy label fails after the disabled label was added', async () => { + const patch = jest.fn().mockResolvedValue(undefined); mockExpiredNamespace(); mockBuildLookup({ id: 1, @@ -559,13 +559,42 @@ describe('TTLCleanupService', () => { mockGetPullRequestLabels.mockResolvedValue(['sample-deploy']); mockAddPullRequestLabels.mockResolvedValue(undefined); mockRemovePullRequestLabel.mockRejectedValue(new Error('GitHub update failed')); + mockCreateOrUpdatePullRequestComment.mockResolvedValue(undefined); await expect(buildService().processTTLCleanupQueue({ data: {} } as any)).resolves.toBeUndefined(); expect(mockAddPullRequestLabels).toHaveBeenCalledWith(expect.objectContaining({ labels: ['sample-disabled'] })); - expect(mockCreateOrUpdatePullRequestComment).not.toHaveBeenCalled(); - expect(patch).not.toHaveBeenCalled(); - expect(mockMetricsIncrement).not.toHaveBeenCalled(); + expect(mockCreateOrUpdatePullRequestComment).toHaveBeenCalledTimes(1); + expect(patch).toHaveBeenCalledWith({ labels: JSON.stringify(['sample-disabled']) }); + expect(mockMetricsIncrement).toHaveBeenCalledWith('total', { dry_run: 'false' }); + }); + + it('retries a pull request left with both the disabled and deploy labels', async () => { + const patch = jest.fn().mockResolvedValue(undefined); + mockExpiredNamespace(); + mockBuildLookup({ + id: 1, + uuid: 'sample-123456', + status: 'deployed', + isStatic: false, + pullRequest: { + status: 'open', + pullRequestNumber: 1, + fullName: 'ExampleOrg/half-done', + labels: JSON.stringify(['sample-disabled', 'sample-deploy']), + repository: { githubInstallationId: 1 }, + $query: jest.fn(() => ({ patch })), + }, + }); + mockGetPullRequestLabels.mockResolvedValue(['sample-disabled', 'sample-deploy']); + mockAddPullRequestLabels.mockResolvedValue(undefined); + mockRemovePullRequestLabel.mockResolvedValue(undefined); + mockCreateOrUpdatePullRequestComment.mockResolvedValue(undefined); + + await buildService().processTTLCleanupQueue({ data: {} } as any); + + expect(mockRemovePullRequestLabel).toHaveBeenCalledWith(expect.objectContaining({ label: 'sample-deploy' })); + expect(mockCreateOrUpdatePullRequestComment).toHaveBeenCalledTimes(1); }); it('renders configured cleanup placeholders using the active label names', async () => { diff --git a/src/server/services/label.ts b/src/server/services/label.ts index cec00157..704cc1b4 100644 --- a/src/server/services/label.ts +++ b/src/server/services/label.ts @@ -20,7 +20,7 @@ import { QUEUE_NAMES } from 'shared/config'; import { redisClient } from 'server/lib/dependencies'; import { withLogContext, getLogger, LogStage, updateLogContext } from 'server/lib/logger'; import { waitForColumnValue } from 'shared/utils'; -import { addPullRequestLabels, removePullRequestLabel } from 'server/lib/github'; +import { addPullRequestLabels, getPullRequestLabels, removePullRequestLabel } from 'server/lib/github'; import { getDeployLabel } from 'server/lib/utils'; interface LabelJob { @@ -114,8 +114,12 @@ export default class LabelService extends Service { } else { const labelsConfig = await this.db.services.GlobalConfig.getLabels(); const deployLabels = labelsConfig.deploy || []; - for (const label of currentLabels.filter((name) => deployLabels.includes(name))) { - await removePullRequestLabel({ ...target, label }); + if (deployLabels.length > 0) { + // The webhook payload can predate a deploy label added after the event. + const liveLabels = await getPullRequestLabels(target); + for (const label of liveLabels.filter((name) => deployLabels.includes(name))) { + await removePullRequestLabel({ ...target, label }); + } } } diff --git a/src/server/services/ttlCleanup.ts b/src/server/services/ttlCleanup.ts index c42596b9..c0ddd9f1 100644 --- a/src/server/services/ttlCleanup.ts +++ b/src/server/services/ttlCleanup.ts @@ -162,6 +162,7 @@ export default class TTLCleanupService extends Service { // Fetch dynamic labels once at the start const keepLabel = await getKeepLabel(); const disabledLabel = await getDisabledLabel(); + const deployLabel = await getDeployLabel(); for (const namespace of namespaces) { const nsName = namespace.metadata?.name; @@ -276,7 +277,7 @@ export default class TTLCleanupService extends Service { continue; } - if (currentLabels.includes(disabledLabel)) { + if (currentLabels.includes(disabledLabel) && !currentLabels.includes(deployLabel)) { getLogger().debug(`Already has ${disabledLabel} label (verified from GitHub), skipping`); continue; } @@ -367,7 +368,13 @@ export default class TTLCleanupService extends Service { fullName: pullRequest.fullName, }; await addPullRequestLabels({ ...target, labels: [disabledLabel] }); - await removePullRequestLabel({ ...target, label: deployLabel }); + // The disabled label alone tears the environment down, so a failed removal must not + // stop the cleanup comment from being posted. + try { + await removePullRequestLabel({ ...target, label: deployLabel }); + } catch (error) { + getLogger().warn({ error }, `TTL: deploy label removal failed pr=${pullRequest.pullRequestNumber}`); + } getLogger().debug(`TTL: labels updated PR#${pullRequest.pullRequestNumber}`);