diff --git a/docs/e2e-testing.md b/docs/e2e-testing.md index 05311fa..3e35e44 100644 --- a/docs/e2e-testing.md +++ b/docs/e2e-testing.md @@ -81,8 +81,8 @@ The join form specs (the table below) are listed with their test counts. The set | Theme | `theme.spec.ts` | Every setting: colour defaults, each colour reaching the form's styles (the primary colour on its buttons), custom CSS applied. | | Copy | `copy.spec.ts` | Every setting: each one's default (or none), the fallbacks for blank sidebar, stage label and joining verb, and where each shows (details, plan and bank-details steps, confirmation step, sidebar). Known problems: JOIN-162 (Home Address Copy never shows), JOIN-163 (day-of-month copy unused), JOIN-164 (organisation details marked required but not enforced). | | Integrations | `integrations-crm.spec.ts`, `integrations-payments.spec.ts`, `integrations-logging.spec.ts` | Every setting. Joins through the real form reach Zetkin (join form, or the API with OAuth; sandbox/live), Mailchimp, Action Network, Auth0 and the join-complete webhook with the credentials and IDs set here; the step webhook; webhook URLs never reach the form. Stripe direct debit and Direct Debit only (the payment methods Stripe's fields are set up with), the subscriptions download; GoCardless sandbox/live; Chargebee hosted pages; getAddress.io and Ideal Postcodes lookups and their key checks; Sentry, Google Cloud Logging and Loggly receiving the plugin's logs (all fake API). Known problem: JOIN-167 (subscriptions download breaks with no Stripe invoices). | -| Logging | | Not yet | -| Debug | | Not yet | +| Logging | `logging-debug.spec.ts` | Every setting. Log Retention: default of 10 days, the set number of days of log files kept, refused unless a whole number of at least 1 (by the browser and by the plugin), and the tab's note on where the number came from. The latest log is shown on the tab. Where else logs go is on the Integrations tab. Known problem: JOIN-172 (the tab can show a stale log file). | +| Debug | `logging-debug.spec.ts` | Every setting: Disable Test Data leaves the join form empty in a test-data build. | ## What is not covered @@ -92,11 +92,11 @@ The `captureJoinBodyViaStripeRedirect` helper bypasses the payment stage entirel ### GoCardless flow -Zero coverage. GoCardless is the primary payment method for Direct Debit members. The hosted redirect flow, API flow, and mandate creation are all untested. There is no equivalent of `captureJoinBodyViaStripeRedirect` for GoCardless. +Partly covered by the settings tests (`tests/settings/integrations-payments.spec.ts`, `features-integrations.spec.ts`): members are sent to GoCardless's hosted billing flow, set up with the access token and environment, and Custom Pages take bank details in the form. Coming back from GoCardless, mandate creation and the join that follows are untested; there is no equivalent of `captureJoinBodyViaStripeRedirect` for GoCardless. ### Chargebee flow -Zero coverage. Neither the hosted pages flow nor the API flow is tested. +Partly covered by the settings tests (`tests/settings/integrations-payments.spec.ts`): with hosted pages on, card payments go to a Chargebee page created with the site and API key; without, the form shows its card fields. Coming back from Chargebee and the join that follows are untested. ### Update flow (`IS_UPDATE_FLOW=true`) @@ -116,7 +116,7 @@ Now covered by the settings tests (`tests/settings/features.spec.ts`). ### Address lookup -`USE_POSTCODE_LOOKUP` enables a postcode-to-address autocomplete. Not tested. +`USE_POSTCODE_LOOKUP` enables a postcode-to-address autocomplete. Now covered by the settings tests (`tests/settings/integrations-payments.spec.ts`), for both getAddress.io and Ideal Postcodes. ### `/join` response handling diff --git a/packages/join-e2e/tests/settings/integrations-logging.spec.ts b/packages/join-e2e/tests/settings/integrations-logging.spec.ts index 8c61761..f71d685 100644 --- a/packages/join-e2e/tests/settings/integrations-logging.spec.ts +++ b/packages/join-e2e/tests/settings/integrations-logging.spec.ts @@ -1,6 +1,16 @@ -import { APIRequestContext } from '@playwright/test'; import { generateKeyPairSync } from 'crypto'; -import { expect, fakeApiRequests, getSettings, joinFormEnv, openJoinForm, test, wpEvalJson } from './fixtures'; +import { + PHP_LOG_FILES, + expect, + fakeApiRequests, + getSettings, + joinFormEnv, + logInfo, + logInvalidPlan, + openJoinForm, + test, + wpEvalJson, +} from './fixtures'; /** * Settings page, Integrations tab: where the plugin's logs go (Sentry, Google @@ -12,34 +22,16 @@ import { expect, fakeApiRequests, getSettings, joinFormEnv, openJoinForm, test, * the service's fake. */ -/** Makes the plugin log an info line ("Recording CK Join form step"). */ -async function logSomething(request: APIRequestContext): Promise { - const response = await request.post('/wp-json/join/v1/step', { data: { email: 'someone@example.com', stage: 'details' } }); - expect(response.ok()).toBe(true); -} - -/** Makes the plugin log an error ("Invalid membership plan: ..."). */ -async function logAnError(request: APIRequestContext): Promise { - const response = await request.post('/wp-json/join/v1/join', { - data: { email: 'someone@example.com', membership: 'no-such-plan' }, - }); - expect(response.status()).toBe(500); -} - /** The size of the plugin's log files on disk, in bytes. */ const logFilesSize = () => - wpEvalJson(` - $dir = \\CommonKnowledge\\JoinBlock\\Logging::getLogDirectory(); - clearstatcache(); - return array_sum(array_map('filesize', glob("$dir/debug-*.log") ?: [])); - `); + wpEvalJson(`clearstatcache(); return array_sum(array_map('filesize', ${PHP_LOG_FILES}));`); -const LOGGING_HOSTS = ['logs-01.loggly.com', 'sentry.e2e.example', 'oauth2.googleapis.com', 'logging.googleapis.com']; +const LOGGING_HOSTS = ['logs-01.loggly.com', 'sentry.e2e.example', 'logging.googleapis.com']; test('by default, logs are written to disk and sent nowhere else', async ({ request }) => { const before = logFilesSize(); - await logSomething(request); - await logAnError(request); + await logInfo(request); + await logInvalidPlan(request); expect(logFilesSize()).toBeGreaterThan(before); const sent = await fakeApiRequests(); @@ -62,7 +54,7 @@ test.describe('Loggly', () => { }); const before = logFilesSize(); - await logSomething(request); + await logInfo(request); const sent = await fakeApiRequests('logs-01.loggly.com'); expect(sent.length).toBeGreaterThan(0); @@ -77,7 +69,7 @@ test.describe('Loggly', () => { await settings.setText('Loggly Customer Token', 'e2e-loggly-token'); await settings.save(); - await logSomething(request); + await logInfo(request); const [first] = await fakeApiRequests('logs-01.loggly.com'); expect(first.headers['x-loggly-tag']).toBe('join-block'); @@ -109,10 +101,10 @@ test.describe('Sentry', () => { expect(getSettings(['sentry_dsn']).sentry_dsn).toBe(DSN); - await logSomething(request); + await logInfo(request); expect(await fakeApiRequests('sentry.e2e.example')).toEqual([]); - await logAnError(request); + await logInvalidPlan(request); const [event] = await fakeApiRequests('sentry.e2e.example'); expect(event).toMatchObject({ method: 'POST', path: '/api/42/envelope/' }); expect(event.headers['x-sentry-auth']).toContain('sentry_key=e2epublickey'); @@ -160,7 +152,7 @@ test.describe('Google Cloud Logging', () => { google_cloud_key_file_contents: keyFile, }); - await logSomething(request); + await logInfo(request); const writes = (await fakeApiRequests('logging.googleapis.com')).filter( (request) => request.method === 'POST' && request.path === '/v2/entries:write', diff --git a/packages/join-e2e/tests/settings/legacy/logging-debug.json b/packages/join-e2e/tests/settings/legacy/logging-debug.json new file mode 100644 index 0000000..2642f88 --- /dev/null +++ b/packages/join-e2e/tests/settings/legacy/logging-debug.json @@ -0,0 +1,4 @@ +{ + "_join_flow_log_retention_days": "4", + "_disable_test_data": "yes" +} diff --git a/packages/join-e2e/tests/settings/logging-debug.spec.ts b/packages/join-e2e/tests/settings/logging-debug.spec.ts new file mode 100644 index 0000000..fa15fef --- /dev/null +++ b/packages/join-e2e/tests/settings/logging-debug.spec.ts @@ -0,0 +1,205 @@ +import { APIRequestContext } from '@playwright/test'; +import { + PHP_LOG_FILES, + STANDARD_JOIN_PAGE, + backUpLogFiles, + expect, + getSettings, + joinFormEnv, + loadLegacySettings, + logInfo, + logInvalidPlan, + openJoinForm, + restoreLogFiles, + test, + wpEvalJson, +} from './fixtures'; + +/** + * Settings page, Logging and Debug tabs. + * + * Logging: how many days of log files are kept, and the log shown on the tab. + * Where else logs go (Sentry, Google Cloud, Loggly) is on the Integrations tab + * (integrations-logging.spec.ts). + * + * Debug: turning off the join form's test data. + */ + +// These tests replace the log files; each test's are put back afterwards. +test.beforeEach(() => backUpLogFiles()); +test.afterEach(() => restoreLogFiles()); + +/** + * Replaces the log files with one for each of the `days` days before today + * (as if the site had been logging that long), so the next log line starts a + * new day's file and the plugin rotates the old ones. Returns the dates laid + * down and the dates kept after rotation, both newest first. + */ +async function rotateLogFiles(request: APIRequestContext, days: number) { + const laidDown = wpEvalJson(` + $dir = \\CommonKnowledge\\JoinBlock\\Logging::getLogDirectory(); + $existing = ${PHP_LOG_FILES}; + preg_match('/debug-([0-9a-f]+)-/', basename($existing[0] ?? 'debug-e2e-'), $match); + $hash = $match[1] ?? bin2hex(random_bytes(18)); + array_map('unlink', $existing); + $dates = []; + for ($i = 1; $i <= ${days}; $i++) { + $dates[] = date('Y-m-d', strtotime("-$i days")); + file_put_contents("$dir/debug-$hash-" . end($dates) . '.log', "old\\n"); + } + return $dates; + `); + await logInfo(request); + const kept = wpEvalJson(`return array_map(fn ($file) => substr(basename($file, '.log'), -10), ${PHP_LOG_FILES});`) + .sort() + .reverse(); + return { laidDown, kept }; +} + +/** + * Checks rotation kept `days` days of files: the new day's, and the newest of + * those laid down. (Worked out from the dates rotateLogFiles saw, so a run + * across midnight still passes.) + */ +function expectKept({ laidDown, kept }: { laidDown: string[]; kept: string[] }, days: number): void { + expect(kept).toHaveLength(days); + expect(kept[0] > laidDown[0]).toBe(true); + expect(kept.slice(1)).toEqual(laidDown.slice(0, days - 1)); +} + +/** JOIN_FLOW_LOG_RETENTION_DAYS from the plugin's environment (packages/join-block/.env), if set. */ +const retentionFromEnvironment = () => + wpEvalJson(`return $_ENV['JOIN_FLOW_LOG_RETENTION_DAYS'] ?? null;`); + +test.describe('Logging', () => { + test('logs are kept for 10 days unless Log Retention says otherwise', async ({ settings, request }) => { + test.skip( + retentionFromEnvironment() !== null, + 'JOIN_FLOW_LOG_RETENTION_DAYS is set in packages/join-block/.env, which replaces the default', + ); + expect(getSettings(['join_flow_log_retention_days']).join_flow_log_retention_days).toBe(''); + + await settings.open('Logging'); + await expect( + settings.page.getByText( + 'Logs are retained for 10 days (default, as neither the Log Retention setting nor JOIN_FLOW_LOG_RETENTION_DAYS is set).', + ), + ).toBeVisible(); + + expectKept(await rotateLogFiles(request, 14), 10); + }); + + test('Log Retention (days) sets how many days of log files are kept', async ({ settings, request }) => { + await settings.open('Logging'); + await settings.setText('Log Retention (days)', '3'); + await settings.save(); + + expect(getSettings(['join_flow_log_retention_days']).join_flow_log_retention_days).toBe('3'); + + await settings.tab('Logging'); + await expect( + settings.page.getByText('Logs are retained for 3 days (from the Log Retention setting above).'), + ).toBeVisible(); + + expectKept(await rotateLogFiles(request, 6), 3); + }); + + for (const value of ['0', '2.5', '-1']) { + test(`Log Retention of ${value} is refused, and nothing is saved`, async ({ settings }) => { + await settings.open('Logging'); + await settings.setText('Log Retention (days)', value); + await settings.tab('Debug'); + await settings.setCheckbox('Disable Test Data', true); + + await settings.expectSaveRefused(); + // The plugin checks it too, for saves that skip the browser's checks. + // (If the refusal above was a reload, the page shows the saved values + // again, so enter the invalid ones again.) + await settings.tab('Logging'); + await settings.setText('Log Retention (days)', value); + await settings.tab('Debug'); + await settings.setCheckbox('Disable Test Data', true); + expect(await settings.saveSkippingPageChecksExpectingErrors()).toEqual([ + 'Log Retention must be a whole number of days, at least 1 (see Logging tab).', + ]); + expect(getSettings(['join_flow_log_retention_days', 'disable_test_data'])).toEqual({ + join_flow_log_retention_days: '', + disable_test_data: false, + }); + }); + } + + test('the tab shows the latest log', async ({ settings, request }) => { + const plan = `log-viewer-${Date.now()}`; + await logInvalidPlan(request, plan); + + await settings.open('Logging'); + await expect(settings.page.locator('pre', { hasText: `Invalid membership plan: ${plan}` })).toBeVisible(); + }); + + // The tab picks its file by a different rule from the plugin's, so with + // another log file in the folder it can show a stale one (Linear JOIN-172). + // Re-enable once it's fixed. + test.fixme('the tab shows the log being written, even with another log file in the folder', async ({ + settings, + request, + }) => { + await logInfo(request); + // A stale file under another name, as an earlier install might leave. + wpEvalJson(` + $dir = \\CommonKnowledge\\JoinBlock\\Logging::getLogDirectory(); + file_put_contents("$dir/debug-zzzz-" . date('Y-m-d', strtotime('-1 day')) . '.log', "stale\\n"); + return true; + `); + const plan = `log-viewer-${Date.now()}`; + await logInvalidPlan(request, plan); + + await settings.open('Logging'); + await expect(settings.page.locator('pre', { hasText: `Invalid membership plan: ${plan}` })).toBeVisible(); + }); +}); + +test.describe('Debug', () => { + test('the join form is filled with test data when it was built with it', async ({ page }) => { + expect(getSettings(['disable_test_data']).disable_test_data).toBe(false); + expect((await joinFormEnv(page, STANDARD_JOIN_PAGE)).DISABLE_TEST_DATA).toBe(false); + + // The E2E suite builds the form with REACT_APP_USE_TEST_DATA (npm test does). + await openJoinForm(page); + await expect(page.locator('input#firstName')).toHaveValue('Test'); + }); + + test('Disable Test Data leaves the join form empty', async ({ settings, page }) => { + await settings.open('Debug'); + await settings.setCheckbox('Disable Test Data', true); + await settings.save(); + + expect(getSettings(['disable_test_data']).disable_test_data).toBe(true); + expect((await joinFormEnv(page, STANDARD_JOIN_PAGE)).DISABLE_TEST_DATA).toBe(true); + + await openJoinForm(page); + await expect(page.locator('input#firstName')).toHaveValue(''); + await expect(page.locator('input#email')).toHaveValue(''); + }); +}); + +test.describe('Logging and Debug settings saved before the settings page was rebuilt', () => { + // See tests/settings/legacy/README.md. + test('are still read and still apply', async ({ settings, page, request }) => { + loadLegacySettings('logging-debug'); + + expect(getSettings(['join_flow_log_retention_days', 'disable_test_data'])).toEqual({ + join_flow_log_retention_days: '4', + disable_test_data: true, + }); + + await openJoinForm(page); + await expect(page.locator('input#firstName')).toHaveValue(''); + + await settings.open('Logging'); + await expect( + settings.page.getByText('Logs are retained for 4 days (from the Log Retention setting above).'), + ).toBeVisible(); + expectKept(await rotateLogFiles(request, 8), 4); + }); +}); diff --git a/packages/join-e2e/tests/settings/support.ts b/packages/join-e2e/tests/settings/support.ts index 5401092..b50966f 100644 --- a/packages/join-e2e/tests/settings/support.ts +++ b/packages/join-e2e/tests/settings/support.ts @@ -151,6 +151,45 @@ export async function postJoin(request: APIRequestContext, data: Record { + const response = await request.post('/wp-json/join/v1/step', { data: { email: 'someone@example.com', stage: 'details' } }); + expect(response.ok()).toBe(true); +} + +/** Makes the plugin log an error naming `plan` ("Invalid membership plan: "). */ +export async function logInvalidPlan(request: APIRequestContext, plan = 'no-such-plan'): Promise { + const response = await request.post('/wp-json/join/v1/join', { data: { email: 'someone@example.com', membership: plan } }); + expect(response.status()).toBe(500); +} + +/** PHP for the paths of the plugin's log files, for use in wpEvalJson(). */ +export const PHP_LOG_FILES = `(glob(\\CommonKnowledge\\JoinBlock\\Logging::getLogDirectory() . '/debug-*.log') ?: [])`; + +const PHP_LOG_BACKUP = `WP_CONTENT_DIR . '/e2e-log-backup'`; + +/** Copies the plugin's log files aside, for tests that change them (see restoreLogFiles). */ +export function backUpLogFiles(): void { + wpEvalJson(` + $backup = ${PHP_LOG_BACKUP}; + wp_mkdir_p($backup); + array_map('unlink', glob("$backup/*") ?: []); + foreach (${PHP_LOG_FILES} as $file) copy($file, "$backup/" . basename($file)); + return true; + `); +} + +/** Puts the plugin's log files back as backUpLogFiles() found them. */ +export function restoreLogFiles(): void { + wpEvalJson(` + $backup = ${PHP_LOG_BACKUP}; + $dir = \\CommonKnowledge\\JoinBlock\\Logging::getLogDirectory(); + array_map('unlink', ${PHP_LOG_FILES}); + foreach (glob("$backup/*") ?: [] as $file) rename($file, "$dir/" . basename($file)); + return true; + `); +} + /** Delivers an event to the plugin's Stripe webhook endpoint, as Stripe would. */ export async function postStripeWebhook(request: APIRequestContext, type: string, data: Record) { const response = await request.post('/wp-json/join/v1/stripe/webhook', { data: { type, data } });