From bf9986f18880ca7829815f0d2e3f9a9f7ab36812 Mon Sep 17 00:00:00 2001 From: chiba Date: Sat, 21 Nov 2026 09:14:38 +0800 Subject: [PATCH] fix(sitemap): write real per-page lastmod from git history - last-updated: read HEAD history for .mdx (keep .md fallback, newer wins); return nothing on shallow clones instead of stamping the build commit date - sitemap-lastmod: map sitemap URLs back to source files via resolveUrl, so frontmatter slugs (how-to-parse-push.mdx -> /docs/socket/protocol/push) resolve; only docs-layout pages under /docs get lastmod, API reference, homepage and marketing pages are left without one - astro.config: sitemap serialize emits lastmod when known - release / canary workflows: fetch-depth 0 so the build has git history Co-Authored-By: Claude Opus 5.5 --- .github/workflows/canary-aliyun.yml | 3 +- .github/workflows/canary-cloudflare.yml | 3 +- .github/workflows/canary.yml | 3 +- .github/workflows/release.yml | 9 ++- astro.config.ts | 10 ++- src/lib/last-updated.test.ts | 68 ++++++++++++++++ src/lib/last-updated.ts | 101 ++++++++++++++---------- src/lib/sitemap-lastmod.test.ts | 84 ++++++++++++++++++++ src/lib/sitemap-lastmod.ts | 78 ++++++++++++++++++ 9 files changed, 312 insertions(+), 47 deletions(-) create mode 100644 src/lib/last-updated.test.ts create mode 100644 src/lib/sitemap-lastmod.test.ts create mode 100644 src/lib/sitemap-lastmod.ts diff --git a/.github/workflows/canary-aliyun.yml b/.github/workflows/canary-aliyun.yml index 8dd8b57c..f6d1586f 100644 --- a/.github/workflows/canary-aliyun.yml +++ b/.github/workflows/canary-aliyun.yml @@ -38,7 +38,8 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 1 + # 全量历史:src/lib/last-updated.ts 按提交记录取文档更新日期(sitemap lastmod、页面 Last Updated),浅克隆时取不到 + fetch-depth: 0 - name: Install ossutil run: | diff --git a/.github/workflows/canary-cloudflare.yml b/.github/workflows/canary-cloudflare.yml index 9b1fa113..852ea4ca 100644 --- a/.github/workflows/canary-cloudflare.yml +++ b/.github/workflows/canary-cloudflare.yml @@ -40,7 +40,8 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 1 + # 全量历史:src/lib/last-updated.ts 按提交记录取文档更新日期(sitemap lastmod、页面 Last Updated),浅克隆时取不到 + fetch-depth: 0 - uses: oven-sh/setup-bun@v2 diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 7c262859..c5e79e3a 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -17,7 +17,8 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 1 + # 全量历史:src/lib/last-updated.ts 按提交记录取文档更新日期(sitemap lastmod、页面 Last Updated),浅克隆时取不到 + fetch-depth: 0 - name: Install ossutil run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2d694791..a95dc444 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,8 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 1 + # 全量历史:src/lib/last-updated.ts 按提交记录取文档更新日期(sitemap lastmod、页面 Last Updated),浅克隆时取不到 + fetch-depth: 0 - name: Download ossutils zip run: | @@ -76,7 +77,8 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 1 + # 全量历史:src/lib/last-updated.ts 按提交记录取文档更新日期(sitemap lastmod、页面 Last Updated),浅克隆时取不到 + fetch-depth: 0 - uses: oven-sh/setup-bun@v2 @@ -113,7 +115,8 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 1 + # 全量历史:src/lib/last-updated.ts 按提交记录取文档更新日期(sitemap lastmod、页面 Last Updated),浅克隆时取不到 + fetch-depth: 0 - name: Download ossutils zip run: | diff --git a/astro.config.ts b/astro.config.ts index 54cedbf9..98940138 100644 --- a/astro.config.ts +++ b/astro.config.ts @@ -13,6 +13,7 @@ import remarkDirective from 'remark-directive' import { remarkCallout } from './src/integrations/remark-callout' import { regionHostnameRewrite } from './src/integrations/region-hostname-rewrite' import { sitemapLocaleFiles } from './src/integrations/sitemap-locale-files' +import { sitemapLastmod } from './src/lib/sitemap-lastmod' import { prebuildMcpTools } from './src/integrations/prebuild-mcp-tools' import { prebuildSkills } from './src/integrations/prebuild-skills' @@ -118,7 +119,14 @@ export default defineConfig({ const isZh = /^\/zh-(CN|HK)(\/|$)/.test(new URL(item.url).pathname) const links = item.links ?? (isZh ? [] : [{ url: item.url, lang: 'en' }]) const en = links.find((l) => l.lang === 'en') - return { ...item, links: en ? [...links, { url: en.url, lang: 'x-default' }] : item.links } + // lastmod = 源文件最近一次提交日期(src/lib/sitemap-lastmod.ts); + // 取不到真实日期的页面(首页、API Reference 等)不写,不用构建当天兜底。 + const lastmod = sitemapLastmod(item.url) + return { + ...item, + ...(lastmod ? { lastmod } : {}), + links: en ? [...links, { url: en.url, lang: 'x-default' }] : item.links, + } }, // 按语言分组,再由 sitemapLocaleFiles 落到 /{en,zh-CN,zh-HK}/sitemap.xml, // 总索引为 /sitemap.xml。 diff --git a/src/lib/last-updated.test.ts b/src/lib/last-updated.test.ts new file mode 100644 index 00000000..6d689198 --- /dev/null +++ b/src/lib/last-updated.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from 'vitest' +import { lookupLastUpdated, parseGitLog } from './last-updated' + +// `git log --no-merges --no-renames --format=%cI --name-only HEAD -- docs`,新提交在前。 +const LOG = `2026-11-09T10:00:00+08:00 + +docs/en/docs/cli/orders/exchange-rate.mdx +docs/zh-CN/docs/cli/orders/exchange-rate.mdx + +2026-10-22T09:30:00+08:00 + +docs/en/docs/socket/protocol/how-to-parse-push.mdx +docs/en/docs/cli/orders/exchange-rate.mdx + +2026-08-17T12:00:00+08:00 + +docs/en/docs/getting-started.mdx +docs/en/docs/cli/index.mdx + +2026-03-01T12:00:00+08:00 + +docs/en/docs/getting-started.md +docs/en/docs/cli.md +docs/en/docs/legacy-only.md +` + +describe('parseGitLog', () => { + const map = parseGitLog(LOG) + + it('每个文件取第一次出现(最近一次提交)的日期', () => { + expect(map.get('docs/en/docs/cli/orders/exchange-rate.mdx')).toBe('2026-11-09T10:00:00+08:00') + expect(map.get('docs/en/docs/socket/protocol/how-to-parse-push.mdx')).toBe('2026-10-22T09:30:00+08:00') + }) + + it('只收 docs/ 下的路径', () => { + expect([...map.keys()].every((k) => k.startsWith('docs/'))).toBe(true) + }) + + it('空输出得到空表', () => { + expect(parseGitLog('').size).toBe(0) + }) +}) + +describe('lookupLastUpdated', () => { + const map = parseGitLog(LOG) + + it('认 .mdx 文件', () => { + expect(lookupLastUpdated(map, 'en/docs/cli/orders/exchange-rate')).toBe('2026-11-09T10:00:00+08:00') + expect(lookupLastUpdated(map, 'zh-CN/docs/cli/orders/exchange-rate')).toBe('2026-11-09T10:00:00+08:00') + }) + + it('.mdx 和迁移前的 .md 都有时取较新的', () => { + expect(lookupLastUpdated(map, 'en/docs/getting-started')).toBe('2026-08-17T12:00:00+08:00') + }) + + it('foo/index.mdx 兜底找迁移前的 foo.md,仍取较新的', () => { + expect(lookupLastUpdated(map, 'en/docs/cli/index')).toBe('2026-08-17T12:00:00+08:00') + }) + + it('只有旧 .md 的记录时用它', () => { + expect(lookupLastUpdated(map, 'en/docs/legacy-only')).toBe('2026-03-01T12:00:00+08:00') + }) + + it('查不到返回 null,不编日期', () => { + expect(lookupLastUpdated(map, 'en/docs/not-in-git')).toBeNull() + expect(lookupLastUpdated(new Map(), 'en/docs/getting-started')).toBeNull() + }) +}) diff --git a/src/lib/last-updated.ts b/src/lib/last-updated.ts index a564f7ee..e9c7d178 100644 --- a/src/lib/last-updated.ts +++ b/src/lib/last-updated.ts @@ -3,41 +3,35 @@ import { execSync } from 'node:child_process' /** * Per-file "last updated" dates for docs, sourced from git. * - * This branch's migration commit bulk-rewrote every `.md` → `.mdx`, so the - * .mdx files on HEAD all share that one commit date — useless for a "Last - * Updated" line. The REAL per-page dates live in the pre-migration `.md` - * files' history on `origin/main`. We read those once (a single `git log` - * for the whole docs tree, memoised at module scope so the whole build/dev - * process pays for it exactly once) and map them back to `.mdx` entry ids. + * One `git log` over the whole docs tree (memoised at module scope, so the + * build pays for it once) gives the latest commit date of every file. Used by + * the "Last Updated" line (DocMeta.astro) and the sitemap `` + * (src/lib/sitemap-lastmod.ts). * - * Everything degrades gracefully: if `origin/main` isn't available (e.g. a - * shallow CI checkout) or git errors, the map is empty and callers simply - * omit the Last Updated line rather than showing a wrong date. + * Files are `.mdx` today; the pre-migration `.md` paths are still looked up as + * a fallback, and when both exist the newer date wins. + * + * Everything degrades gracefully: on a shallow clone (CI `fetch-depth: 1`), + * or when git errors, the map is empty and callers omit the date rather than + * show a wrong one. A shallow clone is treated as "unknown" on purpose: its + * boundary commit looks like a root commit that adds every file, which would + * stamp every page with the build-day commit date. */ let cache: Map | null = null -/** Build `{ "docs/en/docs/getting-started.md" → ISO date }` from origin/main. */ -function buildMap(): Map { +/** + * Parse `git log --format=%cI --name-only` output (newest first) into + * `{ "docs/en/docs/getting-started.mdx" → ISO date }`. The first time a path + * appears is its latest commit. + */ +export function parseGitLog(out: string): Map { const map = new Map() - let out = '' - try { - // Newest-first commit log with the files each touched. `%cI` on its own - // line marks a commit; `docs/…` lines are the files. The FIRST time a - // file appears (walking newest→oldest) is its latest commit. - out = execSync('git log --no-merges --format=%cI --name-only origin/main -- docs', { - encoding: 'utf8', - cwd: process.cwd(), - maxBuffer: 64 * 1024 * 1024, - stdio: ['ignore', 'pipe', 'ignore'], - }) - } catch { - return map // origin/main missing or git unavailable → empty (feature hides) - } let currentDate = '' - for (const line of out.split('\n')) { + for (const raw of out.split('\n')) { + const line = raw.trim() if (/^\d{4}-\d\d-\d\dT/.test(line)) { - currentDate = line.trim() + currentDate = line } else if (line.startsWith('docs/') && currentDate && !map.has(line)) { map.set(line, currentDate) } @@ -45,25 +39,52 @@ function buildMap(): Map { return map } +function git(args: string): string { + return execSync(`git ${args}`, { + encoding: 'utf8', + cwd: process.cwd(), + maxBuffer: 64 * 1024 * 1024, + stdio: ['ignore', 'pipe', 'ignore'], + }) +} + +function buildMap(): Map { + try { + if (git('rev-parse --is-shallow-repository').trim() !== 'false') return new Map() + // HEAD = the commit being built (main on release, the branch on canary). + // --no-renames: list paths as-is, and avoid rename detection fetching + // blobs on partial (blob:none) clones. + return parseGitLog(git('log --no-merges --no-renames --format=%cI --name-only HEAD -- docs')) + } catch { + return new Map() // not a git checkout / git unavailable → feature hides + } +} + function getMap(): Map { if (!cache) cache = buildMap() return cache } +/** Newer of two ISO dates (either may be missing). */ +function newer(a: string | undefined, b: string | undefined): string | undefined { + if (!a) return b + if (!b) return a + return Date.parse(b) > Date.parse(a) ? b : a +} + /** - * Latest commit date (ISO string) for a docs entry, or null if unknown. - * `id` is the collection entry id (file path without extension), e.g. - * `en/docs/getting-started` or `en/docs/cli/index`. + * Look up `id` (collection entry id = file path without extension, e.g. + * `en/docs/getting-started` or `en/docs/cli/index`) in a path → date map. */ +export function lookupLastUpdated(map: ReadonlyMap, id: string): string | null { + let best = map.get(`docs/${id}.mdx`) + best = newer(best, map.get(`docs/${id}.md`)) + // Pages migrated from `foo.md` → `foo/index.mdx`. + if (id.endsWith('/index')) best = newer(best, map.get(`docs/${id.replace(/\/index$/, '')}.md`)) + return best ?? null +} + +/** Latest commit date (ISO string) for a docs entry, or null if unknown. */ export function getLastUpdated(id: string): string | null { - const map = getMap() - // Direct `.md` twin of the current `.mdx` path… - const direct = `docs/${id}.md` - if (map.has(direct)) return map.get(direct)! - // …and the flattened form for pages migrated from `foo.md` → `foo/index.mdx`. - if (id.endsWith('/index')) { - const flat = `docs/${id.replace(/\/index$/, '')}.md` - if (map.has(flat)) return map.get(flat)! - } - return null + return lookupLastUpdated(getMap(), id) } diff --git a/src/lib/sitemap-lastmod.test.ts b/src/lib/sitemap-lastmod.test.ts new file mode 100644 index 00000000..987416fd --- /dev/null +++ b/src/lib/sitemap-lastmod.test.ts @@ -0,0 +1,84 @@ +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { buildUrlToId, createLastmodResolver, readDocSources, urlPath, type DocSource } from './sitemap-lastmod' + +const HOST = 'https://open.longbridge.com' + +const SOURCES: DocSource[] = [ + { id: 'en/index', data: { layout: false } }, + { id: 'en/sdk', data: {} }, + { id: 'en/docs/index', data: {} }, + { id: 'en/docs/api', data: { layout: 'api-reference' } }, + { id: 'en/docs/cli/orders/exchange-rate', data: {} }, + { id: 'en/docs/socket/protocol/how-to-parse-push', data: { slug: '/socket/protocol/push' } }, + { id: 'zh-CN/docs/socket/protocol/how-to-parse-push', data: { slug: '/socket/protocol/push' } }, + { id: 'zh-HK/docs/cli/index', data: {} }, +] + +const DATES: Record = { + 'en/index': '2026-11-20T00:00:00+08:00', + 'en/sdk': '2026-11-20T00:00:00+08:00', + 'en/docs/index': '2026-09-01T00:00:00+08:00', + 'en/docs/api': '2026-11-20T00:00:00+08:00', + 'en/docs/cli/orders/exchange-rate': '2026-11-09T10:00:00+08:00', + 'en/docs/socket/protocol/how-to-parse-push': '2026-10-22T09:30:00+08:00', + 'zh-CN/docs/socket/protocol/how-to-parse-push': '2026-10-23T09:30:00+08:00', +} + +const lastmod = createLastmodResolver(buildUrlToId(SOURCES), (id) => DATES[id] ?? null) + +describe('urlPath', () => { + it('去掉域名、尾斜杠和 .html', () => { + expect(urlPath(`${HOST}/docs/cli/`)).toBe('/docs/cli') + expect(urlPath(`${HOST}/docs/cli.html`)).toBe('/docs/cli') + expect(urlPath(`${HOST}/`)).toBe('/') + }) +}) + +describe('sitemap lastmod', () => { + it('普通 .mdx 文档页按文件路径对上日期', () => { + expect(lastmod(`${HOST}/docs/cli/orders/exchange-rate`)).toBe('2026-11-09T10:00:00+08:00') + }) + + it('frontmatter slug 指定的网址对到真正的源文件', () => { + expect(lastmod(`${HOST}/docs/socket/protocol/push`)).toBe('2026-10-22T09:30:00+08:00') + expect(lastmod(`${HOST}/zh-CN/docs/socket/protocol/push`)).toBe('2026-10-23T09:30:00+08:00') + // 按网址拼出来的文件名不存在,不能误配 + expect(lastmod(`${HOST}/docs/socket/protocol/how-to-parse-push`)).toBeUndefined() + }) + + it('index.mdx 对应目录网址', () => { + expect(lastmod(`${HOST}/docs`)).toBe('2026-09-01T00:00:00+08:00') + }) + + it('取不到日期的不写', () => { + // 源文件没有提交记录 + expect(lastmod(`${HOST}/zh-HK/docs/cli`)).toBeUndefined() + // API Reference:/docs/api 是 api-reference 布局,/docs/api/ 由 openapi.yaml 生成 + expect(lastmod(`${HOST}/docs/api`)).toBeUndefined() + expect(lastmod(`${HOST}/docs/api/get-quote`)).toBeUndefined() + // 首页、营销页内容不在 .mdx 里 + expect(lastmod(`${HOST}/`)).toBeUndefined() + expect(lastmod(`${HOST}/sdk`)).toBeUndefined() + }) + + it('git 历史为空(浅克隆)时一条都不写', () => { + const none = createLastmodResolver(buildUrlToId(SOURCES), () => null) + expect(none(`${HOST}/docs/cli/orders/exchange-rate`)).toBeUndefined() + }) +}) + +describe('readDocSources(仓库真实文档)', () => { + const urlToId = buildUrlToId(readDocSources(join(process.cwd(), 'docs'))) + + it('slug 改过网址的页面三种语言都能对上源文件', () => { + expect(urlToId.get('/docs/socket/protocol/push')).toBe('en/docs/socket/protocol/how-to-parse-push') + expect(urlToId.get('/zh-CN/docs/socket/protocol/push')).toBe('zh-CN/docs/socket/protocol/how-to-parse-push') + expect(urlToId.get('/zh-HK/docs/socket/protocol/push')).toBe('zh-HK/docs/socket/protocol/how-to-parse-push') + }) + + it('不含 API Reference 和首页', () => { + expect(urlToId.has('/docs/api')).toBe(false) + expect(urlToId.has('/')).toBe(false) + }) +}) diff --git a/src/lib/sitemap-lastmod.ts b/src/lib/sitemap-lastmod.ts new file mode 100644 index 00000000..dcf972ef --- /dev/null +++ b/src/lib/sitemap-lastmod.ts @@ -0,0 +1,78 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { globSync } from 'glob' +import matter from 'gray-matter' +import type { CollectionEntry } from 'astro:content' +import { resolveLayoutKind, resolveUrl } from '../../packages/utils/src/slug' +import { getLastUpdated } from './last-updated' + +/** + * Sitemap `` per URL, from the git date of the page's source file + * (src/lib/last-updated.ts). + * + * The sitemap integration only sees URLs, and URLs can't be turned back into + * file paths by string munging: frontmatter `slug` renames many pages (e.g. + * docs/en/docs/socket/protocol/how-to-parse-push.mdx → /docs/socket/protocol/push). + * So we read the same files the `docs` collection loads, resolve each one's + * URL with the same `resolveUrl` the page routes use, and build URL → entry id. + * + * Only docs-layout pages under `/docs/` get a date. Pages whose content + * doesn't live in their .mdx (homepage, sdk/pricing/skill marketing pages, the + * API reference rendered from openapi.yaml) are left without `` — + * no date is better than a wrong one, and we never fall back to the build date. + */ + +export interface DocSource { + /** Collection entry id: path under docs/ without `.mdx`, e.g. `en/docs/cli/index`. */ + id: string + data: { slug?: string; layout?: unknown; docs_layout?: unknown } +} + +const asEntry = (s: DocSource) => s as unknown as CollectionEntry<'docs'> + +/** URL path (`/docs/x`, `/zh-CN/docs/x`) → entry id, for pages that can carry a date. */ +export function buildUrlToId(sources: readonly DocSource[]): Map { + const map = new Map() + for (const s of sources) { + if (s.id.split('/')[1] !== 'docs') continue + if (resolveLayoutKind(asEntry(s)) !== 'docs') continue + map.set(resolveUrl(asEntry(s)), s.id) + } + return map +} + +/** Read every docs collection file (same glob as src/content.config.ts). */ +export function readDocSources(docsDir: string): DocSource[] { + return globSync('{en,zh-CN,zh-HK}/**/*.mdx', { cwd: docsDir, posix: true }).map((rel) => ({ + id: rel.replace(/\.mdx$/, ''), + data: matter(readFileSync(join(docsDir, rel), 'utf8')).data as DocSource['data'], + })) +} + +/** Normalise a sitemap URL to the path form `resolveUrl` produces. */ +export function urlPath(url: string): string { + let p = new URL(url).pathname.replace(/\.html$/, '') + if (p.length > 1 && p.endsWith('/')) p = p.slice(0, -1) + return p +} + +export function createLastmodResolver( + urlToId: ReadonlyMap, + dateOf: (id: string) => string | null +): (url: string) => string | undefined { + return (url) => { + const id = urlToId.get(urlPath(url)) + return (id && dateOf(id)) || undefined + } +} + +let resolver: ((url: string) => string | undefined) | null = null + +/** `` for a sitemap URL, or undefined when there's no real date. */ +export function sitemapLastmod(url: string): string | undefined { + if (!resolver) { + const docsDir = join(process.cwd(), 'docs') + resolver = createLastmodResolver(buildUrlToId(readDocSources(docsDir)), getLastUpdated) + } + return resolver(url) +}