From ba1b0e15fc75c2a6dffa047817a6c0c2c328e1e0 Mon Sep 17 00:00:00 2001 From: Chinesezjc Date: Wed, 12 Aug 2026 16:37:11 +0800 Subject: [PATCH] ci: exempt only push from concurrency cancellation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two self-hosted standby drills each run their complete unsharded aggregate with one gate worker, which takes longer than the interval between master merges, so unconditional cancel-in-progress supersedes a drill before it reaches a verdict and the lane yields no readiness evidence for the failover runbook to point a responder at. Exempt push and nothing else. This has to be decided at workflow level: cancellation applies to the whole superseded run, so a job-level concurrency group cannot exempt its job. The negated form is load-bearing — naming pull_request alone would also stop cancelling workflow_dispatch, and each runner benchmark fans out to twelve larger runners for up to fifteen minutes in this same group on master, so a re-dispatch would queue ahead of a drill instead of replacing a stale measurement. It does not promise every push run finishes: a newer pending run still displaces an older one, only that the lanes periodically reach a verdict. A master push carries only wine-apt-cache and the two drills; every other job is pull-request-gated, workflow_dispatch-gated, or if: false. The spec pins that set and classifies by exact condition, since a negated event test mentions the event it excludes. --- .../2026-07-26-ci-failover-runbook.i18n.yaml | 4 +- .../process/2026-07-26-ci-failover-runbook.md | 4 ++ .../2026-07-26-ci-failover-runbook.zh.md | 4 ++ .github/workflows/ci.yml | 14 +++- scripts/ci-workflow.spec.ts | 65 +++++++++++++++++++ 5 files changed, 88 insertions(+), 3 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-26-ci-failover-runbook.i18n.yaml b/.agents/notes/implemented/process/2026-07-26-ci-failover-runbook.i18n.yaml index 8615d248fc..47fb2a0572 100644 --- a/.agents/notes/implemented/process/2026-07-26-ci-failover-runbook.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-26-ci-failover-runbook.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/process/2026-07-26-ci-failover-runbook.md -2026-07-26-ci-failover-runbook.md: 47901844f4ec581dff8500cc429c54a076b7642b -2026-07-26-ci-failover-runbook.zh.md: 88a793e144a4d06718ffe323a94f8e81f8e62b82 +2026-07-26-ci-failover-runbook.md: 6f2b4bfba9255549b0887aab9061a6513cb3a68f +2026-07-26-ci-failover-runbook.zh.md: 7bb1a025a71eb690d2d90fcdbddbdd56355786fc diff --git a/.agents/notes/implemented/process/2026-07-26-ci-failover-runbook.md b/.agents/notes/implemented/process/2026-07-26-ci-failover-runbook.md index 47901844f4..6f2b4bfba9 100644 --- a/.agents/notes/implemented/process/2026-07-26-ci-failover-runbook.md +++ b/.agents/notes/implemented/process/2026-07-26-ci-failover-runbook.md @@ -12,6 +12,10 @@ The three required Linux worker jobs in [CI](../../../../.github/workflows/ci.ym Each of the three required Linux worker jobs, the independent native Windows job, and the `all checks passed` verdict job — which would otherwise stay queued on the failed pool even after every worker passed — resolves its runner pool through the `DSH_CI_FAILOVER` repository variable. Unset (normal), they run on the hosted enterprise pools. Set to `selfhosted` by any repository writer, all five retarget onto the in-house self-hosted pools: the Linux jobs and verdict onto the `vm-backup` pool, coverage and snapshot concurrency drop to shared-VM bounds, and the hosted-path pnpm cache restores are skipped; the native Windows job onto the `dsh-win-ci` pool. The switch is writer-manageable repository state, not a merge, so it works while every check is red. The in-house pools' readiness is continuously re-proven by the `serial / linux (self-hosted standby)` and `serial / windows (self-hosted standby)` lanes, which run the complete unsharded aggregates on every master push. +`ci.yml` exempts exactly one event from `cancel-in-progress` (`${{ github.event_name != 'push' }}`), so a superseded pull-request run is still cancelled while a push run is left to run. Each drill runs its complete unsharded aggregate with one gate worker, which takes longer than the interval between master merges; under unconditional cancellation a drill is superseded before reaching a verdict and the lane yields no readiness evidence for a responder to check. This is not a guarantee that every push run finishes: GitHub keeps a single pending entry per group, so a newer pending run displaces an older one, and during busy periods intermediate push runs still end as `cancelled`. What the carve-out buys is that the lane periodically reaches a verdict at all, which is what makes it usable as evidence. + +The decision belongs at workflow level because cancellation applies to the whole superseded run: a job-level `concurrency` group does not exempt its job. The negated form is load-bearing rather than cosmetic: naming `pull_request` alone would also stop cancelling `workflow_dispatch`, and each runner benchmark fans out to twelve larger runners for up to fifteen minutes inside this same group on master, so a re-dispatch would queue ahead of a drill instead of replacing a stale measurement. What bounds the cost is that a master push carries only `wine-apt-cache` and these two drills; every other job is pull-request-gated, `workflow_dispatch`-gated, or `if: false`, and `scripts/ci-workflow.spec.ts` pins that set — classifying by exact condition, since a negated event test mentions the event it excludes — so a new push-reachable job cannot quietly start accumulating uncancelled runs. + ### What the in-house pool is `vm-backup`: one 64-core VM, six always-on systemd-managed runner instances. Its image must preinstall Playwright Chromium's Linux system packages; CI downloads the lockfile-selected browser but never runs `apt` on this persistent shared host. Check the latest `serial / linux (self-hosted standby)` run before switching: its aggregate includes browser replay, so a green standby verifies both ordinary capacity and this browser prerequisite. diff --git a/.agents/notes/implemented/process/2026-07-26-ci-failover-runbook.zh.md b/.agents/notes/implemented/process/2026-07-26-ci-failover-runbook.zh.md index 88a793e144..7bb1a025a7 100644 --- a/.agents/notes/implemented/process/2026-07-26-ci-failover-runbook.zh.md +++ b/.agents/notes/implemented/process/2026-07-26-ci-failover-runbook.zh.md @@ -12,6 +12,10 @@ Status: implemented 三个必需的 Linux 工作作业、独立的原生 Windows 作业，以及 `all checks passed` 判定作业（若不随切换，即使全部工作作业通过，它仍会滞留在故障池的队列中）——各自通过仓库变量 `DSH_CI_FAILOVER` 解析运行器池。变量不存在（正常）时它们运行在托管企业池上；由任何具备写权限的协作者设为 `selfhosted` 时，五个作业全部切换到公司自有的自托管池：Linux 作业与判定作业切到 `vm-backup` 池，覆盖率与快照的并发降到共享虚拟机上限，并跳过托管路径的 pnpm 缓存恢复；原生 Windows 作业切到 `dsh-win-ci` 池。这个开关是写者可管理的仓库状态而非一次合并，因此在所有检查都是红色时仍然有效。自有池的就绪状态由 `serial / linux (self-hosted standby)` 与 `serial / windows (self-hosted standby)` 通道持续验证——每次 master 推送都在其上运行完整的未分片聚合流程。 +`ci.yml` 只豁免一个事件不做取消（`${{ github.event_name != 'push' }}`），因此被取代的拉取请求运行仍会取消，而推送运行则留待执行。每次演练以单门禁工作进程执行完整的未分片聚合流程，耗时长于 master 合并的间隔；在无条件取消下，演练会在得出结论前被后续运行取代，该通道无法产出供响应者查看的就绪证据。这并不保证每次推送运行都能跑完：GitHub 每个组只保留一个待运行条目，更新的待运行条目会顶掉更早的，繁忙时段中间的推送运行仍会以 `cancelled` 结束。这项豁免换来的是该通道**周期性**地得出结论，而这正是它能作为证据的前提。 + +这个决定必须放在工作流级：取消作用于被取代的整个运行，作业级 `concurrency` 组并不能豁免其所属作业。采用否定式写法而非仅指名 `pull_request`，是有实质作用的：后者会连 `workflow_dispatch` 一起停止取消，而每次运行器基准测试会在 master 上的同一并发组内同时占用 12 台大规格运行器、最长 15 分钟，届时重复派发会排在演练之前，而不是替换掉已过时的测量。成本之所以可控，是因为一次 master 推送只承载 `wine-apt-cache` 和这两条演练；其余作业都受拉取请求门控、`workflow_dispatch` 门控或 `if: false`，并且 `scripts/ci-workflow.spec.ts` 会锁定这个集合——按条件精确匹配，因为否定式事件判断会包含它所排除的事件名——使新的推送可达作业无法悄悄开始累积未取消的运行。 + ### 自有池是什么 `vm-backup`：一台 64 核虚拟机，6 个常驻 systemd 管理的运行器实例。其镜像必须预装 Playwright Chromium 的 Linux 系统软件包；CI 会下载锁文件选定的浏览器，但绝不在这台持久化共享主机上运行 `apt`。切换前先看 `serial / linux (self-hosted standby)` 最近一次运行：其聚合流程包含浏览器回放，因此绿色热备同时验证常规容量和这项浏览器先决条件。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd3bbb4544..c3549c343a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,9 +15,21 @@ on: - larger-runner-benchmark - consolidated-runner-benchmark +# Cancel a superseded run on every event EXCEPT push. A push run carries the two +# self-hosted standby drills, which take longer than the interval between master +# merges, so cancelling supersedes a drill before it reaches a verdict and the +# lane yields no readiness evidence. Must be decided here: cancellation applies +# to the whole superseded run, so a job-level group cannot exempt its job. +# Negated rather than `== 'pull_request'` so workflow_dispatch keeps cancelling: +# a re-dispatched runner benchmark holds up to 12 larger runners for 15 minutes +# and shares this group with the drills on master, so queueing it would delay +# them. This does not promise every push run finishes (a newer pending run still +# displaces an older one) — only that the lanes periodically reach a verdict. +# Rationale and the bound on this carve-out: +# .agents/notes/implemented/process/2026-07-26-ci-failover-runbook.md concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.event_name != 'push' }} permissions: contents: read diff --git a/scripts/ci-workflow.spec.ts b/scripts/ci-workflow.spec.ts index 2642f234c4..06fb9c2a27 100644 --- a/scripts/ci-workflow.spec.ts +++ b/scripts/ci-workflow.spec.ts @@ -84,6 +84,71 @@ describe('CI workflow', () => { expect(aggregate.needs).not.toContain('serial-windows') }) + it('leaves push runs uncancelled, so the self-hosted standby drills reach a verdict', () => { + const workflow = loadWorkflow('.github/workflows/ci.yml') + if (!isRecord(workflow.jobs) || !isRecord(workflow.concurrency)) { + throw new TypeError('CI workflow must define jobs and a workflow-level concurrency block') + } + + // Cancellation applies to the whole superseded RUN, so this has to be + // decided at workflow level and gated on the event: a job-level group + // cannot exempt its job from its run being cancelled. Only push is exempt — + // a drill takes longer than the interval between master merges. The negated + // form is load-bearing: `== 'pull_request'` would also stop cancelling + // workflow_dispatch, and a re-dispatched runner benchmark holds up to 12 + // larger runners for 15 minutes in this same group on master. + expect(workflow.concurrency['cancel-in-progress']).toBe("${{ github.event_name != 'push' }}") + + // Neither drill may re-introduce a job-level group: it would not help, and + // it would imply the run-scoped cancellation had been solved locally. + for (const name of ['serial-linux-selfhosted', 'serial-windows']) { + const job = workflow.jobs[name] + if (!isRecord(job)) throw new TypeError(`${name} must be defined`) + expect(job.concurrency).toBeUndefined() + // Both stay master-push-only; that is what makes the push carve-out safe. + expect(job.if).toBe("github.event_name == 'push' && github.ref == 'refs/heads/master'") + } + + // What bounds the cost of never cancelling a push run: a master push may + // only carry the cache seeder and the two drills. Any job reachable on push + // would start accumulating uncancelled runs, so the set is pinned here. + // + // Classification is an exact allowlist of the conditions in use, not a + // substring match: `github.event_name != 'pull_request'` mentions + // `pull_request` yet IS push-reachable, so matching on the event name alone + // would silently misclassify it as gated. + const NOT_PUSH_REACHABLE = new Set([ + "github.event_name == 'pull_request'", + "always() && github.event_name == 'pull_request'", + "github.event_name == 'workflow_dispatch' && inputs.suite == 'larger-runner-benchmark'", + "github.event_name == 'workflow_dispatch' && inputs.suite == 'consolidated-runner-benchmark'", + ]) + const pushReachable = Object.entries(workflow.jobs) + .filter(([, job]) => { + if (!isRecord(job)) return false + if (job.if === undefined) return true // unconditional: runs on every event + if (job.if === false) return false // `if: false` parses as a boolean + if (typeof job.if !== 'string') return true // unrecognized shape: surface it + return !NOT_PUSH_REACHABLE.has(job.if.trim()) + }) + .map(([name]) => name) + .sort() + expect(pushReachable).toEqual(['serial-linux-selfhosted', 'serial-windows', 'wine-apt-cache']) + + // Why workflow_dispatch must keep cancelling: each benchmark fans out to a + // dozen larger runners at once, in this same group on master. If it stopped + // cancelling, a re-dispatch would queue ahead of a drill instead of + // replacing the stale measurement. + for (const name of ['larger-runner-benchmark', 'consolidated-runner-benchmark']) { + const job = workflow.jobs[name] + if (!isRecord(job) || !isRecord(job.strategy)) { + throw new TypeError(`${name} must define a matrix strategy`) + } + expect(job.strategy['max-parallel']).toBe(12) + expect(job['timeout-minutes']).toBe(15) + } + }) + it('keeps supported LSP source under native Windows coverage', () => { const config = readFileSync(resolve(root, 'vitest.config.ts'), 'utf8')