diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index a60ad3f0ad..e1f6f39953 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -130,6 +130,7 @@ External packages **directly declared** only by repository tooling, test infrast | [`oxlint-tsgolint`](https://github.com/oxc-project/tsgolint) | MIT | | [`playwright`](https://github.com/microsoft/playwright) | Apache-2.0 | | [`publint`](https://github.com/publint/publint) | MIT | +| [`smol-toml`](https://github.com/squirrelchat/smol-toml) | BSD-3-Clause | | [`tsdown`](https://github.com/rolldown/tsdown) | MIT | | [`typescript-language-server`](https://github.com/typescript-language-server/typescript-language-server) | Apache-2.0 | | [`vite`](https://github.com/vitejs/vite) | MIT | diff --git a/package.json b/package.json index 535d1f4c52..57851edf1c 100644 --- a/package.json +++ b/package.json @@ -141,6 +141,7 @@ "oxlint": "1.76.0", "oxlint-tsgolint": "7.0.2001", "publint": "^0.3.21", + "smol-toml": "^1.7.1", "tsdown": "^0.22.2", "tsx": "^4.22.4", "typescript": "^6.0.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 96a8df6f8f..b309cfbbe2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -99,6 +99,9 @@ importers: publint: specifier: ^0.3.21 version: 0.3.21 + smol-toml: + specifier: ^1.7.1 + version: 1.7.1 tsdown: specifier: ^0.22.2 version: 0.22.2(oxc-resolver@11.20.0)(publint@0.3.21)(tsx@4.22.4)(typescript@6.0.3) @@ -11072,6 +11075,10 @@ packages: resolution: {integrity: sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==} engines: {node: '>= 18'} + smol-toml@1.7.1: + resolution: {integrity: sha512-PPlsspAZ4jbMBu5DMFhfUGDQLu/vrL4SyBROVS37x8ynnVmFIs1VPBz1Co8Xks3TvpIaZXmU85y4DrQ+UyVFoQ==} + engines: {node: '>= 18'} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -16575,6 +16582,8 @@ snapshots: smol-toml@1.6.1: {} + smol-toml@1.7.1: {} + source-map-js@1.2.1: {} source-map@0.6.1: {} diff --git a/scripts/gen-third-party-notices.spec.ts b/scripts/gen-third-party-notices.spec.ts index 7600902bd5..987bf9073b 100644 --- a/scripts/gen-third-party-notices.spec.ts +++ b/scripts/gen-third-party-notices.spec.ts @@ -1,7 +1,7 @@ import { readdirSync, readFileSync } from 'node:fs' import { resolve } from 'node:path' import { describe, expect, it } from 'vitest' -import { isPermissive, type Manifest, manifestPatterns, parsePyprojectRequirements, parsePythonRequirements, parseVendoredRows, render, tierExternalDeps } from './gen-third-party-notices.ts' +import { isPermissive, type Manifest, manifestPatterns, parsePyprojectRequirements, parseVendoredRows, render, tierExternalDeps } from './gen-third-party-notices.ts' const root = resolve(import.meta.dirname, '..') @@ -87,13 +87,6 @@ describe('parseVendoredRows', () => { }) }) -describe('parsePythonRequirements', () => { - it('reads names whether or not the requirement carries a version, extras, or a marker', () => { - expect(parsePythonRequirements('"pydantic>=2.12", "requests", "httpx[http2]", "tomli ; python_version < \'3.11\'", "hatchling >= 1.24.0"')) - .toEqual(['pydantic', 'requests', 'httpx', 'tomli', 'hatchling']) - }) -}) - describe('parsePyprojectRequirements', () => { it('reads the committed manifests', () => { expect(parsePyprojectRequirements(readFileSync(resolve(root, 'python/sdk/pyproject.toml'), 'utf8'))).toContain('pydantic') @@ -127,6 +120,11 @@ describe('parsePyprojectRequirements', () => { .toEqual(['httpx', 'requests']) }) + it('reads names whether or not requirements carry versions, extras, or markers', () => { + expect(parsePyprojectRequirements("[project]\ndependencies = [\"pydantic>=2.12\", \"requests\", \"httpx[http2]\", \"tomli ; python_version < '3.11'\", \"hatchling >= 1.24.0\"]\n")) + .toEqual(['pydantic', 'requests', 'httpx', 'tomli', 'hatchling']) + }) + it('reads single-quoted TOML literals and rejects an unreadable requirement', () => { expect(parsePyprojectRequirements("[project]\ndependencies = ['requests', \"pydantic>=2\"]\n")).toEqual(['requests', 'pydantic']) expect(() => parsePyprojectRequirements('[project]\ndependencies = ["!!broken"]\n')).toThrow(/cannot read a distribution name/) @@ -136,6 +134,27 @@ describe('parsePyprojectRequirements', () => { expect(parsePyprojectRequirements('[project]\ndependencies = [\n "pydantic>=2.12",\n "typing-extensions",\n]\n')) .toEqual(['pydantic', 'typing-extensions']) }) + + it('obeys TOML comments, quoted keys, and escaped strings', () => { + expect(parsePyprojectRequirements([ + '[project] # a legal header comment', + 'dependencies = [', + ' "pydantic", # ] does not close the array', + ' # "old-package" is not a dependency', + ' "tomli; python_version < \'3.11\'",', + ']', + '', + '[dependency-groups]', + '"test.docs" = ["pytest"]', + ].join('\n'))).toEqual(['pydantic', 'tomli', 'pytest']) + }) + + it('accepts dependency-group includes and rejects unsupported requirement shapes', () => { + expect(parsePyprojectRequirements('[dependency-groups]\nbase = ["pytest"]\nall = [{ include-group = "base" }]\n')) + .toEqual(['pytest']) + expect(() => parsePyprojectRequirements('[project]\ndependencies = "pytest"\n')).toThrow(/must be an array/) + expect(() => parsePyprojectRequirements('[dependency-groups]\ntest = [{ unknown = "pytest" }]\n')).toThrow(/unsupported requirement entry/) + }) }) describe('isPermissive', () => { diff --git a/scripts/gen-third-party-notices.ts b/scripts/gen-third-party-notices.ts index 86dde7cf84..c0a3c55887 100644 --- a/scripts/gen-third-party-notices.ts +++ b/scripts/gen-third-party-notices.ts @@ -11,6 +11,7 @@ import { existsSync, globSync, readdirSync, readFileSync, writeFileSync } from 'node:fs' import { resolve } from 'node:path' import * as yaml from 'js-yaml' +import { parse as parseToml, type TomlTableWithoutBigInt, type TomlValueWithoutBigInt } from 'smol-toml' const root = resolve(import.meta.dirname, '..') const OUT = 'THIRD_PARTY_NOTICES.md' @@ -287,83 +288,74 @@ function collectVendored(): VendoredRow[] { return rows } -/** - * Extract the distribution names from one `pyproject.toml` requirement array. - * PEP 508 makes every part after the name optional, so a bare `"requests"` and - * a marker-only `"requests; python_version < '3.11'"` must both be found. - * @param block - the bracketed array text of a requirement list. - * @returns each requirement's distribution name, in file order. - */ -export function parsePythonRequirements(block: string): string[] { - const names: string[] = [] - // TOML strings are single- or double-quoted; every item must yield a name, so - // an unrecognized requirement fails loud instead of dropping a package. - for (const item of block.matchAll(/"([^"]*)"|'([^']*)'/g)) { - const requirement = item[1] ?? item[2] ?? '' - const name = /^\s*([a-zA-Z][a-zA-Z0-9._-]*)\s*(?:\[[^\]]*\])?\s*(?:[<>=!~;@].*)?$/.exec(requirement)?.[1] - if (name === undefined) { - throw new Error(`gen-third-party-notices: cannot read a distribution name from the requirement ${JSON.stringify(requirement)}.`) - } - names.push(name) +/** Whether a parsed TOML value is a table rather than an array or scalar. */ +function isTomlTable(value: TomlValueWithoutBigInt | undefined): value is TomlTableWithoutBigInt { + return value !== undefined && typeof value === 'object' && !Array.isArray(value) +} + +/** Parse one PEP 508 requirement string into its distribution name. */ +function parsePythonRequirement(requirement: string): string { + const name = /^\s*([a-zA-Z][a-zA-Z0-9._-]*)\s*(?:\[[^\]]*\])?\s*(?:[<>=!~;@].*)?$/.exec(requirement)?.[1] + if (name === undefined) { + throw new Error(`gen-third-party-notices: cannot read a distribution name from the requirement ${JSON.stringify(requirement)}.`) } - return names + return name +} + +/** Add the string requirements from one parsed TOML array. */ +function collectPythonRequirementArray( + names: string[], + value: TomlValueWithoutBigInt | undefined, + location: string, + allowGroupIncludes = false, +): void { + if (value === undefined) return + if (!Array.isArray(value)) { + throw new Error(`gen-third-party-notices: ${location} must be an array.`) + } + for (const item of value) { + if (typeof item === 'string') { + names.push(parsePythonRequirement(item)) + continue + } + if (allowGroupIncludes && isTomlTable(item) && typeof item['include-group'] === 'string' && Object.keys(item).length === 1) { + continue + } + throw new Error(`gen-third-party-notices: ${location} contains an unsupported requirement entry.`) + } +} + +/** Read an optional TOML table and reject a present value of another shape. */ +function optionalTomlTable(value: TomlValueWithoutBigInt | undefined, location: string): TomlTableWithoutBigInt | undefined { + if (value === undefined || isTomlTable(value)) return value + throw new Error(`gen-third-party-notices: ${location} must be a table.`) } /** - * Every requirement name a `pyproject.toml` declares, located by TOML table - * rather than by key name: `requires` under `[build-system]`, `dependencies` - * under `[project]`, and every key under `[project.optional-dependencies]` and - * `[dependency-groups]`, whose keys are author-chosen group names. Array bodies - * are scanned with quote awareness, because a requirement may itself contain - * `]` inside extras (`"httpx[http2]"`). + * Every requirement name a `pyproject.toml` declares: `requires` under + * `[build-system]`, `dependencies` under `[project]`, and every key under + * `[project.optional-dependencies]` and `[dependency-groups]`. A TOML parser + * owns comments, quoted keys, escapes, and array boundaries; unsupported + * requirement shapes fail instead of disappearing from the notices. * @param text - the complete `pyproject.toml` contents. * @returns each declared requirement's distribution name, in file order. */ export function parsePyprojectRequirements(text: string): string[] { const names: string[] = [] - let table = '' - const lines = text.split('\n') - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index] ?? '' - const header = /^\s*\[([^\]]+)]\s*$/.exec(line) - if (header?.[1] !== undefined) { - table = header[1] - continue - } - const assignment = /^\s*([A-Za-z0-9._-]+)\s*=\s*\[/.exec(line) - if (assignment?.[1] === undefined) continue - const key = assignment[1] - const bearsRequirements = (table === 'build-system' && key === 'requires') - || (table === 'project' && key === 'dependencies') - || table === 'project.optional-dependencies' - || table === 'dependency-groups' - if (!bearsRequirements) continue + const document = parseToml(text, { integersAsBigInt: false }) + const buildSystem = optionalTomlTable(document['build-system'], '[build-system]') + const project = optionalTomlTable(document.project, '[project]') + collectPythonRequirementArray(names, buildSystem?.requires, '[build-system].requires') + collectPythonRequirementArray(names, project?.dependencies, '[project].dependencies') - // Consume the array body from the opening bracket to its match, ignoring - // brackets inside quoted requirements. - let body = '' - let depth = 0 - let quoted = false - let cursor = index - let column = line.indexOf('[') - scan: for (; cursor < lines.length; cursor += 1) { - const current = lines[cursor] ?? '' - for (; column < current.length; column += 1) { - const character = current[column] ?? '' - if (character === '"' || character === "'") quoted = !quoted - if (!quoted && character === '[') depth += 1 - if (!quoted && character === ']') { - depth -= 1 - if (depth === 0) break scan - } - if (depth > 0) body += character - } - body += '\n' - column = 0 - } - if (depth !== 0) throw new Error(`gen-third-party-notices: unterminated ${key} array in a pyproject.toml table [${table}].`) - names.push(...parsePythonRequirements(body)) - index = cursor + const optional = optionalTomlTable(project?.['optional-dependencies'], '[project.optional-dependencies]') + for (const [group, requirements] of Object.entries(optional ?? {})) { + collectPythonRequirementArray(names, requirements, `[project.optional-dependencies].${group}`) + } + + const groups = optionalTomlTable(document['dependency-groups'], '[dependency-groups]') + for (const [group, requirements] of Object.entries(groups ?? {})) { + collectPythonRequirementArray(names, requirements, `[dependency-groups].${group}`, true) } return names }