diff --git a/.github/workflows/refresh-counts.yml b/.github/workflows/refresh-counts.yml index de6d9b7b..c38d499e 100644 --- a/.github/workflows/refresh-counts.yml +++ b/.github/workflows/refresh-counts.yml @@ -72,86 +72,13 @@ jobs: node-version: '20' - name: Refresh registry counts (npm / PyPI / crates) - # Intentionally NO `set -e`/`-o pipefail` here — matches the original - # update-installs.yml behavior. Single-package curl failures fall - # through (curl exits non-zero, jq emits nothing, `$(())` treats the - # empty capture as 0, NPM_TOTAL just doesn't grow that iteration), - # and per-source HWM logic ensures we never regress. With pipefail, - # one transient registry blip would abort the whole daily refresh. - run: | - CACHE_FILE=".vitepress/theme/installs-cache.json" - TODAY=$(date +%Y-%m-%d) - - # ── Fetch fresh registry counts ── - NPM_TOTAL=0 - for pkg in runcycles @runcycles/mcp-server @runcycles/openclaw-budget-guard; do - COUNT=$(curl -sf "https://api.npmjs.org/downloads/point/2020-01-01:${TODAY}/${pkg}" | jq '.downloads // 0') - NPM_TOTAL=$((NPM_TOTAL + COUNT)) - done - echo "npm=$NPM_TOTAL" - - # PyPI: cumulative non-mirror downloads. /overall returns a daily - # series; sum it. Previously /recent .data.last_month was used, which - # is rolling and broke HWM semantics (a quiet month silently froze - # the displayed count above actual cumulative installs). - PYPI_TOTAL=0 - for pkg in runcycles runcycles-openai-agents; do - COUNT=$(curl -sf "https://pypistats.org/api/packages/${pkg}/overall?mirrors=false" | jq '[.data[].downloads] | add // 0') - PYPI_TOTAL=$((PYPI_TOTAL + COUNT)) - done - echo "pypi=$PYPI_TOTAL" - - CRATES_TOTAL=0 - for pkg in runcycles; do - COUNT=$(curl -sf -H 'User-Agent: runcycles-docs (https://github.com/runcycles/docs)' "https://crates.io/api/v1/crates/${pkg}" | jq '.crate.downloads // 0') - CRATES_TOTAL=$((CRATES_TOTAL + COUNT)) - done - echo "crates=$CRATES_TOTAL" - - # ── Read existing cache ── - if [ -f "$CACHE_FILE" ]; then - CACHED=$(cat "$CACHE_FILE") - else - # Cold start with the current schema — empty everything. - CACHED='{"npm":0,"pypi":0,"crates":0,"clones":0,"clonesByRepo":{},"releases":0,"releasesByRepo":{},"ghPackages":0,"maven":0,"total":0,"fetchedAt":""}' - fi - - # ── Schema-regression guard: refuse to operate on a pre-PR-#515 cache. ── - # If the file is missing `clones` / `clonesByRepo` AND has the legacy - # `ghcr` field, this workflow would silently propagate the wipe by - # merging registry updates over the top. Fail loudly instead. - HAS_CLONES=$(echo "$CACHED" | jq 'has("clones") and has("clonesByRepo")') - HAS_GHCR=$(echo "$CACHED" | jq 'has("ghcr")') - if [ "$HAS_CLONES" = "false" ] && [ "$HAS_GHCR" = "true" ]; then - echo "::error::cache schema regression at $CACHE_FILE: pre-PR-#515 shape detected." - echo "Re-seed via 'GITHUB_TOKEN=... npm run build' locally and commit. See .outreach/installs-cache-runbook.md." - exit 1 - fi - - CACHED_NPM=$(echo "$CACHED" | jq '.npm // 0') - CACHED_PYPI=$(echo "$CACHED" | jq '.pypi // 0') - CACHED_CRATES=$(echo "$CACHED" | jq '.crates // 0') - echo "cached: npm=$CACHED_NPM pypi=$CACHED_PYPI crates=$CACHED_CRATES" - - # ── Per-source HWMs (registry sources only) ── - NPM_HWM=$(( NPM_TOTAL > CACHED_NPM ? NPM_TOTAL : CACHED_NPM )) - PYPI_HWM=$(( PYPI_TOTAL > CACHED_PYPI ? PYPI_TOTAL : CACHED_PYPI )) - CRATES_HWM=$(( CRATES_TOTAL > CACHED_CRATES ? CRATES_TOTAL : CACHED_CRATES )) - echo "hwm: npm=$NPM_HWM pypi=$PYPI_HWM crates=$CRATES_HWM" - - # ── Merge fresh registry HWMs into the cache. `total` and `fetchedAt` - # are intentionally NOT written here — the Node step that follows - # owns those fields, so there is a single canonical writer for the - # displayed total (matching the installs.data.ts loader formula). - NEW=$(echo "$CACHED" | jq --indent 2 \ - --argjson npm "$NPM_HWM" \ - --argjson pypi "$PYPI_HWM" \ - --argjson crates "$CRATES_HWM" \ - '. + { npm: $npm, pypi: $pypi, crates: $crates }') - # Preserve trailing newline to match installs.data.ts's writeFileSync - # (`JSON.stringify(...) + '\n'`); avoids a one-line cosmetic diff - # whenever the build runs after this workflow. - printf '%s\n' "$NEW" > "$CACHE_FILE" + # Per-package HWMs in scripts/update-registry-counts.mjs. Replaces + # the prior inline bash + curl + jq, which used aggregate-only HWMs + # and silently masked legitimate growth in one package whenever + # another package's API call failed on the same run (pypistats.org + # has known intermittent CDN issues — verified 2026-05-09 when two + # consecutive runs returned just one of two packages each). + run: node scripts/update-registry-counts.mjs - name: Refresh GitHub-side counts (clones / releases / ghPackages) env: diff --git a/.vitepress/theme/__tests__/installs.test.ts b/.vitepress/theme/__tests__/installs.test.ts index b557272c..66f9dd1b 100644 --- a/.vitepress/theme/__tests__/installs.test.ts +++ b/.vitepress/theme/__tests__/installs.test.ts @@ -176,3 +176,102 @@ describe('clones day-cursor accumulator', () => { expect(result.lastSeenDay).toBe('2026-04-17') }) }) + +// ── Per-package HWM (registry counts) ──────────────────────────────── +// +// Mirrors hwmPerPackage in installs.data.ts and scripts/update-registry-counts.mjs. +// Per-package HWMs solve the failure mode where one package's API call +// fails on the same run as another's legit growth — aggregate-only HWMs +// would mask that growth; per-package HWMs preserve each independently. +function hwmPerPackage( + packages: readonly string[], + fetched: Record, + cachedByPackage: Record, +): { byPackage: Record; aggregate: number } { + const updated: Record = { ...cachedByPackage } + for (const pkg of packages) { + const fresh = fetched[pkg] + const cached = updated[pkg] ?? 0 + updated[pkg] = fresh != null ? Math.max(fresh, cached) : cached + } + const aggregate = Object.values(updated).reduce( + (sum, v) => sum + (typeof v === 'number' ? v : 0), + 0, + ) + return { byPackage: updated, aggregate } +} + +describe('per-package HWM', () => { + const PACKAGES = ['runcycles', 'runcycles-openai-agents'] as const + + it('cold start: empty cached map, all fetched values become per-package HWMs', () => { + const result = hwmPerPackage(PACKAGES, { 'runcycles': 1620, 'runcycles-openai-agents': 447 }, {}) + expect(result.byPackage).toEqual({ 'runcycles': 1620, 'runcycles-openai-agents': 447 }) + expect(result.aggregate).toBe(2067) + }) + + it('all packages succeed and grow: each package HWMs to its new value', () => { + const cached = { 'runcycles': 1620, 'runcycles-openai-agents': 447 } + const fetched = { 'runcycles': 1700, 'runcycles-openai-agents': 500 } + const result = hwmPerPackage(PACKAGES, fetched, cached) + expect(result.byPackage).toEqual({ 'runcycles': 1700, 'runcycles-openai-agents': 500 }) + expect(result.aggregate).toBe(2200) + }) + + it('one package API fails (null), other succeeds and grows: failed package preserved, growth captured', () => { + // The exact scenario from 2026-05-09: pypistats CDN flakiness + // returned data for one package but errored on the other. With + // aggregate-only HWMs, the surviving package's growth would have + // been masked by an aggregate-HWM "regression" check. + const cached = { 'runcycles': 1620, 'runcycles-openai-agents': 447 } + const fetched = { 'runcycles': 1700, 'runcycles-openai-agents': null } + const result = hwmPerPackage(PACKAGES, fetched, cached) + expect(result.byPackage).toEqual({ 'runcycles': 1700, 'runcycles-openai-agents': 447 }) + expect(result.aggregate).toBe(2147) // captured the +80 growth + }) + + it('all packages fail (all null): aggregate frozen at cached values', () => { + const cached = { 'runcycles': 1620, 'runcycles-openai-agents': 447 } + const fetched = { 'runcycles': null, 'runcycles-openai-agents': null } + const result = hwmPerPackage(PACKAGES, fetched, cached) + expect(result.byPackage).toEqual({ 'runcycles': 1620, 'runcycles-openai-agents': 447 }) + expect(result.aggregate).toBe(2067) + }) + + it('one package returns lower than cached (rolling-window blip): preserved at cached HWM', () => { + // PyPI Stats /overall sometimes returns a truncated daily series; + // today's sum can be less than the cached HWM. Per-package HWM keeps + // the cached value, just like the aggregate-only version did. + const cached = { 'runcycles': 1620, 'runcycles-openai-agents': 447 } + const fetched = { 'runcycles': 1620, 'runcycles-openai-agents': 100 } // dropped from 447 + const result = hwmPerPackage(PACKAGES, fetched, cached) + expect(result.byPackage).toEqual({ 'runcycles': 1620, 'runcycles-openai-agents': 447 }) + expect(result.aggregate).toBe(2067) + }) + + it('successful zero is treated as a real value, not a failure', () => { + // A brand-new package on PyPI may legitimately have 0 downloads in + // its window. fetched=0 (number) is distinct from fetched=null + // (failure). Both result in cached value being preserved when cached + // is higher, but only null preserves cached when fetched is lower. + const cached = { 'pkg-a': 100 } + // Successful 0 + cached 100 → max(0, 100) = 100 (HWM holds) + expect(hwmPerPackage(['pkg-a'], { 'pkg-a': 0 }, cached).byPackage).toEqual({ 'pkg-a': 100 }) + // null + cached 100 → preserved 100 (same outcome here) + expect(hwmPerPackage(['pkg-a'], { 'pkg-a': null }, cached).byPackage).toEqual({ 'pkg-a': 100 }) + // Cold start with successful 0 → 0 (real value, no HWM to fall back on) + expect(hwmPerPackage(['pkg-a'], { 'pkg-a': 0 }, {}).byPackage).toEqual({ 'pkg-a': 0 }) + // Cold start with null → 0 (no fresh, no cached) + expect(hwmPerPackage(['pkg-a'], { 'pkg-a': null }, {}).byPackage).toEqual({ 'pkg-a': 0 }) + }) + + it('package removed from declared list but present in cached map: preserved', () => { + // A package was un-declared in the source list, but its prior count + // stays in the cached map so the aggregate doesn't regress. + const cached = { 'runcycles': 1620, 'old-package': 999 } + const fetched = { 'runcycles': 1700 } // 'old-package' not in fetched + const result = hwmPerPackage(['runcycles'], fetched, cached) + expect(result.byPackage).toEqual({ 'runcycles': 1700, 'old-package': 999 }) + expect(result.aggregate).toBe(2699) + }) +}) diff --git a/.vitepress/theme/installs.data.ts b/.vitepress/theme/installs.data.ts index 1b5adc97..963bb717 100644 --- a/.vitepress/theme/installs.data.ts +++ b/.vitepress/theme/installs.data.ts @@ -35,6 +35,13 @@ interface InstallsCache { npm: number pypi: number crates: number + // Per-package HWMs. Aggregate fields above are derived from these + // (with the aggregate value floored by its previous high during the + // cold-start migration window). Per-package storage means a transient + // failure on one package can't mask legitimate growth in another. + npmByPackage: Record + pypiByPackage: Record + cratesByPackage: Record clones: number clonesByRepo: Record releases: number @@ -60,6 +67,7 @@ function readCache(): InstallsCache { // File missing or unparseable — cold start, allowed. return { npm: 0, pypi: 0, crates: 0, + npmByPackage: {}, pypiByPackage: {}, cratesByPackage: {}, clones: 0, clonesByRepo: {}, releases: 0, releasesByRepo: {}, ghPackages: 0, @@ -86,17 +94,20 @@ function readCache(): InstallsCache { } return { - npm: raw.npm ?? 0, - pypi: raw.pypi ?? 0, - crates: raw.crates ?? 0, - clones: raw.clones ?? 0, - clonesByRepo: raw.clonesByRepo ?? {}, - releases: raw.releases ?? 0, - releasesByRepo: raw.releasesByRepo ?? {}, - ghPackages: raw.ghPackages ?? 0, - maven: raw.maven ?? 0, - total: raw.total ?? 0, - fetchedAt: raw.fetchedAt ?? '', + npm: raw.npm ?? 0, + pypi: raw.pypi ?? 0, + crates: raw.crates ?? 0, + npmByPackage: raw.npmByPackage ?? {}, + pypiByPackage: raw.pypiByPackage ?? {}, + cratesByPackage: raw.cratesByPackage ?? {}, + clones: raw.clones ?? 0, + clonesByRepo: raw.clonesByRepo ?? {}, + releases: raw.releases ?? 0, + releasesByRepo: raw.releasesByRepo ?? {}, + ghPackages: raw.ghPackages ?? 0, + maven: raw.maven ?? 0, + total: raw.total ?? 0, + fetchedAt: raw.fetchedAt ?? '', } } @@ -174,77 +185,118 @@ function ghHeaders(): Record { return h } +// Per-package count fetchers. Return a map keyed by package name; the +// value is the fetched count, or `null` if the API call failed. The +// `null` distinction matters for HWM: a successful 0 (brand-new +// package) is treated as a real value, but a failure preserves the +// cached value instead of comparing against 0. +// +// Package lists are duplicated in scripts/update-registry-counts.mjs +// (the daily refresh workflow). Keep them in sync by convention. + // ── npm ────────────────────────────────────────────────────────────── const NPM_PACKAGES = [ 'runcycles', '@runcycles/mcp-server', '@runcycles/openclaw-budget-guard', -] +] as const -async function fetchNpmDownloads(): Promise { +async function fetchNpmDownloads(): Promise> { const today = new Date().toISOString().slice(0, 10) - const totals = await Promise.all( - NPM_PACKAGES.map(async (pkg) => { + const entries = await Promise.all( + NPM_PACKAGES.map(async (pkg): Promise<[string, number | null]> => { try { const res = await fetch( `https://api.npmjs.org/downloads/point/2020-01-01:${today}/${pkg}` ) - if (!res.ok) return 0 + if (!res.ok) return [pkg, null] const json = await res.json() as { downloads?: number } - return json.downloads ?? 0 + return [pkg, typeof json.downloads === 'number' ? json.downloads : null] } catch { - return 0 + return [pkg, null] } }) ) - return totals.reduce((a, b) => a + b, 0) + return Object.fromEntries(entries) } // ── PyPI ───────────────────────────────────────────────────────────── const PYPI_PACKAGES = [ 'runcycles', 'runcycles-openai-agents', -] +] as const // /overall returns a daily series of non-mirror downloads; summing it // yields a cumulative total compatible with the per-source HWM used // downstream. /recent .last_month was rolling and broke HWM semantics. -async function fetchPypiDownloads(): Promise { - const totals = await Promise.all( - PYPI_PACKAGES.map(async (pkg) => { +async function fetchPypiDownloads(): Promise> { + const entries = await Promise.all( + PYPI_PACKAGES.map(async (pkg): Promise<[string, number | null]> => { try { const res = await fetch(`https://pypistats.org/api/packages/${pkg}/overall?mirrors=false`) - if (!res.ok) return 0 + if (!res.ok) return [pkg, null] const json = await res.json() as { data?: Array<{ downloads?: number }> } - return (json.data ?? []).reduce((sum, row) => sum + (row.downloads ?? 0), 0) + if (!Array.isArray(json.data)) return [pkg, null] + return [pkg, json.data.reduce((sum, row) => sum + (row.downloads ?? 0), 0)] } catch { - return 0 + return [pkg, null] } }) ) - return totals.reduce((a, b) => a + b, 0) + return Object.fromEntries(entries) } // ── crates.io ──────────────────────────────────────────────────────── -const CRATES_PACKAGES = ['runcycles'] +const CRATES_PACKAGES = ['runcycles'] as const -async function fetchCratesDownloads(): Promise { - const totals = await Promise.all( - CRATES_PACKAGES.map(async (pkg) => { +async function fetchCratesDownloads(): Promise> { + const entries = await Promise.all( + CRATES_PACKAGES.map(async (pkg): Promise<[string, number | null]> => { try { const res = await fetch( `https://crates.io/api/v1/crates/${pkg}`, { headers: { 'User-Agent': 'runcycles-docs (https://github.com/runcycles/docs)' } } ) - if (!res.ok) return 0 + if (!res.ok) return [pkg, null] const json = await res.json() as { crate?: { downloads?: number } } - return json.crate?.downloads ?? 0 + return [pkg, typeof json.crate?.downloads === 'number' ? json.crate.downloads : null] } catch { - return 0 + return [pkg, null] } }) ) - return totals.reduce((a, b) => a + b, 0) + return Object.fromEntries(entries) +} + +/** + * Apply per-package HWM and return both the updated map and the aggregate. + * + * For each declared package: + * - if the API call succeeded: HWM = max(fresh, cached_for_this_package) + * - if it failed (fresh is null): preserve the cached value + * + * Packages NOT in the declared list but present in the cached map are + * preserved (e.g., a package was removed from the source list — its + * prior count stays in the aggregate so removal doesn't regress the + * displayed total). The aggregate is sum of the resulting per-package + * map. + */ +function hwmPerPackage( + packages: readonly string[], + fetched: Record, + cachedByPackage: Record, +): { byPackage: Record; aggregate: number } { + const updated: Record = { ...cachedByPackage } + for (const pkg of packages) { + const fresh = fetched[pkg] + const cached = updated[pkg] ?? 0 + updated[pkg] = fresh != null ? Math.max(fresh, cached) : cached + } + const aggregate = Object.values(updated).reduce( + (sum, v) => sum + (typeof v === 'number' ? v : 0), + 0, + ) + return { byPackage: updated, aggregate } } // ── GitHub: list org repos ─────────────────────────────────────────── @@ -420,11 +472,17 @@ export default { Promise.resolve(fetchMavenDownloads()), ]) - // Per-source high-water marks: each source never decreases independently. - const npm = Math.max(npmFetched, cached.npm) - const pypi = Math.max(pypiFetched, cached.pypi) - const crates = Math.max(cratesFetched, cached.crates) - const maven = Math.max(mavenFetched, cached.maven) + // Per-package HWM: each package's count never decreases independently. + // Aggregate is the sum of the per-package map, floored by the previous + // aggregate (cold-start migration safety net — once the per-package + // map is fully populated, this floor becomes redundant). + const npmHwm = hwmPerPackage(NPM_PACKAGES, npmFetched, cached.npmByPackage) + const pypiHwm = hwmPerPackage(PYPI_PACKAGES, pypiFetched, cached.pypiByPackage) + const cratesHwm = hwmPerPackage(CRATES_PACKAGES, cratesFetched, cached.cratesByPackage) + const npm = Math.max(npmHwm.aggregate, cached.npm) + const pypi = Math.max(pypiHwm.aggregate, cached.pypi) + const crates = Math.max(cratesHwm.aggregate, cached.crates) + const maven = Math.max(mavenFetched, cached.maven) // Clones: cumulative via day-cursor; sum of per-repo counts. const clones = Object.values(clonesResult.updatedByRepo).reduce((a, b) => a + b.count, 0) @@ -451,8 +509,8 @@ export default { const total = npm + pypi + crates + releases + ghPackages + maven console.log( - `[installs] npm=${npmFetched}(hwm:${npm}) pypi=${pypiFetched}(hwm:${pypi})` + - ` crates=${cratesFetched}(hwm:${crates})` + + `[installs] npm=${npmHwm.aggregate}(hwm:${npm}) pypi=${pypiHwm.aggregate}(hwm:${pypi})` + + ` crates=${cratesHwm.aggregate}(hwm:${crates})` + ` clones+${clonesResult.totalAdded}(cache:${clones}, NOT in displayed total)` + ` releases=${releases}` + ` ghPackages=${ghPackages}` + @@ -462,6 +520,9 @@ export default { const now = new Date().toISOString() const newCache: InstallsCache = { npm, pypi, crates, + npmByPackage: npmHwm.byPackage, + pypiByPackage: pypiHwm.byPackage, + cratesByPackage: cratesHwm.byPackage, clones, clonesByRepo: clonesResult.updatedByRepo, releases, releasesByRepo: releasesResult.updatedByRepo, ghPackages, diff --git a/scripts/update-registry-counts.mjs b/scripts/update-registry-counts.mjs new file mode 100644 index 00000000..6c887db3 --- /dev/null +++ b/scripts/update-registry-counts.mjs @@ -0,0 +1,184 @@ +#!/usr/bin/env node +/** + * Refresh per-package and aggregate registry counts (npm / PyPI / crates.io) + * in installs-cache.json. Runs as step 1 of refresh-counts.yml, replacing + * the previous inline bash + curl + jq. + * + * Per-package HWMs solve a real failure mode of the aggregate-only + * approach: when one package's API call fails (pypistats.org has known + * intermittent CDN issues), the surviving packages' growth would be + * masked by the aggregate HWM that "looks like a regression". Per- + * package HWMs preserve each package's high-water independently, so + * one transient failure cannot mask another package's legitimate growth. + * + * Aggregate HWMs (npm/pypi/crates as scalars) are kept as a defensive + * backstop: during cold-start migration when the per-package maps are + * empty, the aggregate HWM still protects against regression. + * + * Mirrors the per-source logic in installs.data.ts (the build-time + * loader). The workflow and the build converge on the same cache state. + * + * Does NOT write `total` or `fetchedAt` — those are written by + * scripts/update-github-counts.mjs (step 2), which is the single + * canonical writer for the displayed total. + */ + +import { readFileSync, writeFileSync, existsSync } from 'node:fs' +import { resolve } from 'node:path' + +const ROOT = process.cwd() +const CACHE_PATH = resolve(ROOT, '.vitepress/theme/installs-cache.json') + +// Package lists are duplicated in installs.data.ts (the build-time +// loader). Keep them in sync by convention; both files reference the +// other in their per-source comments. +const NPM_PACKAGES = [ + 'runcycles', + '@runcycles/mcp-server', + '@runcycles/openclaw-budget-guard', +] +const PYPI_PACKAGES = [ + 'runcycles', + 'runcycles-openai-agents', +] +const CRATES_PACKAGES = [ + 'runcycles', +] + +function readCache() { + if (!existsSync(CACHE_PATH)) { + // Cold start with the current schema — empty everything. + return { + npm: 0, pypi: 0, crates: 0, + npmByPackage: {}, pypiByPackage: {}, cratesByPackage: {}, + clones: 0, clonesByRepo: {}, + releases: 0, releasesByRepo: {}, + ghPackages: 0, maven: 0, + total: 0, fetchedAt: '', + } + } + const raw = JSON.parse(readFileSync(CACHE_PATH, 'utf-8')) + // Schema-regression guard (mirrors installs.data.ts and the prior + // bash version). A cache with the legacy `ghcr` field and no + // `clones`/`clonesByRepo` is a wiped accumulator — refuse to operate. + if ('ghcr' in raw && !('clones' in raw && 'clonesByRepo' in raw)) { + console.error(`::error::cache schema regression at ${CACHE_PATH}: pre-PR-#515 shape detected.`) + console.error(`Re-seed via 'GITHUB_TOKEN=... npm run build' locally and commit. See .outreach/installs-cache-runbook.md.`) + process.exit(1) + } + return raw +} + +// `null` return signals "API call failed, do not lower the cached value +// for this package". A successful `0` (e.g., a brand-new package) is +// distinct: it is a real, low number and HWM will treat it as such. +async function fetchNpm(pkg) { + const today = new Date().toISOString().slice(0, 10) + try { + const res = await fetch(`https://api.npmjs.org/downloads/point/2020-01-01:${today}/${pkg}`) + if (!res.ok) return null + const j = await res.json() + return typeof j.downloads === 'number' ? j.downloads : null + } catch { + return null + } +} + +async function fetchPypi(pkg) { + try { + const res = await fetch(`https://pypistats.org/api/packages/${pkg}/overall?mirrors=false`) + if (!res.ok) return null + const j = await res.json() + if (!j || !Array.isArray(j.data)) return null + return j.data.reduce((s, r) => s + (r.downloads ?? 0), 0) + } catch { + return null + } +} + +async function fetchCrates(pkg) { + try { + const res = await fetch(`https://crates.io/api/v1/crates/${pkg}`, { + headers: { 'User-Agent': 'runcycles-docs (https://github.com/runcycles/docs)' }, + }) + if (!res.ok) return null + const j = await res.json() + return typeof j?.crate?.downloads === 'number' ? j.crate.downloads : null + } catch { + return null + } +} + +/** + * Apply per-package HWM and return both the updated map and the aggregate. + * + * For each declared package: + * - if the API call succeeded: HWM = max(fresh, cached_for_this_package) + * - if it failed (fresh is null): preserve the cached value + * + * Packages NOT in the declared list but present in the cached map are + * preserved as-is (e.g., a package was removed from the source list but + * its prior count stays in the cache so the aggregate doesn't regress). + */ +function hwmPerPackage(packages, fetched, cachedByPackage) { + const updated = { ...cachedByPackage } + for (const pkg of packages) { + const fresh = fetched[pkg] + const cached = updated[pkg] ?? 0 + updated[pkg] = fresh != null ? Math.max(fresh, cached) : cached + } + const aggregate = Object.values(updated).reduce((sum, v) => sum + (typeof v === 'number' ? v : 0), 0) + return { byPackage: updated, aggregate } +} + +async function main() { + const cache = readCache() + + // Fetch fresh per-package counts (parallel within each registry). + const [npmEntries, pypiEntries, cratesEntries] = await Promise.all([ + Promise.all(NPM_PACKAGES.map(async (pkg) => [pkg, await fetchNpm(pkg)])), + Promise.all(PYPI_PACKAGES.map(async (pkg) => [pkg, await fetchPypi(pkg)])), + Promise.all(CRATES_PACKAGES.map(async (pkg) => [pkg, await fetchCrates(pkg)])), + ]) + const npmFetched = Object.fromEntries(npmEntries) + const pypiFetched = Object.fromEntries(pypiEntries) + const cratesFetched = Object.fromEntries(cratesEntries) + + console.log('npm fresh: ', JSON.stringify(npmFetched)) + console.log('pypi fresh: ', JSON.stringify(pypiFetched)) + console.log('crates fresh:', JSON.stringify(cratesFetched)) + + // Per-package HWM, then derive aggregate from the per-package map. + const npmRes = hwmPerPackage(NPM_PACKAGES, npmFetched, cache.npmByPackage ?? {}) + const pypiRes = hwmPerPackage(PYPI_PACKAGES, pypiFetched, cache.pypiByPackage ?? {}) + const cratesRes = hwmPerPackage(CRATES_PACKAGES, cratesFetched, cache.cratesByPackage ?? {}) + + cache.npmByPackage = npmRes.byPackage + cache.pypiByPackage = pypiRes.byPackage + cache.cratesByPackage = cratesRes.byPackage + + // Aggregate HWM safety net: during cold-start migration the per-package + // map is empty and the aggregate from per-package would be lower than + // the legacy aggregate. max() preserves the legacy floor until per- + // package data catches up. + cache.npm = Math.max(npmRes.aggregate, typeof cache.npm === 'number' ? cache.npm : 0) + cache.pypi = Math.max(pypiRes.aggregate, typeof cache.pypi === 'number' ? cache.pypi : 0) + cache.crates = Math.max(cratesRes.aggregate, typeof cache.crates === 'number' ? cache.crates : 0) + + console.log( + `hwm: npm=${cache.npm} (per-pkg sum=${npmRes.aggregate})` + + ` pypi=${cache.pypi} (per-pkg sum=${pypiRes.aggregate})` + + ` crates=${cache.crates} (per-pkg sum=${cratesRes.aggregate})` + ) + + // Don't write `total` or `fetchedAt` — step 2 (update-github-counts.mjs) + // owns those fields, so there is a single canonical writer for the + // displayed total (matching the installs.data.ts loader formula). + writeFileSync(CACHE_PATH, JSON.stringify(cache, null, 2) + '\n') + console.log(`wrote ${CACHE_PATH}`) +} + +main().catch((err) => { + console.error('::error::update-registry-counts failed:', err.message ?? err) + process.exit(1) +})