diff --git a/.dockerignore b/.dockerignore index d86227de5..c4adbb98a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -30,6 +30,7 @@ apps/mobile/android/ apps/mobile/.expo/ apps/mobile/.gradle/ infra/docker/data +infra/docker/overrides infra/docker/.trusted-config-generations/ infra/docker/.trusted-config-current infra/docker/.release-evidence/ diff --git a/.gitignore b/.gitignore index 14de5e798..a48268bc9 100644 --- a/.gitignore +++ b/.gitignore @@ -77,6 +77,8 @@ dist/ # Downloaded source data (managed by data-manager service / `openmapx data` CLI) infra/docker/data/ infra/docker/state/ +# The operator's whole-file replacements of release service files +infra/docker/overrides/ # Community integrations (user-installed, not version-controlled) custom_integrations/ diff --git a/apps/api/openapi.json b/apps/api/openapi.json index f1e439ba4..a0d2879b7 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -27692,6 +27692,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -28217,6 +28277,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -28742,6 +28862,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -43259,6 +43439,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -46457,6 +46697,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { diff --git a/apps/api/src/routes/__tests__/admin-services.test.ts b/apps/api/src/routes/__tests__/admin-services.test.ts index 4a07881fd..33bc4be6e 100644 --- a/apps/api/src/routes/__tests__/admin-services.test.ts +++ b/apps/api/src/routes/__tests__/admin-services.test.ts @@ -651,6 +651,28 @@ describe("service credentials", () => { expect(mockRestoreServiceConfig).toHaveBeenCalledTimes(outcome === "not_committed" ? 1 : 0); }, ); + it("answers an apply the operator can fix with its reason, after restoring what was saved", async () => { + const problem = + "infra/docker/overrides/services/martin/config/martin.yaml can't be used: it is a symbolic link. Render refuses it rather than use the release's file in its place: make it a regular file of yours that only you may write, or delete it."; + vi.mocked(applyTrustedConfiguration).mockRejectedValueOnce( + new TrustedConfigurationApplyError({ + outcome: "not_committed", + reason: "failed", + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + problem, + }), + ); + const res = await app.inject({ + method: "POST", + url: "/admin/services/openconditions-ingest/config", + payload: { config: {} }, + }); + expect(res.statusCode).toBe(409); + expect(res.json()).toEqual({ error: problem }); + expect(mockRestoreServiceConfig).toHaveBeenCalledTimes(1); + }); it("persists only the route-validated update through the atomic writer", async () => { mockValidateConfigBody.mockReturnValueOnce({ updates: { RATE_LIMIT_MAX: 240 }, diff --git a/apps/api/src/routes/admin-integration-contract.ts b/apps/api/src/routes/admin-integration-contract.ts index 49cbc905d..76770e02b 100644 --- a/apps/api/src/routes/admin-integration-contract.ts +++ b/apps/api/src/routes/admin-integration-contract.ts @@ -1,4 +1,5 @@ import { integrationAdminOperations } from "@openmapx/core/api-contracts"; +import { TrustedConfigurationApplyError } from "../services/trusted-config-outcome"; import { createAdminRouteRegistrar } from "./admin-contract"; const register = createAdminRouteRegistrar(integrationAdminOperations, { @@ -7,7 +8,11 @@ const register = createAdminRouteRegistrar(integrationAdminOperations, { validationPhase: "preHandler", }); -/** Domain exceptions can include DB/provider inputs. Keep those out of the wire/log error path. */ +/** + * Domain exceptions can include DB/provider inputs. Keep those out of the + * wire/log error path, except a configuration apply the operator can fix (a + * service-file override render may not use), whose reason names files only. + */ export const registerIntegrationAdminRoute: typeof register = (app, operation, handler, options) => register( app, @@ -15,7 +20,10 @@ export const registerIntegrationAdminRoute: typeof register = (app, operation, h async (request, reply) => { try { return await handler(request, reply); - } catch { + } catch (error) { + if (error instanceof TrustedConfigurationApplyError && error.statusCode === 409) { + return reply.code(409).send({ error: error.message }); + } return reply.code(500).send({ error: "Integration administration failed" }); } }, diff --git a/apps/api/src/routes/admin-service-contract.ts b/apps/api/src/routes/admin-service-contract.ts index 717fa08a9..a26abd8a9 100644 --- a/apps/api/src/routes/admin-service-contract.ts +++ b/apps/api/src/routes/admin-service-contract.ts @@ -1,4 +1,5 @@ import { serviceAdminOperations } from "@openmapx/core/api-contracts"; +import { TrustedConfigurationApplyError } from "../services/trusted-config-outcome"; import { createAdminRouteRegistrar } from "./admin-contract"; const register = createAdminRouteRegistrar(serviceAdminOperations, { @@ -7,7 +8,11 @@ const register = createAdminRouteRegistrar(serviceAdminOperations, { validationPhase: "preHandler", }); -/** Domain errors may contain host/configuration inputs; never expose those. */ +/** + * Domain errors may contain host/configuration inputs; never expose those. + * The exception is a configuration apply the operator can fix (an override + * render may not use), whose reason names files only. + */ export const registerServiceAdminRoute: typeof register = (app, operation, handler, options) => register( app, @@ -15,7 +20,10 @@ export const registerServiceAdminRoute: typeof register = (app, operation, handl async (request, reply) => { try { return await handler(request, reply); - } catch { + } catch (error) { + if (error instanceof TrustedConfigurationApplyError && error.statusCode === 409) { + return reply.code(409).send({ error: error.message }); + } return reply.code(500).send({ error: "Service administration failed" }); } }, diff --git a/apps/api/src/services/__tests__/trusted-config-operations.test.ts b/apps/api/src/services/__tests__/trusted-config-operations.test.ts index c5fa889bb..72ca48b81 100644 --- a/apps/api/src/services/__tests__/trusted-config-operations.test.ts +++ b/apps/api/src/services/__tests__/trusted-config-operations.test.ts @@ -155,6 +155,46 @@ describe("applyTrustedConfiguration", () => { expect(executeTrustedConfigurationAndWait).toHaveBeenCalledTimes(1); }, ); + it("asks the agent why an apply was not committed, and gives the admin panel that reason", async () => { + const problem = + "infra/docker/overrides/services/martin/config/martin.yaml can't be used: it has 2 links."; + vi.mocked(executeTrustedConfigurationAndWait).mockResolvedValue({ + outcome: "not_committed", + reason: "failed", + cleanupFailed: false, + }); + inspect.mockImplementation(async (_client, operation: { kind: string }) => + operation.kind === "configuration.failure.inspect" + ? { attemptedRevisionId: `cfg1_${"r".repeat(43)}`, problem } + : { revisionId: null, integrationConfigDigests: [] }, + ); + const error = await applyTrustedConfiguration({ + kind: "stack.render", + operationKey: "opk1_failureReason123", + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(TrustedConfigurationApplyError); + expect(error).toMatchObject({ message: problem, statusCode: 409 }); + expect(isProvenConfigurationNonCommit(error)).toBe(true); + expect(inspect).toHaveBeenCalledWith( + undefined, + { kind: "configuration.failure.inspect", attemptedRevisionId: `cfg1_${"r".repeat(43)}` }, + expect.any(String), + expect.anything(), + ); + + // An agent of an earlier release has no reason to give. + inspect.mockImplementation(async (_client, operation: { kind: string }) => { + if (operation.kind === "configuration.failure.inspect") throw new Error("unknown kind"); + return { revisionId: null, integrationConfigDigests: [] }; + }); + const older = await applyTrustedConfiguration({ + kind: "stack.render", + operationKey: "opk1_failureReason456", + }).catch((caught: unknown) => caught); + expect(older).toMatchObject({ message: "Trusted configuration unavailable" }); + expect((older as { statusCode?: number }).statusCode).toBeUndefined(); + }); + it("does not prepare or publish when authoritative observation is unavailable", async () => { inspect.mockRejectedValueOnce(new Error("synthetic observation failure")); const result = await applyTrustedConfigurationWithEvidence({ diff --git a/apps/api/src/services/trusted-config-operations.ts b/apps/api/src/services/trusted-config-operations.ts index 62528801e..edd219d45 100644 --- a/apps/api/src/services/trusted-config-operations.ts +++ b/apps/api/src/services/trusted-config-operations.ts @@ -189,6 +189,28 @@ export async function applyTrustedConfigurationSnapshot(options: { ); } +/** Why the agent did not commit `revisionId`, when the operator can act on it; null otherwise or unknown. */ +async function readConfigurationFailure( + revisionId: string, + options: ApplyTrustedConfigurationOptions, +): Promise { + if (!configurationGenerationRevisionSchema.safeParse(revisionId).success) return null; + try { + const { problem } = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.failure.inspect", attemptedRevisionId: revisionId }, + createDurableOpsKey( + "trusted-config.failure.inspect", + `${options.operationKey}\0${revisionId}`, + ), + { signal: options.signal }, + ); + return problem; + } catch { + return null; + } +} + async function applySnapshot( options: ApplyTrustedConfigurationOptions, input: TrustedConfigurationPayload, @@ -255,6 +277,15 @@ async function applySnapshot( : submitted ? { outcome: "unknown" as const, reason: "transport_or_result" as const } : { outcome: "not_committed" as const, reason: "failed" as const, cleanupFailed: false }; + // A failure the operator can fix (an override render may not use) has a + // readable reason for the admin panel; an older agent answers none. + const problem = + result.outcome === "not_committed" && result.reason === "failed" + ? await readConfigurationFailure( + (sealed.operation as Extract).revisionId, + options, + ) + : null; let registryUpdateFailed = false; if (result.outcome === "committed") { try { @@ -263,7 +294,12 @@ async function applySnapshot( registryUpdateFailed = true; } } - return { ...result, publisherCleanupFailed: consumed.cleanupFailed, registryUpdateFailed }; + return { + ...result, + publisherCleanupFailed: consumed.cleanupFailed, + registryUpdateFailed, + ...(problem ? { problem } : {}), + }; } catch { // Publication/preparation failed before execute, or an unexpected error after submission. return { diff --git a/apps/api/src/services/trusted-config-outcome.ts b/apps/api/src/services/trusted-config-outcome.ts index 7400d0ee4..220fc1c65 100644 --- a/apps/api/src/services/trusted-config-outcome.ts +++ b/apps/api/src/services/trusted-config-outcome.ts @@ -3,16 +3,25 @@ import type { ConfigurationApplyResult } from "@openmapx/core/ops"; export type TrustedConfigurationApplyOutcome = ( | ConfigurationApplyResult | { outcome: "unknown"; reason: "transport_or_result" } -) & { publisherCleanupFailed: boolean; registryUpdateFailed: boolean }; +) & { + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + /** Why it was not committed, when the operator can fix it (an override render may not use). */ + problem?: string; +}; export class TrustedConfigurationApplyError extends Error { + /** A failure the operator can fix answers 409 with its reason; any other stays a 500. */ + readonly statusCode?: number; + constructor(readonly outcome: TrustedConfigurationApplyOutcome) { super( outcome.outcome === "not_committed" - ? "Trusted configuration unavailable" + ? (outcome.problem ?? "Trusted configuration unavailable") : "Trusted configuration requires reconciliation", ); this.name = "TrustedConfigurationApplyError"; + if (outcome.outcome === "not_committed" && outcome.problem) this.statusCode = 409; } } diff --git a/apps/ops-agent/src/index.ts b/apps/ops-agent/src/index.ts index 2606af8a5..2af0e1162 100644 --- a/apps/ops-agent/src/index.ts +++ b/apps/ops-agent/src/index.ts @@ -36,6 +36,7 @@ import { probePrivacyBackupRuntime, } from "./privacy-backup-runtime"; import { buildOpsAgentServer } from "./server"; +import { describeServiceFilesAtStart } from "./service-files-at-start"; import { afterValidatedServiceAuthority, resolveBootstrapEnabledServiceIds, @@ -62,12 +63,11 @@ async function main(): Promise { // which data-manager can read. OPS_PRIVACY_BACKUP_SCRATCH_ROOT stays set for // agents of earlier releases. janitorPrivacyBackupScratch(config.rootDir); - // The release's built-in manifests apply either way; a checkout that differs - // still supplies the configuration files those services mount. - const drift = services.describeBuiltInServiceDrift( - services.inspectBuiltInServiceDrift(config.rootDir), - ); - if (drift) process.stderr.write(`${drift}\n`); + // The release's manifests and service files apply, with the operator's + // overrides; the checkout's no longer do. Reported, never fatal at start. + for (const line of describeServiceFilesAtStart(join(config.rootDir, "infra", "docker"))) { + process.stderr.write(`${line}\n`); + } await afterValidatedServiceAuthority(config.rootDir, async (releaseAuthority) => { const [apiToken, dataManagerToken] = await Promise.all([ readOpsTokenFile(config.apiTokenFile), diff --git a/apps/ops-agent/src/service-files-at-start.test.ts b/apps/ops-agent/src/service-files-at-start.test.ts new file mode 100644 index 000000000..e89a3da6c --- /dev/null +++ b/apps/ops-agent/src/service-files-at-start.test.ts @@ -0,0 +1,117 @@ +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { services } from "@openmapx/core/server"; +import { afterEach, describe, expect, it } from "vitest"; +import { describeServiceFilesAtStart } from "./service-files-at-start"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function stack() { + const root = mkdtempSync(join(tmpdir(), "openmapx-agent-start-")); + roots.push(root); + const infraDir = join(root, "infra", "docker"); + mkdirSync(infraDir, { recursive: true }); + const directory = join(root, "services", "martin"); + mkdirSync(join(directory, "config"), { recursive: true }); + writeFileSync(join(directory, "config", "martin.yaml"), "listen_addresses: 0.0.0.0:3000\n"); + const martin: services.LoadedService = { + manifest: { + id: "martin", + name: "martin", + version: "1.0.0", + quality: "built-in", + container: { image: "example/martin", tag: "1" }, + bindMounts: [{ source: "config/martin.yaml", target: "/etc/martin/martin.yaml" }], + }, + directory, + isBuiltIn: true, + enabled: true, + }; + const checkout = services.releaseServiceFiles([martin]); + const release: services.ReleaseServiceFiles = { ...checkout, origin: "release" }; + return { infraDir, martin, release }; +} + +const input = (selectedRoots: string[]) => ({ + domain: "maps.example.test", + selectedRoots, + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], +}); + +describe("the agent's start after a panel self-update", () => { + it("starts from its predecessor's generation, which records no service files, and says the next render takes the release's", async () => { + const f = stack(); + // The generation an earlier release rendered: its services mount the checkout. + await services.commitConfigurationGeneration({ + infraDir: f.infraDir, + services: [f.martin], + input: input(["martin"]), + legacyTraefikDynamicMounted: false, + }); + const revision = services.currentConfigurationGeneration(f.infraDir) as string; + const generation = join(f.infraDir, services.CONFIGURATION_GENERATIONS, revision); + rmSync(join(generation, "service-files.json")); + const lines = describeServiceFilesAtStart(f.infraDir, f.release); + expect(lines).toEqual([ + "The applied configuration was rendered by an earlier release, whose services mount their files from the checkout. The next render mounts this release's files instead.", + "The next render imports, once, your edits in the checkout's service files as overrides in infra/docker/overrides/services.", + ]); + + // The update's own render: the import first, then the release's files. + const committed = await services.commitConfigurationGeneration({ + infraDir: f.infraDir, + services: [f.martin], + input: input(["martin"]), + releaseServiceFiles: f.release, + legacyTraefikDynamicMounted: false, + }); + expect(committed.serviceFileReport).toEqual([]); + expect(services.serviceFilesImported(f.infraDir)).toBe(true); + const compose = readFileSync( + join(f.infraDir, services.CURRENT_CONFIGURATION, "docker-compose.generated.yml"), + "utf8", + ); + expect(compose).toMatch( + /\.\.\/\.\.\/service-files\/martin\/[a-f0-9]{64}\/config\/martin\.yaml/, + ); + expect(describeServiceFilesAtStart(f.infraDir, f.release)).toEqual([]); + }); + + it("says the import waits while the checkout's history lists versions the release doesn't", () => { + const f = stack(); + writeFileSync( + join(f.infraDir, "..", "..", "services", "service-file-history.json"), + JSON.stringify({ version: 1, files: { "martin/config/martin.yaml": ["f".repeat(64)] } }), + ); + const shipped = f.release.files.get("martin/config/martin.yaml")?.sha256 ?? ""; + const release = { + ...f.release, + history: new Map([["martin/config/martin.yaml", new Set([shipped])]]), + }; + expect(describeServiceFilesAtStart(f.infraDir, release)).toEqual([ + expect.stringContaining( + "The one-time import of your edits in the checkout's service files waits: the checkout's services/service-file-history.json lists versions of services/martin/config/martin.yaml that the release doesn't", + ), + ]); + }); + + it("reports an override it may not use instead of failing to start", () => { + const f = stack(); + const override = join(f.infraDir, "overrides", "services", "martin", "config", "martin.yaml"); + mkdirSync(join(override, ".."), { recursive: true, mode: 0o755 }); + writeFileSync(override, "x: 1\n"); + chmodSync(override, 0o666); + expect(describeServiceFilesAtStart(f.infraDir, f.release)).toEqual([ + expect.stringContaining("The next render imports, once"), + expect.stringContaining( + "infra/docker/overrides/services/martin/config/martin.yaml can't be used: group or others may write it", + ), + ]); + }); +}); diff --git a/apps/ops-agent/src/service-files-at-start.ts b/apps/ops-agent/src/service-files-at-start.ts new file mode 100644 index 000000000..76b3bc4fb --- /dev/null +++ b/apps/ops-agent/src/service-files-at-start.ts @@ -0,0 +1,52 @@ +import { join, resolve } from "node:path"; +import { services } from "@openmapx/core/server"; + +/** + * What the agent's log says about the service files when it starts. After a + * panel self-update the agent starts from the generation its predecessor + * rendered, which mounts the checkout's files and records none of its own: + * that is expected, and the next render (the update's own) takes the + * release's. Never throws: an unusable override fails that render instead. + */ +export function describeServiceFilesAtStart( + infraDir: string, + // Under OPENMAPX_RELEASE=local none: the checkout's files are the release's. + release: services.ReleaseServiceFiles | null = services.renderedEmbeddedReleaseServiceFiles(), +): string[] { + if (!release) return []; + const lines: string[] = []; + try { + const revisionId = services.currentConfigurationGeneration(infraDir); + const recorded = + revisionId !== null && + services.readGenerationServiceFiles( + join(infraDir, services.CONFIGURATION_GENERATIONS, revisionId), + ) !== null; + if (revisionId !== null && !recorded) { + lines.push( + "The applied configuration was rendered by an earlier release, whose services mount their files from the checkout. The next render mounts this release's files instead.", + ); + } + if (!services.serviceFilesImported(infraDir)) { + const position = services.checkoutPositionFromHistory(resolve(infraDir, "..", ".."), release); + lines.push( + position.atOrBehindRelease + ? "The next render imports, once, your edits in the checkout's service files as overrides in infra/docker/overrides/services." + : services.describeDeferredServiceFileImport(position.reason, []), + ); + } + const overrides = services.readServiceOverrides(infraDir, release); + lines.push( + ...services.describeServiceFiles({ + infraDir, + services: [], + release, + overrides, + stale: services.staleOverrides(infraDir, overrides, { record: false }), + }), + ); + return lines; + } catch (error) { + return [...lines, (error as Error).message]; + } +} diff --git a/apps/ops-agent/src/trusted-config-runtime.test.ts b/apps/ops-agent/src/trusted-config-runtime.test.ts index 11a79510b..531b4d192 100644 --- a/apps/ops-agent/src/trusted-config-runtime.test.ts +++ b/apps/ops-agent/src/trusted-config-runtime.test.ts @@ -1061,4 +1061,79 @@ describe("ops-agent trusted configuration runtime", () => { ); }); }); + + it("does not commit with an override it may not use, and says why for the admin panel", async () => { + const paths = fixture(); + // A built-in that mounts a file of the release. + const directory = join(paths.root, "services", "alpha"); + mkdirSync(join(directory, "config"), { recursive: true }); + writeFileSync(join(directory, "config", "alpha.yml"), "release: true\n"); + const alpha: services.LoadedService = { + ...service("alpha"), + manifest: { + ...service("alpha").manifest, + bindMounts: [{ source: "config/alpha.yml", target: "/etc/alpha.yml" }], + }, + directory, + }; + const override = join(paths.infraDir, "overrides", "services", "alpha", "config", "alpha.yml"); + mkdirSync(dirname(override), { recursive: true, mode: 0o755 }); + writeFileSync(override, "users: ['op:$2y$05$secret-hash']\n"); + chmodSync(override, 0o664); + const runtime = createUnavailableRuntime(); + installTrustedConfigurationRuntime(runtime, { + services: [alpha], + integrationSchemas: new Map([["routing", { enabled: { type: "boolean" } }]]), + infraDir: paths.infraDir, + }); + const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + const apply = (revisionId: string) => { + const operation = { + expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), + kind: "stack.render" as const, + revisionId, + }; + return dispatchOpsOperation(runtime, operation, { + signal: new AbortController().signal, + emitLog: vi.fn(), + claim: claim(operation), + }); + }; + const failure = (attemptedRevisionId: string) => { + const operation = { kind: "configuration.failure.inspect" as const, attemptedRevisionId }; + return dispatchOpsOperation(runtime, operation, { + signal: new AbortController().signal, + emitLog: vi.fn(), + claim: { + fingerprint: opsOperationFingerprint(operation), + operation, + source: "registry", + capability: { revisionId: attemptedRevisionId, values: {} }, + }, + }); + }; + try { + const refused = `cfg1_${"r".repeat(43)}`; + await expect(apply(refused)).resolves.toMatchObject({ + outcome: "not_committed", + reason: "failed", + }); + const problem = + "infra/docker/overrides/services/alpha/config/alpha.yml can't be used: group or others may write it. Render refuses it rather than use the release's file in its place: make it a regular file of yours that only you may write, or delete it."; + await expect(failure(refused)).resolves.toEqual({ attemptedRevisionId: refused, problem }); + expect(stderr.mock.calls.flat().map(String)).toContainEqual(`${problem}\n`); + expect(stderr.mock.calls.flat().join("")).not.toContain("secret-hash"); + expect(services.currentConfigurationGeneration(paths.infraDir)).toBeNull(); + + chmodSync(override, 0o644); + const applied = `cfg1_${"a".repeat(43)}`; + await expect(apply(applied)).resolves.toMatchObject({ outcome: "committed" }); + await expect(failure(applied)).resolves.toEqual({ + attemptedRevisionId: applied, + problem: null, + }); + } finally { + stderr.mockRestore(); + } + }); }); diff --git a/apps/ops-agent/src/trusted-config-runtime.ts b/apps/ops-agent/src/trusted-config-runtime.ts index 26dd7566e..67e4446a4 100644 --- a/apps/ops-agent/src/trusted-config-runtime.ts +++ b/apps/ops-agent/src/trusted-config-runtime.ts @@ -66,12 +66,15 @@ export async function initializeTrustedConfigurationRuntime(infraDir: string): P } } +const MAX_RECORDED_FAILURES = 32; + async function commitGeneration( options: TrustedConfigurationRuntimeOptions, operation: TrustedConfigurationOperation, payload: TrustedConfigurationPayload, claim: OpsTrustedClaim, signal: AbortSignal, + failures: Map, ): Promise { try { return await coreServices.withConfigurationGenerationWriter( @@ -138,7 +141,7 @@ async function commitGeneration( const legacyTraefik = await options.legacyTraefikDynamicMounted?.(); const mounts = await options.containerBindMounts?.(); const committed = await writer.commit({ - containerBindSources: coreServices.bindSources(mounts), + ...(mounts ? { containerBindMounts: mounts } : {}), services, integrationSchemas: authority?.integrationSchemas ?? options.integrationSchemas, input: { ...payload, selectedRoots }, @@ -151,6 +154,7 @@ async function commitGeneration( legacyTraefikDynamicMounted: legacyTraefik, }); for (const problem of committed.stackFileProblems) process.stderr.write(`${problem}\n`); + for (const line of committed.serviceFileReport) process.stderr.write(`${line}\n`); if (legacyTraefik === true) { process.stderr.write(`${coreServices.LEGACY_TRAEFIK_MOUNT_NOTICE}\n`); } @@ -165,11 +169,28 @@ async function commitGeneration( signal, ); } catch (error) { + const problem = operatorProblem(error); + if (problem) { + // The agent's log, and the admin panel through configuration.failure.inspect. + process.stderr.write(`${problem}\n`); + failures.set(operation.revisionId, problem); + while (failures.size > MAX_RECORDED_FAILURES) { + failures.delete(failures.keys().next().value as string); + } + } if (error instanceof coreServices.ConfigurationGenerationError) return error.result; return { outcome: "not_committed", reason: "failed", cleanupFailed: false }; } } +/** The message of a failure the operator can fix (a service file or override), or null. */ +function operatorProblem(error: unknown): string | null { + for (let cause = error; cause instanceof Error; cause = cause.cause) { + if (cause instanceof coreServices.ServiceFileError) return cause.message; + } + return null; +} + function trustedPayload( operation: TrustedConfigurationOperation, context: Parameters[1], @@ -191,6 +212,12 @@ export function installTrustedConfigurationRuntime( runtime: OpsRuntime, options: TrustedConfigurationRuntimeOptions, ): void { + // Why recent applies were not committed, when the operator can act on it. + const failures = new Map(); + runtime["configuration.failure.inspect"] = async (operation) => ({ + attemptedRevisionId: operation.attemptedRevisionId, + problem: failures.get(operation.attemptedRevisionId)?.slice(0, 4_096) ?? null, + }); runtime["configuration.inspect"] = (_operation, context) => coreServices.withConfigurationGenerationWriter( options.infraDir, @@ -204,6 +231,7 @@ export function installTrustedConfigurationRuntime( trustedPayload(operation, context), context.claim, context.signal, + failures, ); // The plan lives in the applied generation, which only this service can // read. Data-manager applies it: its mounts confine links and prunes to the @@ -239,12 +267,13 @@ export function installTrustedConfigurationRuntime( trustedPayload(operation, context), context.claim, context.signal, + failures, ); runtime["serviceConfig.apply"] = async (operation, context) => { const payload = trustedPayload(operation, context); if (!payload.serviceConfigs.some((entry) => entry.serviceId === operation.serviceId)) throw new Error(FAILED); - return commitGeneration(options, operation, payload, context.claim, context.signal); + return commitGeneration(options, operation, payload, context.claim, context.signal, failures); }; runtime["integrationConfig.apply"] = async (operation, context) => { const payload = trustedPayload(operation, context); @@ -253,13 +282,13 @@ export function installTrustedConfigurationRuntime( ) { throw new Error(FAILED); } - return commitGeneration(options, operation, payload, context.claim, context.signal); + return commitGeneration(options, operation, payload, context.claim, context.signal, failures); }; runtime["vault.apply"] = async (operation, context) => { const payload = trustedPayload(operation, context); if (!payload.serviceSecrets.some((entry) => entry.serviceId === operation.serviceId)) throw new Error(FAILED); - return commitGeneration(options, operation, payload, context.claim, context.signal); + return commitGeneration(options, operation, payload, context.claim, context.signal, failures); }; runtime["serviceSelection.inspect"] = async () => { try { diff --git a/apps/web/src/lib/admin/serviceAdmin.test.tsx b/apps/web/src/lib/admin/serviceAdmin.test.tsx index 1d145edcc..e6455d65f 100644 --- a/apps/web/src/lib/admin/serviceAdmin.test.tsx +++ b/apps/web/src/lib/admin/serviceAdmin.test.tsx @@ -1,8 +1,13 @@ import { createApiClient } from "@openmapx/core"; +import * as sdk from "@openmapx/core/api-admin-sdk"; import { useQueries, useQuery } from "@tanstack/react-query"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { act, cleanup, createTestQueryClient, renderHookWithQuery, waitFor } from "@/test"; -import { invalidateServiceAdministration, serviceAdminQueries } from "./serviceAdmin"; +import { + invalidateServiceAdministration, + serviceAdminQueries, + serviceOrThrow, +} from "./serviceAdmin"; let apiUrl = "http://admin.test"; vi.mock("@/integration-api/runtime/EnvProvider", () => ({ useEnv: () => ({ apiUrl }) })); @@ -241,6 +246,16 @@ describe("service administration query acquisition", () => { qc.getQueryData(second.list().queryKey), ); }); + it("shows why a saved configuration was not applied, when the operator can fix it", async () => { + const problem = + "infra/docker/overrides/services/martin/config/martin.yaml can't be used: it is a symbolic link."; + vi.stubGlobal("fetch", () => Response.json({ error: problem }, { status: 409 })); + const client = createApiClient({ baseUrl: apiUrl, credentials: "include" }); + await expect( + serviceOrThrow(sdk.updateAdminServiceConfig("martin", { config: {} }, { client })), + ).rejects.toThrow(problem); + }); + it("forwards query disposal through body cancellation and rejects malformed wire data safely", async () => { let canceled = false; vi.stubGlobal( diff --git a/docs/docs/developer/service-manifest.md b/docs/docs/developer/service-manifest.md index 657897238..21a9705ac 100644 --- a/docs/docs/developer/service-manifest.md +++ b/docs/docs/developer/service-manifest.md @@ -395,12 +395,21 @@ for persistent state instead. ] ``` -| Field | Type | Notes | -| :--------- | :------ | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `source` | string | **Required.** See the first-party source kinds below. Community services cannot declare this field because all community bind mounts are rejected. | -| `target` | string | **Required.** Absolute container path; first-party services may use a Compose-variable reference. No `..`. | -| `readOnly` | boolean | Defaults to `true`. | -| `optional` | boolean | First-party only. When `true`, the mount is silently dropped at render time if the host source is absent. Used for operator-supplied secrets declared once but materialized only when the file is dropped in place. Not allowed with Compose-variable sources. | +| Field | Type | Notes | +| :------------- | :------ | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `source` | string | **Required.** See the first-party source kinds below. Community services cannot declare this field because all community bind mounts are rejected. | +| `target` | string | **Required.** Absolute container path; first-party services may use a Compose-variable reference. No `..`. | +| `readOnly` | boolean | Defaults to `true`. | +| `optional` | boolean | First-party only. When `true`, the mount is silently dropped at render time if the host source is absent. Used for operator-supplied secrets declared once but materialized only when the file is dropped in place. Not allowed with Compose-variable sources. | +| `writableCopy` | string | First-party only, on a writable (`readOnly: false`), non-optional mount of the service's own file that the container rewrites. An `@infra:data/` where render keeps a writable copy, seeded from the release's file or the operator's override and seeded again only when that seed changes. The container mounts the copy. The one-time overrides import never imports such a file. | + +A relative or `@service:` source names a file the release ships. The +ops-agent's bundle embeds it (`pnpm generate:service-files` lists every one, +with the history of its versions, and CI checks both). Render writes it, or the +operator's override, into the generation's content-addressed service-file tree +in `infra/docker/state/ops-agent/service-files`, which the container mounts +read-only (files 0444, scripts 0555, directories 0755). After adding or +changing such a file, run `pnpm generate:service-files` and commit the result. The `source` may be one of the forms below. These are validator capabilities, not permission for an application service to take host authority. Repository diff --git a/docs/docs/guides/map-tiles.md b/docs/docs/guides/map-tiles.md index 31ad62e6e..9836a48dd 100644 --- a/docs/docs/guides/map-tiles.md +++ b/docs/docs/guides/map-tiles.md @@ -239,7 +239,8 @@ against refreshed source data produces a new content-addressed identity. ## Martin overlays Martin serves PostGIS tile functions as dynamic vector tile endpoints. It -serves only the sources declared in `services/martin/config/martin.yaml`, +serves only the sources declared in the release's `services/martin/config/martin.yaml` +(or your [override](../install/managing-services.md#service-file-overrides) of it), the traffic-flow overlay's `segment_flow`, and logs in as `openmapx_tiles`, a role that can call the functions in the `tiles` schema and read no table. Set its password before enabling Martin; PostGIS applies it on diff --git a/docs/docs/guides/routing-engines.md b/docs/docs/guides/routing-engines.md index b36b640b5..f0523bed0 100644 --- a/docs/docs/guides/routing-engines.md +++ b/docs/docs/guides/routing-engines.md @@ -88,9 +88,13 @@ enabled in the manifest environment, independently of those two schema fields. `services/valhalla/config/valhalla.json` sets build concurrency to 8 and bounds request sizes. Current limits include 5,000 km/20 locations for driving, 500 km for cycling, 250 km for walking, and four isochrone contours up to -120 minutes. Change tracked engine configuration only as a deliberate custom -configuration and verify the generated result. Larger limits increase resource -exposure; they do not enlarge geographic data coverage. +120 minutes. Change it only as a deliberate custom configuration, in an +[override](../install/managing-services.md#service-file-overrides) of +`valhalla/config/valhalla.json`, and verify the generated result. Valhalla +rewrites its configuration on every start, so it runs from a writable copy, +`infra/docker/data/valhalla/config/valhalla.json`, which render seeds from the +release's file or your override when that changes. Larger limits increase +resource exposure; they do not enlarge geographic data coverage. ## OSRM: build the graph first diff --git a/docs/docs/guides/supporting-infrastructure.md b/docs/docs/guides/supporting-infrastructure.md index ae579eac5..8986befd0 100644 --- a/docs/docs/guides/supporting-infrastructure.md +++ b/docs/docs/guides/supporting-infrastructure.md @@ -164,12 +164,20 @@ only when it reaches the same service correctly. HTTP/3 also needs UDP 443. If certificates fail, inspect DNS, firewall rules, port conflicts, and proxy logs before retrying. Repeated ACME attempts can hit upstream limits. -A default deployment exposes routed pages publicly. For staging basic auth, -copy `services/traefik/config/dynamic/auth.yml.example` to `auth.yml`, replace -its example hash, and append `basic-auth@file` to the existing middleware list -in `services/traefik/config/traefik.yml`. Render the stack, then recreate -Traefik after changing static configuration; copying the middleware file alone -does not activate it. +A default deployment exposes routed pages publicly. For staging basic auth, add +two [overrides](../install/managing-services.md#service-file-overrides) in +`infra/docker/overrides/services/traefik/config/`: + +- `dynamic/auth.yml`: a copy of `services/traefik/config/dynamic/auth.yml.example` + with your own hash, mode 0600; +- `traefik.yml`: a copy of `services/traefik/config/traefik.yml` with + `basic-auth@file` appended to the existing middleware list. + +Create them, and their directories, under `umask 022`: render refuses overrides +that group or others may write. + +Render the stack, then recreate Traefik, which reads `traefik.yml` only when it +starts; the middleware file alone does not activate it. ### Where Traefik reads its configuration @@ -177,20 +185,18 @@ Traefik watches its dynamic directory and reloads a file as soon as it changes, so that directory is a stable one in the operations agent's state, `infra/docker/state/ops-agent/traefik/dynamic`. Rendering writes `generated-routes.yml` there from validated service exposure declarations, -beside copies of the YAML and TOML files of `services/traefik/config/dynamic`, -each replaced atomically. The static `traefik.yml`, which Traefik reads when it -starts, is copied into each configuration generation. Edit both in -`services/traefik/config` and render; do not edit the copies or the generated -routes. - -Render copies only regular files of the deployment's user with one link, from -directories of that user, and never through a symbolic link. It names any other -file in its output and in `pnpm openmapx check`, without its contents, and -keeps the copy an earlier render made of it; a file you delete is removed from -Traefik's directory by the next render. The directory -is owner-only (0700, files 0600, `auth.yml`'s password hashes included). The -Traefik image runs as root, which reads it through the capabilities Docker -grants by default. +beside the release's YAML and TOML files of `services/traefik/config/dynamic` +(`middlewares.yml`) and your overrides of that directory (`auth.yml`), each +replaced atomically. The static `traefik.yml`, which Traefik reads when it +starts, is copied into each configuration generation, from the release or your +override. Change either through +[overrides](../install/managing-services.md#service-file-overrides); do not +edit the copies or the generated routes. A file removed from your overrides is +removed from Traefik's directory by the next render. + +The directory is owner-only (0700, files 0600, `auth.yml`'s password hashes +included). The Traefik image runs as root, which reads it through the +capabilities Docker grants by default. A Traefik started before this layout reads `services/traefik/config/dynamic` until it is recreated. Render keeps writing the routes there while any diff --git a/docs/docs/guides/transit-engines.md b/docs/docs/guides/transit-engines.md index 79c437015..404416a1c 100644 --- a/docs/docs/guides/transit-engines.md +++ b/docs/docs/guides/transit-engines.md @@ -295,9 +295,13 @@ pnpm openmapx services logs otp --follow ``` Start re-renders and links automatically. The container uses `--load --serve` -and publishes `http://otp:8080` as host loopback `8090`. Build and routing config -files live under `services/otp/config/`. Rebuild for build-config or input changes; -recreate the runtime for container or router configuration changes. +and publishes `http://otp:8080` as host loopback `8090`. The release's build and +routing config files are `services/otp/config/`; change them with +[overrides](../install/managing-services.md#service-file-overrides) of +`otp/config/build-config.json` or `otp/config/router-config.json`. The graph +build reads the copy the applied configuration mounts, so render first. Rebuild +for build-config or input changes; recreate the runtime for container or router +configuration changes. After refreshing the PBF or GTFS files: diff --git a/docs/docs/install/getting-started.md b/docs/docs/install/getting-started.md index d3a3e9ee4..0edf2eb27 100644 --- a/docs/docs/install/getting-started.md +++ b/docs/docs/install/getting-started.md @@ -97,12 +97,14 @@ the defaults, so do not copy a file containing only optional engines. ## 4. Render the compose stack ```bash -pnpm openmapx compose render pnpm openmapx compose release pnpm openmapx compose release --status +pnpm openmapx compose render ``` -Render creates platform secret files and a configuration generation under +Select the release first: render takes the services' files from the selected +release, not from the checkout, and refuses while the checkout's built-in +services differ from that release's. Render creates platform secret files and a configuration generation under `infra/docker/state/ops-agent/configuration/generations/`. The `infra/docker/state/ops-agent/configuration/current` pointer selects the generated Compose file and hardlink plan. Keep these deployment files; do not edit generated YAML. diff --git a/docs/docs/install/managing-services.md b/docs/docs/install/managing-services.md index 9f8f9463c..dfd475592 100644 --- a/docs/docs/install/managing-services.md +++ b/docs/docs/install/managing-services.md @@ -401,6 +401,102 @@ forwarded by `services/data-manager/service.json`; placing an arbitrary variable in `.env` does not pass it to that container. ::: +## Changing a service's files {/* #service-file-overrides */} + +Services mount configuration files and scripts that the release ships: Martin's +`martin.yaml`, OTP's `build-config.json`, Traefik's `traefik.yml` and +middlewares, PostGIS's entrypoint, and others. They come from the release the +stack runs, not from the checkout. Render copies them into +`infra/docker/state/ops-agent/service-files///` with each +configuration generation, and the containers mount those copies read-only. Editing a +file under `services/` in the checkout changes nothing. + +To change one, put the whole file in an override, at the same path below +`infra/docker/overrides/services//`: + +```bash +(umask 022 + mkdir -p infra/docker/overrides/services/martin/config + cp services/martin/config/martin.yaml infra/docker/overrides/services/martin/config/) +"$EDITOR" infra/docker/overrides/services/martin/config/martin.yaml +pnpm openmapx compose render +pnpm openmapx services update martin +``` + +The `umask 022` matters on Debian and Ubuntu, whose default umask of `002` +makes new files and directories group-writable, and render refuses those. To +fix existing ones: `chmod -R g-w,o-w infra/docker/overrides`. + +Render uses the override in place of the release's file. Because the copy's +directory is named after what it holds, `services update` (or **Recreate** in +the admin panel) gives the service its new file. Delete the override to go +back to the release's file. + +- An override is a whole file, owned by the deployment's user, with one link, + that neither its group nor others may write. Its directories must be the + same, and none of them may be a symbolic link. A directory at a file's path + (as `mkdir -p` of the file's own path leaves) is not an override of it. +- If an override replaces a file the release ships but breaks these rules, render + fails and names the file. Using the release's file instead could silently + drop what the override changes, a basic-auth gate for example. A + configuration saved in the admin panel then fails with the same reason, and + is not applied. +- An override of a path the release doesn't ship is reported and ignored. + The one exception is Traefik's dynamic directory. A YAML or TOML file there, + such as `traefik/config/dynamic/auth.yml`, is added to Traefik's + configuration. `generated-routes.yml` is render's own. +- Render's output, the operations agent's log, `pnpm openmapx check` and + **Run diagnostics** list the overrides in use, by name only. + +**When a release changes a file you override.** The agent's state records, for +each override, the release file it replaced. When a later release changes that +file, render, `check` and Run diagnostics say so, and the override keeps +replacing it. Compare your override with the new release file, carry over what +you need, and save the override again. **Editing the override is how you +acknowledge the release's change**: render then records the current release +file as reviewed. If the override needs no change, any change to its bytes +acknowledges it, a comment or a blank line at the end for example. + +**Files a container rewrites.** Valhalla's `configure_valhalla.sh` rewrites +`valhalla.json` on every start. Such a file (`writableCopy` in its manifest) +gets a writable copy, here `infra/docker/data/valhalla/config/valhalla.json`. +Render seeds the copy from the release's file or your override, and seeds it +again only when that seed changes. In between, Valhalla's own rewrite stays. +A missing copy is seeded before the configuration is applied; if that fails, +render fails with the reason, since Docker would otherwise create a directory +in its place and Valhalla could not start. +Override `services/valhalla/config/valhalla.json` to change what it starts +from; editing the copy lasts only until the next seed. + +**Upgrading from a release before this.** The first render with the release's +files imports, once, every service file in the checkout that differs from +every version a release shipped, as an override. This includes an `auth.yml` +and a changed `traefik.yml`. Files a container writes are never imported, and +nothing in the checkout is changed. The render lists what it imported. A +checkout service file you edit afterwards is reported as unused; move it into +`infra/docker/overrides` to keep it. + +The import runs only from a checkout at or behind the selected release's +source revision. A checkout ahead of it can hold newer upstream files, which +also match no released version. Until then the import waits: render imports +and records nothing, says why on every render, in `check` and in Run +diagnostics, lists the checkout files it does not use, and uses the release's. +Traefik is the exception: while the import waits, render refuses if the +checkout holds a Traefik file the import would carry over, an `auth.yml` or +another dynamic YAML or TOML file the release doesn't ship, or a changed +`traefik.yml`. Using the release's files instead could drop a basic-auth gate. +The refusal names the files. Put each in +`infra/docker/overrides/services/traefik/` at the same path, under `umask 022` +(`auth.yml` with mode 0600), and render goes on: an override ends the wait for +that file. +The host CLI asks git; the operations agent goes by the checkout's +`services/service-file-history.json`, which each commit keeps current. A +checkout without that file predates it and is taken to be behind. + +Under `OPENMAPX_RELEASE=local`, for development, the checkout's files are the +release's, for the host CLI and the operations agent alike. Overrides still +apply, and nothing is imported. + ## Exposure: what's reachable from outside By default every service is reachable **only** on the private `openmapx` Docker diff --git a/docs/docs/install/upgrading.md b/docs/docs/install/upgrading.md index 6e6b5949c..01ca27d19 100644 --- a/docs/docs/install/upgrading.md +++ b/docs/docs/install/upgrading.md @@ -99,19 +99,22 @@ pnpm install --frozen-lockfile ``` The ops-agent runs the built-in service and integration manifests of its own -release, not the checkout's, and checks no file permissions in it. The checkout -must still be writable only by the operator's account: services mount -configuration files from it, `services build` uses its build contexts, and -`infra/docker/.env` and the release overlay live in it. A private group per user, the Debian and +release, and the files those services mount, not the checkout's, and checks no +file permissions in it. The checkout must still be writable only by the +operator's account: `services build` uses its build contexts, and +`infra/docker/.env` and the agent's state live in it. A private group per user, the Debian and Ubuntu default, meets this even with their umask of `002`. Where the account's group has other members, clone and pull with umask `022`. -For the same reason, keep the checkout at the selected release. The ops-agent's -startup log -(`pnpm openmapx services logs ops-agent`) and **Run diagnostics** under -Maintenance in the admin panel say when the checkout's built-in services differ -from the release's. The host `pnpm openmapx check` cannot: it runs the -checkout's own code and manifests. +Keep the checkout at the selected release. The host CLI runs the checkout's +own code and manifests, so a host render reads the selected release's service +files from its operations-agent image and refuses while the checkout's built-in +services differ from that release's. It names the differing files and says +what to do: wait until the release of the checkout's commit is published and +run `pnpm openmapx compose release` again, or check out the release's source +revision. A changed service file belongs in an +[override](./managing-services.md#service-file-overrides), which the refusal +ignores. `pnpm openmapx check` reports the same. Releases before the operations agent kept the Transitous pin in its own state wrote an approved pin into the tracked `infra/docker/transitous.lock.json`. If @@ -474,7 +477,8 @@ Without the password, rendering fails with `MARTIN_DB_PASSWORD must be set`. Traefik now reads its routes and middlewares from `infra/docker/state/ops-agent/traefik/dynamic`, and its `traefik.yml` from the -configuration generation, both written by render from `services/traefik/config`. +configuration generation, both written by render from the release's files and +your overrides. A Traefik started by an earlier release keeps reading `services/traefik/config/dynamic` until it is recreated, and render keeps writing the routes there too while any container still mounts it, so routing @@ -492,6 +496,54 @@ it for you. Once no container mounts the checkout's directory, render stops writing there and leaves the last `generated-routes.yml` in place for a rollback. See [where Traefik reads its configuration](../guides/supporting-infrastructure.md#where-traefik-reads-its-configuration). +## Service files come from the release + +Services mount the configuration files and scripts of the release the stack +runs, copied into `infra/docker/state/ops-agent/service-files`, no longer the +checkout's `services/`. Changes of your own go in +[overrides](./managing-services.md#service-file-overrides) in +`infra/docker/overrides/services`. + +- **Your edits are carried over.** The first render of this release imports, + once, every checkout service file that differs from every version a release + shipped, as an override, and lists them. This includes Traefik's `auth.yml` + and a `traefik.yml` naming `basic-auth@file`, so a basic-auth gate stays. + Files a container writes (Valhalla's `valhalla.json`) are never imported. A + checkout an earlier release left, after panel updates, imports nothing. +- **Only from a checkout at or behind the release.** A checkout ahead of the + selected release's source revision can hold newer upstream files, which + match no version the release shipped either. There the import waits: render + imports and records nothing, says why each time, and uses the release's + files meanwhile, so an edit of yours is not applied until the import runs. + Traefik files are the exception: a checkout `auth.yml` (or another dynamic + file the release doesn't ship) or a changed `traefik.yml` makes render + refuse, by name, until you put it in + `infra/docker/overrides/services/traefik/`, so a basic-auth gate can't drop. + A host render asks git and refuses meanwhile. The operations agent, which + has no git, goes by the checkout's `services/service-file-history.json`. To + apply an edit sooner, put it in an override yourself. +- **Valhalla** gets its writable copy in + `infra/docker/data/valhalla/config/valhalla.json`, seeded from the release's + file. Its own rewrite of the checkout's copy is left behind, unused. If the + copy is missing and can't be written, render fails and says why, rather than + let Docker create a directory in its place. +- **Containers recreate when their mounts move.** Until a service is recreated, + its container keeps mounting the checkout's files, which still hold the same + bytes. Render, `check` and Run diagnostics list such containers. The + documented host update, `services update` with a list of services, recreates + only the services listed. The first stack-wide `pnpm openmapx compose up` + recreates every service whose mounts moved, PostGIS and Traefik included: + run it when a restart of the database and the proxy suits you. A panel update + recreates only the application services, which mount no service files. +- **A host render refuses while the checkout differs from the selected + release** (see above). On a host that follows `main`, the newest commit's + release may not be published yet: wait for it and run `compose release` + again. + +A rollback to the release before this one mounts the checkout's files again, +which nothing here changed. Edits you made only in `overrides/` meanwhile do +not apply there. + ## Reclaiming disk after an upgrade Keep prior image digests and the matching database/configuration backup until diff --git a/packages/cli/__tests__/cli-check-command.test.ts b/packages/cli/__tests__/cli-check-command.test.ts index c19b13186..f17414b47 100644 --- a/packages/cli/__tests__/cli-check-command.test.ts +++ b/packages/cli/__tests__/cli-check-command.test.ts @@ -1,12 +1,14 @@ -import { chmodSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { services } from "@openmapx/core/server"; import { Command } from "commander"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { registerCheckCommand } from "../src/commands/check"; -import { legacyTraefikDynamicMounted } from "../src/lib/container-mounts"; +import { checkoutPositionFromGit } from "../src/lib/checkout-position"; +import { containerBindMounts, legacyTraefikDynamicMounted } from "../src/lib/container-mounts"; import { dockerCompose } from "../src/lib/docker"; +import { selectedReleaseServiceFiles } from "../src/lib/release-service-files"; import { ensureTransitousApiKeysFile, legacyTransitousApiKeysPath, @@ -22,6 +24,14 @@ vi.mock("../src/lib/container-mounts", () => ({ legacyTraefikDynamicMounted: vi.fn(async () => false), containerBindMounts: vi.fn(async () => []), })); +// No release is selected unless a test selects one. +vi.mock("../src/lib/release-service-files", () => ({ + selectedReleaseServiceFiles: vi.fn(async () => "none"), +})); +// Git says the checkout is at or behind the release unless a test says otherwise. +vi.mock("../src/lib/checkout-position", () => ({ + checkoutPositionFromGit: vi.fn(async () => ({ atOrBehindRelease: true })), +})); let tmp: string; @@ -130,3 +140,104 @@ it("says while Traefik still reads the checkout's routes", async () => { await check(); expect(traefikLines()).toEqual([]); }); + +it("reports the service files: overrides, the import to come, a host render's refusal, containers on the checkout", async () => { + const directory = join(tmp, "services", "martin"); + mkdirSync(join(directory, "config"), { recursive: true }); + const manifest = { + id: "martin", + name: "martin", + version: "1.0.0", + quality: "built-in", + container: { image: "example/martin", tag: "1" }, + bindMounts: [{ source: "config/martin.yaml", target: "/etc/martin/martin.yaml" }], + }; + writeFileSync(join(directory, "service.json"), JSON.stringify(manifest)); + writeFileSync(join(directory, "config", "martin.yaml"), "listen_addresses: 0.0.0.0:3000\n"); + const override = join(tmp, "infra", "docker", "overrides", "services", "martin", "config"); + mkdirSync(override, { recursive: true, mode: 0o755 }); + writeFileSync(join(override, "martin.yaml"), "users: ['op:$2y$05$secret-hash']\n"); + // Whatever the umask: overrides group or others may write are refused. + chmodSync(join(override, "martin.yaml"), 0o644); + const checkout = services.releaseServiceFiles([ + { manifest, directory, isBuiltIn: true, enabled: true } as services.LoadedService, + ]); + vi.mocked(selectedReleaseServiceFiles).mockResolvedValue({ + files: { ...checkout, origin: "release", manifests: new Map([["martin", "0".repeat(64)]]) }, + release: "r1", + image: "ghcr.io/openmapx/ops-agent@sha256:1", + revision: "a".repeat(40), + }); + vi.mocked(containerBindMounts).mockResolvedValue([ + { name: "docker-martin-1", sources: [join(directory, "config", "martin.yaml")] }, + ]); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const check = async () => { + const program = new Command(); + registerCheckCommand(program); + await expect(program.parseAsync(["check"], { from: "user" })).rejects.toThrow("exit:1"); + }; + const lines = () => errors.mock.calls.flat().map(String); + try { + // Ahead of the release, git says: the import waits, and a host render + // would refuse the differing file. + vi.mocked(checkoutPositionFromGit).mockResolvedValueOnce({ + atOrBehindRelease: false, + reason: "the checkout's commit is neither the release's source revision r nor one before it", + }); + writeFileSync(join(directory, "config", "martin.yaml"), "listen_addresses: 0.0.0.0:3001\n"); + rmSync(join(override, "martin.yaml")); + await check(); + expect(lines()).toContainEqual( + expect.stringContaining( + "The one-time import of your edits in the checkout's service files waits: the checkout's commit is neither", + ), + ); + expect(lines()).toContainEqual( + expect.stringContaining( + "Your edits in the checkout's service files are not imported as overrides meanwhile", + ), + ); + expect(lines()).toContainEqual( + expect.stringContaining("services/martin/config/martin.yaml (changed)"), + ); + writeFileSync(join(directory, "config", "martin.yaml"), "listen_addresses: 0.0.0.0:3000\n"); + writeFileSync(join(override, "martin.yaml"), "users: ['op:$2y$05$secret-hash']\n"); + chmodSync(join(override, "martin.yaml"), 0o644); + errors.mockClear(); + + await check(); + expect(lines()).toContainEqual(expect.stringContaining("The next render imports, once")); + expect(lines()).toContainEqual( + expect.stringContaining( + "Service file overrides in use, in place of the release's files (infra/docker/overrides/services): martin/config/martin.yaml.", + ), + ); + expect(lines()).toContainEqual( + expect.stringContaining( + "1 container still mounts service files from the checkout: docker-martin-1.", + ), + ); + expect(lines().join("\n")).not.toContain("secret-hash"); + + // Once the import ran: what a host render would refuse. + services.importCheckoutServiceFiles({ + infraDir: join(tmp, "infra", "docker"), + services: [], + release: { ...checkout, origin: "release" }, + }); + errors.mockClear(); + await check(); + expect(lines()).toContainEqual( + expect.stringContaining( + "A host render refuses until this is resolved. The checkout's built-in services differ from the selected release r1", + ), + ); + expect(lines()).toContainEqual( + expect.stringContaining("services/martin/service.json (changed)"), + ); + } finally { + vi.mocked(selectedReleaseServiceFiles).mockResolvedValue("none"); + vi.mocked(containerBindMounts).mockResolvedValue([]); + } +}); diff --git a/packages/cli/__tests__/cli-check.test.ts b/packages/cli/__tests__/cli-check.test.ts index 24381bd90..fe38fb864 100644 --- a/packages/cli/__tests__/cli-check.test.ts +++ b/packages/cli/__tests__/cli-check.test.ts @@ -350,6 +350,47 @@ describe("deploymentUserProblems", () => { } }); + it("leaves out Valhalla's writable copy of valhalla.json, which it rewrites as its image's root", () => { + const { root, infraDir, owner } = deployment(""); + try { + writeFileSync(join(infraDir, ".env"), `UID=${owner.uid}\nGID=${owner.gid}\n`, { + mode: 0o600, + }); + const data = join(infraDir, "data"); + const at = (...parts: string[]) => join(data, ...parts); + const user = `${owner.uid}:${owner.gid}`; + const copy = at("valhalla", "config", "valhalla.json"); + const problems = deploymentUserProblems({ + infraDir, + processUid: owner.uid, + composeServices: { + "data-manager": { user, volumes: [{ type: "bind", source: data }] }, + // As render writes it: the copy, read-write, into a service without a user. + valhalla: { volumes: [{ type: "bind", source: copy }] }, + }, + runningUsers: new Map([["data-manager", user]]), + filesystem: fakeFilesystem( + { + [data]: owner.uid, + [at("valhalla")]: owner.uid, + // Render creates the directory as the deployment's user. + [at("valhalla", "config")]: owner.uid, + // configure_valhalla.sh rewrote the file as root. + [copy]: 0, + }, + { + [data]: ["valhalla"], + [at("valhalla")]: ["config"], + [at("valhalla", "config")]: ["valhalla.json"], + }, + ), + }); + expect(problems).toEqual([]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it("leaves out what services running as their image's user write, as on openmapx.com", () => { const { root, infraDir, owner } = deployment(""); try { diff --git a/packages/cli/__tests__/cli-compose.test.ts b/packages/cli/__tests__/cli-compose.test.ts index 79fce8427..6c5f549b4 100644 --- a/packages/cli/__tests__/cli-compose.test.ts +++ b/packages/cli/__tests__/cli-compose.test.ts @@ -1,3 +1,4 @@ +import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import { chmodSync, @@ -27,6 +28,7 @@ import { renderComposeForRepo, rotateRedisPasswordForRepo } from "../src/command import { legacyTraefikDynamicMounted } from "../src/lib/container-mounts"; import { loadInfraEnv } from "../src/lib/infra-env"; import { rotatePlatformSecretFile } from "../src/lib/platform-secret-files"; +import { selectedReleaseServiceFiles } from "../src/lib/release-service-files"; // Docker answers whether a container still mounts the checkout's Traefik // directory, and what containers mount; here, by default, none does. @@ -34,6 +36,10 @@ vi.mock("../src/lib/container-mounts", () => ({ legacyTraefikDynamicMounted: vi.fn(async () => false), containerBindMounts: vi.fn(async () => []), })); +// The selected release's files come from its image; by default none is selected. +vi.mock("../src/lib/release-service-files", () => ({ + selectedReleaseServiceFiles: vi.fn(async () => "none"), +})); /** A file's permission bits, read through a descriptor rather than a second lookup by path. */ function permissions(path: string): number { @@ -523,6 +529,199 @@ describe("renderComposeForRepo", () => { expect(notices()).toEqual([]); }); + /** Martin in a git checkout, and a release of it as its operations-agent image prints it. */ + function martinCheckout() { + const manifest = { + ...baseManifest, + id: "martin", + bindMounts: [{ source: "config/martin.yaml", target: "/etc/martin/martin.yaml" }], + }; + writeManifest("martin", manifest); + const config = join(tmp, "services", "martin", "config"); + mkdirSync(config, { recursive: true }); + const shipped = "listen_addresses: 0.0.0.0:3000\n"; + writeFileSync(join(config, "martin.yaml"), shipped); + const git = (...args: string[]) => + execFileSync( + "git", + [ + "-C", + tmp, + "-c", + "user.name=OpenMapX", + "-c", + "user.email=ops@example.test", + "-c", + "commit.gpgsign=false", + ...args, + ], + { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }, + ).trim(); + /** Commits the checkout's services; returns the commit. */ + const commit = (message: string) => { + git("add", "services"); + git("commit", "-q", "-m", message); + return git("rev-parse", "HEAD"); + }; + git("init", "-q"); + const sha = (text: string) => createHash("sha256").update(text).digest("hex"); + /** Selects the release built from `revision`, whose martin.yaml is `yaml`. */ + const select = (revision: string | null, yaml: string, earlier: string[] = []) => + vi.mocked(selectedReleaseServiceFiles).mockResolvedValue({ + files: { + origin: "release", + files: new Map([ + [ + "martin/config/martin.yaml", + { + serviceId: "martin", + path: "config/martin.yaml", + executable: false, + bytes: Buffer.from(yaml), + sha256: sha(yaml), + }, + ], + ]), + history: new Map([ + ["martin/config/martin.yaml", new Set([sha(yaml), ...earlier.map(sha)])], + ]), + manifests: new Map([["martin", sha(JSON.stringify(manifest))]]), + missing: [], + }, + release: "r1", + image: "ghcr.io/openmapx/ops-agent@sha256:1", + revision, + }); + const render = () => + renderComposeForRepo({ rootDir: tmp, domain: "example.com", services: ["martin"] }); + const mountedFile = () => { + const line = readFileSync(generatedCompose(), "utf8") + .split("\n") + .find((entry) => entry.includes(":/etc/martin/martin.yaml")); + return readFileSync( + join(dirname(generatedCompose()), line?.trim().replace(/^- /, "").split(":")[0] ?? ""), + "utf8", + ); + }; + const overrides = join(tmp, "infra", "docker", "overrides"); + const marker = join(tmp, "infra", "docker", "state", "ops-agent", "overrides-import.json"); + return { + manifest, + config, + shipped, + git, + commit, + select, + render, + mountedFile, + overrides, + marker, + }; + } + + it("renders a selected release's files, keeps checkout edits as overrides, and refuses while its manifests differ", async () => { + const f = martinCheckout(); + try { + // A checkout at the release: the release's file, as the store holds it. + const first = f.commit("the release"); + f.select(first, f.shipped); + await f.render(); + expect(f.mountedFile()).toBe(f.shipped); + + // One a later release left behind: refused, and nothing is imported from it. + const newer = "listen_addresses: 0.0.0.0:3000\nworker_processes: 2\n"; + writeFileSync(join(f.config, "martin.yaml"), newer); + const later = f.commit("the next release"); + f.git("checkout", "-q", first); + rmSync(f.marker); + f.select(later, newer, [f.shipped]); + await expect(f.render()).rejects.toThrow("services/martin/config/martin.yaml (changed)"); + expect(existsSync(f.overrides)).toBe(false); + // Refused before the import, which is still to run. + expect(existsSync(f.marker)).toBe(false); + + // An operator's edit in a checkout behind the release: imported once as + // an override, then used. + writeFileSync(join(f.config, "martin.yaml"), "listen_addresses: 0.0.0.0:3001\n"); + await f.render(); + expect( + readFileSync(join(f.overrides, "services", "martin", "config", "martin.yaml"), "utf8"), + ).toBe("listen_addresses: 0.0.0.0:3001\n"); + expect(f.mountedFile()).toBe("listen_addresses: 0.0.0.0:3001\n"); + expect(existsSync(f.marker)).toBe(true); + + // A checkout whose manifest is not the release's: refused, with what to do. + writeManifest("martin", { ...f.manifest, version: "2.0.0" }); + const refused = await f.render().catch((error: Error) => error.message); + expect(refused).toContain( + `The checkout's built-in services differ from the selected release r1, built from ${later}:\n services/martin/service.json (changed)`, + ); + expect(refused).toContain( + `Either wait until the release of this checkout's commit is published and run \`pnpm openmapx compose release\` again, or check out the selected release's source: git checkout ${later}`, + ); + expect(refused).toContain("infra/docker/overrides/services//"); + } finally { + vi.mocked(selectedReleaseServiceFiles).mockResolvedValue("none"); + } + }); + + it("imports nothing from a checkout one commit ahead of the release with a changed upstream file, and refuses", async () => { + const f = martinCheckout(); + try { + const released = f.commit("the release"); + // Upstream's next commit changes martin.yaml; its release isn't published yet. + writeFileSync( + join(f.config, "martin.yaml"), + "listen_addresses: 0.0.0.0:3000\nworker_processes: 8\n", + ); + f.commit("upstream, ahead of the release"); + f.select(released, f.shipped); + + const refused = await f.render().catch((error: Error) => error.message); + expect(refused).toContain(" services/martin/config/martin.yaml (changed)"); + expect(refused).toContain( + `Your edits in the checkout's service files are not imported as overrides meanwhile: the checkout's commit is neither the release's source revision ${released} nor one before it.`, + ); + expect(existsSync(f.overrides)).toBe(false); + expect(existsSync(f.marker)).toBe(false); + + // A basic-auth file the waiting import would carry over: refused by name, + // until it is an override; then the rest of the refusal stands. + const dynamic = join(tmp, "services", "traefik", "config", "dynamic"); + mkdirSync(dynamic, { recursive: true }); + writeFileSync(join(dynamic, "auth.yml"), "http: {}\n"); + await expect(f.render()).rejects.toThrow( + "the checkout holds Traefik files the import would carry over, and the release's files in their place could drop a basic-auth gate: services/traefik/config/dynamic/auth.yml.", + ); + const overridden = join(f.overrides, "services", "traefik", "config", "dynamic"); + mkdirSync(overridden, { recursive: true, mode: 0o755 }); + for (const directory of [f.overrides, join(f.overrides, "services")]) { + chmodSync(directory, 0o755); + } + for (const directory of ["traefik", "traefik/config", "traefik/config/dynamic"]) { + chmodSync(join(f.overrides, "services", directory), 0o755); + } + writeFileSync(join(overridden, "auth.yml"), "http: {}\n", { mode: 0o600 }); + chmodSync(join(overridden, "auth.yml"), 0o600); + const rest = await f.render().catch((error: Error) => error.message); + expect(rest).toContain(" services/martin/config/martin.yaml (changed)"); + expect(rest).not.toContain("auth.yml"); + expect(existsSync(f.marker)).toBe(false); + rmSync(f.overrides, { recursive: true }); + rmSync(join(dynamic, "auth.yml")); + + // Without a revision to compare with, the import waits too. + f.select(null, f.shipped); + await expect(f.render()).rejects.toThrow( + "not imported as overrides meanwhile: the selected release's operations-agent image names no source revision", + ); + expect(existsSync(f.overrides)).toBe(false); + expect(existsSync(f.marker)).toBe(false); + } finally { + vi.mocked(selectedReleaseServiceFiles).mockResolvedValue("none"); + } + }); + it("rejects pre-existing identical ops-agent role tokens", async () => { writeManifest("alpha", { ...baseManifest, id: "alpha" }); const secretDir = join(tmp, "infra", "docker", "secrets"); diff --git a/packages/cli/src/commands/check.ts b/packages/cli/src/commands/check.ts index 02793e979..7ab2eb4e7 100644 --- a/packages/cli/src/commands/check.ts +++ b/packages/cli/src/commands/check.ts @@ -8,6 +8,7 @@ import { assertCliDeploymentSecrets } from "../lib/deployment-secret-policy"; import { dockerCompose, dockerRun } from "../lib/docker"; import { log, table } from "../lib/output"; import { repoPaths } from "../lib/paths"; +import { serviceFileReport } from "../lib/service-file-report"; import { transitousApiKeysLeftBehindWarning } from "../lib/transitous-api-keys"; interface PsLine { @@ -468,12 +469,9 @@ export function registerCheckCommand(program: Command): void { ? envFilePermissionWarning(envPath, envMode) : null; if (permissionWarning) log.warn(permissionWarning); - // Only the ops-agent's fixed CLI runs embedded manifests; from source the - // checkout is what runs, so there is nothing to compare. - const drift = coreServices.describeBuiltInServiceDrift( - coreServices.inspectBuiltInServiceDrift(repoPaths().root), - ); - if (drift) log.warn(drift); + // The service files: overrides, the import, checkout edits and + // containers still on them, and what a host render would refuse. + for (const { level, message } of await serviceFileReport()) log[level](message); // Integrations the operations agent installed are trusted by its records; // only its fixed CLI reads them, so a check from source has none to report. for (const warning of installedIntegrationWarnings(repoPaths().root)) log.warn(warning); diff --git a/packages/cli/src/commands/compose.ts b/packages/cli/src/commands/compose.ts index a307aa718..d31d1099a 100644 --- a/packages/cli/src/commands/compose.ts +++ b/packages/cli/src/commands/compose.ts @@ -3,6 +3,7 @@ import { join } from "node:path"; import { initializeErasureJournal } from "@openmapx/core/erasure-journal"; import { services as coreServices } from "@openmapx/core/server"; import { type Command, Option } from "commander"; +import { checkoutPositionFromGit } from "../lib/checkout-position"; import { containerBindMounts, legacyTraefikDynamicMounted } from "../lib/container-mounts"; import { assertCliDeploymentSecrets } from "../lib/deployment-secret-policy"; import { dockerComposeStream } from "../lib/docker"; @@ -35,6 +36,7 @@ import { touchesReleasePinnedServices, unpinnedReleaseWarning, } from "../lib/release"; +import { selectedReleaseServiceFiles } from "../lib/release-service-files"; import { applyServiceSelection } from "../lib/service-selection"; import { describeTransitousApiKeysMigration, @@ -334,9 +336,60 @@ export async function renderComposeForRepo(opts: RenderRepoOptions): Promise 0) { + throw new Error(coreServices.describeWaitingTraefikEdits(plan.reason, traefik)); + } + const importing = new Set( + plan?.kind === "import" ? plan.edits.map((edit) => `services/${edit.key}`) : [], + ); + const overrides = coreServices.readServiceOverrides(paths.infraDir, selected.files); + const differences = coreServices + .checkoutDifferencesFromRelease({ + rootDir: paths.root, + services, + release: selected.files, + overrides: new Map( + overrides.active.map((entry) => [`${entry.serviceId}/${entry.path}`, entry.bytes]), + ), + }) + .filter((difference) => !importing.has(difference.path)); + if (differences.length > 0) { + throw new Error( + coreServices.describeCheckoutDifferences(differences, { + name: selected.release, + revision: selected.revision, + importWaits: plan?.kind === "defer" ? plan.reason : null, + }), + ); + } + } const committed = await writer.commit({ legacyTraefikDynamicMounted: legacyTraefik, - containerBindSources: coreServices.bindSources(mounts), + ...(mounts ? { containerBindMounts: mounts } : {}), + ...(selected && selected !== "none" ? { releaseServiceFiles: selected.files } : {}), + ...(checkoutPosition ? { checkoutPosition } : {}), services, integrationSchemas, input: { @@ -351,6 +404,7 @@ export async function renderComposeForRepo(opts: RenderRepoOptions): Promise { + const files = coreServices.readEmbeddedReleaseServiceFiles(); + if (!files) { + log.err( + "Only a release's own CLI, in its operations-agent image, embeds its service files.", + ); + process.exit(1); + } + process.stdout.write(coreServices.serializeReleaseServiceFiles(files)); + }); + compose .command("rotate-redis-password") .description("Atomically rotate Redis authentication files while Redis clients are stopped") diff --git a/packages/cli/src/lib/checkout-position.ts b/packages/cli/src/lib/checkout-position.ts new file mode 100644 index 000000000..14215eb3e --- /dev/null +++ b/packages/cli/src/lib/checkout-position.ts @@ -0,0 +1,42 @@ +import type { services as coreServices } from "@openmapx/core/server"; +import { execa } from "execa"; + +/** + * Where the host's checkout stands against the selected release, by git: at + * or behind the release's source revision, or not (ahead, on another line, + * or unknown). Only at or behind may a checkout file that matches no version + * the release shipped be taken for the operator's edit. The operator's own + * command asks git; the agent never does. + */ +export async function checkoutPositionFromGit( + rootDir: string, + revision: string | null, +): Promise { + if (!revision) { + return { + atOrBehindRelease: false, + reason: + "the selected release's operations-agent image names no source revision to compare the checkout with", + }; + } + const run = await execa("git", ["-C", rootDir, "merge-base", "--is-ancestor", "HEAD", revision], { + reject: false, + timeout: 30_000, + }); + if (run.exitCode === 0) return { atOrBehindRelease: true }; + if (run.exitCode === 1) { + return { + atOrBehindRelease: false, + reason: `the checkout's commit is neither the release's source revision ${revision} nor one before it`, + }; + } + const detail = + String(run.stderr ?? "") + .trim() + .split("\n") + .slice(-1)[0] || `exit ${run.exitCode ?? "unknown"}`; + return { + atOrBehindRelease: false, + reason: `git could not compare the checkout with the release's source revision ${revision} (${detail}); \`git fetch origin\` brings that revision if it is missing`, + }; +} diff --git a/packages/cli/src/lib/release-service-files.ts b/packages/cli/src/lib/release-service-files.ts new file mode 100644 index 000000000..541907044 --- /dev/null +++ b/packages/cli/src/lib/release-service-files.ts @@ -0,0 +1,76 @@ +import { services as coreServices } from "@openmapx/core/server"; +import { execa } from "execa"; + +/** The selected release's service files, as its operations-agent image embeds them. */ +export interface SelectedReleaseServiceFiles { + files: coreServices.ReleaseServiceFiles; + release: string | null; + image: string; + /** The source revision the image was built from, from its OCI label. */ + revision: string | null; +} + +/** What the release image's fixed CLI prints them with. */ +export const RELEASE_FILES_COMMAND = ["node", "dist/cli.mjs", "compose", "release-files"]; + +/** + * The service files a host render uses: those of the release the overlay + * selects, read from its pinned operations-agent image (the host CLI runs + * the checkout's code, which is not the release). `null` when this process + * is the release's own CLI, or under `OPENMAPX_RELEASE=local`, where the + * checkout's files apply; `"none"` before a release is selected. + */ +export async function selectedReleaseServiceFiles( + infraDir: string, +): Promise { + if (coreServices.readEmbeddedReleaseServiceFiles()) return null; + if (coreServices.releaseChannel().kind === "disabled") return null; + const overlay = coreServices.readReleaseOverlayFile( + coreServices.releaseStatePaths(infraDir).overlay, + ); + if (overlay === null) return "none"; + const selection = coreServices.parseReleaseComposeSelection(overlay); + const image = selection.images["ops-agent"]; + if (!image) throw new Error("The release overlay pins no operations-agent image"); + const unreadable = (reason: string) => + new Error( + `The selected release's service files could not be read from its operations-agent image ${image}: ${reason}. Render uses the release's files, so it stops here. If the selected release predates this checkout, select this checkout's release: pnpm openmapx compose release.`, + ); + const run = await execa( + "docker", + ["run", "--rm", "--network", "none", "--read-only", image, ...RELEASE_FILES_COMMAND], + { reject: false, timeout: 300_000, maxBuffer: 64 * 1024 * 1024 }, + ); + if (run.exitCode !== 0) { + throw unreadable( + String(run.stderr ?? "") + .trim() + .split("\n") + .slice(-1)[0] || `exit ${run.exitCode}`, + ); + } + let files: coreServices.ReleaseServiceFiles; + try { + files = coreServices.parseReleaseServiceFiles(String(run.stdout ?? "")); + } catch (error) { + throw unreadable((error as Error).message); + } + const inspected = await execa( + "docker", + [ + "image", + "inspect", + "--format", + '{{index .Config.Labels "org.opencontainers.image.revision"}}', + image, + ], + { reject: false, timeout: 30_000 }, + ); + const revision = String(inspected.stdout ?? "").trim(); + return { + files, + release: selection.release, + image, + revision: inspected.exitCode === 0 && /^[a-f0-9]{40}$/.test(revision) ? revision : null, + }; +} diff --git a/packages/cli/src/lib/service-file-report.ts b/packages/cli/src/lib/service-file-report.ts new file mode 100644 index 000000000..3e0a993f9 --- /dev/null +++ b/packages/cli/src/lib/service-file-report.ts @@ -0,0 +1,116 @@ +import { services as coreServices } from "@openmapx/core/server"; +import { checkoutPositionFromGit } from "./checkout-position"; +import { containerBindMounts } from "./container-mounts"; +import { repoPaths } from "./paths"; +import { selectedReleaseServiceFiles } from "./release-service-files"; + +/** + * What `check` (and so Run diagnostics, which runs the agent's own check) + * says about the service files, without changing anything: the overrides in + * use, flagged and ignored, an import still to run, unused checkout edits, + * containers on the checkout's files, and, run from the checkout, what a + * host render would refuse. Each entry is a warning, or an error when render + * would fail on it. + */ +export async function serviceFileReport(): Promise< + Array<{ level: "warn" | "err"; message: string }> +> { + const paths = repoPaths(); + const lines: Array<{ level: "warn" | "err"; message: string }> = []; + const registry = new coreServices.ServiceRegistry({ rootDir: paths.root }); + await registry.load(); + const services = registry.list(); + let release: coreServices.ReleaseServiceFiles; + let selected: Awaited> = null; + try { + selected = await selectedReleaseServiceFiles(paths.infraDir); + release = + selected && selected !== "none" ? selected.files : coreServices.releaseServiceFiles(services); + } catch (error) { + return [{ level: "err", message: (error as Error).message }]; + } + let overrides: ReturnType; + try { + overrides = coreServices.readServiceOverrides(paths.infraDir, release); + } catch (error) { + return [{ level: "err", message: (error as Error).message }]; + } + // Run from the checkout, git says where it stands; the agent's own check + // goes by the checkout's service file history, as its render does. + const position = + selected && selected !== "none" + ? await checkoutPositionFromGit(paths.root, selected.revision) + : undefined; + let plan: coreServices.ServiceFileImportPlan | null = null; + try { + plan = coreServices.planServiceFileImport({ + infraDir: paths.infraDir, + services, + release, + ...(position ? { position } : {}), + }); + } catch (error) { + lines.push({ level: "err", message: (error as Error).message }); + } + if (plan?.kind === "import") { + const keys = plan.edits.map((edit) => `services/${edit.key}`); + lines.push({ + level: "warn", + message: `The next render imports, once, your edits in the checkout's service files as overrides in infra/docker/overrides/services${keys.length > 0 ? ` (${keys.join(", ")})` : ""}, and from then on uses the release's files.`, + }); + } else if (plan?.kind === "defer") { + const traefik = coreServices.waitingTraefikEdits(plan, release); + lines.push( + traefik.length > 0 + ? { + level: "err", + message: coreServices.describeWaitingTraefikEdits(plan.reason, traefik), + } + : { + level: "warn", + message: coreServices.describeDeferredServiceFileImport(plan.reason, plan.pending), + }, + ); + } + const mounts = await containerBindMounts(); + for (const message of coreServices.describeServiceFiles({ + infraDir: paths.infraDir, + services, + release, + overrides, + stale: coreServices.staleOverrides(paths.infraDir, overrides, { record: false }), + ...(mounts ? { mounts } : {}), + })) { + lines.push({ level: "warn", message }); + } + if (selected && selected !== "none") { + // The edits the next render imports are not what it would refuse. + const importing = new Set( + plan?.kind === "import" ? plan.edits.map((edit) => `services/${edit.key}`) : [], + ); + const differences = coreServices + .checkoutDifferencesFromRelease({ + rootDir: paths.root, + services, + release, + overrides: new Map( + overrides.active.map((entry) => [`${entry.serviceId}/${entry.path}`, entry.bytes]), + ), + }) + .filter((difference) => !importing.has(difference.path)); + if (differences.length > 0) { + lines.push({ + level: "err", + message: `A host render refuses until this is resolved. ${coreServices.describeCheckoutDifferences( + differences, + { + name: selected.release, + revision: selected.revision, + importWaits: plan?.kind === "defer" ? plan.reason : null, + }, + )}`, + }); + } + } + return lines; +} diff --git a/packages/core/src/api/contracts/integration-admin/operations.ts b/packages/core/src/api/contracts/integration-admin/operations.ts index 51921a9d8..f21dbfa79 100644 --- a/packages/core/src/api/contracts/integration-admin/operations.ts +++ b/packages/core/src/api/contracts/integration-admin/operations.ts @@ -45,6 +45,8 @@ const mutation = { body: adminIntegrationEmptyMutationSchema, }; const reloaded = { ...missing, 200: adminJson(adminIntegrationReloadResultSchema) }; +/** Not applied, for a reason the operator can fix (a service-file override render may not use). */ +const notApplied = { 409: adminJson(httpErrorSchema) }; const ok = adminJson(z.strictObject({ ok: z.literal(true) })); /** Complete integration/credential/overview family. Authentication metadata never enforces access. */ @@ -107,6 +109,7 @@ export const integrationAdminOperations = { request: { params: adminIntegrationParamsSchema, body: adminIntegrationConfigPatchSchema }, responses: { ...missing, + ...notApplied, 400: configErrors, 200: adminJson(adminIntegrationConfigUpdatedSchema), }, @@ -116,14 +119,14 @@ export const integrationAdminOperations = { path: "/api/admin/integrations/:id/enable", auth: "admin", request: mutation, - responses: reloaded, + responses: { ...reloaded, ...notApplied }, }, disableAdminIntegration: { method: "POST", path: "/api/admin/integrations/:id/disable", auth: "admin", request: mutation, - responses: reloaded, + responses: { ...reloaded, ...notApplied }, }, getAdminIntegrationHealth: { method: "GET", diff --git a/packages/core/src/api/contracts/service-admin/index.ts b/packages/core/src/api/contracts/service-admin/index.ts index cd707f978..4b76227e5 100644 --- a/packages/core/src/api/contracts/service-admin/index.ts +++ b/packages/core/src/api/contracts/service-admin/index.ts @@ -111,6 +111,8 @@ export const serviceAdminOperations = { 200: adminJson(adminServiceOkSchema), 404: adminJson(httpErrorSchema), 400: adminJson(z.union([httpErrorSchema, adminServiceValidationErrorSchema])), + // Not applied, for a reason the operator can fix (an override render may not use). + 409: adminJson(httpErrorSchema), }, }, getAdminServiceCredentials: { @@ -215,6 +217,8 @@ export const serviceAdminOperations = { responses: { ...adminCommonResponses, 200: adminJson(adminServiceSelectionUpdatedSchema), + // Not applied, for a reason the operator can fix (an override render may not use). + 409: adminJson(httpErrorSchema), 503: adminJson(httpErrorSchema), }, }, diff --git a/packages/core/src/api/generated/admin/integration-admin.generated.ts b/packages/core/src/api/generated/admin/integration-admin.generated.ts index b99e9bd49..e2270c302 100644 --- a/packages/core/src/api/generated/admin/integration-admin.generated.ts +++ b/packages/core/src/api/generated/admin/integration-admin.generated.ts @@ -25,6 +25,7 @@ import type { DisableAdminIntegration401, DisableAdminIntegration403, DisableAdminIntegration404, + DisableAdminIntegration409, DisableAdminIntegration413, DisableAdminIntegration415, DisableAdminIntegration429, @@ -35,6 +36,7 @@ import type { EnableAdminIntegration401, EnableAdminIntegration403, EnableAdminIntegration404, + EnableAdminIntegration409, EnableAdminIntegration413, EnableAdminIntegration415, EnableAdminIntegration429, @@ -140,6 +142,7 @@ import type { PatchAdminIntegrationConfig401, PatchAdminIntegrationConfig403, PatchAdminIntegrationConfig404, + PatchAdminIntegrationConfig409, PatchAdminIntegrationConfig413, PatchAdminIntegrationConfig415, PatchAdminIntegrationConfig429, @@ -1336,6 +1339,11 @@ export type patchAdminIntegrationConfigResponse404 = { status: 404; }; +export type patchAdminIntegrationConfigResponse409 = { + data: PatchAdminIntegrationConfig409; + status: 409; +}; + export type patchAdminIntegrationConfigResponse413 = { data: PatchAdminIntegrationConfig413; status: 413; @@ -1364,6 +1372,7 @@ export type patchAdminIntegrationConfigResponseError = ( | patchAdminIntegrationConfigResponse401 | patchAdminIntegrationConfigResponse403 | patchAdminIntegrationConfigResponse404 + | patchAdminIntegrationConfigResponse409 | patchAdminIntegrationConfigResponse413 | patchAdminIntegrationConfigResponse415 | patchAdminIntegrationConfigResponse429 @@ -1440,6 +1449,11 @@ export type disableAdminIntegrationResponse404 = { status: 404; }; +export type disableAdminIntegrationResponse409 = { + data: DisableAdminIntegration409; + status: 409; +}; + export type disableAdminIntegrationResponse413 = { data: DisableAdminIntegration413; status: 413; @@ -1468,6 +1482,7 @@ export type disableAdminIntegrationResponseError = ( | disableAdminIntegrationResponse401 | disableAdminIntegrationResponse403 | disableAdminIntegrationResponse404 + | disableAdminIntegrationResponse409 | disableAdminIntegrationResponse413 | disableAdminIntegrationResponse415 | disableAdminIntegrationResponse429 @@ -1544,6 +1559,11 @@ export type enableAdminIntegrationResponse404 = { status: 404; }; +export type enableAdminIntegrationResponse409 = { + data: EnableAdminIntegration409; + status: 409; +}; + export type enableAdminIntegrationResponse413 = { data: EnableAdminIntegration413; status: 413; @@ -1572,6 +1592,7 @@ export type enableAdminIntegrationResponseError = ( | enableAdminIntegrationResponse401 | enableAdminIntegrationResponse403 | enableAdminIntegrationResponse404 + | enableAdminIntegrationResponse409 | enableAdminIntegrationResponse413 | enableAdminIntegrationResponse415 | enableAdminIntegrationResponse429 diff --git a/packages/core/src/api/generated/admin/integration-models/disableAdminIntegration409.ts b/packages/core/src/api/generated/admin/integration-models/disableAdminIntegration409.ts new file mode 100644 index 000000000..cf1036277 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/disableAdminIntegration409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DisableAdminIntegration409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/enableAdminIntegration409.ts b/packages/core/src/api/generated/admin/integration-models/enableAdminIntegration409.ts new file mode 100644 index 000000000..95900167d --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/enableAdminIntegration409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type EnableAdminIntegration409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/index.ts b/packages/core/src/api/generated/admin/integration-models/index.ts index 3d75ce4c1..abdee5c9f 100644 --- a/packages/core/src/api/generated/admin/integration-models/index.ts +++ b/packages/core/src/api/generated/admin/integration-models/index.ts @@ -31,6 +31,7 @@ export * from "./disableAdminIntegration400"; export * from "./disableAdminIntegration401"; export * from "./disableAdminIntegration403"; export * from "./disableAdminIntegration404"; +export * from "./disableAdminIntegration409"; export * from "./disableAdminIntegration413"; export * from "./disableAdminIntegration415"; export * from "./disableAdminIntegration429"; @@ -41,6 +42,7 @@ export * from "./enableAdminIntegration400"; export * from "./enableAdminIntegration401"; export * from "./enableAdminIntegration403"; export * from "./enableAdminIntegration404"; +export * from "./enableAdminIntegration409"; export * from "./enableAdminIntegration413"; export * from "./enableAdminIntegration415"; export * from "./enableAdminIntegration429"; @@ -231,6 +233,7 @@ export * from "./patchAdminIntegrationConfig400"; export * from "./patchAdminIntegrationConfig401"; export * from "./patchAdminIntegrationConfig403"; export * from "./patchAdminIntegrationConfig404"; +export * from "./patchAdminIntegrationConfig409"; export * from "./patchAdminIntegrationConfig413"; export * from "./patchAdminIntegrationConfig415"; export * from "./patchAdminIntegrationConfig429"; diff --git a/packages/core/src/api/generated/admin/integration-models/patchAdminIntegrationConfig409.ts b/packages/core/src/api/generated/admin/integration-models/patchAdminIntegrationConfig409.ts new file mode 100644 index 000000000..bce415243 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/patchAdminIntegrationConfig409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PatchAdminIntegrationConfig409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-admin.generated.ts b/packages/core/src/api/generated/admin/service-admin.generated.ts index 500a781d9..66e93c9f7 100644 --- a/packages/core/src/api/generated/admin/service-admin.generated.ts +++ b/packages/core/src/api/generated/admin/service-admin.generated.ts @@ -276,6 +276,7 @@ import type { SetAdminServiceSelection400, SetAdminServiceSelection401, SetAdminServiceSelection403, + SetAdminServiceSelection409, SetAdminServiceSelection413, SetAdminServiceSelection415, SetAdminServiceSelection429, @@ -318,6 +319,7 @@ import type { UpdateAdminServiceConfig401, UpdateAdminServiceConfig403, UpdateAdminServiceConfig404, + UpdateAdminServiceConfig409, UpdateAdminServiceConfig413, UpdateAdminServiceConfig415, UpdateAdminServiceConfig429, @@ -2081,6 +2083,11 @@ export type setAdminServiceSelectionResponse403 = { status: 403; }; +export type setAdminServiceSelectionResponse409 = { + data: SetAdminServiceSelection409; + status: 409; +}; + export type setAdminServiceSelectionResponse413 = { data: SetAdminServiceSelection413; status: 413; @@ -2113,6 +2120,7 @@ export type setAdminServiceSelectionResponseError = ( | setAdminServiceSelectionResponse400 | setAdminServiceSelectionResponse401 | setAdminServiceSelectionResponse403 + | setAdminServiceSelectionResponse409 | setAdminServiceSelectionResponse413 | setAdminServiceSelectionResponse415 | setAdminServiceSelectionResponse429 @@ -2464,6 +2472,11 @@ export type updateAdminServiceConfigResponse404 = { status: 404; }; +export type updateAdminServiceConfigResponse409 = { + data: UpdateAdminServiceConfig409; + status: 409; +}; + export type updateAdminServiceConfigResponse413 = { data: UpdateAdminServiceConfig413; status: 413; @@ -2492,6 +2505,7 @@ export type updateAdminServiceConfigResponseError = ( | updateAdminServiceConfigResponse401 | updateAdminServiceConfigResponse403 | updateAdminServiceConfigResponse404 + | updateAdminServiceConfigResponse409 | updateAdminServiceConfigResponse413 | updateAdminServiceConfigResponse415 | updateAdminServiceConfigResponse429 diff --git a/packages/core/src/api/generated/admin/service-models/index.ts b/packages/core/src/api/generated/admin/service-models/index.ts index c8c67768c..71a117c35 100644 --- a/packages/core/src/api/generated/admin/service-models/index.ts +++ b/packages/core/src/api/generated/admin/service-models/index.ts @@ -379,6 +379,7 @@ export * from "./setAdminServiceSelection200SummarySource"; export * from "./setAdminServiceSelection400"; export * from "./setAdminServiceSelection401"; export * from "./setAdminServiceSelection403"; +export * from "./setAdminServiceSelection409"; export * from "./setAdminServiceSelection413"; export * from "./setAdminServiceSelection415"; export * from "./setAdminServiceSelection429"; @@ -422,6 +423,7 @@ export * from "./updateAdminServiceConfig400"; export * from "./updateAdminServiceConfig401"; export * from "./updateAdminServiceConfig403"; export * from "./updateAdminServiceConfig404"; +export * from "./updateAdminServiceConfig409"; export * from "./updateAdminServiceConfig413"; export * from "./updateAdminServiceConfig415"; export * from "./updateAdminServiceConfig429"; diff --git a/packages/core/src/api/generated/admin/service-models/setAdminServiceSelection409.ts b/packages/core/src/api/generated/admin/service-models/setAdminServiceSelection409.ts new file mode 100644 index 000000000..9379a8a6b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/setAdminServiceSelection409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type SetAdminServiceSelection409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/updateAdminServiceConfig409.ts b/packages/core/src/api/generated/admin/service-models/updateAdminServiceConfig409.ts new file mode 100644 index 000000000..53a8ce991 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/updateAdminServiceConfig409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UpdateAdminServiceConfig409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index c1b078d60..7f1953694 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -37,6 +37,10 @@ const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { kind: "configuration.hardlinks.inspect", revisionId: `cfg1_${"a".repeat(43)}`, }, + "configuration.failure.inspect": { + kind: "configuration.failure.inspect", + attemptedRevisionId: `cfg1_${"a".repeat(43)}`, + }, "stack.render": { expectedCurrentRevisionId: null, kind: "stack.render", diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index 34d69986b..8fdd0430d 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -22,6 +22,7 @@ export const OPS_OPERATION_KINDS = [ "stack.render", "configuration.inspect", "configuration.hardlinks.inspect", + "configuration.failure.inspect", "stack.start", "stack.stop", "service.start", @@ -144,6 +145,7 @@ export const OPS_KIND_POLICIES = { "stack.status": sync("api", 15_000, 256 * 1024), "configuration.inspect": sync("api", 15_000), "configuration.hardlinks.inspect": sync("api", 15_000, 1024 * 1024), + "configuration.failure.inspect": sync("api", 15_000), "stack.render": asyncEffect("api", 120_000), "stack.start": asyncEffect("api", LIFECYCLE_TIMEOUT_MS), "stack.stop": asyncEffect("api", 5 * 60_000), @@ -341,6 +343,13 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ kind: z.literal("configuration.hardlinks.inspect"), revisionId: configurationGenerationRevisionSchema, }), + // Why a configuration apply was not committed, when the operator can act + // on it (an override render may not use), as one line for the admin panel. + // A separate operation, so the apply result itself stays as before. + z.strictObject({ + kind: z.literal("configuration.failure.inspect"), + attemptedRevisionId: configurationGenerationRevisionSchema, + }), kindOnly("stack.start"), kindOnly("stack.stop"), serviceOperation("service.start"), @@ -961,6 +970,10 @@ export const OPS_RESULT_SCHEMAS = { ) .max(256), }), + "configuration.failure.inspect": z.strictObject({ + attemptedRevisionId: configurationGenerationRevisionSchema, + problem: z.string().min(1).max(4_096).nullable(), + }), "configuration.inspect": z.strictObject({ revisionId: configurationGenerationRevisionSchema.nullable(), integrationConfigDigests: z diff --git a/packages/core/src/services/__tests__/release-authority-inventory.test.ts b/packages/core/src/services/__tests__/release-authority-inventory.test.ts index 628e45433..412b05035 100644 --- a/packages/core/src/services/__tests__/release-authority-inventory.test.ts +++ b/packages/core/src/services/__tests__/release-authority-inventory.test.ts @@ -18,8 +18,6 @@ import { embeddedBuiltInServiceManifests as bundledBuiltInServiceManifests } fro import { ServiceRegistry } from "../registry"; import { captureReleaseServiceAuthority, - describeBuiltInServiceDrift, - inspectBuiltInServiceDrift, RELEASE_BUILT_IN_SERVICE_IDS, readBuiltInServices, validateReleaseServiceAuthority, @@ -184,9 +182,6 @@ describe("fixed release service authority", () => { source: releaseSource(RELEASE_BUILT_IN_SERVICE_IDS), }); expect(fromRelease.serviceIds).toEqual([...RELEASE_BUILT_IN_SERVICE_IDS].sort()); - expect( - inspectBuiltInServiceDrift(checkout, releaseSource(RELEASE_BUILT_IN_SERVICE_IDS)), - ).toEqual({ missing: [], changed: [], unexpected: [] }); }); it("runs the release's manifest when the checkout's was tampered with", async () => { @@ -215,10 +210,6 @@ describe("fixed release service authority", () => { }); await registry.load(); expect(registry.get("valhalla")?.manifest.container.image).toBe("ghcr.io/openmapx/valhalla"); - - const drift = inspectBuiltInServiceDrift(checkout, source); - expect(drift).toEqual({ missing: [], changed: ["valhalla"], unexpected: [] }); - expect(describeBuiltInServiceDrift(drift)).toContain("changed valhalla"); }); it("starts a release that adds a built-in service the checkout does not have yet", async () => { @@ -230,12 +221,9 @@ describe("fixed release service authority", () => { expect(captured.services.find((service) => service.manifest.id === "new-engine")).toMatchObject( { directory: join(checkout, "services", "new-engine"), isBuiltIn: true }, ); - const drift = inspectBuiltInServiceDrift(checkout, source); - expect(drift).toEqual({ missing: ["new-engine"], changed: [], unexpected: [] }); - expect(describeBuiltInServiceDrift(drift)).toContain("missing new-engine"); }); - it("reports checkout services the release does not have, without running them", async () => { + it("runs none of the checkout's services the release does not have", async () => { const ids = RELEASE_BUILT_IN_SERVICE_IDS.filter((id) => id !== "brouter"); const checkout = fixture(); mkdirSync(join(checkout, "services", "_template")); @@ -243,17 +231,5 @@ describe("fixed release service authority", () => { const source = releaseSource(ids); const captured = await captureReleaseServiceAuthority(checkout, { source }); expect(captured.serviceIds).not.toContain("brouter"); - expect(inspectBuiltInServiceDrift(checkout, source)).toEqual({ - missing: [], - changed: [], - unexpected: ["brouter"], - }); - }); - - it("reports no drift from source or when the checkout matches", () => { - const checkout = fixture(); - expect(inspectBuiltInServiceDrift(checkout)).toBeNull(); - expect(describeBuiltInServiceDrift(null)).toBeNull(); - expect(describeBuiltInServiceDrift({ missing: [], changed: [], unexpected: [] })).toBeNull(); }); }); diff --git a/packages/core/src/services/__tests__/release-service-files-local.test.ts b/packages/core/src/services/__tests__/release-service-files-local.test.ts index 880138544..15464beb3 100644 --- a/packages/core/src/services/__tests__/release-service-files-local.test.ts +++ b/packages/core/src/services/__tests__/release-service-files-local.test.ts @@ -1,9 +1,21 @@ -import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import { fileURLToPath } from "node:url"; import { afterEach, expect, it, vi } from "vitest"; -import { releaseServiceFiles } from "../release-service-files"; +import { commitConfigurationGeneration } from "../configuration-generation"; +import { CURRENT_CONFIGURATION } from "../deployment"; +import { readBuiltInServices } from "../release-authority-inventory"; +import { releaseServiceFiles, renderedEmbeddedReleaseServiceFiles } from "../release-service-files"; +import { serviceFilesImported } from "../service-file-import"; import type { LoadedService, ServiceManifest } from "../types"; // As the ops-agent's runtime bundle runs: its release's files embedded. @@ -32,6 +44,66 @@ function checkoutMartin(): LoadedService { }; } +it("imports nothing under OPENMAPX_RELEASE=local, and mounts the checkout's edit; a release's render imports it", async () => { + const root = mkdtempSync(join(tmpdir(), "openmapx-local-render-")); + roots.push(root); + cpSync(join(REPOSITORY, "services"), join(root, "services"), { + recursive: true, + filter: (source) => + !source.split(/[/\\]/).includes("node_modules") && + !["generated-routes.yml", "auth.yml"].includes(basename(source)), + }); + const infraDir = join(root, "infra", "docker"); + mkdirSync(infraDir, { recursive: true }); + // The agent's built-ins: the bundle's manifests, mounting from the checkout. + const services = readBuiltInServices(root).services; + const edited = "listen_addresses: 0.0.0.0:3001\n"; + writeFileSync(join(root, "services", "martin", "config", "martin.yaml"), edited); + const commit = () => + commitConfigurationGeneration({ + infraDir, + services, + input: { + domain: "maps.example.test", + selectedRoots: ["martin", "postgis"], + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + }, + legacyTraefikDynamicMounted: false, + }); + const mountedMartin = () => { + const compose = readFileSync( + join(infraDir, CURRENT_CONFIGURATION, "docker-compose.generated.yml"), + "utf8", + ); + const block = compose.slice(compose.indexOf("\n martin:\n")); + const line = block.split("\n").find((entry) => entry.includes(":/etc/martin/martin.yaml")); + const source = line?.trim().replace(/^- /, "").split(":")[0] ?? ""; + return readFileSync(join(infraDir, CURRENT_CONFIGURATION, source), "utf8"); + }; + + vi.stubEnv("OPENMAPX_RELEASE", "local"); + const local = await commit(); + expect(mountedMartin()).toBe(edited); + expect(serviceFilesImported(infraDir)).toBe(false); + expect(existsSync(join(infraDir, "overrides"))).toBe(false); + expect(local.serviceFileReport).toEqual([]); + + vi.unstubAllEnvs(); + const released = await commit(); + expect(released.serviceFileReport[0]).toContain( + "Imported 1 checkout service file that differs from every version a release shipped as overrides", + ); + expect(mountedMartin()).toBe(edited); + expect( + readFileSync( + join(infraDir, "overrides", "services", "martin", "config", "martin.yaml"), + "utf8", + ), + ).toBe(edited); +}); + it("renders the bundle's files, unless OPENMAPX_RELEASE=local makes the checkout's the release's", () => { const martin = checkoutMartin(); const released = releaseServiceFiles([martin]); @@ -40,7 +112,11 @@ it("renders the bundle's files, unless OPENMAPX_RELEASE=local makes the checkout readFileSync(join(REPOSITORY, "services", "martin", "config", "martin.yaml")), ); + expect(renderedEmbeddedReleaseServiceFiles()?.origin).toBe("release"); + vi.stubEnv("OPENMAPX_RELEASE", "local"); + // What the agent's start-up report reads: none, the checkout's are the release's. + expect(renderedEmbeddedReleaseServiceFiles()).toBeNull(); const local = releaseServiceFiles([martin]); expect(local.origin).toBe("checkout"); expect(local.files.get("martin/config/martin.yaml")?.bytes.toString()).toBe( diff --git a/packages/core/src/services/__tests__/service-file-overrides.test.ts b/packages/core/src/services/__tests__/service-file-overrides.test.ts new file mode 100644 index 000000000..3975d8655 --- /dev/null +++ b/packages/core/src/services/__tests__/service-file-overrides.test.ts @@ -0,0 +1,505 @@ +import { createHash } from "node:crypto"; +import { + chmodSync, + cpSync, + existsSync, + linkSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { agentStatePaths } from "../agent-state"; +import { + type ConfigurationInput, + commitConfigurationGeneration, +} from "../configuration-generation"; +import { CURRENT_CONFIGURATION } from "../deployment"; +import { findServiceManifestDirs } from "../manifest-discovery"; +import { + parseServiceFileHistory, + type ReleaseServiceFiles, + readCheckoutServiceFiles, +} from "../release-service-files"; +import { serviceFilesImported } from "../service-file-import"; +import { TRAEFIK_STATIC_GENERATION_FILE } from "../traefik-files"; +import type { LoadedService, ServiceManifest } from "../types"; + +const REPOSITORY = fileURLToPath(new URL("../../../../../", import.meta.url)); +const sha256 = (bytes: Buffer | string) => createHash("sha256").update(bytes).digest("hex"); +const HASH = "'op:$2y$05$secret-hash'"; +const AUTH = `http:\n middlewares:\n basic-auth:\n basicAuth:\n users: [${HASH}]\n`; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +/** + * A checkout: the repository's services copied into a temporary root, as + * `git clone` would leave them. Its files are the release's own. + */ +function checkout() { + const root = mkdtempSync(join(tmpdir(), "openmapx-overrides-")); + roots.push(root); + cpSync(join(REPOSITORY, "services"), join(root, "services"), { + recursive: true, + // Nothing a working checkout adds: dependencies, earlier routes, an operator's auth.yml. + filter: (source) => + !source.split(/[/\\]/).includes("node_modules") && + !["generated-routes.yml", "auth.yml"].includes(basename(source)), + }); + const infraDir = join(root, "infra", "docker"); + mkdirSync(infraDir, { recursive: true }); + const services: LoadedService[] = findServiceManifestDirs(join(root, "services")).map( + (directory) => ({ + manifest: JSON.parse( + readFileSync(join(directory, "service.json"), "utf8"), + ) as ServiceManifest, + directory, + isBuiltIn: true, + enabled: false, + }), + ); + // The release: these same files, before any edit, built at the checkout's + // commit or later, so it knows every version the checkout's history lists. + const shipped = readCheckoutServiceFiles(services, process.geteuid?.() ?? 0); + const listed = parseServiceFileHistory( + readFileSync(join(root, "services", "service-file-history.json"), "utf8"), + ); + const release = ( + changes: Record = {}, + history: Record = {}, + ): ReleaseServiceFiles => { + const files = new Map(shipped.files); + for (const [key, text] of Object.entries(changes)) { + const file = files.get(key); + if (!file) throw new Error(`no ${key}`); + files.set(key, { ...file, bytes: Buffer.from(text), sha256: sha256(text) }); + } + return { + origin: "release", + files, + history: new Map( + [...files].map(([key, file]) => [ + key, + new Set([file.sha256, ...(listed.get(key) ?? []), ...(history[key] ?? [])]), + ]), + ), + manifests: new Map(), + missing: [], + }; + }; + const override = (key: string, text: string, mode = 0o644) => { + const path = join(infraDir, "overrides", "services", key); + mkdirSync(join(path, ".."), { recursive: true, mode: 0o755 }); + writeFileSync(path, text, { mode }); + chmodSync(path, mode); + return path; + }; + return { root, infraDir, services, release, override }; +} + +function input(selectedRoots: string[]): ConfigurationInput { + return { + domain: "maps.example.test", + selectedRoots, + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + }; +} + +const SELECTED = ["traefik", "martin", "postgis", "valhalla"]; + +function mounted(infraDir: string, service: string, target: string): string { + const compose = readFileSync( + join(infraDir, CURRENT_CONFIGURATION, "docker-compose.generated.yml"), + "utf8", + ); + const block = compose.slice(compose.indexOf(`\n ${service}:\n`)); + const line = block.split("\n").find((entry) => entry.includes(`:${target}`)); + if (!line) throw new Error(`${service} does not mount ${target}`); + return join( + infraDir, + CURRENT_CONFIGURATION, + line.trim().replace(/^- /, "").split(":")[0] as string, + ); +} + +describe("service file overrides", () => { + it("use an override in place of the release's file, and name it, never its contents", async () => { + const f = checkout(); + f.override("martin/config/martin.yaml", "listen_addresses: 0.0.0.0:3001\n"); + f.override("traefik/config/dynamic/auth.yml", AUTH, 0o600); + f.override("traefik/config/traefik.yml", "entryPoints: {}\n"); + f.override("valhalla/config/valhalla.json", '{"mjolnir": {"concurrency": 3}}\n'); + const committed = await commitConfigurationGeneration({ + infraDir: f.infraDir, + services: f.services, + input: input(SELECTED), + releaseServiceFiles: f.release(), + legacyTraefikDynamicMounted: false, + }); + + expect(readFileSync(mounted(f.infraDir, "martin", "/etc/martin/martin.yaml"), "utf8")).toBe( + "listen_addresses: 0.0.0.0:3001\n", + ); + const dynamic = agentStatePaths(f.infraDir).traefikDynamic; + expect(readFileSync(join(dynamic, "auth.yml"), "utf8")).toBe(AUTH); + expect( + readFileSync(join(f.infraDir, CURRENT_CONFIGURATION, TRAEFIK_STATIC_GENERATION_FILE), "utf8"), + ).toBe("entryPoints: {}\n"); + expect( + readFileSync(join(f.infraDir, "data", "valhalla", "config", "valhalla.json"), "utf8"), + ).toBe('{"mjolnir": {"concurrency": 3}}\n'); + expect(committed.serviceFileReport).toEqual([ + "Service file overrides in use, in place of the release's files (infra/docker/overrides/services): martin/config/martin.yaml, traefik/config/dynamic/auth.yml (no release file; Traefik's dynamic directory), traefik/config/traefik.yml, valhalla/config/valhalla.json.", + ]); + expect(JSON.stringify(committed)).not.toContain("secret-hash"); + // Removing an override brings the release's file back. + rmSync(join(f.infraDir, "overrides", "services", "traefik", "config", "dynamic", "auth.yml")); + await commitConfigurationGeneration({ + infraDir: f.infraDir, + services: f.services, + input: input(SELECTED), + releaseServiceFiles: f.release(), + legacyTraefikDynamicMounted: false, + }); + expect(existsSync(join(dynamic, "auth.yml"))).toBe(false); + }); + + it("fails the render for an override of a shipped file it may not use, and names it, never its contents", async () => { + const f = checkout(); + const commit = () => + commitConfigurationGeneration({ + infraDir: f.infraDir, + services: f.services, + input: input(SELECTED), + releaseServiceFiles: f.release(), + legacyTraefikDynamicMounted: false, + }); + const auth = f.override("traefik/config/dynamic/middlewares.yml", AUTH, 0o600); + + chmodSync(auth, 0o620); + const writable = await commit().catch((error: Error) => error); + expect(String(writable)).toContain( + "infra/docker/overrides/services/traefik/config/dynamic/middlewares.yml can't be used: group or others may write it", + ); + expect(String(writable)).not.toContain("secret-hash"); + chmodSync(auth, 0o600); + + linkSync(auth, join(f.root, "second-name")); + await expect(commit()).rejects.toThrow("middlewares.yml can't be used: it has 2 links"); + rmSync(join(f.root, "second-name")); + + rmSync(auth); + writeFileSync(join(f.root, "elsewhere.yml"), AUTH); + symlinkSync(join(f.root, "elsewhere.yml"), auth); + await expect(commit()).rejects.toThrow("middlewares.yml can't be used: it is a symbolic link"); + rmSync(auth); + + // A link for a directory that holds shipped paths. + const config = join(f.infraDir, "overrides", "services", "martin", "config"); + mkdirSync(join(f.root, "linked-config")); + // 0755 whatever the umask: under 002 a plain mkdir is group-writable, + // which render refuses before it reaches the link. + mkdirSync(join(config, ".."), { recursive: true, mode: 0o755 }); + symlinkSync(join(f.root, "linked-config"), config); + await expect(commit()).rejects.toThrow( + "infra/docker/overrides/services/martin/config can't be used: it is a symbolic link", + ); + rmSync(config); + + // A directory where the release ships a file, as `mkdir -p` of its path leaves. + mkdirSync(join(config, "martin.yaml"), { recursive: true, mode: 0o755 }); + chmodSync(config, 0o755); + writeFileSync(join(config, "martin.yaml", "martin.yaml"), "listen_addresses: 0.0.0.0:3001\n"); + await expect(commit()).rejects.toThrow( + "infra/docker/overrides/services/martin/config/martin.yaml can't be used: it is a directory", + ); + rmSync(join(f.infraDir, "overrides", "services", "martin"), { recursive: true }); + chmodSync(join(f.infraDir, "overrides", "services"), 0o775); + await expect(commit()).rejects.toThrow( + "infra/docker/overrides/services can't be used: group or others may write it", + ); + chmodSync(join(f.infraDir, "overrides", "services"), 0o755); + await expect(commit()).resolves.toBeDefined(); + }); + + it("reports and ignores an override of a path the release doesn't ship", async () => { + const f = checkout(); + f.override("martin/config/extra.yaml", "x: 1\n"); + f.override("traefik/config/dynamic/generated-routes.yml", "http: {}\n"); + f.override("no-such-service/config.json", "{}\n"); + f.override("well-known/config/html/security.txt", "Contact: mailto:ops@example.test\n"); + const committed = await commitConfigurationGeneration({ + infraDir: f.infraDir, + services: f.services, + input: input([...SELECTED, "well-known"]), + releaseServiceFiles: f.release(), + legacyTraefikDynamicMounted: false, + }); + expect(committed.serviceFileReport).toEqual([ + "infra/docker/overrides/services/martin/config/extra.yaml is ignored: the release has no services/martin/config/extra.yaml for it to replace.", + 'infra/docker/overrides/services/no-such-service is ignored: there is no built-in service "no-such-service" with files to replace.', + "infra/docker/overrides/services/traefik/config/dynamic/generated-routes.yml is ignored: render writes Traefik's routes itself.", + "infra/docker/overrides/services/well-known/config/html/security.txt is ignored: the release has no services/well-known/config/html/security.txt for it to replace.", + ]); + const html = mounted(f.infraDir, "well-known", "/usr/share/nginx/html/.well-known"); + expect(readdirSync(html).sort()).toEqual(["apple-app-site-association", "assetlinks.json"]); + }); + + it("flags an override whose release file changed since, until the override is edited", async () => { + const f = checkout(); + const path = f.override("martin/config/martin.yaml", "listen_addresses: 0.0.0.0:3001\n"); + const commit = (changes: Record = {}) => + commitConfigurationGeneration({ + infraDir: f.infraDir, + services: f.services, + input: input(SELECTED), + releaseServiceFiles: f.release(changes), + legacyTraefikDynamicMounted: false, + }); + const flag = expect.stringContaining( + "The release's services/martin/config/martin.yaml changed since infra/docker/overrides/services/martin/config/martin.yaml was made", + ); + expect((await commit()).serviceFileReport).not.toContainEqual(flag); + const newer = { + "martin/config/martin.yaml": "listen_addresses: 0.0.0.0:3000\nworker_processes: 4\n", + }; + expect((await commit(newer)).serviceFileReport).toContainEqual(flag); + expect((await commit(newer)).serviceFileReport).toContainEqual(flag); + // Editing the override acknowledges the release's change. + writeFileSync(path, "listen_addresses: 0.0.0.0:3001\nworker_processes: 4\n"); + expect((await commit(newer)).serviceFileReport).not.toContainEqual(flag); + }); +}); + +describe("the one-time import of checkout edits", () => { + const commit = (f: ReturnType, release: ReleaseServiceFiles) => + commitConfigurationGeneration({ + infraDir: f.infraDir, + services: f.services, + input: input(SELECTED), + releaseServiceFiles: release, + legacyTraefikDynamicMounted: false, + }); + + it("makes overrides of edits, Traefik's auth.yml and basic-auth reference included, once", async () => { + const f = checkout(); + const traefik = join(f.root, "services", "traefik", "config"); + const gated = readFileSync(join(traefik, "traefik.yml"), "utf8").replace( + "middlewares:", + "middlewares:\n - basic-auth@file", + ); + writeFileSync(join(traefik, "traefik.yml"), gated); + writeFileSync(join(traefik, "dynamic", "auth.yml"), AUTH); + writeFileSync(join(traefik, "dynamic", "generated-routes.yml"), "stale: routes\n"); + writeFileSync(join(f.root, "services", "martin", "config", "martin.yaml"), "edited: true\n"); + chmodSync(join(f.root, "services", "martin", "config", "martin.yaml"), 0o600); + // valhalla-scripted's own rewrite: a file a container writes. + writeFileSync( + join(f.root, "services", "valhalla", "config", "valhalla.json"), + '{"concurrency": "12"}\n', + ); + + const committed = await commit(f, f.release()); + const overrides = join(f.infraDir, "overrides", "services"); + expect(committed.serviceFileReport[0]).toBe( + "Imported 3 checkout service files that differ from every version a release shipped as overrides, used from now on in place of the release's: infra/docker/overrides/services/martin/config/martin.yaml, infra/docker/overrides/services/traefik/config/dynamic/auth.yml, infra/docker/overrides/services/traefik/config/traefik.yml. Edit them there; the checkout's copies are no longer read.", + ); + expect(readFileSync(join(overrides, "traefik", "config", "traefik.yml"), "utf8")).toBe(gated); + expect(readFileSync(join(overrides, "traefik", "config", "dynamic", "auth.yml"), "utf8")).toBe( + AUTH, + ); + expect( + existsSync(join(overrides, "traefik", "config", "dynamic", "generated-routes.yml")), + ).toBe(false); + expect(existsSync(join(overrides, "valhalla"))).toBe(false); + // Never more open than the checkout's file; Traefik's owner-only. + const mode = (path: string) => statSync(path).mode & 0o777; + expect(mode(join(overrides, "martin", "config", "martin.yaml"))).toBe(0o600); + expect(mode(join(overrides, "traefik", "config", "dynamic", "auth.yml"))).toBe(0o600); + expect(mode(join(overrides, "traefik", "config", "traefik.yml"))).toBe(0o600); + // The basic-auth gate survives the switch to the release's files. + expect( + readFileSync(join(f.infraDir, CURRENT_CONFIGURATION, TRAEFIK_STATIC_GENERATION_FILE), "utf8"), + ).toContain("basic-auth@file"); + expect(readFileSync(join(agentStatePaths(f.infraDir).traefikDynamic, "auth.yml"), "utf8")).toBe( + AUTH, + ); + // The checkout is untouched, for a rollback to the release before. + expect(readFileSync(join(traefik, "traefik.yml"), "utf8")).toBe(gated); + expect(JSON.stringify(committed)).not.toContain("secret-hash"); + + // Once: a later checkout edit is reported, not imported. + writeFileSync( + join(f.root, "services", "postgis", "scripts", "sync-password.sh"), + "#!/bin/sh\n", + ); + const later = await commit(f, f.release()); + expect(existsSync(join(overrides, "postgis"))).toBe(false); + expect(later.serviceFileReport).toContainEqual( + "services/postgis/scripts/sync-password.sh differs from every version a release shipped, and render does not use it: services read the release's files, or your overrides. To keep this change, put the whole file in infra/docker/overrides/services/postgis/scripts/sync-password.sh.", + ); + expect(later.serviceFileReport.join("\n")).not.toContain("Imported"); + }); + + it("imports nothing from a checkout an earlier release left, with only Valhalla's own rewrite (openmapx.com)", async () => { + const f = checkout(); + // openmapx.com's checkout predates the service file history, which says + // it is behind the release. + rmSync(join(f.root, "services", "service-file-history.json")); + // The checkout is one release behind: martin.yaml and pelias.json hold + // that release's bytes, which the release's history knows. + const martin = readFileSync( + join(f.root, "services", "martin", "config", "martin.yaml"), + "utf8", + ); + const pelias = readFileSync( + join(f.root, "services", "pelias", "config", "pelias.json"), + "utf8", + ); + const release = f.release( + { + "martin/config/martin.yaml": `${martin}# newer\n`, + "pelias/config/pelias.json": `${pelias.trim()}\n\n`, + }, + { + "martin/config/martin.yaml": [sha256(martin)], + "pelias/config/pelias.json": [sha256(pelias)], + }, + ); + writeFileSync( + join(f.root, "services", "valhalla", "config", "valhalla.json"), + '{"mjolnir": {"concurrency": "12"}}\n', + ); + const committed = await commit(f, release); + expect(existsSync(join(f.infraDir, "overrides"))).toBe(false); + expect(committed.serviceFileReport).toEqual([]); + expect(serviceFilesImported(f.infraDir)).toBe(true); + expect(readFileSync(mounted(f.infraDir, "martin", "/etc/martin/martin.yaml"), "utf8")).toBe( + `${martin}# newer\n`, + ); + }); + + /** + * A checkout one commit ahead of the release: upstream changed martin.yaml, + * and its regenerated history lists the new bytes. + */ + function ahead(f: ReturnType) { + const newer = "listen_addresses: 0.0.0.0:3000\nworker_processes: 8\n"; + const martin = join(f.root, "services", "martin", "config", "martin.yaml"); + const shipped = readFileSync(martin, "utf8"); + writeFileSync(martin, newer); + const historyPath = join(f.root, "services", "service-file-history.json"); + const history = JSON.parse(readFileSync(historyPath, "utf8")) as { + files: Record; + }; + history.files["martin/config/martin.yaml"]?.push(sha256(newer)); + writeFileSync(historyPath, JSON.stringify(history)); + return { newer, shipped, historyPath, history }; + } + + it("waits, importing and recording nothing, while the checkout is ahead of the release", async () => { + const f = checkout(); + const { newer, shipped, historyPath, history } = ahead(f); + // The operator changed OTP's build configuration. + const otp = join(f.root, "services", "otp", "config", "build-config.json"); + writeFileSync(otp, '{"osm": []}\n'); + + const waiting = await commit(f, f.release()); + expect(existsSync(join(f.infraDir, "overrides"))).toBe(false); + expect(serviceFilesImported(f.infraDir)).toBe(false); + expect(waiting.serviceFileReport[0]).toBe( + "The one-time import of your edits in the checkout's service files waits: the checkout's services/service-file-history.json lists versions of services/martin/config/martin.yaml that the release doesn't, so the checkout may be ahead of it. A checkout file ahead of the release could be a newer upstream version rather than your edit, so nothing is imported or recorded until the checkout is at or behind the release's source revision; render tries again each time. Meanwhile render uses the release's files, not these checkout files, which match no version it shipped: services/martin/config/martin.yaml, services/otp/config/build-config.json. To keep an edit among them now, put the whole file in infra/docker/overrides/services//.", + ); + expect(readFileSync(mounted(f.infraDir, "martin", "/etc/martin/martin.yaml"), "utf8")).toBe( + shipped, + ); + // Again on the next render, still without a record. + expect((await commit(f, f.release())).serviceFileReport[0]).toContain("waits"); + expect(serviceFilesImported(f.infraDir)).toBe(false); + + // A history it can't read says nothing about where the checkout stands. + writeFileSync(historyPath, "{}"); + expect((await commit(f, f.release())).serviceFileReport[0]).toContain( + "waits: the checkout's services/service-file-history.json, which says whether it is ahead of the release, is not a service file history.", + ); + writeFileSync(historyPath, JSON.stringify(history)); + + // The release of that commit: upstream's martin.yaml is its own, and only + // the operator's file is imported. + const caughtUp = await commit(f, f.release({ "martin/config/martin.yaml": newer })); + expect(caughtUp.serviceFileReport[0]).toBe( + "Imported 1 checkout service file that differs from every version a release shipped as overrides, used from now on in place of the release's: infra/docker/overrides/services/otp/config/build-config.json. Edit them there; the checkout's copies are no longer read.", + ); + expect(existsSync(join(f.infraDir, "overrides", "services", "martin"))).toBe(false); + expect(serviceFilesImported(f.infraDir)).toBe(true); + }); + + it("refuses, while the import waits, a basic-auth gate it would carry over, until it is in overrides", async () => { + const f = checkout(); + ahead(f); + const traefik = join(f.root, "services", "traefik", "config"); + const gated = readFileSync(join(traefik, "traefik.yml"), "utf8").replace( + "middlewares:", + "middlewares:\n - basic-auth@file", + ); + writeFileSync(join(traefik, "traefik.yml"), gated); + writeFileSync(join(traefik, "dynamic", "auth.yml"), AUTH); + + const refused = await commit(f, f.release()).catch((error: Error) => error.message); + expect(refused).toBe( + "Render refuses while the one-time import of your service file edits waits (the checkout's services/service-file-history.json lists versions of services/martin/config/martin.yaml that the release doesn't, so the checkout may be ahead of it): the checkout holds Traefik files the import would carry over, and the release's files in their place could drop a basic-auth gate: services/traefik/config/dynamic/auth.yml, services/traefik/config/traefik.yml. Put each, as a whole file at the same path, in infra/docker/overrides/services/traefik/, created under umask 022 (an auth.yml holding password hashes with mode 0600), or delete it if you don't need it. An override ends the wait for that file, and render goes on.", + ); + expect(refused).not.toContain("secret-hash"); + expect(existsSync(join(f.infraDir, "overrides"))).toBe(false); + expect(serviceFilesImported(f.infraDir)).toBe(false); + expect(existsSync(join(f.infraDir, CURRENT_CONFIGURATION))).toBe(false); + + // The same files, placed in overrides: render goes on and serves the gate. + f.override("traefik/config/dynamic/auth.yml", AUTH, 0o600); + f.override("traefik/config/traefik.yml", gated); + const committed = await commit(f, f.release()); + expect( + readFileSync(join(f.infraDir, CURRENT_CONFIGURATION, TRAEFIK_STATIC_GENERATION_FILE), "utf8"), + ).toContain("basic-auth@file"); + expect(readFileSync(join(agentStatePaths(f.infraDir).traefikDynamic, "auth.yml"), "utf8")).toBe( + AUTH, + ); + // The rest still waits, reported: upstream's martin.yaml keeps the release's bytes. + expect(committed.serviceFileReport[0]).toContain( + "Meanwhile render uses the release's files, not these checkout files, which match no version it shipped: services/martin/config/martin.yaml.", + ); + expect(serviceFilesImported(f.infraDir)).toBe(false); + }); + + it("runs only for the release's files, and fails rather than lose an edit it may not read", async () => { + const f = checkout(); + const martin = join(f.root, "services", "martin", "config", "martin.yaml"); + // Running from the checkout, under OPENMAPX_RELEASE=local: its edits are + // the files, and nothing is imported. + writeFileSync(martin, "edited: true\n"); + await commit(f, readCheckoutServiceFiles(f.services, process.geteuid?.() ?? 0)); + expect(serviceFilesImported(f.infraDir)).toBe(false); + expect(existsSync(join(f.infraDir, "overrides"))).toBe(false); + + writeFileSync(join(f.root, "elsewhere.yaml"), "edited: true\n"); + rmSync(martin); + symlinkSync(join(f.root, "elsewhere.yaml"), martin); + await expect(commit(f, f.release())).rejects.toThrow( + "services/martin/config/martin.yaml can't be read for the one-time import of your service file edits: it is a symbolic link.", + ); + expect(serviceFilesImported(f.infraDir)).toBe(false); + }); +}); diff --git a/packages/core/src/services/__tests__/service-file-store.test.ts b/packages/core/src/services/__tests__/service-file-store.test.ts index ad299ad59..3e63c7cc0 100644 --- a/packages/core/src/services/__tests__/service-file-store.test.ts +++ b/packages/core/src/services/__tests__/service-file-store.test.ts @@ -28,8 +28,10 @@ import { embeddedBuiltInServiceFiles } from "../embedded-service-files.bundle"; import { findServiceManifestDirs } from "../manifest-discovery"; import { readBuiltInServices } from "../release-authority-inventory"; import { + parseReleaseServiceFiles, type ReleaseServiceFiles, readCheckoutServiceFiles, + serializeReleaseServiceFiles, serviceFileReferences, } from "../release-service-files"; import { readGenerationServiceFiles, serviceFileStore } from "../service-file-store"; @@ -199,14 +201,14 @@ describe("the release's service files in the store", () => { it("removes a tree nothing records or mounts, and keeps one a container mounts or when Docker can't tell", async () => { const { infraDir } = stack(); const services = releaseServices(); - const commit = (containerBindSources?: string[]) => + const commit = (sources?: string[]) => commitConfigurationGeneration({ infraDir, services, input: input(), releaseServiceFiles: release(services), legacyTraefikDynamicMounted: false, - containerBindSources, + ...(sources ? { containerBindMounts: [{ name: "docker-postgis-1", sources }] } : {}), }); await commit([]); // Trees a pruned generation recorded. @@ -376,6 +378,26 @@ describe("the runtime bundle's service files", () => { } }); + it("hands the host CLI its files as an index it verifies", () => { + const set = { + ...release(releaseServices()), + manifests: new Map([["postgis", "a".repeat(64)]]), + }; + const parsed = parseReleaseServiceFiles(serializeReleaseServiceFiles(set)); + expect([...parsed.files.keys()]).toEqual([...set.files.keys()]); + expect(parsed.files.get("postgis/scripts/sync-password.sh")).toMatchObject({ + executable: true, + bytes: set.files.get("postgis/scripts/sync-password.sh")?.bytes, + }); + expect(parsed.manifests).toEqual(set.manifests); + // Bytes that do not match their hash are refused. + const index = JSON.parse(serializeReleaseServiceFiles(set)); + index.files[0].bytes = Buffer.from("tampered").toString("base64"); + expect(() => parseReleaseServiceFiles(JSON.stringify(index))).toThrow( + "does not match its hash", + ); + }); + it("knows every file's current bytes as a version the release shipped", () => { for (const file of files) { expect(history.files[`${file.serviceId}/${file.path}`]).toContain( diff --git a/packages/core/src/services/agent-state.ts b/packages/core/src/services/agent-state.ts index 36249c100..ca81e65f7 100644 --- a/packages/core/src/services/agent-state.ts +++ b/packages/core/src/services/agent-state.ts @@ -1,5 +1,13 @@ import { chmodSync, lstatSync, mkdirSync, renameSync } from "node:fs"; import { dirname, join } from "node:path"; +import { atomicWrite } from "../release-files"; + +/** Replaces one of the agent's state files atomically, creating its directory (0700) first. */ +export function writeAgentStateFile(infraDir: string, name: string, contents: string): void { + const directory = join(infraDir, "state", "ops-agent"); + mkdirSync(directory, { recursive: true, mode: 0o700 }); + atomicWrite(join(directory, name), contents); +} /** * Where the operations agent keeps its own state: the job journal, the diff --git a/packages/core/src/services/checkout-differences.ts b/packages/core/src/services/checkout-differences.ts new file mode 100644 index 000000000..ccd36af57 --- /dev/null +++ b/packages/core/src/services/checkout-differences.ts @@ -0,0 +1,120 @@ +import { lstatSync, readdirSync, type Stats } from "node:fs"; +import { join } from "node:path"; +import { + type ReleaseServiceFiles, + readOwnedRegularFile, + serviceFileReferences, + sha256, +} from "./release-service-files"; +import type { LoadedService } from "./types"; + +/** + * How the checkout's built-in services differ from the selected release's, + * for the host CLI. It runs the checkout's code and manifests, so a host + * render would otherwise let the checkout decide what runs; until the host + * CLI runs from the release's image, it refuses instead. + * + * A service file that differs but has an override is fine: render uses the + * override, not the checkout's file. One a container writes (valhalla.json) + * is never compared. + */ +export interface CheckoutDifference { + /** Relative to the checkout. */ + readonly path: string; + readonly reason: string; +} + +const TRAEFIK_ROUTES = "traefik/config/dynamic/generated-routes.yml"; + +function lstatOrUndefined(path: string): Stats | undefined { + try { + return lstatSync(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } +} + +export function checkoutDifferencesFromRelease(options: { + rootDir: string; + /** The built-in services the checkout has, as the host CLI loaded them. */ + services: readonly LoadedService[]; + release: ReleaseServiceFiles; + overrides: ReadonlyMap; + uid?: number; +}): CheckoutDifference[] { + const { rootDir, release, overrides } = options; + const uid = options.uid ?? process.geteuid?.() ?? 0; + const servicesDir = join(rootDir, "services"); + const differences: CheckoutDifference[] = []; + const compare = (key: string, expected: string) => { + const read = readOwnedRegularFile(join(servicesDir, key), uid); + if ("reason" in read) { + differences.push({ + path: `services/${key}`, + reason: read.reason === "it is missing" ? "missing" : `can't be read: ${read.reason}`, + }); + } else if (sha256(read.bytes) !== expected) { + differences.push({ path: `services/${key}`, reason: "changed" }); + } + }; + for (const [id, hash] of [...release.manifests].sort(([a], [b]) => a.localeCompare(b))) { + compare(`${id}/service.json`, hash); + } + const builtIns = options.services.filter((service) => service.isBuiltIn); + for (const service of builtIns) { + if (!release.manifests.has(service.manifest.id)) { + differences.push({ path: `services/${service.manifest.id}`, reason: "not in this release" }); + } + } + const references = serviceFileReferences(builtIns.map((service) => service.manifest)); + const written = new Set( + references + .filter((reference) => reference.writableCopy) + .map((reference) => `${reference.serviceId}/${reference.path}`), + ); + for (const [key, file] of release.files) { + if (written.has(key) || overrides.has(key)) continue; + compare(key, file.sha256); + } + // A file the checkout adds to a directory the release ships. + const visit = (key: string) => { + const stats = lstatOrUndefined(join(servicesDir, key)); + if (!stats?.isDirectory()) return; + for (const name of readdirSync(join(servicesDir, key)).sort()) { + if (name.startsWith(".")) continue; + const child = `${key}/${name}`; + if (lstatOrUndefined(join(servicesDir, child))?.isDirectory()) visit(child); + else if (!release.files.has(child) && !overrides.has(child) && child !== TRAEFIK_ROUTES) { + differences.push({ path: `services/${child}`, reason: "not in this release" }); + } + } + }; + for (const reference of references) visit(`${reference.serviceId}/${reference.path}`); + return differences; +} + +/** Why a host render refuses, and what the operator can do. */ +export function describeCheckoutDifferences( + differences: readonly CheckoutDifference[], + release: { name: string | null; revision: string | null; importWaits?: string | null }, +): string { + const shown = differences.slice(0, 20).map((entry) => ` ${entry.path} (${entry.reason})`); + if (differences.length > shown.length) + shown.push(` and ${differences.length - shown.length} more`); + const built = release.revision ? `, built from ${release.revision}` : ""; + const checkout = release.revision + ? `git checkout ${release.revision}` + : "git checkout "; + return [ + `The checkout's built-in services differ from the selected release ${release.name ?? "(unknown)"}${built}:`, + ...shown, + "A host render runs this checkout's code and service manifests, so they must be the release's. Either wait until the release of this checkout's commit is published and run `pnpm openmapx compose release` again, or check out the selected release's source: " + + `${checkout}. To keep a change to a service file, put the whole file in infra/docker/overrides/services//; the operations agent, which runs the release's own files, applies it too.`, + ...(release.importWaits + ? [ + `Your edits in the checkout's service files are not imported as overrides meanwhile: ${release.importWaits}. A file there ahead of the release could be a newer upstream version rather than your edit; the one-time import runs once the checkout is at or behind the release's source revision.`, + ] + : []), + ].join("\n"); +} diff --git a/packages/core/src/services/configuration-generation.ts b/packages/core/src/services/configuration-generation.ts index 08e772b2b..4176ed3be 100644 --- a/packages/core/src/services/configuration-generation.ts +++ b/packages/core/src/services/configuration-generation.ts @@ -41,6 +41,7 @@ import { ConfigurationWriterBusyError, } from "./configuration-ownership"; import { validateTrustedConfigurationValues } from "./configuration-schema"; +import type { ContainerBindMounts } from "./container-mounts"; import { CONFIGURATION_GENERATIONS, CURRENT_CONFIGURATION, @@ -58,6 +59,8 @@ import { ServiceFileError, } from "./release-service-files"; import { buildAppApiServiceEnv, expandServiceSelection } from "./selection"; +import { type CheckoutPosition, importCheckoutServiceFiles } from "./service-file-import"; +import { describeServiceFiles } from "./service-file-report"; import { appliedServiceFilePath, collectServiceFileTrees, @@ -70,6 +73,7 @@ import { serviceFileTreePath, writeServiceFileTrees, } from "./service-file-store"; +import { overrideOverlay, readServiceOverrides, staleOverrides } from "./service-overrides"; import { TRAEFIK_STATIC_GENERATION_FILE, writeLegacyTraefikRoutes, @@ -123,11 +127,17 @@ export interface CommitConfigurationOptions { */ releaseServiceFiles?: ReleaseServiceFiles; /** - * The source of every bind mount of every container, running or stopped; - * a service-file tree one of them mounts is kept. Omitted (Docker can't - * tell), no tree is removed. + * Where the checkout stands against the release, for the one-time import + * of its edits: the host CLI asks git. Omitted, the checkout's own service + * file history says (the agent, which has no git). */ - containerBindSources?: readonly string[]; + checkoutPosition?: CheckoutPosition; + /** + * The bind mounts of every container, running or stopped: a service-file + * tree one of them mounts is kept, and those still on the checkout's files + * are reported. Omitted (Docker can't tell), no tree is removed. + */ + containerBindMounts?: readonly ContainerBindMounts[]; } export interface CommittedConfiguration { @@ -137,6 +147,11 @@ export interface CommittedConfiguration { warnings: string[]; /** Files beside the generation that were not written, and why; for every render's output. */ stackFileProblems: string[]; + /** + * The service files: the one-time import, the overrides in use, flagged and + * ignored, unused checkout edits, containers still on the checkout's files. + */ + serviceFileReport: string[]; } /** Evidence about THIS attempt, never inferred from a transport exception. */ @@ -940,14 +955,12 @@ export type RenderConfigurationOptions = Pick< | "input" | "allowMissingSelected" | "releaseServiceFiles" -> & { - /** Replacement bytes for release files, by `/` (the operator's overrides). */ - serviceFileOverlay?: Parameters[2]; -}; +>; /** * What a generation would hold, without writing it: the compose preview and - * every apply render through here. + * every apply render through here. The release's service files, with the + * operator's overrides in their place. */ export function renderConfiguration(options: RenderConfigurationOptions) { const { infraDir, input } = options; @@ -957,11 +970,9 @@ export function renderConfiguration(options: RenderConfigurationOptions) { input, options.allowMissingSelected ?? false, ); - const serviceFiles = resolveServiceFiles( - options.services, - options.releaseServiceFiles ?? releaseServiceFiles(options.services), - options.serviceFileOverlay, - ); + const release = options.releaseServiceFiles ?? releaseServiceFiles(options.services); + const overrides = readServiceOverrides(infraDir, release); + const serviceFiles = resolveServiceFiles(options.services, release, overrideOverlay(overrides)); const context: RenderContext = { domain: input.domain, composeOutDir: join(infraDir, CURRENT_CONFIGURATION), @@ -984,6 +995,7 @@ export function renderConfiguration(options: RenderConfigurationOptions) { context, rendered: renderCompose(resolved.enabled, context), serviceFiles, + overrides, }; } @@ -1017,7 +1029,19 @@ async function commitOwnedConfigurationGeneration( // Its bind mount is optional, left out when the directory is missing, so // it is there before the generation is rendered. ensureTraefikDynamicDirectory(infraDir); - const { resolved, context, rendered, serviceFiles } = renderConfiguration(options); + // The first generation with the release's files takes the operator's + // checkout edits along as overrides, before it renders. + const release = options.releaseServiceFiles ?? releaseServiceFiles(options.services); + const imported = importCheckoutServiceFiles({ + infraDir, + services: options.services, + release, + ...(options.checkoutPosition ? { position: options.checkoutPosition } : {}), + }); + const { resolved, context, rendered, serviceFiles, overrides } = renderConfiguration({ + ...options, + releaseServiceFiles: release, + }); const generations = join(infraDir, CONFIGURATION_GENERATIONS); await initializeOwnedConfigurationGenerations(infraDir); const nonce = randomUUID().replaceAll("-", ""); @@ -1142,6 +1166,7 @@ async function commitOwnedConfigurationGeneration( enabledServiceIds: resolved.enabledIdsOrdered, warnings: rendered.warnings ?? [], stackFileProblems: [], + serviceFileReport: [], }; await syncDirectory(dirname(join(infraDir, CURRENT_CONFIGURATION))); } else { @@ -1150,8 +1175,21 @@ async function commitOwnedConfigurationGeneration( enabledServiceIds: resolved.enabledIdsOrdered, warnings: rendered.warnings ?? [], stackFileProblems: [], + serviceFileReport: [], }; } + committed.serviceFileReport.push( + ...describeServiceFiles({ + infraDir, + services: options.services, + release, + overrides, + // This generation's overrides become the reviewed ones. + stale: staleOverrides(infraDir, overrides, { record: true }), + imported, + ...(options.containerBindMounts ? { mounts: options.containerBindMounts } : {}), + }), + ); committed.stackFileProblems.push( ...writeStackFiles( infraDir, @@ -1166,7 +1204,7 @@ async function commitOwnedConfigurationGeneration( collectServiceFileTrees( infraDir, recordedServiceFileTrees(infraDir), - options.containerBindSources, + options.containerBindMounts?.flatMap((mount) => mount.sources), ); } catch (error) { // They take little space; the next render tries again. diff --git a/packages/core/src/services/container-mounts.ts b/packages/core/src/services/container-mounts.ts index 48f292575..5f2a75123 100644 --- a/packages/core/src/services/container-mounts.ts +++ b/packages/core/src/services/container-mounts.ts @@ -33,9 +33,33 @@ export async function readContainerBindMounts( } } -/** All bind sources of `mounts`, or undefined when not known. */ -export function bindSources( - mounts: readonly ContainerBindMounts[] | undefined, -): string[] | undefined { - return mounts?.flatMap((mount) => mount.sources); +/** + * The containers, running or stopped, that still mount a file or directory + * from the checkout's `services/`: created by a release before services + * mounted the release's files. Traefik's dynamic directory has its own notice. + */ +export function containersOnCheckoutServiceFiles( + rootDir: string, + mounts: readonly ContainerBindMounts[], +): string[] { + const services = `${rootDir.replace(/\/+$/, "")}/services/`; + const traefikDynamic = `${services}traefik/config/dynamic`; + return mounts + .filter((mount) => + mount.sources.some( + (source) => + source.startsWith(services) && + source !== traefikDynamic && + !source.startsWith(`${traefikDynamic}/`), + ), + ) + .map((mount) => mount.name) + .sort(); +} + +export function describeContainersOnCheckoutServiceFiles(names: readonly string[]): string[] { + if (names.length === 0) return []; + return [ + `${names.length} container${names.length === 1 ? "" : "s"} still mount${names.length === 1 ? "s" : ""} service files from the checkout: ${names.join(", ")}. They keep working with the checkout's files until recreated, which then mounts the release's files and your overrides; recreate each when a short restart suits you: pnpm openmapx services update , or Recreate on Admin → Services. A later release stops mounting the checkout into the operations agent and waits for this.`, + ]; } diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index 9ef4240b3..81a751493 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -29,6 +29,11 @@ export { WELL_KNOWN_CAPABILITIES, WELL_KNOWN_DATA_TYPES, } from "./capabilities"; +export { + type CheckoutDifference, + checkoutDifferencesFromRelease, + describeCheckoutDifferences, +} from "./checkout-differences"; export { communityServiceManifestProblems } from "./community-manifest"; export { COMMUNITY_RECORD_VERSION, @@ -94,7 +99,6 @@ export { validateTrustedConfigurationValues, } from "./configuration-schema"; export { - bindSources, type ContainerBindMounts, readContainerBindMounts, } from "./container-mounts"; @@ -190,12 +194,9 @@ export { } from "./manifest-schema"; export { ServiceRegistry, type ServiceRegistryOptions } from "./registry"; export { - type BuiltInServiceDrift, type BuiltInServiceReadOptions, builtInServiceManifestSource, captureReleaseServiceAuthority, - describeBuiltInServiceDrift, - inspectBuiltInServiceDrift, RELEASE_BUILT_IN_SERVICE_IDS, RELEASE_NEVER_MANAGE_SERVICE_IDS, type ReleaseServiceAuthorityCapture, @@ -238,7 +239,9 @@ export { parseReleaseServiceFiles, type ReleaseServiceFile, type ReleaseServiceFiles, + readEmbeddedReleaseServiceFiles, releaseServiceFiles, + renderedEmbeddedReleaseServiceFiles, ServiceFileError, serializeReleaseServiceFiles, serviceFileReferences, @@ -307,7 +310,22 @@ export { parseServiceIdList, SERVICE_SELECTION_ENV, } from "./selection"; -export { serviceFileStore } from "./service-file-store"; +export { + type CheckoutPosition, + checkoutPositionFromHistory, + describeDeferredServiceFileImport, + describeServiceFileImport, + describeWaitingTraefikEdits, + importCheckoutServiceFiles, + planServiceFileImport, + type ServiceFileImport, + type ServiceFileImportPlan, + serviceFilesImported, + waitingTraefikEdits, +} from "./service-file-import"; +export { describeServiceFiles } from "./service-file-report"; +export { readGenerationServiceFiles, serviceFileStore } from "./service-file-store"; +export { readServiceOverrides, staleOverrides } from "./service-overrides"; export { LEGACY_TRAEFIK_MOUNT_NOTICE, legacyTraefikDynamicDirectory, diff --git a/packages/core/src/services/release-authority-inventory.ts b/packages/core/src/services/release-authority-inventory.ts index 26b7fb17a..2f5ccada6 100644 --- a/packages/core/src/services/release-authority-inventory.ts +++ b/packages/core/src/services/release-authority-inventory.ts @@ -1,5 +1,4 @@ import { createHash } from "node:crypto"; -import { readdirSync } from "node:fs"; import { join, resolve } from "node:path"; import { type BuiltInServiceManifestSource, @@ -185,75 +184,3 @@ export async function validateReleaseServiceAuthority(rootDir: string): Promise< const snapshot = await captureReleaseServiceAuthority(rootDir); return new Set(snapshot.serviceIds); } - -/** How the checkout's built-in services differ from the release's. */ -export interface BuiltInServiceDrift { - /** Release services the checkout has no manifest for. */ - readonly missing: readonly string[]; - /** Release services whose checkout manifest has different bytes or cannot be read. */ - readonly changed: readonly string[]; - /** Checkout service directories the release does not have. */ - readonly unexpected: readonly string[]; -} - -/** - * Compares the checkout's built-in manifests with the release's embedded - * ones. Returns `null` when running from source, where the checkout is the - * source. Never throws: drift does not change what runs, the release's - * manifests do, but configuration files still come from the checkout. - */ -export function inspectBuiltInServiceDrift( - rootDir: string, - source: BuiltInServiceManifestSource | null = embeddedBuiltInServiceManifests(), -): BuiltInServiceDrift | null { - if (!source?.embedded) return null; - const ids = [...source.ids].sort(); - const checkout = checkoutBuiltInServiceManifests(rootDir, ids); - const missing: string[] = []; - const changed: string[] = []; - for (const id of ids) { - let release: Buffer; - let current: Buffer; - try { - release = readRegularFileBytes(source.manifestPath(id), MAX_MANIFEST_BYTES); - } catch { - changed.push(id); - continue; - } - try { - current = readRegularFileBytes(checkout.manifestPath(id), MAX_MANIFEST_BYTES); - } catch (error) { - ((error as NodeJS.ErrnoException).code === "ENOENT" ? missing : changed).push(id); - continue; - } - if (!release.equals(current)) changed.push(id); - } - const unexpected: string[] = []; - try { - for (const entry of readdirSync(join(resolve(rootDir), "services"), { withFileTypes: true })) { - if (entry.name.startsWith(".") || entry.name.startsWith("_") || !entry.isDirectory()) { - continue; - } - if (!ids.includes(entry.name)) unexpected.push(entry.name); - } - } catch { - // A checkout without services/ reports every release service as missing. - } - return { missing, changed, unexpected: unexpected.sort() }; -} - -/** One line for an operator, or `null` when the checkout matches the release. */ -export function describeBuiltInServiceDrift(drift: BuiltInServiceDrift | null): string | null { - if (!drift) return null; - const parts = [ - drift.missing.length > 0 ? `missing ${drift.missing.join(", ")}` : null, - drift.changed.length > 0 ? `changed ${drift.changed.join(", ")}` : null, - drift.unexpected.length > 0 ? `not in this release ${drift.unexpected.join(", ")}` : null, - ].filter((part): part is string => part !== null); - if (parts.length === 0) return null; - return ( - `The checkout's built-in services differ from this release (${parts.join("; ")}). ` + - "The release's service manifests apply; update the checkout to the selected release, " + - "because the services still mount their configuration files from it." - ); -} diff --git a/packages/core/src/services/release-service-files.ts b/packages/core/src/services/release-service-files.ts index 0750b2b4b..3e13e308d 100644 --- a/packages/core/src/services/release-service-files.ts +++ b/packages/core/src/services/release-service-files.ts @@ -137,7 +137,8 @@ const historySchema = z.strictObject({ files: z.record(z.string(), z.array(z.string().regex(/^[a-f0-9]{64}$/))), }); -function parseHistory(raw: string): Map> { +/** `services/service-file-history.json`: every version of each file, by `/`. */ +export function parseServiceFileHistory(raw: string): Map> { const parsed = historySchema.parse(JSON.parse(raw)); return new Map(Object.entries(parsed.files).map(([key, hashes]) => [key, new Set(hashes)])); } @@ -168,7 +169,7 @@ export function readEmbeddedReleaseServiceFiles(): ReleaseServiceFiles | null { return { origin: "release", files, - history: parseHistory(readFileSync(fileURLToPath(embedded.history), "utf8")), + history: parseServiceFileHistory(readFileSync(fileURLToPath(embedded.history), "utf8")), manifests, missing: [], }; diff --git a/packages/core/src/services/service-file-import.ts b/packages/core/src/services/service-file-import.ts new file mode 100644 index 000000000..2051c6744 --- /dev/null +++ b/packages/core/src/services/service-file-import.ts @@ -0,0 +1,428 @@ +import { randomBytes } from "node:crypto"; +import { + closeSync, + constants, + existsSync, + fchmodSync, + fsyncSync, + lstatSync, + mkdirSync, + openSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + type Stats, + writeSync, +} from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import z from "zod/v4"; +import { fsyncDirectory } from "../release-files"; +import { agentStatePaths, writeAgentStateFile } from "./agent-state"; +import { + ownedDirectoryProblem, + parseServiceFileHistory, + type ReleaseServiceFiles, + readOwnedRegularFile, + ServiceFileError, + serviceFileReferences, + sha256, +} from "./release-service-files"; +import { + addOverrideRecords, + overridableWithoutRelease, + overridesDirectory, +} from "./service-overrides"; +import type { LoadedService } from "./types"; + +/** + * The one-time import of an operator's edits in the checkout's service files + * into overrides, when render first uses the release's files. + * + * A checkout file is an edit when its bytes match no version that file had + * in a release: a checkout the panel updated still holds an earlier + * release's files, which are not edits. That holds only for a checkout at or + * behind the release: one ahead of it holds upstream versions the release + * has never seen, which match none either. Where the checkout is not known + * to be at or behind the release, the import waits, importing and recording + * nothing, and render says so each time. + * + * A file a container writes (a `writableCopy`, valhalla.json) is never + * imported. Traefik's dynamic YAML and TOML files the release doesn't ship + * (an operator's `auth.yml`) are, and its `traefik.yml` when it differs (the + * `basic-auth@file` reference); the routes earlier releases wrote there never + * are. + * + * Nothing in the checkout is moved or changed. A differing file render may + * not read fails the render, with nothing recorded, rather than lose the + * edit. Afterwards the import is recorded and never runs again; render goes + * on reporting checkout edits that are not overrides. + */ + +const MARKER = "overrides-import.json"; +const markerSchema = z.strictObject({ + version: z.literal(1), + importedAt: z.string(), + imported: z.array(z.string()), +}); +/** Each commit's list of every version of its service files; CI fails one that lacks a file's current bytes. */ +export const CHECKOUT_SERVICE_FILE_HISTORY = "services/service-file-history.json"; + +function markerPath(infraDir: string): string { + return join(agentStatePaths(infraDir).directory, MARKER); +} + +/** Whether the import has run (the checkout's edits are overrides now, or there were none). */ +export function serviceFilesImported(infraDir: string): boolean { + try { + markerSchema.parse(JSON.parse(readFileSync(markerPath(infraDir), "utf8"))); + return true; + } catch { + return false; + } +} + +/** Whether the checkout is at or behind the release, so that its files hold no upstream version newer than the release's. */ +export type CheckoutPosition = + | { readonly atOrBehindRelease: true } + | { readonly atOrBehindRelease: false; readonly reason: string }; + +/** + * Where the checkout stands, without git (the agent): at or behind the + * release when every version of a release file that its own + * `services/service-file-history.json` lists is one the release's lists too. + * A commit that changes a service file adds its bytes to that list, so a + * checkout ahead of the release with a changed file lists a version the + * release doesn't. A file the release no longer names is left out (it is not + * imported), unless it is one of Traefik's dynamic files, which would be. A + * checkout without the list predates the release that introduced it, which + * is this one or earlier, so it is behind. + */ +export function checkoutPositionFromHistory( + rootDir: string, + release: ReleaseServiceFiles, + uid: number = process.geteuid?.() ?? 0, +): CheckoutPosition { + const read = readOwnedRegularFile(join(rootDir, CHECKOUT_SERVICE_FILE_HISTORY), uid); + if ("reason" in read) { + return read.reason === "it is missing" + ? { atOrBehindRelease: true } + : { + atOrBehindRelease: false, + reason: `the checkout's ${CHECKOUT_SERVICE_FILE_HISTORY}, which says whether it is ahead of the release, can't be used: ${read.reason}`, + }; + } + let listed: Map>; + try { + listed = parseServiceFileHistory(read.bytes.toString("utf8")); + } catch { + return { + atOrBehindRelease: false, + reason: `the checkout's ${CHECKOUT_SERVICE_FILE_HISTORY}, which says whether it is ahead of the release, is not a service file history`, + }; + } + const newer: string[] = []; + for (const [key, hashes] of [...listed].sort(([a], [b]) => a.localeCompare(b))) { + const known = release.history.get(key); + const [serviceId = "", ...rest] = key.split("/"); + if (!known && !overridableWithoutRelease(serviceId, rest.join("/"))) continue; + if ([...hashes].some((hash) => !known?.has(hash))) newer.push(`services/${key}`); + } + if (newer.length === 0) return { atOrBehindRelease: true }; + return { + atOrBehindRelease: false, + reason: `the checkout's ${CHECKOUT_SERVICE_FILE_HISTORY} lists versions of ${newer.join(", ")} that the release doesn't, so the checkout may be ahead of it`, + }; +} + +function lstatOrUndefined(path: string): Stats | undefined { + try { + return lstatSync(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } +} + +/** Each release file a container doesn't write, and Traefik's dynamic files the checkout adds. */ +function importCandidates( + rootDir: string, + services: readonly LoadedService[], + release: ReleaseServiceFiles, + uid: number, +): string[] { + const written = new Set( + serviceFileReferences(services.filter((service) => service.isBuiltIn).map((s) => s.manifest)) + .filter((reference) => reference.writableCopy) + .map((reference) => `${reference.serviceId}/${reference.path}`), + ); + const keys = [...release.files.keys()].filter((key) => !written.has(key)); + const dynamic = join(rootDir, "services", "traefik", "config", "dynamic"); + const stats = lstatOrUndefined(dynamic); + if (stats) { + const problem = ownedDirectoryProblem(stats, uid); + if (problem) { + throw new ServiceFileError( + `services/traefik/config/dynamic can't be read for the one-time import of your service file edits: ${problem}. Make it your own directory, or remove it, and render again.`, + ); + } + for (const name of readdirSync(dynamic).sort()) { + const key = `traefik/config/dynamic/${name}`; + if ( + overridableWithoutRelease("traefik", `config/dynamic/${name}`) && + !release.files.has(key) + ) { + keys.push(key); + } + } + } + return keys.sort(); +} + +interface CheckoutEdit { + readonly key: string; + readonly bytes: Buffer; + readonly mode: number; +} + +/** + * The checkout's files that match no version the release shipped and that + * no override carries yet. With `strict`, one that can't be read fails; + * otherwise it is left out. + */ +function differingCheckoutFiles( + infraDir: string, + services: readonly LoadedService[], + release: ReleaseServiceFiles, + uid: number, + strict: boolean, +): CheckoutEdit[] { + const rootDir = resolve(infraDir, "..", ".."); + const overrides = overridesDirectory(infraDir); + const edits: CheckoutEdit[] = []; + let candidates: string[]; + try { + candidates = importCandidates(rootDir, services, release, uid); + } catch (error) { + if (strict) throw error; + return []; + } + for (const key of candidates) { + if (existsSync(join(overrides, key)) || lstatOrUndefined(join(overrides, key))) continue; + const path = join(rootDir, "services", key); + if (!lstatOrUndefined(path)) continue; + const read = readOwnedRegularFile(path, uid); + if ("reason" in read) { + if (!strict) continue; + throw new ServiceFileError( + `services/${key} can't be read for the one-time import of your service file edits: ${read.reason}. Render now uses the release's files, so an edit there would be lost: replace it with a regular file of yours holding your version, or delete it if the release's version will do (git restore services/${key}), and render again.`, + ); + } + const hash = sha256(read.bytes); + const known = release.history.get(key) ?? new Set(); + if (hash === release.files.get(key)?.sha256 || known.has(hash)) continue; + // Never more open than the checkout's file, and never writable by others; + // an operator's Traefik files may hold password hashes, so owner-only. + const mode = read.mode & (key.startsWith("traefik/") ? 0o600 : 0o644); + edits.push({ key, bytes: read.bytes, mode: mode | 0o400 }); + } + return edits; +} + +/** What the import will do: take these edits along, or wait, and why. */ +export type ServiceFileImportPlan = + | { readonly kind: "import"; readonly edits: readonly CheckoutEdit[] } + | { + readonly kind: "defer"; + readonly reason: string; + /** Checkout files matching no version the release shipped, which render does not use meanwhile. */ + readonly pending: readonly string[]; + }; + +/** + * What the import would do now, changing nothing; null when it already ran + * or does not apply (running the checkout's files). Without `position`, the + * checkout's own history says where it stands (the agent, which has no git). + * Throws, when it would import, for a differing file it may not read. + */ +export function planServiceFileImport(options: { + infraDir: string; + services: readonly LoadedService[]; + release: ReleaseServiceFiles; + position?: CheckoutPosition; + uid?: number; +}): ServiceFileImportPlan | null { + const { infraDir, services, release } = options; + if (release.origin !== "release" || serviceFilesImported(infraDir)) return null; + const uid = options.uid ?? process.geteuid?.() ?? 0; + const position = + options.position ?? checkoutPositionFromHistory(resolve(infraDir, "..", ".."), release, uid); + if (!position.atOrBehindRelease) { + return { + kind: "defer", + reason: position.reason, + pending: differingCheckoutFiles(infraDir, services, release, uid, false).map( + (edit) => edit.key, + ), + }; + } + return { kind: "import", edits: differingCheckoutFiles(infraDir, services, release, uid, true) }; +} + +function writeOverride(path: string, bytes: Buffer, mode: number): void { + mkdirSync(dirname(path), { recursive: true, mode: 0o755 }); + const temporary = join(dirname(path), `.${randomBytes(6).toString("hex")}.import.tmp`); + const fd = openSync( + temporary, + constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, + 0o600, + ); + try { + let offset = 0; + while (offset < bytes.length) offset += writeSync(fd, bytes, offset); + fchmodSync(fd, mode); + fsyncSync(fd); + } catch (error) { + closeSync(fd); + rmSync(temporary, { force: true }); + throw error; + } + closeSync(fd); + renameSync(temporary, path); + fsyncDirectory(dirname(path)); +} + +/** + * The Traefik files a waiting import would carry over, which render refuses + * to go without: a dynamic YAML or TOML file the release doesn't ship (an + * operator's `auth.yml`) or a differing `traefik.yml` (its `basic-auth@file` + * reference). Using the release's files in their place could drop a + * basic-auth gate. Other differing files, Traefik's shipped dynamic files + * included, keep the release's bytes and are reported. + */ +export function waitingTraefikEdits( + plan: ServiceFileImportPlan | null, + release: ReleaseServiceFiles, +): string[] { + if (plan?.kind !== "defer") return []; + return plan.pending.filter( + (key) => + key === "traefik/config/traefik.yml" || + (key.startsWith("traefik/") && + overridableWithoutRelease("traefik", key.slice("traefik/".length)) && + !release.files.has(key)), + ); +} + +/** Why render refuses while the import waits on Traefik files, and how to go on. */ +export function describeWaitingTraefikEdits(reason: string, keys: readonly string[]): string { + return `Render refuses while the one-time import of your service file edits waits (${reason}): the checkout holds Traefik files the import would carry over, and the release's files in their place could drop a basic-auth gate: ${keys.map((key) => `services/${key}`).join(", ")}. Put each, as a whole file at the same path, in infra/docker/overrides/services/traefik/, created under umask 022 (an auth.yml holding password hashes with mode 0600), or delete it if you don't need it. An override ends the wait for that file, and render goes on.`; +} + +/** What the import did: the files it made overrides of (`/`), or why it waits. */ +export type ServiceFileImport = + | { readonly imported: readonly string[] } + | { readonly deferred: string; readonly pending: readonly string[] }; + +/** + * Imports the checkout's edits once, when `release` is the release's own + * files and the checkout is at or behind it. Null when the import already + * ran or does not apply (running the checkout's files). + */ +export function importCheckoutServiceFiles(options: { + infraDir: string; + services: readonly LoadedService[]; + release: ReleaseServiceFiles; + position?: CheckoutPosition; + uid?: number; +}): ServiceFileImport | null { + const { infraDir, release } = options; + const plan = planServiceFileImport(options); + if (!plan) return null; + if (plan.kind === "defer") { + const traefik = waitingTraefikEdits(plan, release); + if (traefik.length > 0) { + throw new ServiceFileError(describeWaitingTraefikEdits(plan.reason, traefik)); + } + return { deferred: plan.reason, pending: plan.pending }; + } + const overrides = overridesDirectory(infraDir); + const records: Record = {}; + for (const { key, bytes, mode } of plan.edits) { + writeOverride(join(overrides, key), bytes, mode); + records[key] = { override: sha256(bytes), base: release.files.get(key)?.sha256 ?? null }; + } + if (plan.edits.length > 0) addOverrideRecords(infraDir, records); + const imported = plan.edits.map((edit) => edit.key); + writeAgentStateFile( + infraDir, + MARKER, + `${JSON.stringify({ version: 1, importedAt: new Date().toISOString(), imported })}\n`, + ); + return { imported }; +} + +/** Why the import waits, and what render does meanwhile. */ +export function describeDeferredServiceFileImport( + reason: string, + pending: readonly string[], +): string { + const meanwhile = + pending.length > 0 + ? ` Meanwhile render uses the release's files, not these checkout files, which match no version it shipped: ${pending.map((key) => `services/${key}`).join(", ")}. To keep an edit among them now, put the whole file in infra/docker/overrides/services//.` + : ""; + return `The one-time import of your edits in the checkout's service files waits: ${reason}. A checkout file ahead of the release could be a newer upstream version rather than your edit, so nothing is imported or recorded until the checkout is at or behind the release's source revision; render tries again each time.${meanwhile}`; +} + +/** Said once, when the import made overrides of the checkout's edits, and each time it waits. */ +export function describeServiceFileImport(result: ServiceFileImport | null): string[] { + if (!result) return []; + if ("deferred" in result) + return [describeDeferredServiceFileImport(result.deferred, result.pending)]; + const { imported } = result; + if (imported.length === 0) return []; + return [ + `Imported ${imported.length} checkout service file${imported.length === 1 ? " that differs" : "s that differ"} from every version a release shipped as overrides, used from now on in place of the release's: ${imported.map((key) => `infra/docker/overrides/services/${key}`).join(", ")}. Edit them there; the checkout's copies are no longer read.`, + ]; +} + +/** + * Checkout service files that differ from every version a release shipped + * and that no override carries: an edit render does not use. Unreadable + * files are left out; the import reported them. + */ +export function unusedCheckoutEdits(options: { + infraDir: string; + services: readonly LoadedService[]; + release: ReleaseServiceFiles; + overrides: ReadonlyMap; + uid?: number; +}): string[] { + const { infraDir, services, release, overrides } = options; + if (release.origin !== "release") return []; + const uid = options.uid ?? process.geteuid?.() ?? 0; + const rootDir = resolve(infraDir, "..", ".."); + let candidates: string[]; + try { + candidates = importCandidates(rootDir, services, release, uid); + } catch { + return []; + } + const unused: string[] = []; + for (const key of candidates) { + const read = readOwnedRegularFile(join(rootDir, "services", key), uid); + if ("reason" in read) continue; + const hash = sha256(read.bytes); + if (hash === release.files.get(key)?.sha256 || release.history.get(key)?.has(hash)) continue; + if (overrides.get(key)?.equals(read.bytes)) continue; + unused.push(key); + } + return unused; +} + +export function describeUnusedCheckoutEdits(keys: readonly string[]): string[] { + return keys.map( + (key) => + `services/${key} differs from every version a release shipped, and render does not use it: services read the release's files, or your overrides. To keep this change, put the whole file in infra/docker/overrides/services/${key}.`, + ); +} diff --git a/packages/core/src/services/service-file-report.ts b/packages/core/src/services/service-file-report.ts new file mode 100644 index 000000000..ff3db72f6 --- /dev/null +++ b/packages/core/src/services/service-file-report.ts @@ -0,0 +1,64 @@ +import { resolve } from "node:path"; +import { + type ContainerBindMounts, + containersOnCheckoutServiceFiles, + describeContainersOnCheckoutServiceFiles, +} from "./container-mounts"; +import type { ReleaseServiceFiles } from "./release-service-files"; +import { + describeServiceFileImport, + describeUnusedCheckoutEdits, + type ServiceFileImport, + serviceFilesImported, + unusedCheckoutEdits, +} from "./service-file-import"; +import { + type ActiveOverride, + describeServiceOverrides, + overrideOverlay, + type ServiceOverrides, +} from "./service-overrides"; +import type { LoadedService } from "./types"; + +/** + * What render's output, the agent's log, `check` and Run diagnostics say + * about the service files: the import, once, or why it waits; the overrides in use, those + * whose release file changed since and those ignored; checkout edits no + * override carries; and containers still mounting the checkout's files. + * Names only, never contents. + */ +export function describeServiceFiles(options: { + infraDir: string; + services: readonly LoadedService[]; + release: ReleaseServiceFiles; + overrides: ServiceOverrides; + stale: readonly ActiveOverride[]; + imported?: ServiceFileImport | null; + mounts?: readonly ContainerBindMounts[]; +}): string[] { + const { infraDir, release, overrides } = options; + const lines = [ + ...describeServiceFileImport(options.imported ?? null), + ...describeServiceOverrides(overrides, options.stale), + ]; + if (release.origin === "release" && serviceFilesImported(infraDir)) { + lines.push( + ...describeUnusedCheckoutEdits( + unusedCheckoutEdits({ + infraDir, + services: options.services, + release, + overrides: overrideOverlay(overrides).files, + }), + ), + ); + } + if (options.mounts) { + lines.push( + ...describeContainersOnCheckoutServiceFiles( + containersOnCheckoutServiceFiles(resolve(infraDir, "..", ".."), options.mounts), + ), + ); + } + return lines; +} diff --git a/packages/core/src/services/service-overrides.ts b/packages/core/src/services/service-overrides.ts new file mode 100644 index 000000000..8dd81d581 --- /dev/null +++ b/packages/core/src/services/service-overrides.ts @@ -0,0 +1,288 @@ +import { lstatSync, readdirSync, readFileSync, type Stats } from "node:fs"; +import { join } from "node:path"; +import z from "zod/v4"; +import { agentStatePaths, writeAgentStateFile } from "./agent-state"; +import { + ownedDirectoryProblem, + type ReleaseServiceFiles, + readOwnedRegularFile, + ServiceFileError, + sha256, +} from "./release-service-files"; +import type { ServiceFileOverlay } from "./service-file-store"; + +/** + * An operator's deliberate changes to release service files: whole files in + * `infra/docker/overrides/services//`, read at render + * and used in place of the release's file. + * + * Only a regular file of the deployment's user, with one link and that no + * group or other may write, in directories that are the same, is used; never + * through a link. An override of a path the release ships that is anything + * else fails the render: using the release's file instead could drop what it + * changes (a basic-auth gate). One for a path the release doesn't ship is + * reported and ignored, so a release that drops a file never blocks an + * upgrade. Traefik's dynamic directory is the exception: an operator's YAML + * or TOML file there (`auth.yml`) is an override of that directory. + * + * The agent's state records each override's bytes and the release's file it + * replaced; render flags one whose release file changed since. Editing the + * override is how the operator acknowledges that. + */ + +export const OVERRIDES_DIRECTORY = "overrides"; +const TRAEFIK_DYNAMIC = /^config\/dynamic\/([^/]+\.(?:ya?ml|toml))$/; +const TRAEFIK_ROUTES = "config/dynamic/generated-routes.yml"; + +export interface ActiveOverride { + readonly serviceId: string; + readonly path: string; + readonly bytes: Buffer; + readonly sha256: string; + /** The release's file it replaces, or null for a Traefik dynamic file the release doesn't ship. */ + readonly releaseSha256: string | null; +} + +export interface IgnoredOverride { + /** Below `infra/docker/overrides/services`. */ + readonly path: string; + readonly reason: string; +} + +export interface ServiceOverrides { + readonly active: readonly ActiveOverride[]; + readonly ignored: readonly IgnoredOverride[]; +} + +export function overridesDirectory(infraDir: string): string { + return join(infraDir, OVERRIDES_DIRECTORY, "services"); +} + +/** Whether a release-relative `path` of `serviceId` may be overridden although the release doesn't ship it. */ +export function overridableWithoutRelease(serviceId: string, path: string): boolean { + return serviceId === "traefik" && TRAEFIK_DYNAMIC.test(path) && path !== TRAEFIK_ROUTES; +} + +function lstatOrUndefined(path: string): Stats | undefined { + try { + return lstatSync(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } +} + +/** Why an overrides directory may not be read, or null when it may. */ +function directoryProblem(stats: Stats | undefined, uid: number): string | null { + return ( + ownedDirectoryProblem(stats, uid) ?? + ((stats?.mode ?? 0) & 0o022 ? "group or others may write it" : null) + ); +} + +const unusable = (path: string, reason: string) => + new ServiceFileError( + `infra/docker/overrides/services/${path} can't be used: ${reason}. Render refuses it rather than use the release's file in its place: make it a regular file of yours that only you may write, or delete it.`, + ); + +/** The overrides render uses, and those it ignores; throws for one of a shipped path it may not use. */ +export function readServiceOverrides( + infraDir: string, + release: ReleaseServiceFiles, + uid: number = process.geteuid?.() ?? 0, +): ServiceOverrides { + const root = join(infraDir, OVERRIDES_DIRECTORY); + const active: ActiveOverride[] = []; + const ignored: IgnoredOverride[] = []; + const rootStats = lstatOrUndefined(root); + if (!rootStats) return { active, ignored }; + const rootProblem = directoryProblem(rootStats, uid); + if (rootProblem) { + throw new ServiceFileError( + `infra/docker/overrides can't be used: ${rootProblem}. Render refuses to go on without the overrides it may hold.`, + ); + } + const services = join(root, "services"); + const servicesStats = lstatOrUndefined(services); + if (!servicesStats) return { active, ignored }; + const servicesProblem = directoryProblem(servicesStats, uid); + if (servicesProblem) { + throw new ServiceFileError( + `infra/docker/overrides/services can't be used: ${servicesProblem}. Render refuses to go on without the overrides it may hold.`, + ); + } + const shippedPaths = new Map(); + for (const file of release.files.values()) { + const paths = shippedPaths.get(file.serviceId) ?? []; + paths.push(file.path); + shippedPaths.set(file.serviceId, paths); + } + const shipped = (serviceId: string, path: string) => + release.files.has(`${serviceId}/${path}`) || overridableWithoutRelease(serviceId, path); + /** Whether anything render would use may lie below `path`. */ + const mayHold = (serviceId: string, path: string) => + (shippedPaths.get(serviceId) ?? []).some((file) => file.startsWith(`${path}/`)) || + (serviceId === "traefik" && "config/dynamic".startsWith(path)); + + const visit = (serviceId: string, directory: string, prefix: string) => { + for (const name of readdirSync(directory).sort()) { + const path = prefix ? `${prefix}/${name}` : name; + const shown = `${serviceId}/${path}`; + const absolute = join(directory, name); + const stats = lstatSync(absolute); + if (stats.isDirectory()) { + // In place of a file the release ships, as `mkdir -p` of its path makes one. + if (shipped(serviceId, path)) throw unusable(shown, "it is a directory"); + const problem = directoryProblem(stats, uid); + if (problem && mayHold(serviceId, path)) throw unusable(shown, problem); + if (problem) ignored.push({ path: shown, reason: problem }); + else visit(serviceId, absolute, path); + continue; + } + // A link in place of a directory that holds shipped files. + if (stats.isSymbolicLink() && mayHold(serviceId, path)) { + throw unusable(shown, "it is a symbolic link"); + } + if (!shipped(serviceId, path)) { + ignored.push({ + path: shown, + reason: + serviceId === "traefik" && path === TRAEFIK_ROUTES + ? "render writes Traefik's routes itself" + : `the release has no services/${shown} for it to replace`, + }); + continue; + } + const read = readOwnedRegularFile(absolute, uid); + if ("reason" in read) throw unusable(shown, read.reason); + if (read.mode & 0o022) throw unusable(shown, "group or others may write it"); + active.push({ + serviceId, + path, + bytes: read.bytes, + sha256: sha256(read.bytes), + releaseSha256: release.files.get(shown)?.sha256 ?? null, + }); + } + }; + for (const serviceId of readdirSync(services).sort()) { + const absolute = join(services, serviceId); + const stats = lstatSync(absolute); + if (!shippedPaths.has(serviceId) && serviceId !== "traefik") { + ignored.push({ + path: serviceId, + reason: `there is no built-in service "${serviceId}" with files to replace`, + }); + continue; + } + const problem = directoryProblem(stats, uid); + if (problem) throw unusable(serviceId, problem); + visit(serviceId, absolute, ""); + } + return { active, ignored }; +} + +/** The bytes render uses in place of the release's. */ +export function overrideOverlay(overrides: ServiceOverrides): ServiceFileOverlay { + return { + files: new Map( + overrides.active.map((entry) => [`${entry.serviceId}/${entry.path}`, entry.bytes]), + ), + }; +} + +const recordSchema = z.strictObject({ + version: z.literal(1), + overrides: z.record( + z.string(), + z.strictObject({ + override: z.string().regex(/^[a-f0-9]{64}$/), + base: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable(), + }), + ), +}); +type OverrideRecords = z.infer["overrides"]; + +const RECORDS = "overrides.json"; + +function recordPath(infraDir: string): string { + return join(agentStatePaths(infraDir).directory, RECORDS); +} + +function readRecords(infraDir: string): OverrideRecords { + try { + return recordSchema.parse(JSON.parse(readFileSync(recordPath(infraDir), "utf8"))).overrides; + } catch { + return {}; + } +} + +function writeRecords(infraDir: string, records: OverrideRecords): void { + writeAgentStateFile(infraDir, RECORDS, `${JSON.stringify({ version: 1, overrides: records })}\n`); +} + +/** Records overrides render made itself (the import), each with the release file it replaced. */ +export function addOverrideRecords(infraDir: string, records: OverrideRecords): void { + writeRecords(infraDir, { ...readRecords(infraDir), ...records }); +} + +/** + * The overrides whose release file changed since they were made or last + * edited. With `record`, the agent's state takes each new or edited + * override's release file as its base, and forgets removed ones. + */ +export function staleOverrides( + infraDir: string, + overrides: ServiceOverrides, + options: { record: boolean }, +): ActiveOverride[] { + const recorded = readRecords(infraDir); + const next: OverrideRecords = {}; + const stale: ActiveOverride[] = []; + for (const entry of overrides.active) { + const key = `${entry.serviceId}/${entry.path}`; + const record = recorded[key]; + if (!record || record.override !== entry.sha256) { + next[key] = { override: entry.sha256, base: entry.releaseSha256 }; + continue; + } + next[key] = record; + if (record.base !== entry.releaseSha256) stale.push(entry); + } + if (options.record && JSON.stringify(next) !== JSON.stringify(recorded)) { + writeRecords(infraDir, next); + } + return stale; +} + +/** What render, the agent's log, check and Run diagnostics say about the overrides; names only. */ +export function describeServiceOverrides( + overrides: ServiceOverrides, + stale: readonly ActiveOverride[], +): string[] { + const lines: string[] = []; + if (overrides.active.length > 0) { + const names = overrides.active.map( + (entry) => + `${entry.serviceId}/${entry.path}${entry.releaseSha256 === null ? " (no release file; Traefik's dynamic directory)" : ""}`, + ); + lines.push( + `Service file overrides in use, in place of the release's files (infra/docker/overrides/services): ${names.join(", ")}.`, + ); + } + for (const entry of stale) { + const key = `${entry.serviceId}/${entry.path}`; + lines.push( + entry.releaseSha256 === null + ? `The release no longer ships services/${key}, which infra/docker/overrides/services/${key} was made for. Check that it still applies, then edit the override to acknowledge it.` + : `The release's services/${key} changed since infra/docker/overrides/services/${key} was made, and the override still replaces it. Compare the two, then edit the override to acknowledge the release's change.`, + ); + } + for (const entry of overrides.ignored) { + lines.push(`infra/docker/overrides/services/${entry.path} is ignored: ${entry.reason}.`); + } + return lines; +} diff --git a/packages/core/src/services/writable-copies.ts b/packages/core/src/services/writable-copies.ts index e8c8a720d..5d0f184a7 100644 --- a/packages/core/src/services/writable-copies.ts +++ b/packages/core/src/services/writable-copies.ts @@ -16,8 +16,8 @@ import { } from "node:fs"; import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import z from "zod/v4"; -import { atomicWrite, fsyncDirectory } from "../release-files"; -import { agentStatePaths } from "./agent-state"; +import { fsyncDirectory } from "../release-files"; +import { agentStatePaths, writeAgentStateFile } from "./agent-state"; import { ServiceFileError } from "./release-service-files"; import type { WritableCopySeed } from "./service-file-store"; @@ -240,7 +240,11 @@ function seedCopies( } } if (JSON.stringify(next) !== JSON.stringify(recorded)) { - atomicWrite(recordPath(infraDir), `${JSON.stringify({ version: 1, seeds: next })}\n`); + writeAgentStateFile( + infraDir, + "writable-copies.json", + `${JSON.stringify({ version: 1, seeds: next })}\n`, + ); } return problems; } diff --git a/release-notes/service-files-from-the-release.md b/release-notes/service-files-from-the-release.md new file mode 100644 index 000000000..c19bc1dbf --- /dev/null +++ b/release-notes/service-files-from-the-release.md @@ -0,0 +1,103 @@ +--- +impacts: [migration, operator-action] +--- + +## What changes + +Built-in services mount the configuration files and scripts of the release the +stack runs, not the checkout's `services/`. The operations agent's image embeds +them. Render copies them, with each configuration generation, into +content-addressed directories in `infra/docker/state/ops-agent/service-files`, +which the containers mount read-only. The OTP and Pelias builds read the same +copies. Valhalla, which rewrites its `valhalla.json` on every start, runs from a +writable copy in `infra/docker/data/valhalla/config/valhalla.json`. Render seeds +that copy from the release's file and seeds it again only when that changes. + +Deliberate changes go in overrides: whole files in +`infra/docker/overrides/services//`, used in place of the +release's file. + +- An override render may not use fails the render, by name: a link, a second + link, another owner, one its group or others may write, or a directory in + place of a file. A configuration saved in the admin panel then fails with + that reason. +- An override of a path the release doesn't ship is reported and ignored. The + exception is a YAML or TOML file in Traefik's dynamic directory, such as + `auth.yml`, which is added to Traefik's configuration. +- Render, the agent's log, `pnpm openmapx check` and **Run diagnostics** list + the overrides in use. They also flag those whose release file changed since + the override was made; editing the override acknowledges the change. +- This replaces the earlier warning that the checkout's built-in services + differ from the release's. + +The first render of this release imports, once, every checkout service file +that differs from every version a release shipped, as an override, and lists +them. This covers Traefik's `auth.yml` and a `traefik.yml` naming +`basic-auth@file`, so a basic-auth gate stays. A file a container writes +(`valhalla.json`) is never imported, and nothing in the checkout is changed. A +checkout an earlier release left after panel updates holds only released +versions, so nothing is imported from it. Later checkout edits are reported as +unused. The import runs only from a checkout at or behind the selected +release's source revision, since one ahead of it can hold newer upstream files +that also match no released version. Until then it waits, imports and records +nothing, and render, `check` and Run diagnostics say why and list the checkout +files not used meanwhile. The host CLI asks git; the operations agent goes by +the checkout's `services/service-file-history.json`. Meanwhile render refuses +if the checkout holds a Traefik file the import would carry over (an +`auth.yml`, another dynamic file the release doesn't ship, or a changed +`traefik.yml`), naming it, until it is an override, so a basic-auth gate +can't drop. A configuration saved in the admin panel then fails with that +reason. + +If Valhalla's writable copy is missing and can't be written, render fails with +the reason, rather than let Docker create a root-owned directory in its place. + +The host CLI runs the checkout's code and manifests. A host render now reads +the selected release's service files from its operations-agent image, and +refuses while the checkout's service manifests or files differ from that +release's (overrides aside). It names the files and says what to do: wait until +the release of the checkout's commit is published and run +`pnpm openmapx compose release` again, or check out the release's source +revision. With no release selected yet, render uses the checkout's files and +says so. Under `OPENMAPX_RELEASE=local` the checkout's files are the +release's, for the host CLI and the operations agent alike, and nothing is +imported. + +## Who is affected + +Every installation that runs built-in services. Operators who edited files +under `services/` (Traefik basic auth, Martin sources, OTP configuration, and so +on) find those edits as overrides after the first render. Hosts that follow +`main` with the host CLI may see a render refused until the release of their +commit is published. + +## Required action + +- **After the upgrade, check the render's output, or `pnpm openmapx check`, + for the files it imported, or why the import waits.** If it waits, an edit + of yours is not applied until it runs, and Traefik edits make render refuse: + put the file in an override yourself, or bring the checkout to the release's + source revision. From then on, edit those files in + `infra/docker/overrides/services`, not in the checkout. Create new + overrides under `umask 022`: Debian's and Ubuntu's default `002` makes + them group-writable, and render refuses those. +- **Select the release before rendering on a host.** The documented host + update already does: `git pull`, `pnpm install`, `pnpm openmapx compose +release`, `pnpm openmapx compose render`. A first installation now runs + `compose release` before `compose render`. +- **Choose when services recreate.** A container keeps mounting the checkout's + files until it is recreated. Those files still hold the same bytes, and + render, `check` and Run diagnostics list such containers. The documented host + update, `services update` with a list of services, recreates only the + services listed. A panel update recreates only the application services, + which mount no service files. **The first stack-wide `pnpm openmapx compose +up` recreates every service whose mounts moved, PostGIS and Traefik + included.** Run it when a restart of the database and the proxy suits you, + or recreate services one at a time with `pnpm openmapx services update +`. A later release stops mounting the checkout into the operations + agent, and waits until no container mounts the checkout's service files. + +Rollback: the release before this one mounts the checkout's files again, +which this release leaves untouched. Edits made only in `overrides/` meanwhile +do not apply there; carry them back to the checkout if you need them before +upgrading again. diff --git a/scripts/ops-agent-built-in-services.test.ts b/scripts/ops-agent-built-in-services.test.ts index 30f6cb3ac..680e31efa 100644 --- a/scripts/ops-agent-built-in-services.test.ts +++ b/scripts/ops-agent-built-in-services.test.ts @@ -1,4 +1,5 @@ import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; import { chmodSync, cpSync, @@ -109,12 +110,35 @@ describe("the ops-agent's fixed CLI", () => { expect(missing.stdout).toBe(""); }, 60_000); - it("reports how the checkout differs from the release", async () => { + it("reports the release's service files, whatever the checkout holds, and prints them for the host CLI", async () => { const checkout = hostileCheckout(); const checked = await cli(checkout, "check", "--no-probe"); const output = `${checked.stdout}\n${checked.stderr}`; - expect(output).toContain("missing brouter"); - expect(output).toContain("changed valhalla"); - expect(output).toContain("not in this release unlisted-engine"); + // The checkout's manifests don't matter to the agent any more; its edits + // to service files become overrides at the next render. + expect(output).toContain("The next render imports, once"); + expect(output).not.toContain("missing brouter"); + + const printed = await cli(checkout, "compose", "release-files"); + const index = JSON.parse(printed.stdout) as { + manifests: Record; + files: Array<{ serviceId: string; path: string; executable: boolean }>; + }; + // The release's manifests, brouter included, not the tampered valhalla. + expect(index.manifests.valhalla).toBe( + createHash("sha256") + .update(readFileSync(join(ROOT, "services", "valhalla", "service.json"))) + .digest("hex"), + ); + expect(index.files).toContainEqual( + expect.objectContaining({ serviceId: "brouter", path: "scripts/run.sh" }), + ); + expect(index.files).toContainEqual( + expect.objectContaining({ + serviceId: "postgis", + path: "scripts/sync-password.sh", + executable: true, + }), + ); }, 60_000); }); diff --git a/scripts/ops-authority-inventory.json b/scripts/ops-authority-inventory.json index efdc351c3..11d6137b8 100644 --- a/scripts/ops-authority-inventory.json +++ b/scripts/ops-authority-inventory.json @@ -36,6 +36,7 @@ "backup.list", "backup.restore", "communityServices.inspect", + "configuration.failure.inspect", "configuration.hardlinks.inspect", "configuration.inspect", "data.clean", @@ -195,6 +196,14 @@ "semanticSources": ["apps/api/src/services/admin-ops.ts#applyGenerationHardlinks"], "typedKinds": ["configuration.hardlinks.inspect"] }, + { + "id": "api.configuration-failure", + "callSiteIds": [], + "semanticSources": [ + "apps/api/src/services/trusted-config-operations.ts#readConfigurationFailure" + ], + "typedKinds": ["configuration.failure.inspect"] + }, { "id": "api.compose-runtime", "callSiteIds": [],