From cdba4c31203fe2bdb8cdffe06e960ac37e2e77bc Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Tue, 18 Aug 2026 15:34:48 -0700 Subject: [PATCH 1/9] chore: trigger CI to verify NUT failure logs [do-not-merge] --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 69129dc5..53b78b6c 100644 --- a/README.md +++ b/README.md @@ -2498,3 +2498,5 @@ EXAMPLES _See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/2.2.3/src/commands/agent/validate/authoring-bundle.ts)_ + + From 8b6f429077ff7704f4a70217d996bee89db56891 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Mon, 19 Oct 2026 14:13:47 -0700 Subject: [PATCH 2/9] test(nuts): collapse redundant run-eval invocations to reduce auth churn [do-not-merge] The run-eval NUTs issued the identical JSON-payload command 4x and the identical YAML-spec command 2x, each a fresh CLI process doing a fresh OAuth token refresh against the single shared scratch-org admin user. That burst trips Salesforce's per-user login-rate limit (login rate exceeded), the dominant NUT failure. Run each distinct command once in a before() hook and share the parsed result across the tests that assert against it. All assertions preserved; genuinely-distinct commands untouched. --- test/nuts/agent.test.run-eval.nut.ts | 80 +++++++++++++++------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/test/nuts/agent.test.run-eval.nut.ts b/test/nuts/agent.test.run-eval.nut.ts index 8c73b42c..3533be5b 100644 --- a/test/nuts/agent.test.run-eval.nut.ts +++ b/test/nuts/agent.test.run-eval.nut.ts @@ -29,55 +29,65 @@ describe('agent test run-eval', function () { const yamlSpecPath = join(mockProjectDir, 'eval-test-spec.yaml'); const yamlWithContextPath = join(mockProjectDir, 'eval-with-context.yaml'); + // The JSON-payload run-eval is expensive (~18 min) and hammers the shared admin user's + // OAuth refresh rate limit when invoked repeatedly. Run it exactly once here and share the + // parsed result across every test that asserts against it, so a rate-limit failure surfaces + // once rather than per-test. + let jsonResult: RunEvalResult; + before(async function () { this.timeout(30 * 60 * 1000); // 30 minutes for setup await getTestSession(); + + const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; + const jsonExec = execCmd(jsonCommand, { ensureExitCode: 0 }); + expect(jsonExec.jsonOutput?.result).to.be.ok; + jsonResult = jsonExec.jsonOutput!.result; }); describe('run-eval with JSON file', () => { it('should run evaluation with JSON payload file', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); - - expect(result.jsonOutput?.result).to.be.ok; - expect(result.jsonOutput?.result.tests).to.be.an('array'); - expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0); - expect(result.jsonOutput?.result.summary).to.be.ok; - expect(result.jsonOutput?.result.summary.passed).to.be.a('number'); - expect(result.jsonOutput?.result.summary.failed).to.be.a('number'); - expect(result.jsonOutput?.result.summary.scored).to.be.a('number'); - expect(result.jsonOutput?.result.summary.errors).to.be.a('number'); + expect(jsonResult).to.be.ok; + expect(jsonResult.tests).to.be.an('array'); + expect(jsonResult.tests.length).to.be.greaterThan(0); + expect(jsonResult.summary).to.be.ok; + expect(jsonResult.summary.passed).to.be.a('number'); + expect(jsonResult.summary.failed).to.be.a('number'); + expect(jsonResult.summary.scored).to.be.a('number'); + expect(jsonResult.summary.errors).to.be.a('number'); }); it('should run evaluation with normalized payload', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); - - expect(result.jsonOutput?.result.tests[0]).to.be.ok; - expect(result.jsonOutput?.result.tests[0].id).to.equal('test-topic-routing'); - expect(result.jsonOutput?.result.tests[0].status).to.be.oneOf(['passed', 'failed']); - expect(result.jsonOutput?.result.tests[0].evaluations).to.be.an('array'); + expect(jsonResult.tests[0]).to.be.ok; + expect(jsonResult.tests[0].id).to.equal('test-topic-routing'); + expect(jsonResult.tests[0].status).to.be.oneOf(['passed', 'failed']); + expect(jsonResult.tests[0].evaluations).to.be.an('array'); }); }); describe('run-eval with YAML file', () => { - it('should run evaluation with YAML test spec file', async () => { + // The YAML-spec run-eval is run once and shared across the two tests that assert against it, + // for the same OAuth-rate-limit reason as the JSON-payload run above. + let yamlResult: RunEvalResult; + + before(() => { const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); + const output = execCmd(command, { ensureExitCode: 0 }); + expect(output.jsonOutput?.result).to.be.ok; + yamlResult = output.jsonOutput!.result; + }); - expect(result.jsonOutput?.result).to.be.ok; - expect(result.jsonOutput?.result.tests).to.be.an('array'); - expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0); - expect(result.jsonOutput?.result.summary).to.be.ok; + it('should run evaluation with YAML test spec file', async () => { + expect(yamlResult).to.be.ok; + expect(yamlResult.tests).to.be.an('array'); + expect(yamlResult.tests.length).to.be.greaterThan(0); + expect(yamlResult.summary).to.be.ok; }); it('should auto-infer agent name from YAML subjectName', async () => { - const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); - // Should succeed without explicit --api-name flag - expect(result.jsonOutput?.result).to.be.ok; - expect(result.jsonOutput?.result.tests).to.be.an('array'); + expect(yamlResult).to.be.ok; + expect(yamlResult.tests).to.be.an('array'); }); it('should handle YAML spec with contextVariables', async () => { @@ -173,11 +183,8 @@ describe('agent test run-eval', function () { describe('run-eval output structure', () => { it('should include test summaries with correct structure', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); - - expect(result.jsonOutput?.result.tests).to.be.an('array'); - const firstTest = result.jsonOutput?.result.tests[0]; + expect(jsonResult.tests).to.be.an('array'); + const firstTest = jsonResult.tests[0]; expect(firstTest).to.have.property('id'); expect(firstTest).to.have.property('status'); expect(firstTest).to.have.property('evaluations'); @@ -187,10 +194,7 @@ describe('agent test run-eval', function () { }); it('should include summary with all metrics', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); - - const summary = result.jsonOutput?.result.summary; + const summary = jsonResult.summary; expect(summary).to.have.property('passed'); expect(summary).to.have.property('failed'); expect(summary).to.have.property('scored'); From c2bb5f93d126563cea096b32c0d0d482076d0821 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Wed, 21 Oct 2026 10:57:12 -0700 Subject: [PATCH 3/9] test(nuts): isolate run-eval on a dedicated scratch-org user [do-not-merge] Give the agent test run-eval NUT its own scratch-org user (System Administrator profile + EinsteinGPTPromptTemplateManager, mirroring the shared admin) so its OAuth grant/session churn no longer concentrates on the shared admin and trips the per-user login-rate cap. --- test/nuts/agent.test.run-eval.nut.ts | 24 +++++++-------- test/nuts/shared-setup.ts | 45 +++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/test/nuts/agent.test.run-eval.nut.ts b/test/nuts/agent.test.run-eval.nut.ts index 3533be5b..c6569bae 100644 --- a/test/nuts/agent.test.run-eval.nut.ts +++ b/test/nuts/agent.test.run-eval.nut.ts @@ -18,7 +18,7 @@ import { join } from 'node:path'; import { expect } from 'chai'; import { execCmd } from '@salesforce/cli-plugins-testkit'; import type { RunEvalResult } from '../../src/commands/agent/test/run-eval.js'; -import { getTestSession, getUsername } from './shared-setup.js'; +import { getTestSession, getRunEvalUsername } from './shared-setup.js'; describe('agent test run-eval', function () { // Increase timeout for setup since shared setup includes long waits and deployments @@ -39,7 +39,7 @@ describe('agent test run-eval', function () { this.timeout(30 * 60 * 1000); // 30 minutes for setup await getTestSession(); - const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; + const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; const jsonExec = execCmd(jsonCommand, { ensureExitCode: 0 }); expect(jsonExec.jsonOutput?.result).to.be.ok; jsonResult = jsonExec.jsonOutput!.result; @@ -71,7 +71,7 @@ describe('agent test run-eval', function () { let yamlResult: RunEvalResult; before(() => { - const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getRunEvalUsername()} --json`; const output = execCmd(command, { ensureExitCode: 0 }); expect(output.jsonOutput?.result).to.be.ok; yamlResult = output.jsonOutput!.result; @@ -91,7 +91,7 @@ describe('agent test run-eval', function () { }); it('should handle YAML spec with contextVariables', async () => { - const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getRunEvalUsername()} --json`; const result = execCmd(command, { ensureExitCode: 0 }); // Verify the command succeeds with contextVariables @@ -104,7 +104,7 @@ describe('agent test run-eval', function () { describe('run-eval with flags', () => { it('should respect --no-normalize flag', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getRunEvalUsername()} --json`; const result = execCmd(command, { ensureExitCode: 1 }); expect(result.jsonOutput?.result).to.be.ok; @@ -112,7 +112,7 @@ describe('agent test run-eval', function () { }); it('should use custom batch size', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getRunEvalUsername()} --json`; const result = execCmd(command, { ensureExitCode: 0 }); expect(result.jsonOutput?.result).to.be.ok; @@ -121,20 +121,20 @@ describe('agent test run-eval', function () { it('should support different result formats', async () => { // Test human format (default) - const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getUsername()}`; + const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getRunEvalUsername()}`; const humanResult = execCmd(humanCommand, { ensureExitCode: 0 }); expect(humanResult.shellOutput.stdout).to.be.ok; expect(humanResult.shellOutput.stdout).to.be.a('string'); // Test tap format - const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getUsername()}`; + const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getRunEvalUsername()}`; const tapResult = execCmd(tapCommand, { ensureExitCode: 0 }); expect(tapResult.shellOutput.stdout).to.include('TAP version'); // Test junit format - const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getUsername()}`; + const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getRunEvalUsername()}`; const junitResult = execCmd(junitCommand, { ensureExitCode: 0 }); expect(junitResult.shellOutput.stdout).to.include(' { it('should fail with invalid JSON payload', async () => { const invalidJson = join(mockProjectDir, 'invalid-payload.json'); - const command = `agent test run-eval --spec ${invalidJson} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${invalidJson} --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); @@ -157,7 +157,7 @@ describe('agent test run-eval', function () { }); it('should fail when agent not found', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name NonExistentAgent --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name NonExistentAgent --target-org ${getRunEvalUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); @@ -169,7 +169,7 @@ describe('agent test run-eval', function () { }); it('should require --spec flag', async () => { - const command = `agent test run-eval --api-name Local_Info_Agent --target-org ${getUsername()} --json`; + const command = `agent test run-eval --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); diff --git a/test/nuts/shared-setup.ts b/test/nuts/shared-setup.ts index f3b9176c..97c0b67d 100644 --- a/test/nuts/shared-setup.ts +++ b/test/nuts/shared-setup.ts @@ -17,7 +17,7 @@ import { join } from 'node:path'; import { Duration, TestSession, execCmd } from '@salesforce/cli-plugins-testkit'; import { ComponentSetBuilder, RequestStatus, type ScopedPostDeploy } from '@salesforce/source-deploy-retrieve'; -import { Org, SfError, User, Lifecycle, Connection } from '@salesforce/core'; +import { Org, SfError, User, DefaultUserFields, Lifecycle, Connection } from '@salesforce/core'; import { sleep, ensureArray } from '@salesforce/kit'; /* eslint-disable no-console */ @@ -26,6 +26,9 @@ import { sleep, ensureArray } from '@salesforce/kit'; let testSession: TestSession | undefined; let testSessionPromise: Promise | undefined; let agentUsername: string | undefined; +// Dedicated user for the `agent test run-eval` NUT, isolating its OAuth grant/session churn +// from the shared admin so the full suite's load doesn't trip the per-user login-rate cap. +let runEvalUsername: string | undefined; // Non-sensitive scratch-org identifiers captured at setup for CI failure correlation. let orgId: string | undefined; let orgPod: string | undefined; @@ -129,6 +132,30 @@ export async function getTestSession(): Promise { await user.assignPermissionSets(queryResult.Id, ['EinsteinGPTPromptTemplateManager']); console.log(`Permission set assigned to scratch org user: ${queryResult.Name}`); + // Create a dedicated user (in this same scratch org) for the `agent test run-eval` NUT. + // Running that heavy NUT's commands against a separate user isolates its OAuth + // grant/session churn from the shared admin, so the full suite's load no longer evicts + // the admin's sessions and trips the per-user login-rate cap. Mirror the admin's access: + // System Administrator profile + the same permission set, so run-eval behaves identically. + console.log('Creating dedicated run-eval user...'); + const adminUsername = defaultOrg.username; + const runEvalFields = (await DefaultUserFields.create({ templateUser: adminUsername })).getFields(); + // DefaultUserFields defaults to the Standard User profile; override to System Administrator + // so the dedicated user has admin-equivalent access for run-eval. + const sysAdminProfile = await connection.singleRecordQuery<{ Id: string }>( + "SELECT Id FROM Profile WHERE Name='System Administrator'" + ); + runEvalFields.profileId = sysAdminProfile.Id; + const runEvalUser = await User.create({ org }); + const runEvalAuthInfo = await runEvalUser.createUser(runEvalFields); + runEvalUsername = runEvalAuthInfo.getUsername(); + const runEvalUserId = runEvalAuthInfo.getFields().userId; + if (!runEvalUsername || !runEvalUserId) { + throw new Error('Failed to create run-eval user: no username/id returned'); + } + await runEvalUser.assignPermissionSets(runEvalUserId, ['EinsteinGPTPromptTemplateManager']); + console.log(`[NUT-CONTEXT] runEvalUsername=${runEvalUsername}`); + // Create agent user using the new CLI command console.log('Creating agent user...'); const agentUserResult = execCmd<{ username: string }>( @@ -280,6 +307,22 @@ export function getUsername(): string { return testSession.orgs.get('default')!.username!; } +/** + * Gets the dedicated run-eval user's username. This user lives in the same shared scratch org + * and is created once during getTestSession(), so that `agent test run-eval`'s OAuth grant/session + * churn is isolated from the shared admin user. + * Throws an error if the session (and thus the user) hasn't been created yet. + */ +export function getRunEvalUsername(): string { + if (!testSession) { + throw new Error('Test session not available. Call getTestSession() first.'); + } + if (!runEvalUsername) { + throw new Error('Run-eval user not available. Call getTestSession() first.'); + } + return runEvalUsername; +} + /** * Gets the agent user, username, from the shared test session. * Throws an error if the session hasn't been created yet. From 07e9b2bc982f5ebfac92e63e023d0bb3b737d388 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Wed, 21 Oct 2026 14:21:18 -0700 Subject: [PATCH 4/9] test(nuts): revert dedicated run-eval user; add token-safe auth trace [do-not-merge] Dedicated-user experiment (c2bb5f9) did not clear login-rate: run-eval's first grant on a brand-new user still tripped the cap, proving it is not per-user (org- or IP-wide). Revert that change and instead run the run-eval JSON hook under SF_LOG_LEVEL=trace, dumping the token-redacted OAuth-exchange lines to CI to capture the rate-limit response's HTTP status / endpoint / Retry-After and settle per-org vs per-IP. --- test/nuts/agent.test.run-eval.nut.ts | 40 ++++++++----- test/nuts/shared-setup.ts | 88 +++++++++++++++------------- 2 files changed, 71 insertions(+), 57 deletions(-) diff --git a/test/nuts/agent.test.run-eval.nut.ts b/test/nuts/agent.test.run-eval.nut.ts index c6569bae..63080acf 100644 --- a/test/nuts/agent.test.run-eval.nut.ts +++ b/test/nuts/agent.test.run-eval.nut.ts @@ -18,7 +18,7 @@ import { join } from 'node:path'; import { expect } from 'chai'; import { execCmd } from '@salesforce/cli-plugins-testkit'; import type { RunEvalResult } from '../../src/commands/agent/test/run-eval.js'; -import { getTestSession, getRunEvalUsername } from './shared-setup.js'; +import { getTestSession, getUsername, dumpCliAuthTrace } from './shared-setup.js'; describe('agent test run-eval', function () { // Increase timeout for setup since shared setup includes long waits and deployments @@ -39,10 +39,20 @@ describe('agent test run-eval', function () { this.timeout(30 * 60 * 1000); // 30 minutes for setup await getTestSession(); - const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; - const jsonExec = execCmd(jsonCommand, { ensureExitCode: 0 }); - expect(jsonExec.jsonOutput?.result).to.be.ok; - jsonResult = jsonExec.jsonOutput!.result; + const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; + // DIAGNOSTIC (do-not-merge): run under SF_LOG_LEVEL=trace so the CLI's own (token-redacted) + // log records the OAuth token exchange; dump the relevant lines regardless of pass/fail so CI + // shows the `login rate exceeded` response's HTTP status/endpoint/Retry-After (per-org vs per-IP). + try { + const jsonExec = execCmd(jsonCommand, { + ensureExitCode: 0, + env: { ...process.env, SF_LOG_LEVEL: 'trace' }, + }); + expect(jsonExec.jsonOutput?.result).to.be.ok; + jsonResult = jsonExec.jsonOutput!.result; + } finally { + dumpCliAuthTrace('run-eval-json-before'); + } }); describe('run-eval with JSON file', () => { @@ -71,7 +81,7 @@ describe('agent test run-eval', function () { let yamlResult: RunEvalResult; before(() => { - const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; const output = execCmd(command, { ensureExitCode: 0 }); expect(output.jsonOutput?.result).to.be.ok; yamlResult = output.jsonOutput!.result; @@ -91,7 +101,7 @@ describe('agent test run-eval', function () { }); it('should handle YAML spec with contextVariables', async () => { - const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getUsername()} --json`; const result = execCmd(command, { ensureExitCode: 0 }); // Verify the command succeeds with contextVariables @@ -104,7 +114,7 @@ describe('agent test run-eval', function () { describe('run-eval with flags', () => { it('should respect --no-normalize flag', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getUsername()} --json`; const result = execCmd(command, { ensureExitCode: 1 }); expect(result.jsonOutput?.result).to.be.ok; @@ -112,7 +122,7 @@ describe('agent test run-eval', function () { }); it('should use custom batch size', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getUsername()} --json`; const result = execCmd(command, { ensureExitCode: 0 }); expect(result.jsonOutput?.result).to.be.ok; @@ -121,20 +131,20 @@ describe('agent test run-eval', function () { it('should support different result formats', async () => { // Test human format (default) - const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getRunEvalUsername()}`; + const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getUsername()}`; const humanResult = execCmd(humanCommand, { ensureExitCode: 0 }); expect(humanResult.shellOutput.stdout).to.be.ok; expect(humanResult.shellOutput.stdout).to.be.a('string'); // Test tap format - const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getRunEvalUsername()}`; + const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getUsername()}`; const tapResult = execCmd(tapCommand, { ensureExitCode: 0 }); expect(tapResult.shellOutput.stdout).to.include('TAP version'); // Test junit format - const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getRunEvalUsername()}`; + const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getUsername()}`; const junitResult = execCmd(junitCommand, { ensureExitCode: 0 }); expect(junitResult.shellOutput.stdout).to.include(' { it('should fail with invalid JSON payload', async () => { const invalidJson = join(mockProjectDir, 'invalid-payload.json'); - const command = `agent test run-eval --spec ${invalidJson} --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${invalidJson} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); @@ -157,7 +167,7 @@ describe('agent test run-eval', function () { }); it('should fail when agent not found', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name NonExistentAgent --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name NonExistentAgent --target-org ${getUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); @@ -169,7 +179,7 @@ describe('agent test run-eval', function () { }); it('should require --spec flag', async () => { - const command = `agent test run-eval --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --api-name Local_Info_Agent --target-org ${getUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); diff --git a/test/nuts/shared-setup.ts b/test/nuts/shared-setup.ts index 97c0b67d..dfed61bf 100644 --- a/test/nuts/shared-setup.ts +++ b/test/nuts/shared-setup.ts @@ -15,9 +15,10 @@ */ import { join } from 'node:path'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; import { Duration, TestSession, execCmd } from '@salesforce/cli-plugins-testkit'; import { ComponentSetBuilder, RequestStatus, type ScopedPostDeploy } from '@salesforce/source-deploy-retrieve'; -import { Org, SfError, User, DefaultUserFields, Lifecycle, Connection } from '@salesforce/core'; +import { Org, SfError, User, Lifecycle, Connection } from '@salesforce/core'; import { sleep, ensureArray } from '@salesforce/kit'; /* eslint-disable no-console */ @@ -26,9 +27,6 @@ import { sleep, ensureArray } from '@salesforce/kit'; let testSession: TestSession | undefined; let testSessionPromise: Promise | undefined; let agentUsername: string | undefined; -// Dedicated user for the `agent test run-eval` NUT, isolating its OAuth grant/session churn -// from the shared admin so the full suite's load doesn't trip the per-user login-rate cap. -let runEvalUsername: string | undefined; // Non-sensitive scratch-org identifiers captured at setup for CI failure correlation. let orgId: string | undefined; let orgPod: string | undefined; @@ -132,30 +130,6 @@ export async function getTestSession(): Promise { await user.assignPermissionSets(queryResult.Id, ['EinsteinGPTPromptTemplateManager']); console.log(`Permission set assigned to scratch org user: ${queryResult.Name}`); - // Create a dedicated user (in this same scratch org) for the `agent test run-eval` NUT. - // Running that heavy NUT's commands against a separate user isolates its OAuth - // grant/session churn from the shared admin, so the full suite's load no longer evicts - // the admin's sessions and trips the per-user login-rate cap. Mirror the admin's access: - // System Administrator profile + the same permission set, so run-eval behaves identically. - console.log('Creating dedicated run-eval user...'); - const adminUsername = defaultOrg.username; - const runEvalFields = (await DefaultUserFields.create({ templateUser: adminUsername })).getFields(); - // DefaultUserFields defaults to the Standard User profile; override to System Administrator - // so the dedicated user has admin-equivalent access for run-eval. - const sysAdminProfile = await connection.singleRecordQuery<{ Id: string }>( - "SELECT Id FROM Profile WHERE Name='System Administrator'" - ); - runEvalFields.profileId = sysAdminProfile.Id; - const runEvalUser = await User.create({ org }); - const runEvalAuthInfo = await runEvalUser.createUser(runEvalFields); - runEvalUsername = runEvalAuthInfo.getUsername(); - const runEvalUserId = runEvalAuthInfo.getFields().userId; - if (!runEvalUsername || !runEvalUserId) { - throw new Error('Failed to create run-eval user: no username/id returned'); - } - await runEvalUser.assignPermissionSets(runEvalUserId, ['EinsteinGPTPromptTemplateManager']); - console.log(`[NUT-CONTEXT] runEvalUsername=${runEvalUsername}`); - // Create agent user using the new CLI command console.log('Creating agent user...'); const agentUserResult = execCmd<{ username: string }>( @@ -308,30 +282,60 @@ export function getUsername(): string { } /** - * Gets the dedicated run-eval user's username. This user lives in the same shared scratch org - * and is created once during getTestSession(), so that `agent test run-eval`'s OAuth grant/session - * churn is isolated from the shared admin user. - * Throws an error if the session (and thus the user) hasn't been created yet. + * Gets the agent user, username, from the shared test session. + * Throws an error if the session hasn't been created yet. */ -export function getRunEvalUsername(): string { +export function getAgentUsername(): string | undefined { if (!testSession) { throw new Error('Test session not available. Call getTestSession() first.'); } - if (!runEvalUsername) { - throw new Error('Run-eval user not available. Call getTestSession() first.'); - } - return runEvalUsername; + return agentUsername; } /** - * Gets the agent user, username, from the shared test session. - * Throws an error if the session hasn't been created yet. + * DIAGNOSTIC (do-not-merge): dump the token-redacted lines of the sf CLI's own trace log that + * relate to the OAuth token exchange / login-rate response, so CI can show the HTTP status, + * endpoint, and any Retry-After for a `login rate exceeded` failure — settling whether the cap + * is per-org or per-IP. Uses SF_LOG_LEVEL=trace's log file (which @salesforce/core redacts of + * access/refresh tokens), NOT jsforce DEBUG (which would leak raw tokens into public CI logs). */ -export function getAgentUsername(): string | undefined { +export function dumpCliAuthTrace(label: string): void { if (!testSession) { - throw new Error('Test session not available. Call getTestSession() first.'); + console.log(`[AUTH-TRACE ${label}] no test session; nothing to dump`); + return; + } + // The testkit child shares HOME with the session; sf writes its log under /.sf. + const homes = Array.from(new Set([testSession.homeDir, testSession.dir].filter(Boolean))) as string[]; + const keyword = /(oauth2\/token|login rate|retry-after|invalid_grant|statusCode|refresh token|services\/oauth2)/i; + let printedAny = false; + for (const home of homes) { + const sfDir = join(home, '.sf'); + if (!existsSync(sfDir)) { + continue; + } + for (const logFile of readdirSync(sfDir).filter((f) => f.endsWith('.log'))) { + const full = join(sfDir, logFile); + try { + const matches = readFileSync(full, 'utf8') + .split('\n') + .filter((l) => keyword.test(l)); + if (matches.length) { + printedAny = true; + console.log(`[AUTH-TRACE ${label}] ${full}: ${matches.length} matching line(s), last 40:`); + matches.slice(-40).forEach((l) => console.log(`[AUTH-TRACE] ${l}`)); + } + } catch { + // ignore unreadable/rotated log files + } + } + } + if (!printedAny) { + console.log( + `[AUTH-TRACE ${label}] no matching trace lines found under ${homes + .map((h) => join(h, '.sf')) + .join(', ')} (log path may differ; checked *.log)` + ); } - return agentUsername; } // Root-level hook: on any failing test that used the shared scratch org, emit orgId + pod + From 9883810a8d327f2f08d3617c1e0bf1da28573b1e Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Wed, 21 Oct 2026 15:50:41 -0700 Subject: [PATCH 5/9] test(nuts): fix lint in auth-trace helper (unnecessary type assertion) [do-not-merge] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace Array.from(...) as string[] with a type-guard filter so @typescript-eslint/no-unnecessary-type-assertion passes. (Local hooks had missed it — stale node_modules was missing @eslint-react; reinstalled.) --- test/nuts/shared-setup.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nuts/shared-setup.ts b/test/nuts/shared-setup.ts index dfed61bf..49d6277a 100644 --- a/test/nuts/shared-setup.ts +++ b/test/nuts/shared-setup.ts @@ -305,7 +305,7 @@ export function dumpCliAuthTrace(label: string): void { return; } // The testkit child shares HOME with the session; sf writes its log under /.sf. - const homes = Array.from(new Set([testSession.homeDir, testSession.dir].filter(Boolean))) as string[]; + const homes = [...new Set([testSession.homeDir, testSession.dir].filter((d): d is string => Boolean(d)))]; const keyword = /(oauth2\/token|login rate|retry-after|invalid_grant|statusCode|refresh token|services\/oauth2)/i; let printedAny = false; for (const home of homes) { From 860baa8d542a131624a416f5e7f4230e366e680f Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Thu, 22 Oct 2026 13:59:00 -0700 Subject: [PATCH 6/9] test(nuts): ride out login-rate with backoff on run-eval hooks [do-not-merge] Replace the do-not-merge SF_LOG_LEVEL=trace diagnostic with a login-rate retry helper (execCmdWithLoginRateBackoff) and apply it to the run-eval JSON and YAML before-hooks. The shared scratch org's per-org/per-runner login-rate budget is spent by setup + other NUTs before run-eval runs; retrying with backoff (up to 4 attempts, ~6 min total wait) lets the window recover during the serial waits. Hook timeouts raised to fit the backoff plus the ~18-min eval run. --- test/nuts/agent.test.run-eval.nut.ts | 33 +++++------- test/nuts/shared-setup.ts | 77 ++++++++++++++-------------- 2 files changed, 52 insertions(+), 58 deletions(-) diff --git a/test/nuts/agent.test.run-eval.nut.ts b/test/nuts/agent.test.run-eval.nut.ts index 63080acf..a85168f7 100644 --- a/test/nuts/agent.test.run-eval.nut.ts +++ b/test/nuts/agent.test.run-eval.nut.ts @@ -18,11 +18,13 @@ import { join } from 'node:path'; import { expect } from 'chai'; import { execCmd } from '@salesforce/cli-plugins-testkit'; import type { RunEvalResult } from '../../src/commands/agent/test/run-eval.js'; -import { getTestSession, getUsername, dumpCliAuthTrace } from './shared-setup.js'; +import { getTestSession, getUsername, execCmdWithLoginRateBackoff } from './shared-setup.js'; describe('agent test run-eval', function () { - // Increase timeout for setup since shared setup includes long waits and deployments - this.timeout(30 * 60 * 1000); // 30 minutes + // Increase timeout for setup since shared setup includes long waits and deployments, plus the + // run-eval before-hook rides out the `login rate exceeded` window with backoff (up to ~6 min of + // waiting) before an ~18-min eval run. + this.timeout(60 * 60 * 1000); // 60 minutes const mockProjectDir = join(process.cwd(), 'test', 'mock-projects', 'agent-generate-template', 'specs'); const jsonPayloadPath = join(mockProjectDir, 'eval-payload.json'); @@ -36,23 +38,15 @@ describe('agent test run-eval', function () { let jsonResult: RunEvalResult; before(async function () { - this.timeout(30 * 60 * 1000); // 30 minutes for setup + this.timeout(60 * 60 * 1000); // 60 minutes: setup + login-rate backoff + the eval run await getTestSession(); const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - // DIAGNOSTIC (do-not-merge): run under SF_LOG_LEVEL=trace so the CLI's own (token-redacted) - // log records the OAuth token exchange; dump the relevant lines regardless of pass/fail so CI - // shows the `login rate exceeded` response's HTTP status/endpoint/Retry-After (per-org vs per-IP). - try { - const jsonExec = execCmd(jsonCommand, { - ensureExitCode: 0, - env: { ...process.env, SF_LOG_LEVEL: 'trace' }, - }); - expect(jsonExec.jsonOutput?.result).to.be.ok; - jsonResult = jsonExec.jsonOutput!.result; - } finally { - dumpCliAuthTrace('run-eval-json-before'); - } + // Retry with backoff to ride out the transient `login rate exceeded` window that the shared + // org's auth budget trips when run-eval's first grant fires after suite setup. + const jsonExec = await execCmdWithLoginRateBackoff(jsonCommand); + expect(jsonExec.jsonOutput?.result).to.be.ok; + jsonResult = jsonExec.jsonOutput!.result; }); describe('run-eval with JSON file', () => { @@ -80,9 +74,10 @@ describe('agent test run-eval', function () { // for the same OAuth-rate-limit reason as the JSON-payload run above. let yamlResult: RunEvalResult; - before(() => { + before(async function () { + this.timeout(30 * 60 * 1000); // 30 minutes: login-rate backoff + the eval run const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; - const output = execCmd(command, { ensureExitCode: 0 }); + const output = await execCmdWithLoginRateBackoff(command); expect(output.jsonOutput?.result).to.be.ok; yamlResult = output.jsonOutput!.result; }); diff --git a/test/nuts/shared-setup.ts b/test/nuts/shared-setup.ts index 49d6277a..362e835e 100644 --- a/test/nuts/shared-setup.ts +++ b/test/nuts/shared-setup.ts @@ -15,8 +15,7 @@ */ import { join } from 'node:path'; -import { existsSync, readdirSync, readFileSync } from 'node:fs'; -import { Duration, TestSession, execCmd } from '@salesforce/cli-plugins-testkit'; +import { Duration, TestSession, execCmd, type ExecCmdResult } from '@salesforce/cli-plugins-testkit'; import { ComponentSetBuilder, RequestStatus, type ScopedPostDeploy } from '@salesforce/source-deploy-retrieve'; import { Org, SfError, User, Lifecycle, Connection } from '@salesforce/core'; import { sleep, ensureArray } from '@salesforce/kit'; @@ -292,50 +291,50 @@ export function getAgentUsername(): string | undefined { return agentUsername; } +/** Matches the transient Salesforce OAuth "login rate exceeded" error returned during a token grant. */ +const LOGIN_RATE_RE = /login rate exceeded/i; + /** - * DIAGNOSTIC (do-not-merge): dump the token-redacted lines of the sf CLI's own trace log that - * relate to the OAuth token exchange / login-rate response, so CI can show the HTTP status, - * endpoint, and any Retry-After for a `login rate exceeded` failure — settling whether the cap - * is per-org or per-IP. Uses SF_LOG_LEVEL=trace's log file (which @salesforce/core redacts of - * access/refresh tokens), NOT jsforce DEBUG (which would leak raw tokens into public CI logs). + * Execute a command that is prone to the transient `login rate exceeded` OAuth error, retrying + * with backoff to ride out the (short) rate-limit window. Each attempt runs a fresh `execCmd`; + * on a login-rate failure we wait and retry, on a clean exit we return the result, and on any + * other non-zero exit we fail fast with the command output. Throws once `maxAttempts` login-rate + * failures are exhausted. + * + * This exists because the single shared scratch org's per-org/per-runner login-rate budget is + * spent by suite setup + the other NUTs before the run-eval describe runs; retrying lets the + * window recover (nothing else runs during these serial waits) instead of failing the describe. */ -export function dumpCliAuthTrace(label: string): void { - if (!testSession) { - console.log(`[AUTH-TRACE ${label}] no test session; nothing to dump`); - return; - } - // The testkit child shares HOME with the session; sf writes its log under /.sf. - const homes = [...new Set([testSession.homeDir, testSession.dir].filter((d): d is string => Boolean(d)))]; - const keyword = /(oauth2\/token|login rate|retry-after|invalid_grant|statusCode|refresh token|services\/oauth2)/i; - let printedAny = false; - for (const home of homes) { - const sfDir = join(home, '.sf'); - if (!existsSync(sfDir)) { - continue; - } - for (const logFile of readdirSync(sfDir).filter((f) => f.endsWith('.log'))) { - const full = join(sfDir, logFile); - try { - const matches = readFileSync(full, 'utf8') - .split('\n') - .filter((l) => keyword.test(l)); - if (matches.length) { - printedAny = true; - console.log(`[AUTH-TRACE ${label}] ${full}: ${matches.length} matching line(s), last 40:`); - matches.slice(-40).forEach((l) => console.log(`[AUTH-TRACE] ${l}`)); - } - } catch { - // ignore unreadable/rotated log files +export async function execCmdWithLoginRateBackoff(command: string, maxAttempts = 4): Promise> { + // Waits between attempts (attempt 1->2, 2->3, 3->4); worst case ~6 min of waiting total. + const delays = [Duration.seconds(60), Duration.seconds(120), Duration.seconds(180)]; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const exec = execCmd(command); + if (exec.shellOutput.code === 0) { + if (attempt > 1) { + console.log(`[LOGIN-RATE-BACKOFF] succeeded on attempt ${attempt}/${maxAttempts}: ${command}`); } + return exec; } - } - if (!printedAny) { + const output = `${exec.shellOutput.stdout ?? ''}\n${exec.shellOutput.stderr ?? ''}\n${JSON.stringify( + exec.jsonOutput ?? {} + )}`; + const isLoginRate = LOGIN_RATE_RE.test(output); + if (!isLoginRate || attempt === maxAttempts) { + throw new Error( + `Command failed (exit ${exec.shellOutput.code}) after ${attempt} attempt(s)${ + isLoginRate ? ' — login rate exceeded never cleared' : '' + }: ${command}\n${output}` + ); + } + const delay = delays[Math.min(attempt - 1, delays.length - 1)]; console.log( - `[AUTH-TRACE ${label}] no matching trace lines found under ${homes - .map((h) => join(h, '.sf')) - .join(', ')} (log path may differ; checked *.log)` + `[LOGIN-RATE-BACKOFF] login rate exceeded on attempt ${attempt}/${maxAttempts}; waiting ${delay.seconds}s before retry: ${command}` ); + await sleep(delay); // eslint-disable-line no-await-in-loop } + // Unreachable: the loop either returns on success or throws on the final attempt. + throw new Error(`Command failed after ${maxAttempts} attempts: ${command}`); } // Root-level hook: on any failing test that used the shared scratch org, emit orgId + pod + From 283d6fb3aef9501453e5165a145c22d722d0a0d4 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Fri, 23 Oct 2026 10:08:03 -0700 Subject: [PATCH 7/9] test(nuts): fix login-rate backoff never waiting (Duration identity) [do-not-merge] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The backoff passed a testkit-re-exported Duration to @salesforce/kit's sleep(), whose `instanceof Duration` check missed (different class copy), producing a NaN timeout that resolved instantly — so all 4 attempts fired within ~6s and the window never had time to clear. Pass plain millisecond numbers to sleep() instead. --- test/nuts/shared-setup.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/test/nuts/shared-setup.ts b/test/nuts/shared-setup.ts index 362e835e..9c6f6a1a 100644 --- a/test/nuts/shared-setup.ts +++ b/test/nuts/shared-setup.ts @@ -306,8 +306,11 @@ const LOGIN_RATE_RE = /login rate exceeded/i; * window recover (nothing else runs during these serial waits) instead of failing the describe. */ export async function execCmdWithLoginRateBackoff(command: string, maxAttempts = 4): Promise> { - // Waits between attempts (attempt 1->2, 2->3, 3->4); worst case ~6 min of waiting total. - const delays = [Duration.seconds(60), Duration.seconds(120), Duration.seconds(180)]; + // Wait (in ms) between attempts (attempt 1->2, 2->3, 3->4); worst case ~6 min of waiting total. + // NOTE: pass a plain number to `sleep`, NOT a `Duration` — the `Duration` re-exported by testkit + // is a different class instance than @salesforce/kit's, so `sleep`'s `instanceof Duration` check + // misses, yielding a NaN timeout that resolves instantly (the bug that made the first run not wait). + const delaysMs = [60_000, 120_000, 180_000]; for (let attempt = 1; attempt <= maxAttempts; attempt++) { const exec = execCmd(command); if (exec.shellOutput.code === 0) { @@ -327,11 +330,13 @@ export async function execCmdWithLoginRateBackoff(command: string, maxAttempt }: ${command}\n${output}` ); } - const delay = delays[Math.min(attempt - 1, delays.length - 1)]; + const delayMs = delaysMs[Math.min(attempt - 1, delaysMs.length - 1)]; console.log( - `[LOGIN-RATE-BACKOFF] login rate exceeded on attempt ${attempt}/${maxAttempts}; waiting ${delay.seconds}s before retry: ${command}` + `[LOGIN-RATE-BACKOFF] login rate exceeded on attempt ${attempt}/${maxAttempts}; waiting ${ + delayMs / 1000 + }s before retry: ${command}` ); - await sleep(delay); // eslint-disable-line no-await-in-loop + await sleep(delayMs); // eslint-disable-line no-await-in-loop } // Unreachable: the loop either returns on success or throws on the final attempt. throw new Error(`Command failed after ${maxAttempts} attempts: ${command}`); From 2bc5f4ce5e1b7d04eff95a2e0af340dd55cde301 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Fri, 23 Oct 2026 13:25:15 -0700 Subject: [PATCH 8/9] test(nuts): give run-eval its own scratch org (per-org isolation probe) [do-not-merge] Revert the login-rate backoff (CI run 39992637604 proved it: the Duration identity fix made sleep actually wait, but 6 min of backoff never cleared the window, and it pushed the suite into the 60-min job timeout). Instead, isolate run-eval on a DEDICATED TestSession/scratch org: - extract org provisioning into reusable provisionEvalOrg(session) - add getRunEvalTestSession()/getRunEvalUsername(); restore AGENT_USER_USERNAME to the shared org's agent user afterward so z2 publish isn't contaminated - rewire agent.test.run-eval.nut.ts to the dedicated org Discriminates per-ORG vs per-IP: if run-eval's first grant succeeds on a fresh org, the cap is per-org; if it still trips login rate exceeded, it's per-IP. --- test/nuts/agent.test.run-eval.nut.ts | 50 ++- test/nuts/shared-setup.ts | 437 +++++++++++++++------------ 2 files changed, 267 insertions(+), 220 deletions(-) diff --git a/test/nuts/agent.test.run-eval.nut.ts b/test/nuts/agent.test.run-eval.nut.ts index a85168f7..e0a3e63c 100644 --- a/test/nuts/agent.test.run-eval.nut.ts +++ b/test/nuts/agent.test.run-eval.nut.ts @@ -18,12 +18,12 @@ import { join } from 'node:path'; import { expect } from 'chai'; import { execCmd } from '@salesforce/cli-plugins-testkit'; import type { RunEvalResult } from '../../src/commands/agent/test/run-eval.js'; -import { getTestSession, getUsername, execCmdWithLoginRateBackoff } from './shared-setup.js'; +import { getRunEvalTestSession, getRunEvalUsername } from './shared-setup.js'; describe('agent test run-eval', function () { - // Increase timeout for setup since shared setup includes long waits and deployments, plus the - // run-eval before-hook rides out the `login rate exceeded` window with backoff (up to ~6 min of - // waiting) before an ~18-min eval run. + // run-eval gets its OWN dedicated scratch org (see shared-setup.ts, per-org isolation + // experiment), so this before-hook provisions a fresh org (~20-25 min: create + 2 deploys + + // Einstein wait) and then runs the ~18-min eval. Generous suite timeout accordingly. this.timeout(60 * 60 * 1000); // 60 minutes const mockProjectDir = join(process.cwd(), 'test', 'mock-projects', 'agent-generate-template', 'specs'); @@ -31,20 +31,18 @@ describe('agent test run-eval', function () { const yamlSpecPath = join(mockProjectDir, 'eval-test-spec.yaml'); const yamlWithContextPath = join(mockProjectDir, 'eval-with-context.yaml'); - // The JSON-payload run-eval is expensive (~18 min) and hammers the shared admin user's - // OAuth refresh rate limit when invoked repeatedly. Run it exactly once here and share the - // parsed result across every test that asserts against it, so a rate-limit failure surfaces - // once rather than per-test. + // The JSON-payload run-eval is expensive (~18 min) and each invocation is a fresh OAuth token + // grant, so repeated invocations press the org's login-rate budget. Run it exactly once here + // and share the parsed result across every test that asserts against it, so a rate-limit + // failure surfaces once rather than per-test. let jsonResult: RunEvalResult; before(async function () { - this.timeout(60 * 60 * 1000); // 60 minutes: setup + login-rate backoff + the eval run - await getTestSession(); + this.timeout(60 * 60 * 1000); // 60 minutes: dedicated-org provisioning + the eval run + await getRunEvalTestSession(); - const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - // Retry with backoff to ride out the transient `login rate exceeded` window that the shared - // org's auth budget trips when run-eval's first grant fires after suite setup. - const jsonExec = await execCmdWithLoginRateBackoff(jsonCommand); + const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; + const jsonExec = execCmd(jsonCommand, { ensureExitCode: 0 }); expect(jsonExec.jsonOutput?.result).to.be.ok; jsonResult = jsonExec.jsonOutput!.result; }); @@ -75,9 +73,9 @@ describe('agent test run-eval', function () { let yamlResult: RunEvalResult; before(async function () { - this.timeout(30 * 60 * 1000); // 30 minutes: login-rate backoff + the eval run - const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; - const output = await execCmdWithLoginRateBackoff(command); + this.timeout(30 * 60 * 1000); // 30 minutes: the eval run (org already provisioned) + const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getRunEvalUsername()} --json`; + const output = execCmd(command, { ensureExitCode: 0 }); expect(output.jsonOutput?.result).to.be.ok; yamlResult = output.jsonOutput!.result; }); @@ -96,7 +94,7 @@ describe('agent test run-eval', function () { }); it('should handle YAML spec with contextVariables', async () => { - const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getRunEvalUsername()} --json`; const result = execCmd(command, { ensureExitCode: 0 }); // Verify the command succeeds with contextVariables @@ -109,7 +107,7 @@ describe('agent test run-eval', function () { describe('run-eval with flags', () => { it('should respect --no-normalize flag', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getRunEvalUsername()} --json`; const result = execCmd(command, { ensureExitCode: 1 }); expect(result.jsonOutput?.result).to.be.ok; @@ -117,7 +115,7 @@ describe('agent test run-eval', function () { }); it('should use custom batch size', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getRunEvalUsername()} --json`; const result = execCmd(command, { ensureExitCode: 0 }); expect(result.jsonOutput?.result).to.be.ok; @@ -126,20 +124,20 @@ describe('agent test run-eval', function () { it('should support different result formats', async () => { // Test human format (default) - const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getUsername()}`; + const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getRunEvalUsername()}`; const humanResult = execCmd(humanCommand, { ensureExitCode: 0 }); expect(humanResult.shellOutput.stdout).to.be.ok; expect(humanResult.shellOutput.stdout).to.be.a('string'); // Test tap format - const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getUsername()}`; + const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getRunEvalUsername()}`; const tapResult = execCmd(tapCommand, { ensureExitCode: 0 }); expect(tapResult.shellOutput.stdout).to.include('TAP version'); // Test junit format - const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getUsername()}`; + const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getRunEvalUsername()}`; const junitResult = execCmd(junitCommand, { ensureExitCode: 0 }); expect(junitResult.shellOutput.stdout).to.include(' { it('should fail with invalid JSON payload', async () => { const invalidJson = join(mockProjectDir, 'invalid-payload.json'); - const command = `agent test run-eval --spec ${invalidJson} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${invalidJson} --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); @@ -162,7 +160,7 @@ describe('agent test run-eval', function () { }); it('should fail when agent not found', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name NonExistentAgent --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name NonExistentAgent --target-org ${getRunEvalUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); @@ -174,7 +172,7 @@ describe('agent test run-eval', function () { }); it('should require --spec flag', async () => { - const command = `agent test run-eval --api-name Local_Info_Agent --target-org ${getUsername()} --json`; + const command = `agent test run-eval --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); diff --git a/test/nuts/shared-setup.ts b/test/nuts/shared-setup.ts index 9c6f6a1a..e14869a6 100644 --- a/test/nuts/shared-setup.ts +++ b/test/nuts/shared-setup.ts @@ -15,14 +15,14 @@ */ import { join } from 'node:path'; -import { Duration, TestSession, execCmd, type ExecCmdResult } from '@salesforce/cli-plugins-testkit'; +import { Duration, TestSession, execCmd } from '@salesforce/cli-plugins-testkit'; import { ComponentSetBuilder, RequestStatus, type ScopedPostDeploy } from '@salesforce/source-deploy-retrieve'; import { Org, SfError, User, Lifecycle, Connection } from '@salesforce/core'; import { sleep, ensureArray } from '@salesforce/kit'; /* eslint-disable no-console */ -// Module-level variables to ensure only one TestSession is created +// The shared TestSession + one scratch org used by the bulk of the suite. let testSession: TestSession | undefined; let testSessionPromise: Promise | undefined; let agentUsername: string | undefined; @@ -30,6 +30,20 @@ let agentUsername: string | undefined; let orgId: string | undefined; let orgPod: string | undefined; +// A DEDICATED TestSession + its own scratch org for the `agent test run-eval` NUTs. +// PER-ORG ISOLATION EXPERIMENT: run-eval's first OAuth token grant trips `login rate +// exceeded` because the shared org's (or the CI runner IP's) login-rate budget is already +// spent by the rest of the suite before run-eval runs. A dedicated USER in the SAME org did +// not help (the cap is not per-user). Giving run-eval its OWN scratch org tests whether the +// cap is per-ORG (a fresh org has its own budget → run-eval succeeds) or per-IP (all orgs +// share the runner IP → still fails). +let runEvalSession: TestSession | undefined; +let runEvalSessionPromise: Promise | undefined; + +// The deploy progress listeners attach to the global Lifecycle singleton, so register them +// exactly once even though two sessions get provisioned. +let deployListenersRegistered = false; + // Format a UTC timestamp as MM/DD/YYYY:HH:mm:ss for CI log correlation (Splunk-friendly). function fmtUtc(d: Date): string { const p = (n: number): string => String(n).padStart(2, '0'); @@ -65,6 +79,169 @@ async function waitForEinsteinReady(connection: Connection, maxAttempts = 30): P throw new Error(`Einstein AI did not become ready within ${timeoutSeconds} seconds timeout`); } +// Register the deploy progress logging once against the global Lifecycle singleton. The +// listeners reset their own progress counters on each `scopedPreDeploy`, so a single +// registration correctly handles the deploys of both the shared and the run-eval sessions. +function registerDeployListeners(): void { + if (deployListenersRegistered) { + return; + } + deployListenersRegistered = true; + + const lifecycle = Lifecycle.getInstance(); + let lastDeployed = 0; + let lastTotal = 0; + + lifecycle.on('scopedPreDeploy', () => { + console.log('[DEPLOY] Starting deployment...'); + // Reset progress tracking for new deployment + lastDeployed = 0; + lastTotal = 0; + return Promise.resolve(); + }); + + lifecycle.on('scopedPostDeploy', (result: ScopedPostDeploy) => { + const deployResult = result.deployResult.response; + const status = deployResult.status; + const numberComponentErrors = deployResult.numberComponentErrors ?? 0; + const numberComponentsDeployed = deployResult.numberComponentsDeployed ?? 0; + const numberComponentsTotal = deployResult.numberComponentsTotal ?? 0; + const numberTestErrors = deployResult.numberTestErrors ?? 0; + const numberTestsCompleted = deployResult.numberTestsCompleted ?? 0; + const numberTestsTotal = deployResult.numberTestsTotal ?? 0; + + // Log progress during polling (only if changed) + if ( + (numberComponentsDeployed !== lastDeployed || numberComponentsTotal !== lastTotal) && + numberComponentsTotal > 0 + ) { + console.log( + `[DEPLOY] Progress: ${numberComponentsDeployed}/${numberComponentsTotal} components deployed${ + numberComponentErrors > 0 ? `, ${numberComponentErrors} errors` : '' + }` + ); + lastDeployed = numberComponentsDeployed; + lastTotal = numberComponentsTotal; + } + + // Log final status when deployment is complete + const isComplete = + status === RequestStatus.Succeeded || status === RequestStatus.Failed || status === RequestStatus.Canceled; + if (isComplete) { + console.log(`[DEPLOY] Deployment completed - Status: ${status}`); + console.log( + `[DEPLOY] Components: ${numberComponentsDeployed}/${numberComponentsTotal} deployed, ${numberComponentErrors} errors` + ); + if (numberTestsTotal > 0) { + console.log( + `[DEPLOY] Tests: ${numberTestsCompleted}/${numberTestsTotal} completed, ${numberTestErrors} errors` + ); + } + const componentFailures = ensureArray(deployResult.details?.componentFailures); + if (componentFailures.length > 0) { + console.log(`[DEPLOY] Component failures: ${componentFailures.length}`); + componentFailures.slice(0, 5).forEach((failure, idx) => { + console.log( + `[DEPLOY] Failure ${idx + 1}: ${failure.fullName ?? 'unknown'} - ${failure.problemType ?? 'unknown'}: ${ + failure.problem ?? 'unknown' + }` + ); + }); + } + } + return Promise.resolve(); + }); +} + +/** + * Provision a freshly-created scratch org so it can run agent evaluations: assign the Einstein + * perm set to the admin, create the agent runtime user, wait for Einstein AI to come up, and + * deploy the agent metadata (first without, then with the AiEvaluationDefinition). Used for BOTH + * the shared session and the dedicated run-eval session. + * + * Side effect: sets `process.env.AGENT_USER_USERNAME` to THIS org's agent user, because the + * metadata deploys use it as an SDR `replaceWithEnv` token (see mock project sfdx-project.json). + * Any caller that provisions a second org must restore the env var afterward if a later deploy + * targets a different org. + */ +async function provisionEvalOrg( + session: TestSession +): Promise<{ username: string; agentUser: string; orgId: string; pod: string }> { + const defaultOrg = session.orgs.get('default'); + if (!defaultOrg?.username) { + throw new Error('No default scratch org found in the test session.'); + } + const username = defaultOrg.username; + console.log(`Using scratch org: ${username}`); + + const org = await Org.create({ aliasOrUsername: username }); + const connection = org.getConnection(); + + // Capture non-sensitive org identifiers for CI failure correlation (orgId + pod + UTC timestamp). + const resolvedOrgId = org.getOrgId(); + const orgInfo = await connection.singleRecordQuery<{ InstanceName: string }>('SELECT InstanceName FROM Organization'); + const pod = orgInfo.InstanceName; + console.log(`[NUT-CONTEXT] utc=${fmtUtc(new Date())} orgId=${resolvedOrgId} pod=${pod}`); + + // assign the EinsteinGPTPromptTemplateManager to the scratch org admin user + const queryResult = await connection.singleRecordQuery<{ Id: string; Name: string }>( + `SELECT Id, Name FROM User WHERE Username='${username}'` + ); + const user = await User.create({ org }); + await user.assignPermissionSets(queryResult.Id, ['EinsteinGPTPromptTemplateManager']); + console.log(`Permission set assigned to scratch org user: ${queryResult.Name}`); + + // Create agent user using the new CLI command + console.log('Creating agent user...'); + const agentUserResult = execCmd<{ username: string }>(`org create agent-user --target-org ${username} --json`, { + ensureExitCode: 0, + cli: 'sf', + }); + const agentUser = agentUserResult.jsonOutput?.result?.username; + if (!agentUser) { + throw new Error('Failed to create agent user: no username returned'); + } + console.log(`Agent user created: ${agentUser}`); + + // Wait for Einstein AI services to be ready + console.log('Waiting for Einstein AI services to be ready...'); + await waitForEinsteinReady(connection); + console.log('Einstein AI services are ready'); + + // Set environment variable for string replacement (SDR `replaceWithEnv`) used by the deploys. + process.env.AGENT_USER_USERNAME = agentUser; + process.env.SF_AAB_COMPILATION = 'false'; + + registerDeployListeners(); + + console.log('deploying metadata (no AiEvaluationDefinition)'); + const cs1 = await ComponentSetBuilder.build({ + manifest: { + manifestPath: join(session.project.dir, 'noTest.xml'), + directoryPaths: [session.homeDir], + }, + }); + const deploy1 = await cs1.deploy({ usernameOrConnection: username }); + // pollStatus waits until deployment completes - will throw if deployment fails + await deploy1.pollStatus({ frequency: Duration.seconds(10), timeout: Duration.minutes(30) }); + console.log('[DEPLOY] First deployment completed successfully'); + + console.log('deploying metadata (AiEvaluationDefinition)'); + const cs2 = await ComponentSetBuilder.build({ + manifest: { + manifestPath: join(session.project.dir, 'test.xml'), + directoryPaths: [session.homeDir], + }, + }); + const deploy2 = await cs2.deploy({ usernameOrConnection: username }); + // pollStatus waits until deployment completes - will throw if deployment fails + await deploy2.pollStatus({ frequency: Duration.seconds(10), timeout: Duration.minutes(30) }); + console.log('[DEPLOY] Second deployment completed successfully'); + console.log('[DEPLOY] All deployments complete, tests can now run'); + + return { username, agentUser, orgId: resolvedOrgId, pod }; +} + /** * Gets the shared TestSession with a scratch org. This ensures only one TestSession * is created per test run, even if called from multiple test files simultaneously. @@ -103,154 +280,10 @@ export async function getTestSession(): Promise { testSession = session; console.log('TestSession created successfully'); - // Get the scratch org username and assign permission set - const orgs = session.orgs; - const defaultOrg = orgs.get('default'); - - if (orgs && orgs.size > 0) { - if (defaultOrg?.username) { - console.log(`Using scratch org: ${defaultOrg.username}`); - const org = await Org.create({ aliasOrUsername: defaultOrg.username }); - const connection = org.getConnection(); - - // Capture non-sensitive org identifiers for CI failure correlation (orgId + pod + UTC timestamp). - orgId = org.getOrgId(); - const orgInfo = await connection.singleRecordQuery<{ InstanceName: string }>( - 'SELECT InstanceName FROM Organization' - ); - orgPod = orgInfo.InstanceName; - console.log(`[NUT-CONTEXT] utc=${fmtUtc(new Date())} orgId=${orgId} pod=${orgPod}`); - - // assign the EinsteinGPTPromptTemplateManager to the scratch org admin user - const queryResult = await connection.singleRecordQuery<{ Id: string; Name: string }>( - `SELECT Id, Name FROM User WHERE Username='${defaultOrg.username}'` - ); - const user = await User.create({ org }); - await user.assignPermissionSets(queryResult.Id, ['EinsteinGPTPromptTemplateManager']); - console.log(`Permission set assigned to scratch org user: ${queryResult.Name}`); - - // Create agent user using the new CLI command - console.log('Creating agent user...'); - const agentUserResult = execCmd<{ username: string }>( - `org create agent-user --target-org ${defaultOrg.username} --json`, - { ensureExitCode: 0, cli: 'sf' } - ); - - if (!agentUserResult.jsonOutput?.result?.username) { - throw new Error('Failed to create agent user: no username returned'); - } - - agentUsername = agentUserResult.jsonOutput.result.username; - console.log(`Agent user created: ${agentUsername}`); - - // Wait for Einstein AI services to be ready - console.log('Waiting for Einstein AI services to be ready...'); - await waitForEinsteinReady(connection); - console.log('Einstein AI services are ready'); - - // Set environment variable for string replacement - process.env.AGENT_USER_USERNAME = agentUsername; - process.env.SF_AAB_COMPILATION = 'false'; - - // Set up deploy event listeners to log progress - const lifecycle = Lifecycle.getInstance(); - let lastDeployed = 0; - let lastTotal = 0; - - lifecycle.on('scopedPreDeploy', () => { - console.log('[DEPLOY] Starting deployment...'); - // Reset progress tracking for new deployment - lastDeployed = 0; - lastTotal = 0; - return Promise.resolve(); - }); - - lifecycle.on('scopedPostDeploy', (result: ScopedPostDeploy) => { - const deployResult = result.deployResult.response; - const status = deployResult.status; - const numberComponentErrors = deployResult.numberComponentErrors ?? 0; - const numberComponentsDeployed = deployResult.numberComponentsDeployed ?? 0; - const numberComponentsTotal = deployResult.numberComponentsTotal ?? 0; - const numberTestErrors = deployResult.numberTestErrors ?? 0; - const numberTestsCompleted = deployResult.numberTestsCompleted ?? 0; - const numberTestsTotal = deployResult.numberTestsTotal ?? 0; - - // Log progress during polling (only if changed) - if ( - (numberComponentsDeployed !== lastDeployed || numberComponentsTotal !== lastTotal) && - numberComponentsTotal > 0 - ) { - console.log( - `[DEPLOY] Progress: ${numberComponentsDeployed}/${numberComponentsTotal} components deployed${ - numberComponentErrors > 0 ? `, ${numberComponentErrors} errors` : '' - }` - ); - lastDeployed = numberComponentsDeployed; - lastTotal = numberComponentsTotal; - } - - // Log final status when deployment is complete - const isComplete = - status === RequestStatus.Succeeded || - status === RequestStatus.Failed || - status === RequestStatus.Canceled; - if (isComplete) { - console.log(`[DEPLOY] Deployment completed - Status: ${status}`); - console.log( - `[DEPLOY] Components: ${numberComponentsDeployed}/${numberComponentsTotal} deployed, ${numberComponentErrors} errors` - ); - if (numberTestsTotal > 0) { - console.log( - `[DEPLOY] Tests: ${numberTestsCompleted}/${numberTestsTotal} completed, ${numberTestErrors} errors` - ); - } - const componentFailures = ensureArray(deployResult.details?.componentFailures); - if (componentFailures.length > 0) { - console.log(`[DEPLOY] Component failures: ${componentFailures.length}`); - componentFailures.slice(0, 5).forEach((failure, idx) => { - console.log( - `[DEPLOY] Failure ${idx + 1}: ${failure.fullName ?? 'unknown'} - ${ - failure.problemType ?? 'unknown' - }: ${failure.problem ?? 'unknown'}` - ); - }); - } - } - return Promise.resolve(); - }); - - console.log('deploying metadata (no AiEvaluationDefinition)'); - - const cs1 = await ComponentSetBuilder.build({ - manifest: { - manifestPath: join(testSession.project.dir, 'noTest.xml'), - directoryPaths: [testSession.homeDir], - }, - }); - const deploy1 = await cs1.deploy({ usernameOrConnection: defaultOrg.username }); - // pollStatus waits until deployment completes - will throw if deployment fails - await deploy1.pollStatus({ frequency: Duration.seconds(10), timeout: Duration.minutes(30) }); - console.log('[DEPLOY] First deployment completed successfully'); - - // Reset for second deployment - lastDeployed = 0; - lastTotal = 0; - - console.log('deploying metadata (AiEvaluationDefinition)'); - - const cs2 = await ComponentSetBuilder.build({ - manifest: { - manifestPath: join(testSession.project.dir, 'test.xml'), - directoryPaths: [testSession.homeDir], - }, - }); - const deploy2 = await cs2.deploy({ usernameOrConnection: defaultOrg.username }); - // pollStatus waits until deployment completes - will throw if deployment fails - await deploy2.pollStatus({ frequency: Duration.seconds(10), timeout: Duration.minutes(30) }); - console.log('[DEPLOY] Second deployment completed successfully'); - console.log('[DEPLOY] All deployments complete, tests can now run'); - } - } + const provisioned = await provisionEvalOrg(session); + agentUsername = provisioned.agentUser; + orgId = provisioned.orgId; + orgPod = provisioned.pod; return session; } catch (e) { @@ -291,55 +324,71 @@ export function getAgentUsername(): string | undefined { return agentUsername; } -/** Matches the transient Salesforce OAuth "login rate exceeded" error returned during a token grant. */ -const LOGIN_RATE_RE = /login rate exceeded/i; - /** - * Execute a command that is prone to the transient `login rate exceeded` OAuth error, retrying - * with backoff to ride out the (short) rate-limit window. Each attempt runs a fresh `execCmd`; - * on a login-rate failure we wait and retry, on a clean exit we return the result, and on any - * other non-zero exit we fail fast with the command output. Throws once `maxAttempts` login-rate - * failures are exhausted. - * - * This exists because the single shared scratch org's per-org/per-runner login-rate budget is - * spent by suite setup + the other NUTs before the run-eval describe runs; retrying lets the - * window recover (nothing else runs during these serial waits) instead of failing the describe. + * Gets the DEDICATED run-eval TestSession, creating its own separate scratch org (isolated from + * the shared org used by the rest of the suite). See the per-org isolation experiment note above. + * After provisioning, restores `process.env.AGENT_USER_USERNAME` to the shared org's agent user so + * later shared-org deploys (e.g. z2 publish of Willie_Resort_Manager.agent) are not contaminated + * with this org's agent user. */ -export async function execCmdWithLoginRateBackoff(command: string, maxAttempts = 4): Promise> { - // Wait (in ms) between attempts (attempt 1->2, 2->3, 3->4); worst case ~6 min of waiting total. - // NOTE: pass a plain number to `sleep`, NOT a `Duration` — the `Duration` re-exported by testkit - // is a different class instance than @salesforce/kit's, so `sleep`'s `instanceof Duration` check - // misses, yielding a NaN timeout that resolves instantly (the bug that made the first run not wait). - const delaysMs = [60_000, 120_000, 180_000]; - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - const exec = execCmd(command); - if (exec.shellOutput.code === 0) { - if (attempt > 1) { - console.log(`[LOGIN-RATE-BACKOFF] succeeded on attempt ${attempt}/${maxAttempts}: ${command}`); +export async function getRunEvalTestSession(): Promise { + if (runEvalSession) { + return runEvalSession; + } + if (runEvalSessionPromise) { + return runEvalSessionPromise; + } + + // @ts-expect-error it can throw an error + runEvalSessionPromise = (async (): Promise => { + try { + console.log('Creating DEDICATED run-eval TestSession with its own scratch org...'); + const session = await TestSession.create({ + project: { + sourceDir: join('test', 'mock-projects', 'agent-generate-template'), + }, + devhubAuthStrategy: 'AUTO', + scratchOrgs: [ + { + alias: 'default', + setDefault: true, + config: 'config/project-scratch-def.json', + }, + ], + }); + + runEvalSession = session; + console.log('run-eval TestSession created successfully'); + + const provisioned = await provisionEvalOrg(session); + console.log(`[NUT-CONTEXT] run-eval dedicated org username=${provisioned.username} orgId=${provisioned.orgId}`); + + // Restore the SDR replacement env var to the shared org's agent user, so later shared-org + // deploys are not contaminated by the run-eval org's agent user. + if (agentUsername) { + process.env.AGENT_USER_USERNAME = agentUsername; } - return exec; - } - const output = `${exec.shellOutput.stdout ?? ''}\n${exec.shellOutput.stderr ?? ''}\n${JSON.stringify( - exec.jsonOutput ?? {} - )}`; - const isLoginRate = LOGIN_RATE_RE.test(output); - if (!isLoginRate || attempt === maxAttempts) { - throw new Error( - `Command failed (exit ${exec.shellOutput.code}) after ${attempt} attempt(s)${ - isLoginRate ? ' — login rate exceeded never cleared' : '' - }: ${command}\n${output}` - ); + + return session; + } catch (e) { + console.log('XXXXXX RUN-EVAL SESSION ERROR XXXXXXX'); + console.dir(SfError.wrap(e).toObject()); + void Promise.reject(SfError.wrap(e)); } - const delayMs = delaysMs[Math.min(attempt - 1, delaysMs.length - 1)]; - console.log( - `[LOGIN-RATE-BACKOFF] login rate exceeded on attempt ${attempt}/${maxAttempts}; waiting ${ - delayMs / 1000 - }s before retry: ${command}` - ); - await sleep(delayMs); // eslint-disable-line no-await-in-loop + })(); + + return runEvalSessionPromise; +} + +/** + * Gets the scratch org username from the dedicated run-eval test session. + * Throws an error if the session hasn't been created yet. + */ +export function getRunEvalUsername(): string { + if (!runEvalSession) { + throw new Error('Run-eval test session not available. Call getRunEvalTestSession() first.'); } - // Unreachable: the loop either returns on success or throws on the final attempt. - throw new Error(`Command failed after ${maxAttempts} attempts: ${command}`); + return runEvalSession.orgs.get('default')!.username!; } // Root-level hook: on any failing test that used the shared scratch org, emit orgId + pod + From 2d7d42146ab86f821a8dee220d64798e618d14a4 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Fri, 23 Oct 2026 15:59:00 -0700 Subject: [PATCH 9/9] test(nuts): run-eval isolation via 2nd org in one session, not 2nd session [do-not-merge] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prior attempt (2bc5f4c) failed fast: a second TestSession.create() throws 'Attempted to wrap cwd which is already wrapped' and corrupts the first session (testkit wraps process.cwd() globally — one TestSession per process). Keep the per-org isolation experiment but change the mechanism: declare a second scratch org (alias 'runeval') in the SAME shared TestSession, provision it lazily, and point the run-eval NUTs at it. provisionEvalOrg is now parameterized by alias. Still discriminates per-ORG vs per-IP for the login-rate cap. --- test/nuts/shared-setup.ts | 121 +++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 68 deletions(-) diff --git a/test/nuts/shared-setup.ts b/test/nuts/shared-setup.ts index e14869a6..586fb8be 100644 --- a/test/nuts/shared-setup.ts +++ b/test/nuts/shared-setup.ts @@ -30,18 +30,23 @@ let agentUsername: string | undefined; let orgId: string | undefined; let orgPod: string | undefined; -// A DEDICATED TestSession + its own scratch org for the `agent test run-eval` NUTs. -// PER-ORG ISOLATION EXPERIMENT: run-eval's first OAuth token grant trips `login rate -// exceeded` because the shared org's (or the CI runner IP's) login-rate budget is already -// spent by the rest of the suite before run-eval runs. A dedicated USER in the SAME org did -// not help (the cap is not per-user). Giving run-eval its OWN scratch org tests whether the -// cap is per-ORG (a fresh org has its own budget → run-eval succeeds) or per-IP (all orgs +// A DEDICATED SECOND scratch org (alias `runeval`) in the SAME TestSession for the +// `agent test run-eval` NUTs. PER-ORG ISOLATION EXPERIMENT: run-eval's first OAuth token grant +// trips `login rate exceeded` because the shared org's (or the CI runner IP's) login-rate budget +// is already spent by the rest of the suite before run-eval runs. A dedicated USER in the SAME +// org did not help (the cap is not per-user). Giving run-eval its OWN scratch org tests whether +// the cap is per-ORG (a fresh org has its own budget → run-eval succeeds) or per-IP (all orgs // share the runner IP → still fails). -let runEvalSession: TestSession | undefined; -let runEvalSessionPromise: Promise | undefined; +// +// NOTE: this must be a second ORG in the one TestSession, NOT a second TestSession — testkit wraps +// process.cwd() globally and a second TestSession.create() throws "Attempted to wrap cwd which is +// already wrapped" and corrupts the first session. +const RUN_EVAL_ORG_ALIAS = 'runeval'; +let runEvalUsername: string | undefined; +let runEvalProvisionPromise: Promise | undefined; // The deploy progress listeners attach to the global Lifecycle singleton, so register them -// exactly once even though two sessions get provisioned. +// exactly once even though two orgs get provisioned. let deployListenersRegistered = false; // Format a UTC timestamp as MM/DD/YYYY:HH:mm:ss for CI log correlation (Splunk-friendly). @@ -165,14 +170,15 @@ function registerDeployListeners(): void { * targets a different org. */ async function provisionEvalOrg( - session: TestSession + session: TestSession, + alias = 'default' ): Promise<{ username: string; agentUser: string; orgId: string; pod: string }> { - const defaultOrg = session.orgs.get('default'); - if (!defaultOrg?.username) { - throw new Error('No default scratch org found in the test session.'); + const scratchOrg = session.orgs.get(alias); + if (!scratchOrg?.username) { + throw new Error(`No '${alias}' scratch org found in the test session.`); } - const username = defaultOrg.username; - console.log(`Using scratch org: ${username}`); + const username = scratchOrg.username; + console.log(`Using scratch org (${alias}): ${username}`); const org = await Org.create({ aliasOrUsername: username }); const connection = org.getConnection(); @@ -274,6 +280,13 @@ export async function getTestSession(): Promise { setDefault: true, config: 'config/project-scratch-def.json', }, + { + // Dedicated org for the run-eval NUTs (per-org login-rate isolation experiment). + // Created here, provisioned lazily in getRunEvalTestSession(). + alias: RUN_EVAL_ORG_ALIAS, + setDefault: false, + config: 'config/project-scratch-def.json', + }, ], }); @@ -325,70 +338,42 @@ export function getAgentUsername(): string | undefined { } /** - * Gets the DEDICATED run-eval TestSession, creating its own separate scratch org (isolated from - * the shared org used by the rest of the suite). See the per-org isolation experiment note above. - * After provisioning, restores `process.env.AGENT_USER_USERNAME` to the shared org's agent user so - * later shared-org deploys (e.g. z2 publish of Willie_Resort_Manager.agent) are not contaminated - * with this org's agent user. + * Ensures the DEDICATED run-eval scratch org (alias `runeval`, created alongside the shared org in + * the one TestSession) is provisioned and ready. Provisioned lazily and exactly once. See the + * per-org isolation experiment note above. After provisioning, restores + * `process.env.AGENT_USER_USERNAME` to the shared org's agent user so later shared-org deploys + * (e.g. z2 publish of Willie_Resort_Manager.agent) are not contaminated with this org's agent user. */ export async function getRunEvalTestSession(): Promise { - if (runEvalSession) { - return runEvalSession; - } - if (runEvalSessionPromise) { - return runEvalSessionPromise; - } - - // @ts-expect-error it can throw an error - runEvalSessionPromise = (async (): Promise => { - try { - console.log('Creating DEDICATED run-eval TestSession with its own scratch org...'); - const session = await TestSession.create({ - project: { - sourceDir: join('test', 'mock-projects', 'agent-generate-template'), - }, - devhubAuthStrategy: 'AUTO', - scratchOrgs: [ - { - alias: 'default', - setDefault: true, - config: 'config/project-scratch-def.json', - }, - ], - }); - - runEvalSession = session; - console.log('run-eval TestSession created successfully'); - - const provisioned = await provisionEvalOrg(session); - console.log(`[NUT-CONTEXT] run-eval dedicated org username=${provisioned.username} orgId=${provisioned.orgId}`); - - // Restore the SDR replacement env var to the shared org's agent user, so later shared-org - // deploys are not contaminated by the run-eval org's agent user. - if (agentUsername) { - process.env.AGENT_USER_USERNAME = agentUsername; - } - - return session; - } catch (e) { - console.log('XXXXXX RUN-EVAL SESSION ERROR XXXXXXX'); - console.dir(SfError.wrap(e).toObject()); - void Promise.reject(SfError.wrap(e)); + // Ensure the shared session (which also created the `runeval` org) exists first. + const session = await getTestSession(); + + runEvalProvisionPromise ??= (async (): Promise => { + console.log('Provisioning DEDICATED run-eval scratch org...'); + const provisioned = await provisionEvalOrg(session, RUN_EVAL_ORG_ALIAS); + runEvalUsername = provisioned.username; + console.log(`[NUT-CONTEXT] run-eval dedicated org username=${provisioned.username} orgId=${provisioned.orgId}`); + + // Restore the SDR replacement env var to the shared org's agent user, so later shared-org + // deploys are not contaminated by the run-eval org's agent user. + if (agentUsername) { + process.env.AGENT_USER_USERNAME = agentUsername; } })(); + await runEvalProvisionPromise; - return runEvalSessionPromise; + return session; } /** - * Gets the scratch org username from the dedicated run-eval test session. - * Throws an error if the session hasn't been created yet. + * Gets the username of the dedicated run-eval scratch org. + * Throws an error if it hasn't been provisioned yet. */ export function getRunEvalUsername(): string { - if (!runEvalSession) { - throw new Error('Run-eval test session not available. Call getRunEvalTestSession() first.'); + if (!runEvalUsername) { + throw new Error('Run-eval org not provisioned. Call getRunEvalTestSession() first.'); } - return runEvalSession.orgs.get('default')!.username!; + return runEvalUsername; } // Root-level hook: on any failing test that used the shared scratch org, emit orgId + pod +