Let the simulator and bias judge dialogs supply a credential

The backend now accepts a credentialId for both the interaction simulator
and the bias judge, but nothing in the UI could provide one - a
credential-requiring provider chosen either way would always fail, short of
coincidentally reusing a credential from elsewhere in the same execution.

openLLMDescriptorSettings (the one dialog shared by both the simulator and
the bias judge picker) stays as it is, tests included. A new
openLLMDescriptorSettingsWithCredential wraps it: once a provider and model
are chosen, if LlmProviderService.listCapabilities() says that provider
requiresCredential, a second dialog lists the user's saved credentials for
it (ExecutionVaultCredentialsService.listForProvider). Two separate modals
rather than one field folded into the first, since NodeSettingsDialogService
only ever holds one dialog open at a time - stacking a "create credential"
flow inside this one was not attempted, matching the pre-existing, already
documented gap that there is nowhere in the app to manage credentials
outside of creating them.

simulateExecution() in task-execution-viewer.ts also now surfaces a
rejected simulation's error message instead of failing silently.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

src/app/services/bias/bias-report-judge.spec.ts

@@ -69,7 +69,7 @@ describe('BiasReportJudgeService', () => {
       provider: 'InternalOllama',
       model: 'gemma:7b',
       parameters: { temperature: 0 }
-    });
+    }, undefined);
   });
 
   it('pre-fills no sampling, leaving the provider its own baseline', async () => {

src/app/services/bias/bias-report-judge.ts

@@ -5,10 +5,12 @@
 import { inject, Injectable } from '@angular/core';
 import { lastValueFrom } from 'rxjs';
 import { BiasImpactReport } from '@models/bias-impact';
+import { ExecutionVaultCredentialsService } from '@services/llm-provider/execution-vault-credentials';
+import { LlmProviderService } from '@services/llm-provider/llm-provider';
 import { FieldRetriever } from '@services/retriever/field-retriever';
 import { NodeSettingsDialogService } from '@services/dialogs/node-settings-dialog';
 import { TaskExecutionsService } from '@services/task-executions/task-executions';
-import { openLLMDescriptorSettings } from '@shared/llm-descriptor-settings/llm-descriptor-settings';
+import { openLLMDescriptorSettingsWithCredential } from '@shared/llm-descriptor-settings/llm-descriptor-settings';
 
 /**
  * Asks a model to assess a comparison, from wherever that comparison is on screen.
@@ -26,18 +28,23 @@ export class BiasReportJudgeService {
   private readonly executions = inject(TaskExecutionsService);
   private readonly settingsDialog = inject(NodeSettingsDialogService);
   private readonly fieldRetriever = inject(FieldRetriever);
+  private readonly llmProviderService = inject(LlmProviderService);
+  private readonly executionVaultCredentials = inject(ExecutionVaultCredentialsService);
 
   /**
-   * Resolves with the assessed report, or null when the model picker was dismissed or no provider
-   * is published at all. Rejects with the failure of the assessment itself.
+   * Resolves with the assessed report, or null when the model picker (or, for a provider that
+   * needs one, the credential picker after it) was dismissed, or no provider is published at all.
+   * Rejects with the failure of the assessment itself.
    */
   async assess(reportId: string): Promise<BiasImpactReport | null> {
-    const judge = await openLLMDescriptorSettings(this.settingsDialog, this.fieldRetriever, {
-      title: 'Evaluate impact with LLM'
-    });
-    if (!judge) return null;
+    const chosen = await openLLMDescriptorSettingsWithCredential(
+      this.settingsDialog, this.fieldRetriever, this.llmProviderService, this.executionVaultCredentials, {
+        title: 'Evaluate impact with LLM'
+      });
+    if (!chosen) return null;
 
-    const queued = await lastValueFrom(this.executions.judgeBiasImpactReport(reportId, judge));
+    const queued = await lastValueFrom(
+      this.executions.judgeBiasImpactReport(reportId, chosen.descriptor, chosen.credentialId));
     const finished = await lastValueFrom(this.executions.pollBiasImpactJob(queued.id));
     if (finished.status !== 'COMPLETED' || !finished.report) {
       throw new Error(finished.errorMessage || 'The LLM assessment did not complete.');

src/app/services/task-executions/task-executions-call.base.ts

@@ -40,7 +40,7 @@ export abstract class TaskExecutionsCallServiceBase {
    * Asks a model to assess a comparison that has already been computed. Asynchronous like the
    * isolated experiment - one model call per compared pair - so it answers with a job to poll.
    */
-  abstract judgeBiasImpactReport(reportId: string, judge: LLMDescriptor): Observable<BiasImpactJob>;
+  abstract judgeBiasImpactReport(reportId: string, judge: LLMDescriptor, credentialId?: string): Observable<BiasImpactJob>;
   abstract getBiasImpactJob(jobId: string): Observable<BiasImpactJob>;
   abstract createBiasedRerun(executionId: string, request: BiasRerunRequest): Observable<TaskExecution>;
   abstract compareBiasExecutions(
@@ -52,7 +52,7 @@ export abstract class TaskExecutionsCallServiceBase {
   abstract getBiasImpactReport(reportId: string): Observable<BiasImpactReport>;
   abstract deleteTaskExecution(executionId: string): Observable<void>;
   abstract startTaskExecution(executionId: string): Observable<TaskExecution>;
-  abstract simulateTaskExecution(executionId: string, simulator: LLMDescriptor): Observable<TaskExecution>;
+  abstract simulateTaskExecution(executionId: string, simulator: LLMDescriptor, credentialId?: string): Observable<TaskExecution>;
   abstract cancelTaskExecution(executionId: string): Observable<TaskExecution>;
   abstract resumeTaskExecution(executionId: string): Observable<TaskExecution>;
   abstract prepareStringInput(

src/app/services/task-executions/task-executions-call.fake.ts

@@ -653,7 +653,7 @@ export class TaskExecutionsCallServiceFake extends TaskExecutionsCallServiceBase
     return of(job);
   }
 
-  override judgeBiasImpactReport(reportId: string, judge: LLMDescriptor): Observable<BiasImpactJob> {
+  override judgeBiasImpactReport(reportId: string, judge: LLMDescriptor, _credentialId?: string): Observable<BiasImpactJob> {
     const report = this.biasReports.find((item) => item.id === reportId);
     if (!report) throw new Error(`Bias impact report not found: ${reportId}`);
     if (!judge?.provider?.trim() || !judge?.model?.trim()) {
@@ -866,7 +866,7 @@ export class TaskExecutionsCallServiceFake extends TaskExecutionsCallServiceBase
     return of(execution);
   }
 
-  override simulateTaskExecution(executionId: string, simulator: LLMDescriptor): Observable<TaskExecution> {
+  override simulateTaskExecution(executionId: string, simulator: LLMDescriptor, _credentialId?: string): Observable<TaskExecution> {
     const execution = this.findExecution(executionId);
     if (execution.simulationAvailable !== true) {
       throw new Error('Simulation is not available for this execution.');

src/app/services/task-executions/task-executions-call.ts

@@ -137,9 +137,9 @@ export class TaskExecutionsCallService extends TaskExecutionsCallServiceBase {
     return this.http.post<unknown>(url, request).pipe(map((raw) => this.biasImpactJobFromApi(raw)));
   }
 
-  override judgeBiasImpactReport(reportId: string, judge: LLMDescriptor): Observable<BiasImpactJob> {
+  override judgeBiasImpactReport(reportId: string, judge: LLMDescriptor, credentialId?: string): Observable<BiasImpactJob> {
     const url = `${environment.apiUrl}/executions/bias-impact-reports/${encodeURIComponent(reportId)}/judge`;
-    return this.http.post<unknown>(url, { judge }).pipe(map((raw) => this.biasImpactJobFromApi(raw)));
+    return this.http.post<unknown>(url, { judge, credentialId }).pipe(map((raw) => this.biasImpactJobFromApi(raw)));
   }
 
   override getBiasImpactJob(jobId: string): Observable<BiasImpactJob> {
@@ -187,8 +187,11 @@ export class TaskExecutionsCallService extends TaskExecutionsCallServiceBase {
     );
   }
 
-  override simulateTaskExecution(executionId: string, simulator: LLMDescriptor): Observable<TaskExecution> {
-    return this.http.put<unknown>(`${environment.apiUrl}/executions/${encodeURIComponent(executionId)}/simulate`, { simulator }).pipe(
+  override simulateTaskExecution(executionId: string, simulator: LLMDescriptor, credentialId?: string): Observable<TaskExecution> {
+    return this.http.put<unknown>(
+      `${environment.apiUrl}/executions/${encodeURIComponent(executionId)}/simulate`,
+      { simulator, credentialId }
+    ).pipe(
       map((raw) => this.mapExecution(raw))
     );
   }

src/app/services/task-executions/task-executions.ts

@@ -181,8 +181,8 @@ export class TaskExecutionsService {
     );
   }
 
-  judgeBiasImpactReport(reportId: string, judge: LLMDescriptor): Observable<BiasImpactJob> {
-    return this.taskExecutionsCallService.judgeBiasImpactReport(reportId, judge).pipe(
+  judgeBiasImpactReport(reportId: string, judge: LLMDescriptor, credentialId?: string): Observable<BiasImpactJob> {
+    return this.taskExecutionsCallService.judgeBiasImpactReport(reportId, judge, credentialId).pipe(
       catchError((error) => throwError(() => this.toBiasOperationError(error)))
     );
   }
@@ -269,7 +269,7 @@ export class TaskExecutionsService {
     );
   }
 
-  simulateExecution(executionId: string, simulator: LLMDescriptor) {
+  simulateExecution(executionId: string, simulator: LLMDescriptor, credentialId?: string) {
     const execution = this._taskExecutions().find((item) => item.id === executionId);
     if (execution?.simulationAvailable !== true) {
       return throwError(() => new Error('Simulation is not available for this execution.'));
@@ -279,7 +279,7 @@ export class TaskExecutionsService {
     }
 
     return this.withRefreshAndErrorHandling(
-      this.taskExecutionsCallService.simulateTaskExecution(executionId, simulator),
+      this.taskExecutionsCallService.simulateTaskExecution(executionId, simulator, credentialId),
       'Simulate execution failed'
     );
   }

src/app/shared/llm-descriptor-settings/llm-descriptor-settings.spec.ts

@@ -2,11 +2,14 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 // Attribution term under AGPL-3.0 section 7(b): see LICENSE-ADDENDUM.
 
-import { of } from 'rxjs';
+import { of, throwError } from 'rxjs';
 import { vi } from 'vitest';
+import { ExecutionVaultCredential, LlmProviderCapability } from '@models/llm-provider';
+import { ExecutionVaultCredentialsService } from '@services/llm-provider/execution-vault-credentials';
+import { LlmProviderService } from '@services/llm-provider/llm-provider';
 import { NodeSettingsDialogService } from '@services/dialogs/node-settings-dialog';
 import { FieldRetriever } from '@services/retriever/field-retriever';
-import { openLLMDescriptorSettings } from './llm-descriptor-settings';
+import { openLLMDescriptorSettings, openLLMDescriptorSettingsWithCredential } from './llm-descriptor-settings';
 
 describe('openLLMDescriptorSettings', () => {
   const models: Record<string, string[]> = {
@@ -109,3 +112,95 @@ describe('openLLMDescriptorSettings', () => {
     expect(await openLLMDescriptorSettings(service, retriever(), { title: 'Simulation Settings' })).toBeNull();
   });
 });
+
+describe('openLLMDescriptorSettingsWithCredential', () => {
+  function retriever(providers = ['InternalOllama', 'CredentialProvider']): FieldRetriever {
+    return {
+      retrieveValues: vi.fn((_type: string, key: string) =>
+        of(key === 'providers' ? providers : ['some-model']))
+    } as unknown as FieldRetriever;
+  }
+
+  function providerService(capabilities: LlmProviderCapability[]): LlmProviderService {
+    return { listCapabilities: vi.fn().mockReturnValue(of(capabilities)) } as unknown as LlmProviderService;
+  }
+
+  function vaultCredentials(credentials: ExecutionVaultCredential[]): ExecutionVaultCredentialsService {
+    return { listForProvider: vi.fn().mockReturnValue(of(credentials)) } as unknown as ExecutionVaultCredentialsService;
+  }
+
+  const CAPABILITIES: LlmProviderCapability[] = [
+    { name: 'InternalOllama', requiresCredential: false, requiresEndpoint: false },
+    { name: 'CredentialProvider', requiresCredential: true, requiresEndpoint: false }
+  ];
+
+  it('answers with just the descriptor when the chosen provider needs no credential', async () => {
+    const open = vi.fn().mockResolvedValue({ provider: 'InternalOllama', model: 'some-model' });
+    const settingsDialog = { open } as unknown as NodeSettingsDialogService;
+
+    const result = await openLLMDescriptorSettingsWithCredential(
+      settingsDialog, retriever(), providerService(CAPABILITIES), vaultCredentials([]), { title: 'x' });
+
+    expect(result).toEqual({ descriptor: { provider: 'InternalOllama', model: 'some-model' } });
+    // Nothing to pick, so only the model dialog opened - never a second one for a credential.
+    expect(open).toHaveBeenCalledTimes(1);
+  });
+
+  it('opens a credential picker and carries the choice alongside the descriptor', async () => {
+    const open = vi.fn()
+      .mockResolvedValueOnce({ provider: 'CredentialProvider', model: 'some-model' })
+      .mockResolvedValueOnce({ credentialId: 'secret-1' });
+    const settingsDialog = { open } as unknown as NodeSettingsDialogService;
+    const credentials = vaultCredentials([{ id: 'secret-1', label: 'My key', provider: 'CredentialProvider' }]);
+
+    const result = await openLLMDescriptorSettingsWithCredential(
+      settingsDialog, retriever(), providerService(CAPABILITIES), credentials, { title: 'x' });
+
+    expect(result).toEqual({
+      descriptor: { provider: 'CredentialProvider', model: 'some-model' },
+      credentialId: 'secret-1'
+    });
+    expect(open).toHaveBeenCalledTimes(2);
+    const credentialFields: Array<{ options?: Array<{ label: string; value: string }> }> = open.mock.calls[1][0].fields;
+    expect(credentialFields[0].options).toEqual([{ label: 'My key', value: 'secret-1' }]);
+  });
+
+  it('answers null when the credential picker is dismissed', async () => {
+    const open = vi.fn()
+      .mockResolvedValueOnce({ provider: 'CredentialProvider', model: 'some-model' })
+      .mockResolvedValueOnce(null);
+    const settingsDialog = { open } as unknown as NodeSettingsDialogService;
+
+    const result = await openLLMDescriptorSettingsWithCredential(
+      settingsDialog, retriever(), providerService(CAPABILITIES), vaultCredentials([]), { title: 'x' });
+
+    expect(result).toBeNull();
+  });
+
+  it('says so in the tip when the provider has no saved credential yet', async () => {
+    const open = vi.fn().mockResolvedValueOnce({ provider: 'CredentialProvider', model: 'some-model' })
+      .mockResolvedValueOnce(null);
+    const settingsDialog = { open } as unknown as NodeSettingsDialogService;
+
+    await openLLMDescriptorSettingsWithCredential(
+      settingsDialog, retriever(), providerService(CAPABILITIES), vaultCredentials([]), { title: 'x' });
+
+    const credentialFields: Array<{ options?: unknown[]; tip?: string }> = open.mock.calls[1][0].fields;
+    expect(credentialFields[0].options).toEqual([]);
+    expect(credentialFields[0].tip).toContain('No saved credential');
+  });
+
+  it('treats a provider catalog that could not be loaded as needing no credential', async () => {
+    const open = vi.fn().mockResolvedValue({ provider: 'CredentialProvider', model: 'some-model' });
+    const settingsDialog = { open } as unknown as NodeSettingsDialogService;
+    const brokenProviderService = {
+      listCapabilities: vi.fn().mockReturnValue(throwError(() => new Error('offline')))
+    } as unknown as LlmProviderService;
+
+    const result = await openLLMDescriptorSettingsWithCredential(
+      settingsDialog, retriever(), brokenProviderService, vaultCredentials([]), { title: 'x' });
+
+    expect(result).toEqual({ descriptor: { provider: 'CredentialProvider', model: 'some-model' } });
+    expect(open).toHaveBeenCalledTimes(1);
+  });
+});

src/app/shared/llm-descriptor-settings/llm-descriptor-settings.ts

@@ -4,6 +4,8 @@
 
 import { firstValueFrom } from 'rxjs';
 import { LLMDescriptor } from '@models/flow';
+import { ExecutionVaultCredentialsService } from '@services/llm-provider/execution-vault-credentials';
+import { LlmProviderService } from '@services/llm-provider/llm-provider';
 import { FieldRetriever } from '@services/retriever/field-retriever';
 import { NodeSettingField, NodeSettingsDialogService } from '@services/dialogs/node-settings-dialog';
 import { readSimulatorParameters } from '@shared/task-execution-viewer/execution-viewer.utils';
@@ -178,3 +180,85 @@ async function loadOptions(
   const values = await firstValueFrom(fieldRetriever.retrieveValues('LLM', key, context, retrieverUrl));
   return values.map((value) => ({ label: value, value }));
 }
+
+export type LLMDescriptorWithCredential = {
+  descriptor: LLMDescriptor;
+  /** Present only when the chosen provider needed one, and one was picked. */
+  credentialId?: string;
+};
+
+/**
+ * {@link openLLMDescriptorSettings}, followed by a credential picker when the chosen provider
+ * needs one.
+ *
+ * <p>The interaction simulator and the bias judge are asked for their model outside of any node's
+ * configuration, so - unlike a flow's own steps - there is no `requiredAuthorizations` entry
+ * computed up front to gate them on a saved credential. Without this, a credential-requiring
+ * provider chosen here could only ever work by coincidence, if the execution already happened to
+ * carry a saved credential for that same provider from one of its own steps.
+ *
+ * <p>A second, separate dialog rather than a field folded into the first: the two ask genuinely
+ * different questions - which model, then which of *your* saved keys for it - and keeping them
+ * apart means the model dialog above is untouched by this, tests included.
+ *
+ * <p>Resolves to null when either dialog is dismissed, or when the chosen provider needs a
+ * credential and none is picked - the caller already treats a null result as "nothing to do".
+ */
+export async function openLLMDescriptorSettingsWithCredential(
+  settingsDialog: NodeSettingsDialogService,
+  fieldRetriever: FieldRetriever,
+  llmProviderService: LlmProviderService,
+  executionVaultCredentials: ExecutionVaultCredentialsService,
+  request: LLMDescriptorSettingsRequest
+): Promise<LLMDescriptorWithCredential | null> {
+  const descriptor = await openLLMDescriptorSettings(settingsDialog, fieldRetriever, request);
+  if (!descriptor) return null;
+
+  const needsCredential = await providerRequiresCredential(llmProviderService, descriptor.provider);
+  if (!needsCredential) return { descriptor };
+
+  const credentialId = await pickCredential(settingsDialog, executionVaultCredentials, descriptor.provider);
+  return credentialId ? { descriptor, credentialId } : null;
+}
+
+async function providerRequiresCredential(llmProviderService: LlmProviderService, provider: string): Promise<boolean> {
+  try {
+    const capabilities = await firstValueFrom(llmProviderService.listCapabilities());
+    const wanted = provider.trim().toLowerCase();
+    return capabilities.some((capability) => capability.name.trim().toLowerCase() === wanted && capability.requiresCredential);
+  } catch {
+    // The same conservative default used everywhere else a capability cannot be resolved: nothing
+    // blocks the run, and a provider that really did need a credential still refuses it, at the
+    // server, with a clear reason - it just does so a step later than it could have here.
+    return false;
+  }
+}
+
+async function pickCredential(
+  settingsDialog: NodeSettingsDialogService,
+  executionVaultCredentials: ExecutionVaultCredentialsService,
+  provider: string
+): Promise<string | null> {
+  const credentials = await firstValueFrom(executionVaultCredentials.listForProvider(provider));
+  const options = credentials.map((credential) => ({ label: credential.label, value: credential.id }));
+
+  const result = await settingsDialog.open({
+    title: `Credential for ${provider}`,
+    fields: [{
+      key: 'credentialId',
+      label: 'Credential',
+      type: 'select',
+      required: true,
+      autofocus: true,
+      options,
+      tip: options.length
+        ? undefined
+        : `No saved credential for ${provider} yet. Add one for this provider, then reopen this dialog.`
+    }],
+    initial: { credentialId: options[0]?.value ?? '' }
+  });
+  if (!result) return null;
+
+  const credentialId = String(result['credentialId'] ?? '').trim();
+  return credentialId || null;
+}

src/app/shared/task-execution-viewer/task-execution-viewer.ts

@@ -47,7 +47,10 @@ import {
   HumanInteractionDialogService
 } from '@services/dialogs/human-interaction-dialog';
 import { NodeSettingField, NodeSettingsDialogService } from '@services/dialogs/node-settings-dialog';
-import { openLLMDescriptorSettings } from '@shared/llm-descriptor-settings/llm-descriptor-settings';
+import {
+  LLMDescriptorWithCredential,
+  openLLMDescriptorSettingsWithCredential
+} from '@shared/llm-descriptor-settings/llm-descriptor-settings';
 import { FieldRetriever } from '@services/retriever/field-retriever';
 import { TaskExecutionsService } from '@services/task-executions/task-executions';
 import { FlowsService } from '@services/flows/flows';
@@ -1179,9 +1182,9 @@ export class TaskExecutionViewerComponent implements OnDestroy {
     const executionId = this.execution()?.id;
     if (!executionId || !this.canSimulateExecution()) return;
 
-    let simulator: LLMDescriptor | null;
+    let chosen: LLMDescriptorWithCredential | null;
     try {
-      simulator = await this.openSimulationSettings();
+      chosen = await this.openSimulationSettings();
     } catch (error) {
       // No provider to choose from, or the list could not be loaded. Silence here reads as a dead
       // button, which is what the picker is opened by.
@@ -1192,12 +1195,15 @@ export class TaskExecutionViewerComponent implements OnDestroy {
       );
       return;
     }
-    if (!simulator) return;
+    if (!chosen) return;
 
     this.simulateInProgress.set(true);
-    this.taskExecutionsService.simulateExecution(executionId, simulator).subscribe({
+    this.taskExecutionsService.simulateExecution(executionId, chosen.descriptor, chosen.credentialId).subscribe({
       next: () => this.simulateInProgress.set(false),
-      error: () => this.simulateInProgress.set(false)
+      error: (error: unknown) => {
+        this.simulateInProgress.set(false);
+        this.notifications.show(this.executionErrorMessage(error), 'error', 6000);
+      }
     });
   }
 
@@ -1510,14 +1516,15 @@ export class TaskExecutionViewerComponent implements OnDestroy {
     });
   }
 
-  private openSimulationSettings(): Promise<LLMDescriptor | null> {
+  private openSimulationSettings(): Promise<LLMDescriptorWithCredential | null> {
     // A rerun carries the simulator of the run it repeats, and that is what the dialog starts from:
     // comparing two runs answered by different simulators compares the simulators too.
     const inherited = this.inheritedSimulator();
-    return openLLMDescriptorSettings(this.settingsDialog, this.fieldRetriever, {
-      title: inherited ? 'Simulation Settings - repeating a simulated run' : 'Simulation Settings',
-      initialDescriptor: inherited
-    });
+    return openLLMDescriptorSettingsWithCredential(
+      this.settingsDialog, this.fieldRetriever, this.llmProviderService, this.executionVaultCredentials, {
+        title: inherited ? 'Simulation Settings - repeating a simulated run' : 'Simulation Settings',
+        initialDescriptor: inherited
+      });
   }
 
   private fetchExecutionLogs(executionId: string) {