diff --git a/src/app/services/bias/bias-report-judge.spec.ts b/src/app/services/bias/bias-report-judge.spec.ts index e96364c..22d7aa9 100644 --- a/src/app/services/bias/bias-report-judge.spec.ts +++ b/src/app/services/bias/bias-report-judge.spec.ts @@ -69,7 +69,7 @@ describe('BiasReportJudgeService', () => { provider: 'InternalOllama', model: 'gemma:7b', parameters: { temperature: 0 } - }); + }, undefined); }); it('pre-fills no sampling, leaving the provider its own baseline', async () => { diff --git a/src/app/services/bias/bias-report-judge.ts b/src/app/services/bias/bias-report-judge.ts index 95a9796..95c382b 100644 --- a/src/app/services/bias/bias-report-judge.ts +++ b/src/app/services/bias/bias-report-judge.ts @@ -5,10 +5,12 @@ import { inject, Injectable } from '@angular/core'; import { lastValueFrom } from 'rxjs'; import { BiasImpactReport } from '@models/bias-impact'; +import { ExecutionVaultCredentialsService } from '@services/llm-provider/execution-vault-credentials'; +import { LlmProviderService } from '@services/llm-provider/llm-provider'; import { FieldRetriever } from '@services/retriever/field-retriever'; import { NodeSettingsDialogService } from '@services/dialogs/node-settings-dialog'; import { TaskExecutionsService } from '@services/task-executions/task-executions'; -import { openLLMDescriptorSettings } from '@shared/llm-descriptor-settings/llm-descriptor-settings'; +import { openLLMDescriptorSettingsWithCredential } from '@shared/llm-descriptor-settings/llm-descriptor-settings'; /** * Asks a model to assess a comparison, from wherever that comparison is on screen. @@ -26,18 +28,23 @@ export class BiasReportJudgeService { private readonly executions = inject(TaskExecutionsService); private readonly settingsDialog = inject(NodeSettingsDialogService); private readonly fieldRetriever = inject(FieldRetriever); + private readonly llmProviderService = inject(LlmProviderService); + private readonly executionVaultCredentials = inject(ExecutionVaultCredentialsService); /** - * Resolves with the assessed report, or null when the model picker was dismissed or no provider - * is published at all. Rejects with the failure of the assessment itself. + * Resolves with the assessed report, or null when the model picker (or, for a provider that + * needs one, the credential picker after it) was dismissed, or no provider is published at all. + * Rejects with the failure of the assessment itself. */ async assess(reportId: string): Promise { - const judge = await openLLMDescriptorSettings(this.settingsDialog, this.fieldRetriever, { - title: 'Evaluate impact with LLM' - }); - if (!judge) return null; + const chosen = await openLLMDescriptorSettingsWithCredential( + this.settingsDialog, this.fieldRetriever, this.llmProviderService, this.executionVaultCredentials, { + title: 'Evaluate impact with LLM' + }); + if (!chosen) return null; - const queued = await lastValueFrom(this.executions.judgeBiasImpactReport(reportId, judge)); + const queued = await lastValueFrom( + this.executions.judgeBiasImpactReport(reportId, chosen.descriptor, chosen.credentialId)); const finished = await lastValueFrom(this.executions.pollBiasImpactJob(queued.id)); if (finished.status !== 'COMPLETED' || !finished.report) { throw new Error(finished.errorMessage || 'The LLM assessment did not complete.'); diff --git a/src/app/services/task-executions/task-executions-call.base.ts b/src/app/services/task-executions/task-executions-call.base.ts index a39017e..5b2d58d 100644 --- a/src/app/services/task-executions/task-executions-call.base.ts +++ b/src/app/services/task-executions/task-executions-call.base.ts @@ -40,7 +40,7 @@ export abstract class TaskExecutionsCallServiceBase { * Asks a model to assess a comparison that has already been computed. Asynchronous like the * isolated experiment - one model call per compared pair - so it answers with a job to poll. */ - abstract judgeBiasImpactReport(reportId: string, judge: LLMDescriptor): Observable; + abstract judgeBiasImpactReport(reportId: string, judge: LLMDescriptor, credentialId?: string): Observable; abstract getBiasImpactJob(jobId: string): Observable; abstract createBiasedRerun(executionId: string, request: BiasRerunRequest): Observable; abstract compareBiasExecutions( @@ -52,7 +52,7 @@ export abstract class TaskExecutionsCallServiceBase { abstract getBiasImpactReport(reportId: string): Observable; abstract deleteTaskExecution(executionId: string): Observable; abstract startTaskExecution(executionId: string): Observable; - abstract simulateTaskExecution(executionId: string, simulator: LLMDescriptor): Observable; + abstract simulateTaskExecution(executionId: string, simulator: LLMDescriptor, credentialId?: string): Observable; abstract cancelTaskExecution(executionId: string): Observable; abstract resumeTaskExecution(executionId: string): Observable; abstract prepareStringInput( diff --git a/src/app/services/task-executions/task-executions-call.fake.ts b/src/app/services/task-executions/task-executions-call.fake.ts index e367752..0fe7df7 100644 --- a/src/app/services/task-executions/task-executions-call.fake.ts +++ b/src/app/services/task-executions/task-executions-call.fake.ts @@ -653,7 +653,7 @@ export class TaskExecutionsCallServiceFake extends TaskExecutionsCallServiceBase return of(job); } - override judgeBiasImpactReport(reportId: string, judge: LLMDescriptor): Observable { + override judgeBiasImpactReport(reportId: string, judge: LLMDescriptor, _credentialId?: string): Observable { const report = this.biasReports.find((item) => item.id === reportId); if (!report) throw new Error(`Bias impact report not found: ${reportId}`); if (!judge?.provider?.trim() || !judge?.model?.trim()) { @@ -866,7 +866,7 @@ export class TaskExecutionsCallServiceFake extends TaskExecutionsCallServiceBase return of(execution); } - override simulateTaskExecution(executionId: string, simulator: LLMDescriptor): Observable { + override simulateTaskExecution(executionId: string, simulator: LLMDescriptor, _credentialId?: string): Observable { const execution = this.findExecution(executionId); if (execution.simulationAvailable !== true) { throw new Error('Simulation is not available for this execution.'); diff --git a/src/app/services/task-executions/task-executions-call.ts b/src/app/services/task-executions/task-executions-call.ts index 9ed1712..a831dfa 100644 --- a/src/app/services/task-executions/task-executions-call.ts +++ b/src/app/services/task-executions/task-executions-call.ts @@ -137,9 +137,9 @@ export class TaskExecutionsCallService extends TaskExecutionsCallServiceBase { return this.http.post(url, request).pipe(map((raw) => this.biasImpactJobFromApi(raw))); } - override judgeBiasImpactReport(reportId: string, judge: LLMDescriptor): Observable { + override judgeBiasImpactReport(reportId: string, judge: LLMDescriptor, credentialId?: string): Observable { const url = `${environment.apiUrl}/executions/bias-impact-reports/${encodeURIComponent(reportId)}/judge`; - return this.http.post(url, { judge }).pipe(map((raw) => this.biasImpactJobFromApi(raw))); + return this.http.post(url, { judge, credentialId }).pipe(map((raw) => this.biasImpactJobFromApi(raw))); } override getBiasImpactJob(jobId: string): Observable { @@ -187,8 +187,11 @@ export class TaskExecutionsCallService extends TaskExecutionsCallServiceBase { ); } - override simulateTaskExecution(executionId: string, simulator: LLMDescriptor): Observable { - return this.http.put(`${environment.apiUrl}/executions/${encodeURIComponent(executionId)}/simulate`, { simulator }).pipe( + override simulateTaskExecution(executionId: string, simulator: LLMDescriptor, credentialId?: string): Observable { + return this.http.put( + `${environment.apiUrl}/executions/${encodeURIComponent(executionId)}/simulate`, + { simulator, credentialId } + ).pipe( map((raw) => this.mapExecution(raw)) ); } diff --git a/src/app/services/task-executions/task-executions.ts b/src/app/services/task-executions/task-executions.ts index 18fd0f8..ca83a68 100644 --- a/src/app/services/task-executions/task-executions.ts +++ b/src/app/services/task-executions/task-executions.ts @@ -181,8 +181,8 @@ export class TaskExecutionsService { ); } - judgeBiasImpactReport(reportId: string, judge: LLMDescriptor): Observable { - return this.taskExecutionsCallService.judgeBiasImpactReport(reportId, judge).pipe( + judgeBiasImpactReport(reportId: string, judge: LLMDescriptor, credentialId?: string): Observable { + return this.taskExecutionsCallService.judgeBiasImpactReport(reportId, judge, credentialId).pipe( catchError((error) => throwError(() => this.toBiasOperationError(error))) ); } @@ -269,7 +269,7 @@ export class TaskExecutionsService { ); } - simulateExecution(executionId: string, simulator: LLMDescriptor) { + simulateExecution(executionId: string, simulator: LLMDescriptor, credentialId?: string) { const execution = this._taskExecutions().find((item) => item.id === executionId); if (execution?.simulationAvailable !== true) { return throwError(() => new Error('Simulation is not available for this execution.')); @@ -279,7 +279,7 @@ export class TaskExecutionsService { } return this.withRefreshAndErrorHandling( - this.taskExecutionsCallService.simulateTaskExecution(executionId, simulator), + this.taskExecutionsCallService.simulateTaskExecution(executionId, simulator, credentialId), 'Simulate execution failed' ); } diff --git a/src/app/shared/llm-descriptor-settings/llm-descriptor-settings.spec.ts b/src/app/shared/llm-descriptor-settings/llm-descriptor-settings.spec.ts index cae5960..f96a133 100644 --- a/src/app/shared/llm-descriptor-settings/llm-descriptor-settings.spec.ts +++ b/src/app/shared/llm-descriptor-settings/llm-descriptor-settings.spec.ts @@ -2,11 +2,14 @@ // SPDX-License-Identifier: AGPL-3.0-or-later // Attribution term under AGPL-3.0 section 7(b): see LICENSE-ADDENDUM. -import { of } from 'rxjs'; +import { of, throwError } from 'rxjs'; import { vi } from 'vitest'; +import { ExecutionVaultCredential, LlmProviderCapability } from '@models/llm-provider'; +import { ExecutionVaultCredentialsService } from '@services/llm-provider/execution-vault-credentials'; +import { LlmProviderService } from '@services/llm-provider/llm-provider'; import { NodeSettingsDialogService } from '@services/dialogs/node-settings-dialog'; import { FieldRetriever } from '@services/retriever/field-retriever'; -import { openLLMDescriptorSettings } from './llm-descriptor-settings'; +import { openLLMDescriptorSettings, openLLMDescriptorSettingsWithCredential } from './llm-descriptor-settings'; describe('openLLMDescriptorSettings', () => { const models: Record = { @@ -109,3 +112,95 @@ describe('openLLMDescriptorSettings', () => { expect(await openLLMDescriptorSettings(service, retriever(), { title: 'Simulation Settings' })).toBeNull(); }); }); + +describe('openLLMDescriptorSettingsWithCredential', () => { + function retriever(providers = ['InternalOllama', 'CredentialProvider']): FieldRetriever { + return { + retrieveValues: vi.fn((_type: string, key: string) => + of(key === 'providers' ? providers : ['some-model'])) + } as unknown as FieldRetriever; + } + + function providerService(capabilities: LlmProviderCapability[]): LlmProviderService { + return { listCapabilities: vi.fn().mockReturnValue(of(capabilities)) } as unknown as LlmProviderService; + } + + function vaultCredentials(credentials: ExecutionVaultCredential[]): ExecutionVaultCredentialsService { + return { listForProvider: vi.fn().mockReturnValue(of(credentials)) } as unknown as ExecutionVaultCredentialsService; + } + + const CAPABILITIES: LlmProviderCapability[] = [ + { name: 'InternalOllama', requiresCredential: false, requiresEndpoint: false }, + { name: 'CredentialProvider', requiresCredential: true, requiresEndpoint: false } + ]; + + it('answers with just the descriptor when the chosen provider needs no credential', async () => { + const open = vi.fn().mockResolvedValue({ provider: 'InternalOllama', model: 'some-model' }); + const settingsDialog = { open } as unknown as NodeSettingsDialogService; + + const result = await openLLMDescriptorSettingsWithCredential( + settingsDialog, retriever(), providerService(CAPABILITIES), vaultCredentials([]), { title: 'x' }); + + expect(result).toEqual({ descriptor: { provider: 'InternalOllama', model: 'some-model' } }); + // Nothing to pick, so only the model dialog opened - never a second one for a credential. + expect(open).toHaveBeenCalledTimes(1); + }); + + it('opens a credential picker and carries the choice alongside the descriptor', async () => { + const open = vi.fn() + .mockResolvedValueOnce({ provider: 'CredentialProvider', model: 'some-model' }) + .mockResolvedValueOnce({ credentialId: 'secret-1' }); + const settingsDialog = { open } as unknown as NodeSettingsDialogService; + const credentials = vaultCredentials([{ id: 'secret-1', label: 'My key', provider: 'CredentialProvider' }]); + + const result = await openLLMDescriptorSettingsWithCredential( + settingsDialog, retriever(), providerService(CAPABILITIES), credentials, { title: 'x' }); + + expect(result).toEqual({ + descriptor: { provider: 'CredentialProvider', model: 'some-model' }, + credentialId: 'secret-1' + }); + expect(open).toHaveBeenCalledTimes(2); + const credentialFields: Array<{ options?: Array<{ label: string; value: string }> }> = open.mock.calls[1][0].fields; + expect(credentialFields[0].options).toEqual([{ label: 'My key', value: 'secret-1' }]); + }); + + it('answers null when the credential picker is dismissed', async () => { + const open = vi.fn() + .mockResolvedValueOnce({ provider: 'CredentialProvider', model: 'some-model' }) + .mockResolvedValueOnce(null); + const settingsDialog = { open } as unknown as NodeSettingsDialogService; + + const result = await openLLMDescriptorSettingsWithCredential( + settingsDialog, retriever(), providerService(CAPABILITIES), vaultCredentials([]), { title: 'x' }); + + expect(result).toBeNull(); + }); + + it('says so in the tip when the provider has no saved credential yet', async () => { + const open = vi.fn().mockResolvedValueOnce({ provider: 'CredentialProvider', model: 'some-model' }) + .mockResolvedValueOnce(null); + const settingsDialog = { open } as unknown as NodeSettingsDialogService; + + await openLLMDescriptorSettingsWithCredential( + settingsDialog, retriever(), providerService(CAPABILITIES), vaultCredentials([]), { title: 'x' }); + + const credentialFields: Array<{ options?: unknown[]; tip?: string }> = open.mock.calls[1][0].fields; + expect(credentialFields[0].options).toEqual([]); + expect(credentialFields[0].tip).toContain('No saved credential'); + }); + + it('treats a provider catalog that could not be loaded as needing no credential', async () => { + const open = vi.fn().mockResolvedValue({ provider: 'CredentialProvider', model: 'some-model' }); + const settingsDialog = { open } as unknown as NodeSettingsDialogService; + const brokenProviderService = { + listCapabilities: vi.fn().mockReturnValue(throwError(() => new Error('offline'))) + } as unknown as LlmProviderService; + + const result = await openLLMDescriptorSettingsWithCredential( + settingsDialog, retriever(), brokenProviderService, vaultCredentials([]), { title: 'x' }); + + expect(result).toEqual({ descriptor: { provider: 'CredentialProvider', model: 'some-model' } }); + expect(open).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/app/shared/llm-descriptor-settings/llm-descriptor-settings.ts b/src/app/shared/llm-descriptor-settings/llm-descriptor-settings.ts index c1a90b8..3edf47a 100644 --- a/src/app/shared/llm-descriptor-settings/llm-descriptor-settings.ts +++ b/src/app/shared/llm-descriptor-settings/llm-descriptor-settings.ts @@ -4,6 +4,8 @@ import { firstValueFrom } from 'rxjs'; import { LLMDescriptor } from '@models/flow'; +import { ExecutionVaultCredentialsService } from '@services/llm-provider/execution-vault-credentials'; +import { LlmProviderService } from '@services/llm-provider/llm-provider'; import { FieldRetriever } from '@services/retriever/field-retriever'; import { NodeSettingField, NodeSettingsDialogService } from '@services/dialogs/node-settings-dialog'; import { readSimulatorParameters } from '@shared/task-execution-viewer/execution-viewer.utils'; @@ -178,3 +180,85 @@ async function loadOptions( const values = await firstValueFrom(fieldRetriever.retrieveValues('LLM', key, context, retrieverUrl)); return values.map((value) => ({ label: value, value })); } + +export type LLMDescriptorWithCredential = { + descriptor: LLMDescriptor; + /** Present only when the chosen provider needed one, and one was picked. */ + credentialId?: string; +}; + +/** + * {@link openLLMDescriptorSettings}, followed by a credential picker when the chosen provider + * needs one. + * + *

The interaction simulator and the bias judge are asked for their model outside of any node's + * configuration, so - unlike a flow's own steps - there is no `requiredAuthorizations` entry + * computed up front to gate them on a saved credential. Without this, a credential-requiring + * provider chosen here could only ever work by coincidence, if the execution already happened to + * carry a saved credential for that same provider from one of its own steps. + * + *

A second, separate dialog rather than a field folded into the first: the two ask genuinely + * different questions - which model, then which of *your* saved keys for it - and keeping them + * apart means the model dialog above is untouched by this, tests included. + * + *

Resolves to null when either dialog is dismissed, or when the chosen provider needs a + * credential and none is picked - the caller already treats a null result as "nothing to do". + */ +export async function openLLMDescriptorSettingsWithCredential( + settingsDialog: NodeSettingsDialogService, + fieldRetriever: FieldRetriever, + llmProviderService: LlmProviderService, + executionVaultCredentials: ExecutionVaultCredentialsService, + request: LLMDescriptorSettingsRequest +): Promise { + const descriptor = await openLLMDescriptorSettings(settingsDialog, fieldRetriever, request); + if (!descriptor) return null; + + const needsCredential = await providerRequiresCredential(llmProviderService, descriptor.provider); + if (!needsCredential) return { descriptor }; + + const credentialId = await pickCredential(settingsDialog, executionVaultCredentials, descriptor.provider); + return credentialId ? { descriptor, credentialId } : null; +} + +async function providerRequiresCredential(llmProviderService: LlmProviderService, provider: string): Promise { + try { + const capabilities = await firstValueFrom(llmProviderService.listCapabilities()); + const wanted = provider.trim().toLowerCase(); + return capabilities.some((capability) => capability.name.trim().toLowerCase() === wanted && capability.requiresCredential); + } catch { + // The same conservative default used everywhere else a capability cannot be resolved: nothing + // blocks the run, and a provider that really did need a credential still refuses it, at the + // server, with a clear reason - it just does so a step later than it could have here. + return false; + } +} + +async function pickCredential( + settingsDialog: NodeSettingsDialogService, + executionVaultCredentials: ExecutionVaultCredentialsService, + provider: string +): Promise { + const credentials = await firstValueFrom(executionVaultCredentials.listForProvider(provider)); + const options = credentials.map((credential) => ({ label: credential.label, value: credential.id })); + + const result = await settingsDialog.open({ + title: `Credential for ${provider}`, + fields: [{ + key: 'credentialId', + label: 'Credential', + type: 'select', + required: true, + autofocus: true, + options, + tip: options.length + ? undefined + : `No saved credential for ${provider} yet. Add one for this provider, then reopen this dialog.` + }], + initial: { credentialId: options[0]?.value ?? '' } + }); + if (!result) return null; + + const credentialId = String(result['credentialId'] ?? '').trim(); + return credentialId || null; +} diff --git a/src/app/shared/task-execution-viewer/task-execution-viewer.ts b/src/app/shared/task-execution-viewer/task-execution-viewer.ts index 8ab9769..c67a922 100644 --- a/src/app/shared/task-execution-viewer/task-execution-viewer.ts +++ b/src/app/shared/task-execution-viewer/task-execution-viewer.ts @@ -47,7 +47,10 @@ import { HumanInteractionDialogService } from '@services/dialogs/human-interaction-dialog'; import { NodeSettingField, NodeSettingsDialogService } from '@services/dialogs/node-settings-dialog'; -import { openLLMDescriptorSettings } from '@shared/llm-descriptor-settings/llm-descriptor-settings'; +import { + LLMDescriptorWithCredential, + openLLMDescriptorSettingsWithCredential +} from '@shared/llm-descriptor-settings/llm-descriptor-settings'; import { FieldRetriever } from '@services/retriever/field-retriever'; import { TaskExecutionsService } from '@services/task-executions/task-executions'; import { FlowsService } from '@services/flows/flows'; @@ -1179,9 +1182,9 @@ export class TaskExecutionViewerComponent implements OnDestroy { const executionId = this.execution()?.id; if (!executionId || !this.canSimulateExecution()) return; - let simulator: LLMDescriptor | null; + let chosen: LLMDescriptorWithCredential | null; try { - simulator = await this.openSimulationSettings(); + chosen = await this.openSimulationSettings(); } catch (error) { // No provider to choose from, or the list could not be loaded. Silence here reads as a dead // button, which is what the picker is opened by. @@ -1192,12 +1195,15 @@ export class TaskExecutionViewerComponent implements OnDestroy { ); return; } - if (!simulator) return; + if (!chosen) return; this.simulateInProgress.set(true); - this.taskExecutionsService.simulateExecution(executionId, simulator).subscribe({ + this.taskExecutionsService.simulateExecution(executionId, chosen.descriptor, chosen.credentialId).subscribe({ next: () => this.simulateInProgress.set(false), - error: () => this.simulateInProgress.set(false) + error: (error: unknown) => { + this.simulateInProgress.set(false); + this.notifications.show(this.executionErrorMessage(error), 'error', 6000); + } }); } @@ -1510,14 +1516,15 @@ export class TaskExecutionViewerComponent implements OnDestroy { }); } - private openSimulationSettings(): Promise { + private openSimulationSettings(): Promise { // A rerun carries the simulator of the run it repeats, and that is what the dialog starts from: // comparing two runs answered by different simulators compares the simulators too. const inherited = this.inheritedSimulator(); - return openLLMDescriptorSettings(this.settingsDialog, this.fieldRetriever, { - title: inherited ? 'Simulation Settings - repeating a simulated run' : 'Simulation Settings', - initialDescriptor: inherited - }); + return openLLMDescriptorSettingsWithCredential( + this.settingsDialog, this.fieldRetriever, this.llmProviderService, this.executionVaultCredentials, { + title: inherited ? 'Simulation Settings - repeating a simulated run' : 'Simulation Settings', + initialDescriptor: inherited + }); } private fetchExecutionLogs(executionId: string) {