From 7bfef72f798e75b5def1510a63e01c95c9e07c8d Mon Sep 17 00:00:00 2001 From: Lucio Lelii Date: Tue, 21 Jul 2026 10:34:22 +0200 Subject: [PATCH] feat: add bias impact experiments and biased reruns --- src/app/app.html | 2 + src/app/app.ts | 4 +- src/app/models/bias-impact.spec.ts | 102 +++++++++ src/app/models/bias-impact.ts | 155 +++++++++++++ src/app/models/flow.ts | 27 +++ src/app/models/task-execution.ts | 2 + src/app/services/blocks/block-call.base.ts | 5 + src/app/services/blocks/blocks-call.fake.ts | 37 +++ src/app/services/blocks/blocks-call.spec.ts | 52 +++++ src/app/services/blocks/blocks-call.ts | 47 +++- src/app/services/blocks/blocks.ts | 20 +- .../bias-impact-experiment-dialog.spec.ts | 24 ++ .../dialogs/bias-impact-experiment-dialog.ts | 28 +++ .../dialogs/bias-rerun-dialog.spec.ts | 20 ++ src/app/services/dialogs/bias-rerun-dialog.ts | 29 +++ .../task-executions-call.base.ts | 20 ++ .../task-executions-call.fake.ts | 144 +++++++++++- .../task-executions-call.spec.ts | 115 ++++++++++ .../task-executions/task-executions-call.ts | 169 +++++++++++++- .../task-executions/task-executions.spec.ts | 87 +++++++ .../task-executions/task-executions.ts | 149 +++++++++++- .../behavioral-probe-editor.css | 15 ++ .../behavioral-probe-editor.html | 107 +++++++++ .../behavioral-probe-editor.spec.ts | 89 ++++++++ .../behavioral-probe-editor.ts | 216 ++++++++++++++++++ .../bias-annotations/bias-annotations.css | 2 + .../bias-annotations/bias-annotations.html | 7 + .../bias-annotations/bias-annotations.spec.ts | 28 ++- .../bias-annotations/bias-annotations.ts | 27 ++- .../bias-impact-experiment-dialog.css | 7 + .../bias-impact-experiment-dialog.html | 24 ++ .../bias-impact-experiment-dialog.ts | 135 +++++++++++ .../bias-impact-report-viewer.css | 21 ++ .../bias-impact-report-viewer.html | 89 ++++++++ .../bias-impact-report-viewer.spec.ts | 64 ++++++ .../bias-impact-report-viewer.ts | 32 +++ .../bias-output-diff/bias-output-diff.css | 9 + .../bias-output-diff/bias-output-diff.html | 26 +++ .../bias-output-diff/bias-output-diff.spec.ts | 30 +++ .../bias-output-diff/bias-output-diff.ts | 22 ++ .../bias-rerun-dialog/bias-rerun-dialog.css | 5 + .../bias-rerun-dialog/bias-rerun-dialog.html | 18 ++ .../bias-rerun-dialog/bias-rerun-dialog.ts | 80 +++++++ src/app/shared/json-viewer/json-viewer.css | 8 + src/app/shared/json-viewer/json-viewer.html | 34 +++ .../shared/json-viewer/json-viewer.spec.ts | 28 +++ src/app/shared/json-viewer/json-viewer.ts | 62 +++++ .../nodes/generic-node/generic-node.html | 1 + .../shared/nodes/generic-node/generic-node.ts | 20 +- .../nodes/task-step-node/task-step-node.css | 14 ++ .../nodes/task-step-node/task-step-node.html | 11 + .../nodes/task-step-node/task-step-node.ts | 53 ++++- .../side-effect-policy-selector.css | 5 + .../side-effect-policy-selector.html | 10 + .../side-effect-policy-selector.spec.ts | 23 ++ .../side-effect-policy-selector.ts | 22 ++ .../task-execution-viewer.css | 10 + .../task-execution-viewer.html | 17 ++ .../task-execution-viewer.ts | 60 ++++- 59 files changed, 2656 insertions(+), 13 deletions(-) create mode 100644 src/app/models/bias-impact.spec.ts create mode 100644 src/app/models/bias-impact.ts create mode 100644 src/app/services/dialogs/bias-impact-experiment-dialog.spec.ts create mode 100644 src/app/services/dialogs/bias-impact-experiment-dialog.ts create mode 100644 src/app/services/dialogs/bias-rerun-dialog.spec.ts create mode 100644 src/app/services/dialogs/bias-rerun-dialog.ts create mode 100644 src/app/services/task-executions/task-executions-call.spec.ts create mode 100644 src/app/services/task-executions/task-executions.spec.ts create mode 100644 src/app/shared/behavioral-probe-editor/behavioral-probe-editor.css create mode 100644 src/app/shared/behavioral-probe-editor/behavioral-probe-editor.html create mode 100644 src/app/shared/behavioral-probe-editor/behavioral-probe-editor.spec.ts create mode 100644 src/app/shared/behavioral-probe-editor/behavioral-probe-editor.ts create mode 100644 src/app/shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog.css create mode 100644 src/app/shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog.html create mode 100644 src/app/shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog.ts create mode 100644 src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.css create mode 100644 src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.html create mode 100644 src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.spec.ts create mode 100644 src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.ts create mode 100644 src/app/shared/bias-output-diff/bias-output-diff.css create mode 100644 src/app/shared/bias-output-diff/bias-output-diff.html create mode 100644 src/app/shared/bias-output-diff/bias-output-diff.spec.ts create mode 100644 src/app/shared/bias-output-diff/bias-output-diff.ts create mode 100644 src/app/shared/bias-rerun-dialog/bias-rerun-dialog.css create mode 100644 src/app/shared/bias-rerun-dialog/bias-rerun-dialog.html create mode 100644 src/app/shared/bias-rerun-dialog/bias-rerun-dialog.ts create mode 100644 src/app/shared/json-viewer/json-viewer.css create mode 100644 src/app/shared/json-viewer/json-viewer.html create mode 100644 src/app/shared/json-viewer/json-viewer.spec.ts create mode 100644 src/app/shared/json-viewer/json-viewer.ts create mode 100644 src/app/shared/side-effect-policy-selector/side-effect-policy-selector.css create mode 100644 src/app/shared/side-effect-policy-selector/side-effect-policy-selector.html create mode 100644 src/app/shared/side-effect-policy-selector/side-effect-policy-selector.spec.ts create mode 100644 src/app/shared/side-effect-policy-selector/side-effect-policy-selector.ts diff --git a/src/app/app.html b/src/app/app.html index 294bd3b..a2be06e 100644 --- a/src/app/app.html +++ b/src/app/app.html @@ -4,3 +4,5 @@ + + diff --git a/src/app/app.ts b/src/app/app.ts index 2c8834e..050f53b 100644 --- a/src/app/app.ts +++ b/src/app/app.ts @@ -5,10 +5,12 @@ import { GlobalNotificationComponent } from '@shared/global-notification/global- import { HumanInteractionDialogHostComponent } from '@shared/human-interaction-dialog/human-interaction-dialog'; import { NodeSettingsDialogHostComponent } from '@shared/node-settings-dialog/node-settings-dialog'; import { SubflowPreviewDialogHostComponent } from '@shared/subflow-preview-dialog/subflow-preview-dialog'; +import { BiasImpactExperimentDialogHostComponent } from '@shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog'; +import { BiasRerunDialogHostComponent } from '@shared/bias-rerun-dialog/bias-rerun-dialog'; @Component({ selector: 'app-root', - imports: [RouterOutlet, ConfirmDialogHostComponent, GlobalNotificationComponent, HumanInteractionDialogHostComponent, NodeSettingsDialogHostComponent, SubflowPreviewDialogHostComponent], + imports: [RouterOutlet, ConfirmDialogHostComponent, GlobalNotificationComponent, HumanInteractionDialogHostComponent, NodeSettingsDialogHostComponent, SubflowPreviewDialogHostComponent, BiasImpactExperimentDialogHostComponent, BiasRerunDialogHostComponent], templateUrl: './app.html', styleUrl: './app.css', changeDetection: ChangeDetectionStrategy.OnPush diff --git a/src/app/models/bias-impact.spec.ts b/src/app/models/bias-impact.spec.ts new file mode 100644 index 0000000..6330953 --- /dev/null +++ b/src/app/models/bias-impact.spec.ts @@ -0,0 +1,102 @@ +import { + BIAS_EXPERIMENT_ERROR_CODES, + BIAS_PROBE_ERROR_CODES, + BiasImpactJob, + BiasImpactReport +} from './bias-impact'; +import { BehavioralProbe, isProbeExecutable } from './flow'; +import { TaskExecution } from './task-execution'; + +describe('bias impact models', () => { + it('recognizes executable instruction-based probes', () => { + expect(isProbeExecutable(undefined)).toBe(false); + expect(isProbeExecutable({ activationMode: 'PROMPT_DIRECTIVE' })).toBe(false); + expect(isProbeExecutable({ + activationMode: 'INPUT_TRANSFORMATION', + instruction: ' ${original} with experimental framing ' + })).toBe(true); + }); + + it('requires typed outputs for new MOCK_RESPONSE probes', () => { + const legacyProbe: BehavioralProbe = { + activationMode: 'MOCK_RESPONSE', + instruction: 'legacy mock response' + }; + const typedProbe: BehavioralProbe = { + activationMode: 'MOCK_RESPONSE', + mockOutputs: { body: 'controlled response', success: true } + }; + + expect(isProbeExecutable(legacyProbe)).toBe(false); + expect(isProbeExecutable(typedProbe)).toBe(true); + }); + + it('models completed jobs with the definitive report field names', () => { + const report: BiasImpactReport = { + id: 'report-1', + experimentId: 'experiment-1', + kind: 'ISOLATED_STEP', + baselineExecutionId: 'baseline-1', + biasedExecutionId: null, + nodeId: 'node-1', + annotationIds: ['annotation-1'], + repetitions: 3, + createdAt: '2026-07-21T10:00:00', + rawOutputsIncluded: true, + immediateImpact: { + outputChanged: true, + maximumTextDifference: 0.5, + changeRate: 1, + baselineOutput: { response: 'baseline' }, + biasedOutputs: [{ response: 'biased' }] + }, + downstreamImpact: [], + routingChanges: [], + mockedSideEffects: [{ nodeId: 'http-1', nodeName: 'HTTP call', kind: 'HTTP' }], + summary: 'The output changed.', + warnings: [] + }; + const job: BiasImpactJob = { + id: 'job-1', + status: 'COMPLETED', + executionId: 'baseline-1', + stepId: 'node-1', + createdAt: '2026-07-21T09:59:00', + startedAt: '2026-07-21T09:59:01', + completedAt: '2026-07-21T10:00:00', + reportId: report.id, + report, + errorCode: null, + errorMessage: null, + terminal: true + }; + + expect(job.report?.immediateImpact.biasedOutputs).toEqual([{ response: 'biased' }]); + expect(job.report?.mockedSideEffects[0]?.kind).toBe('HTTP'); + }); + + it('keeps bias execution context optional for existing executions', () => { + const execution = { + id: 'execution-1', + name: 'Legacy execution', + creationTime: 0, + context: { + inputs: {}, + result: {}, + errors: {}, + warnings: {}, + steps: {}, + status: 'SUCCESS', + waitingSteps: [] + } + } satisfies TaskExecution; + + expect(execution).not.toHaveProperty('biasExecutionContext'); + }); + + it('exports the backend error codes needed by probe and experiment flows', () => { + expect(BIAS_PROBE_ERROR_CODES).toContain('BIAS_PROBE_MOCK_OUTPUT_TYPE_MISMATCH'); + expect(BIAS_EXPERIMENT_ERROR_CODES).toContain('BIAS_SIDE_EFFECT_CONFIRMATION_REQUIRED'); + expect(BIAS_EXPERIMENT_ERROR_CODES).toContain('BIAS_EXECUTION_HISTORY_MISMATCH'); + }); +}); diff --git a/src/app/models/bias-impact.ts b/src/app/models/bias-impact.ts new file mode 100644 index 0000000..6890195 --- /dev/null +++ b/src/app/models/bias-impact.ts @@ -0,0 +1,155 @@ +import { BiasActivationMode } from './flow'; + +export type BiasCapabilities = { + blockType: string; + supported: boolean; + isolatedExperimentSupported: boolean; + fullFlowExperimentSupported: boolean; + externalSideEffects: boolean; + configurationDependent: boolean; + activationModes: BiasActivationMode[]; +}; + +export type ExternalSideEffectPolicy = 'BLOCK' | 'MOCK' | 'REQUIRE_CONFIRMATION'; + +export type BiasImpactExperimentRequest = { + annotationIds: string[]; + repetitions: number; + includeRawOutputs: boolean; + externalSideEffectPolicy: ExternalSideEffectPolicy; + confirmExternalSideEffects: boolean; +}; + +export type BiasRerunActivation = { + nodeId: string; + annotationIds: string[]; +}; + +export type BiasRerunRequest = { + activations: BiasRerunActivation[]; + externalSideEffectPolicy: ExternalSideEffectPolicy; + confirmExternalSideEffects: boolean; +}; + +export type BiasExecutionMode = 'NORMAL' | 'BIAS_VARIANT' | string; + +export type BiasExecutionContext = { + experimentId: string; + mode: BiasExecutionMode; + activeAnnotationIdsByNode: Record; + externalSideEffectPolicy: ExternalSideEffectPolicy; + externalSideEffectsConfirmed: boolean; +}; + +export type BiasImpactJobStatus = 'QUEUED' | 'RUNNING' | 'COMPLETED' | 'FAILED'; + +export type BiasImpactJob = { + id: string; + status: BiasImpactJobStatus; + executionId: string; + stepId: string; + createdAt: string; + startedAt: string | null; + completedAt: string | null; + reportId: string | null; + report: BiasImpactReport | null; + errorCode: string | null; + errorMessage: string | null; + terminal: boolean; +}; + +export type BiasImpactReportKind = 'ISOLATED_STEP' | 'FULL_FLOW'; + +export type BiasImmediateImpact = { + outputChanged: boolean; + maximumTextDifference: number; + changeRate: number; + baselineOutput: unknown; + biasedOutputs: unknown[]; +}; + +export type BiasDownstreamImpactEntry = { + nodeId: string; + nodeName: string; + baselineStatus: string; + biasedStatus: string; + changed: boolean; + baselineOutputs: unknown; + biasedOutputs: unknown; +}; + +export type BiasRoutingChangeEntry = { + nodeId: string; + baselineBranch: string; + biasedBranch: string; +}; + +export type BiasMockedSideEffectKind = + | 'HTTP' + | 'MCP_AGENT' + | 'MCP_AGENT_CHAT' + | 'EXTERNAL'; + +export type BiasMockedSideEffect = { + nodeId: string; + nodeName: string; + kind: BiasMockedSideEffectKind; +}; + +export type BiasImpactReport = { + id: string; + experimentId: string; + kind: BiasImpactReportKind; + baselineExecutionId: string; + biasedExecutionId: string | null; + nodeId: string | null; + annotationIds: string[]; + repetitions: number; + createdAt: string; + rawOutputsIncluded: boolean; + immediateImpact: BiasImmediateImpact; + downstreamImpact: BiasDownstreamImpactEntry[]; + routingChanges: BiasRoutingChangeEntry[]; + mockedSideEffects: BiasMockedSideEffect[]; + summary: string; + warnings: string[]; +}; + +export const BIAS_PROBE_ERROR_CODES = [ + 'BIAS_PROBE_MODE_REQUIRED', + 'BIAS_PROBE_INSTRUCTION_REQUIRED', + 'BIAS_PROBE_MODE_UNSUPPORTED', + 'BIAS_PROBE_TARGET_INPUT_NOT_FOUND', + 'BIAS_PROBE_MOCK_OUTPUTS_REQUIRED', + 'BIAS_PROBE_MOCK_OUTPUT_NOT_FOUND', + 'BIAS_PROBE_MOCK_OUTPUT_TYPE_MISMATCH', + 'DUPLICATE_BIAS_ANNOTATION_ID', + 'TOO_MANY_BIAS_ANNOTATIONS', + 'BIAS_FIELD_TOO_LONG' +] as const; + +export type BiasProbeErrorCode = typeof BIAS_PROBE_ERROR_CODES[number]; + +export const BIAS_EXPERIMENT_ERROR_CODES = [ + 'BIAS_BASELINE_NOT_FINAL', + 'BIAS_STEP_NOT_FOUND', + 'BIAS_BLOCK_NOT_SUPPORTED', + 'BIAS_ANNOTATION_NOT_FOUND', + 'BIAS_ANNOTATION_NOT_EXECUTABLE', + 'BIAS_EXECUTION_NOT_FINAL', + 'BIAS_EXECUTION_NOT_VARIANT', + 'BIAS_EXECUTION_HISTORY_MISMATCH', + 'BIAS_SIDE_EFFECT_BLOCKED', + 'BIAS_SIDE_EFFECT_CONFIRMATION_REQUIRED', + 'BIAS_JOB_NOT_FOUND', + 'BIAS_REPORT_NOT_FOUND', + 'BIAS_EXPERIMENT_FAILED' +] as const; + +export type BiasExperimentErrorCode = typeof BIAS_EXPERIMENT_ERROR_CODES[number]; + +export type BiasSideEffectError = { + reason: 'SIDE_EFFECT_BLOCKED' | 'CONFIRMATION_REQUIRED'; + code: 'BIAS_SIDE_EFFECT_BLOCKED' | 'BIAS_SIDE_EFFECT_CONFIRMATION_REQUIRED'; + message: string; +}; diff --git a/src/app/models/flow.ts b/src/app/models/flow.ts index 79cc17b..364534b 100644 --- a/src/app/models/flow.ts +++ b/src/app/models/flow.ts @@ -86,6 +86,32 @@ export type FlowNodeBase = { nodeFamily?: NodeFamily; }; +export type BiasActivationMode = + | 'PROMPT_DIRECTIVE' + | 'INPUT_TRANSFORMATION' + | 'OUTPUT_TRANSFORMATION' + | 'ROUTING_OVERRIDE' + | 'MOCK_RESPONSE' + | string; + +export type BehavioralProbe = { + activationMode?: BiasActivationMode; + instruction?: string; + targetInputs?: string[]; + expectedImpact?: string; + mockOutputs?: Record; +}; + +export function isProbeExecutable(probe: BehavioralProbe | null | undefined): boolean { + if (!probe?.activationMode) return false; + + if (probe.activationMode === 'MOCK_RESPONSE') { + return !!probe.mockOutputs && Object.keys(probe.mockOutputs).length > 0; + } + + return typeof probe.instruction === 'string' && probe.instruction.trim().length > 0; +} + export type BiasAnnotation = Record & { id?: string; category?: string; @@ -96,6 +122,7 @@ export type BiasAnnotation = Record & { status?: string; source?: string; analysisId?: string; + behavioralProbe?: BehavioralProbe; }; export type BiasAnnotationOption = { diff --git a/src/app/models/task-execution.ts b/src/app/models/task-execution.ts index 0331e4a..cdeb286 100644 --- a/src/app/models/task-execution.ts +++ b/src/app/models/task-execution.ts @@ -1,4 +1,5 @@ import { FlowBlockConnection, FlowNode, FlowNodeDependency, FlowPort, LLMDescriptor } from './flow'; +import { BiasExecutionContext } from './bias-impact'; export type TaskExecutionStatus = 'CREATED' | 'READY' | 'RUNNING' | 'WAITING' | 'SUSPENDED' | 'SUCCESS' | 'ERROR' | 'CANCELLED'; export type TaskExecutionStatusGroup = 'INIT' | 'RUNNING' | 'PAUSED' | 'FINAL'; @@ -25,6 +26,7 @@ export type TaskExecution = { sourceFlowId?: string | null; runNumber?: number | null; rerunOfExecutionId?: string | null; + biasExecutionContext?: BiasExecutionContext; context: TaskExecutionContext; interactionSimulationEnabled?: boolean; simulationAvailable?: boolean; diff --git a/src/app/services/blocks/block-call.base.ts b/src/app/services/blocks/block-call.base.ts index 65b090d..fcbb623 100644 --- a/src/app/services/blocks/block-call.base.ts +++ b/src/app/services/blocks/block-call.base.ts @@ -1,4 +1,5 @@ import { BiasAnnotationsDescriptor, BlockType, BlockTypeName, FlowBlock } from "@models/flow"; +import { BiasCapabilities } from "@models/bias-impact"; import { Observable } from "rxjs"; export type BlockDraftContext = { @@ -12,6 +13,10 @@ export abstract class BlocksCallServiceBase { abstract retrieveBiasAnnotationsDescriptor(): Observable; + abstract retrieveBiasCapabilities(blockType: string): Observable; + + abstract retrieveBiasCapabilitiesForInstance(blockType: string, block: FlowBlock): Observable; + abstract createEmptyBlock(blockType: BlockTypeName, context?: BlockDraftContext) : Observable; abstract updateBlock(blockId : string, configuration : any, context?: BlockDraftContext) : Observable; diff --git a/src/app/services/blocks/blocks-call.fake.ts b/src/app/services/blocks/blocks-call.fake.ts index 90b8fba..9cbaabd 100644 --- a/src/app/services/blocks/blocks-call.fake.ts +++ b/src/app/services/blocks/blocks-call.fake.ts @@ -1,4 +1,5 @@ import { BiasAnnotationsDescriptor, BlockType, FlowBlock } from "@models/flow"; +import { BiasCapabilities } from '@models/bias-impact'; import { Observable, of } from "rxjs"; import { BlockDraftContext, BlocksCallServiceBase } from "./block-call.base"; @@ -179,6 +180,20 @@ export class BlocksCallServiceFake extends BlocksCallServiceBase { }); } + override retrieveBiasCapabilities(blockType: string): Observable { + return of(this.biasCapabilities(blockType, false)); + } + + override retrieveBiasCapabilitiesForInstance(blockType: string, block: FlowBlock): Observable { + const usesLlm = Boolean((block.specificConfiguration as Record)['useLlm']); + return of({ + ...this.biasCapabilities(blockType, true), + activationModes: usesLlm + ? ['PROMPT_DIRECTIVE', 'INPUT_TRANSFORMATION', 'OUTPUT_TRANSFORMATION'] + : ['INPUT_TRANSFORMATION', 'OUTPUT_TRANSFORMATION'] + }); + } + override createEmptyBlock(blockType: string, _context?: BlockDraftContext): Observable { const descriptor = this.blockTypes.find((b) => b.type === blockType); const typeName = descriptor?.type ?? blockType ?? "LLMBlock"; @@ -263,6 +278,28 @@ export class BlocksCallServiceFake extends BlocksCallServiceBase { return sanitized; } + private biasCapabilities(blockType: string, instanceSpecific: boolean): BiasCapabilities { + const configurationDependent = blockType === 'ConditionalBlock' || blockType === 'SwitchBlock'; + const externalSideEffects = blockType === 'HTTPServerCallBlock' + || blockType === 'MCPAgentBlock' + || blockType === 'MCPAgentChatBlock'; + const activationModes = externalSideEffects + ? ['MOCK_RESPONSE'] + : configurationDependent && !instanceSpecific + ? ['INPUT_TRANSFORMATION', 'OUTPUT_TRANSFORMATION'] + : ['PROMPT_DIRECTIVE', 'INPUT_TRANSFORMATION', 'OUTPUT_TRANSFORMATION']; + + return { + blockType, + supported: activationModes.length > 0, + isolatedExperimentSupported: true, + fullFlowExperimentSupported: activationModes.length > 0, + externalSideEffects, + configurationDependent, + activationModes + }; + } + private sanitizeSchemaValue( value: unknown, schemaNode: Record | null, diff --git a/src/app/services/blocks/blocks-call.spec.ts b/src/app/services/blocks/blocks-call.spec.ts index b3747ed..4890f2f 100644 --- a/src/app/services/blocks/blocks-call.spec.ts +++ b/src/app/services/blocks/blocks-call.spec.ts @@ -116,4 +116,56 @@ describe('BlocksCallService', () => { options: { category: [{ value: 'DYNAMIC_VALUE', label: 'Dynamic label', description: 'Dynamic description' }] } })); }); + + it('retrieves and caches type-level bias capabilities', async () => { + const first = firstValueFrom(service.retrieveBiasCapabilities('LLMBlock')); + const request = httpMock.expectOne(`${environment.apiUrl}/blocks/types/LLMBlock/bias-capabilities`); + expect(request.request.method).toBe('GET'); + request.flush({ + blockType: 'LLMBlock', + supported: true, + isolatedExperimentSupported: true, + fullFlowExperimentSupported: true, + externalSideEffects: false, + configurationDependent: false, + activationModes: ['PROMPT_DIRECTIVE'] + }); + + await expect(first).resolves.toEqual(expect.objectContaining({ + blockType: 'LLMBlock', activationModes: ['PROMPT_DIRECTIVE'] + })); + await expect(firstValueFrom(service.retrieveBiasCapabilities('LLMBlock'))).resolves.toEqual(expect.objectContaining({ + supported: true + })); + }); + + it('posts the configured block for instance-specific bias capabilities', async () => { + const block = { + id: 'conditional-1', + name: 'Conditional', + inputs: [], + outputs: [{ name: 'true', type: 'TEXT', multiple: false }], + specificConfiguration: { useLlm: true }, + typeName: 'ConditionalBlock', + nodeFamily: 'block' as const + }; + const response = firstValueFrom(service.retrieveBiasCapabilitiesForInstance('ConditionalBlock', block)); + const request = httpMock.expectOne(`${environment.apiUrl}/blocks/types/ConditionalBlock/bias-capabilities`); + expect(request.request.method).toBe('POST'); + expect(request.request.body).toEqual(block); + request.flush({ + blockType: 'ConditionalBlock', + supported: true, + isolatedExperimentSupported: true, + fullFlowExperimentSupported: true, + externalSideEffects: false, + configurationDependent: true, + activationModes: ['PROMPT_DIRECTIVE', 'INPUT_TRANSFORMATION'] + }); + + await expect(response).resolves.toEqual(expect.objectContaining({ + configurationDependent: true, + activationModes: ['PROMPT_DIRECTIVE', 'INPUT_TRANSFORMATION'] + })); + }); }); diff --git a/src/app/services/blocks/blocks-call.ts b/src/app/services/blocks/blocks-call.ts index 256cde3..f467e1b 100644 --- a/src/app/services/blocks/blocks-call.ts +++ b/src/app/services/blocks/blocks-call.ts @@ -1,4 +1,5 @@ -import { BiasAnnotationOption, BiasAnnotationsDescriptor, BlockType, FlowBlock } from "@models/flow"; +import { BiasActivationMode, BiasAnnotationOption, BiasAnnotationsDescriptor, BlockType, FlowBlock } from "@models/flow"; +import { BiasCapabilities } from '@models/bias-impact'; import { HttpClient, HttpParams } from "@angular/common/http"; import { inject } from "@angular/core"; import { environment } from "@environment"; @@ -8,6 +9,7 @@ import { BlockDraftContext, BlocksCallServiceBase } from "./block-call.base"; export class BlocksCallService extends BlocksCallServiceBase { private readonly http = inject(HttpClient); private blockTypesCache: BlockType[] | null = null; + private readonly biasCapabilitiesCache = new Map(); override retrieveAllBlocksTypes(): Observable { return this.http @@ -27,6 +29,30 @@ export class BlocksCallService extends BlocksCallServiceBase { .pipe(map((raw) => this.biasAnnotationsDescriptorFromApi(raw))); } + override retrieveBiasCapabilities(blockType: string): Observable { + const cached = this.biasCapabilitiesCache.get(blockType); + if (cached) return of(cached); + + return this.http + .get(`${environment.apiUrl}/blocks/types/${encodeURIComponent(blockType)}/bias-capabilities`) + .pipe( + map((raw) => this.biasCapabilitiesFromApi(raw, blockType)), + map((capabilities) => { + this.biasCapabilitiesCache.set(blockType, capabilities); + return capabilities; + }) + ); + } + + override retrieveBiasCapabilitiesForInstance(blockType: string, block: FlowBlock): Observable { + return this.http + .post( + `${environment.apiUrl}/blocks/types/${encodeURIComponent(blockType)}/bias-capabilities`, + block + ) + .pipe(map((raw) => this.biasCapabilitiesFromApi(raw, blockType))); + } + override createEmptyBlock(blockType: string, context?: BlockDraftContext): Observable { return this.getBlockTypesForCreate().pipe( take(1), @@ -186,6 +212,25 @@ export class BlocksCallService extends BlocksCallServiceBase { }; } + private biasCapabilitiesFromApi(raw: unknown, fallbackBlockType: string): BiasCapabilities { + const value = this.toRecord(raw); + const activationModes = Array.isArray(value['activationModes']) + ? value['activationModes'] + .filter((mode): mode is string => typeof mode === 'string') + .map((mode) => mode as BiasActivationMode) + : []; + + return { + blockType: String(value['blockType'] ?? fallbackBlockType), + supported: value['supported'] === true, + isolatedExperimentSupported: value['isolatedExperimentSupported'] === true, + fullFlowExperimentSupported: value['fullFlowExperimentSupported'] === true, + externalSideEffects: value['externalSideEffects'] === true, + configurationDependent: value['configurationDependent'] === true, + activationModes + }; + } + private toPorts(raw: unknown, fallback: Array<{ name: string; type: string; multiple: boolean }>) { if (!Array.isArray(raw)) return fallback; return raw diff --git a/src/app/services/blocks/blocks.ts b/src/app/services/blocks/blocks.ts index 853185c..90f0c59 100644 --- a/src/app/services/blocks/blocks.ts +++ b/src/app/services/blocks/blocks.ts @@ -1,8 +1,9 @@ import { computed, Injectable, signal } from '@angular/core'; import { environment } from '@environment'; import { BiasAnnotationsDescriptor, BlockType, BlockTypeName, FlowBlock } from '@models/flow'; +import { BiasCapabilities } from '@models/bias-impact'; import { BlockDraftContext, BlocksCallServiceBase } from './block-call.base'; -import { catchError, finalize, firstValueFrom, map, Observable, of, shareReplay, throwError } from 'rxjs'; +import { catchError, finalize, firstValueFrom, map, Observable, of, shareReplay, tap, throwError } from 'rxjs'; @Injectable({ providedIn: 'root', @@ -19,11 +20,13 @@ export class BlocksService { private _blockTypes = signal([]); private readonly _biasAnnotationsDescriptor = signal(null); + private readonly _biasCapabilities = signal>({}); private biasDescriptorPromise: Promise | null = null; readonly hasPendingServerSync = computed(() => this.pendingServerSyncCount() > 0); readonly blockTypes = this._blockTypes.asReadonly(); readonly catalogLoading = this._catalogLoading.asReadonly(); readonly biasAnnotationsDescriptor = this._biasAnnotationsDescriptor.asReadonly(); + readonly biasCapabilities = this._biasCapabilities.asReadonly(); async getBiasAnnotationsDescriptor(force = false): Promise { const cached = this._biasAnnotationsDescriptor(); @@ -39,6 +42,21 @@ export class BlocksService { return this.biasDescriptorPromise; } + retrieveBiasCapabilities(blockType: string, force = false): Observable { + const cached = this._biasCapabilities()[blockType]; + if (cached && !force) return of(cached); + + return this.blocksCallService.retrieveBiasCapabilities(blockType).pipe( + tap((capabilities) => { + this._biasCapabilities.update((current) => ({ ...current, [blockType]: capabilities })); + }) + ); + } + + retrieveBiasCapabilitiesForInstance(blockType: string, block: FlowBlock): Observable { + return this.blocksCallService.retrieveBiasCapabilitiesForInstance(blockType, block); + } + hasLoadedBlockTypes() { return this._blockTypes().length > 0 || (!this.toInit && !this.loadingPromise); } diff --git a/src/app/services/dialogs/bias-impact-experiment-dialog.spec.ts b/src/app/services/dialogs/bias-impact-experiment-dialog.spec.ts new file mode 100644 index 0000000..7e19563 --- /dev/null +++ b/src/app/services/dialogs/bias-impact-experiment-dialog.spec.ts @@ -0,0 +1,24 @@ +import { TestBed } from '@angular/core/testing'; +import { BiasImpactExperimentDialogService } from './bias-impact-experiment-dialog'; + +describe('BiasImpactExperimentDialogService', () => { + let service: BiasImpactExperimentDialogService; + + beforeEach(() => { + TestBed.configureTestingModule({}); + service = TestBed.inject(BiasImpactExperimentDialogService); + }); + + it('opens only when at least one annotation has an executable probe', () => { + service.open({ + executionId: 'execution', stepId: 'step', nodeId: 'node', nodeName: 'Node', + capabilities: { blockType: 'LLM', supported: true, isolatedExperimentSupported: true, fullFlowExperimentSupported: true, externalSideEffects: false, configurationDependent: false, activationModes: [] }, + annotations: [ + { id: 'not-executable', behavioralProbe: { activationMode: 'PROMPT_DIRECTIVE' } }, + { id: 'executable', behavioralProbe: { activationMode: 'PROMPT_DIRECTIVE', instruction: 'Apply probe' } } + ] + }); + + expect(service.state()?.annotations.map((annotation) => annotation.id)).toEqual(['executable']); + }); +}); diff --git a/src/app/services/dialogs/bias-impact-experiment-dialog.ts b/src/app/services/dialogs/bias-impact-experiment-dialog.ts new file mode 100644 index 0000000..26cc0a5 --- /dev/null +++ b/src/app/services/dialogs/bias-impact-experiment-dialog.ts @@ -0,0 +1,28 @@ +import { Injectable, signal } from '@angular/core'; +import { BiasAnnotation, isProbeExecutable } from '@models/flow'; +import { BiasCapabilities } from '@models/bias-impact'; + +export type BiasImpactExperimentDialogInput = { + executionId: string; + stepId: string; + nodeId: string; + nodeName: string; + annotations: BiasAnnotation[]; + capabilities: BiasCapabilities; +}; + +@Injectable({ providedIn: 'root' }) +export class BiasImpactExperimentDialogService { + private readonly _state = signal(null); + readonly state = this._state.asReadonly(); + + open(input: BiasImpactExperimentDialogInput) { + const annotations = input.annotations.filter((annotation) => isProbeExecutable(annotation.behavioralProbe)); + if (!annotations.length) return; + this._state.set({ ...input, annotations }); + } + + close() { + this._state.set(null); + } +} diff --git a/src/app/services/dialogs/bias-rerun-dialog.spec.ts b/src/app/services/dialogs/bias-rerun-dialog.spec.ts new file mode 100644 index 0000000..232990d --- /dev/null +++ b/src/app/services/dialogs/bias-rerun-dialog.spec.ts @@ -0,0 +1,20 @@ +import { TestBed } from '@angular/core/testing'; +import { BiasRerunDialogService } from './bias-rerun-dialog'; + +describe('BiasRerunDialogService', () => { + it('keeps a dialog state only when there are eligible block candidates', () => { + const service = TestBed.inject(BiasRerunDialogService); + const base = { executionId: 'baseline', onCreated: () => undefined }; + service.open({ ...base, candidates: [] }); + expect(service.state()).toBeNull(); + + service.open({ + ...base, + candidates: [{ + nodeId: 'node-1', nodeName: 'Node 1', annotations: [], + capabilities: { blockType: 'LLM', supported: true, isolatedExperimentSupported: true, fullFlowExperimentSupported: true, externalSideEffects: false, configurationDependent: false, activationModes: [] } + }] + }); + expect(service.state()?.executionId).toBe('baseline'); + }); +}); diff --git a/src/app/services/dialogs/bias-rerun-dialog.ts b/src/app/services/dialogs/bias-rerun-dialog.ts new file mode 100644 index 0000000..1dc9546 --- /dev/null +++ b/src/app/services/dialogs/bias-rerun-dialog.ts @@ -0,0 +1,29 @@ +import { Injectable, signal } from '@angular/core'; +import { BiasAnnotation } from '@models/flow'; +import { BiasCapabilities } from '@models/bias-impact'; +import { TaskExecution } from '@models/task-execution'; + +export type BiasRerunCandidate = { + nodeId: string; + nodeName: string; + annotations: BiasAnnotation[]; + capabilities: BiasCapabilities; +}; + +export type BiasRerunDialogInput = { + executionId: string; + candidates: BiasRerunCandidate[]; + onCreated: (execution: TaskExecution) => void; +}; + +@Injectable({ providedIn: 'root' }) +export class BiasRerunDialogService { + private readonly _state = signal(null); + readonly state = this._state.asReadonly(); + + open(input: BiasRerunDialogInput) { + if (input.candidates.length) this._state.set(input); + } + + close() { this._state.set(null); } +} diff --git a/src/app/services/task-executions/task-executions-call.base.ts b/src/app/services/task-executions/task-executions-call.base.ts index 5255690..e78a007 100644 --- a/src/app/services/task-executions/task-executions-call.base.ts +++ b/src/app/services/task-executions/task-executions-call.base.ts @@ -1,4 +1,10 @@ import { LLMDescriptor } from '@models/flow'; +import { + BiasImpactExperimentRequest, + BiasImpactJob, + BiasImpactReport, + BiasRerunRequest +} from '@models/bias-impact'; import { ExecutionEventLogEntry, TaskExecution, TaskExecutionGroup } from '@models/task-execution'; import { Observable } from 'rxjs'; @@ -8,6 +14,20 @@ export abstract class TaskExecutionsCallServiceBase { abstract retrieveExecutionEvents(executionId: string): Observable; abstract createTaskExecution(flowId: string): Observable; abstract rerunTaskExecution(executionId: string): Observable; + abstract runBiasImpactExperiment( + executionId: string, + stepId: string, + request: BiasImpactExperimentRequest + ): Observable; + abstract getBiasImpactJob(jobId: string): Observable; + abstract createBiasedRerun(executionId: string, request: BiasRerunRequest): Observable; + abstract compareBiasExecutions( + baselineExecutionId: string, + biasedExecutionId: string, + includeRawOutputs: boolean + ): Observable; + abstract listBiasImpactReports(executionId: string): Observable; + abstract getBiasImpactReport(reportId: string): Observable; abstract deleteTaskExecution(executionId: string): Observable; abstract startTaskExecution(executionId: string): Observable; abstract simulateTaskExecution(executionId: string, simulator: LLMDescriptor): Observable; diff --git a/src/app/services/task-executions/task-executions-call.fake.ts b/src/app/services/task-executions/task-executions-call.fake.ts index aafb692..8e90937 100644 --- a/src/app/services/task-executions/task-executions-call.fake.ts +++ b/src/app/services/task-executions/task-executions-call.fake.ts @@ -1,9 +1,17 @@ import { LLMDescriptor } from '@models/flow'; +import { + BiasImpactExperimentRequest, + BiasImpactJob, + BiasImpactReport, + BiasRerunRequest +} from '@models/bias-impact'; import { ExecutionEventLogEntry, TaskExecution, TaskExecutionGroup } from '@models/task-execution'; -import { Observable, of } from 'rxjs'; +import { map, Observable, of } from 'rxjs'; import { TaskExecutionsCallServiceBase } from './task-executions-call.base'; export class TaskExecutionsCallServiceFake extends TaskExecutionsCallServiceBase { + private readonly biasJobs = new Map(); + private readonly biasReports: BiasImpactReport[] = []; private readonly data: TaskExecution[] = [ { id: 'c106be9d-5467-428c-8992-0b5f40a59aac', @@ -518,6 +526,113 @@ export class TaskExecutionsCallServiceFake extends TaskExecutionsCallServiceBase return of(this.withSimulationAvailability(execution)); } + override runBiasImpactExperiment( + executionId: string, + stepId: string, + request: BiasImpactExperimentRequest + ): Observable { + const job: BiasImpactJob = { + id: crypto.randomUUID(), + status: 'QUEUED', + executionId, + stepId, + createdAt: new Date().toISOString(), + startedAt: null, + completedAt: null, + reportId: null, + report: null, + errorCode: null, + errorMessage: null, + terminal: false + }; + this.biasJobs.set(job.id, { polls: 0, job: { ...job, report: null } }); + return of(job); + } + + override getBiasImpactJob(jobId: string): Observable { + const entry = this.biasJobs.get(jobId); + if (!entry) { + throw new Error(`Bias impact job not found: ${jobId}`); + } + + entry.polls += 1; + if (entry.polls === 1) { + entry.job = { ...entry.job, status: 'RUNNING', startedAt: new Date().toISOString() }; + } else if (!entry.job.terminal) { + const report = this.createIsolatedReport(entry.job.executionId, entry.job.stepId); + this.biasReports.unshift(report); + entry.job = { + ...entry.job, + status: 'COMPLETED', + completedAt: new Date().toISOString(), + reportId: report.id, + report, + terminal: true + }; + } + return of(entry.job); + } + + override createBiasedRerun(executionId: string, request: BiasRerunRequest): Observable { + return this.rerunTaskExecution(executionId).pipe( + map((execution) => { + const biasedExecution: TaskExecution = { + ...execution, + biasExecutionContext: { + experimentId: crypto.randomUUID(), + mode: 'BIAS_VARIANT', + activeAnnotationIdsByNode: Object.fromEntries( + request.activations.map((activation) => [activation.nodeId, activation.annotationIds]) + ), + externalSideEffectPolicy: request.externalSideEffectPolicy, + externalSideEffectsConfirmed: request.confirmExternalSideEffects + } + }; + const index = this.data.findIndex((item) => item.id === biasedExecution.id); + if (index >= 0) this.data[index] = biasedExecution; + return biasedExecution; + }) + ); + } + + override compareBiasExecutions( + baselineExecutionId: string, + biasedExecutionId: string, + includeRawOutputs: boolean + ): Observable { + const existing = this.biasReports.find((report) => + report.baselineExecutionId === baselineExecutionId + && report.biasedExecutionId === biasedExecutionId + && report.rawOutputsIncluded === includeRawOutputs + ); + if (existing) return of(existing); + + const report: BiasImpactReport = { + ...this.createIsolatedReport(baselineExecutionId, 'biased-node'), + id: crypto.randomUUID(), + kind: 'FULL_FLOW', + biasedExecutionId, + nodeId: null, + repetitions: 1, + rawOutputsIncluded: includeRawOutputs, + summary: 'Observed a changed downstream node in the biased rerun.' + }; + this.biasReports.unshift(report); + return of(report); + } + + override listBiasImpactReports(executionId: string): Observable { + return of(this.biasReports.filter((report) => + report.baselineExecutionId === executionId || report.biasedExecutionId === executionId + )); + } + + override getBiasImpactReport(reportId: string): Observable { + const report = this.biasReports.find((item) => item.id === reportId); + if (!report) throw new Error(`Bias impact report not found: ${reportId}`); + return of(report); + } + override deleteTaskExecution(executionId: string): Observable { const index = this.data.findIndex((item) => item.id === executionId); if (index >= 0) { @@ -727,6 +842,33 @@ export class TaskExecutionsCallServiceFake extends TaskExecutionsCallServiceBase return of(execution); } + private createIsolatedReport(baselineExecutionId: string, stepId: string): BiasImpactReport { + return { + id: crypto.randomUUID(), + experimentId: crypto.randomUUID(), + kind: 'ISOLATED_STEP', + baselineExecutionId, + biasedExecutionId: null, + nodeId: stepId, + annotationIds: ['demo-bias-annotation'], + repetitions: 3, + createdAt: new Date().toISOString(), + rawOutputsIncluded: true, + immediateImpact: { + outputChanged: true, + maximumTextDifference: 0.4, + changeRate: 1, + baselineOutput: { output: 'Baseline result' }, + biasedOutputs: [{ output: 'Biased result' }] + }, + downstreamImpact: [], + routingChanges: [], + mockedSideEffects: [], + summary: 'The selected bias changed the observed output.', + warnings: ['This comparison can include normal model non-determinism.'] + }; + } + private findExecution(executionId: string): TaskExecution { const execution = this.data.find((item) => item.id === executionId); if (!execution) { diff --git a/src/app/services/task-executions/task-executions-call.spec.ts b/src/app/services/task-executions/task-executions-call.spec.ts new file mode 100644 index 0000000..e667de0 --- /dev/null +++ b/src/app/services/task-executions/task-executions-call.spec.ts @@ -0,0 +1,115 @@ +import { provideHttpClient } from '@angular/common/http'; +import { HttpTestingController, provideHttpClientTesting } from '@angular/common/http/testing'; +import { TestBed } from '@angular/core/testing'; +import { environment } from '@environment'; +import { firstValueFrom } from 'rxjs'; + +import { TaskExecutionsCallService } from './task-executions-call'; + +const report = { + id: 'report-1', + experimentId: 'experiment-1', + kind: 'ISOLATED_STEP', + baselineExecutionId: 'execution-1', + biasedExecutionId: null, + nodeId: 'step-1', + annotationIds: ['annotation-1'], + repetitions: 3, + createdAt: '2026-07-21T10:00:00', + rawOutputsIncluded: true, + immediateImpact: { + outputChanged: true, + maximumTextDifference: 0.4, + changeRate: 1, + baselineOutput: { output: 'baseline' }, + biasedOutputs: [{ output: 'biased' }] + }, + downstreamImpact: [], + routingChanges: [], + mockedSideEffects: [{ nodeId: 'http-1', nodeName: 'HTTP call', kind: 'HTTP' }], + summary: 'Changed output', + warnings: [] +}; + +describe('TaskExecutionsCallService bias APIs', () => { + let service: TaskExecutionsCallService; + let httpMock: HttpTestingController; + + beforeEach(() => { + TestBed.configureTestingModule({ + providers: [TaskExecutionsCallService, provideHttpClient(), provideHttpClientTesting()] + }); + service = TestBed.inject(TaskExecutionsCallService); + httpMock = TestBed.inject(HttpTestingController); + }); + + afterEach(() => httpMock.verify()); + + it('starts an asynchronous impact experiment and maps the job response', async () => { + const result = firstValueFrom(service.runBiasImpactExperiment('execution-1', 'step-1', { + annotationIds: ['annotation-1'], + repetitions: 3, + includeRawOutputs: true, + externalSideEffectPolicy: 'BLOCK', + confirmExternalSideEffects: false + })); + const request = httpMock.expectOne(`${environment.apiUrl}/executions/execution-1/steps/step-1/bias-impact`); + expect(request.request.method).toBe('POST'); + request.flush({ + id: 'job-1', status: 'QUEUED', executionId: 'execution-1', stepId: 'step-1', + createdAt: '2026-07-21T10:00:00', startedAt: null, completedAt: null, + reportId: null, report: null, errorCode: null, errorMessage: null, terminal: false + }); + + await expect(result).resolves.toEqual(expect.objectContaining({ id: 'job-1', status: 'QUEUED', terminal: false })); + }); + + it('retrieves a completed job with its persisted report', async () => { + const result = firstValueFrom(service.getBiasImpactJob('job-1')); + const request = httpMock.expectOne(`${environment.apiUrl}/executions/bias-impact-jobs/job-1`); + expect(request.request.method).toBe('GET'); + request.flush({ + id: 'job-1', status: 'COMPLETED', executionId: 'execution-1', stepId: 'step-1', + createdAt: '2026-07-21T10:00:00', startedAt: '2026-07-21T10:00:01', completedAt: '2026-07-21T10:00:02', + reportId: 'report-1', report, errorCode: null, errorMessage: null, terminal: true + }); + + await expect(result).resolves.toEqual(expect.objectContaining({ + status: 'COMPLETED', reportId: 'report-1', report: expect.objectContaining({ mockedSideEffects: report.mockedSideEffects }) + })); + }); + + it('uses the confirmed biased rerun and comparison routes', async () => { + const rerun = firstValueFrom(service.createBiasedRerun('baseline-1', { + activations: [{ nodeId: 'node-1', annotationIds: ['annotation-1'] }], + externalSideEffectPolicy: 'MOCK', + confirmExternalSideEffects: false + })); + const rerunRequest = httpMock.expectOne(`${environment.apiUrl}/executions/baseline-1/bias-rerun`); + expect(rerunRequest.request.method).toBe('POST'); + rerunRequest.flush({ id: 'variant-1', name: 'Variant', creationTime: 1, context: {} }); + await expect(rerun).resolves.toEqual(expect.objectContaining({ id: 'variant-1' })); + + const comparison = firstValueFrom(service.compareBiasExecutions('baseline-1', 'variant-1', true)); + const comparisonRequest = httpMock.expectOne( + `${environment.apiUrl}/executions/baseline-1/bias-compare/variant-1?includeRawOutputs=true` + ); + expect(comparisonRequest.request.method).toBe('POST'); + comparisonRequest.flush({ ...report, baselineExecutionId: 'baseline-1', biasedExecutionId: 'variant-1', kind: 'FULL_FLOW' }); + await expect(comparison).resolves.toEqual(expect.objectContaining({ kind: 'FULL_FLOW', biasedExecutionId: 'variant-1' })); + }); + + it('lists and retrieves persisted reports', async () => { + const listed = firstValueFrom(service.listBiasImpactReports('execution-1')); + const listRequest = httpMock.expectOne(`${environment.apiUrl}/executions/execution-1/bias-impact-reports`); + expect(listRequest.request.method).toBe('GET'); + listRequest.flush([report]); + await expect(listed).resolves.toEqual([expect.objectContaining({ id: 'report-1' })]); + + const detail = firstValueFrom(service.getBiasImpactReport('report-1')); + const detailRequest = httpMock.expectOne(`${environment.apiUrl}/executions/bias-impact-reports/report-1`); + expect(detailRequest.request.method).toBe('GET'); + detailRequest.flush(report); + await expect(detail).resolves.toEqual(expect.objectContaining({ id: 'report-1' })); + }); +}); diff --git a/src/app/services/task-executions/task-executions-call.ts b/src/app/services/task-executions/task-executions-call.ts index 5c47680..b55f052 100644 --- a/src/app/services/task-executions/task-executions-call.ts +++ b/src/app/services/task-executions/task-executions-call.ts @@ -1,7 +1,18 @@ -import { HttpClient } from '@angular/common/http'; +import { HttpClient, HttpParams } from '@angular/common/http'; import { inject } from '@angular/core'; import { environment } from '@environment'; import { LLMDescriptor } from '@models/flow'; +import { + BiasDownstreamImpactEntry, + BiasImpactExperimentRequest, + BiasImpactJob, + BiasImpactJobStatus, + BiasImpactReport, + BiasImpactReportKind, + BiasMockedSideEffect, + BiasRerunRequest, + BiasRoutingChangeEntry +} from '@models/bias-impact'; import { ExecutionEventLogEntry, TaskExecution, TaskExecutionGroup } from '@models/task-execution'; import { map, Observable } from 'rxjs'; import { TaskExecutionsCallServiceBase } from './task-executions-call.base'; @@ -37,6 +48,50 @@ export class TaskExecutionsCallService extends TaskExecutionsCallServiceBase { ); } + override runBiasImpactExperiment( + executionId: string, + stepId: string, + request: BiasImpactExperimentRequest + ): Observable { + const url = `${environment.apiUrl}/executions/${encodeURIComponent(executionId)}/steps/${encodeURIComponent(stepId)}/bias-impact`; + return this.http.post(url, request).pipe(map((raw) => this.biasImpactJobFromApi(raw))); + } + + override getBiasImpactJob(jobId: string): Observable { + return this.http + .get(`${environment.apiUrl}/executions/bias-impact-jobs/${encodeURIComponent(jobId)}`) + .pipe(map((raw) => this.biasImpactJobFromApi(raw))); + } + + override createBiasedRerun(executionId: string, request: BiasRerunRequest): Observable { + return this.http + .post(`${environment.apiUrl}/executions/${encodeURIComponent(executionId)}/bias-rerun`, request) + .pipe(map((raw) => this.mapExecution(raw))); + } + + override compareBiasExecutions( + baselineExecutionId: string, + biasedExecutionId: string, + includeRawOutputs: boolean + ): Observable { + const url = `${environment.apiUrl}/executions/${encodeURIComponent(baselineExecutionId)}/bias-compare/${encodeURIComponent(biasedExecutionId)}`; + return this.http.post(url, null, { + params: new HttpParams().set('includeRawOutputs', String(includeRawOutputs)) + }).pipe(map((raw) => this.biasImpactReportFromApi(raw))); + } + + override listBiasImpactReports(executionId: string): Observable { + return this.http + .get(`${environment.apiUrl}/executions/${encodeURIComponent(executionId)}/bias-impact-reports`) + .pipe(map((raw) => Array.isArray(raw) ? raw.map((item) => this.biasImpactReportFromApi(item)) : [])); + } + + override getBiasImpactReport(reportId: string): Observable { + return this.http + .get(`${environment.apiUrl}/executions/bias-impact-reports/${encodeURIComponent(reportId)}`) + .pipe(map((raw) => this.biasImpactReportFromApi(raw))); + } + override deleteTaskExecution(executionId: string): Observable { return this.http.delete(`${environment.apiUrl}/executions/${encodeURIComponent(executionId)}`); } @@ -203,6 +258,114 @@ export class TaskExecutionsCallService extends TaskExecutionsCallServiceBase { }; } + private biasImpactJobFromApi(raw: unknown): BiasImpactJob { + const value = this.toRecord(raw); + const status = this.toBiasImpactJobStatus(value['status']); + const rawReport = value['report']; + return { + id: String(value['id'] ?? ''), + status, + executionId: String(value['executionId'] ?? ''), + stepId: String(value['stepId'] ?? ''), + createdAt: String(value['createdAt'] ?? ''), + startedAt: this.toNullableString(value['startedAt']), + completedAt: this.toNullableString(value['completedAt']), + reportId: this.toNullableString(value['reportId']), + report: rawReport && typeof rawReport === 'object' ? this.biasImpactReportFromApi(rawReport) : null, + errorCode: this.toNullableString(value['errorCode']), + errorMessage: this.toNullableString(value['errorMessage']), + terminal: value['terminal'] === true || status === 'COMPLETED' || status === 'FAILED' + }; + } + + private biasImpactReportFromApi(raw: unknown): BiasImpactReport { + const value = this.toRecord(raw); + const immediate = this.toRecord(value['immediateImpact']); + return { + id: String(value['id'] ?? ''), + experimentId: String(value['experimentId'] ?? ''), + kind: this.toBiasImpactReportKind(value['kind']), + baselineExecutionId: String(value['baselineExecutionId'] ?? ''), + biasedExecutionId: this.toNullableString(value['biasedExecutionId']), + nodeId: this.toNullableString(value['nodeId']), + annotationIds: this.toStringArray(value['annotationIds']), + repetitions: this.toNumber(value['repetitions'], 0), + createdAt: String(value['createdAt'] ?? ''), + rawOutputsIncluded: value['rawOutputsIncluded'] === true, + immediateImpact: { + outputChanged: immediate['outputChanged'] === true, + maximumTextDifference: this.toNumber(immediate['maximumTextDifference'], 0), + changeRate: this.toNumber(immediate['changeRate'], 0), + baselineOutput: immediate['baselineOutput'] ?? {}, + biasedOutputs: Array.isArray(immediate['biasedOutputs']) ? immediate['biasedOutputs'] : [] + }, + downstreamImpact: this.toDownstreamImpact(value['downstreamImpact']), + routingChanges: this.toRoutingChanges(value['routingChanges']), + mockedSideEffects: this.toMockedSideEffects(value['mockedSideEffects']), + summary: String(value['summary'] ?? ''), + warnings: this.toStringArray(value['warnings']) + }; + } + + private toDownstreamImpact(raw: unknown): BiasDownstreamImpactEntry[] { + if (!Array.isArray(raw)) return []; + return raw.map((item) => { + const value = this.toRecord(item); + return { + nodeId: String(value['nodeId'] ?? ''), + nodeName: String(value['nodeName'] ?? ''), + baselineStatus: String(value['baselineStatus'] ?? ''), + biasedStatus: String(value['biasedStatus'] ?? ''), + changed: value['changed'] === true, + baselineOutputs: value['baselineOutputs'] ?? {}, + biasedOutputs: value['biasedOutputs'] ?? {} + }; + }); + } + + private toRoutingChanges(raw: unknown): BiasRoutingChangeEntry[] { + if (!Array.isArray(raw)) return []; + return raw.map((item) => { + const value = this.toRecord(item); + return { + nodeId: String(value['nodeId'] ?? ''), + baselineBranch: String(value['baselineBranch'] ?? ''), + biasedBranch: String(value['biasedBranch'] ?? '') + }; + }); + } + + private toMockedSideEffects(raw: unknown): BiasMockedSideEffect[] { + if (!Array.isArray(raw)) return []; + return raw.map((item) => { + const value = this.toRecord(item); + const kind = String(value['kind'] ?? 'EXTERNAL'); + return { + nodeId: String(value['nodeId'] ?? ''), + nodeName: String(value['nodeName'] ?? ''), + kind: kind === 'HTTP' || kind === 'MCP_AGENT' || kind === 'MCP_AGENT_CHAT' ? kind : 'EXTERNAL' + }; + }); + } + + private toBiasImpactJobStatus(value: unknown): BiasImpactJobStatus { + return value === 'RUNNING' || value === 'COMPLETED' || value === 'FAILED' ? value : 'QUEUED'; + } + + private toBiasImpactReportKind(value: unknown): BiasImpactReportKind { + return value === 'FULL_FLOW' ? value : 'ISOLATED_STEP'; + } + + private toStringArray(value: unknown): string[] { + return Array.isArray(value) ? value.map(String) : []; + } + + private toRecord(value: unknown): Record { + return value && typeof value === 'object' && !Array.isArray(value) + ? value as Record + : {}; + } + private mapExecutionGroup(raw: unknown): TaskExecutionGroup { const group = (raw ?? {}) as Partial & Record; const executions = Array.isArray(group['executions']) @@ -264,6 +427,10 @@ export class TaskExecutionsCallService extends TaskExecutionsCallServiceBase { return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null; } + private toNullableString(value: unknown): string | null { + return typeof value === 'string' && value.trim().length > 0 ? value : null; + } + private toTimestamp(value: unknown, fallback: number): number { const timestamp = typeof value === 'number' ? value : Number(value); return Number.isFinite(timestamp) ? timestamp : fallback; diff --git a/src/app/services/task-executions/task-executions.spec.ts b/src/app/services/task-executions/task-executions.spec.ts new file mode 100644 index 0000000..504d6b6 --- /dev/null +++ b/src/app/services/task-executions/task-executions.spec.ts @@ -0,0 +1,87 @@ +import { TestBed } from '@angular/core/testing'; +import { BiasImpactJob } from '@models/bias-impact'; +import { lastValueFrom, of, throwError } from 'rxjs'; +import { vi } from 'vitest'; + +import { TaskExecutionsService } from './task-executions'; + +const job = (status: BiasImpactJob['status'], terminal: boolean): BiasImpactJob => ({ + id: 'job-1', + status, + executionId: 'execution-1', + stepId: 'step-1', + createdAt: '2026-07-21T10:00:00', + startedAt: null, + completedAt: terminal ? '2026-07-21T10:00:10' : null, + reportId: null, + report: null, + errorCode: null, + errorMessage: null, + terminal +}); + +describe('TaskExecutionsService bias operations', () => { + let service: TaskExecutionsService; + + beforeEach(() => { + TestBed.configureTestingModule({}); + service = TestBed.inject(TaskExecutionsService); + }); + + it('maps side-effect policy conflicts using the backend error code', async () => { + service.taskExecutionsCallService = { + runBiasImpactExperiment: () => throwError(() => ({ + status: 409, + error: { + detail: 'External side effects are blocked', + errors: [{ code: 'BIAS_SIDE_EFFECT_BLOCKED', message: 'External side effects are blocked' }] + } + })) + } as unknown as typeof service.taskExecutionsCallService; + + await expect(lastValueFrom(service.runBiasImpactExperiment('execution-1', 'step-1', { + annotationIds: ['annotation-1'], + repetitions: 3, + includeRawOutputs: true, + externalSideEffectPolicy: 'BLOCK', + confirmExternalSideEffects: false + }))).rejects.toEqual({ + reason: 'SIDE_EFFECT_BLOCKED', + code: 'BIAS_SIDE_EFFECT_BLOCKED', + message: 'External side effects are blocked' + }); + expect(service.biasExperimentInProgress()).toBe(false); + }); + + it('polls until a terminal job and stops after completion', async () => { + vi.useFakeTimers(); + const getBiasImpactJob = vi.fn() + .mockReturnValueOnce(of(job('QUEUED', false))) + .mockReturnValueOnce(of(job('RUNNING', false))) + .mockReturnValueOnce(of(job('COMPLETED', true))); + service.taskExecutionsCallService = { getBiasImpactJob } as unknown as typeof service.taskExecutionsCallService; + + const terminalJob = lastValueFrom(service.pollBiasImpactJob('job-1')); + await vi.advanceTimersByTimeAsync(3_000); + + await expect(terminalJob).resolves.toEqual(expect.objectContaining({ status: 'COMPLETED', terminal: true })); + expect(getBiasImpactJob).toHaveBeenCalledTimes(3); + vi.useRealTimers(); + }); + + it('retries a transient polling failure without abandoning the job', async () => { + vi.useFakeTimers(); + const getBiasImpactJob = vi.fn() + .mockReturnValueOnce(throwError(() => ({ status: 0 }))) + .mockReturnValueOnce(of(job('RUNNING', false))) + .mockReturnValueOnce(of(job('COMPLETED', true))); + service.taskExecutionsCallService = { getBiasImpactJob } as unknown as typeof service.taskExecutionsCallService; + + const terminalJob = lastValueFrom(service.pollBiasImpactJob('job-1')); + await vi.advanceTimersByTimeAsync(5_000); + + await expect(terminalJob).resolves.toEqual(expect.objectContaining({ status: 'COMPLETED' })); + expect(getBiasImpactJob).toHaveBeenCalledTimes(3); + vi.useRealTimers(); + }); +}); diff --git a/src/app/services/task-executions/task-executions.ts b/src/app/services/task-executions/task-executions.ts index 3dde227..8852722 100644 --- a/src/app/services/task-executions/task-executions.ts +++ b/src/app/services/task-executions/task-executions.ts @@ -1,8 +1,30 @@ import { DestroyRef, inject, Injectable, signal } from '@angular/core'; import { environment } from '@environment'; import { LLMDescriptor } from '@models/flow'; +import { + BiasImpactExperimentRequest, + BiasImpactJob, + BiasImpactReport, + BiasRerunRequest, + BiasSideEffectError +} from '@models/bias-impact'; import { ExecutionEventLogEntry, getExecutionStatusGroup, TaskExecution, TaskExecutionGroup } from '@models/task-execution'; -import { catchError, finalize, Observable, tap, throwError } from 'rxjs'; +import { + catchError, + defer, + EMPTY, + expand, + filter, + finalize, + map, + Observable, + of, + switchMap, + tap, + throwError, + timeout, + timer +} from 'rxjs'; import { TaskExecutionsCallServiceBase } from './task-executions-call.base'; @Injectable({ @@ -18,10 +40,14 @@ export class TaskExecutionsService { private _taskExecutions = signal([]); private _taskExecutionGroups = signal([]); private _pendingExecutionCreation = signal(false); + private _biasExperimentInProgress = signal(false); + private _biasRerunInProgress = signal(false); taskExecutions = this._taskExecutions.asReadonly(); taskExecutionGroups = this._taskExecutionGroups.asReadonly(); pendingExecutionCreation = this._pendingExecutionCreation.asReadonly(); + biasExperimentInProgress = this._biasExperimentInProgress.asReadonly(); + biasRerunInProgress = this._biasRerunInProgress.asReadonly(); init() { if (this.initialized) return; @@ -78,6 +104,86 @@ export class TaskExecutionsService { ); } + runBiasImpactExperiment( + executionId: string, + stepId: string, + request: BiasImpactExperimentRequest + ): Observable { + this._biasExperimentInProgress.set(true); + return this.taskExecutionsCallService.runBiasImpactExperiment(executionId, stepId, request).pipe( + finalize(() => this._biasExperimentInProgress.set(false)), + catchError((error) => throwError(() => this.toBiasOperationError(error))) + ); + } + + getBiasImpactJob(jobId: string): Observable { + return this.taskExecutionsCallService.getBiasImpactJob(jobId).pipe( + catchError((error) => throwError(() => this.toBiasOperationError(error))) + ); + } + + pollBiasImpactJob(jobId: string): Observable { + type PollState = { job: BiasImpactJob | null; failures: number }; + const poll = (failures: number): Observable => this.getBiasImpactJob(jobId).pipe( + timeout({ first: 15_000 }), + map((job) => ({ job, failures: 0 })), + catchError((error) => { + if (!this.isRetryablePollingError(error)) { + return throwError(() => error); + } + console.warn('Bias impact job polling request failed; retrying', error); + return of({ job: null, failures: failures + 1 }); + }) + ); + + return defer(() => poll(0)).pipe( + expand((state) => { + if (state.job?.terminal) return EMPTY; + const delay = state.job + ? 1_500 + : Math.min(5_000, 1_500 * (2 ** Math.min(state.failures, 2))); + return timer(delay).pipe(switchMap(() => poll(state.failures))); + }), + filter((state): state is { job: BiasImpactJob; failures: number } => state.job !== null), + map((state) => state.job) + ); + } + + createBiasedRerun(executionId: string, request: BiasRerunRequest): Observable { + this._biasRerunInProgress.set(true); + this._pendingExecutionCreation.set(true); + return this.taskExecutionsCallService.createBiasedRerun(executionId, request).pipe( + tap(() => this.refresh()), + finalize(() => { + this._pendingExecutionCreation.set(false); + this._biasRerunInProgress.set(false); + }), + catchError((error) => throwError(() => this.toBiasOperationError(error))) + ); + } + + compareBiasExecutions( + baselineExecutionId: string, + biasedExecutionId: string, + includeRawOutputs: boolean + ): Observable { + return this.taskExecutionsCallService + .compareBiasExecutions(baselineExecutionId, biasedExecutionId, includeRawOutputs) + .pipe(catchError((error) => throwError(() => this.toBiasOperationError(error)))); + } + + listBiasImpactReports(executionId: string): Observable { + return this.taskExecutionsCallService.listBiasImpactReports(executionId).pipe( + catchError((error) => throwError(() => this.toBiasOperationError(error))) + ); + } + + getBiasImpactReport(reportId: string): Observable { + return this.taskExecutionsCallService.getBiasImpactReport(reportId).pipe( + catchError((error) => throwError(() => this.toBiasOperationError(error))) + ); + } + deleteExecution(executionId: string) { return this.withRefreshAndErrorHandling( this.taskExecutionsCallService.deleteTaskExecution(executionId), @@ -245,4 +351,45 @@ export class TaskExecutionsService { }) ); } + + private toBiasOperationError(error: unknown): unknown { + const response = error as { status?: unknown; error?: unknown }; + if (response?.status !== 409 || !response.error || typeof response.error !== 'object') { + return error; + } + + const body = response.error as Record; + const errors = Array.isArray(body['errors']) ? body['errors'] : []; + const first = errors[0]; + if (!first || typeof first !== 'object') return error; + const record = first as Record; + const code = record['code']; + const message = typeof record['message'] === 'string' + ? record['message'] + : typeof body['detail'] === 'string' ? body['detail'] : 'Bias experiment request was rejected'; + + if (code === 'BIAS_SIDE_EFFECT_BLOCKED') { + const mapped: BiasSideEffectError = { + reason: 'SIDE_EFFECT_BLOCKED', + code, + message + }; + return mapped; + } + if (code === 'BIAS_SIDE_EFFECT_CONFIRMATION_REQUIRED') { + const mapped: BiasSideEffectError = { + reason: 'CONFIRMATION_REQUIRED', + code, + message + }; + return mapped; + } + return error; + } + + private isRetryablePollingError(error: unknown): boolean { + const value = error as { status?: unknown; name?: unknown }; + const status = typeof value?.status === 'number' ? value.status : null; + return value?.name === 'TimeoutError' || status === 0 || (status != null && status >= 500); + } } diff --git a/src/app/shared/behavioral-probe-editor/behavioral-probe-editor.css b/src/app/shared/behavioral-probe-editor/behavioral-probe-editor.css new file mode 100644 index 0000000..a6201c3 --- /dev/null +++ b/src/app/shared/behavioral-probe-editor/behavioral-probe-editor.css @@ -0,0 +1,15 @@ +:host { display: block; } +.probe-editor { display: grid; gap: 12px; padding: 12px; border: 1px solid #cbd5e1; border-radius: 9px; background: #f8fafc; } +.probe-header h4 { margin: 0; font-size: 13px; } +.probe-header p, small { margin: 3px 0 0; color: #64748b; font-size: 11px; font-weight: 400; } +label { display: grid; gap: 5px; font-size: 12px; font-weight: 650; } +input[type='text'], select, textarea { width: 100%; box-sizing: border-box; border: 1px solid #cbd5e1; border-radius: 7px; padding: 8px; font: inherit; font-weight: 400; background: white; } +textarea { resize: vertical; } +fieldset { display: grid; gap: 5px; border: 1px solid #dbe3ee; border-radius: 7px; padding: 8px; } +legend { padding: 0 4px; font-size: 12px; font-weight: 650; } +.probe-check { display: flex; align-items: center; gap: 6px; font-weight: 400; } +.probe-check input { margin: 0; } +.probe-state, .probe-note { border-radius: 7px; padding: 8px; background: #e0f2fe; color: #075985; font-size: 12px; } +.probe-error, em { color: #b91c1c; font-size: 11px; font-style: normal; } +.mock-output-list { display: grid; gap: 8px; } +code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; } diff --git a/src/app/shared/behavioral-probe-editor/behavioral-probe-editor.html b/src/app/shared/behavioral-probe-editor/behavioral-probe-editor.html new file mode 100644 index 0000000..6d3e25a --- /dev/null +++ b/src/app/shared/behavioral-probe-editor/behavioral-probe-editor.html @@ -0,0 +1,107 @@ +
+
+
+

Behavioral probe

+

Optional controlled behavior used only by bias experiments.

+
+
+ + @if (loadingCapabilities) { +
Loading compatible activation modes…
+ } @else if (capabilityError) { +
{{ capabilityError }}
+ } @else if (capabilities && !supported) { +
This block can be annotated, but it does not support a bias experiment.
+ } @else { + + + @if (currentProbe.activationMode === 'PROMPT_DIRECTIVE') { + + } + + @if (currentProbe.activationMode === 'INPUT_TRANSFORMATION') { + +
+ Target inputs Leave all unchecked to target every textual input. + @for (input of block?.inputs ?? []; track input.name) { + + } +
+ } + + @if (currentProbe.activationMode === 'OUTPUT_TRANSFORMATION') { + + } + + @if (currentProbe.activationMode === 'ROUTING_OVERRIDE') { + + } + + @if (isMockResponse) { +
Mock responses use typed output values. No new mock instruction text is saved.
+ @if (currentProbe.instruction) { + + } +
+ @for (output of block?.outputs ?? []; track output.name) { + + } +
+ } + + @if (currentProbe.activationMode) { + + } + } +
diff --git a/src/app/shared/behavioral-probe-editor/behavioral-probe-editor.spec.ts b/src/app/shared/behavioral-probe-editor/behavioral-probe-editor.spec.ts new file mode 100644 index 0000000..e996d2a --- /dev/null +++ b/src/app/shared/behavioral-probe-editor/behavioral-probe-editor.spec.ts @@ -0,0 +1,89 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { FlowBlock } from '@models/flow'; +import { BlocksService } from '@services/blocks/blocks'; +import { of } from 'rxjs'; +import { vi } from 'vitest'; + +import { BehavioralProbeEditorComponent } from './behavioral-probe-editor'; + +const block: FlowBlock = { + id: 'conditional-1', + name: 'Conditional', + inputs: [{ name: 'input', type: 'TEXT', multiple: true }], + outputs: [ + { name: 'response', type: 'TEXT', multiple: false }, + { name: 'accepted', type: 'BOOLEAN', multiple: false }, + { name: 'items', type: 'TEXT', multiple: true } + ], + specificConfiguration: { useLlm: true }, + typeName: 'ConditionalBlock', + nodeFamily: 'block' +}; + +describe('BehavioralProbeEditorComponent', () => { + let fixture: ComponentFixture; + let component: BehavioralProbeEditorComponent; + const retrieveBiasCapabilities = vi.fn(); + const retrieveBiasCapabilitiesForInstance = vi.fn(); + + beforeEach(async () => { + retrieveBiasCapabilities.mockReset(); + retrieveBiasCapabilitiesForInstance.mockReset(); + retrieveBiasCapabilities.mockReturnValue(of({ + blockType: 'ConditionalBlock', supported: true, isolatedExperimentSupported: true, + fullFlowExperimentSupported: true, externalSideEffects: false, + configurationDependent: true, activationModes: ['INPUT_TRANSFORMATION'] + })); + retrieveBiasCapabilitiesForInstance.mockReturnValue(of({ + blockType: 'ConditionalBlock', supported: true, isolatedExperimentSupported: true, + fullFlowExperimentSupported: true, externalSideEffects: false, + configurationDependent: true, activationModes: ['PROMPT_DIRECTIVE', 'MOCK_RESPONSE', 'ROUTING_OVERRIDE'] + })); + await TestBed.configureTestingModule({ + imports: [BehavioralProbeEditorComponent], + providers: [{ + provide: BlocksService, + useValue: { retrieveBiasCapabilities, retrieveBiasCapabilitiesForInstance } + }] + }).compileComponents(); + fixture = TestBed.createComponent(BehavioralProbeEditorComponent); + component = fixture.componentInstance; + fixture.componentRef.setInput('block', block); + fixture.detectChanges(); + }); + + it('uses instance capabilities when the type-level response is configuration dependent', () => { + expect(retrieveBiasCapabilities).toHaveBeenCalledWith('ConditionalBlock'); + expect(retrieveBiasCapabilitiesForInstance).toHaveBeenCalledWith('ConditionalBlock', block); + expect(component.activationModes).toEqual(['PROMPT_DIRECTIVE', 'MOCK_RESPONSE', 'ROUTING_OVERRIDE']); + }); + + it('emits typed complete mock outputs and never writes a new mock instruction', () => { + const changed = vi.fn(); + component.probeChange.subscribe(changed); + + component.selectActivationMode('MOCK_RESPONSE'); + + expect(changed).toHaveBeenLastCalledWith(expect.objectContaining({ + activationMode: 'MOCK_RESPONSE', + instruction: undefined, + mockOutputs: { response: '', accepted: false, items: [] } + })); + }); + + it('updates templates, target inputs and routing branch for the selected mode', () => { + const changed = vi.fn(); + component.probeChange.subscribe(changed); + fixture.componentRef.setInput('probe', { activationMode: 'INPUT_TRANSFORMATION' }); + fixture.detectChanges(); + + component.setInstruction('Frame ${original} as confirming evidence.'); + component.toggleTargetInput('input', true); + expect(changed).toHaveBeenLastCalledWith(expect.objectContaining({ targetInputs: ['input'] })); + + fixture.componentRef.setInput('probe', { activationMode: 'ROUTING_OVERRIDE' }); + fixture.detectChanges(); + component.setRoutingOutput('accepted'); + expect(changed).toHaveBeenLastCalledWith(expect.objectContaining({ instruction: 'accepted' })); + }); +}); diff --git a/src/app/shared/behavioral-probe-editor/behavioral-probe-editor.ts b/src/app/shared/behavioral-probe-editor/behavioral-probe-editor.ts new file mode 100644 index 0000000..96e5ad9 --- /dev/null +++ b/src/app/shared/behavioral-probe-editor/behavioral-probe-editor.ts @@ -0,0 +1,216 @@ +import { CommonModule } from '@angular/common'; +import { ChangeDetectionStrategy, Component, Input, OnChanges, Output, EventEmitter, inject } from '@angular/core'; +import { FormsModule } from '@angular/forms'; +import { BiasCapabilities } from '@models/bias-impact'; +import { BehavioralProbe, BiasActivationMode, FlowBlock, FlowPort } from '@models/flow'; +import { BlocksService } from '@services/blocks/blocks'; +import { take } from 'rxjs'; + +@Component({ + selector: 'app-behavioral-probe-editor', + standalone: true, + imports: [CommonModule, FormsModule], + templateUrl: './behavioral-probe-editor.html', + styleUrl: './behavioral-probe-editor.css', + changeDetection: ChangeDetectionStrategy.OnPush +}) +export class BehavioralProbeEditorComponent implements OnChanges { + private readonly blocks = inject(BlocksService); + private capabilityRequestVersion = 0; + private capabilityKey: string | null = null; + + @Input() block: FlowBlock | null = null; + @Input() probe: BehavioralProbe | undefined; + @Input() readonly = false; + @Output() probeChange = new EventEmitter(); + + capabilities: BiasCapabilities | null = null; + loadingCapabilities = false; + capabilityError: string | null = null; + mockValueErrors: Record = {}; + + get blockType(): string | null { + const value = this.block?.typeName; + return typeof value === 'string' && value.trim().length > 0 ? value : null; + } + + get activationModes(): BiasActivationMode[] { + return this.capabilities?.activationModes ?? []; + } + + get supported(): boolean { + return this.capabilities?.supported !== false; + } + + get currentProbe(): BehavioralProbe { + return this.probe ?? {}; + } + + get isMockResponse(): boolean { + return this.currentProbe.activationMode === 'MOCK_RESPONSE'; + } + + get selectedTargetInputs(): string[] { + return this.currentProbe.targetInputs ?? []; + } + + ngOnChanges() { + void this.loadCapabilities(); + } + + selectActivationMode(mode: string) { + const activationMode = mode as BiasActivationMode; + if (!activationMode) { + this.emit({ ...this.currentProbe, activationMode: undefined }); + return; + } + + if (activationMode === 'MOCK_RESPONSE') { + this.mockValueErrors = {}; + this.emit({ + ...this.currentProbe, + activationMode, + instruction: undefined, + targetInputs: [], + mockOutputs: this.normalizedMockOutputs(this.currentProbe.mockOutputs) + }); + return; + } + + this.emit({ + ...this.currentProbe, + activationMode, + mockOutputs: undefined + }); + } + + setInstruction(instruction: string) { + this.emit({ ...this.currentProbe, instruction }); + } + + setExpectedImpact(expectedImpact: string) { + this.emit({ ...this.currentProbe, expectedImpact }); + } + + toggleTargetInput(name: string, checked: boolean) { + const selected = new Set(this.selectedTargetInputs); + if (checked) selected.add(name); + else selected.delete(name); + this.emit({ ...this.currentProbe, targetInputs: [...selected] }); + } + + setRoutingOutput(name: string) { + this.emit({ ...this.currentProbe, instruction: name }); + } + + setTextMockOutput(port: FlowPort, value: string) { + this.setMockOutput(port.name, value); + } + + setBooleanMockOutput(port: FlowPort, checked: boolean) { + this.setMockOutput(port.name, checked); + } + + setJsonMockOutput(port: FlowPort, value: string) { + try { + const parsed = JSON.parse(value); + if (port.multiple && !Array.isArray(parsed)) { + throw new Error('Use a JSON array for a multiple output.'); + } + this.mockValueErrors = { ...this.mockValueErrors, [port.name]: '' }; + this.setMockOutput(port.name, parsed); + } catch (error) { + this.mockValueErrors = { + ...this.mockValueErrors, + [port.name]: error instanceof Error ? error.message : 'Invalid JSON value.' + }; + } + } + + mockValueText(port: FlowPort): string { + const value = this.currentProbe.mockOutputs?.[port.name]; + if (typeof value === 'string') return value; + if (value === undefined) return ''; + return JSON.stringify(value, null, 2); + } + + mockBooleanValue(port: FlowPort): boolean { + return this.currentProbe.mockOutputs?.[port.name] === true; + } + + private async loadCapabilities() { + const blockType = this.blockType; + if (!blockType) { + this.capabilities = null; + this.capabilityKey = null; + return; + } + + const nextCapabilityKey = `${blockType}:${this.configurationFingerprint()}`; + if (nextCapabilityKey === this.capabilityKey && (this.capabilities || this.loadingCapabilities)) return; + this.capabilityKey = nextCapabilityKey; + + const requestVersion = ++this.capabilityRequestVersion; + this.loadingCapabilities = true; + this.capabilityError = null; + this.blocks.retrieveBiasCapabilities(blockType).pipe(take(1)).subscribe({ + next: (capabilities) => { + if (requestVersion !== this.capabilityRequestVersion) return; + if (!capabilities.configurationDependent || !this.block) { + this.capabilities = capabilities; + this.loadingCapabilities = false; + return; + } + this.blocks.retrieveBiasCapabilitiesForInstance(blockType, this.block).pipe(take(1)).subscribe({ + next: (instanceCapabilities) => { + if (requestVersion !== this.capabilityRequestVersion) return; + this.capabilities = instanceCapabilities; + this.loadingCapabilities = false; + }, + error: () => { + if (requestVersion !== this.capabilityRequestVersion) return; + this.capabilityError = 'Unable to load the configured block capabilities.'; + this.loadingCapabilities = false; + } + }); + }, + error: () => { + if (requestVersion !== this.capabilityRequestVersion) return; + this.capabilityError = 'Unable to load bias capabilities.'; + this.loadingCapabilities = false; + } + }); + } + + private setMockOutput(name: string, value: unknown) { + this.emit({ + ...this.currentProbe, + mockOutputs: { ...this.normalizedMockOutputs(this.currentProbe.mockOutputs), [name]: value } + }); + } + + private normalizedMockOutputs(existing: Record | undefined): Record { + return (this.block?.outputs ?? []).reduce>((result, port) => { + result[port.name] = existing?.[port.name] ?? this.defaultMockValue(port); + return result; + }, {}); + } + + private defaultMockValue(port: FlowPort): unknown { + if (port.multiple) return []; + if (port.type === 'BOOLEAN') return false; + return ''; + } + + private configurationFingerprint(): string { + try { + return JSON.stringify(this.block?.specificConfiguration ?? {}); + } catch { + return String(this.block?.specificConfiguration ?? ''); + } + } + + private emit(probe: BehavioralProbe) { + this.probeChange.emit(probe); + } +} diff --git a/src/app/shared/bias-annotations/bias-annotations.css b/src/app/shared/bias-annotations/bias-annotations.css index 41ef8c6..8ae83b5 100644 --- a/src/app/shared/bias-annotations/bias-annotations.css +++ b/src/app/shared/bias-annotations/bias-annotations.css @@ -13,6 +13,8 @@ button:disabled { cursor: not-allowed; opacity: .5; } .bias-badge { border-radius: 999px; padding: 2px 7px; background: #e2e8f0; font-size: 10px; font-weight: 650; } .bias-badge.severity { background: #fee2e2; color: #991b1b; } .bias-badge.status { background: #dbeafe; color: #1e40af; } +.bias-badge.source { background: #f3e8ff; color: #6b21a8; } +.bias-badge.probe { background: #dcfce7; color: #166534; } .bias-issue { margin: 7px 0; font-size: 12px; white-space: pre-wrap; } .bias-actions { justify-content: flex-end; } .bias-actions .danger { color: #b91c1c; } diff --git a/src/app/shared/bias-annotations/bias-annotations.html b/src/app/shared/bias-annotations/bias-annotations.html index 53ec493..b4c90c3 100644 --- a/src/app/shared/bias-annotations/bias-annotations.html +++ b/src/app/shared/bias-annotations/bias-annotations.html @@ -21,6 +21,8 @@ {{ optionLabel('category', annotation.category) }} {{ optionLabel('severity', annotation.severity) }} {{ optionLabel('status', annotation.status) }} + {{ optionLabel('source', annotation.source) }} + @if (probeExecutable(annotation)) { Executable probe }
{{ annotation.issue }}
@if (serverError(index); as error) {
{{ error }}
} @@ -62,6 +64,11 @@ @if (clientErrors[field.key]; as error) { {{ error }} } } +
diff --git a/src/app/shared/bias-annotations/bias-annotations.spec.ts b/src/app/shared/bias-annotations/bias-annotations.spec.ts index 048ed9a..ba37a2f 100644 --- a/src/app/shared/bias-annotations/bias-annotations.spec.ts +++ b/src/app/shared/bias-annotations/bias-annotations.spec.ts @@ -4,6 +4,7 @@ import { BiasAnnotationsDescriptor } from '@models/flow'; import { BlocksService } from '@services/blocks/blocks'; import { EditorStateHolder } from '@stores/flow-editor'; import { vi } from 'vitest'; +import { of } from 'rxjs'; import { BiasAnnotationsComponent } from './bias-annotations'; const descriptor: BiasAnnotationsDescriptor = { @@ -33,7 +34,22 @@ describe('BiasAnnotationsComponent', () => { await TestBed.configureTestingModule({ imports: [BiasAnnotationsComponent], providers: [ - { provide: BlocksService, useValue: { biasAnnotationsDescriptor: signal(descriptor) } }, + { + provide: BlocksService, + useValue: { + biasAnnotationsDescriptor: signal(descriptor), + retrieveBiasCapabilities: () => of({ + blockType: 'LLMBlock', supported: true, isolatedExperimentSupported: true, + fullFlowExperimentSupported: true, externalSideEffects: false, + configurationDependent: false, activationModes: ['PROMPT_DIRECTIVE'] + }), + retrieveBiasCapabilitiesForInstance: () => of({ + blockType: 'LLMBlock', supported: true, isolatedExperimentSupported: true, + fullFlowExperimentSupported: true, externalSideEffects: false, + configurationDependent: false, activationModes: ['PROMPT_DIRECTIVE'] + }) + } + }, { provide: EditorStateHolder, useValue: { flowValidationErrors: validationErrors } } ] }).compileComponents(); @@ -93,4 +109,14 @@ describe('BiasAnnotationsComponent', () => { expect(component.serverError(0, 'category')).toBe('Category required'); expect(component.serverError(1, 'category')).toBeNull(); }); + + it('maps typed mock-output probe errors to nested fields', () => { + validationErrors.set([{ + code: 'BIAS_PROBE_MOCK_OUTPUT_TYPE_MISMATCH', + id: 'block-1', + field: 'biasAnnotations[0].behavioralProbe.mockOutputs.response', + message: 'Response must be text' + }]); + expect(component.serverError(0, 'behavioralProbe.mockOutputs.response')).toBe('Response must be text'); + }); }); diff --git a/src/app/shared/bias-annotations/bias-annotations.ts b/src/app/shared/bias-annotations/bias-annotations.ts index 2174d3f..4c72e30 100644 --- a/src/app/shared/bias-annotations/bias-annotations.ts +++ b/src/app/shared/bias-annotations/bias-annotations.ts @@ -1,9 +1,19 @@ import { CommonModule } from '@angular/common'; import { ChangeDetectionStrategy, Component, EventEmitter, Input, Output, inject } from '@angular/core'; import { FormsModule } from '@angular/forms'; -import { BiasAnnotation, BiasAnnotationOption, BiasAnnotationsDescriptor, FlowValidationError } from '@models/flow'; +import { + BiasAnnotation, + BiasAnnotationOption, + BiasAnnotationsDescriptor, + BehavioralProbe, + FlowBlock, + FlowValidationError, + isProbeExecutable +} from '@models/flow'; +import { BIAS_PROBE_ERROR_CODES } from '@models/bias-impact'; import { BlocksService } from '@services/blocks/blocks'; import { EditorStateHolder } from '@stores/flow-editor'; +import { BehavioralProbeEditorComponent } from '@shared/behavioral-probe-editor/behavioral-probe-editor'; type BiasField = { key: string; @@ -18,13 +28,14 @@ type BiasField = { const BIAS_ERROR_CODES = new Set([ 'TOO_MANY_BIAS_ANNOTATIONS', 'NULL_BIAS_ANNOTATION', 'DUPLICATE_BIAS_ANNOTATION_ID', - 'BIAS_CATEGORY_REQUIRED', 'BIAS_SEVERITY_REQUIRED', 'BIAS_ISSUE_REQUIRED', 'BIAS_FIELD_TOO_LONG' + 'BIAS_CATEGORY_REQUIRED', 'BIAS_SEVERITY_REQUIRED', 'BIAS_ISSUE_REQUIRED', 'BIAS_FIELD_TOO_LONG', + ...BIAS_PROBE_ERROR_CODES ]); @Component({ selector: 'app-bias-annotations', standalone: true, - imports: [CommonModule, FormsModule], + imports: [CommonModule, FormsModule, BehavioralProbeEditorComponent], templateUrl: './bias-annotations.html', styleUrl: './bias-annotations.css', changeDetection: ChangeDetectionStrategy.OnPush @@ -34,6 +45,7 @@ export class BiasAnnotationsComponent { private readonly editorState = inject(EditorStateHolder); @Input({ required: true }) blockId = ''; + @Input() block: FlowBlock | null = null; @Input() annotations: BiasAnnotation[] = []; @Input() readonly = false; @Output() annotationsChange = new EventEmitter(); @@ -62,6 +74,7 @@ export class BiasAnnotationsComponent { return Object.entries(properties) .filter(([key]) => !generated.has(key)) + .filter(([key]) => key !== 'behavioralProbe') .map(([key, raw]) => { const field = this.record(raw); const maxLength = Number(field['maxLength']); @@ -152,6 +165,14 @@ export class BiasAnnotationsComponent { return field.options.find((option) => option.value === value)?.description ?? field.description ?? null; } + probeExecutable(annotation: BiasAnnotation): boolean { + return isProbeExecutable(annotation.behavioralProbe); + } + + updateProbe(probe: BehavioralProbe | undefined) { + this.draft = { ...this.draft, behavioralProbe: probe }; + } + valueLength(field: string): number { return String(this.draft[field] ?? '').length; } diff --git a/src/app/shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog.css b/src/app/shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog.css new file mode 100644 index 0000000..221dee5 --- /dev/null +++ b/src/app/shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog.css @@ -0,0 +1,7 @@ +.bias-experiment-dialog__backdrop { background: rgb(0 0 0 / .5); inset: 0; position: fixed; z-index: 10020; } +.bias-experiment-dialog { background: #fff; border: 1px solid #cbd5e1; border-radius: 1rem; box-shadow: 0 25px 50px rgb(15 23 42 / .25); display: flex; flex-direction: column; left: 50%; max-height: min(88vh, 800px); max-width: 680px; position: fixed; top: 50%; transform: translate(-50%, -50%); width: min(94vw, 680px); z-index: 10021; } +header, footer { align-items: center; display: flex; gap: 1rem; justify-content: space-between; padding: 1rem 1.25rem; } header { border-bottom: 1px solid #e2e8f0; } header h3, header p { margin: 0; } header p { color: #64748b; font-size: .85rem; } +.bias-experiment-dialog__content { display: grid; gap: .9rem; overflow: auto; padding: 1.25rem; } .bias-experiment-dialog__content > p { margin: 0; } +fieldset { display: grid; gap: .5rem; } fieldset label, .bias-experiment-dialog__content > label { color: #334155; font-size: .87rem; } +.bias-experiment-dialog__progress { background: #eff6ff; border-left: 3px solid #2563eb; color: #1e3a8a; padding: .6rem .7rem; } +.bias-experiment-dialog__error { background: #fff1f2; border-left: 3px solid #e11d48; color: #9f1239; margin: 0; padding: .6rem .7rem; } footer { border-top: 1px solid #e2e8f0; justify-content: flex-end; } diff --git a/src/app/shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog.html b/src/app/shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog.html new file mode 100644 index 0000000..9e7188c --- /dev/null +++ b/src/app/shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog.html @@ -0,0 +1,24 @@ +@if (state(); as currentState) { +
+ +} diff --git a/src/app/shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog.ts b/src/app/shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog.ts new file mode 100644 index 0000000..de9b54c --- /dev/null +++ b/src/app/shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog.ts @@ -0,0 +1,135 @@ +import { ChangeDetectionStrategy, Component, DestroyRef, effect, inject, signal } from '@angular/core'; +import { FormsModule } from '@angular/forms'; +import { MatButtonModule } from '@angular/material/button'; +import { BiasImpactReportViewerComponent } from '@shared/bias-impact-report-viewer/bias-impact-report-viewer'; +import { SideEffectPolicySelectorComponent } from '@shared/side-effect-policy-selector/side-effect-policy-selector'; +import { ConfirmDialogService } from '@services/dialogs/confirm-dialog'; +import { BiasImpactExperimentDialogService } from '@services/dialogs/bias-impact-experiment-dialog'; +import { NotificationService } from '@services/notifications/notification'; +import { TaskExecutionsService } from '@services/task-executions/task-executions'; +import { BiasImpactJob, BiasImpactReport, BiasSideEffectError, ExternalSideEffectPolicy } from '@models/bias-impact'; +import { Subscription } from 'rxjs'; + +@Component({ + selector: 'app-bias-impact-experiment-dialog-host', + standalone: true, + imports: [FormsModule, MatButtonModule, BiasImpactReportViewerComponent, SideEffectPolicySelectorComponent], + templateUrl: './bias-impact-experiment-dialog.html', + styleUrl: './bias-impact-experiment-dialog.css', + changeDetection: ChangeDetectionStrategy.OnPush +}) +export class BiasImpactExperimentDialogHostComponent { + private readonly dialog = inject(BiasImpactExperimentDialogService); + private readonly executions = inject(TaskExecutionsService); + private readonly confirmation = inject(ConfirmDialogService); + private readonly notifications = inject(NotificationService); + private readonly destroyRef = inject(DestroyRef); + private pollSubscription: Subscription | null = null; + + readonly state = this.dialog.state; + readonly selectedAnnotationIds = signal([]); + readonly repetitions = signal(3); + readonly includeRawOutputs = signal(true); + readonly policy = signal('BLOCK'); + readonly submitting = signal(false); + readonly currentJob = signal(null); + readonly inlineError = signal(null); + readonly report = signal(null); + + constructor() { + effect(() => { + const state = this.state(); + this.cancelPolling(); + this.selectedAnnotationIds.set(state?.annotations.map((annotation) => String(annotation.id ?? '')).filter(Boolean) ?? []); + this.repetitions.set(3); + this.includeRawOutputs.set(true); + this.policy.set('BLOCK'); + this.submitting.set(false); + this.currentJob.set(null); + this.inlineError.set(null); + this.report.set(null); + }); + this.destroyRef.onDestroy(() => this.cancelPolling()); + } + + toggleAnnotation(id: string, checked: boolean) { + this.selectedAnnotationIds.update((current) => checked ? [...new Set([...current, id])] : current.filter((value) => value !== id)); + } + + updateRepetitions(value: number) { + this.repetitions.set(Math.min(10, Math.max(1, Number.isFinite(value) ? Math.round(value) : 3))); + } + + async submit() { + const state = this.state(); + if (!state || this.submitting() || this.currentJob()) return; + if (!this.selectedAnnotationIds().length) { + this.inlineError.set('Select at least one executable bias annotation.'); + return; + } + + let confirmExternalSideEffects = false; + if (this.policy() === 'REQUIRE_CONFIRMATION') { + confirmExternalSideEffects = await this.confirmation.open('This experiment may invoke real external HTTP or MCP calls. Do you want to continue?'); + if (!confirmExternalSideEffects) return; + } + + this.submitting.set(true); + this.inlineError.set(null); + this.executions.runBiasImpactExperiment(state.executionId, state.stepId, { + annotationIds: this.selectedAnnotationIds(), + repetitions: this.repetitions(), + includeRawOutputs: this.includeRawOutputs(), + externalSideEffectPolicy: this.policy(), + confirmExternalSideEffects + }).subscribe({ + next: (job) => this.startPolling(job), + error: (error) => this.handleInitialError(error) + }); + } + + close() { + this.cancelPolling(); + this.dialog.close(); + } + + private startPolling(job: BiasImpactJob) { + this.currentJob.set(job); + this.pollSubscription = this.executions.pollBiasImpactJob(job.id).subscribe({ + next: (nextJob) => { + this.currentJob.set(nextJob); + if (!nextJob.terminal) return; + this.submitting.set(false); + this.cancelPolling(); + if (nextJob.status === 'COMPLETED' && nextJob.report) this.report.set(nextJob.report); + else this.inlineError.set(nextJob.errorMessage || 'The bias impact experiment failed.'); + }, + error: (error) => { + this.submitting.set(false); + this.currentJob.set(null); + this.inlineError.set(error instanceof Error ? error.message : 'Unable to retrieve the experiment status.'); + } + }); + } + + private handleInitialError(error: unknown) { + this.submitting.set(false); + const sideEffectError = error as Partial; + if (sideEffectError.reason === 'SIDE_EFFECT_BLOCKED' || sideEffectError.reason === 'CONFIRMATION_REQUIRED') { + this.inlineError.set(sideEffectError.message ?? 'External side effects require a different policy.'); + return; + } + const status = (error as { status?: number })?.status; + if (status === 404 || status === 400) { + this.notifications.show(error instanceof Error ? error.message : 'The execution is no longer eligible for this experiment.', 'error'); + this.close(); + return; + } + this.inlineError.set(error instanceof Error ? error.message : 'Unable to start the bias impact experiment.'); + } + + private cancelPolling() { + this.pollSubscription?.unsubscribe(); + this.pollSubscription = null; + } +} diff --git a/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.css b/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.css new file mode 100644 index 0000000..9cdb582 --- /dev/null +++ b/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.css @@ -0,0 +1,21 @@ +.bias-impact-report { color: #1e293b; display: grid; gap: 1rem; } +.bias-impact-report__header, .bias-impact-report__section-heading, .bias-impact-report__node-heading, .bias-impact-report__routing { align-items: center; display: flex; gap: .65rem; justify-content: space-between; } +.bias-impact-report__header h3, .bias-impact-report__section h4 { margin: 0; } +.bias-impact-report__header time { color: #64748b; font-size: .84rem; } +.bias-impact-report__kind { background: #e0e7ff; border-radius: 999px; color: #3730a3; font-size: .72rem; font-weight: 700; padding: .2rem .5rem; } +.bias-impact-report__metadata { display: grid; gap: .55rem; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); margin: 0; } +.bias-impact-report__metadata div { background: #f8fafc; border-radius: .4rem; padding: .55rem; } +dt { color: #64748b; font-size: .73rem; } dd { font-size: .82rem; margin: .15rem 0 0; overflow-wrap: anywhere; } +.bias-impact-report__summary { background: #eff6ff; border-left: 3px solid #2563eb; margin: 0; padding: .6rem .75rem; } +.bias-impact-report__section { border-top: 1px solid #e2e8f0; padding-top: .9rem; } +.bias-impact-report__metrics { display: flex; flex-wrap: wrap; gap: .5rem 1.2rem; font-size: .86rem; margin: .6rem 0; } +.bias-impact-report__empty { color: #64748b; font-size: .87rem; } +.bias-impact-report__downstream { background: #f8fafc; border: 1px solid #e2e8f0; border-radius: .45rem; margin-top: .7rem; padding: .7rem; } +.bias-impact-report__node-heading { justify-content: flex-start; } .bias-impact-report__node-heading span { color: #64748b; font-size: .8rem; } .bias-impact-report__node-heading span.changed { color: #b45309; font-weight: 700; } +.bias-impact-report__downstream p { font-size: .84rem; margin: .5rem 0; } +.bias-impact-report__routing { border-bottom: 1px solid #f1f5f9; font-size: .86rem; padding: .45rem 0; justify-content: flex-start; } +.bias-impact-report__side-effects { margin: .55rem 0 0; padding-left: 1.2rem; } .bias-impact-report__side-effects li { margin: .3rem 0; } .bias-impact-report__side-effects span { color: #7c3aed; font-size: .78rem; font-weight: 700; } +.bias-impact-report__notice { background: #fffbeb; border-left: 3px solid #f59e0b; font-size: .84rem; margin: 0; padding: .6rem .75rem; } +.bias-impact-report__warnings { background: #fff7ed; border: 1px solid #fed7aa; border-radius: .45rem; padding: .8rem; } +code { font-size: .78rem; overflow-wrap: anywhere; } +@media (max-width: 700px) { .bias-impact-report__header, .bias-impact-report__section-heading { align-items: flex-start; flex-direction: column; } } diff --git a/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.html b/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.html new file mode 100644 index 0000000..e850c18 --- /dev/null +++ b/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.html @@ -0,0 +1,89 @@ +@if (report; as currentReport) { +
+
+
+ {{ currentReport.kind }} +

Bias impact report

+
+ +
+ +
+
Baseline execution
{{ currentReport.baselineExecutionId }}
+ @if (currentReport.biasedExecutionId) {
Biased execution
{{ currentReport.biasedExecutionId }}
} +
Experiment
{{ currentReport.experimentId }}
+ @if (currentReport.nodeId) {
Node
{{ currentReport.nodeId }}
} +
Annotations
{{ currentReport.annotationIds.join(', ') || 'None' }}
+
Repetitions
{{ currentReport.repetitions }}
+
+ +

{{ currentReport.summary }}

+ +
+

Immediate impact

+
+ Output changed: {{ currentReport.immediateImpact.outputChanged ? 'Yes' : 'No' }} + Change rate: {{ percent(currentReport.immediateImpact.changeRate) }} + Maximum text difference: {{ textDifference(currentReport.immediateImpact.maximumTextDifference) }} +
+ +
+ +
+
+

Downstream impact

+ +
+ @if (downstreamEntries.length === 0) { +

No downstream impact was recorded. This is normal for an isolated step experiment.

+ } + @for (entry of downstreamEntries; track entry.nodeId) { +
+
+ {{ entry.nodeName || entry.nodeId }} + {{ entry.nodeId }} + {{ entry.changed ? 'Changed' : 'Unchanged' }} +
+

Status: {{ entry.baselineStatus }} → {{ entry.biasedStatus }}

+ +
+ } +
+ +
+

Routing changes

+ @if (currentReport.routingChanges.length === 0) {

No routing changes recorded.

} + @for (change of currentReport.routingChanges; track change.nodeId) { +
{{ change.nodeId }}{{ change.baselineBranch }} → {{ change.biasedBranch }}
+ } +
+ + @if (currentReport.mockedSideEffects.length > 0) { +
+

Mocked side effects

+
    + @for (sideEffect of currentReport.mockedSideEffects; track sideEffect.nodeId) { +
  • {{ sideEffect.nodeName || sideEffect.nodeId }} {{ sideEffect.nodeId }} {{ sideEffect.kind }}
    • + } +
+
+ } + + @if (!currentReport.rawOutputsIncluded) { +

Raw output maps and lists were intentionally omitted from this report.

+ } + +
+

Warnings

+ @if (currentReport.warnings.length === 0) {

No warnings reported.

} + @for (warning of currentReport.warnings; track warning) {

{{ warning }}

} +
+
+} diff --git a/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.spec.ts b/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.spec.ts new file mode 100644 index 0000000..e71d77a --- /dev/null +++ b/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.spec.ts @@ -0,0 +1,64 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { BiasImpactReport } from '@models/bias-impact'; +import { BiasImpactReportViewerComponent } from './bias-impact-report-viewer'; + +describe('BiasImpactReportViewerComponent', () => { + let fixture: ComponentFixture; + + const report: BiasImpactReport = { + id: 'report-1', + experimentId: 'experiment-1', + kind: 'FULL_FLOW', + baselineExecutionId: 'execution-base', + biasedExecutionId: 'execution-biased', + nodeId: null, + annotationIds: ['annotation-1'], + repetitions: 3, + createdAt: '2026-07-21T10:00:00.000Z', + rawOutputsIncluded: false, + immediateImpact: { + outputChanged: true, + changeRate: .5, + maximumTextDifference: .25, + baselineOutput: 'baseline', + biasedOutputs: ['biased'] + }, + downstreamImpact: [ + { nodeId: 'node-changed', nodeName: 'Changed node', baselineStatus: 'COMPLETED', biasedStatus: 'COMPLETED', changed: true, baselineOutputs: 'a', biasedOutputs: 'b' }, + { nodeId: 'node-same', nodeName: 'Same node', baselineStatus: 'COMPLETED', biasedStatus: 'COMPLETED', changed: false, baselineOutputs: 'a', biasedOutputs: 'a' } + ], + routingChanges: [{ nodeId: 'router', baselineBranch: 'yes', biasedBranch: 'no' }], + mockedSideEffects: [{ nodeId: 'http', nodeName: 'HTTP request', kind: 'HTTP' }], + summary: 'The biased variant changed the output.', + warnings: [] + }; + + beforeEach(async () => { + await TestBed.configureTestingModule({ imports: [BiasImpactReportViewerComponent] }).compileComponents(); + fixture = TestBed.createComponent(BiasImpactReportViewerComponent); + fixture.componentInstance.report = report; + }); + + it('renders metadata, impact sections, side effects and the raw-output notice', () => { + fixture.detectChanges(); + const text = fixture.nativeElement.textContent; + + expect(text).toContain('execution-base'); + expect(text).toContain('execution-biased'); + expect(text).not.toContain('Node\n'); + expect(text).toContain('Change rate: 50.0%'); + expect(text).toContain('router'); + expect(text).toContain('HTTP request'); + expect(text).toContain('intentionally omitted'); + expect(text).toContain('No warnings reported.'); + }); + + it('filters downstream entries to changed nodes', () => { + fixture.componentInstance.changedOnly = true; + fixture.detectChanges(); + + const text = fixture.nativeElement.textContent; + expect(text).toContain('Changed node'); + expect(text).not.toContain('Same node'); + }); +}); diff --git a/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.ts b/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.ts new file mode 100644 index 0000000..0df5ff4 --- /dev/null +++ b/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.ts @@ -0,0 +1,32 @@ +import { CommonModule, DatePipe } from '@angular/common'; +import { ChangeDetectionStrategy, Component, Input } from '@angular/core'; +import { FormsModule } from '@angular/forms'; +import { BiasDownstreamImpactEntry, BiasImpactReport } from '@models/bias-impact'; +import { BiasOutputDiffComponent } from '../bias-output-diff/bias-output-diff'; + +@Component({ + selector: 'app-bias-impact-report-viewer', + standalone: true, + imports: [CommonModule, DatePipe, FormsModule, BiasOutputDiffComponent], + templateUrl: './bias-impact-report-viewer.html', + styleUrl: './bias-impact-report-viewer.css', + changeDetection: ChangeDetectionStrategy.OnPush +}) +export class BiasImpactReportViewerComponent { + @Input() report: BiasImpactReport | null = null; + + changedOnly = false; + + get downstreamEntries(): BiasDownstreamImpactEntry[] { + const entries = this.report?.downstreamImpact ?? []; + return this.changedOnly ? entries.filter((entry) => entry.changed) : entries; + } + + percent(value: number): string { + return `${(value * 100).toFixed(1)}%`; + } + + textDifference(value: number): string { + return `${value.toFixed(3)} (0 = identical, 1 = maximally different)`; + } +} diff --git a/src/app/shared/bias-output-diff/bias-output-diff.css b/src/app/shared/bias-output-diff/bias-output-diff.css new file mode 100644 index 0000000..6543e7b --- /dev/null +++ b/src/app/shared/bias-output-diff/bias-output-diff.css @@ -0,0 +1,9 @@ +.bias-output-diff { display: grid; gap: .8rem; grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); } +.bias-output-diff__variants { display: grid; gap: .8rem; } +.bias-output-diff__column { background: #f8fafc; border: 1px solid #dbe4ef; border-radius: .45rem; min-width: 0; padding: .7rem; } +.bias-output-diff__baseline { border-left: 3px solid #2563eb; } +.bias-output-diff__biased { border-left: 3px solid #a855f7; } +h4 { color: #334155; font-size: .78rem; margin: 0 0 .5rem; text-transform: uppercase; } +pre { font-family: inherit; margin: 0; overflow: auto; white-space: pre-wrap; word-break: break-word; } +.bias-output-diff__empty { color: #64748b; margin: .4rem 0; } +@media (max-width: 700px) { .bias-output-diff { grid-template-columns: 1fr; } } diff --git a/src/app/shared/bias-output-diff/bias-output-diff.html b/src/app/shared/bias-output-diff/bias-output-diff.html new file mode 100644 index 0000000..55e22e0 --- /dev/null +++ b/src/app/shared/bias-output-diff/bias-output-diff.html @@ -0,0 +1,26 @@ +
+
+

{{ baselineLabel }}

+ @if (isText(baselineOutput)) { + 
{{ baselineOutput }}
+ } @else { + + } +
+ +
+ @if (biasedOutputs.length === 0) { +

No biased output is available.

+ } + @for (output of biasedOutputs; track $index) { +
+

{{ variantLabel }}{{ biasedOutputs.length > 1 ? ' ' + ($index + 1) : '' }}

+ @if (isText(output)) { + 
{{ output }}
+ } @else { + + } +
+ } +
+
diff --git a/src/app/shared/bias-output-diff/bias-output-diff.spec.ts b/src/app/shared/bias-output-diff/bias-output-diff.spec.ts new file mode 100644 index 0000000..f0c3f31 --- /dev/null +++ b/src/app/shared/bias-output-diff/bias-output-diff.spec.ts @@ -0,0 +1,30 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { BiasOutputDiffComponent } from './bias-output-diff'; + +describe('BiasOutputDiffComponent', () => { + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ imports: [BiasOutputDiffComponent] }).compileComponents(); + fixture = TestBed.createComponent(BiasOutputDiffComponent); + }); + + it('renders text baseline and every biased variant side by side', () => { + fixture.componentInstance.baselineOutput = 'neutral'; + fixture.componentInstance.biasedOutputs = ['biased one', 'biased two']; + fixture.detectChanges(); + + const text = fixture.nativeElement.textContent; + expect(text).toContain('neutral'); + expect(text).toContain('Biased output 1'); + expect(text).toContain('Biased output 2'); + }); + + it('uses the JSON viewer for complex outputs', () => { + fixture.componentInstance.baselineOutput = { result: true }; + fixture.componentInstance.biasedOutputs = [{ result: false }]; + fixture.detectChanges(); + + expect(fixture.nativeElement.querySelectorAll('app-json-viewer').length).toBe(2); + }); +}); diff --git a/src/app/shared/bias-output-diff/bias-output-diff.ts b/src/app/shared/bias-output-diff/bias-output-diff.ts new file mode 100644 index 0000000..e6182f6 --- /dev/null +++ b/src/app/shared/bias-output-diff/bias-output-diff.ts @@ -0,0 +1,22 @@ +import { CommonModule } from '@angular/common'; +import { ChangeDetectionStrategy, Component, Input } from '@angular/core'; +import { JsonViewerComponent } from '../json-viewer/json-viewer'; + +@Component({ + selector: 'app-bias-output-diff', + standalone: true, + imports: [CommonModule, JsonViewerComponent], + templateUrl: './bias-output-diff.html', + styleUrl: './bias-output-diff.css', + changeDetection: ChangeDetectionStrategy.OnPush +}) +export class BiasOutputDiffComponent { + @Input() baselineOutput: unknown; + @Input() biasedOutputs: unknown[] = []; + @Input() baselineLabel = 'Baseline'; + @Input() variantLabel = 'Biased output'; + + isText(value: unknown): value is string { + return typeof value === 'string'; + } +} diff --git a/src/app/shared/bias-rerun-dialog/bias-rerun-dialog.css b/src/app/shared/bias-rerun-dialog/bias-rerun-dialog.css new file mode 100644 index 0000000..77c2003 --- /dev/null +++ b/src/app/shared/bias-rerun-dialog/bias-rerun-dialog.css @@ -0,0 +1,5 @@ +.bias-rerun-dialog__backdrop { background: rgb(0 0 0 / .5); inset: 0; position: fixed; z-index: 10020; } +.bias-rerun-dialog { background: #fff; border: 1px solid #cbd5e1; border-radius: 1rem; box-shadow: 0 25px 50px rgb(15 23 42 / .25); display: flex; flex-direction: column; left: 50%; max-height: min(88vh, 800px); max-width: 680px; position: fixed; top: 50%; transform: translate(-50%, -50%); width: min(94vw, 680px); z-index: 10021; } +header, footer { align-items: center; display: flex; gap: 1rem; justify-content: space-between; padding: 1rem 1.25rem; } header { border-bottom: 1px solid #e2e8f0; } h3, p { margin: 0; } header p { color: #64748b; font-size: .85rem; } +.bias-rerun-dialog__content { display: grid; gap: .9rem; overflow: auto; padding: 1.25rem; } fieldset { display: grid; gap: .5rem; } label { color: #334155; font-size: .87rem; } +.bias-rerun-dialog__error { background: #fff1f2; border-left: 3px solid #e11d48; color: #9f1239; padding: .6rem .7rem; } footer { border-top: 1px solid #e2e8f0; justify-content: flex-end; } diff --git a/src/app/shared/bias-rerun-dialog/bias-rerun-dialog.html b/src/app/shared/bias-rerun-dialog/bias-rerun-dialog.html new file mode 100644 index 0000000..5f604b7 --- /dev/null +++ b/src/app/shared/bias-rerun-dialog/bias-rerun-dialog.html @@ -0,0 +1,18 @@ +@if (state(); as currentState) { +
+ +} diff --git a/src/app/shared/bias-rerun-dialog/bias-rerun-dialog.ts b/src/app/shared/bias-rerun-dialog/bias-rerun-dialog.ts new file mode 100644 index 0000000..a7862ac --- /dev/null +++ b/src/app/shared/bias-rerun-dialog/bias-rerun-dialog.ts @@ -0,0 +1,80 @@ +import { ChangeDetectionStrategy, Component, effect, inject, signal } from '@angular/core'; +import { FormsModule } from '@angular/forms'; +import { MatButtonModule } from '@angular/material/button'; +import { ExternalSideEffectPolicy, BiasSideEffectError } from '@models/bias-impact'; +import { ConfirmDialogService } from '@services/dialogs/confirm-dialog'; +import { BiasRerunDialogService } from '@services/dialogs/bias-rerun-dialog'; +import { TaskExecutionsService } from '@services/task-executions/task-executions'; +import { SideEffectPolicySelectorComponent } from '@shared/side-effect-policy-selector/side-effect-policy-selector'; + +@Component({ + selector: 'app-bias-rerun-dialog-host', + standalone: true, + imports: [FormsModule, MatButtonModule, SideEffectPolicySelectorComponent], + templateUrl: './bias-rerun-dialog.html', + styleUrl: './bias-rerun-dialog.css', + changeDetection: ChangeDetectionStrategy.OnPush +}) +export class BiasRerunDialogHostComponent { + private readonly dialog = inject(BiasRerunDialogService); + private readonly executions = inject(TaskExecutionsService); + private readonly confirmation = inject(ConfirmDialogService); + readonly state = this.dialog.state; + readonly selectedAnnotationIdsByNode = signal>({}); + readonly policy = signal('BLOCK'); + readonly creating = signal(false); + readonly inlineError = signal(null); + + constructor() { + effect(() => { + const state = this.state(); + this.selectedAnnotationIdsByNode.set(Object.fromEntries((state?.candidates ?? []).map((candidate) => [candidate.nodeId, []]))); + this.policy.set('BLOCK'); + this.creating.set(false); + this.inlineError.set(null); + }); + } + + selectedIds(nodeId: string): string[] { return this.selectedAnnotationIdsByNode()[nodeId] ?? []; } + + toggleAnnotation(nodeId: string, annotationId: string, checked: boolean) { + this.selectedAnnotationIdsByNode.update((current) => { + const selected = new Set(current[nodeId] ?? []); + if (checked) selected.add(annotationId); else selected.delete(annotationId); + return { ...current, [nodeId]: [...selected] }; + }); + } + + hasExternalSideEffects(): boolean { + return (this.state()?.candidates ?? []).some((candidate) => this.selectedIds(candidate.nodeId).length > 0 && candidate.capabilities.externalSideEffects); + } + + async submit() { + const state = this.state(); + if (!state || this.creating()) return; + const activations = Object.entries(this.selectedAnnotationIdsByNode()) + .filter(([, annotationIds]) => annotationIds.length > 0) + .map(([nodeId, annotationIds]) => ({ nodeId, annotationIds })); + if (!activations.length) { this.inlineError.set('Select at least one executable annotation.'); return; } + + let confirmExternalSideEffects = false; + if (this.policy() === 'REQUIRE_CONFIRMATION') { + confirmExternalSideEffects = await this.confirmation.open('This biased rerun may invoke real external HTTP or MCP calls. Do you want to continue?'); + if (!confirmExternalSideEffects) return; + } + this.creating.set(true); + this.inlineError.set(null); + this.executions.createBiasedRerun(state.executionId, { activations, externalSideEffectPolicy: this.policy(), confirmExternalSideEffects }).subscribe({ + next: (execution) => { state.onCreated(execution); this.dialog.close(); }, + error: (error) => { + this.creating.set(false); + const sideEffectError = error as Partial; + this.inlineError.set(sideEffectError.reason === 'SIDE_EFFECT_BLOCKED' || sideEffectError.reason === 'CONFIRMATION_REQUIRED' + ? sideEffectError.message ?? 'External side effect policy prevented the rerun.' + : error instanceof Error ? error.message : 'Unable to create the biased rerun.'); + } + }); + } + + close() { if (!this.creating()) this.dialog.close(); } +} diff --git a/src/app/shared/json-viewer/json-viewer.css b/src/app/shared/json-viewer/json-viewer.css new file mode 100644 index 0000000..43ed85f --- /dev/null +++ b/src/app/shared/json-viewer/json-viewer.css @@ -0,0 +1,8 @@ +.json-viewer { font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; font-size: .78rem; overflow: auto; } +.json-viewer__label { color: #475569; font-family: inherit; font-weight: 700; margin-bottom: .35rem; } +.json-viewer__group { min-width: max-content; } +.json-viewer__group > summary { cursor: pointer; color: #1e40af; outline-offset: 2px; } +.json-viewer__children { border-left: 1px solid #cbd5e1; margin: .3rem 0 .2rem .35rem; padding-left: .7rem; } +.json-viewer__entry { align-items: flex-start; display: flex; gap: .45rem; line-height: 1.55; } +.json-viewer__key { color: #7c3aed; white-space: nowrap; } +.json-viewer__primitive { color: #0f172a; font-family: inherit; white-space: pre-wrap; word-break: break-word; } diff --git a/src/app/shared/json-viewer/json-viewer.html b/src/app/shared/json-viewer/json-viewer.html new file mode 100644 index 0000000..990bd5a --- /dev/null +++ b/src/app/shared/json-viewer/json-viewer.html @@ -0,0 +1,34 @@ +
+ @if (label) { +
{{ label }}
+ } + + + @if (isExpandable(value)) { +
+ {{ summary(value) }} +
+ @if (isArray(value)) { + @for (item of value; track $index) { +
+ [{{ $index }}] + +
+ } + } @else if (isObject(value)) { + @for (entry of entries(value); track entry.key) { +
+ {{ entry.key }} + +
+ } + } +
+
+ } @else { + {{ formatPrimitive(value) }} + } + + + +
diff --git a/src/app/shared/json-viewer/json-viewer.spec.ts b/src/app/shared/json-viewer/json-viewer.spec.ts new file mode 100644 index 0000000..01cb1b5 --- /dev/null +++ b/src/app/shared/json-viewer/json-viewer.spec.ts @@ -0,0 +1,28 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { JsonViewerComponent } from './json-viewer'; + +describe('JsonViewerComponent', () => { + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ imports: [JsonViewerComponent] }).compileComponents(); + fixture = TestBed.createComponent(JsonViewerComponent); + }); + + it('renders an expandable recursive object tree', () => { + fixture.componentInstance.value = { answer: { enabled: true }, values: [1, 2] }; + fixture.detectChanges(); + + const text = fixture.nativeElement.textContent; + expect(text).toContain('Object (2)'); + expect(text).toContain('answer'); + expect(text).toContain('enabled'); + expect(text).toContain('true'); + expect(text).toContain('[0]'); + }); + + it('formats primitive strings as JSON strings', () => { + expect(fixture.componentInstance.formatPrimitive('hello')).toBe('"hello"'); + expect(fixture.componentInstance.formatPrimitive(null)).toBe('null'); + }); +}); diff --git a/src/app/shared/json-viewer/json-viewer.ts b/src/app/shared/json-viewer/json-viewer.ts new file mode 100644 index 0000000..2a27271 --- /dev/null +++ b/src/app/shared/json-viewer/json-viewer.ts @@ -0,0 +1,62 @@ +import { CommonModule } from '@angular/common'; +import { ChangeDetectionStrategy, Component, Input } from '@angular/core'; + +/** Lightweight, dependency-free JSON tree for diagnostic and report views. */ +@Component({ + selector: 'app-json-viewer', + standalone: true, + imports: [CommonModule], + templateUrl: './json-viewer.html', + styleUrl: './json-viewer.css', + changeDetection: ChangeDetectionStrategy.OnPush +}) +export class JsonViewerComponent { + @Input() value: unknown; + @Input() label: string | null = null; + @Input() initiallyExpanded = true; + + private readonly collapsedPaths = new Set(); + + isArray(value: unknown): value is unknown[] { + return Array.isArray(value); + } + + isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); + } + + isExpandable(value: unknown): boolean { + return this.isArray(value) || this.isObject(value); + } + + entries(value: Record): Array<{ key: string; value: unknown }> { + return Object.entries(value).map(([key, entryValue]) => ({ key, value: entryValue })); + } + + summary(value: unknown): string { + if (this.isArray(value)) return `Array (${value.length})`; + if (this.isObject(value)) return `Object (${Object.keys(value).length})`; + return this.formatPrimitive(value); + } + + formatPrimitive(value: unknown): string { + if (typeof value === 'string') return `"${value}"`; + if (value === undefined) return 'undefined'; + if (value === null) return 'null'; + try { + return JSON.stringify(value); + } catch { + return String(value); + } + } + + isExpanded(path: string): boolean { + return this.initiallyExpanded && !this.collapsedPaths.has(path); + } + + onToggle(path: string, event: Event) { + const expanded = (event.target as HTMLDetailsElement).open; + if (expanded) this.collapsedPaths.delete(path); + else this.collapsedPaths.add(path); + } +} diff --git a/src/app/shared/nodes/generic-node/generic-node.html b/src/app/shared/nodes/generic-node/generic-node.html index ee71bb4..92850a9 100644 --- a/src/app/shared/nodes/generic-node/generic-node.html +++ b/src/app/shared/nodes/generic-node/generic-node.html @@ -512,6 +512,7 @@ diff --git a/src/app/shared/nodes/generic-node/generic-node.ts b/src/app/shared/nodes/generic-node/generic-node.ts index 5aadee3..098bf5e 100644 --- a/src/app/shared/nodes/generic-node/generic-node.ts +++ b/src/app/shared/nodes/generic-node/generic-node.ts @@ -2,7 +2,7 @@ import { CommonModule } from '@angular/common'; import { ChangeDetectionStrategy, ChangeDetectorRef, Component, effect, ElementRef, HostBinding, HostListener, inject, Input, OnDestroy } from '@angular/core'; import { FormsModule } from '@angular/forms'; import { MatTooltipModule } from '@angular/material/tooltip'; -import { BiasAnnotation, BlockType, currentFlowPortValueKind, flowValueKindLabel, FlowData, FlowPort, FlowValueKind, FLOW_DEPENDANT_PORT_KEY, FLOW_DEPENDENCY_PORT_KEY, normalizeFlowPortValueKinds } from '@models/flow'; +import { BiasAnnotation, BlockType, currentFlowPortValueKind, flowValueKindLabel, FlowBlock, FlowData, FlowPort, FlowValueKind, FLOW_DEPENDANT_PORT_KEY, FLOW_DEPENDENCY_PORT_KEY, normalizeFlowPortValueKinds } from '@models/flow'; import { BiasAnnotationsComponent } from '../../bias-annotations/bias-annotations'; import { ClassicPreset } from 'rete'; import { ReteModule } from 'rete-angular-plugin/21'; @@ -694,6 +694,24 @@ export class GenericNodeComponent implements OnDestroy { return Array.isArray(value) ? value as BiasAnnotation[] : []; } + get biasBlock(): FlowBlock | null { + const nodeData = this.data?.data as Record | undefined; + const blockId = this.blockId; + const typeName = this.blockType; + if (!nodeData || !blockId || !typeName) return null; + return { + id: blockId, + name: this.name, + position: nodeData['position'] as { x: number; y: number } | undefined, + inputs: this.resolvePorts('input').map((port) => ({ ...port })), + outputs: this.resolvePorts('output').map((port) => ({ ...port })), + specificConfiguration: this.blockConfiguration ?? {}, + typeName, + nodeFamily: 'block', + biasAnnotations: this.biasAnnotations + }; + } + private get biasAnnotationsProperty(): string { const descriptorSignal = (this.blocksService as BlocksService & { biasAnnotationsDescriptor?: () => { blockProperty?: string } | null diff --git a/src/app/shared/nodes/task-step-node/task-step-node.css b/src/app/shared/nodes/task-step-node/task-step-node.css index 5635a6f..0380a45 100644 --- a/src/app/shared/nodes/task-step-node/task-step-node.css +++ b/src/app/shared/nodes/task-step-node/task-step-node.css @@ -616,3 +616,17 @@ white-space: pre-wrap; word-break: break-word; } +.llm-bias-impact-trigger { + align-items: center; + background: #eff6ff; + border: 1px solid #93c5fd; + border-radius: .35rem; + color: #1d4ed8; + cursor: pointer; + display: inline-flex; + font-size: .9rem; + justify-content: center; + margin-left: auto; + padding: .3rem .4rem; +} +.llm-bias-impact-trigger:hover { background: #dbeafe; } diff --git a/src/app/shared/nodes/task-step-node/task-step-node.html b/src/app/shared/nodes/task-step-node/task-step-node.html index 7072908..4ea85fc 100644 --- a/src/app/shared/nodes/task-step-node/task-step-node.html +++ b/src/app/shared/nodes/task-step-node/task-step-node.html @@ -59,6 +59,17 @@ {{ name }}
+ @if (canMeasureBiasImpact()) { + + }
diff --git a/src/app/shared/nodes/task-step-node/task-step-node.ts b/src/app/shared/nodes/task-step-node/task-step-node.ts index b5dc3e2..1d30b0a 100644 --- a/src/app/shared/nodes/task-step-node/task-step-node.ts +++ b/src/app/shared/nodes/task-step-node/task-step-node.ts @@ -2,13 +2,16 @@ import { CommonModule } from '@angular/common'; import { ChangeDetectionStrategy, ChangeDetectorRef, Component, HostBinding, Input, inject } from '@angular/core'; import { ClassicPreset } from 'rete'; import { ReteModule } from 'rete-angular-plugin/21'; -import { BlockInteractionContract, BlockType, FlowData, FlowPort, FLOW_DEPENDANT_PORT_KEY, FLOW_DEPENDENCY_PORT_KEY } from '@models/flow'; +import { BiasAnnotation, BlockInteractionContract, BlockType, FlowData, FlowPort, isProbeExecutable, FLOW_DEPENDANT_PORT_KEY, FLOW_DEPENDENCY_PORT_KEY } from '@models/flow'; +import { BiasCapabilities } from '@models/bias-impact'; import { BlocksService } from '@services/blocks/blocks'; import { ContainersService } from '@services/containers/containers'; import { NodeSettingsDialogService } from '@services/dialogs/node-settings-dialog'; import { SubflowPreviewDialogService } from '@services/dialogs/subflow-preview-dialog'; import { HumanInteractionDialogService } from '@services/dialogs/human-interaction-dialog'; import { TaskExecutionsService } from '@services/task-executions/task-executions'; +import { BiasImpactExperimentDialogService } from '@services/dialogs/bias-impact-experiment-dialog'; +import { take } from 'rxjs'; import { collectSchemaFlowDataFields, isFlowDataFieldPath, @@ -114,6 +117,7 @@ export class TaskStepNodeComponent { private cdr = inject(ChangeDetectorRef); private humanInteractionDialog = inject(HumanInteractionDialogService); private taskExecutionsService = inject(TaskExecutionsService); + private biasImpactExperimentDialog = inject(BiasImpactExperimentDialogService); @Input() data!: any; @Input() emit!: (data: any) => void; @@ -137,6 +141,7 @@ export class TaskStepNodeComponent { mainContentFields: MainContentView[] = []; interactionSubmitting = false; schemaReady = false; + biasCapabilities: BiasCapabilities | null = null; private blockSchema: Record | null = null; private blockDescriptor: BlockType | null = null; @@ -170,6 +175,7 @@ export class TaskStepNodeComponent { this.rebuildDisplayState(); void this.loadSchemaContext(); + this.loadBiasCapabilities(); } private rebuildDisplayState() { @@ -423,6 +429,36 @@ export class TaskStepNodeComponent { return this.stepStatus() === 'COMPLETED'; } + executableBiasAnnotations(): BiasAnnotation[] { + const annotations = this.data?.data?.biasAnnotations; + return Array.isArray(annotations) + ? annotations.filter((annotation): annotation is BiasAnnotation => !!annotation && isProbeExecutable(annotation.behavioralProbe)) + : []; + } + + canMeasureBiasImpact(): boolean { + return this.blockConfiguration?.['__executionStatusGroup'] === 'FINAL' + && this.executableBiasAnnotations().length > 0 + && this.biasCapabilities?.isolatedExperimentSupported === true; + } + + measureBiasImpact(event: Event) { + event.preventDefault(); + event.stopPropagation(); + const executionId = this.executionId(); + const stepId = this.executionNodeId(); + const capabilities = this.biasCapabilities; + if (!executionId || !stepId || !capabilities || !this.canMeasureBiasImpact()) return; + this.biasImpactExperimentDialog.open({ + executionId, + stepId, + nodeId: String(this.data?.id ?? stepId), + nodeName: this.nodeTitle(), + annotations: this.executableBiasAnnotations(), + capabilities + }); + } + isRunning(): boolean { return this.stepStatus() === 'RUNNING'; } @@ -485,6 +521,21 @@ export class TaskStepNodeComponent { return this.data?.data?.specificConfiguration ?? null; } + private loadBiasCapabilities() { + const blockType = this.blockType; + if (!blockType || this.isContainerNode()) return; + this.blocksService.retrieveBiasCapabilities(blockType).pipe(take(1)).subscribe({ + next: (capabilities) => { + this.biasCapabilities = capabilities; + this.cdr.markForCheck(); + }, + error: () => { + this.biasCapabilities = null; + this.cdr.markForCheck(); + } + }); + } + private get blockType(): string | null { const typeName = this.data?.data?.typeName; return typeof typeName === 'string' && typeName.length > 0 ? typeName : null; diff --git a/src/app/shared/side-effect-policy-selector/side-effect-policy-selector.css b/src/app/shared/side-effect-policy-selector/side-effect-policy-selector.css new file mode 100644 index 0000000..60c747d --- /dev/null +++ b/src/app/shared/side-effect-policy-selector/side-effect-policy-selector.css @@ -0,0 +1,5 @@ +.side-effect-policy-selector { display: grid; gap: .45rem; } +.side-effect-policy-selector h4 { font-size: .9rem; margin: 0; } +.side-effect-policy-selector label { color: #334155; font-size: .86rem; } +.side-effect-policy-selector__warning { background: #fff7ed; border-left: 3px solid #f97316; color: #9a3412; font-size: .84rem; margin: 0; padding: .55rem .65rem; } +.side-effect-policy-selector__note { color: #0369a1; font-size: .82rem; margin: 0; } diff --git a/src/app/shared/side-effect-policy-selector/side-effect-policy-selector.html b/src/app/shared/side-effect-policy-selector/side-effect-policy-selector.html new file mode 100644 index 0000000..aa6a8ba --- /dev/null +++ b/src/app/shared/side-effect-policy-selector/side-effect-policy-selector.html @@ -0,0 +1,10 @@ +
+

External side effects

+ @if (externalSideEffects) { +

This block can invoke external services. Choose how the experiment handles those calls.

+ } + + + + @if (policy === 'MOCK') {

HTTP and MCP calls will not be invoked for real.

} +
diff --git a/src/app/shared/side-effect-policy-selector/side-effect-policy-selector.spec.ts b/src/app/shared/side-effect-policy-selector/side-effect-policy-selector.spec.ts new file mode 100644 index 0000000..a235c22 --- /dev/null +++ b/src/app/shared/side-effect-policy-selector/side-effect-policy-selector.spec.ts @@ -0,0 +1,23 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { SideEffectPolicySelectorComponent } from './side-effect-policy-selector'; + +describe('SideEffectPolicySelectorComponent', () => { + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ imports: [SideEffectPolicySelectorComponent] }).compileComponents(); + fixture = TestBed.createComponent(SideEffectPolicySelectorComponent); + }); + + it('emits the selected policy and explains mock mode', () => { + const emitted: string[] = []; + fixture.componentInstance.policyChange.subscribe((policy) => emitted.push(policy)); + fixture.componentInstance.policy = 'MOCK'; + fixture.componentInstance.externalSideEffects = true; + fixture.detectChanges(); + fixture.componentInstance.select('BLOCK'); + + expect(emitted).toEqual(['BLOCK']); + expect(fixture.nativeElement.textContent).toContain('will not be invoked for real'); + }); +}); diff --git a/src/app/shared/side-effect-policy-selector/side-effect-policy-selector.ts b/src/app/shared/side-effect-policy-selector/side-effect-policy-selector.ts new file mode 100644 index 0000000..6d5ac58 --- /dev/null +++ b/src/app/shared/side-effect-policy-selector/side-effect-policy-selector.ts @@ -0,0 +1,22 @@ +import { CommonModule } from '@angular/common'; +import { ChangeDetectionStrategy, Component, EventEmitter, Input, Output } from '@angular/core'; +import { ExternalSideEffectPolicy } from '@models/bias-impact'; + +@Component({ + selector: 'app-side-effect-policy-selector', + standalone: true, + imports: [CommonModule], + templateUrl: './side-effect-policy-selector.html', + styleUrl: './side-effect-policy-selector.css', + changeDetection: ChangeDetectionStrategy.OnPush +}) +export class SideEffectPolicySelectorComponent { + @Input() policy: ExternalSideEffectPolicy = 'BLOCK'; + @Input() externalSideEffects = false; + @Input() disabled = false; + @Output() policyChange = new EventEmitter(); + + select(policy: ExternalSideEffectPolicy) { + if (!this.disabled) this.policyChange.emit(policy); + } +} diff --git a/src/app/shared/task-execution-viewer/task-execution-viewer.css b/src/app/shared/task-execution-viewer/task-execution-viewer.css index 026b611..016c09b 100644 --- a/src/app/shared/task-execution-viewer/task-execution-viewer.css +++ b/src/app/shared/task-execution-viewer/task-execution-viewer.css @@ -411,3 +411,13 @@ overflow-y: auto; overscroll-behavior: contain; } +.execution-bias-variant-context { + align-items: center; + color: #5b21b6; + display: flex; + flex-wrap: wrap; + font-size: .78rem; + gap: .35rem .75rem; + margin-top: .45rem; +} +.execution-graph-action-button-bias { background: #f3e8ff; color: #7e22ce; } diff --git a/src/app/shared/task-execution-viewer/task-execution-viewer.html b/src/app/shared/task-execution-viewer/task-execution-viewer.html index d636a13..0c8e14a 100644 --- a/src/app/shared/task-execution-viewer/task-execution-viewer.html +++ b/src/app/shared/task-execution-viewer/task-execution-viewer.html @@ -11,6 +11,14 @@ @if (simulationDescriptorLabel(); as simulationDescriptor) {

Simulator: {{ simulationDescriptor }}

} + @if (execution()!.biasExecutionContext; as biasContext) { +
+ Bias variant + Experiment: {{ biasContext.experimentId }} + Baseline: {{ execution()!.rerunOfExecutionId || 'not available' }} + Active annotations: {{ biasContext.activeAnnotationIdsByNode | json }} +
+ } }
@@ -92,6 +100,15 @@ } + ('inputs'); readonly startInProgress = signal(false); readonly simulateInProgress = signal(false); + readonly biasRerunOpening = signal(false); readonly cancelInProgress = signal(false); readonly resumeInProgress = signal(false); readonly savingInputs = signal>({}); @@ -404,6 +412,10 @@ export class TaskExecutionViewerComponent implements OnDestroy { }); readonly isSimulatedExecution = computed(() => this.execution()?.interactionSimulationEnabled === true); + readonly isBiasVariant = computed(() => !!this.execution()?.biasExecutionContext); + readonly canCreateBiasedRerun = computed(() => + getExecutionStatusGroup(this.execution()?.context.status) === 'FINAL' && !this.biasRerunOpening() + ); readonly simulationDescriptorLabel = computed(() => { const descriptor = this.execution()?.interactionSimulationDescriptor; if (!descriptor) return null; @@ -584,6 +596,30 @@ export class TaskExecutionViewerComponent implements OnDestroy { }); } + async openBiasedRerunDialog() { + const execution = this.execution(); + if (!execution || !this.canCreateBiasedRerun()) return; + this.biasRerunOpening.set(true); + try { + const candidates = await this.biasRerunCandidates(); + if (!candidates.length) return; + this.biasRerunDialog.open({ + executionId: execution.id, + candidates, + onCreated: (variant) => { + void this.router.navigate([], { + relativeTo: this.route, + queryParams: { executionId: variant.id }, + queryParamsHandling: 'merge', + replaceUrl: true + }); + } + }); + } finally { + this.biasRerunOpening.set(false); + } + } + onTextInputChange(input: EditableExecutionInput, value: string | string[]) { if (this.inputsReadOnly()) return; this.pendingTextInputs.update((current) => ({ ...current, [input.key]: value })); @@ -959,6 +995,28 @@ export class TaskExecutionViewerComponent implements OnDestroy { return (this.execution()?.stepDependencies ?? []).some((dependency) => String(dependency.targetId) === stepId); } + private async biasRerunCandidates(): Promise { + const candidates = this.stepsArray().flatMap((step): Array<{ nodeId: string; nodeName: string; block: FlowBlock }> => { + const node = getTaskExecutionStepNode(step); + if (!node || node.nodeFamily === 'container') return []; + const block = node as FlowBlock; + const annotations = (block.biasAnnotations ?? []).filter((annotation) => isProbeExecutable(annotation.behavioralProbe)); + return annotations.length ? [{ nodeId: step.id, nodeName: block.name || step.id, block: { ...block, biasAnnotations: annotations } }] : []; + }); + + const resolved = await Promise.all(candidates.map(async (candidate) => { + const capabilities = await firstValueFrom(this.blocksService.retrieveBiasCapabilities(candidate.block.typeName)); + if (!capabilities.fullFlowExperimentSupported) return null; + return { + nodeId: candidate.nodeId, + nodeName: candidate.nodeName, + annotations: candidate.block.biasAnnotations ?? [], + capabilities + } satisfies BiasRerunCandidate; + })); + return resolved.filter((candidate): candidate is BiasRerunCandidate => candidate !== null); + } + private hasOutgoingDependency(stepId: string): boolean { return (this.execution()?.stepDependencies ?? []).some((dependency) => String(dependency.sourceId) === stepId); }