feat: add bias impact experiments and biased reruns
This commit is contained in:
parent
909c4e6724
commit
7bfef72f79
|
|
@ -4,3 +4,5 @@
|
|||
<app-human-interaction-dialog-host></app-human-interaction-dialog-host>
|
||||
<app-node-settings-dialog-host></app-node-settings-dialog-host>
|
||||
<app-subflow-preview-dialog-host></app-subflow-preview-dialog-host>
|
||||
<app-bias-impact-experiment-dialog-host></app-bias-impact-experiment-dialog-host>
|
||||
<app-bias-rerun-dialog-host></app-bias-rerun-dialog-host>
|
||||
|
|
|
|||
|
|
@ -5,10 +5,12 @@ import { GlobalNotificationComponent } from '@shared/global-notification/global-
|
|||
import { HumanInteractionDialogHostComponent } from '@shared/human-interaction-dialog/human-interaction-dialog';
|
||||
import { NodeSettingsDialogHostComponent } from '@shared/node-settings-dialog/node-settings-dialog';
|
||||
import { SubflowPreviewDialogHostComponent } from '@shared/subflow-preview-dialog/subflow-preview-dialog';
|
||||
import { BiasImpactExperimentDialogHostComponent } from '@shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog';
|
||||
import { BiasRerunDialogHostComponent } from '@shared/bias-rerun-dialog/bias-rerun-dialog';
|
||||
|
||||
@Component({
|
||||
selector: 'app-root',
|
||||
imports: [RouterOutlet, ConfirmDialogHostComponent, GlobalNotificationComponent, HumanInteractionDialogHostComponent, NodeSettingsDialogHostComponent, SubflowPreviewDialogHostComponent],
|
||||
imports: [RouterOutlet, ConfirmDialogHostComponent, GlobalNotificationComponent, HumanInteractionDialogHostComponent, NodeSettingsDialogHostComponent, SubflowPreviewDialogHostComponent, BiasImpactExperimentDialogHostComponent, BiasRerunDialogHostComponent],
|
||||
templateUrl: './app.html',
|
||||
styleUrl: './app.css',
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
|
|
|
|||
|
|
@ -0,0 +1,102 @@
|
|||
import {
|
||||
BIAS_EXPERIMENT_ERROR_CODES,
|
||||
BIAS_PROBE_ERROR_CODES,
|
||||
BiasImpactJob,
|
||||
BiasImpactReport
|
||||
} from './bias-impact';
|
||||
import { BehavioralProbe, isProbeExecutable } from './flow';
|
||||
import { TaskExecution } from './task-execution';
|
||||
|
||||
describe('bias impact models', () => {
|
||||
it('recognizes executable instruction-based probes', () => {
|
||||
expect(isProbeExecutable(undefined)).toBe(false);
|
||||
expect(isProbeExecutable({ activationMode: 'PROMPT_DIRECTIVE' })).toBe(false);
|
||||
expect(isProbeExecutable({
|
||||
activationMode: 'INPUT_TRANSFORMATION',
|
||||
instruction: ' ${original} with experimental framing '
|
||||
})).toBe(true);
|
||||
});
|
||||
|
||||
it('requires typed outputs for new MOCK_RESPONSE probes', () => {
|
||||
const legacyProbe: BehavioralProbe = {
|
||||
activationMode: 'MOCK_RESPONSE',
|
||||
instruction: 'legacy mock response'
|
||||
};
|
||||
const typedProbe: BehavioralProbe = {
|
||||
activationMode: 'MOCK_RESPONSE',
|
||||
mockOutputs: { body: 'controlled response', success: true }
|
||||
};
|
||||
|
||||
expect(isProbeExecutable(legacyProbe)).toBe(false);
|
||||
expect(isProbeExecutable(typedProbe)).toBe(true);
|
||||
});
|
||||
|
||||
it('models completed jobs with the definitive report field names', () => {
|
||||
const report: BiasImpactReport = {
|
||||
id: 'report-1',
|
||||
experimentId: 'experiment-1',
|
||||
kind: 'ISOLATED_STEP',
|
||||
baselineExecutionId: 'baseline-1',
|
||||
biasedExecutionId: null,
|
||||
nodeId: 'node-1',
|
||||
annotationIds: ['annotation-1'],
|
||||
repetitions: 3,
|
||||
createdAt: '2026-07-21T10:00:00',
|
||||
rawOutputsIncluded: true,
|
||||
immediateImpact: {
|
||||
outputChanged: true,
|
||||
maximumTextDifference: 0.5,
|
||||
changeRate: 1,
|
||||
baselineOutput: { response: 'baseline' },
|
||||
biasedOutputs: [{ response: 'biased' }]
|
||||
},
|
||||
downstreamImpact: [],
|
||||
routingChanges: [],
|
||||
mockedSideEffects: [{ nodeId: 'http-1', nodeName: 'HTTP call', kind: 'HTTP' }],
|
||||
summary: 'The output changed.',
|
||||
warnings: []
|
||||
};
|
||||
const job: BiasImpactJob = {
|
||||
id: 'job-1',
|
||||
status: 'COMPLETED',
|
||||
executionId: 'baseline-1',
|
||||
stepId: 'node-1',
|
||||
createdAt: '2026-07-21T09:59:00',
|
||||
startedAt: '2026-07-21T09:59:01',
|
||||
completedAt: '2026-07-21T10:00:00',
|
||||
reportId: report.id,
|
||||
report,
|
||||
errorCode: null,
|
||||
errorMessage: null,
|
||||
terminal: true
|
||||
};
|
||||
|
||||
expect(job.report?.immediateImpact.biasedOutputs).toEqual([{ response: 'biased' }]);
|
||||
expect(job.report?.mockedSideEffects[0]?.kind).toBe('HTTP');
|
||||
});
|
||||
|
||||
it('keeps bias execution context optional for existing executions', () => {
|
||||
const execution = {
|
||||
id: 'execution-1',
|
||||
name: 'Legacy execution',
|
||||
creationTime: 0,
|
||||
context: {
|
||||
inputs: {},
|
||||
result: {},
|
||||
errors: {},
|
||||
warnings: {},
|
||||
steps: {},
|
||||
status: 'SUCCESS',
|
||||
waitingSteps: []
|
||||
}
|
||||
} satisfies TaskExecution;
|
||||
|
||||
expect(execution).not.toHaveProperty('biasExecutionContext');
|
||||
});
|
||||
|
||||
it('exports the backend error codes needed by probe and experiment flows', () => {
|
||||
expect(BIAS_PROBE_ERROR_CODES).toContain('BIAS_PROBE_MOCK_OUTPUT_TYPE_MISMATCH');
|
||||
expect(BIAS_EXPERIMENT_ERROR_CODES).toContain('BIAS_SIDE_EFFECT_CONFIRMATION_REQUIRED');
|
||||
expect(BIAS_EXPERIMENT_ERROR_CODES).toContain('BIAS_EXECUTION_HISTORY_MISMATCH');
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
import { BiasActivationMode } from './flow';
|
||||
|
||||
export type BiasCapabilities = {
|
||||
blockType: string;
|
||||
supported: boolean;
|
||||
isolatedExperimentSupported: boolean;
|
||||
fullFlowExperimentSupported: boolean;
|
||||
externalSideEffects: boolean;
|
||||
configurationDependent: boolean;
|
||||
activationModes: BiasActivationMode[];
|
||||
};
|
||||
|
||||
export type ExternalSideEffectPolicy = 'BLOCK' | 'MOCK' | 'REQUIRE_CONFIRMATION';
|
||||
|
||||
export type BiasImpactExperimentRequest = {
|
||||
annotationIds: string[];
|
||||
repetitions: number;
|
||||
includeRawOutputs: boolean;
|
||||
externalSideEffectPolicy: ExternalSideEffectPolicy;
|
||||
confirmExternalSideEffects: boolean;
|
||||
};
|
||||
|
||||
export type BiasRerunActivation = {
|
||||
nodeId: string;
|
||||
annotationIds: string[];
|
||||
};
|
||||
|
||||
export type BiasRerunRequest = {
|
||||
activations: BiasRerunActivation[];
|
||||
externalSideEffectPolicy: ExternalSideEffectPolicy;
|
||||
confirmExternalSideEffects: boolean;
|
||||
};
|
||||
|
||||
export type BiasExecutionMode = 'NORMAL' | 'BIAS_VARIANT' | string;
|
||||
|
||||
export type BiasExecutionContext = {
|
||||
experimentId: string;
|
||||
mode: BiasExecutionMode;
|
||||
activeAnnotationIdsByNode: Record<string, string[]>;
|
||||
externalSideEffectPolicy: ExternalSideEffectPolicy;
|
||||
externalSideEffectsConfirmed: boolean;
|
||||
};
|
||||
|
||||
export type BiasImpactJobStatus = 'QUEUED' | 'RUNNING' | 'COMPLETED' | 'FAILED';
|
||||
|
||||
export type BiasImpactJob = {
|
||||
id: string;
|
||||
status: BiasImpactJobStatus;
|
||||
executionId: string;
|
||||
stepId: string;
|
||||
createdAt: string;
|
||||
startedAt: string | null;
|
||||
completedAt: string | null;
|
||||
reportId: string | null;
|
||||
report: BiasImpactReport | null;
|
||||
errorCode: string | null;
|
||||
errorMessage: string | null;
|
||||
terminal: boolean;
|
||||
};
|
||||
|
||||
export type BiasImpactReportKind = 'ISOLATED_STEP' | 'FULL_FLOW';
|
||||
|
||||
export type BiasImmediateImpact = {
|
||||
outputChanged: boolean;
|
||||
maximumTextDifference: number;
|
||||
changeRate: number;
|
||||
baselineOutput: unknown;
|
||||
biasedOutputs: unknown[];
|
||||
};
|
||||
|
||||
export type BiasDownstreamImpactEntry = {
|
||||
nodeId: string;
|
||||
nodeName: string;
|
||||
baselineStatus: string;
|
||||
biasedStatus: string;
|
||||
changed: boolean;
|
||||
baselineOutputs: unknown;
|
||||
biasedOutputs: unknown;
|
||||
};
|
||||
|
||||
export type BiasRoutingChangeEntry = {
|
||||
nodeId: string;
|
||||
baselineBranch: string;
|
||||
biasedBranch: string;
|
||||
};
|
||||
|
||||
export type BiasMockedSideEffectKind =
|
||||
| 'HTTP'
|
||||
| 'MCP_AGENT'
|
||||
| 'MCP_AGENT_CHAT'
|
||||
| 'EXTERNAL';
|
||||
|
||||
export type BiasMockedSideEffect = {
|
||||
nodeId: string;
|
||||
nodeName: string;
|
||||
kind: BiasMockedSideEffectKind;
|
||||
};
|
||||
|
||||
export type BiasImpactReport = {
|
||||
id: string;
|
||||
experimentId: string;
|
||||
kind: BiasImpactReportKind;
|
||||
baselineExecutionId: string;
|
||||
biasedExecutionId: string | null;
|
||||
nodeId: string | null;
|
||||
annotationIds: string[];
|
||||
repetitions: number;
|
||||
createdAt: string;
|
||||
rawOutputsIncluded: boolean;
|
||||
immediateImpact: BiasImmediateImpact;
|
||||
downstreamImpact: BiasDownstreamImpactEntry[];
|
||||
routingChanges: BiasRoutingChangeEntry[];
|
||||
mockedSideEffects: BiasMockedSideEffect[];
|
||||
summary: string;
|
||||
warnings: string[];
|
||||
};
|
||||
|
||||
export const BIAS_PROBE_ERROR_CODES = [
|
||||
'BIAS_PROBE_MODE_REQUIRED',
|
||||
'BIAS_PROBE_INSTRUCTION_REQUIRED',
|
||||
'BIAS_PROBE_MODE_UNSUPPORTED',
|
||||
'BIAS_PROBE_TARGET_INPUT_NOT_FOUND',
|
||||
'BIAS_PROBE_MOCK_OUTPUTS_REQUIRED',
|
||||
'BIAS_PROBE_MOCK_OUTPUT_NOT_FOUND',
|
||||
'BIAS_PROBE_MOCK_OUTPUT_TYPE_MISMATCH',
|
||||
'DUPLICATE_BIAS_ANNOTATION_ID',
|
||||
'TOO_MANY_BIAS_ANNOTATIONS',
|
||||
'BIAS_FIELD_TOO_LONG'
|
||||
] as const;
|
||||
|
||||
export type BiasProbeErrorCode = typeof BIAS_PROBE_ERROR_CODES[number];
|
||||
|
||||
export const BIAS_EXPERIMENT_ERROR_CODES = [
|
||||
'BIAS_BASELINE_NOT_FINAL',
|
||||
'BIAS_STEP_NOT_FOUND',
|
||||
'BIAS_BLOCK_NOT_SUPPORTED',
|
||||
'BIAS_ANNOTATION_NOT_FOUND',
|
||||
'BIAS_ANNOTATION_NOT_EXECUTABLE',
|
||||
'BIAS_EXECUTION_NOT_FINAL',
|
||||
'BIAS_EXECUTION_NOT_VARIANT',
|
||||
'BIAS_EXECUTION_HISTORY_MISMATCH',
|
||||
'BIAS_SIDE_EFFECT_BLOCKED',
|
||||
'BIAS_SIDE_EFFECT_CONFIRMATION_REQUIRED',
|
||||
'BIAS_JOB_NOT_FOUND',
|
||||
'BIAS_REPORT_NOT_FOUND',
|
||||
'BIAS_EXPERIMENT_FAILED'
|
||||
] as const;
|
||||
|
||||
export type BiasExperimentErrorCode = typeof BIAS_EXPERIMENT_ERROR_CODES[number];
|
||||
|
||||
export type BiasSideEffectError = {
|
||||
reason: 'SIDE_EFFECT_BLOCKED' | 'CONFIRMATION_REQUIRED';
|
||||
code: 'BIAS_SIDE_EFFECT_BLOCKED' | 'BIAS_SIDE_EFFECT_CONFIRMATION_REQUIRED';
|
||||
message: string;
|
||||
};
|
||||
|
|
@ -86,6 +86,32 @@ export type FlowNodeBase = {
|
|||
nodeFamily?: NodeFamily;
|
||||
};
|
||||
|
||||
export type BiasActivationMode =
|
||||
| 'PROMPT_DIRECTIVE'
|
||||
| 'INPUT_TRANSFORMATION'
|
||||
| 'OUTPUT_TRANSFORMATION'
|
||||
| 'ROUTING_OVERRIDE'
|
||||
| 'MOCK_RESPONSE'
|
||||
| string;
|
||||
|
||||
export type BehavioralProbe = {
|
||||
activationMode?: BiasActivationMode;
|
||||
instruction?: string;
|
||||
targetInputs?: string[];
|
||||
expectedImpact?: string;
|
||||
mockOutputs?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export function isProbeExecutable(probe: BehavioralProbe | null | undefined): boolean {
|
||||
if (!probe?.activationMode) return false;
|
||||
|
||||
if (probe.activationMode === 'MOCK_RESPONSE') {
|
||||
return !!probe.mockOutputs && Object.keys(probe.mockOutputs).length > 0;
|
||||
}
|
||||
|
||||
return typeof probe.instruction === 'string' && probe.instruction.trim().length > 0;
|
||||
}
|
||||
|
||||
export type BiasAnnotation = Record<string, unknown> & {
|
||||
id?: string;
|
||||
category?: string;
|
||||
|
|
@ -96,6 +122,7 @@ export type BiasAnnotation = Record<string, unknown> & {
|
|||
status?: string;
|
||||
source?: string;
|
||||
analysisId?: string;
|
||||
behavioralProbe?: BehavioralProbe;
|
||||
};
|
||||
|
||||
export type BiasAnnotationOption = {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { FlowBlockConnection, FlowNode, FlowNodeDependency, FlowPort, LLMDescriptor } from './flow';
|
||||
import { BiasExecutionContext } from './bias-impact';
|
||||
|
||||
export type TaskExecutionStatus = 'CREATED' | 'READY' | 'RUNNING' | 'WAITING' | 'SUSPENDED' | 'SUCCESS' | 'ERROR' | 'CANCELLED';
|
||||
export type TaskExecutionStatusGroup = 'INIT' | 'RUNNING' | 'PAUSED' | 'FINAL';
|
||||
|
|
@ -25,6 +26,7 @@ export type TaskExecution = {
|
|||
sourceFlowId?: string | null;
|
||||
runNumber?: number | null;
|
||||
rerunOfExecutionId?: string | null;
|
||||
biasExecutionContext?: BiasExecutionContext;
|
||||
context: TaskExecutionContext;
|
||||
interactionSimulationEnabled?: boolean;
|
||||
simulationAvailable?: boolean;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { BiasAnnotationsDescriptor, BlockType, BlockTypeName, FlowBlock } from "@models/flow";
|
||||
import { BiasCapabilities } from "@models/bias-impact";
|
||||
import { Observable } from "rxjs";
|
||||
|
||||
export type BlockDraftContext = {
|
||||
|
|
@ -12,6 +13,10 @@ export abstract class BlocksCallServiceBase {
|
|||
|
||||
abstract retrieveBiasAnnotationsDescriptor(): Observable<BiasAnnotationsDescriptor>;
|
||||
|
||||
abstract retrieveBiasCapabilities(blockType: string): Observable<BiasCapabilities>;
|
||||
|
||||
abstract retrieveBiasCapabilitiesForInstance(blockType: string, block: FlowBlock): Observable<BiasCapabilities>;
|
||||
|
||||
abstract createEmptyBlock(blockType: BlockTypeName, context?: BlockDraftContext) : Observable<FlowBlock>;
|
||||
|
||||
abstract updateBlock(blockId : string, configuration : any, context?: BlockDraftContext) : Observable<FlowBlock>;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { BiasAnnotationsDescriptor, BlockType, FlowBlock } from "@models/flow";
|
||||
import { BiasCapabilities } from '@models/bias-impact';
|
||||
import { Observable, of } from "rxjs";
|
||||
import { BlockDraftContext, BlocksCallServiceBase } from "./block-call.base";
|
||||
|
||||
|
|
@ -179,6 +180,20 @@ export class BlocksCallServiceFake extends BlocksCallServiceBase {
|
|||
});
|
||||
}
|
||||
|
||||
override retrieveBiasCapabilities(blockType: string): Observable<BiasCapabilities> {
|
||||
return of(this.biasCapabilities(blockType, false));
|
||||
}
|
||||
|
||||
override retrieveBiasCapabilitiesForInstance(blockType: string, block: FlowBlock): Observable<BiasCapabilities> {
|
||||
const usesLlm = Boolean((block.specificConfiguration as Record<string, unknown>)['useLlm']);
|
||||
return of({
|
||||
...this.biasCapabilities(blockType, true),
|
||||
activationModes: usesLlm
|
||||
? ['PROMPT_DIRECTIVE', 'INPUT_TRANSFORMATION', 'OUTPUT_TRANSFORMATION']
|
||||
: ['INPUT_TRANSFORMATION', 'OUTPUT_TRANSFORMATION']
|
||||
});
|
||||
}
|
||||
|
||||
override createEmptyBlock(blockType: string, _context?: BlockDraftContext): Observable<FlowBlock> {
|
||||
const descriptor = this.blockTypes.find((b) => b.type === blockType);
|
||||
const typeName = descriptor?.type ?? blockType ?? "LLMBlock";
|
||||
|
|
@ -263,6 +278,28 @@ export class BlocksCallServiceFake extends BlocksCallServiceBase {
|
|||
return sanitized;
|
||||
}
|
||||
|
||||
private biasCapabilities(blockType: string, instanceSpecific: boolean): BiasCapabilities {
|
||||
const configurationDependent = blockType === 'ConditionalBlock' || blockType === 'SwitchBlock';
|
||||
const externalSideEffects = blockType === 'HTTPServerCallBlock'
|
||||
|| blockType === 'MCPAgentBlock'
|
||||
|| blockType === 'MCPAgentChatBlock';
|
||||
const activationModes = externalSideEffects
|
||||
? ['MOCK_RESPONSE']
|
||||
: configurationDependent && !instanceSpecific
|
||||
? ['INPUT_TRANSFORMATION', 'OUTPUT_TRANSFORMATION']
|
||||
: ['PROMPT_DIRECTIVE', 'INPUT_TRANSFORMATION', 'OUTPUT_TRANSFORMATION'];
|
||||
|
||||
return {
|
||||
blockType,
|
||||
supported: activationModes.length > 0,
|
||||
isolatedExperimentSupported: true,
|
||||
fullFlowExperimentSupported: activationModes.length > 0,
|
||||
externalSideEffects,
|
||||
configurationDependent,
|
||||
activationModes
|
||||
};
|
||||
}
|
||||
|
||||
private sanitizeSchemaValue(
|
||||
value: unknown,
|
||||
schemaNode: Record<string, unknown> | null,
|
||||
|
|
|
|||
|
|
@ -116,4 +116,56 @@ describe('BlocksCallService', () => {
|
|||
options: { category: [{ value: 'DYNAMIC_VALUE', label: 'Dynamic label', description: 'Dynamic description' }] }
|
||||
}));
|
||||
});
|
||||
|
||||
it('retrieves and caches type-level bias capabilities', async () => {
|
||||
const first = firstValueFrom(service.retrieveBiasCapabilities('LLMBlock'));
|
||||
const request = httpMock.expectOne(`${environment.apiUrl}/blocks/types/LLMBlock/bias-capabilities`);
|
||||
expect(request.request.method).toBe('GET');
|
||||
request.flush({
|
||||
blockType: 'LLMBlock',
|
||||
supported: true,
|
||||
isolatedExperimentSupported: true,
|
||||
fullFlowExperimentSupported: true,
|
||||
externalSideEffects: false,
|
||||
configurationDependent: false,
|
||||
activationModes: ['PROMPT_DIRECTIVE']
|
||||
});
|
||||
|
||||
await expect(first).resolves.toEqual(expect.objectContaining({
|
||||
blockType: 'LLMBlock', activationModes: ['PROMPT_DIRECTIVE']
|
||||
}));
|
||||
await expect(firstValueFrom(service.retrieveBiasCapabilities('LLMBlock'))).resolves.toEqual(expect.objectContaining({
|
||||
supported: true
|
||||
}));
|
||||
});
|
||||
|
||||
it('posts the configured block for instance-specific bias capabilities', async () => {
|
||||
const block = {
|
||||
id: 'conditional-1',
|
||||
name: 'Conditional',
|
||||
inputs: [],
|
||||
outputs: [{ name: 'true', type: 'TEXT', multiple: false }],
|
||||
specificConfiguration: { useLlm: true },
|
||||
typeName: 'ConditionalBlock',
|
||||
nodeFamily: 'block' as const
|
||||
};
|
||||
const response = firstValueFrom(service.retrieveBiasCapabilitiesForInstance('ConditionalBlock', block));
|
||||
const request = httpMock.expectOne(`${environment.apiUrl}/blocks/types/ConditionalBlock/bias-capabilities`);
|
||||
expect(request.request.method).toBe('POST');
|
||||
expect(request.request.body).toEqual(block);
|
||||
request.flush({
|
||||
blockType: 'ConditionalBlock',
|
||||
supported: true,
|
||||
isolatedExperimentSupported: true,
|
||||
fullFlowExperimentSupported: true,
|
||||
externalSideEffects: false,
|
||||
configurationDependent: true,
|
||||
activationModes: ['PROMPT_DIRECTIVE', 'INPUT_TRANSFORMATION']
|
||||
});
|
||||
|
||||
await expect(response).resolves.toEqual(expect.objectContaining({
|
||||
configurationDependent: true,
|
||||
activationModes: ['PROMPT_DIRECTIVE', 'INPUT_TRANSFORMATION']
|
||||
}));
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { BiasAnnotationOption, BiasAnnotationsDescriptor, BlockType, FlowBlock } from "@models/flow";
|
||||
import { BiasActivationMode, BiasAnnotationOption, BiasAnnotationsDescriptor, BlockType, FlowBlock } from "@models/flow";
|
||||
import { BiasCapabilities } from '@models/bias-impact';
|
||||
import { HttpClient, HttpParams } from "@angular/common/http";
|
||||
import { inject } from "@angular/core";
|
||||
import { environment } from "@environment";
|
||||
|
|
@ -8,6 +9,7 @@ import { BlockDraftContext, BlocksCallServiceBase } from "./block-call.base";
|
|||
export class BlocksCallService extends BlocksCallServiceBase {
|
||||
private readonly http = inject(HttpClient);
|
||||
private blockTypesCache: BlockType[] | null = null;
|
||||
private readonly biasCapabilitiesCache = new Map<string, BiasCapabilities>();
|
||||
|
||||
override retrieveAllBlocksTypes(): Observable<BlockType[]> {
|
||||
return this.http
|
||||
|
|
@ -27,6 +29,30 @@ export class BlocksCallService extends BlocksCallServiceBase {
|
|||
.pipe(map((raw) => this.biasAnnotationsDescriptorFromApi(raw)));
|
||||
}
|
||||
|
||||
override retrieveBiasCapabilities(blockType: string): Observable<BiasCapabilities> {
|
||||
const cached = this.biasCapabilitiesCache.get(blockType);
|
||||
if (cached) return of(cached);
|
||||
|
||||
return this.http
|
||||
.get<unknown>(`${environment.apiUrl}/blocks/types/${encodeURIComponent(blockType)}/bias-capabilities`)
|
||||
.pipe(
|
||||
map((raw) => this.biasCapabilitiesFromApi(raw, blockType)),
|
||||
map((capabilities) => {
|
||||
this.biasCapabilitiesCache.set(blockType, capabilities);
|
||||
return capabilities;
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
override retrieveBiasCapabilitiesForInstance(blockType: string, block: FlowBlock): Observable<BiasCapabilities> {
|
||||
return this.http
|
||||
.post<unknown>(
|
||||
`${environment.apiUrl}/blocks/types/${encodeURIComponent(blockType)}/bias-capabilities`,
|
||||
block
|
||||
)
|
||||
.pipe(map((raw) => this.biasCapabilitiesFromApi(raw, blockType)));
|
||||
}
|
||||
|
||||
override createEmptyBlock(blockType: string, context?: BlockDraftContext): Observable<FlowBlock> {
|
||||
return this.getBlockTypesForCreate().pipe(
|
||||
take(1),
|
||||
|
|
@ -186,6 +212,25 @@ export class BlocksCallService extends BlocksCallServiceBase {
|
|||
};
|
||||
}
|
||||
|
||||
private biasCapabilitiesFromApi(raw: unknown, fallbackBlockType: string): BiasCapabilities {
|
||||
const value = this.toRecord(raw);
|
||||
const activationModes = Array.isArray(value['activationModes'])
|
||||
? value['activationModes']
|
||||
.filter((mode): mode is string => typeof mode === 'string')
|
||||
.map((mode) => mode as BiasActivationMode)
|
||||
: [];
|
||||
|
||||
return {
|
||||
blockType: String(value['blockType'] ?? fallbackBlockType),
|
||||
supported: value['supported'] === true,
|
||||
isolatedExperimentSupported: value['isolatedExperimentSupported'] === true,
|
||||
fullFlowExperimentSupported: value['fullFlowExperimentSupported'] === true,
|
||||
externalSideEffects: value['externalSideEffects'] === true,
|
||||
configurationDependent: value['configurationDependent'] === true,
|
||||
activationModes
|
||||
};
|
||||
}
|
||||
|
||||
private toPorts(raw: unknown, fallback: Array<{ name: string; type: string; multiple: boolean }>) {
|
||||
if (!Array.isArray(raw)) return fallback;
|
||||
return raw
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
import { computed, Injectable, signal } from '@angular/core';
|
||||
import { environment } from '@environment';
|
||||
import { BiasAnnotationsDescriptor, BlockType, BlockTypeName, FlowBlock } from '@models/flow';
|
||||
import { BiasCapabilities } from '@models/bias-impact';
|
||||
import { BlockDraftContext, BlocksCallServiceBase } from './block-call.base';
|
||||
import { catchError, finalize, firstValueFrom, map, Observable, of, shareReplay, throwError } from 'rxjs';
|
||||
import { catchError, finalize, firstValueFrom, map, Observable, of, shareReplay, tap, throwError } from 'rxjs';
|
||||
|
||||
@Injectable({
|
||||
providedIn: 'root',
|
||||
|
|
@ -19,11 +20,13 @@ export class BlocksService {
|
|||
|
||||
private _blockTypes = signal<BlockType[]>([]);
|
||||
private readonly _biasAnnotationsDescriptor = signal<BiasAnnotationsDescriptor | null>(null);
|
||||
private readonly _biasCapabilities = signal<Record<string, BiasCapabilities>>({});
|
||||
private biasDescriptorPromise: Promise<BiasAnnotationsDescriptor> | null = null;
|
||||
readonly hasPendingServerSync = computed(() => this.pendingServerSyncCount() > 0);
|
||||
readonly blockTypes = this._blockTypes.asReadonly();
|
||||
readonly catalogLoading = this._catalogLoading.asReadonly();
|
||||
readonly biasAnnotationsDescriptor = this._biasAnnotationsDescriptor.asReadonly();
|
||||
readonly biasCapabilities = this._biasCapabilities.asReadonly();
|
||||
|
||||
async getBiasAnnotationsDescriptor(force = false): Promise<BiasAnnotationsDescriptor> {
|
||||
const cached = this._biasAnnotationsDescriptor();
|
||||
|
|
@ -39,6 +42,21 @@ export class BlocksService {
|
|||
return this.biasDescriptorPromise;
|
||||
}
|
||||
|
||||
retrieveBiasCapabilities(blockType: string, force = false): Observable<BiasCapabilities> {
|
||||
const cached = this._biasCapabilities()[blockType];
|
||||
if (cached && !force) return of(cached);
|
||||
|
||||
return this.blocksCallService.retrieveBiasCapabilities(blockType).pipe(
|
||||
tap((capabilities) => {
|
||||
this._biasCapabilities.update((current) => ({ ...current, [blockType]: capabilities }));
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
retrieveBiasCapabilitiesForInstance(blockType: string, block: FlowBlock): Observable<BiasCapabilities> {
|
||||
return this.blocksCallService.retrieveBiasCapabilitiesForInstance(blockType, block);
|
||||
}
|
||||
|
||||
hasLoadedBlockTypes() {
|
||||
return this._blockTypes().length > 0 || (!this.toInit && !this.loadingPromise);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
import { TestBed } from '@angular/core/testing';
|
||||
import { BiasImpactExperimentDialogService } from './bias-impact-experiment-dialog';
|
||||
|
||||
describe('BiasImpactExperimentDialogService', () => {
|
||||
let service: BiasImpactExperimentDialogService;
|
||||
|
||||
beforeEach(() => {
|
||||
TestBed.configureTestingModule({});
|
||||
service = TestBed.inject(BiasImpactExperimentDialogService);
|
||||
});
|
||||
|
||||
it('opens only when at least one annotation has an executable probe', () => {
|
||||
service.open({
|
||||
executionId: 'execution', stepId: 'step', nodeId: 'node', nodeName: 'Node',
|
||||
capabilities: { blockType: 'LLM', supported: true, isolatedExperimentSupported: true, fullFlowExperimentSupported: true, externalSideEffects: false, configurationDependent: false, activationModes: [] },
|
||||
annotations: [
|
||||
{ id: 'not-executable', behavioralProbe: { activationMode: 'PROMPT_DIRECTIVE' } },
|
||||
{ id: 'executable', behavioralProbe: { activationMode: 'PROMPT_DIRECTIVE', instruction: 'Apply probe' } }
|
||||
]
|
||||
});
|
||||
|
||||
expect(service.state()?.annotations.map((annotation) => annotation.id)).toEqual(['executable']);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
import { Injectable, signal } from '@angular/core';
|
||||
import { BiasAnnotation, isProbeExecutable } from '@models/flow';
|
||||
import { BiasCapabilities } from '@models/bias-impact';
|
||||
|
||||
export type BiasImpactExperimentDialogInput = {
|
||||
executionId: string;
|
||||
stepId: string;
|
||||
nodeId: string;
|
||||
nodeName: string;
|
||||
annotations: BiasAnnotation[];
|
||||
capabilities: BiasCapabilities;
|
||||
};
|
||||
|
||||
@Injectable({ providedIn: 'root' })
|
||||
export class BiasImpactExperimentDialogService {
|
||||
private readonly _state = signal<BiasImpactExperimentDialogInput | null>(null);
|
||||
readonly state = this._state.asReadonly();
|
||||
|
||||
open(input: BiasImpactExperimentDialogInput) {
|
||||
const annotations = input.annotations.filter((annotation) => isProbeExecutable(annotation.behavioralProbe));
|
||||
if (!annotations.length) return;
|
||||
this._state.set({ ...input, annotations });
|
||||
}
|
||||
|
||||
close() {
|
||||
this._state.set(null);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
import { TestBed } from '@angular/core/testing';
|
||||
import { BiasRerunDialogService } from './bias-rerun-dialog';
|
||||
|
||||
describe('BiasRerunDialogService', () => {
|
||||
it('keeps a dialog state only when there are eligible block candidates', () => {
|
||||
const service = TestBed.inject(BiasRerunDialogService);
|
||||
const base = { executionId: 'baseline', onCreated: () => undefined };
|
||||
service.open({ ...base, candidates: [] });
|
||||
expect(service.state()).toBeNull();
|
||||
|
||||
service.open({
|
||||
...base,
|
||||
candidates: [{
|
||||
nodeId: 'node-1', nodeName: 'Node 1', annotations: [],
|
||||
capabilities: { blockType: 'LLM', supported: true, isolatedExperimentSupported: true, fullFlowExperimentSupported: true, externalSideEffects: false, configurationDependent: false, activationModes: [] }
|
||||
}]
|
||||
});
|
||||
expect(service.state()?.executionId).toBe('baseline');
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
import { Injectable, signal } from '@angular/core';
|
||||
import { BiasAnnotation } from '@models/flow';
|
||||
import { BiasCapabilities } from '@models/bias-impact';
|
||||
import { TaskExecution } from '@models/task-execution';
|
||||
|
||||
export type BiasRerunCandidate = {
|
||||
nodeId: string;
|
||||
nodeName: string;
|
||||
annotations: BiasAnnotation[];
|
||||
capabilities: BiasCapabilities;
|
||||
};
|
||||
|
||||
export type BiasRerunDialogInput = {
|
||||
executionId: string;
|
||||
candidates: BiasRerunCandidate[];
|
||||
onCreated: (execution: TaskExecution) => void;
|
||||
};
|
||||
|
||||
@Injectable({ providedIn: 'root' })
|
||||
export class BiasRerunDialogService {
|
||||
private readonly _state = signal<BiasRerunDialogInput | null>(null);
|
||||
readonly state = this._state.asReadonly();
|
||||
|
||||
open(input: BiasRerunDialogInput) {
|
||||
if (input.candidates.length) this._state.set(input);
|
||||
}
|
||||
|
||||
close() { this._state.set(null); }
|
||||
}
|
||||
|
|
@ -1,4 +1,10 @@
|
|||
import { LLMDescriptor } from '@models/flow';
|
||||
import {
|
||||
BiasImpactExperimentRequest,
|
||||
BiasImpactJob,
|
||||
BiasImpactReport,
|
||||
BiasRerunRequest
|
||||
} from '@models/bias-impact';
|
||||
import { ExecutionEventLogEntry, TaskExecution, TaskExecutionGroup } from '@models/task-execution';
|
||||
import { Observable } from 'rxjs';
|
||||
|
||||
|
|
@ -8,6 +14,20 @@ export abstract class TaskExecutionsCallServiceBase {
|
|||
abstract retrieveExecutionEvents(executionId: string): Observable<ExecutionEventLogEntry[]>;
|
||||
abstract createTaskExecution(flowId: string): Observable<TaskExecution>;
|
||||
abstract rerunTaskExecution(executionId: string): Observable<TaskExecution>;
|
||||
abstract runBiasImpactExperiment(
|
||||
executionId: string,
|
||||
stepId: string,
|
||||
request: BiasImpactExperimentRequest
|
||||
): Observable<BiasImpactJob>;
|
||||
abstract getBiasImpactJob(jobId: string): Observable<BiasImpactJob>;
|
||||
abstract createBiasedRerun(executionId: string, request: BiasRerunRequest): Observable<TaskExecution>;
|
||||
abstract compareBiasExecutions(
|
||||
baselineExecutionId: string,
|
||||
biasedExecutionId: string,
|
||||
includeRawOutputs: boolean
|
||||
): Observable<BiasImpactReport>;
|
||||
abstract listBiasImpactReports(executionId: string): Observable<BiasImpactReport[]>;
|
||||
abstract getBiasImpactReport(reportId: string): Observable<BiasImpactReport>;
|
||||
abstract deleteTaskExecution(executionId: string): Observable<void>;
|
||||
abstract startTaskExecution(executionId: string): Observable<TaskExecution>;
|
||||
abstract simulateTaskExecution(executionId: string, simulator: LLMDescriptor): Observable<TaskExecution>;
|
||||
|
|
|
|||
|
|
@ -1,9 +1,17 @@
|
|||
import { LLMDescriptor } from '@models/flow';
|
||||
import {
|
||||
BiasImpactExperimentRequest,
|
||||
BiasImpactJob,
|
||||
BiasImpactReport,
|
||||
BiasRerunRequest
|
||||
} from '@models/bias-impact';
|
||||
import { ExecutionEventLogEntry, TaskExecution, TaskExecutionGroup } from '@models/task-execution';
|
||||
import { Observable, of } from 'rxjs';
|
||||
import { map, Observable, of } from 'rxjs';
|
||||
import { TaskExecutionsCallServiceBase } from './task-executions-call.base';
|
||||
|
||||
export class TaskExecutionsCallServiceFake extends TaskExecutionsCallServiceBase {
|
||||
private readonly biasJobs = new Map<string, { polls: number; job: BiasImpactJob }>();
|
||||
private readonly biasReports: BiasImpactReport[] = [];
|
||||
private readonly data: TaskExecution[] = [
|
||||
{
|
||||
id: 'c106be9d-5467-428c-8992-0b5f40a59aac',
|
||||
|
|
@ -518,6 +526,113 @@ export class TaskExecutionsCallServiceFake extends TaskExecutionsCallServiceBase
|
|||
return of(this.withSimulationAvailability(execution));
|
||||
}
|
||||
|
||||
override runBiasImpactExperiment(
|
||||
executionId: string,
|
||||
stepId: string,
|
||||
request: BiasImpactExperimentRequest
|
||||
): Observable<BiasImpactJob> {
|
||||
const job: BiasImpactJob = {
|
||||
id: crypto.randomUUID(),
|
||||
status: 'QUEUED',
|
||||
executionId,
|
||||
stepId,
|
||||
createdAt: new Date().toISOString(),
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
reportId: null,
|
||||
report: null,
|
||||
errorCode: null,
|
||||
errorMessage: null,
|
||||
terminal: false
|
||||
};
|
||||
this.biasJobs.set(job.id, { polls: 0, job: { ...job, report: null } });
|
||||
return of(job);
|
||||
}
|
||||
|
||||
override getBiasImpactJob(jobId: string): Observable<BiasImpactJob> {
|
||||
const entry = this.biasJobs.get(jobId);
|
||||
if (!entry) {
|
||||
throw new Error(`Bias impact job not found: ${jobId}`);
|
||||
}
|
||||
|
||||
entry.polls += 1;
|
||||
if (entry.polls === 1) {
|
||||
entry.job = { ...entry.job, status: 'RUNNING', startedAt: new Date().toISOString() };
|
||||
} else if (!entry.job.terminal) {
|
||||
const report = this.createIsolatedReport(entry.job.executionId, entry.job.stepId);
|
||||
this.biasReports.unshift(report);
|
||||
entry.job = {
|
||||
...entry.job,
|
||||
status: 'COMPLETED',
|
||||
completedAt: new Date().toISOString(),
|
||||
reportId: report.id,
|
||||
report,
|
||||
terminal: true
|
||||
};
|
||||
}
|
||||
return of(entry.job);
|
||||
}
|
||||
|
||||
override createBiasedRerun(executionId: string, request: BiasRerunRequest): Observable<TaskExecution> {
|
||||
return this.rerunTaskExecution(executionId).pipe(
|
||||
map((execution) => {
|
||||
const biasedExecution: TaskExecution = {
|
||||
...execution,
|
||||
biasExecutionContext: {
|
||||
experimentId: crypto.randomUUID(),
|
||||
mode: 'BIAS_VARIANT',
|
||||
activeAnnotationIdsByNode: Object.fromEntries(
|
||||
request.activations.map((activation) => [activation.nodeId, activation.annotationIds])
|
||||
),
|
||||
externalSideEffectPolicy: request.externalSideEffectPolicy,
|
||||
externalSideEffectsConfirmed: request.confirmExternalSideEffects
|
||||
}
|
||||
};
|
||||
const index = this.data.findIndex((item) => item.id === biasedExecution.id);
|
||||
if (index >= 0) this.data[index] = biasedExecution;
|
||||
return biasedExecution;
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
override compareBiasExecutions(
|
||||
baselineExecutionId: string,
|
||||
biasedExecutionId: string,
|
||||
includeRawOutputs: boolean
|
||||
): Observable<BiasImpactReport> {
|
||||
const existing = this.biasReports.find((report) =>
|
||||
report.baselineExecutionId === baselineExecutionId
|
||||
&& report.biasedExecutionId === biasedExecutionId
|
||||
&& report.rawOutputsIncluded === includeRawOutputs
|
||||
);
|
||||
if (existing) return of(existing);
|
||||
|
||||
const report: BiasImpactReport = {
|
||||
...this.createIsolatedReport(baselineExecutionId, 'biased-node'),
|
||||
id: crypto.randomUUID(),
|
||||
kind: 'FULL_FLOW',
|
||||
biasedExecutionId,
|
||||
nodeId: null,
|
||||
repetitions: 1,
|
||||
rawOutputsIncluded: includeRawOutputs,
|
||||
summary: 'Observed a changed downstream node in the biased rerun.'
|
||||
};
|
||||
this.biasReports.unshift(report);
|
||||
return of(report);
|
||||
}
|
||||
|
||||
override listBiasImpactReports(executionId: string): Observable<BiasImpactReport[]> {
|
||||
return of(this.biasReports.filter((report) =>
|
||||
report.baselineExecutionId === executionId || report.biasedExecutionId === executionId
|
||||
));
|
||||
}
|
||||
|
||||
override getBiasImpactReport(reportId: string): Observable<BiasImpactReport> {
|
||||
const report = this.biasReports.find((item) => item.id === reportId);
|
||||
if (!report) throw new Error(`Bias impact report not found: ${reportId}`);
|
||||
return of(report);
|
||||
}
|
||||
|
||||
override deleteTaskExecution(executionId: string): Observable<void> {
|
||||
const index = this.data.findIndex((item) => item.id === executionId);
|
||||
if (index >= 0) {
|
||||
|
|
@ -727,6 +842,33 @@ export class TaskExecutionsCallServiceFake extends TaskExecutionsCallServiceBase
|
|||
return of(execution);
|
||||
}
|
||||
|
||||
private createIsolatedReport(baselineExecutionId: string, stepId: string): BiasImpactReport {
|
||||
return {
|
||||
id: crypto.randomUUID(),
|
||||
experimentId: crypto.randomUUID(),
|
||||
kind: 'ISOLATED_STEP',
|
||||
baselineExecutionId,
|
||||
biasedExecutionId: null,
|
||||
nodeId: stepId,
|
||||
annotationIds: ['demo-bias-annotation'],
|
||||
repetitions: 3,
|
||||
createdAt: new Date().toISOString(),
|
||||
rawOutputsIncluded: true,
|
||||
immediateImpact: {
|
||||
outputChanged: true,
|
||||
maximumTextDifference: 0.4,
|
||||
changeRate: 1,
|
||||
baselineOutput: { output: 'Baseline result' },
|
||||
biasedOutputs: [{ output: 'Biased result' }]
|
||||
},
|
||||
downstreamImpact: [],
|
||||
routingChanges: [],
|
||||
mockedSideEffects: [],
|
||||
summary: 'The selected bias changed the observed output.',
|
||||
warnings: ['This comparison can include normal model non-determinism.']
|
||||
};
|
||||
}
|
||||
|
||||
private findExecution(executionId: string): TaskExecution {
|
||||
const execution = this.data.find((item) => item.id === executionId);
|
||||
if (!execution) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,115 @@
|
|||
import { provideHttpClient } from '@angular/common/http';
|
||||
import { HttpTestingController, provideHttpClientTesting } from '@angular/common/http/testing';
|
||||
import { TestBed } from '@angular/core/testing';
|
||||
import { environment } from '@environment';
|
||||
import { firstValueFrom } from 'rxjs';
|
||||
|
||||
import { TaskExecutionsCallService } from './task-executions-call';
|
||||
|
||||
const report = {
|
||||
id: 'report-1',
|
||||
experimentId: 'experiment-1',
|
||||
kind: 'ISOLATED_STEP',
|
||||
baselineExecutionId: 'execution-1',
|
||||
biasedExecutionId: null,
|
||||
nodeId: 'step-1',
|
||||
annotationIds: ['annotation-1'],
|
||||
repetitions: 3,
|
||||
createdAt: '2026-07-21T10:00:00',
|
||||
rawOutputsIncluded: true,
|
||||
immediateImpact: {
|
||||
outputChanged: true,
|
||||
maximumTextDifference: 0.4,
|
||||
changeRate: 1,
|
||||
baselineOutput: { output: 'baseline' },
|
||||
biasedOutputs: [{ output: 'biased' }]
|
||||
},
|
||||
downstreamImpact: [],
|
||||
routingChanges: [],
|
||||
mockedSideEffects: [{ nodeId: 'http-1', nodeName: 'HTTP call', kind: 'HTTP' }],
|
||||
summary: 'Changed output',
|
||||
warnings: []
|
||||
};
|
||||
|
||||
describe('TaskExecutionsCallService bias APIs', () => {
|
||||
let service: TaskExecutionsCallService;
|
||||
let httpMock: HttpTestingController;
|
||||
|
||||
beforeEach(() => {
|
||||
TestBed.configureTestingModule({
|
||||
providers: [TaskExecutionsCallService, provideHttpClient(), provideHttpClientTesting()]
|
||||
});
|
||||
service = TestBed.inject(TaskExecutionsCallService);
|
||||
httpMock = TestBed.inject(HttpTestingController);
|
||||
});
|
||||
|
||||
afterEach(() => httpMock.verify());
|
||||
|
||||
it('starts an asynchronous impact experiment and maps the job response', async () => {
|
||||
const result = firstValueFrom(service.runBiasImpactExperiment('execution-1', 'step-1', {
|
||||
annotationIds: ['annotation-1'],
|
||||
repetitions: 3,
|
||||
includeRawOutputs: true,
|
||||
externalSideEffectPolicy: 'BLOCK',
|
||||
confirmExternalSideEffects: false
|
||||
}));
|
||||
const request = httpMock.expectOne(`${environment.apiUrl}/executions/execution-1/steps/step-1/bias-impact`);
|
||||
expect(request.request.method).toBe('POST');
|
||||
request.flush({
|
||||
id: 'job-1', status: 'QUEUED', executionId: 'execution-1', stepId: 'step-1',
|
||||
createdAt: '2026-07-21T10:00:00', startedAt: null, completedAt: null,
|
||||
reportId: null, report: null, errorCode: null, errorMessage: null, terminal: false
|
||||
});
|
||||
|
||||
await expect(result).resolves.toEqual(expect.objectContaining({ id: 'job-1', status: 'QUEUED', terminal: false }));
|
||||
});
|
||||
|
||||
it('retrieves a completed job with its persisted report', async () => {
|
||||
const result = firstValueFrom(service.getBiasImpactJob('job-1'));
|
||||
const request = httpMock.expectOne(`${environment.apiUrl}/executions/bias-impact-jobs/job-1`);
|
||||
expect(request.request.method).toBe('GET');
|
||||
request.flush({
|
||||
id: 'job-1', status: 'COMPLETED', executionId: 'execution-1', stepId: 'step-1',
|
||||
createdAt: '2026-07-21T10:00:00', startedAt: '2026-07-21T10:00:01', completedAt: '2026-07-21T10:00:02',
|
||||
reportId: 'report-1', report, errorCode: null, errorMessage: null, terminal: true
|
||||
});
|
||||
|
||||
await expect(result).resolves.toEqual(expect.objectContaining({
|
||||
status: 'COMPLETED', reportId: 'report-1', report: expect.objectContaining({ mockedSideEffects: report.mockedSideEffects })
|
||||
}));
|
||||
});
|
||||
|
||||
it('uses the confirmed biased rerun and comparison routes', async () => {
|
||||
const rerun = firstValueFrom(service.createBiasedRerun('baseline-1', {
|
||||
activations: [{ nodeId: 'node-1', annotationIds: ['annotation-1'] }],
|
||||
externalSideEffectPolicy: 'MOCK',
|
||||
confirmExternalSideEffects: false
|
||||
}));
|
||||
const rerunRequest = httpMock.expectOne(`${environment.apiUrl}/executions/baseline-1/bias-rerun`);
|
||||
expect(rerunRequest.request.method).toBe('POST');
|
||||
rerunRequest.flush({ id: 'variant-1', name: 'Variant', creationTime: 1, context: {} });
|
||||
await expect(rerun).resolves.toEqual(expect.objectContaining({ id: 'variant-1' }));
|
||||
|
||||
const comparison = firstValueFrom(service.compareBiasExecutions('baseline-1', 'variant-1', true));
|
||||
const comparisonRequest = httpMock.expectOne(
|
||||
`${environment.apiUrl}/executions/baseline-1/bias-compare/variant-1?includeRawOutputs=true`
|
||||
);
|
||||
expect(comparisonRequest.request.method).toBe('POST');
|
||||
comparisonRequest.flush({ ...report, baselineExecutionId: 'baseline-1', biasedExecutionId: 'variant-1', kind: 'FULL_FLOW' });
|
||||
await expect(comparison).resolves.toEqual(expect.objectContaining({ kind: 'FULL_FLOW', biasedExecutionId: 'variant-1' }));
|
||||
});
|
||||
|
||||
it('lists and retrieves persisted reports', async () => {
|
||||
const listed = firstValueFrom(service.listBiasImpactReports('execution-1'));
|
||||
const listRequest = httpMock.expectOne(`${environment.apiUrl}/executions/execution-1/bias-impact-reports`);
|
||||
expect(listRequest.request.method).toBe('GET');
|
||||
listRequest.flush([report]);
|
||||
await expect(listed).resolves.toEqual([expect.objectContaining({ id: 'report-1' })]);
|
||||
|
||||
const detail = firstValueFrom(service.getBiasImpactReport('report-1'));
|
||||
const detailRequest = httpMock.expectOne(`${environment.apiUrl}/executions/bias-impact-reports/report-1`);
|
||||
expect(detailRequest.request.method).toBe('GET');
|
||||
detailRequest.flush(report);
|
||||
await expect(detail).resolves.toEqual(expect.objectContaining({ id: 'report-1' }));
|
||||
});
|
||||
});
|
||||
|
|
@ -1,7 +1,18 @@
|
|||
import { HttpClient } from '@angular/common/http';
|
||||
import { HttpClient, HttpParams } from '@angular/common/http';
|
||||
import { inject } from '@angular/core';
|
||||
import { environment } from '@environment';
|
||||
import { LLMDescriptor } from '@models/flow';
|
||||
import {
|
||||
BiasDownstreamImpactEntry,
|
||||
BiasImpactExperimentRequest,
|
||||
BiasImpactJob,
|
||||
BiasImpactJobStatus,
|
||||
BiasImpactReport,
|
||||
BiasImpactReportKind,
|
||||
BiasMockedSideEffect,
|
||||
BiasRerunRequest,
|
||||
BiasRoutingChangeEntry
|
||||
} from '@models/bias-impact';
|
||||
import { ExecutionEventLogEntry, TaskExecution, TaskExecutionGroup } from '@models/task-execution';
|
||||
import { map, Observable } from 'rxjs';
|
||||
import { TaskExecutionsCallServiceBase } from './task-executions-call.base';
|
||||
|
|
@ -37,6 +48,50 @@ export class TaskExecutionsCallService extends TaskExecutionsCallServiceBase {
|
|||
);
|
||||
}
|
||||
|
||||
override runBiasImpactExperiment(
|
||||
executionId: string,
|
||||
stepId: string,
|
||||
request: BiasImpactExperimentRequest
|
||||
): Observable<BiasImpactJob> {
|
||||
const url = `${environment.apiUrl}/executions/${encodeURIComponent(executionId)}/steps/${encodeURIComponent(stepId)}/bias-impact`;
|
||||
return this.http.post<unknown>(url, request).pipe(map((raw) => this.biasImpactJobFromApi(raw)));
|
||||
}
|
||||
|
||||
override getBiasImpactJob(jobId: string): Observable<BiasImpactJob> {
|
||||
return this.http
|
||||
.get<unknown>(`${environment.apiUrl}/executions/bias-impact-jobs/${encodeURIComponent(jobId)}`)
|
||||
.pipe(map((raw) => this.biasImpactJobFromApi(raw)));
|
||||
}
|
||||
|
||||
override createBiasedRerun(executionId: string, request: BiasRerunRequest): Observable<TaskExecution> {
|
||||
return this.http
|
||||
.post<unknown>(`${environment.apiUrl}/executions/${encodeURIComponent(executionId)}/bias-rerun`, request)
|
||||
.pipe(map((raw) => this.mapExecution(raw)));
|
||||
}
|
||||
|
||||
override compareBiasExecutions(
|
||||
baselineExecutionId: string,
|
||||
biasedExecutionId: string,
|
||||
includeRawOutputs: boolean
|
||||
): Observable<BiasImpactReport> {
|
||||
const url = `${environment.apiUrl}/executions/${encodeURIComponent(baselineExecutionId)}/bias-compare/${encodeURIComponent(biasedExecutionId)}`;
|
||||
return this.http.post<unknown>(url, null, {
|
||||
params: new HttpParams().set('includeRawOutputs', String(includeRawOutputs))
|
||||
}).pipe(map((raw) => this.biasImpactReportFromApi(raw)));
|
||||
}
|
||||
|
||||
override listBiasImpactReports(executionId: string): Observable<BiasImpactReport[]> {
|
||||
return this.http
|
||||
.get<unknown>(`${environment.apiUrl}/executions/${encodeURIComponent(executionId)}/bias-impact-reports`)
|
||||
.pipe(map((raw) => Array.isArray(raw) ? raw.map((item) => this.biasImpactReportFromApi(item)) : []));
|
||||
}
|
||||
|
||||
override getBiasImpactReport(reportId: string): Observable<BiasImpactReport> {
|
||||
return this.http
|
||||
.get<unknown>(`${environment.apiUrl}/executions/bias-impact-reports/${encodeURIComponent(reportId)}`)
|
||||
.pipe(map((raw) => this.biasImpactReportFromApi(raw)));
|
||||
}
|
||||
|
||||
override deleteTaskExecution(executionId: string): Observable<void> {
|
||||
return this.http.delete<void>(`${environment.apiUrl}/executions/${encodeURIComponent(executionId)}`);
|
||||
}
|
||||
|
|
@ -203,6 +258,114 @@ export class TaskExecutionsCallService extends TaskExecutionsCallServiceBase {
|
|||
};
|
||||
}
|
||||
|
||||
private biasImpactJobFromApi(raw: unknown): BiasImpactJob {
|
||||
const value = this.toRecord(raw);
|
||||
const status = this.toBiasImpactJobStatus(value['status']);
|
||||
const rawReport = value['report'];
|
||||
return {
|
||||
id: String(value['id'] ?? ''),
|
||||
status,
|
||||
executionId: String(value['executionId'] ?? ''),
|
||||
stepId: String(value['stepId'] ?? ''),
|
||||
createdAt: String(value['createdAt'] ?? ''),
|
||||
startedAt: this.toNullableString(value['startedAt']),
|
||||
completedAt: this.toNullableString(value['completedAt']),
|
||||
reportId: this.toNullableString(value['reportId']),
|
||||
report: rawReport && typeof rawReport === 'object' ? this.biasImpactReportFromApi(rawReport) : null,
|
||||
errorCode: this.toNullableString(value['errorCode']),
|
||||
errorMessage: this.toNullableString(value['errorMessage']),
|
||||
terminal: value['terminal'] === true || status === 'COMPLETED' || status === 'FAILED'
|
||||
};
|
||||
}
|
||||
|
||||
private biasImpactReportFromApi(raw: unknown): BiasImpactReport {
|
||||
const value = this.toRecord(raw);
|
||||
const immediate = this.toRecord(value['immediateImpact']);
|
||||
return {
|
||||
id: String(value['id'] ?? ''),
|
||||
experimentId: String(value['experimentId'] ?? ''),
|
||||
kind: this.toBiasImpactReportKind(value['kind']),
|
||||
baselineExecutionId: String(value['baselineExecutionId'] ?? ''),
|
||||
biasedExecutionId: this.toNullableString(value['biasedExecutionId']),
|
||||
nodeId: this.toNullableString(value['nodeId']),
|
||||
annotationIds: this.toStringArray(value['annotationIds']),
|
||||
repetitions: this.toNumber(value['repetitions'], 0),
|
||||
createdAt: String(value['createdAt'] ?? ''),
|
||||
rawOutputsIncluded: value['rawOutputsIncluded'] === true,
|
||||
immediateImpact: {
|
||||
outputChanged: immediate['outputChanged'] === true,
|
||||
maximumTextDifference: this.toNumber(immediate['maximumTextDifference'], 0),
|
||||
changeRate: this.toNumber(immediate['changeRate'], 0),
|
||||
baselineOutput: immediate['baselineOutput'] ?? {},
|
||||
biasedOutputs: Array.isArray(immediate['biasedOutputs']) ? immediate['biasedOutputs'] : []
|
||||
},
|
||||
downstreamImpact: this.toDownstreamImpact(value['downstreamImpact']),
|
||||
routingChanges: this.toRoutingChanges(value['routingChanges']),
|
||||
mockedSideEffects: this.toMockedSideEffects(value['mockedSideEffects']),
|
||||
summary: String(value['summary'] ?? ''),
|
||||
warnings: this.toStringArray(value['warnings'])
|
||||
};
|
||||
}
|
||||
|
||||
private toDownstreamImpact(raw: unknown): BiasDownstreamImpactEntry[] {
|
||||
if (!Array.isArray(raw)) return [];
|
||||
return raw.map((item) => {
|
||||
const value = this.toRecord(item);
|
||||
return {
|
||||
nodeId: String(value['nodeId'] ?? ''),
|
||||
nodeName: String(value['nodeName'] ?? ''),
|
||||
baselineStatus: String(value['baselineStatus'] ?? ''),
|
||||
biasedStatus: String(value['biasedStatus'] ?? ''),
|
||||
changed: value['changed'] === true,
|
||||
baselineOutputs: value['baselineOutputs'] ?? {},
|
||||
biasedOutputs: value['biasedOutputs'] ?? {}
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
private toRoutingChanges(raw: unknown): BiasRoutingChangeEntry[] {
|
||||
if (!Array.isArray(raw)) return [];
|
||||
return raw.map((item) => {
|
||||
const value = this.toRecord(item);
|
||||
return {
|
||||
nodeId: String(value['nodeId'] ?? ''),
|
||||
baselineBranch: String(value['baselineBranch'] ?? ''),
|
||||
biasedBranch: String(value['biasedBranch'] ?? '')
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
private toMockedSideEffects(raw: unknown): BiasMockedSideEffect[] {
|
||||
if (!Array.isArray(raw)) return [];
|
||||
return raw.map((item) => {
|
||||
const value = this.toRecord(item);
|
||||
const kind = String(value['kind'] ?? 'EXTERNAL');
|
||||
return {
|
||||
nodeId: String(value['nodeId'] ?? ''),
|
||||
nodeName: String(value['nodeName'] ?? ''),
|
||||
kind: kind === 'HTTP' || kind === 'MCP_AGENT' || kind === 'MCP_AGENT_CHAT' ? kind : 'EXTERNAL'
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
private toBiasImpactJobStatus(value: unknown): BiasImpactJobStatus {
|
||||
return value === 'RUNNING' || value === 'COMPLETED' || value === 'FAILED' ? value : 'QUEUED';
|
||||
}
|
||||
|
||||
private toBiasImpactReportKind(value: unknown): BiasImpactReportKind {
|
||||
return value === 'FULL_FLOW' ? value : 'ISOLATED_STEP';
|
||||
}
|
||||
|
||||
private toStringArray(value: unknown): string[] {
|
||||
return Array.isArray(value) ? value.map(String) : [];
|
||||
}
|
||||
|
||||
private toRecord(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === 'object' && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: {};
|
||||
}
|
||||
|
||||
private mapExecutionGroup(raw: unknown): TaskExecutionGroup {
|
||||
const group = (raw ?? {}) as Partial<TaskExecutionGroup> & Record<string, unknown>;
|
||||
const executions = Array.isArray(group['executions'])
|
||||
|
|
@ -264,6 +427,10 @@ export class TaskExecutionsCallService extends TaskExecutionsCallServiceBase {
|
|||
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
private toNullableString(value: unknown): string | null {
|
||||
return typeof value === 'string' && value.trim().length > 0 ? value : null;
|
||||
}
|
||||
|
||||
private toTimestamp(value: unknown, fallback: number): number {
|
||||
const timestamp = typeof value === 'number' ? value : Number(value);
|
||||
return Number.isFinite(timestamp) ? timestamp : fallback;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,87 @@
|
|||
import { TestBed } from '@angular/core/testing';
|
||||
import { BiasImpactJob } from '@models/bias-impact';
|
||||
import { lastValueFrom, of, throwError } from 'rxjs';
|
||||
import { vi } from 'vitest';
|
||||
|
||||
import { TaskExecutionsService } from './task-executions';
|
||||
|
||||
const job = (status: BiasImpactJob['status'], terminal: boolean): BiasImpactJob => ({
|
||||
id: 'job-1',
|
||||
status,
|
||||
executionId: 'execution-1',
|
||||
stepId: 'step-1',
|
||||
createdAt: '2026-07-21T10:00:00',
|
||||
startedAt: null,
|
||||
completedAt: terminal ? '2026-07-21T10:00:10' : null,
|
||||
reportId: null,
|
||||
report: null,
|
||||
errorCode: null,
|
||||
errorMessage: null,
|
||||
terminal
|
||||
});
|
||||
|
||||
describe('TaskExecutionsService bias operations', () => {
|
||||
let service: TaskExecutionsService;
|
||||
|
||||
beforeEach(() => {
|
||||
TestBed.configureTestingModule({});
|
||||
service = TestBed.inject(TaskExecutionsService);
|
||||
});
|
||||
|
||||
it('maps side-effect policy conflicts using the backend error code', async () => {
|
||||
service.taskExecutionsCallService = {
|
||||
runBiasImpactExperiment: () => throwError(() => ({
|
||||
status: 409,
|
||||
error: {
|
||||
detail: 'External side effects are blocked',
|
||||
errors: [{ code: 'BIAS_SIDE_EFFECT_BLOCKED', message: 'External side effects are blocked' }]
|
||||
}
|
||||
}))
|
||||
} as unknown as typeof service.taskExecutionsCallService;
|
||||
|
||||
await expect(lastValueFrom(service.runBiasImpactExperiment('execution-1', 'step-1', {
|
||||
annotationIds: ['annotation-1'],
|
||||
repetitions: 3,
|
||||
includeRawOutputs: true,
|
||||
externalSideEffectPolicy: 'BLOCK',
|
||||
confirmExternalSideEffects: false
|
||||
}))).rejects.toEqual({
|
||||
reason: 'SIDE_EFFECT_BLOCKED',
|
||||
code: 'BIAS_SIDE_EFFECT_BLOCKED',
|
||||
message: 'External side effects are blocked'
|
||||
});
|
||||
expect(service.biasExperimentInProgress()).toBe(false);
|
||||
});
|
||||
|
||||
it('polls until a terminal job and stops after completion', async () => {
|
||||
vi.useFakeTimers();
|
||||
const getBiasImpactJob = vi.fn()
|
||||
.mockReturnValueOnce(of(job('QUEUED', false)))
|
||||
.mockReturnValueOnce(of(job('RUNNING', false)))
|
||||
.mockReturnValueOnce(of(job('COMPLETED', true)));
|
||||
service.taskExecutionsCallService = { getBiasImpactJob } as unknown as typeof service.taskExecutionsCallService;
|
||||
|
||||
const terminalJob = lastValueFrom(service.pollBiasImpactJob('job-1'));
|
||||
await vi.advanceTimersByTimeAsync(3_000);
|
||||
|
||||
await expect(terminalJob).resolves.toEqual(expect.objectContaining({ status: 'COMPLETED', terminal: true }));
|
||||
expect(getBiasImpactJob).toHaveBeenCalledTimes(3);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('retries a transient polling failure without abandoning the job', async () => {
|
||||
vi.useFakeTimers();
|
||||
const getBiasImpactJob = vi.fn()
|
||||
.mockReturnValueOnce(throwError(() => ({ status: 0 })))
|
||||
.mockReturnValueOnce(of(job('RUNNING', false)))
|
||||
.mockReturnValueOnce(of(job('COMPLETED', true)));
|
||||
service.taskExecutionsCallService = { getBiasImpactJob } as unknown as typeof service.taskExecutionsCallService;
|
||||
|
||||
const terminalJob = lastValueFrom(service.pollBiasImpactJob('job-1'));
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
|
||||
await expect(terminalJob).resolves.toEqual(expect.objectContaining({ status: 'COMPLETED' }));
|
||||
expect(getBiasImpactJob).toHaveBeenCalledTimes(3);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
});
|
||||
|
|
@ -1,8 +1,30 @@
|
|||
import { DestroyRef, inject, Injectable, signal } from '@angular/core';
|
||||
import { environment } from '@environment';
|
||||
import { LLMDescriptor } from '@models/flow';
|
||||
import {
|
||||
BiasImpactExperimentRequest,
|
||||
BiasImpactJob,
|
||||
BiasImpactReport,
|
||||
BiasRerunRequest,
|
||||
BiasSideEffectError
|
||||
} from '@models/bias-impact';
|
||||
import { ExecutionEventLogEntry, getExecutionStatusGroup, TaskExecution, TaskExecutionGroup } from '@models/task-execution';
|
||||
import { catchError, finalize, Observable, tap, throwError } from 'rxjs';
|
||||
import {
|
||||
catchError,
|
||||
defer,
|
||||
EMPTY,
|
||||
expand,
|
||||
filter,
|
||||
finalize,
|
||||
map,
|
||||
Observable,
|
||||
of,
|
||||
switchMap,
|
||||
tap,
|
||||
throwError,
|
||||
timeout,
|
||||
timer
|
||||
} from 'rxjs';
|
||||
import { TaskExecutionsCallServiceBase } from './task-executions-call.base';
|
||||
|
||||
@Injectable({
|
||||
|
|
@ -18,10 +40,14 @@ export class TaskExecutionsService {
|
|||
private _taskExecutions = signal<TaskExecution[]>([]);
|
||||
private _taskExecutionGroups = signal<TaskExecutionGroup[]>([]);
|
||||
private _pendingExecutionCreation = signal(false);
|
||||
private _biasExperimentInProgress = signal(false);
|
||||
private _biasRerunInProgress = signal(false);
|
||||
|
||||
taskExecutions = this._taskExecutions.asReadonly();
|
||||
taskExecutionGroups = this._taskExecutionGroups.asReadonly();
|
||||
pendingExecutionCreation = this._pendingExecutionCreation.asReadonly();
|
||||
biasExperimentInProgress = this._biasExperimentInProgress.asReadonly();
|
||||
biasRerunInProgress = this._biasRerunInProgress.asReadonly();
|
||||
|
||||
init() {
|
||||
if (this.initialized) return;
|
||||
|
|
@ -78,6 +104,86 @@ export class TaskExecutionsService {
|
|||
);
|
||||
}
|
||||
|
||||
runBiasImpactExperiment(
|
||||
executionId: string,
|
||||
stepId: string,
|
||||
request: BiasImpactExperimentRequest
|
||||
): Observable<BiasImpactJob> {
|
||||
this._biasExperimentInProgress.set(true);
|
||||
return this.taskExecutionsCallService.runBiasImpactExperiment(executionId, stepId, request).pipe(
|
||||
finalize(() => this._biasExperimentInProgress.set(false)),
|
||||
catchError((error) => throwError(() => this.toBiasOperationError(error)))
|
||||
);
|
||||
}
|
||||
|
||||
getBiasImpactJob(jobId: string): Observable<BiasImpactJob> {
|
||||
return this.taskExecutionsCallService.getBiasImpactJob(jobId).pipe(
|
||||
catchError((error) => throwError(() => this.toBiasOperationError(error)))
|
||||
);
|
||||
}
|
||||
|
||||
pollBiasImpactJob(jobId: string): Observable<BiasImpactJob> {
|
||||
type PollState = { job: BiasImpactJob | null; failures: number };
|
||||
const poll = (failures: number): Observable<PollState> => this.getBiasImpactJob(jobId).pipe(
|
||||
timeout({ first: 15_000 }),
|
||||
map((job) => ({ job, failures: 0 })),
|
||||
catchError((error) => {
|
||||
if (!this.isRetryablePollingError(error)) {
|
||||
return throwError(() => error);
|
||||
}
|
||||
console.warn('Bias impact job polling request failed; retrying', error);
|
||||
return of({ job: null, failures: failures + 1 });
|
||||
})
|
||||
);
|
||||
|
||||
return defer(() => poll(0)).pipe(
|
||||
expand((state) => {
|
||||
if (state.job?.terminal) return EMPTY;
|
||||
const delay = state.job
|
||||
? 1_500
|
||||
: Math.min(5_000, 1_500 * (2 ** Math.min(state.failures, 2)));
|
||||
return timer(delay).pipe(switchMap(() => poll(state.failures)));
|
||||
}),
|
||||
filter((state): state is { job: BiasImpactJob; failures: number } => state.job !== null),
|
||||
map((state) => state.job)
|
||||
);
|
||||
}
|
||||
|
||||
createBiasedRerun(executionId: string, request: BiasRerunRequest): Observable<TaskExecution> {
|
||||
this._biasRerunInProgress.set(true);
|
||||
this._pendingExecutionCreation.set(true);
|
||||
return this.taskExecutionsCallService.createBiasedRerun(executionId, request).pipe(
|
||||
tap(() => this.refresh()),
|
||||
finalize(() => {
|
||||
this._pendingExecutionCreation.set(false);
|
||||
this._biasRerunInProgress.set(false);
|
||||
}),
|
||||
catchError((error) => throwError(() => this.toBiasOperationError(error)))
|
||||
);
|
||||
}
|
||||
|
||||
compareBiasExecutions(
|
||||
baselineExecutionId: string,
|
||||
biasedExecutionId: string,
|
||||
includeRawOutputs: boolean
|
||||
): Observable<BiasImpactReport> {
|
||||
return this.taskExecutionsCallService
|
||||
.compareBiasExecutions(baselineExecutionId, biasedExecutionId, includeRawOutputs)
|
||||
.pipe(catchError((error) => throwError(() => this.toBiasOperationError(error))));
|
||||
}
|
||||
|
||||
listBiasImpactReports(executionId: string): Observable<BiasImpactReport[]> {
|
||||
return this.taskExecutionsCallService.listBiasImpactReports(executionId).pipe(
|
||||
catchError((error) => throwError(() => this.toBiasOperationError(error)))
|
||||
);
|
||||
}
|
||||
|
||||
getBiasImpactReport(reportId: string): Observable<BiasImpactReport> {
|
||||
return this.taskExecutionsCallService.getBiasImpactReport(reportId).pipe(
|
||||
catchError((error) => throwError(() => this.toBiasOperationError(error)))
|
||||
);
|
||||
}
|
||||
|
||||
deleteExecution(executionId: string) {
|
||||
return this.withRefreshAndErrorHandling(
|
||||
this.taskExecutionsCallService.deleteTaskExecution(executionId),
|
||||
|
|
@ -245,4 +351,45 @@ export class TaskExecutionsService {
|
|||
})
|
||||
);
|
||||
}
|
||||
|
||||
private toBiasOperationError(error: unknown): unknown {
|
||||
const response = error as { status?: unknown; error?: unknown };
|
||||
if (response?.status !== 409 || !response.error || typeof response.error !== 'object') {
|
||||
return error;
|
||||
}
|
||||
|
||||
const body = response.error as Record<string, unknown>;
|
||||
const errors = Array.isArray(body['errors']) ? body['errors'] : [];
|
||||
const first = errors[0];
|
||||
if (!first || typeof first !== 'object') return error;
|
||||
const record = first as Record<string, unknown>;
|
||||
const code = record['code'];
|
||||
const message = typeof record['message'] === 'string'
|
||||
? record['message']
|
||||
: typeof body['detail'] === 'string' ? body['detail'] : 'Bias experiment request was rejected';
|
||||
|
||||
if (code === 'BIAS_SIDE_EFFECT_BLOCKED') {
|
||||
const mapped: BiasSideEffectError = {
|
||||
reason: 'SIDE_EFFECT_BLOCKED',
|
||||
code,
|
||||
message
|
||||
};
|
||||
return mapped;
|
||||
}
|
||||
if (code === 'BIAS_SIDE_EFFECT_CONFIRMATION_REQUIRED') {
|
||||
const mapped: BiasSideEffectError = {
|
||||
reason: 'CONFIRMATION_REQUIRED',
|
||||
code,
|
||||
message
|
||||
};
|
||||
return mapped;
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
private isRetryablePollingError(error: unknown): boolean {
|
||||
const value = error as { status?: unknown; name?: unknown };
|
||||
const status = typeof value?.status === 'number' ? value.status : null;
|
||||
return value?.name === 'TimeoutError' || status === 0 || (status != null && status >= 500);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
:host { display: block; }
|
||||
.probe-editor { display: grid; gap: 12px; padding: 12px; border: 1px solid #cbd5e1; border-radius: 9px; background: #f8fafc; }
|
||||
.probe-header h4 { margin: 0; font-size: 13px; }
|
||||
.probe-header p, small { margin: 3px 0 0; color: #64748b; font-size: 11px; font-weight: 400; }
|
||||
label { display: grid; gap: 5px; font-size: 12px; font-weight: 650; }
|
||||
input[type='text'], select, textarea { width: 100%; box-sizing: border-box; border: 1px solid #cbd5e1; border-radius: 7px; padding: 8px; font: inherit; font-weight: 400; background: white; }
|
||||
textarea { resize: vertical; }
|
||||
fieldset { display: grid; gap: 5px; border: 1px solid #dbe3ee; border-radius: 7px; padding: 8px; }
|
||||
legend { padding: 0 4px; font-size: 12px; font-weight: 650; }
|
||||
.probe-check { display: flex; align-items: center; gap: 6px; font-weight: 400; }
|
||||
.probe-check input { margin: 0; }
|
||||
.probe-state, .probe-note { border-radius: 7px; padding: 8px; background: #e0f2fe; color: #075985; font-size: 12px; }
|
||||
.probe-error, em { color: #b91c1c; font-size: 11px; font-style: normal; }
|
||||
.mock-output-list { display: grid; gap: 8px; }
|
||||
code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }
|
||||
|
|
@ -0,0 +1,107 @@
|
|||
<section class="probe-editor">
|
||||
<div class="probe-header">
|
||||
<div>
|
||||
<h4>Behavioral probe</h4>
|
||||
<p>Optional controlled behavior used only by bias experiments.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@if (loadingCapabilities) {
|
||||
<div class="probe-state">Loading compatible activation modes…</div>
|
||||
} @else if (capabilityError) {
|
||||
<div class="probe-error">{{ capabilityError }}</div>
|
||||
} @else if (capabilities && !supported) {
|
||||
<div class="probe-state">This block can be annotated, but it does not support a bias experiment.</div>
|
||||
} @else {
|
||||
<label>
|
||||
<span>Activation mode</span>
|
||||
<select
|
||||
[ngModel]="currentProbe.activationMode ?? ''"
|
||||
[disabled]="readonly || !activationModes.length"
|
||||
(ngModelChange)="selectActivationMode($event)">
|
||||
<option value="">Select an activation mode</option>
|
||||
@for (mode of activationModes; track mode) {
|
||||
<option [value]="mode">{{ mode }}</option>
|
||||
}
|
||||
</select>
|
||||
</label>
|
||||
|
||||
@if (currentProbe.activationMode === 'PROMPT_DIRECTIVE') {
|
||||
<label>
|
||||
<span>Experimental instruction</span>
|
||||
<textarea [ngModel]="currentProbe.instruction ?? ''" [disabled]="readonly" rows="4" (ngModelChange)="setInstruction($event)"></textarea>
|
||||
</label>
|
||||
}
|
||||
|
||||
@if (currentProbe.activationMode === 'INPUT_TRANSFORMATION') {
|
||||
<label>
|
||||
<span>Transformation template</span>
|
||||
<textarea [ngModel]="currentProbe.instruction ?? ''" [disabled]="readonly" rows="4" (ngModelChange)="setInstruction($event)"></textarea>
|
||||
<small><code>${original}</code> is the only placeholder. Without it, the instruction is prepended to the original value. Multiple inputs are transformed item by item.</small>
|
||||
</label>
|
||||
<fieldset [disabled]="readonly">
|
||||
<legend>Target inputs <small>Leave all unchecked to target every textual input.</small></legend>
|
||||
@for (input of block?.inputs ?? []; track input.name) {
|
||||
<label class="probe-check">
|
||||
<input type="checkbox" [checked]="selectedTargetInputs.includes(input.name)" (change)="toggleTargetInput(input.name, $any($event.target).checked)" />
|
||||
<span>{{ input.name }} <small>{{ input.type }}{{ input.multiple ? ' · multiple' : '' }}</small></span>
|
||||
</label>
|
||||
}
|
||||
</fieldset>
|
||||
}
|
||||
|
||||
@if (currentProbe.activationMode === 'OUTPUT_TRANSFORMATION') {
|
||||
<label>
|
||||
<span>Transformation template</span>
|
||||
<textarea [ngModel]="currentProbe.instruction ?? ''" [disabled]="readonly" rows="4" (ngModelChange)="setInstruction($event)"></textarea>
|
||||
<small><code>${original}</code> is the only placeholder. Without it, the instruction is prepended to the original value.</small>
|
||||
</label>
|
||||
}
|
||||
|
||||
@if (currentProbe.activationMode === 'ROUTING_OVERRIDE') {
|
||||
<label>
|
||||
<span>Forced output branch</span>
|
||||
<select [ngModel]="currentProbe.instruction ?? ''" [disabled]="readonly" (ngModelChange)="setRoutingOutput($event)">
|
||||
<option value="">Select an output branch</option>
|
||||
@for (output of block?.outputs ?? []; track output.name) {
|
||||
<option [value]="output.name">{{ output.name }}</option>
|
||||
}
|
||||
</select>
|
||||
</label>
|
||||
}
|
||||
|
||||
@if (isMockResponse) {
|
||||
<div class="probe-note">Mock responses use typed output values. No new mock instruction text is saved.</div>
|
||||
@if (currentProbe.instruction) {
|
||||
<label>
|
||||
<span>Legacy mock instruction</span>
|
||||
<textarea [ngModel]="currentProbe.instruction" rows="2" readonly></textarea>
|
||||
<small>Retained only to display annotations saved before typed mock outputs were introduced.</small>
|
||||
</label>
|
||||
}
|
||||
<div class="mock-output-list">
|
||||
@for (output of block?.outputs ?? []; track output.name) {
|
||||
<label>
|
||||
<span>{{ output.name }} <small>{{ output.type }}{{ output.multiple ? ' · multiple' : '' }}</small></span>
|
||||
@if (output.type === 'BOOLEAN' && !output.multiple) {
|
||||
<input type="checkbox" [checked]="mockBooleanValue(output)" [disabled]="readonly" (change)="setBooleanMockOutput(output, $any($event.target).checked)" />
|
||||
} @else if (output.multiple || output.type === 'ANY') {
|
||||
<textarea [ngModel]="mockValueText(output)" [disabled]="readonly" rows="3" placeholder="Valid JSON value" (ngModelChange)="setJsonMockOutput(output, $event)"></textarea>
|
||||
} @else {
|
||||
<input type="text" [ngModel]="mockValueText(output)" [disabled]="readonly" (ngModelChange)="setTextMockOutput(output, $event)" />
|
||||
}
|
||||
@if (mockValueErrors[output.name]) { <em>{{ mockValueErrors[output.name] }}</em> }
|
||||
</label>
|
||||
}
|
||||
</div>
|
||||
}
|
||||
|
||||
@if (currentProbe.activationMode) {
|
||||
<label>
|
||||
<span>Expected impact</span>
|
||||
<textarea [ngModel]="currentProbe.expectedImpact ?? ''" [disabled]="readonly" rows="3" (ngModelChange)="setExpectedImpact($event)"></textarea>
|
||||
<small>This describes the expected observation; it does not change the execution.</small>
|
||||
</label>
|
||||
}
|
||||
}
|
||||
</section>
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { FlowBlock } from '@models/flow';
|
||||
import { BlocksService } from '@services/blocks/blocks';
|
||||
import { of } from 'rxjs';
|
||||
import { vi } from 'vitest';
|
||||
|
||||
import { BehavioralProbeEditorComponent } from './behavioral-probe-editor';
|
||||
|
||||
const block: FlowBlock = {
|
||||
id: 'conditional-1',
|
||||
name: 'Conditional',
|
||||
inputs: [{ name: 'input', type: 'TEXT', multiple: true }],
|
||||
outputs: [
|
||||
{ name: 'response', type: 'TEXT', multiple: false },
|
||||
{ name: 'accepted', type: 'BOOLEAN', multiple: false },
|
||||
{ name: 'items', type: 'TEXT', multiple: true }
|
||||
],
|
||||
specificConfiguration: { useLlm: true },
|
||||
typeName: 'ConditionalBlock',
|
||||
nodeFamily: 'block'
|
||||
};
|
||||
|
||||
describe('BehavioralProbeEditorComponent', () => {
|
||||
let fixture: ComponentFixture<BehavioralProbeEditorComponent>;
|
||||
let component: BehavioralProbeEditorComponent;
|
||||
const retrieveBiasCapabilities = vi.fn();
|
||||
const retrieveBiasCapabilitiesForInstance = vi.fn();
|
||||
|
||||
beforeEach(async () => {
|
||||
retrieveBiasCapabilities.mockReset();
|
||||
retrieveBiasCapabilitiesForInstance.mockReset();
|
||||
retrieveBiasCapabilities.mockReturnValue(of({
|
||||
blockType: 'ConditionalBlock', supported: true, isolatedExperimentSupported: true,
|
||||
fullFlowExperimentSupported: true, externalSideEffects: false,
|
||||
configurationDependent: true, activationModes: ['INPUT_TRANSFORMATION']
|
||||
}));
|
||||
retrieveBiasCapabilitiesForInstance.mockReturnValue(of({
|
||||
blockType: 'ConditionalBlock', supported: true, isolatedExperimentSupported: true,
|
||||
fullFlowExperimentSupported: true, externalSideEffects: false,
|
||||
configurationDependent: true, activationModes: ['PROMPT_DIRECTIVE', 'MOCK_RESPONSE', 'ROUTING_OVERRIDE']
|
||||
}));
|
||||
await TestBed.configureTestingModule({
|
||||
imports: [BehavioralProbeEditorComponent],
|
||||
providers: [{
|
||||
provide: BlocksService,
|
||||
useValue: { retrieveBiasCapabilities, retrieveBiasCapabilitiesForInstance }
|
||||
}]
|
||||
}).compileComponents();
|
||||
fixture = TestBed.createComponent(BehavioralProbeEditorComponent);
|
||||
component = fixture.componentInstance;
|
||||
fixture.componentRef.setInput('block', block);
|
||||
fixture.detectChanges();
|
||||
});
|
||||
|
||||
it('uses instance capabilities when the type-level response is configuration dependent', () => {
|
||||
expect(retrieveBiasCapabilities).toHaveBeenCalledWith('ConditionalBlock');
|
||||
expect(retrieveBiasCapabilitiesForInstance).toHaveBeenCalledWith('ConditionalBlock', block);
|
||||
expect(component.activationModes).toEqual(['PROMPT_DIRECTIVE', 'MOCK_RESPONSE', 'ROUTING_OVERRIDE']);
|
||||
});
|
||||
|
||||
it('emits typed complete mock outputs and never writes a new mock instruction', () => {
|
||||
const changed = vi.fn();
|
||||
component.probeChange.subscribe(changed);
|
||||
|
||||
component.selectActivationMode('MOCK_RESPONSE');
|
||||
|
||||
expect(changed).toHaveBeenLastCalledWith(expect.objectContaining({
|
||||
activationMode: 'MOCK_RESPONSE',
|
||||
instruction: undefined,
|
||||
mockOutputs: { response: '', accepted: false, items: [] }
|
||||
}));
|
||||
});
|
||||
|
||||
it('updates templates, target inputs and routing branch for the selected mode', () => {
|
||||
const changed = vi.fn();
|
||||
component.probeChange.subscribe(changed);
|
||||
fixture.componentRef.setInput('probe', { activationMode: 'INPUT_TRANSFORMATION' });
|
||||
fixture.detectChanges();
|
||||
|
||||
component.setInstruction('Frame ${original} as confirming evidence.');
|
||||
component.toggleTargetInput('input', true);
|
||||
expect(changed).toHaveBeenLastCalledWith(expect.objectContaining({ targetInputs: ['input'] }));
|
||||
|
||||
fixture.componentRef.setInput('probe', { activationMode: 'ROUTING_OVERRIDE' });
|
||||
fixture.detectChanges();
|
||||
component.setRoutingOutput('accepted');
|
||||
expect(changed).toHaveBeenLastCalledWith(expect.objectContaining({ instruction: 'accepted' }));
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,216 @@
|
|||
import { CommonModule } from '@angular/common';
|
||||
import { ChangeDetectionStrategy, Component, Input, OnChanges, Output, EventEmitter, inject } from '@angular/core';
|
||||
import { FormsModule } from '@angular/forms';
|
||||
import { BiasCapabilities } from '@models/bias-impact';
|
||||
import { BehavioralProbe, BiasActivationMode, FlowBlock, FlowPort } from '@models/flow';
|
||||
import { BlocksService } from '@services/blocks/blocks';
|
||||
import { take } from 'rxjs';
|
||||
|
||||
@Component({
|
||||
selector: 'app-behavioral-probe-editor',
|
||||
standalone: true,
|
||||
imports: [CommonModule, FormsModule],
|
||||
templateUrl: './behavioral-probe-editor.html',
|
||||
styleUrl: './behavioral-probe-editor.css',
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
})
|
||||
export class BehavioralProbeEditorComponent implements OnChanges {
|
||||
private readonly blocks = inject(BlocksService);
|
||||
private capabilityRequestVersion = 0;
|
||||
private capabilityKey: string | null = null;
|
||||
|
||||
@Input() block: FlowBlock | null = null;
|
||||
@Input() probe: BehavioralProbe | undefined;
|
||||
@Input() readonly = false;
|
||||
@Output() probeChange = new EventEmitter<BehavioralProbe | undefined>();
|
||||
|
||||
capabilities: BiasCapabilities | null = null;
|
||||
loadingCapabilities = false;
|
||||
capabilityError: string | null = null;
|
||||
mockValueErrors: Record<string, string> = {};
|
||||
|
||||
get blockType(): string | null {
|
||||
const value = this.block?.typeName;
|
||||
return typeof value === 'string' && value.trim().length > 0 ? value : null;
|
||||
}
|
||||
|
||||
get activationModes(): BiasActivationMode[] {
|
||||
return this.capabilities?.activationModes ?? [];
|
||||
}
|
||||
|
||||
get supported(): boolean {
|
||||
return this.capabilities?.supported !== false;
|
||||
}
|
||||
|
||||
get currentProbe(): BehavioralProbe {
|
||||
return this.probe ?? {};
|
||||
}
|
||||
|
||||
get isMockResponse(): boolean {
|
||||
return this.currentProbe.activationMode === 'MOCK_RESPONSE';
|
||||
}
|
||||
|
||||
get selectedTargetInputs(): string[] {
|
||||
return this.currentProbe.targetInputs ?? [];
|
||||
}
|
||||
|
||||
ngOnChanges() {
|
||||
void this.loadCapabilities();
|
||||
}
|
||||
|
||||
selectActivationMode(mode: string) {
|
||||
const activationMode = mode as BiasActivationMode;
|
||||
if (!activationMode) {
|
||||
this.emit({ ...this.currentProbe, activationMode: undefined });
|
||||
return;
|
||||
}
|
||||
|
||||
if (activationMode === 'MOCK_RESPONSE') {
|
||||
this.mockValueErrors = {};
|
||||
this.emit({
|
||||
...this.currentProbe,
|
||||
activationMode,
|
||||
instruction: undefined,
|
||||
targetInputs: [],
|
||||
mockOutputs: this.normalizedMockOutputs(this.currentProbe.mockOutputs)
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
this.emit({
|
||||
...this.currentProbe,
|
||||
activationMode,
|
||||
mockOutputs: undefined
|
||||
});
|
||||
}
|
||||
|
||||
setInstruction(instruction: string) {
|
||||
this.emit({ ...this.currentProbe, instruction });
|
||||
}
|
||||
|
||||
setExpectedImpact(expectedImpact: string) {
|
||||
this.emit({ ...this.currentProbe, expectedImpact });
|
||||
}
|
||||
|
||||
toggleTargetInput(name: string, checked: boolean) {
|
||||
const selected = new Set(this.selectedTargetInputs);
|
||||
if (checked) selected.add(name);
|
||||
else selected.delete(name);
|
||||
this.emit({ ...this.currentProbe, targetInputs: [...selected] });
|
||||
}
|
||||
|
||||
setRoutingOutput(name: string) {
|
||||
this.emit({ ...this.currentProbe, instruction: name });
|
||||
}
|
||||
|
||||
setTextMockOutput(port: FlowPort, value: string) {
|
||||
this.setMockOutput(port.name, value);
|
||||
}
|
||||
|
||||
setBooleanMockOutput(port: FlowPort, checked: boolean) {
|
||||
this.setMockOutput(port.name, checked);
|
||||
}
|
||||
|
||||
setJsonMockOutput(port: FlowPort, value: string) {
|
||||
try {
|
||||
const parsed = JSON.parse(value);
|
||||
if (port.multiple && !Array.isArray(parsed)) {
|
||||
throw new Error('Use a JSON array for a multiple output.');
|
||||
}
|
||||
this.mockValueErrors = { ...this.mockValueErrors, [port.name]: '' };
|
||||
this.setMockOutput(port.name, parsed);
|
||||
} catch (error) {
|
||||
this.mockValueErrors = {
|
||||
...this.mockValueErrors,
|
||||
[port.name]: error instanceof Error ? error.message : 'Invalid JSON value.'
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
mockValueText(port: FlowPort): string {
|
||||
const value = this.currentProbe.mockOutputs?.[port.name];
|
||||
if (typeof value === 'string') return value;
|
||||
if (value === undefined) return '';
|
||||
return JSON.stringify(value, null, 2);
|
||||
}
|
||||
|
||||
mockBooleanValue(port: FlowPort): boolean {
|
||||
return this.currentProbe.mockOutputs?.[port.name] === true;
|
||||
}
|
||||
|
||||
private async loadCapabilities() {
|
||||
const blockType = this.blockType;
|
||||
if (!blockType) {
|
||||
this.capabilities = null;
|
||||
this.capabilityKey = null;
|
||||
return;
|
||||
}
|
||||
|
||||
const nextCapabilityKey = `${blockType}:${this.configurationFingerprint()}`;
|
||||
if (nextCapabilityKey === this.capabilityKey && (this.capabilities || this.loadingCapabilities)) return;
|
||||
this.capabilityKey = nextCapabilityKey;
|
||||
|
||||
const requestVersion = ++this.capabilityRequestVersion;
|
||||
this.loadingCapabilities = true;
|
||||
this.capabilityError = null;
|
||||
this.blocks.retrieveBiasCapabilities(blockType).pipe(take(1)).subscribe({
|
||||
next: (capabilities) => {
|
||||
if (requestVersion !== this.capabilityRequestVersion) return;
|
||||
if (!capabilities.configurationDependent || !this.block) {
|
||||
this.capabilities = capabilities;
|
||||
this.loadingCapabilities = false;
|
||||
return;
|
||||
}
|
||||
this.blocks.retrieveBiasCapabilitiesForInstance(blockType, this.block).pipe(take(1)).subscribe({
|
||||
next: (instanceCapabilities) => {
|
||||
if (requestVersion !== this.capabilityRequestVersion) return;
|
||||
this.capabilities = instanceCapabilities;
|
||||
this.loadingCapabilities = false;
|
||||
},
|
||||
error: () => {
|
||||
if (requestVersion !== this.capabilityRequestVersion) return;
|
||||
this.capabilityError = 'Unable to load the configured block capabilities.';
|
||||
this.loadingCapabilities = false;
|
||||
}
|
||||
});
|
||||
},
|
||||
error: () => {
|
||||
if (requestVersion !== this.capabilityRequestVersion) return;
|
||||
this.capabilityError = 'Unable to load bias capabilities.';
|
||||
this.loadingCapabilities = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private setMockOutput(name: string, value: unknown) {
|
||||
this.emit({
|
||||
...this.currentProbe,
|
||||
mockOutputs: { ...this.normalizedMockOutputs(this.currentProbe.mockOutputs), [name]: value }
|
||||
});
|
||||
}
|
||||
|
||||
private normalizedMockOutputs(existing: Record<string, unknown> | undefined): Record<string, unknown> {
|
||||
return (this.block?.outputs ?? []).reduce<Record<string, unknown>>((result, port) => {
|
||||
result[port.name] = existing?.[port.name] ?? this.defaultMockValue(port);
|
||||
return result;
|
||||
}, {});
|
||||
}
|
||||
|
||||
private defaultMockValue(port: FlowPort): unknown {
|
||||
if (port.multiple) return [];
|
||||
if (port.type === 'BOOLEAN') return false;
|
||||
return '';
|
||||
}
|
||||
|
||||
private configurationFingerprint(): string {
|
||||
try {
|
||||
return JSON.stringify(this.block?.specificConfiguration ?? {});
|
||||
} catch {
|
||||
return String(this.block?.specificConfiguration ?? '');
|
||||
}
|
||||
}
|
||||
|
||||
private emit(probe: BehavioralProbe) {
|
||||
this.probeChange.emit(probe);
|
||||
}
|
||||
}
|
||||
|
|
@ -13,6 +13,8 @@ button:disabled { cursor: not-allowed; opacity: .5; }
|
|||
.bias-badge { border-radius: 999px; padding: 2px 7px; background: #e2e8f0; font-size: 10px; font-weight: 650; }
|
||||
.bias-badge.severity { background: #fee2e2; color: #991b1b; }
|
||||
.bias-badge.status { background: #dbeafe; color: #1e40af; }
|
||||
.bias-badge.source { background: #f3e8ff; color: #6b21a8; }
|
||||
.bias-badge.probe { background: #dcfce7; color: #166534; }
|
||||
.bias-issue { margin: 7px 0; font-size: 12px; white-space: pre-wrap; }
|
||||
.bias-actions { justify-content: flex-end; }
|
||||
.bias-actions .danger { color: #b91c1c; }
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@
|
|||
<span class="bias-badge category">{{ optionLabel('category', annotation.category) }}</span>
|
||||
<span class="bias-badge severity">{{ optionLabel('severity', annotation.severity) }}</span>
|
||||
<span class="bias-badge status">{{ optionLabel('status', annotation.status) }}</span>
|
||||
<span class="bias-badge source">{{ optionLabel('source', annotation.source) }}</span>
|
||||
@if (probeExecutable(annotation)) { <span class="bias-badge probe">Executable probe</span> }
|
||||
</div>
|
||||
<div class="bias-issue">{{ annotation.issue }}</div>
|
||||
@if (serverError(index); as error) { <div class="bias-error">{{ error }}</div> }
|
||||
|
|
@ -62,6 +64,11 @@
|
|||
@if (clientErrors[field.key]; as error) { <em>{{ error }}</em> }
|
||||
</label>
|
||||
}
|
||||
<app-behavioral-probe-editor
|
||||
[block]="block"
|
||||
[probe]="draft.behavioralProbe"
|
||||
[readonly]="readonly"
|
||||
(probeChange)="updateProbe($event)" />
|
||||
</div>
|
||||
<div class="bias-modal-actions">
|
||||
<button type="button" (click)="close($event)">Cancel</button>
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { BiasAnnotationsDescriptor } from '@models/flow';
|
|||
import { BlocksService } from '@services/blocks/blocks';
|
||||
import { EditorStateHolder } from '@stores/flow-editor';
|
||||
import { vi } from 'vitest';
|
||||
import { of } from 'rxjs';
|
||||
import { BiasAnnotationsComponent } from './bias-annotations';
|
||||
|
||||
const descriptor: BiasAnnotationsDescriptor = {
|
||||
|
|
@ -33,7 +34,22 @@ describe('BiasAnnotationsComponent', () => {
|
|||
await TestBed.configureTestingModule({
|
||||
imports: [BiasAnnotationsComponent],
|
||||
providers: [
|
||||
{ provide: BlocksService, useValue: { biasAnnotationsDescriptor: signal(descriptor) } },
|
||||
{
|
||||
provide: BlocksService,
|
||||
useValue: {
|
||||
biasAnnotationsDescriptor: signal(descriptor),
|
||||
retrieveBiasCapabilities: () => of({
|
||||
blockType: 'LLMBlock', supported: true, isolatedExperimentSupported: true,
|
||||
fullFlowExperimentSupported: true, externalSideEffects: false,
|
||||
configurationDependent: false, activationModes: ['PROMPT_DIRECTIVE']
|
||||
}),
|
||||
retrieveBiasCapabilitiesForInstance: () => of({
|
||||
blockType: 'LLMBlock', supported: true, isolatedExperimentSupported: true,
|
||||
fullFlowExperimentSupported: true, externalSideEffects: false,
|
||||
configurationDependent: false, activationModes: ['PROMPT_DIRECTIVE']
|
||||
})
|
||||
}
|
||||
},
|
||||
{ provide: EditorStateHolder, useValue: { flowValidationErrors: validationErrors } }
|
||||
]
|
||||
}).compileComponents();
|
||||
|
|
@ -93,4 +109,14 @@ describe('BiasAnnotationsComponent', () => {
|
|||
expect(component.serverError(0, 'category')).toBe('Category required');
|
||||
expect(component.serverError(1, 'category')).toBeNull();
|
||||
});
|
||||
|
||||
it('maps typed mock-output probe errors to nested fields', () => {
|
||||
validationErrors.set([{
|
||||
code: 'BIAS_PROBE_MOCK_OUTPUT_TYPE_MISMATCH',
|
||||
id: 'block-1',
|
||||
field: 'biasAnnotations[0].behavioralProbe.mockOutputs.response',
|
||||
message: 'Response must be text'
|
||||
}]);
|
||||
expect(component.serverError(0, 'behavioralProbe.mockOutputs.response')).toBe('Response must be text');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,9 +1,19 @@
|
|||
import { CommonModule } from '@angular/common';
|
||||
import { ChangeDetectionStrategy, Component, EventEmitter, Input, Output, inject } from '@angular/core';
|
||||
import { FormsModule } from '@angular/forms';
|
||||
import { BiasAnnotation, BiasAnnotationOption, BiasAnnotationsDescriptor, FlowValidationError } from '@models/flow';
|
||||
import {
|
||||
BiasAnnotation,
|
||||
BiasAnnotationOption,
|
||||
BiasAnnotationsDescriptor,
|
||||
BehavioralProbe,
|
||||
FlowBlock,
|
||||
FlowValidationError,
|
||||
isProbeExecutable
|
||||
} from '@models/flow';
|
||||
import { BIAS_PROBE_ERROR_CODES } from '@models/bias-impact';
|
||||
import { BlocksService } from '@services/blocks/blocks';
|
||||
import { EditorStateHolder } from '@stores/flow-editor';
|
||||
import { BehavioralProbeEditorComponent } from '@shared/behavioral-probe-editor/behavioral-probe-editor';
|
||||
|
||||
type BiasField = {
|
||||
key: string;
|
||||
|
|
@ -18,13 +28,14 @@ type BiasField = {
|
|||
|
||||
const BIAS_ERROR_CODES = new Set([
|
||||
'TOO_MANY_BIAS_ANNOTATIONS', 'NULL_BIAS_ANNOTATION', 'DUPLICATE_BIAS_ANNOTATION_ID',
|
||||
'BIAS_CATEGORY_REQUIRED', 'BIAS_SEVERITY_REQUIRED', 'BIAS_ISSUE_REQUIRED', 'BIAS_FIELD_TOO_LONG'
|
||||
'BIAS_CATEGORY_REQUIRED', 'BIAS_SEVERITY_REQUIRED', 'BIAS_ISSUE_REQUIRED', 'BIAS_FIELD_TOO_LONG',
|
||||
...BIAS_PROBE_ERROR_CODES
|
||||
]);
|
||||
|
||||
@Component({
|
||||
selector: 'app-bias-annotations',
|
||||
standalone: true,
|
||||
imports: [CommonModule, FormsModule],
|
||||
imports: [CommonModule, FormsModule, BehavioralProbeEditorComponent],
|
||||
templateUrl: './bias-annotations.html',
|
||||
styleUrl: './bias-annotations.css',
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
|
|
@ -34,6 +45,7 @@ export class BiasAnnotationsComponent {
|
|||
private readonly editorState = inject(EditorStateHolder);
|
||||
|
||||
@Input({ required: true }) blockId = '';
|
||||
@Input() block: FlowBlock | null = null;
|
||||
@Input() annotations: BiasAnnotation[] = [];
|
||||
@Input() readonly = false;
|
||||
@Output() annotationsChange = new EventEmitter<BiasAnnotation[]>();
|
||||
|
|
@ -62,6 +74,7 @@ export class BiasAnnotationsComponent {
|
|||
|
||||
return Object.entries(properties)
|
||||
.filter(([key]) => !generated.has(key))
|
||||
.filter(([key]) => key !== 'behavioralProbe')
|
||||
.map(([key, raw]) => {
|
||||
const field = this.record(raw);
|
||||
const maxLength = Number(field['maxLength']);
|
||||
|
|
@ -152,6 +165,14 @@ export class BiasAnnotationsComponent {
|
|||
return field.options.find((option) => option.value === value)?.description ?? field.description ?? null;
|
||||
}
|
||||
|
||||
probeExecutable(annotation: BiasAnnotation): boolean {
|
||||
return isProbeExecutable(annotation.behavioralProbe);
|
||||
}
|
||||
|
||||
updateProbe(probe: BehavioralProbe | undefined) {
|
||||
this.draft = { ...this.draft, behavioralProbe: probe };
|
||||
}
|
||||
|
||||
valueLength(field: string): number {
|
||||
return String(this.draft[field] ?? '').length;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
.bias-experiment-dialog__backdrop { background: rgb(0 0 0 / .5); inset: 0; position: fixed; z-index: 10020; }
|
||||
.bias-experiment-dialog { background: #fff; border: 1px solid #cbd5e1; border-radius: 1rem; box-shadow: 0 25px 50px rgb(15 23 42 / .25); display: flex; flex-direction: column; left: 50%; max-height: min(88vh, 800px); max-width: 680px; position: fixed; top: 50%; transform: translate(-50%, -50%); width: min(94vw, 680px); z-index: 10021; }
|
||||
header, footer { align-items: center; display: flex; gap: 1rem; justify-content: space-between; padding: 1rem 1.25rem; } header { border-bottom: 1px solid #e2e8f0; } header h3, header p { margin: 0; } header p { color: #64748b; font-size: .85rem; }
|
||||
.bias-experiment-dialog__content { display: grid; gap: .9rem; overflow: auto; padding: 1.25rem; } .bias-experiment-dialog__content > p { margin: 0; }
|
||||
fieldset { display: grid; gap: .5rem; } fieldset label, .bias-experiment-dialog__content > label { color: #334155; font-size: .87rem; }
|
||||
.bias-experiment-dialog__progress { background: #eff6ff; border-left: 3px solid #2563eb; color: #1e3a8a; padding: .6rem .7rem; }
|
||||
.bias-experiment-dialog__error { background: #fff1f2; border-left: 3px solid #e11d48; color: #9f1239; margin: 0; padding: .6rem .7rem; } footer { border-top: 1px solid #e2e8f0; justify-content: flex-end; }
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
@if (state(); as currentState) {
|
||||
<div class="bias-experiment-dialog__backdrop" (click)="close()"></div>
|
||||
<section class="bias-experiment-dialog" role="dialog" aria-modal="true" aria-label="Measure bias impact">
|
||||
<header><div><h3>Measure bias impact</h3><p>{{ currentState.nodeName }}</p></div><button type="button" mat-stroked-button (click)="close()">Close</button></header>
|
||||
@if (report(); as completedReport) {
|
||||
<div class="bias-experiment-dialog__content"><app-bias-impact-report-viewer [report]="completedReport" /></div>
|
||||
} @else {
|
||||
<div class="bias-experiment-dialog__content">
|
||||
<p>Run the selected probes against this completed execution step.</p>
|
||||
<fieldset [disabled]="submitting() || !!currentJob()"><legend>Executable annotations</legend>
|
||||
@for (annotation of currentState.annotations; track annotation.id) {
|
||||
<label><input type="checkbox" [checked]="selectedAnnotationIds().includes($any(annotation.id))" (change)="toggleAnnotation($any(annotation.id), $any($event.target).checked)"> {{ annotation.category || 'Uncategorized' }} — {{ annotation.issue || annotation.rationale || annotation.id }}</label>
|
||||
}
|
||||
</fieldset>
|
||||
<label>Repetitions <input type="number" min="1" max="10" [ngModel]="repetitions()" (ngModelChange)="updateRepetitions($event)" [disabled]="submitting() || !!currentJob()"></label>
|
||||
<label><input type="checkbox" [ngModel]="includeRawOutputs()" (ngModelChange)="includeRawOutputs.set($event)" [disabled]="submitting() || !!currentJob()"> Include raw outputs in the report</label>
|
||||
<app-side-effect-policy-selector [policy]="policy()" [externalSideEffects]="currentState.capabilities.externalSideEffects" [disabled]="submitting() || !!currentJob()" (policyChange)="policy.set($event)" />
|
||||
@if (currentJob(); as job) { <p class="bias-experiment-dialog__progress">{{ job.status === 'QUEUED' ? 'Experiment queued…' : 'Experiment running…' }}</p> }
|
||||
@if (inlineError(); as error) { <p class="bias-experiment-dialog__error">{{ error }}</p> }
|
||||
</div>
|
||||
<footer><button type="button" mat-stroked-button (click)="close()">Cancel</button><button type="button" mat-flat-button [disabled]="submitting() || !!currentJob()" (click)="submit()">{{ submitting() || currentJob() ? 'Measuring…' : 'Measure impact' }}</button></footer>
|
||||
}
|
||||
</section>
|
||||
}
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
import { ChangeDetectionStrategy, Component, DestroyRef, effect, inject, signal } from '@angular/core';
|
||||
import { FormsModule } from '@angular/forms';
|
||||
import { MatButtonModule } from '@angular/material/button';
|
||||
import { BiasImpactReportViewerComponent } from '@shared/bias-impact-report-viewer/bias-impact-report-viewer';
|
||||
import { SideEffectPolicySelectorComponent } from '@shared/side-effect-policy-selector/side-effect-policy-selector';
|
||||
import { ConfirmDialogService } from '@services/dialogs/confirm-dialog';
|
||||
import { BiasImpactExperimentDialogService } from '@services/dialogs/bias-impact-experiment-dialog';
|
||||
import { NotificationService } from '@services/notifications/notification';
|
||||
import { TaskExecutionsService } from '@services/task-executions/task-executions';
|
||||
import { BiasImpactJob, BiasImpactReport, BiasSideEffectError, ExternalSideEffectPolicy } from '@models/bias-impact';
|
||||
import { Subscription } from 'rxjs';
|
||||
|
||||
@Component({
|
||||
selector: 'app-bias-impact-experiment-dialog-host',
|
||||
standalone: true,
|
||||
imports: [FormsModule, MatButtonModule, BiasImpactReportViewerComponent, SideEffectPolicySelectorComponent],
|
||||
templateUrl: './bias-impact-experiment-dialog.html',
|
||||
styleUrl: './bias-impact-experiment-dialog.css',
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
})
|
||||
export class BiasImpactExperimentDialogHostComponent {
|
||||
private readonly dialog = inject(BiasImpactExperimentDialogService);
|
||||
private readonly executions = inject(TaskExecutionsService);
|
||||
private readonly confirmation = inject(ConfirmDialogService);
|
||||
private readonly notifications = inject(NotificationService);
|
||||
private readonly destroyRef = inject(DestroyRef);
|
||||
private pollSubscription: Subscription | null = null;
|
||||
|
||||
readonly state = this.dialog.state;
|
||||
readonly selectedAnnotationIds = signal<string[]>([]);
|
||||
readonly repetitions = signal(3);
|
||||
readonly includeRawOutputs = signal(true);
|
||||
readonly policy = signal<ExternalSideEffectPolicy>('BLOCK');
|
||||
readonly submitting = signal(false);
|
||||
readonly currentJob = signal<BiasImpactJob | null>(null);
|
||||
readonly inlineError = signal<string | null>(null);
|
||||
readonly report = signal<BiasImpactReport | null>(null);
|
||||
|
||||
constructor() {
|
||||
effect(() => {
|
||||
const state = this.state();
|
||||
this.cancelPolling();
|
||||
this.selectedAnnotationIds.set(state?.annotations.map((annotation) => String(annotation.id ?? '')).filter(Boolean) ?? []);
|
||||
this.repetitions.set(3);
|
||||
this.includeRawOutputs.set(true);
|
||||
this.policy.set('BLOCK');
|
||||
this.submitting.set(false);
|
||||
this.currentJob.set(null);
|
||||
this.inlineError.set(null);
|
||||
this.report.set(null);
|
||||
});
|
||||
this.destroyRef.onDestroy(() => this.cancelPolling());
|
||||
}
|
||||
|
||||
toggleAnnotation(id: string, checked: boolean) {
|
||||
this.selectedAnnotationIds.update((current) => checked ? [...new Set([...current, id])] : current.filter((value) => value !== id));
|
||||
}
|
||||
|
||||
updateRepetitions(value: number) {
|
||||
this.repetitions.set(Math.min(10, Math.max(1, Number.isFinite(value) ? Math.round(value) : 3)));
|
||||
}
|
||||
|
||||
async submit() {
|
||||
const state = this.state();
|
||||
if (!state || this.submitting() || this.currentJob()) return;
|
||||
if (!this.selectedAnnotationIds().length) {
|
||||
this.inlineError.set('Select at least one executable bias annotation.');
|
||||
return;
|
||||
}
|
||||
|
||||
let confirmExternalSideEffects = false;
|
||||
if (this.policy() === 'REQUIRE_CONFIRMATION') {
|
||||
confirmExternalSideEffects = await this.confirmation.open('This experiment may invoke real external HTTP or MCP calls. Do you want to continue?');
|
||||
if (!confirmExternalSideEffects) return;
|
||||
}
|
||||
|
||||
this.submitting.set(true);
|
||||
this.inlineError.set(null);
|
||||
this.executions.runBiasImpactExperiment(state.executionId, state.stepId, {
|
||||
annotationIds: this.selectedAnnotationIds(),
|
||||
repetitions: this.repetitions(),
|
||||
includeRawOutputs: this.includeRawOutputs(),
|
||||
externalSideEffectPolicy: this.policy(),
|
||||
confirmExternalSideEffects
|
||||
}).subscribe({
|
||||
next: (job) => this.startPolling(job),
|
||||
error: (error) => this.handleInitialError(error)
|
||||
});
|
||||
}
|
||||
|
||||
close() {
|
||||
this.cancelPolling();
|
||||
this.dialog.close();
|
||||
}
|
||||
|
||||
private startPolling(job: BiasImpactJob) {
|
||||
this.currentJob.set(job);
|
||||
this.pollSubscription = this.executions.pollBiasImpactJob(job.id).subscribe({
|
||||
next: (nextJob) => {
|
||||
this.currentJob.set(nextJob);
|
||||
if (!nextJob.terminal) return;
|
||||
this.submitting.set(false);
|
||||
this.cancelPolling();
|
||||
if (nextJob.status === 'COMPLETED' && nextJob.report) this.report.set(nextJob.report);
|
||||
else this.inlineError.set(nextJob.errorMessage || 'The bias impact experiment failed.');
|
||||
},
|
||||
error: (error) => {
|
||||
this.submitting.set(false);
|
||||
this.currentJob.set(null);
|
||||
this.inlineError.set(error instanceof Error ? error.message : 'Unable to retrieve the experiment status.');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private handleInitialError(error: unknown) {
|
||||
this.submitting.set(false);
|
||||
const sideEffectError = error as Partial<BiasSideEffectError>;
|
||||
if (sideEffectError.reason === 'SIDE_EFFECT_BLOCKED' || sideEffectError.reason === 'CONFIRMATION_REQUIRED') {
|
||||
this.inlineError.set(sideEffectError.message ?? 'External side effects require a different policy.');
|
||||
return;
|
||||
}
|
||||
const status = (error as { status?: number })?.status;
|
||||
if (status === 404 || status === 400) {
|
||||
this.notifications.show(error instanceof Error ? error.message : 'The execution is no longer eligible for this experiment.', 'error');
|
||||
this.close();
|
||||
return;
|
||||
}
|
||||
this.inlineError.set(error instanceof Error ? error.message : 'Unable to start the bias impact experiment.');
|
||||
}
|
||||
|
||||
private cancelPolling() {
|
||||
this.pollSubscription?.unsubscribe();
|
||||
this.pollSubscription = null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
.bias-impact-report { color: #1e293b; display: grid; gap: 1rem; }
|
||||
.bias-impact-report__header, .bias-impact-report__section-heading, .bias-impact-report__node-heading, .bias-impact-report__routing { align-items: center; display: flex; gap: .65rem; justify-content: space-between; }
|
||||
.bias-impact-report__header h3, .bias-impact-report__section h4 { margin: 0; }
|
||||
.bias-impact-report__header time { color: #64748b; font-size: .84rem; }
|
||||
.bias-impact-report__kind { background: #e0e7ff; border-radius: 999px; color: #3730a3; font-size: .72rem; font-weight: 700; padding: .2rem .5rem; }
|
||||
.bias-impact-report__metadata { display: grid; gap: .55rem; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); margin: 0; }
|
||||
.bias-impact-report__metadata div { background: #f8fafc; border-radius: .4rem; padding: .55rem; }
|
||||
dt { color: #64748b; font-size: .73rem; } dd { font-size: .82rem; margin: .15rem 0 0; overflow-wrap: anywhere; }
|
||||
.bias-impact-report__summary { background: #eff6ff; border-left: 3px solid #2563eb; margin: 0; padding: .6rem .75rem; }
|
||||
.bias-impact-report__section { border-top: 1px solid #e2e8f0; padding-top: .9rem; }
|
||||
.bias-impact-report__metrics { display: flex; flex-wrap: wrap; gap: .5rem 1.2rem; font-size: .86rem; margin: .6rem 0; }
|
||||
.bias-impact-report__empty { color: #64748b; font-size: .87rem; }
|
||||
.bias-impact-report__downstream { background: #f8fafc; border: 1px solid #e2e8f0; border-radius: .45rem; margin-top: .7rem; padding: .7rem; }
|
||||
.bias-impact-report__node-heading { justify-content: flex-start; } .bias-impact-report__node-heading span { color: #64748b; font-size: .8rem; } .bias-impact-report__node-heading span.changed { color: #b45309; font-weight: 700; }
|
||||
.bias-impact-report__downstream p { font-size: .84rem; margin: .5rem 0; }
|
||||
.bias-impact-report__routing { border-bottom: 1px solid #f1f5f9; font-size: .86rem; padding: .45rem 0; justify-content: flex-start; }
|
||||
.bias-impact-report__side-effects { margin: .55rem 0 0; padding-left: 1.2rem; } .bias-impact-report__side-effects li { margin: .3rem 0; } .bias-impact-report__side-effects span { color: #7c3aed; font-size: .78rem; font-weight: 700; }
|
||||
.bias-impact-report__notice { background: #fffbeb; border-left: 3px solid #f59e0b; font-size: .84rem; margin: 0; padding: .6rem .75rem; }
|
||||
.bias-impact-report__warnings { background: #fff7ed; border: 1px solid #fed7aa; border-radius: .45rem; padding: .8rem; }
|
||||
code { font-size: .78rem; overflow-wrap: anywhere; }
|
||||
@media (max-width: 700px) { .bias-impact-report__header, .bias-impact-report__section-heading { align-items: flex-start; flex-direction: column; } }
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
@if (report; as currentReport) {
|
||||
<article class="bias-impact-report">
|
||||
<header class="bias-impact-report__header">
|
||||
<div>
|
||||
<span class="bias-impact-report__kind">{{ currentReport.kind }}</span>
|
||||
<h3>Bias impact report</h3>
|
||||
</div>
|
||||
<time [attr.datetime]="currentReport.createdAt">{{ currentReport.createdAt | date:'medium' }}</time>
|
||||
</header>
|
||||
|
||||
<dl class="bias-impact-report__metadata">
|
||||
<div><dt>Baseline execution</dt><dd>{{ currentReport.baselineExecutionId }}</dd></div>
|
||||
@if (currentReport.biasedExecutionId) { <div><dt>Biased execution</dt><dd>{{ currentReport.biasedExecutionId }}</dd></div> }
|
||||
<div><dt>Experiment</dt><dd>{{ currentReport.experimentId }}</dd></div>
|
||||
@if (currentReport.nodeId) { <div><dt>Node</dt><dd>{{ currentReport.nodeId }}</dd></div> }
|
||||
<div><dt>Annotations</dt><dd>{{ currentReport.annotationIds.join(', ') || 'None' }}</dd></div>
|
||||
<div><dt>Repetitions</dt><dd>{{ currentReport.repetitions }}</dd></div>
|
||||
</dl>
|
||||
|
||||
<p class="bias-impact-report__summary">{{ currentReport.summary }}</p>
|
||||
|
||||
<section class="bias-impact-report__section">
|
||||
<h4>Immediate impact</h4>
|
||||
<div class="bias-impact-report__metrics">
|
||||
<span>Output changed: <strong>{{ currentReport.immediateImpact.outputChanged ? 'Yes' : 'No' }}</strong></span>
|
||||
<span>Change rate: <strong>{{ percent(currentReport.immediateImpact.changeRate) }}</strong></span>
|
||||
<span>Maximum text difference: <strong>{{ textDifference(currentReport.immediateImpact.maximumTextDifference) }}</strong></span>
|
||||
</div>
|
||||
<app-bias-output-diff
|
||||
[baselineOutput]="currentReport.immediateImpact.baselineOutput"
|
||||
[biasedOutputs]="currentReport.immediateImpact.biasedOutputs"
|
||||
/>
|
||||
</section>
|
||||
|
||||
<section class="bias-impact-report__section">
|
||||
<div class="bias-impact-report__section-heading">
|
||||
<h4>Downstream impact</h4>
|
||||
<label><input type="checkbox" [(ngModel)]="changedOnly"> Changed nodes only</label>
|
||||
</div>
|
||||
@if (downstreamEntries.length === 0) {
|
||||
<p class="bias-impact-report__empty">No downstream impact was recorded. This is normal for an isolated step experiment.</p>
|
||||
}
|
||||
@for (entry of downstreamEntries; track entry.nodeId) {
|
||||
<article class="bias-impact-report__downstream">
|
||||
<div class="bias-impact-report__node-heading">
|
||||
<strong>{{ entry.nodeName || entry.nodeId }}</strong>
|
||||
<code>{{ entry.nodeId }}</code>
|
||||
<span [class.changed]="entry.changed">{{ entry.changed ? 'Changed' : 'Unchanged' }}</span>
|
||||
</div>
|
||||
<p>Status: {{ entry.baselineStatus }} → {{ entry.biasedStatus }}</p>
|
||||
<app-bias-output-diff
|
||||
[baselineOutput]="entry.baselineOutputs"
|
||||
[biasedOutputs]="[entry.biasedOutputs]"
|
||||
variantLabel="Biased output"
|
||||
/>
|
||||
</article>
|
||||
}
|
||||
</section>
|
||||
|
||||
<section class="bias-impact-report__section">
|
||||
<h4>Routing changes</h4>
|
||||
@if (currentReport.routingChanges.length === 0) { <p class="bias-impact-report__empty">No routing changes recorded.</p> }
|
||||
@for (change of currentReport.routingChanges; track change.nodeId) {
|
||||
<div class="bias-impact-report__routing"><code>{{ change.nodeId }}</code><span>{{ change.baselineBranch }} → {{ change.biasedBranch }}</span></div>
|
||||
}
|
||||
</section>
|
||||
|
||||
@if (currentReport.mockedSideEffects.length > 0) {
|
||||
<section class="bias-impact-report__section">
|
||||
<h4>Mocked side effects</h4>
|
||||
<ul class="bias-impact-report__side-effects">
|
||||
@for (sideEffect of currentReport.mockedSideEffects; track sideEffect.nodeId) {
|
||||
<li><strong>{{ sideEffect.nodeName || sideEffect.nodeId }}</strong> <code>{{ sideEffect.nodeId }}</code> <span>{{ sideEffect.kind }}</span></li>
|
||||
}
|
||||
</ul>
|
||||
</section>
|
||||
}
|
||||
|
||||
@if (!currentReport.rawOutputsIncluded) {
|
||||
<p class="bias-impact-report__notice">Raw output maps and lists were intentionally omitted from this report.</p>
|
||||
}
|
||||
|
||||
<section class="bias-impact-report__section bias-impact-report__warnings">
|
||||
<h4>Warnings</h4>
|
||||
@if (currentReport.warnings.length === 0) { <p class="bias-impact-report__empty">No warnings reported.</p> }
|
||||
@for (warning of currentReport.warnings; track warning) { <p>{{ warning }}</p> }
|
||||
</section>
|
||||
</article>
|
||||
}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { BiasImpactReport } from '@models/bias-impact';
|
||||
import { BiasImpactReportViewerComponent } from './bias-impact-report-viewer';
|
||||
|
||||
describe('BiasImpactReportViewerComponent', () => {
|
||||
let fixture: ComponentFixture<BiasImpactReportViewerComponent>;
|
||||
|
||||
const report: BiasImpactReport = {
|
||||
id: 'report-1',
|
||||
experimentId: 'experiment-1',
|
||||
kind: 'FULL_FLOW',
|
||||
baselineExecutionId: 'execution-base',
|
||||
biasedExecutionId: 'execution-biased',
|
||||
nodeId: null,
|
||||
annotationIds: ['annotation-1'],
|
||||
repetitions: 3,
|
||||
createdAt: '2026-07-21T10:00:00.000Z',
|
||||
rawOutputsIncluded: false,
|
||||
immediateImpact: {
|
||||
outputChanged: true,
|
||||
changeRate: .5,
|
||||
maximumTextDifference: .25,
|
||||
baselineOutput: 'baseline',
|
||||
biasedOutputs: ['biased']
|
||||
},
|
||||
downstreamImpact: [
|
||||
{ nodeId: 'node-changed', nodeName: 'Changed node', baselineStatus: 'COMPLETED', biasedStatus: 'COMPLETED', changed: true, baselineOutputs: 'a', biasedOutputs: 'b' },
|
||||
{ nodeId: 'node-same', nodeName: 'Same node', baselineStatus: 'COMPLETED', biasedStatus: 'COMPLETED', changed: false, baselineOutputs: 'a', biasedOutputs: 'a' }
|
||||
],
|
||||
routingChanges: [{ nodeId: 'router', baselineBranch: 'yes', biasedBranch: 'no' }],
|
||||
mockedSideEffects: [{ nodeId: 'http', nodeName: 'HTTP request', kind: 'HTTP' }],
|
||||
summary: 'The biased variant changed the output.',
|
||||
warnings: []
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({ imports: [BiasImpactReportViewerComponent] }).compileComponents();
|
||||
fixture = TestBed.createComponent(BiasImpactReportViewerComponent);
|
||||
fixture.componentInstance.report = report;
|
||||
});
|
||||
|
||||
it('renders metadata, impact sections, side effects and the raw-output notice', () => {
|
||||
fixture.detectChanges();
|
||||
const text = fixture.nativeElement.textContent;
|
||||
|
||||
expect(text).toContain('execution-base');
|
||||
expect(text).toContain('execution-biased');
|
||||
expect(text).not.toContain('Node\n');
|
||||
expect(text).toContain('Change rate: 50.0%');
|
||||
expect(text).toContain('router');
|
||||
expect(text).toContain('HTTP request');
|
||||
expect(text).toContain('intentionally omitted');
|
||||
expect(text).toContain('No warnings reported.');
|
||||
});
|
||||
|
||||
it('filters downstream entries to changed nodes', () => {
|
||||
fixture.componentInstance.changedOnly = true;
|
||||
fixture.detectChanges();
|
||||
|
||||
const text = fixture.nativeElement.textContent;
|
||||
expect(text).toContain('Changed node');
|
||||
expect(text).not.toContain('Same node');
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
import { CommonModule, DatePipe } from '@angular/common';
|
||||
import { ChangeDetectionStrategy, Component, Input } from '@angular/core';
|
||||
import { FormsModule } from '@angular/forms';
|
||||
import { BiasDownstreamImpactEntry, BiasImpactReport } from '@models/bias-impact';
|
||||
import { BiasOutputDiffComponent } from '../bias-output-diff/bias-output-diff';
|
||||
|
||||
@Component({
|
||||
selector: 'app-bias-impact-report-viewer',
|
||||
standalone: true,
|
||||
imports: [CommonModule, DatePipe, FormsModule, BiasOutputDiffComponent],
|
||||
templateUrl: './bias-impact-report-viewer.html',
|
||||
styleUrl: './bias-impact-report-viewer.css',
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
})
|
||||
export class BiasImpactReportViewerComponent {
|
||||
@Input() report: BiasImpactReport | null = null;
|
||||
|
||||
changedOnly = false;
|
||||
|
||||
get downstreamEntries(): BiasDownstreamImpactEntry[] {
|
||||
const entries = this.report?.downstreamImpact ?? [];
|
||||
return this.changedOnly ? entries.filter((entry) => entry.changed) : entries;
|
||||
}
|
||||
|
||||
percent(value: number): string {
|
||||
return `${(value * 100).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
textDifference(value: number): string {
|
||||
return `${value.toFixed(3)} (0 = identical, 1 = maximally different)`;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
.bias-output-diff { display: grid; gap: .8rem; grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); }
|
||||
.bias-output-diff__variants { display: grid; gap: .8rem; }
|
||||
.bias-output-diff__column { background: #f8fafc; border: 1px solid #dbe4ef; border-radius: .45rem; min-width: 0; padding: .7rem; }
|
||||
.bias-output-diff__baseline { border-left: 3px solid #2563eb; }
|
||||
.bias-output-diff__biased { border-left: 3px solid #a855f7; }
|
||||
h4 { color: #334155; font-size: .78rem; margin: 0 0 .5rem; text-transform: uppercase; }
|
||||
pre { font-family: inherit; margin: 0; overflow: auto; white-space: pre-wrap; word-break: break-word; }
|
||||
.bias-output-diff__empty { color: #64748b; margin: .4rem 0; }
|
||||
@media (max-width: 700px) { .bias-output-diff { grid-template-columns: 1fr; } }
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
<section class="bias-output-diff" aria-label="Output comparison">
|
||||
<div class="bias-output-diff__column bias-output-diff__baseline">
|
||||
<h4>{{ baselineLabel }}</h4>
|
||||
@if (isText(baselineOutput)) {
|
||||
<pre>{{ baselineOutput }}</pre>
|
||||
} @else {
|
||||
<app-json-viewer [value]="baselineOutput" />
|
||||
}
|
||||
</div>
|
||||
|
||||
<div class="bias-output-diff__variants">
|
||||
@if (biasedOutputs.length === 0) {
|
||||
<p class="bias-output-diff__empty">No biased output is available.</p>
|
||||
}
|
||||
@for (output of biasedOutputs; track $index) {
|
||||
<div class="bias-output-diff__column bias-output-diff__biased">
|
||||
<h4>{{ variantLabel }}{{ biasedOutputs.length > 1 ? ' ' + ($index + 1) : '' }}</h4>
|
||||
@if (isText(output)) {
|
||||
<pre>{{ output }}</pre>
|
||||
} @else {
|
||||
<app-json-viewer [value]="output" />
|
||||
}
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
</section>
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { BiasOutputDiffComponent } from './bias-output-diff';
|
||||
|
||||
describe('BiasOutputDiffComponent', () => {
|
||||
let fixture: ComponentFixture<BiasOutputDiffComponent>;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({ imports: [BiasOutputDiffComponent] }).compileComponents();
|
||||
fixture = TestBed.createComponent(BiasOutputDiffComponent);
|
||||
});
|
||||
|
||||
it('renders text baseline and every biased variant side by side', () => {
|
||||
fixture.componentInstance.baselineOutput = 'neutral';
|
||||
fixture.componentInstance.biasedOutputs = ['biased one', 'biased two'];
|
||||
fixture.detectChanges();
|
||||
|
||||
const text = fixture.nativeElement.textContent;
|
||||
expect(text).toContain('neutral');
|
||||
expect(text).toContain('Biased output 1');
|
||||
expect(text).toContain('Biased output 2');
|
||||
});
|
||||
|
||||
it('uses the JSON viewer for complex outputs', () => {
|
||||
fixture.componentInstance.baselineOutput = { result: true };
|
||||
fixture.componentInstance.biasedOutputs = [{ result: false }];
|
||||
fixture.detectChanges();
|
||||
|
||||
expect(fixture.nativeElement.querySelectorAll('app-json-viewer').length).toBe(2);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
import { CommonModule } from '@angular/common';
|
||||
import { ChangeDetectionStrategy, Component, Input } from '@angular/core';
|
||||
import { JsonViewerComponent } from '../json-viewer/json-viewer';
|
||||
|
||||
@Component({
|
||||
selector: 'app-bias-output-diff',
|
||||
standalone: true,
|
||||
imports: [CommonModule, JsonViewerComponent],
|
||||
templateUrl: './bias-output-diff.html',
|
||||
styleUrl: './bias-output-diff.css',
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
})
|
||||
export class BiasOutputDiffComponent {
|
||||
@Input() baselineOutput: unknown;
|
||||
@Input() biasedOutputs: unknown[] = [];
|
||||
@Input() baselineLabel = 'Baseline';
|
||||
@Input() variantLabel = 'Biased output';
|
||||
|
||||
isText(value: unknown): value is string {
|
||||
return typeof value === 'string';
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
.bias-rerun-dialog__backdrop { background: rgb(0 0 0 / .5); inset: 0; position: fixed; z-index: 10020; }
|
||||
.bias-rerun-dialog { background: #fff; border: 1px solid #cbd5e1; border-radius: 1rem; box-shadow: 0 25px 50px rgb(15 23 42 / .25); display: flex; flex-direction: column; left: 50%; max-height: min(88vh, 800px); max-width: 680px; position: fixed; top: 50%; transform: translate(-50%, -50%); width: min(94vw, 680px); z-index: 10021; }
|
||||
header, footer { align-items: center; display: flex; gap: 1rem; justify-content: space-between; padding: 1rem 1.25rem; } header { border-bottom: 1px solid #e2e8f0; } h3, p { margin: 0; } header p { color: #64748b; font-size: .85rem; }
|
||||
.bias-rerun-dialog__content { display: grid; gap: .9rem; overflow: auto; padding: 1.25rem; } fieldset { display: grid; gap: .5rem; } label { color: #334155; font-size: .87rem; }
|
||||
.bias-rerun-dialog__error { background: #fff1f2; border-left: 3px solid #e11d48; color: #9f1239; padding: .6rem .7rem; } footer { border-top: 1px solid #e2e8f0; justify-content: flex-end; }
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
@if (state(); as currentState) {
|
||||
<div class="bias-rerun-dialog__backdrop" (click)="close()"></div>
|
||||
<section class="bias-rerun-dialog" role="dialog" aria-modal="true" aria-label="Create biased rerun">
|
||||
<header><div><h3>Create biased rerun</h3><p>Select the executable annotations to activate for the full flow.</p></div><button type="button" mat-stroked-button (click)="close()">Close</button></header>
|
||||
<div class="bias-rerun-dialog__content">
|
||||
@for (candidate of currentState.candidates; track candidate.nodeId) {
|
||||
<fieldset [disabled]="creating()"><legend>{{ candidate.nodeName }}</legend>
|
||||
@for (annotation of candidate.annotations; track annotation.id) {
|
||||
<label><input type="checkbox" [checked]="selectedIds(candidate.nodeId).includes($any(annotation.id))" (change)="toggleAnnotation(candidate.nodeId, $any(annotation.id), $any($event.target).checked)"> {{ annotation.category || 'Uncategorized' }} — {{ annotation.issue || annotation.rationale || annotation.id }}</label>
|
||||
}
|
||||
</fieldset>
|
||||
}
|
||||
<app-side-effect-policy-selector [policy]="policy()" [externalSideEffects]="hasExternalSideEffects()" [disabled]="creating()" (policyChange)="policy.set($event)" />
|
||||
@if (inlineError(); as error) { <p class="bias-rerun-dialog__error">{{ error }}</p> }
|
||||
</div>
|
||||
<footer><button type="button" mat-stroked-button (click)="close()">Cancel</button><button type="button" mat-flat-button [disabled]="creating()" (click)="submit()">{{ creating() ? 'Creating…' : 'Create biased rerun' }}</button></footer>
|
||||
</section>
|
||||
}
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
import { ChangeDetectionStrategy, Component, effect, inject, signal } from '@angular/core';
|
||||
import { FormsModule } from '@angular/forms';
|
||||
import { MatButtonModule } from '@angular/material/button';
|
||||
import { ExternalSideEffectPolicy, BiasSideEffectError } from '@models/bias-impact';
|
||||
import { ConfirmDialogService } from '@services/dialogs/confirm-dialog';
|
||||
import { BiasRerunDialogService } from '@services/dialogs/bias-rerun-dialog';
|
||||
import { TaskExecutionsService } from '@services/task-executions/task-executions';
|
||||
import { SideEffectPolicySelectorComponent } from '@shared/side-effect-policy-selector/side-effect-policy-selector';
|
||||
|
||||
@Component({
|
||||
selector: 'app-bias-rerun-dialog-host',
|
||||
standalone: true,
|
||||
imports: [FormsModule, MatButtonModule, SideEffectPolicySelectorComponent],
|
||||
templateUrl: './bias-rerun-dialog.html',
|
||||
styleUrl: './bias-rerun-dialog.css',
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
})
|
||||
export class BiasRerunDialogHostComponent {
|
||||
private readonly dialog = inject(BiasRerunDialogService);
|
||||
private readonly executions = inject(TaskExecutionsService);
|
||||
private readonly confirmation = inject(ConfirmDialogService);
|
||||
readonly state = this.dialog.state;
|
||||
readonly selectedAnnotationIdsByNode = signal<Record<string, string[]>>({});
|
||||
readonly policy = signal<ExternalSideEffectPolicy>('BLOCK');
|
||||
readonly creating = signal(false);
|
||||
readonly inlineError = signal<string | null>(null);
|
||||
|
||||
constructor() {
|
||||
effect(() => {
|
||||
const state = this.state();
|
||||
this.selectedAnnotationIdsByNode.set(Object.fromEntries((state?.candidates ?? []).map((candidate) => [candidate.nodeId, []])));
|
||||
this.policy.set('BLOCK');
|
||||
this.creating.set(false);
|
||||
this.inlineError.set(null);
|
||||
});
|
||||
}
|
||||
|
||||
selectedIds(nodeId: string): string[] { return this.selectedAnnotationIdsByNode()[nodeId] ?? []; }
|
||||
|
||||
toggleAnnotation(nodeId: string, annotationId: string, checked: boolean) {
|
||||
this.selectedAnnotationIdsByNode.update((current) => {
|
||||
const selected = new Set(current[nodeId] ?? []);
|
||||
if (checked) selected.add(annotationId); else selected.delete(annotationId);
|
||||
return { ...current, [nodeId]: [...selected] };
|
||||
});
|
||||
}
|
||||
|
||||
hasExternalSideEffects(): boolean {
|
||||
return (this.state()?.candidates ?? []).some((candidate) => this.selectedIds(candidate.nodeId).length > 0 && candidate.capabilities.externalSideEffects);
|
||||
}
|
||||
|
||||
async submit() {
|
||||
const state = this.state();
|
||||
if (!state || this.creating()) return;
|
||||
const activations = Object.entries(this.selectedAnnotationIdsByNode())
|
||||
.filter(([, annotationIds]) => annotationIds.length > 0)
|
||||
.map(([nodeId, annotationIds]) => ({ nodeId, annotationIds }));
|
||||
if (!activations.length) { this.inlineError.set('Select at least one executable annotation.'); return; }
|
||||
|
||||
let confirmExternalSideEffects = false;
|
||||
if (this.policy() === 'REQUIRE_CONFIRMATION') {
|
||||
confirmExternalSideEffects = await this.confirmation.open('This biased rerun may invoke real external HTTP or MCP calls. Do you want to continue?');
|
||||
if (!confirmExternalSideEffects) return;
|
||||
}
|
||||
this.creating.set(true);
|
||||
this.inlineError.set(null);
|
||||
this.executions.createBiasedRerun(state.executionId, { activations, externalSideEffectPolicy: this.policy(), confirmExternalSideEffects }).subscribe({
|
||||
next: (execution) => { state.onCreated(execution); this.dialog.close(); },
|
||||
error: (error) => {
|
||||
this.creating.set(false);
|
||||
const sideEffectError = error as Partial<BiasSideEffectError>;
|
||||
this.inlineError.set(sideEffectError.reason === 'SIDE_EFFECT_BLOCKED' || sideEffectError.reason === 'CONFIRMATION_REQUIRED'
|
||||
? sideEffectError.message ?? 'External side effect policy prevented the rerun.'
|
||||
: error instanceof Error ? error.message : 'Unable to create the biased rerun.');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
close() { if (!this.creating()) this.dialog.close(); }
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
.json-viewer { font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; font-size: .78rem; overflow: auto; }
|
||||
.json-viewer__label { color: #475569; font-family: inherit; font-weight: 700; margin-bottom: .35rem; }
|
||||
.json-viewer__group { min-width: max-content; }
|
||||
.json-viewer__group > summary { cursor: pointer; color: #1e40af; outline-offset: 2px; }
|
||||
.json-viewer__children { border-left: 1px solid #cbd5e1; margin: .3rem 0 .2rem .35rem; padding-left: .7rem; }
|
||||
.json-viewer__entry { align-items: flex-start; display: flex; gap: .45rem; line-height: 1.55; }
|
||||
.json-viewer__key { color: #7c3aed; white-space: nowrap; }
|
||||
.json-viewer__primitive { color: #0f172a; font-family: inherit; white-space: pre-wrap; word-break: break-word; }
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
<section class="json-viewer" [attr.aria-label]="label ?? 'JSON value'">
|
||||
@if (label) {
|
||||
<div class="json-viewer__label">{{ label }}</div>
|
||||
}
|
||||
|
||||
<ng-template #renderValue let-value let-path="path">
|
||||
@if (isExpandable(value)) {
|
||||
<details class="json-viewer__group" [open]="isExpanded(path)" (toggle)="onToggle(path, $event)">
|
||||
<summary>{{ summary(value) }}</summary>
|
||||
<div class="json-viewer__children">
|
||||
@if (isArray(value)) {
|
||||
@for (item of value; track $index) {
|
||||
<div class="json-viewer__entry">
|
||||
<span class="json-viewer__key">[{{ $index }}]</span>
|
||||
<ng-container *ngTemplateOutlet="renderValue; context: { $implicit: item, path: path + '[' + $index + ']' }" />
|
||||
</div>
|
||||
}
|
||||
} @else if (isObject(value)) {
|
||||
@for (entry of entries(value); track entry.key) {
|
||||
<div class="json-viewer__entry">
|
||||
<span class="json-viewer__key">{{ entry.key }}</span>
|
||||
<ng-container *ngTemplateOutlet="renderValue; context: { $implicit: entry.value, path: path + '.' + entry.key }" />
|
||||
</div>
|
||||
}
|
||||
}
|
||||
</div>
|
||||
</details>
|
||||
} @else {
|
||||
<code class="json-viewer__primitive">{{ formatPrimitive(value) }}</code>
|
||||
}
|
||||
</ng-template>
|
||||
|
||||
<ng-container *ngTemplateOutlet="renderValue; context: { $implicit: value, path: '$' }" />
|
||||
</section>
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { JsonViewerComponent } from './json-viewer';
|
||||
|
||||
describe('JsonViewerComponent', () => {
|
||||
let fixture: ComponentFixture<JsonViewerComponent>;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({ imports: [JsonViewerComponent] }).compileComponents();
|
||||
fixture = TestBed.createComponent(JsonViewerComponent);
|
||||
});
|
||||
|
||||
it('renders an expandable recursive object tree', () => {
|
||||
fixture.componentInstance.value = { answer: { enabled: true }, values: [1, 2] };
|
||||
fixture.detectChanges();
|
||||
|
||||
const text = fixture.nativeElement.textContent;
|
||||
expect(text).toContain('Object (2)');
|
||||
expect(text).toContain('answer');
|
||||
expect(text).toContain('enabled');
|
||||
expect(text).toContain('true');
|
||||
expect(text).toContain('[0]');
|
||||
});
|
||||
|
||||
it('formats primitive strings as JSON strings', () => {
|
||||
expect(fixture.componentInstance.formatPrimitive('hello')).toBe('"hello"');
|
||||
expect(fixture.componentInstance.formatPrimitive(null)).toBe('null');
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
import { CommonModule } from '@angular/common';
|
||||
import { ChangeDetectionStrategy, Component, Input } from '@angular/core';
|
||||
|
||||
/** Lightweight, dependency-free JSON tree for diagnostic and report views. */
|
||||
@Component({
|
||||
selector: 'app-json-viewer',
|
||||
standalone: true,
|
||||
imports: [CommonModule],
|
||||
templateUrl: './json-viewer.html',
|
||||
styleUrl: './json-viewer.css',
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
})
|
||||
export class JsonViewerComponent {
|
||||
@Input() value: unknown;
|
||||
@Input() label: string | null = null;
|
||||
@Input() initiallyExpanded = true;
|
||||
|
||||
private readonly collapsedPaths = new Set<string>();
|
||||
|
||||
isArray(value: unknown): value is unknown[] {
|
||||
return Array.isArray(value);
|
||||
}
|
||||
|
||||
isObject(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
isExpandable(value: unknown): boolean {
|
||||
return this.isArray(value) || this.isObject(value);
|
||||
}
|
||||
|
||||
entries(value: Record<string, unknown>): Array<{ key: string; value: unknown }> {
|
||||
return Object.entries(value).map(([key, entryValue]) => ({ key, value: entryValue }));
|
||||
}
|
||||
|
||||
summary(value: unknown): string {
|
||||
if (this.isArray(value)) return `Array (${value.length})`;
|
||||
if (this.isObject(value)) return `Object (${Object.keys(value).length})`;
|
||||
return this.formatPrimitive(value);
|
||||
}
|
||||
|
||||
formatPrimitive(value: unknown): string {
|
||||
if (typeof value === 'string') return `"${value}"`;
|
||||
if (value === undefined) return 'undefined';
|
||||
if (value === null) return 'null';
|
||||
try {
|
||||
return JSON.stringify(value);
|
||||
} catch {
|
||||
return String(value);
|
||||
}
|
||||
}
|
||||
|
||||
isExpanded(path: string): boolean {
|
||||
return this.initiallyExpanded && !this.collapsedPaths.has(path);
|
||||
}
|
||||
|
||||
onToggle(path: string, event: Event) {
|
||||
const expanded = (event.target as HTMLDetailsElement).open;
|
||||
if (expanded) this.collapsedPaths.delete(path);
|
||||
else this.collapsedPaths.add(path);
|
||||
}
|
||||
}
|
||||
|
|
@ -512,6 +512,7 @@
|
|||
|
||||
<app-bias-annotations
|
||||
[blockId]="blockId ?? ''"
|
||||
[block]="biasBlock"
|
||||
[annotations]="biasAnnotations"
|
||||
[readonly]="isReadonly"
|
||||
(annotationsChange)="updateBiasAnnotations($event)" />
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { CommonModule } from '@angular/common';
|
|||
import { ChangeDetectionStrategy, ChangeDetectorRef, Component, effect, ElementRef, HostBinding, HostListener, inject, Input, OnDestroy } from '@angular/core';
|
||||
import { FormsModule } from '@angular/forms';
|
||||
import { MatTooltipModule } from '@angular/material/tooltip';
|
||||
import { BiasAnnotation, BlockType, currentFlowPortValueKind, flowValueKindLabel, FlowData, FlowPort, FlowValueKind, FLOW_DEPENDANT_PORT_KEY, FLOW_DEPENDENCY_PORT_KEY, normalizeFlowPortValueKinds } from '@models/flow';
|
||||
import { BiasAnnotation, BlockType, currentFlowPortValueKind, flowValueKindLabel, FlowBlock, FlowData, FlowPort, FlowValueKind, FLOW_DEPENDANT_PORT_KEY, FLOW_DEPENDENCY_PORT_KEY, normalizeFlowPortValueKinds } from '@models/flow';
|
||||
import { BiasAnnotationsComponent } from '../../bias-annotations/bias-annotations';
|
||||
import { ClassicPreset } from 'rete';
|
||||
import { ReteModule } from 'rete-angular-plugin/21';
|
||||
|
|
@ -694,6 +694,24 @@ export class GenericNodeComponent implements OnDestroy {
|
|||
return Array.isArray(value) ? value as BiasAnnotation[] : [];
|
||||
}
|
||||
|
||||
get biasBlock(): FlowBlock | null {
|
||||
const nodeData = this.data?.data as Record<string, unknown> | undefined;
|
||||
const blockId = this.blockId;
|
||||
const typeName = this.blockType;
|
||||
if (!nodeData || !blockId || !typeName) return null;
|
||||
return {
|
||||
id: blockId,
|
||||
name: this.name,
|
||||
position: nodeData['position'] as { x: number; y: number } | undefined,
|
||||
inputs: this.resolvePorts('input').map((port) => ({ ...port })),
|
||||
outputs: this.resolvePorts('output').map((port) => ({ ...port })),
|
||||
specificConfiguration: this.blockConfiguration ?? {},
|
||||
typeName,
|
||||
nodeFamily: 'block',
|
||||
biasAnnotations: this.biasAnnotations
|
||||
};
|
||||
}
|
||||
|
||||
private get biasAnnotationsProperty(): string {
|
||||
const descriptorSignal = (this.blocksService as BlocksService & {
|
||||
biasAnnotationsDescriptor?: () => { blockProperty?: string } | null
|
||||
|
|
|
|||
|
|
@ -616,3 +616,17 @@
|
|||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
.llm-bias-impact-trigger {
|
||||
align-items: center;
|
||||
background: #eff6ff;
|
||||
border: 1px solid #93c5fd;
|
||||
border-radius: .35rem;
|
||||
color: #1d4ed8;
|
||||
cursor: pointer;
|
||||
display: inline-flex;
|
||||
font-size: .9rem;
|
||||
justify-content: center;
|
||||
margin-left: auto;
|
||||
padding: .3rem .4rem;
|
||||
}
|
||||
.llm-bias-impact-trigger:hover { background: #dbeafe; }
|
||||
|
|
|
|||
|
|
@ -59,6 +59,17 @@
|
|||
<span class="llm-subtitle">{{ name }}</span>
|
||||
</div>
|
||||
</div>
|
||||
@if (canMeasureBiasImpact()) {
|
||||
<button
|
||||
type="button"
|
||||
class="llm-bias-impact-trigger"
|
||||
aria-label="Measure bias impact"
|
||||
title="Measure bias impact"
|
||||
(pointerdown)="$event.stopPropagation()"
|
||||
(click)="measureBiasImpact($event)">
|
||||
<i class="bi bi-bar-chart-line"></i>
|
||||
</button>
|
||||
}
|
||||
</div>
|
||||
|
||||
<div class="llm-body">
|
||||
|
|
|
|||
|
|
@ -2,13 +2,16 @@ import { CommonModule } from '@angular/common';
|
|||
import { ChangeDetectionStrategy, ChangeDetectorRef, Component, HostBinding, Input, inject } from '@angular/core';
|
||||
import { ClassicPreset } from 'rete';
|
||||
import { ReteModule } from 'rete-angular-plugin/21';
|
||||
import { BlockInteractionContract, BlockType, FlowData, FlowPort, FLOW_DEPENDANT_PORT_KEY, FLOW_DEPENDENCY_PORT_KEY } from '@models/flow';
|
||||
import { BiasAnnotation, BlockInteractionContract, BlockType, FlowData, FlowPort, isProbeExecutable, FLOW_DEPENDANT_PORT_KEY, FLOW_DEPENDENCY_PORT_KEY } from '@models/flow';
|
||||
import { BiasCapabilities } from '@models/bias-impact';
|
||||
import { BlocksService } from '@services/blocks/blocks';
|
||||
import { ContainersService } from '@services/containers/containers';
|
||||
import { NodeSettingsDialogService } from '@services/dialogs/node-settings-dialog';
|
||||
import { SubflowPreviewDialogService } from '@services/dialogs/subflow-preview-dialog';
|
||||
import { HumanInteractionDialogService } from '@services/dialogs/human-interaction-dialog';
|
||||
import { TaskExecutionsService } from '@services/task-executions/task-executions';
|
||||
import { BiasImpactExperimentDialogService } from '@services/dialogs/bias-impact-experiment-dialog';
|
||||
import { take } from 'rxjs';
|
||||
import {
|
||||
collectSchemaFlowDataFields,
|
||||
isFlowDataFieldPath,
|
||||
|
|
@ -114,6 +117,7 @@ export class TaskStepNodeComponent {
|
|||
private cdr = inject(ChangeDetectorRef);
|
||||
private humanInteractionDialog = inject(HumanInteractionDialogService);
|
||||
private taskExecutionsService = inject(TaskExecutionsService);
|
||||
private biasImpactExperimentDialog = inject(BiasImpactExperimentDialogService);
|
||||
|
||||
@Input() data!: any;
|
||||
@Input() emit!: (data: any) => void;
|
||||
|
|
@ -137,6 +141,7 @@ export class TaskStepNodeComponent {
|
|||
mainContentFields: MainContentView[] = [];
|
||||
interactionSubmitting = false;
|
||||
schemaReady = false;
|
||||
biasCapabilities: BiasCapabilities | null = null;
|
||||
|
||||
private blockSchema: Record<string, any> | null = null;
|
||||
private blockDescriptor: BlockType | null = null;
|
||||
|
|
@ -170,6 +175,7 @@ export class TaskStepNodeComponent {
|
|||
|
||||
this.rebuildDisplayState();
|
||||
void this.loadSchemaContext();
|
||||
this.loadBiasCapabilities();
|
||||
}
|
||||
|
||||
private rebuildDisplayState() {
|
||||
|
|
@ -423,6 +429,36 @@ export class TaskStepNodeComponent {
|
|||
return this.stepStatus() === 'COMPLETED';
|
||||
}
|
||||
|
||||
executableBiasAnnotations(): BiasAnnotation[] {
|
||||
const annotations = this.data?.data?.biasAnnotations;
|
||||
return Array.isArray(annotations)
|
||||
? annotations.filter((annotation): annotation is BiasAnnotation => !!annotation && isProbeExecutable(annotation.behavioralProbe))
|
||||
: [];
|
||||
}
|
||||
|
||||
canMeasureBiasImpact(): boolean {
|
||||
return this.blockConfiguration?.['__executionStatusGroup'] === 'FINAL'
|
||||
&& this.executableBiasAnnotations().length > 0
|
||||
&& this.biasCapabilities?.isolatedExperimentSupported === true;
|
||||
}
|
||||
|
||||
measureBiasImpact(event: Event) {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
const executionId = this.executionId();
|
||||
const stepId = this.executionNodeId();
|
||||
const capabilities = this.biasCapabilities;
|
||||
if (!executionId || !stepId || !capabilities || !this.canMeasureBiasImpact()) return;
|
||||
this.biasImpactExperimentDialog.open({
|
||||
executionId,
|
||||
stepId,
|
||||
nodeId: String(this.data?.id ?? stepId),
|
||||
nodeName: this.nodeTitle(),
|
||||
annotations: this.executableBiasAnnotations(),
|
||||
capabilities
|
||||
});
|
||||
}
|
||||
|
||||
isRunning(): boolean {
|
||||
return this.stepStatus() === 'RUNNING';
|
||||
}
|
||||
|
|
@ -485,6 +521,21 @@ export class TaskStepNodeComponent {
|
|||
return this.data?.data?.specificConfiguration ?? null;
|
||||
}
|
||||
|
||||
private loadBiasCapabilities() {
|
||||
const blockType = this.blockType;
|
||||
if (!blockType || this.isContainerNode()) return;
|
||||
this.blocksService.retrieveBiasCapabilities(blockType).pipe(take(1)).subscribe({
|
||||
next: (capabilities) => {
|
||||
this.biasCapabilities = capabilities;
|
||||
this.cdr.markForCheck();
|
||||
},
|
||||
error: () => {
|
||||
this.biasCapabilities = null;
|
||||
this.cdr.markForCheck();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private get blockType(): string | null {
|
||||
const typeName = this.data?.data?.typeName;
|
||||
return typeof typeName === 'string' && typeName.length > 0 ? typeName : null;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
.side-effect-policy-selector { display: grid; gap: .45rem; }
|
||||
.side-effect-policy-selector h4 { font-size: .9rem; margin: 0; }
|
||||
.side-effect-policy-selector label { color: #334155; font-size: .86rem; }
|
||||
.side-effect-policy-selector__warning { background: #fff7ed; border-left: 3px solid #f97316; color: #9a3412; font-size: .84rem; margin: 0; padding: .55rem .65rem; }
|
||||
.side-effect-policy-selector__note { color: #0369a1; font-size: .82rem; margin: 0; }
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
<section class="side-effect-policy-selector">
|
||||
<h4>External side effects</h4>
|
||||
@if (externalSideEffects) {
|
||||
<p class="side-effect-policy-selector__warning">This block can invoke external services. Choose how the experiment handles those calls.</p>
|
||||
}
|
||||
<label><input type="radio" name="side-effect-policy" [checked]="policy === 'BLOCK'" [disabled]="disabled" (change)="select('BLOCK')"> Block external calls</label>
|
||||
<label><input type="radio" name="side-effect-policy" [checked]="policy === 'MOCK'" [disabled]="disabled" (change)="select('MOCK')"> Mock external calls</label>
|
||||
<label><input type="radio" name="side-effect-policy" [checked]="policy === 'REQUIRE_CONFIRMATION'" [disabled]="disabled" (change)="select('REQUIRE_CONFIRMATION')"> Allow after confirmation</label>
|
||||
@if (policy === 'MOCK') { <p class="side-effect-policy-selector__note">HTTP and MCP calls will not be invoked for real.</p> }
|
||||
</section>
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { SideEffectPolicySelectorComponent } from './side-effect-policy-selector';
|
||||
|
||||
describe('SideEffectPolicySelectorComponent', () => {
|
||||
let fixture: ComponentFixture<SideEffectPolicySelectorComponent>;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({ imports: [SideEffectPolicySelectorComponent] }).compileComponents();
|
||||
fixture = TestBed.createComponent(SideEffectPolicySelectorComponent);
|
||||
});
|
||||
|
||||
it('emits the selected policy and explains mock mode', () => {
|
||||
const emitted: string[] = [];
|
||||
fixture.componentInstance.policyChange.subscribe((policy) => emitted.push(policy));
|
||||
fixture.componentInstance.policy = 'MOCK';
|
||||
fixture.componentInstance.externalSideEffects = true;
|
||||
fixture.detectChanges();
|
||||
fixture.componentInstance.select('BLOCK');
|
||||
|
||||
expect(emitted).toEqual(['BLOCK']);
|
||||
expect(fixture.nativeElement.textContent).toContain('will not be invoked for real');
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
import { CommonModule } from '@angular/common';
|
||||
import { ChangeDetectionStrategy, Component, EventEmitter, Input, Output } from '@angular/core';
|
||||
import { ExternalSideEffectPolicy } from '@models/bias-impact';
|
||||
|
||||
@Component({
|
||||
selector: 'app-side-effect-policy-selector',
|
||||
standalone: true,
|
||||
imports: [CommonModule],
|
||||
templateUrl: './side-effect-policy-selector.html',
|
||||
styleUrl: './side-effect-policy-selector.css',
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
})
|
||||
export class SideEffectPolicySelectorComponent {
|
||||
@Input() policy: ExternalSideEffectPolicy = 'BLOCK';
|
||||
@Input() externalSideEffects = false;
|
||||
@Input() disabled = false;
|
||||
@Output() policyChange = new EventEmitter<ExternalSideEffectPolicy>();
|
||||
|
||||
select(policy: ExternalSideEffectPolicy) {
|
||||
if (!this.disabled) this.policyChange.emit(policy);
|
||||
}
|
||||
}
|
||||
|
|
@ -411,3 +411,13 @@
|
|||
overflow-y: auto;
|
||||
overscroll-behavior: contain;
|
||||
}
|
||||
.execution-bias-variant-context {
|
||||
align-items: center;
|
||||
color: #5b21b6;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
font-size: .78rem;
|
||||
gap: .35rem .75rem;
|
||||
margin-top: .45rem;
|
||||
}
|
||||
.execution-graph-action-button-bias { background: #f3e8ff; color: #7e22ce; }
|
||||
|
|
|
|||
|
|
@ -11,6 +11,14 @@
|
|||
@if (simulationDescriptorLabel(); as simulationDescriptor) {
|
||||
<p class="text-xs text-sky-900">Simulator: {{ simulationDescriptor }}</p>
|
||||
}
|
||||
@if (execution()!.biasExecutionContext; as biasContext) {
|
||||
<div class="execution-bias-variant-context">
|
||||
<strong>Bias variant</strong>
|
||||
<span>Experiment: {{ biasContext.experimentId }}</span>
|
||||
<span>Baseline: {{ execution()!.rerunOfExecutionId || 'not available' }}</span>
|
||||
<span>Active annotations: {{ biasContext.activeAnnotationIdsByNode | json }}</span>
|
||||
</div>
|
||||
}
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -92,6 +100,15 @@
|
|||
<mat-icon fontIcon="science"></mat-icon>
|
||||
</button>
|
||||
}
|
||||
<button
|
||||
type="button"
|
||||
class="execution-action-button execution-graph-action-button execution-action-button-bias"
|
||||
matTooltip="Create biased rerun"
|
||||
aria-label="Create biased rerun"
|
||||
[disabled]="!canCreateBiasedRerun()"
|
||||
(click)="openBiasedRerunDialog()">
|
||||
<mat-icon fontIcon="account_tree"></mat-icon>
|
||||
</button>
|
||||
</div>
|
||||
<app-rete-editor
|
||||
[flowId]="execution()!.id"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { CommonModule } from '@angular/common';
|
||||
import { ChangeDetectionStrategy, Component, computed, effect, ElementRef, inject, input, OnDestroy, signal, viewChild } from '@angular/core';
|
||||
import { ActivatedRoute, Router } from '@angular/router';
|
||||
import { MatButtonModule } from '@angular/material/button';
|
||||
import { MatIconModule } from '@angular/material/icon';
|
||||
import { MatTooltipModule } from '@angular/material/tooltip';
|
||||
|
|
@ -11,7 +12,7 @@ import {
|
|||
FlowData,
|
||||
LLMDescriptor,
|
||||
FlowNode,
|
||||
FlowNodeDependency,
|
||||
FlowNodeDependency, isProbeExecutable,
|
||||
normalizeFlowPortValueKinds
|
||||
} from '@models/flow';
|
||||
import {
|
||||
|
|
@ -35,6 +36,8 @@ import { NodeSettingField, NodeSettingsDialogService } from '@services/dialogs/n
|
|||
import { FieldRetriever } from '@services/retriever/field-retriever';
|
||||
import { TaskExecutionsService } from '@services/task-executions/task-executions';
|
||||
import { ContainersService } from '@services/containers/containers';
|
||||
import { BlocksService } from '@services/blocks/blocks';
|
||||
import { BiasRerunDialogService, BiasRerunCandidate } from '@services/dialogs/bias-rerun-dialog';
|
||||
import { firstValueFrom } from 'rxjs';
|
||||
import {
|
||||
ExecutionOutputEntry,
|
||||
|
|
@ -79,6 +82,10 @@ export class TaskExecutionViewerComponent implements OnDestroy {
|
|||
private settingsDialog = inject(NodeSettingsDialogService);
|
||||
private fieldRetriever = inject(FieldRetriever);
|
||||
private containersService = inject(ContainersService);
|
||||
private blocksService = inject(BlocksService);
|
||||
private biasRerunDialog = inject(BiasRerunDialogService);
|
||||
private router = inject(Router);
|
||||
private route = inject(ActivatedRoute);
|
||||
private lastExecutionId: string | null = null;
|
||||
private lastExecutionStatus: string | null = null;
|
||||
private static readonly SIMULATOR_PROVIDER_RETRIEVER_URL = '/retriever/LLM/providers';
|
||||
|
|
@ -88,6 +95,7 @@ export class TaskExecutionViewerComponent implements OnDestroy {
|
|||
readonly activeAsideTab = signal<'inputs' | 'intermediate' | 'logs' | 'output'>('inputs');
|
||||
readonly startInProgress = signal(false);
|
||||
readonly simulateInProgress = signal(false);
|
||||
readonly biasRerunOpening = signal(false);
|
||||
readonly cancelInProgress = signal(false);
|
||||
readonly resumeInProgress = signal(false);
|
||||
readonly savingInputs = signal<Record<string, boolean>>({});
|
||||
|
|
@ -404,6 +412,10 @@ export class TaskExecutionViewerComponent implements OnDestroy {
|
|||
});
|
||||
|
||||
readonly isSimulatedExecution = computed(() => this.execution()?.interactionSimulationEnabled === true);
|
||||
readonly isBiasVariant = computed(() => !!this.execution()?.biasExecutionContext);
|
||||
readonly canCreateBiasedRerun = computed(() =>
|
||||
getExecutionStatusGroup(this.execution()?.context.status) === 'FINAL' && !this.biasRerunOpening()
|
||||
);
|
||||
readonly simulationDescriptorLabel = computed(() => {
|
||||
const descriptor = this.execution()?.interactionSimulationDescriptor;
|
||||
if (!descriptor) return null;
|
||||
|
|
@ -584,6 +596,30 @@ export class TaskExecutionViewerComponent implements OnDestroy {
|
|||
});
|
||||
}
|
||||
|
||||
async openBiasedRerunDialog() {
|
||||
const execution = this.execution();
|
||||
if (!execution || !this.canCreateBiasedRerun()) return;
|
||||
this.biasRerunOpening.set(true);
|
||||
try {
|
||||
const candidates = await this.biasRerunCandidates();
|
||||
if (!candidates.length) return;
|
||||
this.biasRerunDialog.open({
|
||||
executionId: execution.id,
|
||||
candidates,
|
||||
onCreated: (variant) => {
|
||||
void this.router.navigate([], {
|
||||
relativeTo: this.route,
|
||||
queryParams: { executionId: variant.id },
|
||||
queryParamsHandling: 'merge',
|
||||
replaceUrl: true
|
||||
});
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
this.biasRerunOpening.set(false);
|
||||
}
|
||||
}
|
||||
|
||||
onTextInputChange(input: EditableExecutionInput, value: string | string[]) {
|
||||
if (this.inputsReadOnly()) return;
|
||||
this.pendingTextInputs.update((current) => ({ ...current, [input.key]: value }));
|
||||
|
|
@ -959,6 +995,28 @@ export class TaskExecutionViewerComponent implements OnDestroy {
|
|||
return (this.execution()?.stepDependencies ?? []).some((dependency) => String(dependency.targetId) === stepId);
|
||||
}
|
||||
|
||||
private async biasRerunCandidates(): Promise<BiasRerunCandidate[]> {
|
||||
const candidates = this.stepsArray().flatMap((step): Array<{ nodeId: string; nodeName: string; block: FlowBlock }> => {
|
||||
const node = getTaskExecutionStepNode(step);
|
||||
if (!node || node.nodeFamily === 'container') return [];
|
||||
const block = node as FlowBlock;
|
||||
const annotations = (block.biasAnnotations ?? []).filter((annotation) => isProbeExecutable(annotation.behavioralProbe));
|
||||
return annotations.length ? [{ nodeId: step.id, nodeName: block.name || step.id, block: { ...block, biasAnnotations: annotations } }] : [];
|
||||
});
|
||||
|
||||
const resolved = await Promise.all(candidates.map(async (candidate) => {
|
||||
const capabilities = await firstValueFrom(this.blocksService.retrieveBiasCapabilities(candidate.block.typeName));
|
||||
if (!capabilities.fullFlowExperimentSupported) return null;
|
||||
return {
|
||||
nodeId: candidate.nodeId,
|
||||
nodeName: candidate.nodeName,
|
||||
annotations: candidate.block.biasAnnotations ?? [],
|
||||
capabilities
|
||||
} satisfies BiasRerunCandidate;
|
||||
}));
|
||||
return resolved.filter((candidate): candidate is BiasRerunCandidate => candidate !== null);
|
||||
}
|
||||
|
||||
private hasOutgoingDependency(stepId: string): boolean {
|
||||
return (this.execution()?.stepDependencies ?? []).some((dependency) => String(dependency.sourceId) === stepId);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue