diff --git a/src/app/models/bias-impact.ts b/src/app/models/bias-impact.ts index d4b95ff..3b27878 100644 --- a/src/app/models/bias-impact.ts +++ b/src/app/models/bias-impact.ts @@ -167,6 +167,13 @@ export type BiasJudgeSummary = { judgedPairs: number; skippedPairs: number; errors: string[]; + /** + * What this assessment said about each compared pair, by the path the comparison hands out. + * + * Carried so that a second opinion does not erase the first. The per-subject badges follow the + * most recent assessment, which the service attaches to the pairs themselves. + */ + verdicts?: Record; }; /** One element of a compared list value, which for an iterator container is one subject. */ @@ -297,8 +304,8 @@ export type BiasImpactReport = { warnings: string[]; interventionDirection?: BiasInterventionDirection; schemaVersion?: number; - /** Present only once someone has asked a model to assess the comparison. */ - judge?: BiasJudgeSummary | null; + /** Every assessment asked of a model, newest first. Empty until one is asked for. */ + judgements?: BiasJudgeSummary[]; simulation?: BiasSimulationContext | null; }; diff --git a/src/app/services/task-executions/task-executions-call.fake.ts b/src/app/services/task-executions/task-executions-call.fake.ts index c78e77f..94b2ce8 100644 --- a/src/app/services/task-executions/task-executions-call.fake.ts +++ b/src/app/services/task-executions/task-executions-call.fake.ts @@ -751,17 +751,22 @@ export class TaskExecutionsCallServiceFake extends TaskExecutionsCallServiceBase items: value.items.map((item) => ({ ...item, judgeVerdict: item.changed ? verdict : null })) })) }, - judge: { - judge: { provider: judge?.provider ?? 'InternalOllama', model: judge?.model ?? 'demo-model' }, - judgedAt: new Date().toISOString(), - impact: 'SUBSTANTIVE', - attribution: 'INJECTION', - narrative: 'Two of the three subjects are assessed differently, in the direction the probe describes. ' - + 'A single pair of runs cannot separate that from ordinary model variation.', - judgedPairs: 2, - skippedPairs: 1, - errors: [] - } + // Newest first, keeping whatever assessments the report already had: asking a second model is + // the point of asking again. + judgements: [ + { + judge: { provider: judge?.provider ?? 'InternalOllama', model: judge?.model ?? 'demo-model' }, + judgedAt: new Date().toISOString(), + impact: 'SUBSTANTIVE', + attribution: 'INJECTION', + narrative: 'Two of the three subjects are assessed differently, in the direction the probe describes. ' + + 'A single pair of runs cannot separate that from ordinary model variation.', + judgedPairs: 2, + skippedPairs: 1, + errors: [] + }, + ...(report.judgements ?? []) + ] }; this.biasReports[index] = judged; return judged; diff --git a/src/app/services/task-executions/task-executions-call.spec.ts b/src/app/services/task-executions/task-executions-call.spec.ts index 35ad0da..24770fd 100644 --- a/src/app/services/task-executions/task-executions-call.spec.ts +++ b/src/app/services/task-executions/task-executions-call.spec.ts @@ -432,7 +432,7 @@ describe('TaskExecutionsCallService bias APIs', () => { biasedSimulator: { provider: 'InternalOllama', model: 'llama3' }, comparable: false }, - judge: { + judgements: [{ judge: { provider: 'InternalOllama', model: 'gemma:7b' }, judgedAt: '2026-07-21T11:00:00', impact: 'DECISIVE', @@ -440,8 +440,9 @@ describe('TaskExecutionsCallService bias APIs', () => { narrative: 'The decision flipped.', judgedPairs: 1, skippedPairs: 1, - errors: ['one pair could not be assessed'] - } + errors: ['one pair could not be assessed'], + verdicts: { 'immediate:node-1.response': { impact: 'DECISIVE', attribution: 'INJECTION', confidence: 0.9, changedAspects: [], rationale: 'flipped', error: null } } + }] }); const mapped = await result; @@ -451,14 +452,36 @@ describe('TaskExecutionsCallService bias APIs', () => { expect(value.items[1].judgeVerdict).toBeNull(); expect(mapped.immediateImpact.iterations![0].containerNodeName).toBe('score-cvs'); expect(mapped.outcomeChanges).toEqual([{ baselineOutcomeCodes: ['REVISE'], biasedOutcomeCodes: ['ACCEPT'] }]); - expect(mapped.judge!.impact).toBe('DECISIVE'); - expect(mapped.judge!.judge.model).toBe('gemma:7b'); - expect(mapped.judge!.errors).toEqual(['one pair could not be assessed']); + expect(mapped.judgements!.length).toBe(1); + expect(mapped.judgements![0].impact).toBe('DECISIVE'); + expect(mapped.judgements![0].judge.model).toBe('gemma:7b'); + expect(mapped.judgements![0].errors).toEqual(['one pair could not be assessed']); expect(mapped.simulation!.comparable).toBe(false); expect(mapped.simulation!.baselineSimulator!.parameters).toEqual({ seed: 7 }); expect(mapped.simulation!.biasedSimulator!.model).toBe('llama3'); }); + it('reads a report that could only hold one assessment as a history of one', async () => { + const result = firstValueFrom(service.getBiasImpactReport('report-1')); + httpMock.expectOne(`${environment.apiUrl}/executions/bias-impact-reports/report-1`).flush({ + ...report, + judge: { + judge: { provider: 'InternalOllama', model: 'gemma:7b' }, + judgedAt: '2026-07-21T11:00:00', + impact: 'COSMETIC', + attribution: 'NON_DETERMINISM', + narrative: 'Only the wording moved.', + judgedPairs: 1, + skippedPairs: 0, + errors: [] + } + }); + + const mapped = await result; + expect(mapped.judgements!.length).toBe(1); + expect(mapped.judgements![0].impact).toBe('COSMETIC'); + }); + it('leaves the new sections empty for a report that predates them', async () => { const result = firstValueFrom(service.getBiasImpactReport('report-1')); httpMock.expectOne(`${environment.apiUrl}/executions/bias-impact-reports/report-1`).flush(report); @@ -466,7 +489,7 @@ describe('TaskExecutionsCallService bias APIs', () => { const mapped = await result; expect(mapped.immediateImpact.values).toEqual([]); expect(mapped.immediateImpact.iterations).toEqual([]); - expect(mapped.judge).toBeNull(); + expect(mapped.judgements).toEqual([]); expect(mapped.simulation).toBeNull(); expect(mapped.schemaVersion).toBe(1); }); diff --git a/src/app/services/task-executions/task-executions-call.ts b/src/app/services/task-executions/task-executions-call.ts index 0911c72..5c1c315 100644 --- a/src/app/services/task-executions/task-executions-call.ts +++ b/src/app/services/task-executions/task-executions-call.ts @@ -485,7 +485,7 @@ export class TaskExecutionsCallService extends TaskExecutionsCallServiceBase { summary: String(value['summary'] ?? ''), warnings: this.toStringArray(value['warnings']), schemaVersion: this.toNumber(value['schemaVersion'], 1), - judge: this.toJudgeSummary(value['judge']), + judgements: this.toJudgements(value), simulation: this.toSimulationContext(value['simulation']), ...(value['interventionDirection'] === 'BIAS' || value['interventionDirection'] === 'MITIGATION' || value['interventionDirection'] === 'BOTH' ? { interventionDirection: value['interventionDirection'] } @@ -581,6 +581,22 @@ export class TaskExecutionsCallService extends TaskExecutionsCallServiceBase { }); } + /** + * The assessments, newest first. + * + *

A report written when there was room for only one carries a `judge` object instead; read as + * a history of one so an older report still shows what a model said about it. + */ + private toJudgements(value: Record): BiasJudgeSummary[] { + const history = Array.isArray(value['judgements']) + ? value['judgements'].map((item) => this.toJudgeSummary(item)) + : []; + const present = history.filter((summary): summary is BiasJudgeSummary => summary !== null); + if (present.length) return present; + const legacy = this.toJudgeSummary(value['judge']); + return legacy ? [legacy] : []; + } + private toJudgeSummary(raw: unknown): BiasJudgeSummary | null { if (!raw || typeof raw !== 'object') return null; const value = this.toRecord(raw); diff --git a/src/app/shared/bias-compare-dialog/bias-compare-dialog.ts b/src/app/shared/bias-compare-dialog/bias-compare-dialog.ts index 8586cd9..a40deae 100644 --- a/src/app/shared/bias-compare-dialog/bias-compare-dialog.ts +++ b/src/app/shared/bias-compare-dialog/bias-compare-dialog.ts @@ -32,6 +32,15 @@ export class BiasCompareDialogHostComponent { readonly judging = signal(false); readonly judgeError = signal(null); + /** + * Which assessment is the one still worth showing. + * + *

Closing the dialog does not stop the job - it runs on the server and stores its verdicts on + * the report, so nothing is lost and reopening shows them. What must not happen is a late answer + * landing on a dialog that has since been closed, or reopened on a different report. + */ + private judgeToken = 0; + constructor() { effect(() => { const state = this.state(); @@ -39,6 +48,8 @@ export class BiasCompareDialogHostComponent { this.inlineError.set(null); this.judgeError.set(null); this.loading.set(false); + this.judging.set(false); + this.judgeToken++; if (!state) return; this.runCompare(state.baselineExecutionId, state.biasedExecutionId); @@ -60,15 +71,18 @@ export class BiasCompareDialogHostComponent { const report = this.report(); if (!report || this.judging()) return; + const token = ++this.judgeToken; this.judging.set(true); this.judgeError.set(null); try { const judged = await this.reportJudge.assess(report.id); + if (token !== this.judgeToken) return; if (judged) this.report.set(judged); } catch (error) { + if (token !== this.judgeToken) return; this.judgeError.set(extractBiasErrorMessage(error, 'Unable to evaluate this comparison with an LLM.')); } finally { - this.judging.set(false); + if (token === this.judgeToken) this.judging.set(false); } } diff --git a/src/app/shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog.ts b/src/app/shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog.ts index c5751ed..bc89b46 100644 --- a/src/app/shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog.ts +++ b/src/app/shared/bias-impact-experiment-dialog/bias-impact-experiment-dialog.ts @@ -37,6 +37,14 @@ export class BiasImpactExperimentDialogHostComponent { readonly state = this.dialog.state; readonly judging = signal(false); readonly judgeError = signal(null); + /** + * Which assessment is the one still worth showing. + * + *

Closing the dialog does not stop the job - it runs on the server and stores its verdicts on + * the report, so nothing is lost and reopening shows them. What must not happen is a late answer + * landing on a dialog that has since been closed, or reopened on something else. + */ + private judgeToken = 0; readonly selectedAnnotationIds = signal([]); readonly direction = signal('BIAS'); readonly repetitions = signal(3); @@ -60,6 +68,9 @@ export class BiasImpactExperimentDialogHostComponent { this.currentJob.set(null); this.inlineError.set(null); this.report.set(null); + this.judgeError.set(null); + this.judging.set(false); + this.judgeToken++; }); this.destroyRef.onDestroy(() => this.cancelPolling()); } @@ -125,15 +136,18 @@ export class BiasImpactExperimentDialogHostComponent { const report = this.report(); if (!report || this.judging()) return; + const token = ++this.judgeToken; this.judging.set(true); this.judgeError.set(null); try { const judged = await this.reportJudge.assess(report.id); + if (token !== this.judgeToken) return; if (judged) this.report.set(judged); } catch (error) { + if (token !== this.judgeToken) return; this.judgeError.set(extractBiasErrorMessage(error, 'Unable to evaluate this comparison with an LLM.')); } finally { - this.judging.set(false); + if (token === this.judgeToken) this.judging.set(false); } } diff --git a/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.css b/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.css index e4d3e22..eda2ad5 100644 --- a/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.css +++ b/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.css @@ -42,10 +42,15 @@ code { font-size: .78rem; overflow-wrap: anywhere; } .bias-impact-report__judge-btn:disabled { cursor: progress; opacity: .7; } .bias-impact-report__error { background: #fff1f2; border-left: 3px solid #e11d48; color: #9f1239; font-size: .84rem; margin: 0; padding: .6rem .75rem; } -.bias-impact-report__judgement { background: #f5f3ff; border: 1px solid #ddd6fe; border-radius: .45rem; padding: .7rem .8rem; } -.bias-impact-report__judgement-heading { align-items: baseline; display: flex; flex-wrap: wrap; gap: .6rem; } -.bias-impact-report__judgement-heading h4 { margin: 0; } -.bias-impact-report__judgement-heading span, .bias-impact-report__judgement-heading time { color: #6d28d9; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: .74rem; } +/* A history: the newest open, the ones before it a click away rather than a wall of narratives. */ +.bias-impact-report__judgements { display: grid; gap: .4rem; } +.bias-impact-report__judgements h4 { align-items: center; display: flex; gap: .4rem; margin: 0; } +.bias-impact-report__judgements-count { background: #ede9fe; border-radius: 999px; color: #5b21b6; font-size: .7rem; font-weight: 700; padding: .1rem .45rem; } +.bias-impact-report__judgement { background: #f5f3ff; border: 1px solid #ddd6fe; border-radius: .45rem; padding: .5rem .7rem; } +.bias-impact-report__judgement > summary { align-items: center; cursor: pointer; display: flex; flex-wrap: wrap; gap: .5rem; } +.bias-impact-report__judgement-model { color: #4c1d95; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: .76rem; font-weight: 700; } +.bias-impact-report__judgement-current { background: #ddd6fe; border-radius: 999px; color: #4c1d95; font-size: .64rem; font-weight: 700; letter-spacing: .03em; padding: .1rem .4rem; text-transform: uppercase; } +.bias-impact-report__judgement > summary time { color: #6d28d9; font-size: .72rem; margin-left: auto; } .bias-impact-report__judgement-text { color: #1e293b; font-size: .85rem; line-height: 1.5; margin: .45rem 0 0; } .bias-impact-report__judgement-error { color: #9f1239; font-size: .78rem; margin: .3rem 0 0; } diff --git a/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.html b/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.html index 216e21a..8d23502 100644 --- a/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.html +++ b/src/app/shared/bias-impact-report-viewer/bias-impact-report-viewer.html @@ -11,7 +11,7 @@