diff --git a/src/app/services/dialogs/human-interaction-dialog.ts b/src/app/services/dialogs/human-interaction-dialog.ts index bbbdf80..6c0e7a4 100644 --- a/src/app/services/dialogs/human-interaction-dialog.ts +++ b/src/app/services/dialogs/human-interaction-dialog.ts @@ -16,6 +16,13 @@ export type HumanDecisionOption = { label: string; }; +/** One thing a tester is asked to judge, as the evaluation form renders it. */ +export type EvaluationCriterionView = { + name: string; + description: string; + scale: 'PASS_FAIL' | 'SCORE_1_5'; +}; + export type HumanInteractionRuntimeInput = { name: string; value: string; @@ -45,6 +52,21 @@ export type HumanInteractionDialogInput = { decisionOptions?: HumanDecisionOption[]; rationaleRequired?: boolean; rationaleLabel?: string; + /** Evaluation form: what to open, the script to follow, and what to score. */ + evaluationTarget?: string; + taskScript?: string[]; + criteria?: EvaluationCriterionView[]; + evidenceRequired?: boolean; + evidenceCount?: number; + /** + * A reference verdict the node is keeping out of sight until this evaluation is submitted. The + * point is not secrecy - it is that a judgement made after reading it is a different measurement. + */ + referenceLabel?: string | null; + referenceValue?: string | null; + referenceHidden?: boolean; + onRevealReference?: () => void; + onAttachEvidence?: (files: File[]) => void; /** Flat substitution map for `${{name}}` / `${{global.name}}` / `${{vars.name}}` placeholders in `question`/`actionDescription`. */ templateValues?: Record; onSubmit?: (value: HumanInteractionDialogResult) => void; @@ -52,7 +74,8 @@ export type HumanInteractionDialogInput = { export type HumanInteractionDialogResult = | { mode: 'message' | 'complete'; value: string } - | { mode: 'decision'; choice: string; rationale: string }; + | { mode: 'decision'; choice: string; rationale: string } + | { mode: 'evaluation'; verdict: Record; notes: string }; export type HumanInteractionDialogState = { executionId: string | null; @@ -78,6 +101,16 @@ export type HumanInteractionDialogState = { decisionOptions: HumanDecisionOption[]; rationaleRequired: boolean; rationaleLabel: string; + evaluationTarget: string; + taskScript: string[]; + criteria: EvaluationCriterionView[]; + evidenceRequired: boolean; + evidenceCount: number; + referenceLabel: string | null; + referenceValue: string | null; + referenceHidden: boolean; + onRevealReference: (() => void) | null; + onAttachEvidence: ((files: File[]) => void) | null; templateValues: Record; onSubmit: ((value: HumanInteractionDialogResult) => void) | null; resolve: (value: HumanInteractionDialogResult | null) => void; @@ -115,6 +148,16 @@ export class HumanInteractionDialogService { decisionOptions: input.decisionOptions ?? [], rationaleRequired: input.rationaleRequired === true, rationaleLabel: input.rationaleLabel ?? 'Rationale', + evaluationTarget: input.evaluationTarget ?? '', + taskScript: input.taskScript ?? [], + criteria: input.criteria ?? [], + evidenceRequired: input.evidenceRequired === true, + evidenceCount: input.evidenceCount ?? 0, + referenceLabel: input.referenceLabel ?? null, + referenceValue: input.referenceValue ?? null, + referenceHidden: input.referenceHidden === true, + onRevealReference: input.onRevealReference ?? null, + onAttachEvidence: input.onAttachEvidence ?? null, templateValues: input.templateValues ?? {}, onSubmit: input.onSubmit ?? null, resolve @@ -153,6 +196,16 @@ export class HumanInteractionDialogService { decisionOptions: input.decisionOptions ?? state.decisionOptions, rationaleRequired: input.rationaleRequired ?? state.rationaleRequired, rationaleLabel: input.rationaleLabel ?? state.rationaleLabel, + evaluationTarget: input.evaluationTarget ?? state.evaluationTarget, + taskScript: input.taskScript ?? state.taskScript, + criteria: input.criteria ?? state.criteria, + evidenceRequired: input.evidenceRequired ?? state.evidenceRequired, + evidenceCount: input.evidenceCount ?? state.evidenceCount, + referenceLabel: input.referenceLabel !== undefined ? input.referenceLabel : state.referenceLabel, + referenceValue: input.referenceValue !== undefined ? input.referenceValue : state.referenceValue, + referenceHidden: input.referenceHidden ?? state.referenceHidden, + onRevealReference: input.onRevealReference ?? state.onRevealReference, + onAttachEvidence: input.onAttachEvidence ?? state.onAttachEvidence, templateValues: input.templateValues ?? state.templateValues, onSubmit: input.onSubmit !== undefined ? input.onSubmit : state.onSubmit }); diff --git a/src/app/services/task-executions/task-executions-call.base.ts b/src/app/services/task-executions/task-executions-call.base.ts index 8e6cfff..45379c7 100644 --- a/src/app/services/task-executions/task-executions-call.base.ts +++ b/src/app/services/task-executions/task-executions-call.base.ts @@ -102,6 +102,26 @@ export abstract class TaskExecutionsCallServiceBase { fieldName: string, value: string ): Observable; + /** + * Submits a human evaluation whole - every criterion and the notes in one call - because the + * fields are one judgement, and half of one recorded on the step is worse than none. + */ + abstract submitEvaluation( + executionId: string, + nodeId: string, + verdict: Record, + notes: string + ): Observable; + abstract attachEvaluationEvidence( + executionId: string, + nodeId: string, + files: File[] + ): Observable; + /** Asks to see the reference verdict; the execution records that it was seen before judging. */ + abstract revealEvaluationReference( + executionId: string, + nodeId: string + ): Observable; abstract provideAuthorization( executionId: string, key: string, diff --git a/src/app/services/task-executions/task-executions-call.fake.ts b/src/app/services/task-executions/task-executions-call.fake.ts index 7eb5ba7..55b9eef 100644 --- a/src/app/services/task-executions/task-executions-call.fake.ts +++ b/src/app/services/task-executions/task-executions-call.fake.ts @@ -1005,6 +1005,46 @@ export class TaskExecutionsCallServiceFake extends TaskExecutionsCallServiceBase return of(execution); } + override submitEvaluation( + executionId: string, + nodeId: string, + verdict: Record, + notes: string + ): Observable { + const execution = this.findExecution(executionId); + const step = execution.context.steps[nodeId]; + execution.context.result[`${nodeId}:verdict`] = verdict; + execution.context.result[`${nodeId}:notes`] = notes; + execution.context.waitingSteps = execution.context.waitingSteps.filter((stepId) => stepId !== nodeId); + if (step) step.status = 'COMPLETED'; + execution.context.status = execution.context.waitingSteps.length ? 'WAITING' : 'RUNNING'; + return of(execution); + } + + override attachEvaluationEvidence( + executionId: string, + nodeId: string, + files: File[] + ): Observable { + const execution = this.findExecution(executionId); + const key = `${nodeId}:evidence`; + const existing = execution.context.partialResult?.[key]; + execution.context.partialResult = { + ...(execution.context.partialResult ?? {}), + [key]: [...(Array.isArray(existing) ? existing : []), ...files.map((file) => file.name)] + }; + return of(execution); + } + + override revealEvaluationReference(executionId: string, nodeId: string): Observable { + const execution = this.findExecution(executionId); + execution.context.partialResult = { + ...(execution.context.partialResult ?? {}), + [`${nodeId}:__referenceRevealed`]: true + }; + return of(execution); + } + override provideAuthorization( executionId: string, key: string, diff --git a/src/app/services/task-executions/task-executions-call.ts b/src/app/services/task-executions/task-executions-call.ts index 3bc0510..40b4cfe 100644 --- a/src/app/services/task-executions/task-executions-call.ts +++ b/src/app/services/task-executions/task-executions-call.ts @@ -273,6 +273,34 @@ export class TaskExecutionsCallService extends TaskExecutionsCallServiceBase { }).pipe(map((raw) => this.mapExecution(raw))); } + override submitEvaluation( + executionId: string, + nodeId: string, + verdict: Record, + notes: string + ): Observable { + const url = `${environment.apiUrl}/executions/${encodeURIComponent(executionId)}/node/${encodeURIComponent(nodeId)}/evaluation`; + return this.http.put(url, { verdict, notes }).pipe(map((raw) => this.mapExecution(raw))); + } + + override attachEvaluationEvidence( + executionId: string, + nodeId: string, + files: File[] + ): Observable { + const url = `${environment.apiUrl}/executions/${encodeURIComponent(executionId)}/node/${encodeURIComponent(nodeId)}/evaluation/evidence`; + const formData = new FormData(); + for (const file of files) { + formData.append('files', file); + } + return this.http.put(url, formData).pipe(map((raw) => this.mapExecution(raw))); + } + + override revealEvaluationReference(executionId: string, nodeId: string): Observable { + const url = `${environment.apiUrl}/executions/${encodeURIComponent(executionId)}/node/${encodeURIComponent(nodeId)}/evaluation/reveal`; + return this.http.put(url, {}).pipe(map((raw) => this.mapExecution(raw))); + } + override provideAuthorization( executionId: string, key: string, diff --git a/src/app/services/task-executions/task-executions.ts b/src/app/services/task-executions/task-executions.ts index 5cda7bd..8eafc23 100644 --- a/src/app/services/task-executions/task-executions.ts +++ b/src/app/services/task-executions/task-executions.ts @@ -360,6 +360,46 @@ export class TaskExecutionsService { ); } + submitEvaluation(executionId: string, nodeId: string, verdict: Record, notes: string) { + const execution = this._taskExecutions().find((item) => item.id === executionId); + if (execution?.interactionSimulationEnabled === true) { + return throwError(() => new Error('Manual interaction is disabled for simulated executions.')); + } + + return this.withRefreshAndErrorHandling( + this.taskExecutionsCallService.submitEvaluation(executionId, nodeId, verdict, notes).pipe( + tap((updatedExecution) => { + if (updatedExecution.executionKind === 'SUBFLOW') { + this.cacheFollowedExecution(updatedExecution); + } else { + this.replaceExecution(updatedExecution); + } + }) + ), + 'Submit evaluation failed' + ); + } + + attachEvaluationEvidence(executionId: string, nodeId: string, files: File[]) { + return this.withRefreshAndErrorHandling( + this.taskExecutionsCallService.attachEvaluationEvidence(executionId, nodeId, files).pipe( + tap((execution) => this.replaceExecution(execution)) + ), + 'Attach evaluation evidence failed', + false + ); + } + + revealEvaluationReference(executionId: string, nodeId: string) { + return this.withRefreshAndErrorHandling( + this.taskExecutionsCallService.revealEvaluationReference(executionId, nodeId).pipe( + tap((execution) => this.replaceExecution(execution)) + ), + 'Reveal reference verdict failed', + false + ); + } + provideAuthorization(executionId: string, key: string, value: string) { return this.withRefreshAndErrorHandling( this.taskExecutionsCallService.provideAuthorization(executionId, key, value).pipe( diff --git a/src/app/shared/human-decision-interaction/human-decision-interaction.spec.ts b/src/app/shared/human-decision-interaction/human-decision-interaction.spec.ts index bda8fc4..b83de1c 100644 --- a/src/app/shared/human-decision-interaction/human-decision-interaction.spec.ts +++ b/src/app/shared/human-decision-interaction/human-decision-interaction.spec.ts @@ -34,6 +34,16 @@ function state(overrides: Partial = {}): HumanInter ], rationaleRequired: true, rationaleLabel: 'Evidence-based rationale', + evaluationTarget: '', + taskScript: [], + criteria: [], + evidenceRequired: false, + evidenceCount: 0, + referenceLabel: null, + referenceValue: null, + referenceHidden: false, + onRevealReference: null, + onAttachEvidence: null, templateValues: {}, onSubmit: null, resolve: () => undefined, diff --git a/src/app/shared/human-evaluation-interaction/human-evaluation-interaction.css b/src/app/shared/human-evaluation-interaction/human-evaluation-interaction.css new file mode 100644 index 0000000..edf98a3 --- /dev/null +++ b/src/app/shared/human-evaluation-interaction/human-evaluation-interaction.css @@ -0,0 +1,34 @@ +/* + * SPDX-FileCopyrightText: 2025-2026 Lucio Lelii - ISTI-CNR + * SPDX-License-Identifier: AGPL-3.0-or-later + * Attribution term under AGPL-3.0 section 7(b): see LICENSE-ADDENDUM. + */ + +/* Wide enough to hit without aiming, since a tester scores several of these in a row. */ +.evaluation-score { + min-width: 56px; + padding: 7px 14px; + border: 1px solid #cbd5e1; + border-radius: 999px; + background: #fff; + color: #1e293b; + font-size: 0.875rem; + cursor: pointer; +} + +.evaluation-score:hover:not(:disabled) { + border-color: #64748b; + background: #f8fafc; +} + +.evaluation-score-selected { + border-color: #2563eb; + background: #eff6ff; + color: #1d4ed8; + font-weight: 600; +} + +.evaluation-score:disabled { + cursor: not-allowed; + opacity: 0.65; +} diff --git a/src/app/shared/human-evaluation-interaction/human-evaluation-interaction.html b/src/app/shared/human-evaluation-interaction/human-evaluation-interaction.html new file mode 100644 index 0000000..fc09879 --- /dev/null +++ b/src/app/shared/human-evaluation-interaction/human-evaluation-interaction.html @@ -0,0 +1,137 @@ + + +
+ @if (state.evaluationTarget) { +
+
What to evaluate
+
+ +
+
+ } + + @if (state.taskScript.length) { +
+ Steps to perform +
    + @for (step of state.taskScript; track $index) { +
  1. {{ step }}
    2. + } +
+
+ } + + @if (state.referenceLabel) { +
+ Reference verdict + @if (state.referenceHidden) { +
+

+ Hidden until you submit, so your judgement is your own. Opening it is recorded with the result. +

+
+ +
+
+ } @else { +
{{ state.referenceValue || '-' }}
+ } +
+ } + + @if (state.runtimeInputs.length) { +
+ Runtime inputs +
+ @for (input of state.runtimeInputs; track input.name) { +
+
{{ input.name }}
+
{{ input.value || '-' }}
+
+ } +
+
+ } + +
+ Your verdict + @for (criterion of state.criteria; track criterion.name) { +
+
{{ criterion.name }}
+ @if (criterion.description) { +
{{ criterion.description }}
+ } +
+ @for (value of valuesFor(criterion.scale); track value) { + + } +
+
+ } +
+ + @if (!state.criteria.length) { +
+ This evaluation defines no criteria, so there is nothing to score. +
+ } + +
+ + Evidence{{ state.evidenceRequired ? ' *' : '' }} + +
+ + + {{ state.evidenceCount }} attached + +
+ @if (state.evidenceRequired && !state.evidenceCount) { +

This evaluation needs at least one attachment.

+ } +
+ + @if (state.submitError) { +
+ {{ state.submitError }} +
+ } + + + Notes + + + +
+ @if (missingCriteria().length) { + Still to score: {{ missingCriteria().join(', ') }} + } @else { + + } + +
+
diff --git a/src/app/shared/human-evaluation-interaction/human-evaluation-interaction.spec.ts b/src/app/shared/human-evaluation-interaction/human-evaluation-interaction.spec.ts new file mode 100644 index 0000000..85ef553 --- /dev/null +++ b/src/app/shared/human-evaluation-interaction/human-evaluation-interaction.spec.ts @@ -0,0 +1,158 @@ +// SPDX-FileCopyrightText: 2025-2026 Lucio Lelii - ISTI-CNR +// SPDX-License-Identifier: AGPL-3.0-or-later +// Attribution term under AGPL-3.0 section 7(b): see LICENSE-ADDENDUM. + +import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { HumanInteractionDialogState } from '@services/dialogs/human-interaction-dialog'; +import { vi } from 'vitest'; +import { HumanEvaluationInteractionComponent } from './human-evaluation-interaction'; + +function state(overrides: Partial = {}): HumanInteractionDialogState { + return { + executionId: 'execution-1', + nodeId: 'evaluation-1', + title: 'Evaluation', + kind: 'evaluation-form', + actionDescription: '', + currentInput: '', + runtimeInputs: [{ name: 'deployUrl', value: 'https://staging.example' }], + history: [], + latestResponse: '', + historyField: null, + responseField: 'verdict', + messageField: 'notes', + completionField: 'verdict', + pendingUserMessage: null, + awaitingAssistantResponse: false, + assistantResponseBaseline: '', + isRunning: false, + isSubmitting: false, + submitError: null, + question: '', + decisionOptions: [], + rationaleRequired: false, + rationaleLabel: 'Rationale', + evaluationTarget: 'Open ${{deployUrl}}', + taskScript: ['Sign in', 'Create an order'], + criteria: [ + { name: 'works', description: 'Does the order go through?', scale: 'PASS_FAIL' }, + { name: 'clarity', description: 'How clear are the errors?', scale: 'SCORE_1_5' } + ], + evidenceRequired: false, + evidenceCount: 0, + referenceLabel: null, + referenceValue: null, + referenceHidden: false, + onRevealReference: null, + onAttachEvidence: null, + templateValues: {}, + onSubmit: null, + resolve: () => undefined, + ...overrides + }; +} + +describe('HumanEvaluationInteractionComponent', () => { + let fixture: ComponentFixture; + let component: HumanEvaluationInteractionComponent; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + imports: [HumanEvaluationInteractionComponent] + }).compileComponents(); + fixture = TestBed.createComponent(HumanEvaluationInteractionComponent); + component = fixture.componentInstance; + fixture.componentRef.setInput('state', state()); + fixture.detectChanges(); + }); + + it('will not submit until every criterion has a verdict', () => { + // A half-scored form is not a smaller judgement, it is an unfinished one. + expect(component.canSubmit()).toBe(false); + component.score('works', 'pass'); + expect(component.canSubmit()).toBe(false); + component.score('clarity', '4'); + expect(component.canSubmit()).toBe(true); + }); + + it('offers each criterion the values its own scale allows', () => { + expect(component.valuesFor('PASS_FAIL')).toEqual(['pass', 'fail']); + expect(component.valuesFor('SCORE_1_5')).toEqual(['1', '2', '3', '4', '5']); + }); + + it('emits one verdict per criterion, keyed by name', () => { + const emitted: unknown[] = []; + component.submitEvaluation.subscribe((value) => emitted.push(value)); + component.score('works', 'fail'); + component.score('clarity', '2'); + component.notes = ' Checkout throws on empty cart '; + component.submit(); + + expect(emitted).toEqual([{ + mode: 'evaluation', + verdict: { works: 'fail', clarity: '2' }, + notes: 'Checkout throws on empty cart' + }]); + }); + + it('names what is still unscored, rather than only refusing', () => { + component.score('works', 'pass'); + expect(component.missingCriteria()).toEqual(['clarity']); + }); + + it('refuses a verdict with no attachment when the node requires evidence', () => { + fixture.componentRef.setInput('state', state({ evidenceRequired: true })); + fixture.detectChanges(); + component.score('works', 'pass'); + component.score('clarity', '3'); + + expect(component.canSubmit()).toBe(false); + + fixture.componentRef.setInput('state', state({ evidenceRequired: true, evidenceCount: 1 })); + fixture.detectChanges(); + expect(component.canSubmit()).toBe(true); + }); + + it('keeps the reference verdict out of sight until it is asked for', () => { + const onRevealReference = vi.fn(); + fixture.componentRef.setInput('state', state({ + referenceLabel: 'agentReport', + referenceValue: null, + referenceHidden: true, + onRevealReference + })); + fixture.detectChanges(); + + expect(fixture.nativeElement.textContent).not.toContain('All good, shipped'); + component.revealReference(); + expect(onRevealReference).toHaveBeenCalledOnce(); + }); + + it('shows the reference once revealed', () => { + fixture.componentRef.setInput('state', state({ + referenceLabel: 'agentReport', + referenceValue: 'All good, shipped', + referenceHidden: false + })); + fixture.detectChanges(); + + expect(fixture.nativeElement.textContent).toContain('All good, shipped'); + }); + + it('keeps the scores already given when an error updates the dialog state', () => { + component.score('works', 'pass'); + fixture.componentRef.setInput('state', state({ submitError: 'Network error' })); + fixture.detectChanges(); + + expect(component.isScored('works', 'pass')).toBe(true); + expect(fixture.nativeElement.textContent).toContain('Network error'); + }); + + it('scores nothing while the step is submitting', () => { + fixture.componentRef.setInput('state', state({ isSubmitting: true })); + fixture.detectChanges(); + component.score('works', 'pass'); + + expect(component.isScored('works', 'pass')).toBe(false); + }); +}); diff --git a/src/app/shared/human-evaluation-interaction/human-evaluation-interaction.ts b/src/app/shared/human-evaluation-interaction/human-evaluation-interaction.ts new file mode 100644 index 0000000..aaf159b --- /dev/null +++ b/src/app/shared/human-evaluation-interaction/human-evaluation-interaction.ts @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: 2025-2026 Lucio Lelii - ISTI-CNR +// SPDX-License-Identifier: AGPL-3.0-or-later +// Attribution term under AGPL-3.0 section 7(b): see LICENSE-ADDENDUM. + +import { ChangeDetectionStrategy, Component, EventEmitter, Input, Output } from '@angular/core'; +import { FormsModule } from '@angular/forms'; +import { MatButtonModule } from '@angular/material/button'; +import { MatFormFieldModule } from '@angular/material/form-field'; +import { MatInputModule } from '@angular/material/input'; +import { + HumanInteractionDialogResult, + HumanInteractionDialogState +} from '@services/dialogs/human-interaction-dialog'; +import { TemplatePlaceholderTextComponent } from '@shared/template-placeholder-text/template-placeholder-text'; + +/** The scores a PASS_FAIL and a SCORE_1_5 criterion offer, in the order they are shown. */ +const SCALE_VALUES: Record = { + PASS_FAIL: ['pass', 'fail'], + SCORE_1_5: ['1', '2', '3', '4', '5'] +}; + +@Component({ + selector: 'app-human-evaluation-interaction', + standalone: true, + imports: [FormsModule, MatButtonModule, MatFormFieldModule, MatInputModule, TemplatePlaceholderTextComponent], + templateUrl: './human-evaluation-interaction.html', + styleUrl: './human-evaluation-interaction.css', + changeDetection: ChangeDetectionStrategy.OnPush +}) +export class HumanEvaluationInteractionComponent { + @Input({ required: true }) state!: HumanInteractionDialogState; + @Output() submitEvaluation = new EventEmitter>(); + + verdict: Record = {}; + notes = ''; + + valuesFor(scale: string): string[] { + return SCALE_VALUES[scale] ?? SCALE_VALUES['PASS_FAIL']; + } + + score(criterion: string, value: string) { + if (this.busy()) return; + this.verdict = { ...this.verdict, [criterion]: value }; + } + + isScored(criterion: string, value: string): boolean { + return this.verdict[criterion] === value; + } + + busy(): boolean { + return this.state.isSubmitting || this.state.isRunning; + } + + /** + * Revealing is a one-way act on purpose: the node records that this judgement was made after + * seeing the other one, and taking it back would make that record a lie. + */ + revealReference(event?: Event) { + event?.preventDefault(); + event?.stopPropagation(); + if (this.busy()) return; + this.state.onRevealReference?.(); + } + + attachEvidence(event: Event) { + const input = event.target as HTMLInputElement; + const files = Array.from(input.files ?? []); + if (files.length) { + this.state.onAttachEvidence?.(files); + } + input.value = ''; + } + + missingCriteria(): string[] { + return this.state.criteria + .filter((criterion) => !this.verdict[criterion.name]) + .map((criterion) => criterion.name); + } + + canSubmit(): boolean { + if (this.busy()) return false; + if (!this.state.criteria.length) return false; + if (this.missingCriteria().length) return false; + return !this.state.evidenceRequired || this.state.evidenceCount > 0; + } + + submit(event?: Event) { + event?.preventDefault(); + event?.stopPropagation(); + if (!this.canSubmit()) return; + this.submitEvaluation.emit({ + mode: 'evaluation', + verdict: { ...this.verdict }, + notes: this.notes.trim() + }); + } +} diff --git a/src/app/shared/human-interaction-dialog/human-interaction-dialog.html b/src/app/shared/human-interaction-dialog/human-interaction-dialog.html index a53026e..93748e3 100644 --- a/src/app/shared/human-interaction-dialog/human-interaction-dialog.html +++ b/src/app/shared/human-interaction-dialog/human-interaction-dialog.html @@ -19,6 +19,8 @@ Select one option and document the rationale for the decision. } @else if (currentState.kind === 'single-response') { Provide the requested response to complete this activity. + } @else if (currentState.kind === 'evaluation-form') { + Exercise the target, then score every criterion. } @else { This interaction type is not supported. } @@ -113,6 +115,12 @@ [state]="currentState" (submitResponse)="submitTextResponse($event)" /> + } @else if (currentState.kind === 'evaluation-form') { +
+ +
} @else {
Unsupported interaction contract: {{ currentState.kind }} diff --git a/src/app/shared/human-interaction-dialog/human-interaction-dialog.ts b/src/app/shared/human-interaction-dialog/human-interaction-dialog.ts index ccab122..54da116 100644 --- a/src/app/shared/human-interaction-dialog/human-interaction-dialog.ts +++ b/src/app/shared/human-interaction-dialog/human-interaction-dialog.ts @@ -12,6 +12,7 @@ import { HumanInteractionDialogService } from '@services/dialogs/human-interaction-dialog'; import { HumanDecisionInteractionComponent } from '@shared/human-decision-interaction/human-decision-interaction'; +import { HumanEvaluationInteractionComponent } from '@shared/human-evaluation-interaction/human-evaluation-interaction'; import { HumanTextInteractionComponent } from '@shared/human-text-interaction/human-text-interaction'; @Component({ @@ -23,6 +24,7 @@ import { HumanTextInteractionComponent } from '@shared/human-text-interaction/hu MatFormFieldModule, MatInputModule, HumanDecisionInteractionComponent, + HumanEvaluationInteractionComponent, HumanTextInteractionComponent ], templateUrl: './human-interaction-dialog.html', @@ -136,6 +138,10 @@ export class HumanInteractionDialogHostComponent { this.dialog.submit(result); } + submitHumanEvaluation(result: Extract) { + this.dialog.submit(result); + } + canSendEditedOutput(): boolean { const state = this.state(); if (state?.isSubmitting || state?.isRunning) return false; diff --git a/src/app/shared/nodes/generic-node/generic-node.spec.ts b/src/app/shared/nodes/generic-node/generic-node.spec.ts index d636f29..eb89ded 100644 --- a/src/app/shared/nodes/generic-node/generic-node.spec.ts +++ b/src/app/shared/nodes/generic-node/generic-node.spec.ts @@ -1372,4 +1372,15 @@ describe('GenericNodeComponent', () => { expect(summary(definition, { source: 'INPUT', name: 'planDoc', multiple: true }, 0)) .toBe('INPUT ยท planDoc'); }); + + it('reads back a plain-value row as its value, not as its position', () => { + // A task script is a list of strings: "Item 1 / Item 2" hid the very text being configured. + const definition = { path: 'taskScript', label: 'Steps', itemSchema: { type: 'string' }, uniqueBy: null } as any; + const summary = (component as any).toArrayItemSummary.bind(component); + + expect(summary(definition, 'Sign in with the test account', 0)).toBe('Sign in with the test account'); + expect(summary(definition, ' Create an order ', 1)).toBe('Create an order'); + // An empty string identifies nothing, so the position is all that is left to say. + expect(summary(definition, ' ', 2)).toBe('Item 3'); + }); }); diff --git a/src/app/shared/nodes/generic-node/generic-node.ts b/src/app/shared/nodes/generic-node/generic-node.ts index a46e7fa..8561ee3 100644 --- a/src/app/shared/nodes/generic-node/generic-node.ts +++ b/src/app/shared/nodes/generic-node/generic-node.ts @@ -1956,6 +1956,11 @@ export class GenericNodeComponent implements OnDestroy { } private toArrayItemSummary(definition: ArrayFieldDefinition, item: unknown, index: number) { + // An array of plain values - a task script, a list of names - is its own summary. Reading back + // "Item 1 / Item 2" where the row holds "Sign in" tells the reader nothing they can act on. + if (typeof item === 'string') return item.trim() || `Item ${index + 1}`; + if (typeof item === 'number' || typeof item === 'boolean') return String(item); + if (!item || typeof item !== 'object' || Array.isArray(item)) { return `Item ${index + 1}`; } diff --git a/src/app/shared/nodes/task-step-node/task-step-node.ts b/src/app/shared/nodes/task-step-node/task-step-node.ts index 78c9402..cadaa23 100644 --- a/src/app/shared/nodes/task-step-node/task-step-node.ts +++ b/src/app/shared/nodes/task-step-node/task-step-node.ts @@ -17,6 +17,7 @@ import { HumanDecisionOption, HumanInteractionDialogResult, HumanInteractionDialogService, + EvaluationCriterionView, HumanInteractionRuntimeInput } from '@services/dialogs/human-interaction-dialog'; import { TaskExecutionsService } from '@services/task-executions/task-executions'; @@ -131,6 +132,9 @@ export class TaskStepNodeComponent { name = 'Step'; mainContentFields: MainContentView[] = []; interactionSubmitting = false; + /** Per dialog session: the step holds the truth, these only drive what the form shows. */ + private evaluationEvidenceCount = 0; + private evaluationReferenceRevealed = false; schemaReady = false; biasCapabilities: BiasCapabilities | null = null; @@ -981,7 +985,6 @@ export class TaskStepNodeComponent { kind: contract.kind, actionDescription: this.actionDescriptionValue(), currentInput: this.currentInputValue(), - runtimeInputs: this.interactionRuntimeInputs(), history: this.chatHistory(contract), latestResponse: this.latestInteractionResponse(contract), historyField: contract.historyField, @@ -998,10 +1001,109 @@ export class TaskStepNodeComponent { decisionOptions: this.decisionOptions(), rationaleRequired: this.blockConfiguration?.['rationaleRequired'] === true, rationaleLabel: this.decisionRationaleLabel(), + evaluationTarget: this.evaluationTarget(), + taskScript: this.evaluationTaskScript(), + criteria: this.evaluationCriteria(), + evidenceRequired: this.blockConfiguration?.['evidenceRequired'] === true, + evidenceCount: 0, + referenceLabel: this.evaluationReferenceInput(), + referenceValue: this.evaluationReferenceValue(), + referenceHidden: this.isEvaluationReferenceHidden(), + runtimeInputs: this.evaluationVisibleRuntimeInputs(), + onRevealReference: () => this.revealEvaluationReference(executionId, executionNodeId), + onAttachEvidence: (files: File[]) => this.attachEvaluationEvidence(executionId, executionNodeId, files), templateValues: this.templateSubstitutions() }; } + /** What the node says to open, unresolved: the dialog renders its placeholders itself. */ + private evaluationTarget(): string { + const target = this.blockConfiguration?.['target']; + return typeof target === 'string' ? target : ''; + } + + private evaluationTaskScript(): string[] { + const script = this.blockConfiguration?.['taskScript']; + if (!Array.isArray(script)) return []; + return script.filter((step): step is string => typeof step === 'string' && step.trim().length > 0); + } + + private evaluationCriteria(): EvaluationCriterionView[] { + const criteria = this.blockConfiguration?.['criteria']; + if (!Array.isArray(criteria)) return []; + return criteria.flatMap((criterion) => { + if (!criterion || typeof criterion !== 'object' || Array.isArray(criterion)) return []; + const record = criterion as Record; + const name = typeof record['name'] === 'string' ? record['name'].trim() : ''; + if (!name) return []; + const description = typeof record['description'] === 'string' ? record['description'].trim() : ''; + const scale = record['scale'] === 'SCORE_1_5' ? 'SCORE_1_5' as const : 'PASS_FAIL' as const; + return [{ name, description, scale }]; + }); + } + + private evaluationReferenceInput(): string | null { + const reference = this.blockConfiguration?.['referenceInput']; + return typeof reference === 'string' && reference.trim().length > 0 ? reference.trim() : null; + } + + /** + * Hidden while the node asks for a blind judgement and nobody has asked to see it yet. The + * hiding is what makes the recorded flag mean something, so it is decided here rather than left + * to the form. + */ + private isEvaluationReferenceHidden(): boolean { + if (this.blockConfiguration?.['blindUntilSubmitted'] !== true) return false; + return !this.evaluationReferenceRevealed; + } + + private evaluationReferenceValue(): string | null { + const reference = this.evaluationReferenceInput(); + if (!reference || this.isEvaluationReferenceHidden()) return null; + return this.executionInputTooltip(reference) ?? null; + } + + /** The reference is pulled out of the ordinary inputs while it is meant to stay unseen. */ + private evaluationVisibleRuntimeInputs(): HumanInteractionRuntimeInput[] { + const inputs = this.interactionRuntimeInputs(); + const reference = this.evaluationReferenceInput(); + if (!reference || !this.isEvaluationReferenceHidden()) return inputs; + return inputs.filter((input) => input.name !== reference); + } + + private revealEvaluationReference(executionId: string, executionNodeId: string) { + this.taskExecutionsService.revealEvaluationReference(executionId, executionNodeId).subscribe({ + next: () => { + this.evaluationReferenceRevealed = true; + this.humanInteractionDialog.update({ + referenceHidden: false, + referenceValue: this.executionInputTooltip(this.evaluationReferenceInput() ?? '') ?? '', + runtimeInputs: this.interactionRuntimeInputs() + }); + }, + error: (error) => { + this.humanInteractionDialog.update({ submitError: 'Failed to reveal the reference verdict.' }); + console.error('Reveal reference verdict failed', error); + } + }); + } + + private attachEvaluationEvidence(executionId: string, executionNodeId: string, files: File[]) { + this.taskExecutionsService.attachEvaluationEvidence(executionId, executionNodeId, files).subscribe({ + next: () => { + this.evaluationEvidenceCount += files.length; + this.humanInteractionDialog.update({ + evidenceCount: this.evaluationEvidenceCount, + submitError: null + }); + }, + error: (error) => { + this.humanInteractionDialog.update({ submitError: 'Failed to attach the evidence.' }); + console.error('Attach evaluation evidence failed', error); + } + }); + } + private submitInteractionResult( executionId: string, executionNodeId: string, @@ -1018,6 +1120,10 @@ export class TaskStepNodeComponent { this.submitHumanDecision(executionId, executionNodeId, contract, result); return; } + if (result.mode === 'evaluation') { + this.submitHumanEvaluation(executionId, executionNodeId, result); + return; + } const interactionFieldName = result.mode === 'message' ? contract.messageField @@ -1060,6 +1166,44 @@ export class TaskStepNodeComponent { }); } + private submitHumanEvaluation( + executionId: string, + executionNodeId: string, + result: Extract + ) { + const scored = this.evaluationCriteria().filter((criterion) => !result.verdict[criterion.name]); + if (scored.length) { + this.humanInteractionDialog.update({ + submitError: `Score every criterion first: ${scored.map((criterion) => criterion.name).join(', ')}.` + }); + return; + } + + this.interactionSubmitting = true; + this.humanInteractionDialog.update({ isSubmitting: true, submitError: null }); + this.taskExecutionsService.submitEvaluation( + executionId, + executionNodeId, + result.verdict, + result.notes + ).subscribe({ + next: () => { + this.interactionSubmitting = false; + this.evaluationEvidenceCount = 0; + this.evaluationReferenceRevealed = false; + this.humanInteractionDialog.close(result); + }, + error: (error) => { + this.interactionSubmitting = false; + this.humanInteractionDialog.update({ + isSubmitting: false, + submitError: 'Failed to submit the evaluation.' + }); + console.error('Submit evaluation failed', error); + } + }); + } + private submitHumanDecision( executionId: string, executionNodeId: string,