feat: support provider selection for flow assistant
This commit is contained in:
parent
1728ab3c9b
commit
b27cfc8888
|
|
@ -0,0 +1,45 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
if [[ -f local.env ]]; then
|
||||
set -a
|
||||
source local.env
|
||||
set +a
|
||||
fi
|
||||
|
||||
POSTGRES_CONTAINER="${POSTGRES_CONTAINER:-humainflow-postgres}"
|
||||
POSTGRES_IMAGE="${POSTGRES_IMAGE:-postgres:17-alpine}"
|
||||
POSTGRES_DB="${POSTGRES_DB:-mydb}"
|
||||
POSTGRES_USER="${POSTGRES_USER:-${DB_USER:-lucio}}"
|
||||
POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-${DB_PASSWORD:-password}}"
|
||||
|
||||
if docker container inspect "$POSTGRES_CONTAINER" >/dev/null 2>&1; then
|
||||
docker rm -f "$POSTGRES_CONTAINER" >/dev/null
|
||||
fi
|
||||
|
||||
docker run \
|
||||
--name "$POSTGRES_CONTAINER" \
|
||||
--rm \
|
||||
-e POSTGRES_DB="$POSTGRES_DB" \
|
||||
-e POSTGRES_USER="$POSTGRES_USER" \
|
||||
-e POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \
|
||||
-p 5432:5432 \
|
||||
-d "$POSTGRES_IMAGE" >/dev/null
|
||||
|
||||
cleanup() {
|
||||
docker stop "$POSTGRES_CONTAINER" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
until docker exec "$POSTGRES_CONTAINER" pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; do
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Il database è usa-e-getta: crea lo schema dalle entity e non applica le
|
||||
# migration Flyway, che richiedono uno schema base già esistente.
|
||||
./mvnw spring-boot:run \
|
||||
-Dspring-boot.run.jvmArguments="-Dddl-auto=create-drop -Dspring.flyway.enabled=false"
|
||||
|
|
@ -32,9 +32,9 @@ import it.cnr.isti.workflow.manager.assistant.model.AssistantFixRequest;
|
|||
import it.cnr.isti.workflow.manager.assistant.model.AssistantFlowResponse;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantGenerationRequest;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantIntent;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantLlmSelection;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantMessageRole;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantMessageView;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantModelSelection;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantRefineRequest;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantSessionCreateRequest;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantSessionMessageRequest;
|
||||
|
|
@ -70,8 +70,7 @@ public class AssistantConversationService {
|
|||
SessionState session = new SessionState(
|
||||
UUID.randomUUID().toString(),
|
||||
owner,
|
||||
request.model(),
|
||||
request.phaseModels(),
|
||||
request.llmSelection(),
|
||||
now,
|
||||
now);
|
||||
sessions.put(session.id, session);
|
||||
|
|
@ -147,8 +146,8 @@ public class AssistantConversationService {
|
|||
switch (intent) {
|
||||
case DRAFT -> {
|
||||
AssistantFlowResponse result = flowAssistantService.draft(
|
||||
new AssistantGenerationRequest(contextualPrompt, session.model, DEFAULT_MAX_REPAIR_ATTEMPTS,
|
||||
session.phaseModels),
|
||||
new AssistantGenerationRequest(contextualPrompt, DEFAULT_MAX_REPAIR_ATTEMPTS,
|
||||
session.llmSelection),
|
||||
(phase, progressMessage) -> updateProgress(call, phase, progressMessage));
|
||||
ensureNotCancelled(call);
|
||||
call.flowResult = result;
|
||||
|
|
@ -158,8 +157,8 @@ public class AssistantConversationService {
|
|||
}
|
||||
case REFINE -> {
|
||||
AssistantFlowResponse result = flowAssistantService.refine(
|
||||
new AssistantRefineRequest(contextualPrompt, session.currentFlow, session.model,
|
||||
DEFAULT_MAX_REPAIR_ATTEMPTS, session.phaseModels),
|
||||
new AssistantRefineRequest(contextualPrompt, session.currentFlow, DEFAULT_MAX_REPAIR_ATTEMPTS,
|
||||
session.llmSelection),
|
||||
(phase, progressMessage) -> updateProgress(call, phase, progressMessage));
|
||||
ensureNotCancelled(call);
|
||||
call.flowResult = result;
|
||||
|
|
@ -170,7 +169,7 @@ public class AssistantConversationService {
|
|||
case FIX -> {
|
||||
AssistantFlowResponse result = flowAssistantService.fix(
|
||||
new AssistantFixRequest(contextualPrompt, session.currentFlow, session.lastValidationErrors,
|
||||
session.model, DEFAULT_MAX_REPAIR_ATTEMPTS, session.phaseModels),
|
||||
DEFAULT_MAX_REPAIR_ATTEMPTS, session.llmSelection),
|
||||
(phase, progressMessage) -> updateProgress(call, phase, progressMessage));
|
||||
ensureNotCancelled(call);
|
||||
call.flowResult = result;
|
||||
|
|
@ -181,7 +180,7 @@ public class AssistantConversationService {
|
|||
case EXPLAIN -> {
|
||||
updateProgress(call, "explaining", "Explaining the current flow");
|
||||
AssistantExplainResponse result = flowAssistantService.explain(
|
||||
new AssistantExplainRequest(session.currentFlow, contextualPrompt, session.model));
|
||||
new AssistantExplainRequest(session.currentFlow, contextualPrompt, session.llmSelection));
|
||||
ensureNotCancelled(call);
|
||||
call.explainResult = result;
|
||||
session.appendMessage(AssistantMessageRole.ASSISTANT, result.explanation(), call.id);
|
||||
|
|
@ -382,8 +381,7 @@ public class AssistantConversationService {
|
|||
private static final class SessionState {
|
||||
private final String id;
|
||||
private final String owner;
|
||||
private final String model;
|
||||
private final AssistantModelSelection phaseModels;
|
||||
private final AssistantLlmSelection llmSelection;
|
||||
private final Instant createdAt;
|
||||
private volatile Instant updatedAt;
|
||||
private volatile String lastCallId;
|
||||
|
|
@ -393,12 +391,11 @@ public class AssistantConversationService {
|
|||
private final AtomicBoolean running = new AtomicBoolean(false);
|
||||
private final List<AssistantMessageView> messages = java.util.Collections.synchronizedList(new ArrayList<>());
|
||||
|
||||
private SessionState(String id, String owner, String model, AssistantModelSelection phaseModels,
|
||||
private SessionState(String id, String owner, AssistantLlmSelection llmSelection,
|
||||
Instant createdAt, Instant updatedAt) {
|
||||
this.id = id;
|
||||
this.owner = owner;
|
||||
this.model = model;
|
||||
this.phaseModels = phaseModels;
|
||||
this.llmSelection = llmSelection;
|
||||
this.createdAt = createdAt;
|
||||
this.updatedAt = updatedAt;
|
||||
}
|
||||
|
|
@ -416,7 +413,7 @@ public class AssistantConversationService {
|
|||
return new AssistantSessionView(
|
||||
id,
|
||||
owner,
|
||||
model,
|
||||
llmSelection,
|
||||
createdAt,
|
||||
updatedAt,
|
||||
lastCallId,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
package it.cnr.isti.workflow.manager.assistant;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* Configuration shared by the flow assistant. Provider credentials are deliberately kept
|
||||
* server-side: clients can choose a registered provider and model, but never send API keys in
|
||||
* assistant requests or session state.
|
||||
*/
|
||||
@Component
|
||||
@ConfigurationProperties(prefix = "app.assistant")
|
||||
public class AssistantProperties {
|
||||
|
||||
private String defaultProvider = "InternalOllama";
|
||||
private Map<String, String> providerAuthorizations = new LinkedHashMap<>();
|
||||
|
||||
public String getDefaultProvider() {
|
||||
return defaultProvider;
|
||||
}
|
||||
|
||||
public void setDefaultProvider(String defaultProvider) {
|
||||
this.defaultProvider = defaultProvider;
|
||||
}
|
||||
|
||||
public Map<String, String> getProviderAuthorizations() {
|
||||
return providerAuthorizations;
|
||||
}
|
||||
|
||||
public void setProviderAuthorizations(Map<String, String> providerAuthorizations) {
|
||||
this.providerAuthorizations = providerAuthorizations == null
|
||||
? new LinkedHashMap<>()
|
||||
: new LinkedHashMap<>(providerAuthorizations);
|
||||
}
|
||||
}
|
||||
|
|
@ -282,7 +282,7 @@ public class FlowAssistantPromptService {
|
|||
}
|
||||
}
|
||||
|
||||
Selected internal model:
|
||||
Selected assistant model:
|
||||
%s
|
||||
|
||||
Flow plan:
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ import it.cnr.isti.workflow.manager.assistant.model.AssistantExplainResponse;
|
|||
import it.cnr.isti.workflow.manager.assistant.model.AssistantFixRequest;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantFlowResponse;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantGenerationRequest;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantLlmSelection;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantModelSelection;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantRefineRequest;
|
||||
import it.cnr.isti.workflow.manager.blocks.Block;
|
||||
|
|
@ -82,7 +83,6 @@ public class FlowAssistantService {
|
|||
private static final Logger log = LoggerFactory.getLogger(FlowAssistantService.class);
|
||||
private static final Logger assistantResponseLog = LoggerFactory.getLogger("assistant.responses");
|
||||
|
||||
private static final String INTERNAL_PROVIDER_NAME = "InternalOllama";
|
||||
private static final String SHARED_MEMORY_SESSION_NAME = "sharedMemorySession";
|
||||
private static final int DEFAULT_PROVIDER_RETRY_ATTEMPTS = 3;
|
||||
private static final int DEFAULT_MAX_REPAIR_ATTEMPTS = 2;
|
||||
|
|
@ -237,9 +237,15 @@ public class FlowAssistantService {
|
|||
private record ResolvedAssistantModels(String planningModel, String jsonModel, String repairModel) {
|
||||
}
|
||||
|
||||
private record ResolvedAssistantSelection(String provider, String model, ResolvedAssistantModels phaseModels) {
|
||||
}
|
||||
|
||||
@Autowired
|
||||
private Map<String, LLMProvider> llmProviders;
|
||||
|
||||
@Autowired
|
||||
private AssistantProperties assistantProperties;
|
||||
|
||||
@Autowired
|
||||
private BlockCatalogService blockCatalogService;
|
||||
|
||||
|
|
@ -282,6 +288,15 @@ public class FlowAssistantService {
|
|||
@Value("${app.assistant.provider-retry-attempts:" + DEFAULT_PROVIDER_RETRY_ATTEMPTS + "}")
|
||||
private int providerRetryAttempts;
|
||||
|
||||
@Value("${app.assistant.default-model}")
|
||||
private String defaultAssistantModel;
|
||||
|
||||
@Value("${app.assistant.generated-flow-provider:InternalOllama}")
|
||||
private String generatedFlowProvider;
|
||||
|
||||
@Value("${app.assistant.generated-flow-model:${app.assistant.default-model}}")
|
||||
private String generatedFlowModel;
|
||||
|
||||
@Value("${app.assistant.default-planning-model:${app.assistant.default-model}}")
|
||||
private String defaultPlanningModel;
|
||||
|
||||
|
|
@ -298,8 +313,8 @@ public class FlowAssistantService {
|
|||
public AssistantFlowResponse draft(AssistantGenerationRequest request, ProgressListener progressListener) {
|
||||
boolean directMdc = ensureAssistantRequestMdc(OperationMode.DRAFT.name());
|
||||
try {
|
||||
return generateFlow(OperationMode.DRAFT, request.userPrompt(), null, List.of(), request.model(),
|
||||
request.phaseModels(), request.maxRepairAttempts(), progressListener);
|
||||
return generateFlow(OperationMode.DRAFT, request.userPrompt(), null, List.of(), request.llmSelection(),
|
||||
request.maxRepairAttempts(), progressListener);
|
||||
} finally {
|
||||
clearAssistantRequestMdc(directMdc);
|
||||
}
|
||||
|
|
@ -312,8 +327,8 @@ public class FlowAssistantService {
|
|||
public AssistantFlowResponse refine(AssistantRefineRequest request, ProgressListener progressListener) {
|
||||
boolean directMdc = ensureAssistantRequestMdc(OperationMode.REFINE.name());
|
||||
try {
|
||||
return generateFlow(OperationMode.REFINE, request.userPrompt(), request.flow(), List.of(), request.model(),
|
||||
request.phaseModels(), request.maxRepairAttempts(), progressListener);
|
||||
return generateFlow(OperationMode.REFINE, request.userPrompt(), request.flow(), List.of(), request.llmSelection(),
|
||||
request.maxRepairAttempts(), progressListener);
|
||||
} finally {
|
||||
clearAssistantRequestMdc(directMdc);
|
||||
}
|
||||
|
|
@ -329,8 +344,8 @@ public class FlowAssistantService {
|
|||
: request.validationErrors();
|
||||
boolean directMdc = ensureAssistantRequestMdc(OperationMode.FIX.name());
|
||||
try {
|
||||
return generateFlow(OperationMode.FIX, request.userPrompt(), request.flow(), initialErrors, request.model(),
|
||||
request.phaseModels(), request.maxRepairAttempts(), progressListener);
|
||||
return generateFlow(OperationMode.FIX, request.userPrompt(), request.flow(), initialErrors, request.llmSelection(),
|
||||
request.maxRepairAttempts(), progressListener);
|
||||
} finally {
|
||||
clearAssistantRequestMdc(directMdc);
|
||||
}
|
||||
|
|
@ -343,11 +358,13 @@ public class FlowAssistantService {
|
|||
public AssistantExplainResponse explain(AssistantExplainRequest request, ProgressListener progressListener) {
|
||||
boolean directMdc = ensureAssistantRequestMdc("EXPLAIN");
|
||||
try {
|
||||
LLMProvider provider = resolveInternalProvider();
|
||||
ResolvedAssistantSelection selection = resolveAssistantSelection(request.llmSelection());
|
||||
LLMProvider provider = resolveProvider(selection.provider());
|
||||
String authorization = resolveProviderAuthorization(provider);
|
||||
progressListener.onProgress("explaining", "Explaining the current flow");
|
||||
String prompt = promptService.buildExplainPrompt(request.flow(), request.userPrompt());
|
||||
String rawResponse = invokeProvider(provider, request.model(), prompt);
|
||||
logAssistantRawResponse("explain", request.model(), "text", 1, 1, rawResponse);
|
||||
String rawResponse = invokeProvider(provider, selection.model(), prompt, authorization);
|
||||
logAssistantRawResponse("explain", selection.model(), "text", 1, 1, rawResponse);
|
||||
AssistantExplainResponse response = new AssistantExplainResponse(rawResponse);
|
||||
progressListener.onProgress("completed", "Flow explanation ready");
|
||||
return response;
|
||||
|
|
@ -357,10 +374,13 @@ public class FlowAssistantService {
|
|||
}
|
||||
|
||||
private AssistantFlowResponse generateFlow(OperationMode initialMode, String userPrompt, FlowCreateRequest currentFlow,
|
||||
List<ValidationError> initialErrors, String workflowModel, AssistantModelSelection requestedPhaseModels,
|
||||
Integer maxRepairAttempts, ProgressListener progressListener) {
|
||||
LLMProvider provider = resolveInternalProvider();
|
||||
ResolvedAssistantModels phaseModels = resolveAssistantModels(workflowModel, requestedPhaseModels);
|
||||
List<ValidationError> initialErrors, AssistantLlmSelection requestedSelection, Integer maxRepairAttempts,
|
||||
ProgressListener progressListener) {
|
||||
ResolvedAssistantSelection selection = resolveAssistantSelection(requestedSelection);
|
||||
LLMProvider provider = resolveProvider(selection.provider());
|
||||
String authorization = resolveProviderAuthorization(provider);
|
||||
String assistantModel = selection.model();
|
||||
ResolvedAssistantModels phaseModels = selection.phaseModels();
|
||||
int allowedRepairs = maxRepairAttempts == null ? DEFAULT_MAX_REPAIR_ATTEMPTS : maxRepairAttempts;
|
||||
int repairs = 0;
|
||||
OperationMode mode = initialMode;
|
||||
|
|
@ -370,8 +390,8 @@ public class FlowAssistantService {
|
|||
List<ValidationError> errors = List.of();
|
||||
|
||||
while (true) {
|
||||
assembled = assembleFlow(provider, workflowModel, phaseModels, mode, userPrompt, flowContext, errorContext,
|
||||
progressListener);
|
||||
assembled = assembleFlow(provider, authorization, assistantModel, generatedFlowProvider, generatedFlowModel,
|
||||
phaseModels, mode, userPrompt, flowContext, errorContext, progressListener);
|
||||
progressListener.onProgress("validating", "Validating the assembled flow");
|
||||
errors = validate(assembled.flow());
|
||||
if (errors.isEmpty() || repairs >= allowedRepairs) {
|
||||
|
|
@ -405,9 +425,9 @@ public class FlowAssistantService {
|
|||
}
|
||||
}
|
||||
|
||||
private AssembledFlow assembleFlow(LLMProvider provider, String workflowModel, ResolvedAssistantModels phaseModels,
|
||||
OperationMode mode, String userPrompt, FlowCreateRequest currentFlow, List<ValidationError> errors,
|
||||
ProgressListener progressListener) {
|
||||
private AssembledFlow assembleFlow(LLMProvider provider, String authorization, String assistantModel,
|
||||
String flowProvider, String flowModel, ResolvedAssistantModels phaseModels, OperationMode mode, String userPrompt,
|
||||
FlowCreateRequest currentFlow, List<ValidationError> errors, ProgressListener progressListener) {
|
||||
List<BlockCatalogService.AssistantPromptBlockDescriptor> catalog = blockCatalogService.getPromptCatalog();
|
||||
Map<String, BlockCatalogService.AssistantPromptBlockDescriptor> catalogByType = new LinkedHashMap<>();
|
||||
for (BlockCatalogService.AssistantPromptBlockDescriptor descriptor : catalog) {
|
||||
|
|
@ -423,7 +443,7 @@ public class FlowAssistantService {
|
|||
} else {
|
||||
String planPrompt = promptService.buildPlanPrompt(mode, userPrompt, currentFlow, errors, catalog,
|
||||
mcpServerCatalogEntries());
|
||||
parsedPlan = invokeStructuredAndValidate(provider, planningModelFor(mode, phaseModels),
|
||||
parsedPlan = invokeStructuredAndValidate(provider, authorization, planningModelFor(mode, phaseModels),
|
||||
phaseModels.repairModel(), planPrompt, "plan", rawResponse -> {
|
||||
ParsedPlan plan = parsePlan(rawResponse);
|
||||
AssistantFlowPlan normalizedPlan = validateAndNormalizePlan(plan.plan(), mode, userPrompt, currentFlow,
|
||||
|
|
@ -474,16 +494,16 @@ public class FlowAssistantService {
|
|||
} else {
|
||||
progressListener.onProgress("configuring_blocks", "Configuring block " + blockPlan.blockId());
|
||||
String blockPrompt = promptService.buildBlockConfigurationPrompt(mode, userPrompt, descriptor,
|
||||
parsedPlan.plan(), blockPlan, currentFlow, errors, workflowModel, mcpServerCatalogEntries());
|
||||
parsedPlan.plan(), blockPlan, currentFlow, errors, assistantModel, mcpServerCatalogEntries());
|
||||
int currentBlockIndex = blockIndex;
|
||||
int blockPlanCount = blockPlans.size();
|
||||
ConfiguredBlockResult configuredBlock = invokeStructuredAndValidate(provider,
|
||||
ConfiguredBlockResult configuredBlock = invokeStructuredAndValidate(provider, authorization,
|
||||
jsonModelFor(mode, phaseModels), phaseModels.repairModel(), blockPrompt,
|
||||
"block configuration for " + blockPlan.blockId(), rawResponse -> {
|
||||
ParsedBlockDraft parsedBlock = parseBlockDraft(rawResponse);
|
||||
AssistantConfiguredBlockDraft normalizedDraft = normalizeBlockDraft(blockPlan,
|
||||
parsedBlock.block());
|
||||
Block<?> newBlock = buildBlock(descriptor, blockPlan, normalizedDraft, workflowModel,
|
||||
Block<?> newBlock = buildBlock(descriptor, blockPlan, normalizedDraft, flowProvider, flowModel,
|
||||
requireSharedMemorySemantics, currentBlockIndex, blockPlanCount);
|
||||
return new ConfiguredBlockResult(parsedBlock, newBlock);
|
||||
});
|
||||
|
|
@ -542,7 +562,7 @@ public class FlowAssistantService {
|
|||
// are reused and existing inner connections preserved (incremental diffing); on ADD
|
||||
// there is nothing to reuse.
|
||||
Container<?> baseContainer = containerOperation == PlanOperation.UPDATE ? existingContainer : null;
|
||||
assembledContainer = assembleContainer(provider, workflowModel, phaseModels, mode, userPrompt,
|
||||
assembledContainer = assembleContainer(provider, authorization, assistantModel, flowProvider, flowModel, phaseModels, mode, userPrompt,
|
||||
containerPlan, baseContainer, catalogByType, rationaleParts, progressListener);
|
||||
}
|
||||
|
||||
|
|
@ -576,7 +596,7 @@ public class FlowAssistantService {
|
|||
} else {
|
||||
String connectionsPrompt = promptService.buildConnectionsPrompt(mode, userPrompt, parsedPlan.plan(),
|
||||
configuredNodes, currentFlow, errors);
|
||||
parsedConnections = invokeStructuredAndValidate(provider, jsonModelFor(mode, phaseModels),
|
||||
parsedConnections = invokeStructuredAndValidate(provider, authorization, jsonModelFor(mode, phaseModels),
|
||||
phaseModels.repairModel(), connectionsPrompt, "connections", rawResponse -> {
|
||||
ParsedConnections parsed = parseConnectionsOrInferSequential(rawResponse, assembledBlocks);
|
||||
// MCP shared-session ordering is expressed as a Dependency (see
|
||||
|
|
@ -622,8 +642,8 @@ public class FlowAssistantService {
|
|||
* only ADD/UPDATE inner blocks are reconfigured. When it is null (ADD, or FIX), every inner
|
||||
* block is configured fresh.
|
||||
*/
|
||||
private Container<?> assembleContainer(LLMProvider provider, String workflowModel,
|
||||
ResolvedAssistantModels phaseModels, OperationMode mode, String userPrompt,
|
||||
private Container<?> assembleContainer(LLMProvider provider, String authorization, String assistantModel,
|
||||
String flowProvider, String flowModel, ResolvedAssistantModels phaseModels, OperationMode mode, String userPrompt,
|
||||
AssistantContainerPlan containerPlan, Container<?> existingContainer,
|
||||
Map<String, BlockCatalogService.AssistantPromptBlockDescriptor> catalogByType,
|
||||
List<String> rationaleParts, ProgressListener progressListener) {
|
||||
|
|
@ -674,17 +694,17 @@ public class FlowAssistantService {
|
|||
progressListener.onProgress("configuring_blocks",
|
||||
"Configuring block " + blockPlan.blockId() + " in container " + containerPlan.containerId());
|
||||
String blockPrompt = promptService.buildBlockConfigurationPrompt(mode, userPrompt, descriptor,
|
||||
containerInnerPlan(containerPlan), blockPlan, null, List.of(), workflowModel,
|
||||
containerInnerPlan(containerPlan), blockPlan, null, List.of(), assistantModel,
|
||||
mcpServerCatalogEntries());
|
||||
int currentBlockIndex = blockIndex;
|
||||
int blockPlanCount = innerBlockPlans.size();
|
||||
ConfiguredBlockResult configuredBlock = invokeStructuredAndValidate(provider,
|
||||
ConfiguredBlockResult configuredBlock = invokeStructuredAndValidate(provider, authorization,
|
||||
jsonModelFor(mode, phaseModels), phaseModels.repairModel(), blockPrompt,
|
||||
"block configuration for " + blockPlan.blockId() + " in container " + containerPlan.containerId(),
|
||||
rawResponse -> {
|
||||
ParsedBlockDraft parsedBlock = parseBlockDraft(rawResponse);
|
||||
AssistantConfiguredBlockDraft normalizedDraft = normalizeBlockDraft(blockPlan, parsedBlock.block());
|
||||
Block<?> newBlock = buildBlock(descriptor, blockPlan, normalizedDraft, workflowModel,
|
||||
Block<?> newBlock = buildBlock(descriptor, blockPlan, normalizedDraft, flowProvider, flowModel,
|
||||
innerRequiresSharedMemory, currentBlockIndex, blockPlanCount);
|
||||
return new ConfiguredBlockResult(parsedBlock, newBlock);
|
||||
});
|
||||
|
|
@ -725,7 +745,7 @@ public class FlowAssistantService {
|
|||
} else {
|
||||
String connectionsPrompt = promptService.buildConnectionsPrompt(mode, userPrompt,
|
||||
containerInnerPlan(containerPlan), innerConfiguredNodes, null, List.of());
|
||||
innerConnections = invokeStructuredAndValidate(provider, jsonModelFor(mode, phaseModels),
|
||||
innerConnections = invokeStructuredAndValidate(provider, authorization, jsonModelFor(mode, phaseModels),
|
||||
phaseModels.repairModel(), connectionsPrompt,
|
||||
"connections for container " + containerPlan.containerId(), rawResponse -> {
|
||||
ParsedConnections parsed = parseConnectionsOrInferSequential(rawResponse, innerBlocks);
|
||||
|
|
@ -777,7 +797,7 @@ public class FlowAssistantService {
|
|||
// The guard subflow is built deterministically from the natural-language
|
||||
// guardCondition - the assistant never authors its rigid structure (switch
|
||||
// outputs named guard/feedback, ${{outputs.x}} references, boolean coercion).
|
||||
.guardSubFlow(buildLoopGuardSubFlow(containerPlan, subFlow, workflowModel))
|
||||
.guardSubFlow(buildLoopGuardSubFlow(containerPlan, subFlow, flowProvider, flowModel))
|
||||
// feedbackInput may be null: LoopContainerConfiguration infers it when the body
|
||||
// has exactly one open non-multiple input; the model supplies it when the body
|
||||
// has several (else inference is ambiguous and validation fails into repair).
|
||||
|
|
@ -823,12 +843,12 @@ public class FlowAssistantService {
|
|||
* only supplies the natural-language guardCondition - none of this rigid wiring.
|
||||
*/
|
||||
private FlowData buildLoopGuardSubFlow(AssistantContainerPlan containerPlan, FlowData bodySubFlow,
|
||||
String workflowModel) {
|
||||
String workflowProvider, String workflowModel) {
|
||||
String bodyOutputName = ContainerFlowInterfaceResolver.resolveGenericOutputs(bodySubFlow).stream()
|
||||
.findFirst()
|
||||
.map(IODescriptor::getName)
|
||||
.orElse("response");
|
||||
LLMDescriptor descriptor = LLMDescriptor.builder().provider(INTERNAL_PROVIDER_NAME).model(workflowModel).build();
|
||||
LLMDescriptor descriptor = LLMDescriptor.builder().provider(workflowProvider).model(workflowModel).build();
|
||||
|
||||
Block<?> guardEvaluator = llmBlockFactory.create(LLMBlockConfiguration.builder()
|
||||
.name(containerPlan.containerId() + "-guard-evaluator")
|
||||
|
|
@ -994,7 +1014,7 @@ public class FlowAssistantService {
|
|||
}
|
||||
}
|
||||
|
||||
private <T> T invokeStructuredAndValidate(LLMProvider provider, String model, String repairModel, String prompt,
|
||||
private <T> T invokeStructuredAndValidate(LLMProvider provider, String authorization, String model, String repairModel, String prompt,
|
||||
String taskName, StructuredResponseParser<T> parser) {
|
||||
String currentPrompt = prompt;
|
||||
String rawResponse = null;
|
||||
|
|
@ -1004,7 +1024,7 @@ public class FlowAssistantService {
|
|||
for (int attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
String attemptModel = attempt == 1 ? model : repairModel;
|
||||
try {
|
||||
rawResponse = invokeStructuredProvider(provider, attemptModel, repairModel, currentPrompt, taskName,
|
||||
rawResponse = invokeStructuredProvider(provider, authorization, attemptModel, repairModel, currentPrompt, taskName,
|
||||
attempt, maxAttempts);
|
||||
if (log.isTraceEnabled()) {
|
||||
log.trace(
|
||||
|
|
@ -1105,23 +1125,6 @@ public class FlowAssistantService {
|
|||
MDC.remove("assistantPhase");
|
||||
}
|
||||
|
||||
private ResolvedAssistantModels resolveAssistantModels(String workflowModel,
|
||||
AssistantModelSelection requestedPhaseModels) {
|
||||
return new ResolvedAssistantModels(
|
||||
firstNonBlank(
|
||||
requestedPhaseModels == null ? null : requestedPhaseModels.planningModel(),
|
||||
defaultPlanningModel,
|
||||
workflowModel),
|
||||
firstNonBlank(
|
||||
requestedPhaseModels == null ? null : requestedPhaseModels.jsonModel(),
|
||||
defaultJsonModel,
|
||||
workflowModel),
|
||||
firstNonBlank(
|
||||
requestedPhaseModels == null ? null : requestedPhaseModels.repairModel(),
|
||||
defaultRepairModel,
|
||||
workflowModel));
|
||||
}
|
||||
|
||||
private String planningModelFor(OperationMode mode, ResolvedAssistantModels phaseModels) {
|
||||
return mode == OperationMode.FIX ? phaseModels.repairModel() : phaseModels.planningModel();
|
||||
}
|
||||
|
|
@ -1184,7 +1187,7 @@ public class FlowAssistantService {
|
|||
}
|
||||
|
||||
private Block<?> buildBlock(BlockCatalogService.AssistantPromptBlockDescriptor descriptor, AssistantBlockPlan blockPlan,
|
||||
AssistantConfiguredBlockDraft draft, String model, boolean requireSharedMemorySemantics, int blockIndex,
|
||||
AssistantConfiguredBlockDraft draft, String provider, String model, boolean requireSharedMemorySemantics, int blockIndex,
|
||||
int blockCount) {
|
||||
if (!(draft.config() instanceof ObjectNode configNode)) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_GATEWAY,
|
||||
|
|
@ -1194,7 +1197,7 @@ public class FlowAssistantService {
|
|||
ObjectNode normalizedConfig = configNode.deepCopy();
|
||||
normalizedConfig.put("type", descriptor.configurationType());
|
||||
normalizedConfig.put("name", defaultIfBlank(draft.name(), defaultIfBlank(blockPlan.purpose(), blockPlan.blockType())));
|
||||
injectSystemManagedFields(normalizedConfig, descriptor, model);
|
||||
injectSystemManagedFields(normalizedConfig, descriptor, provider, model);
|
||||
ensureRequiredTextDefaults(normalizedConfig, descriptor, blockPlan);
|
||||
normalizeHumanDecisionOptions(normalizedConfig, descriptor);
|
||||
normalizeHttpServerCallAuthorization(normalizedConfig, blockPlan);
|
||||
|
|
@ -1415,12 +1418,12 @@ public class FlowAssistantService {
|
|||
}
|
||||
|
||||
private void injectSystemManagedFields(ObjectNode config, BlockCatalogService.AssistantPromptBlockDescriptor descriptor,
|
||||
String model) {
|
||||
String provider, String model) {
|
||||
String configurationType = descriptor.configurationType();
|
||||
removeSystemManagedFields(config);
|
||||
if (isConfigurationType(configurationType, "LLMBlockConfiguration")
|
||||
|| isConfigurationType(configurationType, "ChatInteractionBlockConfiguration")) {
|
||||
config.set("llmDescriptor", llmDescriptorNode(model));
|
||||
config.set("llmDescriptor", llmDescriptorNode(provider, model));
|
||||
return;
|
||||
}
|
||||
if (isConfigurationType(configurationType, "ConditionalBlockConfiguration")
|
||||
|
|
@ -1428,7 +1431,7 @@ public class FlowAssistantService {
|
|||
boolean useLlm = inferConditionalUseLlm(config);
|
||||
config.put("useLlm", useLlm);
|
||||
if (useLlm) {
|
||||
config.set("llmDescriptor", llmDescriptorNode(model));
|
||||
config.set("llmDescriptor", llmDescriptorNode(provider, model));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -3039,9 +3042,9 @@ public class FlowAssistantService {
|
|||
|| normalized.contains("nessuna connessione");
|
||||
}
|
||||
|
||||
private ObjectNode llmDescriptorNode(String model) {
|
||||
private ObjectNode llmDescriptorNode(String provider, String model) {
|
||||
ObjectNode llmDescriptor = ObjectMapperHolder.mapper.createObjectNode();
|
||||
llmDescriptor.put("provider", INTERNAL_PROVIDER_NAME);
|
||||
llmDescriptor.put("provider", provider);
|
||||
llmDescriptor.put("model", model);
|
||||
return llmDescriptor;
|
||||
}
|
||||
|
|
@ -3073,29 +3076,68 @@ public class FlowAssistantService {
|
|||
return node == null ? null : node.stringValueOpt().orElse(null);
|
||||
}
|
||||
|
||||
private LLMProvider resolveInternalProvider() {
|
||||
// Try by exact provider name first (stable, independent of Spring bean name)
|
||||
LLMProvider byName = llmProviders.values().stream()
|
||||
.filter(candidate -> INTERNAL_PROVIDER_NAME.equals(candidate.getName()))
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
if (byName != null) {
|
||||
return byName;
|
||||
private ResolvedAssistantSelection resolveAssistantSelection(AssistantLlmSelection requestedSelection) {
|
||||
if (requestedSelection == null) {
|
||||
String defaultProvider = trimToNull(assistantProperties.getDefaultProvider());
|
||||
if (defaultProvider == null) {
|
||||
throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE,
|
||||
"Assistant default provider is not configured");
|
||||
}
|
||||
String defaultModel = trimToNull(defaultAssistantModel);
|
||||
if (defaultModel == null) {
|
||||
throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE,
|
||||
"Assistant default model is not configured");
|
||||
}
|
||||
return new ResolvedAssistantSelection(defaultProvider, defaultModel,
|
||||
new ResolvedAssistantModels(defaultPlanningModel, defaultJsonModel, defaultRepairModel));
|
||||
}
|
||||
// Fallback: legacy bean-name key kept for backwards compatibility with custom deployments
|
||||
LLMProvider byBeanName = llmProviders.get("internalOllamaLLMProvider");
|
||||
if (byBeanName != null) {
|
||||
return byBeanName;
|
||||
|
||||
String provider = trimToNull(requestedSelection.provider());
|
||||
String model = trimToNull(requestedSelection.model());
|
||||
if (provider == null || model == null) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST,
|
||||
"llmSelection.provider and llmSelection.model are required when selecting an assistant LLM");
|
||||
}
|
||||
throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE,
|
||||
"Internal assistant provider not available: " + INTERNAL_PROVIDER_NAME
|
||||
+ ". Registered providers: " + llmProviders.values().stream()
|
||||
.map(LLMProvider::getName).toList());
|
||||
AssistantModelSelection requestedPhaseModels = requestedSelection.phaseModels();
|
||||
return new ResolvedAssistantSelection(provider, model,
|
||||
new ResolvedAssistantModels(
|
||||
firstNonBlank(requestedPhaseModels == null ? null : requestedPhaseModels.planningModel(), model),
|
||||
firstNonBlank(requestedPhaseModels == null ? null : requestedPhaseModels.jsonModel(), model),
|
||||
firstNonBlank(requestedPhaseModels == null ? null : requestedPhaseModels.repairModel(), model)));
|
||||
}
|
||||
|
||||
private String invokeProvider(LLMProvider provider, String model, String prompt) {
|
||||
private LLMProvider resolveProvider(String providerName) {
|
||||
return llmProviders.values().stream()
|
||||
.filter(candidate -> providerName.equalsIgnoreCase(candidate.getName()))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new ResponseStatusException(HttpStatus.BAD_REQUEST,
|
||||
"Assistant provider not found: " + providerName + ". Registered providers: "
|
||||
+ llmProviders.values().stream().map(LLMProvider::getName)
|
||||
.filter(Objects::nonNull).distinct().sorted().toList()));
|
||||
}
|
||||
|
||||
private String resolveProviderAuthorization(LLMProvider provider) {
|
||||
if (!provider.requiresAuthorization()) {
|
||||
return null;
|
||||
}
|
||||
String authorization = assistantProperties.getProviderAuthorizations().entrySet().stream()
|
||||
.filter(entry -> provider.getName().equalsIgnoreCase(entry.getKey()))
|
||||
.map(Map.Entry::getValue)
|
||||
.filter(value -> value != null && !value.isBlank())
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
if (authorization == null) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST,
|
||||
"Assistant provider requires server-side authorization: " + provider.getName());
|
||||
}
|
||||
return authorization;
|
||||
}
|
||||
|
||||
private String invokeProvider(LLMProvider provider, String model, String prompt, String authorization) {
|
||||
try {
|
||||
return provider.generate(model, prompt);
|
||||
return provider.requiresAuthorization()
|
||||
? provider.generate(model, prompt, authorization)
|
||||
: provider.generate(model, prompt);
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage(), e);
|
||||
} catch (ResponseStatusException e) {
|
||||
|
|
@ -3107,11 +3149,13 @@ public class FlowAssistantService {
|
|||
}
|
||||
}
|
||||
|
||||
private String invokeStructuredProvider(LLMProvider provider, String model, String repairModel, String prompt,
|
||||
private String invokeStructuredProvider(LLMProvider provider, String authorization, String model, String repairModel, String prompt,
|
||||
String taskName, int attempt, int maxAttempts) {
|
||||
RuntimeException structuredFailure = null;
|
||||
try {
|
||||
String structuredResponse = provider.generateJson(model, prompt);
|
||||
String structuredResponse = provider.requiresAuthorization()
|
||||
? provider.generateJson(model, prompt, authorization)
|
||||
: provider.generateJson(model, prompt);
|
||||
if (structuredResponse != null) {
|
||||
logAssistantRawResponse(taskName, model, "json", attempt, maxAttempts, structuredResponse);
|
||||
}
|
||||
|
|
@ -3133,7 +3177,9 @@ public class FlowAssistantService {
|
|||
}
|
||||
|
||||
try {
|
||||
String fallbackResponse = provider.generate(model, prompt);
|
||||
String fallbackResponse = provider.requiresAuthorization()
|
||||
? provider.generate(model, prompt, authorization)
|
||||
: provider.generate(model, prompt);
|
||||
if (fallbackResponse != null) {
|
||||
logAssistantRawResponse(taskName, model, "text-fallback", attempt, maxAttempts, fallbackResponse);
|
||||
}
|
||||
|
|
@ -3141,8 +3187,11 @@ public class FlowAssistantService {
|
|||
if (!looksLikeStructuredJson(fallbackResponse)) {
|
||||
try {
|
||||
String reformatModel = firstNonBlank(repairModel, model);
|
||||
String reformatted = provider.generateJson(reformatModel,
|
||||
promptService.buildJsonReformatPrompt(prompt, fallbackResponse));
|
||||
String reformatted = provider.requiresAuthorization()
|
||||
? provider.generateJson(reformatModel,
|
||||
promptService.buildJsonReformatPrompt(prompt, fallbackResponse), authorization)
|
||||
: provider.generateJson(reformatModel,
|
||||
promptService.buildJsonReformatPrompt(prompt, fallbackResponse));
|
||||
if (reformatted != null) {
|
||||
logAssistantRawResponse(taskName, reformatModel, "json-reformat", attempt, maxAttempts,
|
||||
reformatted);
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
package it.cnr.isti.workflow.manager.assistant.model;
|
||||
|
||||
public record AssistantConfigView(
|
||||
String provider,
|
||||
String defaultProvider,
|
||||
String defaultModel,
|
||||
String availableModelsRetrieverUrl,
|
||||
String availableProvidersRetrieverUrl,
|
||||
String availableModelsRetrieverUrlTemplate,
|
||||
AssistantModelSelection defaultPhaseModels) {
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,11 +2,14 @@ package it.cnr.isti.workflow.manager.assistant.model;
|
|||
|
||||
import it.cnr.isti.workflow.manager.flows.model.FlowCreateRequest;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
|
||||
public record AssistantExplainRequest(
|
||||
@Valid @NotNull FlowCreateRequest flow,
|
||||
String userPrompt,
|
||||
@NotBlank String model) {
|
||||
@Valid AssistantLlmSelection llmSelection) {
|
||||
|
||||
public AssistantExplainRequest(FlowCreateRequest flow, String userPrompt, String model) {
|
||||
this(flow, userPrompt, model == null ? null : new AssistantLlmSelection("InternalOllama", model, null));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,19 +7,25 @@ import it.cnr.isti.workflow.manager.flows.validation.ValidationError;
|
|||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.Max;
|
||||
import jakarta.validation.constraints.Min;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
|
||||
public record AssistantFixRequest(
|
||||
String userPrompt,
|
||||
@Valid @NotNull FlowCreateRequest flow,
|
||||
List<ValidationError> validationErrors,
|
||||
@NotBlank String model,
|
||||
@Min(0) @Max(3) Integer maxRepairAttempts,
|
||||
@Valid AssistantModelSelection phaseModels) {
|
||||
@Valid AssistantLlmSelection llmSelection) {
|
||||
|
||||
public AssistantFixRequest(String userPrompt, FlowCreateRequest flow,
|
||||
List<ValidationError> validationErrors, String model, Integer maxRepairAttempts) {
|
||||
this(userPrompt, flow, validationErrors, model, maxRepairAttempts, null);
|
||||
this(userPrompt, flow, validationErrors, maxRepairAttempts,
|
||||
model == null ? null : new AssistantLlmSelection("InternalOllama", model, null));
|
||||
}
|
||||
|
||||
public AssistantFixRequest(String userPrompt, FlowCreateRequest flow,
|
||||
List<ValidationError> validationErrors, String model, Integer maxRepairAttempts,
|
||||
AssistantModelSelection phaseModels) {
|
||||
this(userPrompt, flow, validationErrors, maxRepairAttempts,
|
||||
model == null ? null : new AssistantLlmSelection("InternalOllama", model, phaseModels));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,11 +7,17 @@ import jakarta.validation.Valid;
|
|||
|
||||
public record AssistantGenerationRequest(
|
||||
@NotBlank String userPrompt,
|
||||
@NotBlank String model,
|
||||
@Min(0) @Max(3) Integer maxRepairAttempts,
|
||||
@Valid AssistantModelSelection phaseModels) {
|
||||
@Valid AssistantLlmSelection llmSelection) {
|
||||
|
||||
public AssistantGenerationRequest(String userPrompt, String model, Integer maxRepairAttempts) {
|
||||
this(userPrompt, model, maxRepairAttempts, null);
|
||||
this(userPrompt, maxRepairAttempts,
|
||||
model == null ? null : new AssistantLlmSelection("InternalOllama", model, null));
|
||||
}
|
||||
|
||||
public AssistantGenerationRequest(String userPrompt, String model, Integer maxRepairAttempts,
|
||||
AssistantModelSelection phaseModels) {
|
||||
this(userPrompt, maxRepairAttempts,
|
||||
model == null ? null : new AssistantLlmSelection("InternalOllama", model, phaseModels));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
package it.cnr.isti.workflow.manager.assistant.model;
|
||||
|
||||
import jakarta.validation.Valid;
|
||||
|
||||
/**
|
||||
* Optional client override for the LLM used by the creation assistant. When absent, the service
|
||||
* uses the globally configured assistant defaults.
|
||||
*/
|
||||
public record AssistantLlmSelection(
|
||||
String provider,
|
||||
String model,
|
||||
@Valid AssistantModelSelection phaseModels) {
|
||||
}
|
||||
|
|
@ -10,12 +10,18 @@ import jakarta.validation.constraints.NotNull;
|
|||
public record AssistantRefineRequest(
|
||||
@NotBlank String userPrompt,
|
||||
@Valid @NotNull FlowCreateRequest flow,
|
||||
@NotBlank String model,
|
||||
@Min(0) @Max(3) Integer maxRepairAttempts,
|
||||
@Valid AssistantModelSelection phaseModels) {
|
||||
@Valid AssistantLlmSelection llmSelection) {
|
||||
|
||||
public AssistantRefineRequest(String userPrompt, FlowCreateRequest flow, String model,
|
||||
Integer maxRepairAttempts) {
|
||||
this(userPrompt, flow, model, maxRepairAttempts, null);
|
||||
this(userPrompt, flow, maxRepairAttempts,
|
||||
model == null ? null : new AssistantLlmSelection("InternalOllama", model, null));
|
||||
}
|
||||
|
||||
public AssistantRefineRequest(String userPrompt, FlowCreateRequest flow, String model,
|
||||
Integer maxRepairAttempts, AssistantModelSelection phaseModels) {
|
||||
this(userPrompt, flow, maxRepairAttempts,
|
||||
model == null ? null : new AssistantLlmSelection("InternalOllama", model, phaseModels));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
package it.cnr.isti.workflow.manager.assistant.model;
|
||||
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
|
||||
public record AssistantSessionCreateRequest(
|
||||
@NotBlank String model,
|
||||
@Valid AssistantModelSelection phaseModels) {
|
||||
@Valid AssistantLlmSelection llmSelection) {
|
||||
|
||||
public AssistantSessionCreateRequest(String model) {
|
||||
this(model, null);
|
||||
this(model == null ? null : new AssistantLlmSelection("InternalOllama", model, null));
|
||||
}
|
||||
|
||||
public AssistantSessionCreateRequest(String model, AssistantModelSelection phaseModels) {
|
||||
this(model == null ? null : new AssistantLlmSelection("InternalOllama", model, phaseModels));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import it.cnr.isti.workflow.manager.flows.validation.ValidationError;
|
|||
public record AssistantSessionView(
|
||||
String id,
|
||||
String owner,
|
||||
String model,
|
||||
AssistantLlmSelection llmSelection,
|
||||
Instant createdAt,
|
||||
Instant updatedAt,
|
||||
String lastCallId,
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
|
|||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import it.cnr.isti.workflow.manager.assistant.AssistantConversationService;
|
||||
import it.cnr.isti.workflow.manager.assistant.AssistantProperties;
|
||||
import it.cnr.isti.workflow.manager.assistant.FlowAssistantService;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantExplainRequest;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantExplainResponse;
|
||||
|
|
@ -35,8 +36,8 @@ import jakarta.validation.Valid;
|
|||
@RequestMapping("/assistant")
|
||||
public class AssistantController {
|
||||
|
||||
private static final String INTERNAL_PROVIDER_NAME = "InternalOllama";
|
||||
private static final String MODELS_RETRIEVER_URL = "/retriever/LLM/models?provider=InternalOllama";
|
||||
private static final String PROVIDERS_RETRIEVER_URL = "/retriever/LLM/providers";
|
||||
private static final String MODELS_RETRIEVER_URL_TEMPLATE = "/retriever/LLM/models?provider={provider}";
|
||||
|
||||
@Autowired
|
||||
private FlowAssistantService flowAssistantService;
|
||||
|
|
@ -44,6 +45,9 @@ public class AssistantController {
|
|||
@Autowired
|
||||
private AssistantConversationService assistantConversationService;
|
||||
|
||||
@Autowired
|
||||
private AssistantProperties assistantProperties;
|
||||
|
||||
@Value("${app.assistant.default-model}")
|
||||
private String defaultAssistantModel;
|
||||
|
||||
|
|
@ -89,15 +93,16 @@ public class AssistantController {
|
|||
@Operation(summary = "Get assistant config", description = "Returns assistant configuration for the GUI, including the default model and retriever URL for dynamic model selection.")
|
||||
public AssistantConfigView getConfig() {
|
||||
return new AssistantConfigView(
|
||||
INTERNAL_PROVIDER_NAME,
|
||||
assistantProperties.getDefaultProvider(),
|
||||
defaultAssistantModel,
|
||||
MODELS_RETRIEVER_URL,
|
||||
PROVIDERS_RETRIEVER_URL,
|
||||
MODELS_RETRIEVER_URL_TEMPLATE,
|
||||
new AssistantModelSelection(defaultPlanningModel, defaultJsonModel, defaultRepairModel));
|
||||
}
|
||||
|
||||
@PostMapping("/sessions")
|
||||
@SecurityRequirement(name = "bearerAuth")
|
||||
@Operation(summary = "Create assistant session", description = "Creates a chat assistant session bound to the selected internal model.")
|
||||
@Operation(summary = "Create assistant session", description = "Creates a chat assistant session using the default LLM configuration or a client-selected provider and model.")
|
||||
public AssistantSessionView createSession(@RequestBody @Valid AssistantSessionCreateRequest request,
|
||||
@AuthenticationPrincipal LoginEntity userDetails) {
|
||||
return assistantConversationService.createSession(userDetails.getUsername(), request);
|
||||
|
|
|
|||
|
|
@ -55,7 +55,8 @@ public class BiasAnnotationsController {
|
|||
options.put("severity", describe(BiasSeverity.class));
|
||||
options.put("status", describe(BiasAnnotationStatus.class));
|
||||
options.put("source", describe(BiasAnnotationSource.class));
|
||||
options.put("behavioralProbe.activationMode", describe(BiasActivationMode.class));
|
||||
options.put("biasProbe.activationMode", describe(BiasActivationMode.class));
|
||||
options.put("mitigationProbe.activationMode", describe(BiasActivationMode.class));
|
||||
|
||||
return new BiasAnnotationDescriptor(
|
||||
BlockBiasAnnotation.class.getSimpleName(),
|
||||
|
|
|
|||
|
|
@ -427,11 +427,13 @@ public class ExecutionsService {
|
|||
String experimentId = java.util.UUID.randomUUID().toString();
|
||||
BiasExecutionContext biasContext = new BiasExecutionContext(
|
||||
experimentId,
|
||||
BiasExecutionMode.BIAS_VARIANT,
|
||||
resolution.annotationIdsByNode(),
|
||||
BiasExecutionMode.EXPERIMENT,
|
||||
resolution.biasAnnotationIdsByNode(),
|
||||
request.externalSideEffectPolicy(),
|
||||
request.confirmExternalSideEffects(),
|
||||
resolution.subflowActivatedContainerIds());
|
||||
resolution.biasSubflowActivatedContainerIds(),
|
||||
resolution.mitigationAnnotationIdsByNode(),
|
||||
resolution.mitigationSubflowActivatedContainerIds());
|
||||
String runGroupId = resolveHistoryGroupId(source);
|
||||
int nextRunNumber = nextRunNumber(source.getSourceFlowId(), runGroupId, owner, source.getRunNumber());
|
||||
ExecutionObject rerun = createExecution(source.getName(), source.getFlow(), owner, runGroupId,
|
||||
|
|
@ -444,8 +446,10 @@ public class ExecutionsService {
|
|||
private BiasActivationResolution validateBiasActivations(FlowData flow, List<BiasActivation> activations) {
|
||||
Map<String, it.cnr.isti.workflow.manager.flows.model.FlowNode> nodesById = flow.getNodes().stream()
|
||||
.collect(Collectors.toMap(it.cnr.isti.workflow.manager.flows.model.FlowNode::getId, node -> node));
|
||||
Map<String, List<String>> result = new LinkedHashMap<>();
|
||||
Set<String> subflowActivatedContainerIds = new LinkedHashSet<>();
|
||||
Map<String, List<String>> biasResult = new LinkedHashMap<>();
|
||||
Map<String, List<String>> mitigationResult = new LinkedHashMap<>();
|
||||
Set<String> biasSubflowActivatedContainerIds = new LinkedHashSet<>();
|
||||
Set<String> mitigationSubflowActivatedContainerIds = new LinkedHashSet<>();
|
||||
for (BiasActivation activation : activations) {
|
||||
it.cnr.isti.workflow.manager.flows.model.FlowNode node = nodesById.get(activation.nodeId());
|
||||
if (node == null) {
|
||||
|
|
@ -460,12 +464,18 @@ public class ExecutionsService {
|
|||
"includeSubflow can only be used on a container node: " + node.getId());
|
||||
}
|
||||
List<FlowData> subFlows = subFlowsOf(container);
|
||||
if (!BiasContainerPropagation.hasExecutableAnnotations(subFlows)) {
|
||||
if (!BiasContainerPropagation.hasExecutableAnnotations(subFlows, activation.direction())) {
|
||||
throw new BiasApiException(HttpStatus.BAD_REQUEST, ValidationErrorCode.BIAS_SUBFLOW_NOT_EXECUTABLE,
|
||||
entity, node.getId(), "includeSubflow",
|
||||
"Container subflow has no executable bias annotation: " + node.getId());
|
||||
"Container subflow has no executable annotation for direction "
|
||||
+ activation.direction() + ": " + node.getId());
|
||||
}
|
||||
if (activation.direction().includesBias()) {
|
||||
biasSubflowActivatedContainerIds.add(node.getId());
|
||||
}
|
||||
if (activation.direction().includesMitigation()) {
|
||||
mitigationSubflowActivatedContainerIds.add(node.getId());
|
||||
}
|
||||
subflowActivatedContainerIds.add(node.getId());
|
||||
} else if (activation.annotationIds().isEmpty()) {
|
||||
throw new BiasApiException(HttpStatus.BAD_REQUEST, ValidationErrorCode.BIAS_ACTIVATION_ANNOTATIONS_REQUIRED,
|
||||
entity, node.getId(), "annotationIds",
|
||||
|
|
@ -482,19 +492,35 @@ public class ExecutionsService {
|
|||
entity, node.getId(), "biasAnnotations",
|
||||
"Bias annotation " + annotationId + " not found on node " + node.getId());
|
||||
}
|
||||
if (annotation.behavioralProbe() == null) {
|
||||
if (activation.direction().includesBias() && annotation.biasProbe() == null) {
|
||||
throw new BiasApiException(HttpStatus.BAD_REQUEST, ValidationErrorCode.BIAS_ANNOTATION_NOT_EXECUTABLE,
|
||||
entity, node.getId(), "biasAnnotations",
|
||||
"Bias annotation " + annotationId + " has no behavioralProbe");
|
||||
"Bias annotation " + annotationId + " has no biasProbe");
|
||||
}
|
||||
if (activation.direction().includesMitigation() && annotation.mitigationProbe() == null) {
|
||||
throw new BiasApiException(HttpStatus.BAD_REQUEST, ValidationErrorCode.BIAS_ANNOTATION_NOT_EXECUTABLE,
|
||||
entity, node.getId(), "biasAnnotations",
|
||||
"Bias annotation " + annotationId + " has no mitigationProbe");
|
||||
}
|
||||
}
|
||||
if (!activation.annotationIds().isEmpty()) {
|
||||
result.computeIfAbsent(node.getId(), ignored -> new java.util.ArrayList<>())
|
||||
.addAll(activation.annotationIds());
|
||||
if (activation.direction().includesBias()) {
|
||||
biasResult.computeIfAbsent(node.getId(), ignored -> new java.util.ArrayList<>())
|
||||
.addAll(activation.annotationIds());
|
||||
}
|
||||
if (activation.direction().includesMitigation()) {
|
||||
mitigationResult.computeIfAbsent(node.getId(), ignored -> new java.util.ArrayList<>())
|
||||
.addAll(activation.annotationIds());
|
||||
}
|
||||
}
|
||||
}
|
||||
result.replaceAll((nodeId, annotationIds) -> annotationIds.stream().distinct().toList());
|
||||
return new BiasActivationResolution(Map.copyOf(result), Set.copyOf(subflowActivatedContainerIds));
|
||||
biasResult.replaceAll((nodeId, annotationIds) -> annotationIds.stream().distinct().toList());
|
||||
mitigationResult.replaceAll((nodeId, annotationIds) -> annotationIds.stream().distinct().toList());
|
||||
return new BiasActivationResolution(
|
||||
Map.copyOf(biasResult),
|
||||
Map.copyOf(mitigationResult),
|
||||
Set.copyOf(biasSubflowActivatedContainerIds),
|
||||
Set.copyOf(mitigationSubflowActivatedContainerIds));
|
||||
}
|
||||
|
||||
private List<FlowData> subFlowsOf(Container<?> container) {
|
||||
|
|
@ -509,8 +535,11 @@ public class ExecutionsService {
|
|||
return subFlows;
|
||||
}
|
||||
|
||||
private record BiasActivationResolution(Map<String, List<String>> annotationIdsByNode,
|
||||
Set<String> subflowActivatedContainerIds) {
|
||||
private record BiasActivationResolution(
|
||||
Map<String, List<String>> biasAnnotationIdsByNode,
|
||||
Map<String, List<String>> mitigationAnnotationIdsByNode,
|
||||
Set<String> biasSubflowActivatedContainerIds,
|
||||
Set<String> mitigationSubflowActivatedContainerIds) {
|
||||
}
|
||||
|
||||
@Transactional
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package it.cnr.isti.workflow.manager.executions.api;
|
||||
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasActivationMode;
|
||||
import it.cnr.isti.workflow.manager.executions.bias.BiasInterventionDirection;
|
||||
|
||||
/**
|
||||
* Bias behavioural probe that is active on a step during a bias variant rerun.
|
||||
|
|
@ -9,6 +10,7 @@ import it.cnr.isti.workflow.manager.flows.model.bias.BiasActivationMode;
|
|||
*/
|
||||
public record ActiveBiasProbeView(
|
||||
String annotationId,
|
||||
BiasInterventionDirection direction,
|
||||
BiasActivationMode activationMode,
|
||||
String instruction) {
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import java.util.Map;
|
|||
import java.util.stream.Collectors;
|
||||
|
||||
import it.cnr.isti.workflow.manager.executions.bias.BiasExecutionContext;
|
||||
import it.cnr.isti.workflow.manager.executions.bias.BiasInterventionDirection;
|
||||
import it.cnr.isti.workflow.manager.executions.steps.Input;
|
||||
import it.cnr.isti.workflow.manager.executions.steps.Output;
|
||||
import it.cnr.isti.workflow.manager.executions.steps.Step;
|
||||
|
|
@ -54,20 +55,31 @@ public class ExecutionStepView {
|
|||
private static List<ActiveBiasProbeView> activeBiasProbes(Step<?> step) {
|
||||
FlowNode node = step.getNode();
|
||||
BiasExecutionContext biasContext = step.getBiasExecutionContext();
|
||||
if (node == null || biasContext == null || !biasContext.isVariantFor(node.getId())) {
|
||||
if (node == null || biasContext == null || !biasContext.isExperimentFor(node.getId())) {
|
||||
return List.of();
|
||||
}
|
||||
List<String> activeIds = biasContext.annotationIdsFor(node.getId());
|
||||
Map<String, BlockBiasAnnotation> byId = node.getBiasAnnotations().stream()
|
||||
.filter(annotation -> annotation != null && annotation.behavioralProbe() != null)
|
||||
.collect(Collectors.toMap(BlockBiasAnnotation::id, annotation -> annotation, (first, ignored) -> first));
|
||||
return activeIds.stream()
|
||||
.map(byId::get)
|
||||
.filter(annotation -> annotation != null)
|
||||
.collect(Collectors.toMap(BlockBiasAnnotation::id, annotation -> annotation, (first, ignored) -> first));
|
||||
List<ActiveBiasProbeView> active = new java.util.ArrayList<>();
|
||||
biasContext.biasAnnotationIdsFor(node.getId()).stream()
|
||||
.map(byId::get)
|
||||
.filter(annotation -> annotation != null && annotation.biasProbe() != null)
|
||||
.map(annotation -> new ActiveBiasProbeView(
|
||||
annotation.id(),
|
||||
annotation.behavioralProbe().activationMode(),
|
||||
annotation.behavioralProbe().instruction()))
|
||||
.toList();
|
||||
BiasInterventionDirection.BIAS,
|
||||
annotation.biasProbe().activationMode(),
|
||||
annotation.biasProbe().instruction()))
|
||||
.forEach(active::add);
|
||||
biasContext.mitigationAnnotationIdsFor(node.getId()).stream()
|
||||
.map(byId::get)
|
||||
.filter(annotation -> annotation != null && annotation.mitigationProbe() != null)
|
||||
.map(annotation -> new ActiveBiasProbeView(
|
||||
annotation.id(),
|
||||
BiasInterventionDirection.MITIGATION,
|
||||
annotation.mitigationProbe().activationMode(),
|
||||
annotation.mitigationProbe().instruction()))
|
||||
.forEach(active::add);
|
||||
return List.copyOf(active);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package it.cnr.isti.workflow.manager.executions.bias;
|
|||
import java.util.List;
|
||||
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
|
||||
/**
|
||||
* Activates a subset of a node's bias annotations for a bias rerun.
|
||||
|
|
@ -16,13 +17,11 @@ import jakarta.validation.constraints.NotBlank;
|
|||
public record BiasActivation(
|
||||
@NotBlank String nodeId,
|
||||
List<@NotBlank String> annotationIds,
|
||||
boolean includeSubflow) {
|
||||
boolean includeSubflow,
|
||||
@NotNull BiasInterventionDirection direction) {
|
||||
|
||||
public BiasActivation {
|
||||
annotationIds = annotationIds == null ? List.of() : annotationIds.stream().distinct().toList();
|
||||
}
|
||||
|
||||
public BiasActivation(String nodeId, List<String> annotationIds) {
|
||||
this(nodeId, annotationIds, false);
|
||||
direction = java.util.Objects.requireNonNull(direction, "direction");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,46 +11,69 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
|
|||
public record BiasExecutionContext(
|
||||
String experimentId,
|
||||
BiasExecutionMode mode,
|
||||
Map<String, List<String>> activeAnnotationIdsByNode,
|
||||
Map<String, List<String>> activeBiasAnnotationIdsByNode,
|
||||
ExternalSideEffectPolicy externalSideEffectPolicy,
|
||||
boolean externalSideEffectsConfirmed,
|
||||
Set<String> subflowActivatedContainerIds) {
|
||||
Set<String> biasSubflowActivatedContainerIds,
|
||||
Map<String, List<String>> activeMitigationAnnotationIdsByNode,
|
||||
Set<String> mitigationSubflowActivatedContainerIds) {
|
||||
|
||||
public BiasExecutionContext {
|
||||
mode = mode == null ? BiasExecutionMode.NORMAL : mode;
|
||||
externalSideEffectPolicy = externalSideEffectPolicy == null ? ExternalSideEffectPolicy.BLOCK : externalSideEffectPolicy;
|
||||
Map<String, List<String>> normalized = new LinkedHashMap<>();
|
||||
if (activeAnnotationIdsByNode != null) {
|
||||
activeAnnotationIdsByNode.forEach((nodeId, annotationIds) -> {
|
||||
if (activeBiasAnnotationIdsByNode != null) {
|
||||
activeBiasAnnotationIdsByNode.forEach((nodeId, annotationIds) -> {
|
||||
if (nodeId != null && !nodeId.isBlank()) {
|
||||
normalized.put(nodeId, annotationIds == null ? List.of() : annotationIds.stream().distinct().toList());
|
||||
}
|
||||
});
|
||||
}
|
||||
activeAnnotationIdsByNode = Map.copyOf(normalized);
|
||||
subflowActivatedContainerIds = subflowActivatedContainerIds == null
|
||||
activeBiasAnnotationIdsByNode = Map.copyOf(normalized);
|
||||
Map<String, List<String>> normalizedMitigations = new LinkedHashMap<>();
|
||||
if (activeMitigationAnnotationIdsByNode != null) {
|
||||
activeMitigationAnnotationIdsByNode.forEach((nodeId, annotationIds) -> {
|
||||
if (nodeId != null && !nodeId.isBlank()) {
|
||||
normalizedMitigations.put(nodeId,
|
||||
annotationIds == null ? List.of() : annotationIds.stream().distinct().toList());
|
||||
}
|
||||
});
|
||||
}
|
||||
activeMitigationAnnotationIdsByNode = Map.copyOf(normalizedMitigations);
|
||||
biasSubflowActivatedContainerIds = biasSubflowActivatedContainerIds == null
|
||||
? Set.of()
|
||||
: subflowActivatedContainerIds.stream()
|
||||
: biasSubflowActivatedContainerIds.stream()
|
||||
.filter(id -> id != null && !id.isBlank())
|
||||
.collect(Collectors.toUnmodifiableSet());
|
||||
mitigationSubflowActivatedContainerIds = mitigationSubflowActivatedContainerIds == null
|
||||
? Set.of()
|
||||
: mitigationSubflowActivatedContainerIds.stream()
|
||||
.filter(id -> id != null && !id.isBlank())
|
||||
.collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
public BiasExecutionContext(String experimentId, BiasExecutionMode mode,
|
||||
Map<String, List<String>> activeAnnotationIdsByNode, ExternalSideEffectPolicy externalSideEffectPolicy,
|
||||
boolean externalSideEffectsConfirmed) {
|
||||
this(experimentId, mode, activeAnnotationIdsByNode, externalSideEffectPolicy, externalSideEffectsConfirmed, Set.of());
|
||||
}
|
||||
|
||||
public static BiasExecutionContext normal() {
|
||||
return new BiasExecutionContext(null, BiasExecutionMode.NORMAL, Map.of(), ExternalSideEffectPolicy.BLOCK, false, Set.of());
|
||||
return new BiasExecutionContext(null, BiasExecutionMode.NORMAL, Map.of(), ExternalSideEffectPolicy.BLOCK,
|
||||
false, Set.of(), Map.of(), Set.of());
|
||||
}
|
||||
|
||||
public List<String> annotationIdsFor(String nodeId) {
|
||||
return activeAnnotationIdsByNode.getOrDefault(nodeId, List.of());
|
||||
public List<String> biasAnnotationIdsFor(String nodeId) {
|
||||
return activeBiasAnnotationIdsByNode.getOrDefault(nodeId, List.of());
|
||||
}
|
||||
|
||||
public boolean isVariantFor(String nodeId) {
|
||||
return mode == BiasExecutionMode.BIAS_VARIANT && !annotationIdsFor(nodeId).isEmpty();
|
||||
public List<String> mitigationAnnotationIdsFor(String nodeId) {
|
||||
return activeMitigationAnnotationIdsByNode.getOrDefault(nodeId, List.of());
|
||||
}
|
||||
|
||||
public List<String> allAnnotationIdsFor(String nodeId) {
|
||||
return java.util.stream.Stream.concat(biasAnnotationIdsFor(nodeId).stream(),
|
||||
mitigationAnnotationIdsFor(nodeId).stream())
|
||||
.distinct()
|
||||
.toList();
|
||||
}
|
||||
|
||||
public boolean isExperimentFor(String nodeId) {
|
||||
return mode == BiasExecutionMode.EXPERIMENT && !allAnnotationIdsFor(nodeId).isEmpty();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -58,7 +81,26 @@ public record BiasExecutionContext(
|
|||
* propagated to its nodes (the "coarse" container propagation model).
|
||||
*/
|
||||
public boolean isSubflowActivatedFor(String containerId) {
|
||||
return mode == BiasExecutionMode.BIAS_VARIANT && subflowActivatedContainerIds.contains(containerId);
|
||||
return isBiasSubflowActivatedFor(containerId) || isMitigationSubflowActivatedFor(containerId);
|
||||
}
|
||||
|
||||
public boolean isBiasSubflowActivatedFor(String containerId) {
|
||||
return mode == BiasExecutionMode.EXPERIMENT && biasSubflowActivatedContainerIds.contains(containerId);
|
||||
}
|
||||
|
||||
public boolean isMitigationSubflowActivatedFor(String containerId) {
|
||||
return mode == BiasExecutionMode.EXPERIMENT && mitigationSubflowActivatedContainerIds.contains(containerId);
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
public BiasInterventionDirection interventionDirection() {
|
||||
boolean bias = !activeBiasAnnotationIdsByNode.isEmpty() || !biasSubflowActivatedContainerIds.isEmpty();
|
||||
boolean mitigation = !activeMitigationAnnotationIdsByNode.isEmpty()
|
||||
|| !mitigationSubflowActivatedContainerIds.isEmpty();
|
||||
if (bias && mitigation) {
|
||||
return BiasInterventionDirection.BOTH;
|
||||
}
|
||||
return mitigation ? BiasInterventionDirection.MITIGATION : BiasInterventionDirection.BIAS;
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
|
|
|
|||
|
|
@ -2,6 +2,5 @@ package it.cnr.isti.workflow.manager.executions.bias;
|
|||
|
||||
public enum BiasExecutionMode {
|
||||
NORMAL,
|
||||
BIAS_BASELINE,
|
||||
BIAS_VARIANT
|
||||
EXPERIMENT
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,13 +5,15 @@ import java.util.List;
|
|||
import jakarta.validation.constraints.Max;
|
||||
import jakarta.validation.constraints.Min;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
|
||||
public record BiasImpactExperimentRequest(
|
||||
@NotEmpty List<String> annotationIds,
|
||||
@Min(1) @Max(10) Integer repetitions,
|
||||
Boolean includeRawOutputs,
|
||||
ExternalSideEffectPolicy externalSideEffectPolicy,
|
||||
Boolean confirmExternalSideEffects) {
|
||||
Boolean confirmExternalSideEffects,
|
||||
@NotNull BiasInterventionDirection direction) {
|
||||
|
||||
public BiasImpactExperimentRequest {
|
||||
annotationIds = annotationIds == null ? List.of() : annotationIds.stream().distinct().toList();
|
||||
|
|
@ -19,5 +21,6 @@ public record BiasImpactExperimentRequest(
|
|||
includeRawOutputs = includeRawOutputs == null || includeRawOutputs;
|
||||
externalSideEffectPolicy = externalSideEffectPolicy == null ? ExternalSideEffectPolicy.BLOCK : externalSideEffectPolicy;
|
||||
confirmExternalSideEffects = Boolean.TRUE.equals(confirmExternalSideEffects);
|
||||
direction = java.util.Objects.requireNonNull(direction, "direction");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ public record BiasImpactReport(
|
|||
String id,
|
||||
String experimentId,
|
||||
BiasExperimentKind kind,
|
||||
BiasInterventionDirection interventionDirection,
|
||||
String baselineExecutionId,
|
||||
String biasedExecutionId,
|
||||
String nodeId,
|
||||
|
|
@ -23,6 +24,7 @@ public record BiasImpactReport(
|
|||
List<String> warnings) {
|
||||
|
||||
public BiasImpactReport {
|
||||
interventionDirection = java.util.Objects.requireNonNull(interventionDirection, "interventionDirection");
|
||||
annotationIds = annotationIds == null ? List.of() : List.copyOf(annotationIds);
|
||||
downstreamImpact = downstreamImpact == null ? List.of() : List.copyOf(downstreamImpact);
|
||||
routingChanges = routingChanges == null ? List.of() : List.copyOf(routingChanges);
|
||||
|
|
|
|||
|
|
@ -56,10 +56,13 @@ public class BiasImpactService {
|
|||
String experimentId = UUID.randomUUID().toString();
|
||||
BiasExecutionContext variantContext = new BiasExecutionContext(
|
||||
experimentId,
|
||||
BiasExecutionMode.BIAS_VARIANT,
|
||||
Map.of(block.getId(), request.annotationIds()),
|
||||
BiasExecutionMode.EXPERIMENT,
|
||||
request.direction().includesBias() ? Map.of(block.getId(), request.annotationIds()) : Map.of(),
|
||||
request.externalSideEffectPolicy(),
|
||||
request.confirmExternalSideEffects());
|
||||
request.confirmExternalSideEffects(),
|
||||
Set.of(),
|
||||
request.direction().includesMitigation() ? Map.of(block.getId(), request.annotationIds()) : Map.of(),
|
||||
Set.of());
|
||||
|
||||
Map<String, Object> baselineOutput = outputsOf(baseline, stepId);
|
||||
List<Map<String, Object>> biasedOutputs = new ArrayList<>();
|
||||
|
|
@ -85,6 +88,7 @@ public class BiasImpactService {
|
|||
UUID.randomUUID().toString(),
|
||||
experimentId,
|
||||
BiasExperimentKind.ISOLATED_STEP,
|
||||
request.direction(),
|
||||
baseline.getId(),
|
||||
null,
|
||||
block.getId(),
|
||||
|
|
@ -98,9 +102,10 @@ public class BiasImpactService {
|
|||
List.of(),
|
||||
mockedSideEffects,
|
||||
impact.outputChanged()
|
||||
? "The activated bias changed the observed output of the selected block."
|
||||
? "The activated " + request.direction().name().toLowerCase()
|
||||
+ " intervention changed the observed output of the selected block."
|
||||
: "No output change was observed for the selected block.",
|
||||
List.of("The baseline output was captured from the selected completed execution; only biased variants were repeated."));
|
||||
List.of("The baseline output was captured from the selected completed execution; only experimental variants were repeated."));
|
||||
return persist(report, owner, request.includeRawOutputs());
|
||||
}
|
||||
|
||||
|
|
@ -130,7 +135,7 @@ public class BiasImpactService {
|
|||
throw new BiasApiException(HttpStatus.BAD_REQUEST, ValidationErrorCode.BIAS_BLOCK_NOT_SUPPORTED,
|
||||
"block", block.getId(), "The block does not support an isolated bias experiment: " + block.getName());
|
||||
}
|
||||
validateAnnotationIds(block, request.annotationIds());
|
||||
validateAnnotationIds(block, request.annotationIds(), request.direction());
|
||||
if (adapterRegistry.hasExternalSideEffects(block)) {
|
||||
if (request.externalSideEffectPolicy() == ExternalSideEffectPolicy.BLOCK) {
|
||||
throw new BiasApiException(HttpStatus.CONFLICT, ValidationErrorCode.BIAS_SIDE_EFFECT_BLOCKED,
|
||||
|
|
@ -159,7 +164,7 @@ public class BiasImpactService {
|
|||
requireFinal(baseline, "baseline");
|
||||
requireFinal(biased, "biased");
|
||||
BiasExecutionContext context = biased.getBiasExecutionContext();
|
||||
if (context == null || context.mode() != BiasExecutionMode.BIAS_VARIANT) {
|
||||
if (context == null || context.mode() != BiasExecutionMode.EXPERIMENT) {
|
||||
throw new BiasApiException(HttpStatus.BAD_REQUEST, ValidationErrorCode.BIAS_EXECUTION_NOT_VARIANT,
|
||||
"execution", biasedExecutionId, "The compared execution is not a bias variant");
|
||||
}
|
||||
|
|
@ -176,8 +181,10 @@ public class BiasImpactService {
|
|||
return existingReport.get().getReport();
|
||||
}
|
||||
|
||||
Set<String> activatedNodeIds = new LinkedHashSet<>(context.activeAnnotationIdsByNode().keySet());
|
||||
activatedNodeIds.addAll(context.subflowActivatedContainerIds());
|
||||
Set<String> activatedNodeIds = new LinkedHashSet<>(context.activeBiasAnnotationIdsByNode().keySet());
|
||||
activatedNodeIds.addAll(context.activeMitigationAnnotationIdsByNode().keySet());
|
||||
activatedNodeIds.addAll(context.biasSubflowActivatedContainerIds());
|
||||
activatedNodeIds.addAll(context.mitigationSubflowActivatedContainerIds());
|
||||
Set<String> downstreamIds = downstreamNodeIds(baseline, activatedNodeIds);
|
||||
List<BiasDownstreamImpact> downstream = downstreamIds.stream()
|
||||
.map(nodeId -> compareStep(baseline, biased, nodeId, includeRawOutputs))
|
||||
|
|
@ -189,7 +196,9 @@ public class BiasImpactService {
|
|||
List<BiasRoutingChange> routing = activatedNodeIds.stream()
|
||||
.flatMap(nodeId -> routingChanges(nodeId, outputsOf(baseline, nodeId), outputsOf(biased, nodeId)).stream())
|
||||
.toList();
|
||||
List<String> annotationIds = context.activeAnnotationIdsByNode().values().stream()
|
||||
List<String> annotationIds = java.util.stream.Stream.concat(
|
||||
context.activeBiasAnnotationIdsByNode().values().stream(),
|
||||
context.activeMitigationAnnotationIdsByNode().values().stream())
|
||||
.flatMap(Collection::stream)
|
||||
.distinct()
|
||||
.toList();
|
||||
|
|
@ -211,6 +220,7 @@ public class BiasImpactService {
|
|||
UUID.randomUUID().toString(),
|
||||
context.experimentId(),
|
||||
BiasExperimentKind.FULL_FLOW,
|
||||
context.interventionDirection(),
|
||||
baseline.getId(),
|
||||
biased.getId(),
|
||||
activatedNodeIds.size() == 1 ? activatedNodeIds.iterator().next() : null,
|
||||
|
|
@ -264,7 +274,8 @@ public class BiasImpactService {
|
|||
return report;
|
||||
}
|
||||
|
||||
private void validateAnnotationIds(Block<?> block, List<String> annotationIds) {
|
||||
private void validateAnnotationIds(Block<?> block, List<String> annotationIds,
|
||||
BiasInterventionDirection direction) {
|
||||
Map<String, it.cnr.isti.workflow.manager.flows.model.bias.BlockBiasAnnotation> byId = new LinkedHashMap<>();
|
||||
block.getBiasAnnotations().forEach(annotation -> byId.put(annotation.id(), annotation));
|
||||
for (String annotationId : annotationIds) {
|
||||
|
|
@ -274,10 +285,15 @@ public class BiasImpactService {
|
|||
"block", block.getId(), "biasAnnotations",
|
||||
"Bias annotation " + annotationId + " not found on block " + block.getId());
|
||||
}
|
||||
if (annotation.behavioralProbe() == null) {
|
||||
if (direction.includesBias() && annotation.biasProbe() == null) {
|
||||
throw new BiasApiException(HttpStatus.BAD_REQUEST, ValidationErrorCode.BIAS_ANNOTATION_NOT_EXECUTABLE,
|
||||
"block", block.getId(), "biasAnnotations",
|
||||
"Bias annotation " + annotationId + " has no behavioralProbe");
|
||||
"Bias annotation " + annotationId + " has no biasProbe");
|
||||
}
|
||||
if (direction.includesMitigation() && annotation.mitigationProbe() == null) {
|
||||
throw new BiasApiException(HttpStatus.BAD_REQUEST, ValidationErrorCode.BIAS_ANNOTATION_NOT_EXECUTABLE,
|
||||
"block", block.getId(), "biasAnnotations",
|
||||
"Bias annotation " + annotationId + " has no mitigationProbe");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
package it.cnr.isti.workflow.manager.executions.bias;
|
||||
|
||||
/** Selects which executable side of a bias annotation is applied. */
|
||||
public enum BiasInterventionDirection {
|
||||
BIAS,
|
||||
MITIGATION,
|
||||
BOTH;
|
||||
|
||||
public boolean includesBias() {
|
||||
return this == BIAS || this == BOTH;
|
||||
}
|
||||
|
||||
public boolean includesMitigation() {
|
||||
return this == MITIGATION || this == BOTH;
|
||||
}
|
||||
}
|
||||
|
|
@ -4,13 +4,13 @@ import java.util.Set;
|
|||
|
||||
import it.cnr.isti.workflow.manager.flows.model.FlowNode;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasActivationMode;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BlockBiasAnnotation;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasBehavioralProbe;
|
||||
|
||||
public interface BiasBehaviorAdapter {
|
||||
|
||||
boolean supports(FlowNode node, BiasActivationMode mode);
|
||||
|
||||
void apply(BiasPreparedExecution execution, BlockBiasAnnotation annotation);
|
||||
void apply(BiasPreparedExecution execution, BiasBehavioralProbe probe);
|
||||
|
||||
Set<BiasActivationMode> supportedModes(FlowNode node);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import it.cnr.isti.workflow.manager.executions.ExecutionEventLogger;
|
|||
import it.cnr.isti.workflow.manager.executions.ExecutionEventType;
|
||||
import it.cnr.isti.workflow.manager.executions.bias.BiasExecutionContext;
|
||||
import it.cnr.isti.workflow.manager.executions.bias.BiasExecutionMode;
|
||||
import it.cnr.isti.workflow.manager.executions.bias.BiasInterventionDirection;
|
||||
import it.cnr.isti.workflow.manager.executions.bias.ExternalSideEffectPolicy;
|
||||
import it.cnr.isti.workflow.manager.executions.bias.BiasApiException;
|
||||
import it.cnr.isti.workflow.manager.executions.bias.BiasMockedSideEffect;
|
||||
|
|
@ -25,6 +26,7 @@ import it.cnr.isti.workflow.manager.flows.validation.ValidationErrorCode;
|
|||
import it.cnr.isti.workflow.manager.executions.steps.Input;
|
||||
import it.cnr.isti.workflow.manager.flows.model.FlowNode;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasActivationMode;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasBehavioralProbe;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BlockBiasAnnotation;
|
||||
|
||||
@Component
|
||||
|
|
@ -39,34 +41,40 @@ public class BiasBehaviorAdapterRegistry {
|
|||
public static BiasPreparedExecution prepare(FlowNode node, List<Input> inputs, Map<String, Object> executionVariables,
|
||||
BiasExecutionContext context, ExecutionEventLogger eventLogger) {
|
||||
BiasExecutionContext effectiveContext = context == null ? BiasExecutionContext.normal() : context;
|
||||
List<BlockBiasAnnotation> activeAnnotations = resolveActiveAnnotations(node, effectiveContext);
|
||||
List<ActiveProbe> activeProbes = resolveActiveProbes(node, effectiveContext);
|
||||
List<BlockBiasAnnotation> activeAnnotations = activeProbes.stream()
|
||||
.map(ActiveProbe::annotation)
|
||||
.distinct()
|
||||
.toList();
|
||||
BiasPreparedExecution prepared = new BiasPreparedExecution(node, inputs, executionVariables, activeAnnotations);
|
||||
|
||||
enforceExternalSideEffectPolicy(prepared, effectiveContext, eventLogger);
|
||||
if (effectiveContext.mode() != BiasExecutionMode.BIAS_VARIANT) {
|
||||
if (effectiveContext.mode() != BiasExecutionMode.EXPERIMENT) {
|
||||
return prepared;
|
||||
}
|
||||
|
||||
for (BlockBiasAnnotation annotation : activeAnnotations) {
|
||||
if (annotation.behavioralProbe() == null) {
|
||||
throw new BiasApiException(HttpStatus.BAD_REQUEST, ValidationErrorCode.BIAS_ANNOTATION_NOT_EXECUTABLE,
|
||||
entityType(node), node.getId(), "biasAnnotations",
|
||||
"Bias annotation " + annotation.id() + " has no behavioralProbe");
|
||||
}
|
||||
BiasActivationMode mode = annotation.behavioralProbe().activationMode();
|
||||
// Bias is deliberately applied before mitigation so BOTH models a
|
||||
// corrective intervention over an already distorted execution.
|
||||
for (ActiveProbe activeProbe : activeProbes) {
|
||||
BiasActivationMode mode = activeProbe.probe().activationMode();
|
||||
BiasBehaviorAdapter adapter = adapters().stream()
|
||||
.filter(candidate -> candidate.supports(node, mode))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new BiasApiException(HttpStatus.BAD_REQUEST,
|
||||
ValidationErrorCode.BIAS_PROBE_MODE_UNSUPPORTED, entityType(node), node.getId(),
|
||||
"biasAnnotations", "Activation mode " + mode + " is not supported by node " + node.getName()));
|
||||
adapter.apply(prepared, annotation);
|
||||
adapter.apply(prepared, activeProbe.probe());
|
||||
}
|
||||
|
||||
if (!activeAnnotations.isEmpty() && eventLogger != null) {
|
||||
if (!activeProbes.isEmpty() && eventLogger != null) {
|
||||
eventLogger.info(ExecutionEventType.BIAS_EXPERIMENT_APPLIED,
|
||||
"Applied bias experiment to node " + node.getName(),
|
||||
Map.of("annotationIds", activeAnnotations.stream().map(BlockBiasAnnotation::id).toList()));
|
||||
"Applied bias experiment interventions to node " + node.getName(),
|
||||
Map.of(
|
||||
"annotationIds", activeAnnotations.stream().map(BlockBiasAnnotation::id).toList(),
|
||||
"directions", activeProbes.stream()
|
||||
.map(active -> active.direction().name())
|
||||
.distinct()
|
||||
.toList()));
|
||||
}
|
||||
return prepared;
|
||||
}
|
||||
|
|
@ -105,12 +113,13 @@ public class BiasBehaviorAdapterRegistry {
|
|||
return block != null && !block.isUserInteractive() && !supportedModes(block).isEmpty();
|
||||
}
|
||||
|
||||
private static List<BlockBiasAnnotation> resolveActiveAnnotations(FlowNode node, BiasExecutionContext context) {
|
||||
if (node == null || context.mode() != BiasExecutionMode.BIAS_VARIANT) {
|
||||
private static List<ActiveProbe> resolveActiveProbes(FlowNode node, BiasExecutionContext context) {
|
||||
if (node == null || context.mode() != BiasExecutionMode.EXPERIMENT) {
|
||||
return List.of();
|
||||
}
|
||||
List<String> activeIds = context.annotationIdsFor(node.getId());
|
||||
if (activeIds.isEmpty()) {
|
||||
List<String> biasIds = context.biasAnnotationIdsFor(node.getId());
|
||||
List<String> mitigationIds = context.mitigationAnnotationIdsFor(node.getId());
|
||||
if (biasIds.isEmpty() && mitigationIds.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
Map<String, BlockBiasAnnotation> byId = new LinkedHashMap<>();
|
||||
|
|
@ -119,17 +128,36 @@ public class BiasBehaviorAdapterRegistry {
|
|||
byId.put(annotation.id(), annotation);
|
||||
}
|
||||
});
|
||||
List<BlockBiasAnnotation> resolved = new ArrayList<>();
|
||||
for (String annotationId : activeIds) {
|
||||
List<ActiveProbe> resolved = new ArrayList<>();
|
||||
resolveActiveProbes(node, biasIds, byId, BiasInterventionDirection.BIAS, resolved);
|
||||
resolveActiveProbes(node, mitigationIds, byId, BiasInterventionDirection.MITIGATION, resolved);
|
||||
return List.copyOf(resolved);
|
||||
}
|
||||
|
||||
private static void resolveActiveProbes(FlowNode node, List<String> annotationIds,
|
||||
Map<String, BlockBiasAnnotation> byId, BiasInterventionDirection direction, List<ActiveProbe> resolved) {
|
||||
for (String annotationId : annotationIds) {
|
||||
BlockBiasAnnotation annotation = byId.get(annotationId);
|
||||
if (annotation == null) {
|
||||
throw new BiasApiException(HttpStatus.BAD_REQUEST, ValidationErrorCode.BIAS_ANNOTATION_NOT_FOUND,
|
||||
entityType(node), node.getId(), "biasAnnotations",
|
||||
"Bias annotation " + annotationId + " not found on node " + node.getId());
|
||||
}
|
||||
resolved.add(annotation);
|
||||
BiasBehavioralProbe probe = direction == BiasInterventionDirection.BIAS
|
||||
? annotation.biasProbe()
|
||||
: annotation.mitigationProbe();
|
||||
if (probe == null) {
|
||||
String field = direction == BiasInterventionDirection.BIAS ? "biasProbe" : "mitigationProbe";
|
||||
throw new BiasApiException(HttpStatus.BAD_REQUEST, ValidationErrorCode.BIAS_ANNOTATION_NOT_EXECUTABLE,
|
||||
entityType(node), node.getId(), "biasAnnotations",
|
||||
"Bias annotation " + annotation.id() + " has no " + field);
|
||||
}
|
||||
resolved.add(new ActiveProbe(annotation, probe, direction));
|
||||
}
|
||||
return List.copyOf(resolved);
|
||||
}
|
||||
|
||||
private record ActiveProbe(BlockBiasAnnotation annotation, BiasBehavioralProbe probe,
|
||||
BiasInterventionDirection direction) {
|
||||
}
|
||||
|
||||
private static void enforceExternalSideEffectPolicy(BiasPreparedExecution prepared, BiasExecutionContext context,
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import java.util.Objects;
|
|||
|
||||
import it.cnr.isti.workflow.manager.executions.bias.BiasExecutionContext;
|
||||
import it.cnr.isti.workflow.manager.executions.bias.BiasExecutionMode;
|
||||
import it.cnr.isti.workflow.manager.executions.bias.BiasInterventionDirection;
|
||||
import it.cnr.isti.workflow.manager.flows.model.FlowData;
|
||||
import it.cnr.isti.workflow.manager.flows.model.FlowNode;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BlockBiasAnnotation;
|
||||
|
|
@ -32,21 +33,38 @@ public final class BiasContainerPropagation {
|
|||
if (outerContext == null || !outerContext.isSubflowActivatedFor(containerId)) {
|
||||
return BiasExecutionContext.normal();
|
||||
}
|
||||
Map<String, List<String>> activeAnnotationIdsByNode = executableAnnotationIdsByNode(subFlows);
|
||||
if (activeAnnotationIdsByNode.isEmpty()) {
|
||||
Map<String, List<String>> activeBiasAnnotationIdsByNode = outerContext.isBiasSubflowActivatedFor(containerId)
|
||||
? executableAnnotationIdsByNode(subFlows, BiasInterventionDirection.BIAS)
|
||||
: Map.of();
|
||||
Map<String, List<String>> activeMitigationAnnotationIdsByNode =
|
||||
outerContext.isMitigationSubflowActivatedFor(containerId)
|
||||
? executableAnnotationIdsByNode(subFlows, BiasInterventionDirection.MITIGATION)
|
||||
: Map.of();
|
||||
if (activeBiasAnnotationIdsByNode.isEmpty() && activeMitigationAnnotationIdsByNode.isEmpty()) {
|
||||
return BiasExecutionContext.normal();
|
||||
}
|
||||
return new BiasExecutionContext(outerContext.experimentId(), BiasExecutionMode.BIAS_VARIANT,
|
||||
activeAnnotationIdsByNode, outerContext.externalSideEffectPolicy(),
|
||||
outerContext.externalSideEffectsConfirmed());
|
||||
return new BiasExecutionContext(outerContext.experimentId(), BiasExecutionMode.EXPERIMENT,
|
||||
activeBiasAnnotationIdsByNode, outerContext.externalSideEffectPolicy(),
|
||||
outerContext.externalSideEffectsConfirmed(), java.util.Set.of(),
|
||||
activeMitigationAnnotationIdsByNode, java.util.Set.of());
|
||||
}
|
||||
|
||||
/** Whether at least one node across the given subflows carries an executable probe. */
|
||||
public static boolean hasExecutableAnnotations(List<FlowData> subFlows) {
|
||||
return !executableAnnotationIdsByNode(subFlows).isEmpty();
|
||||
return hasExecutableAnnotations(subFlows, BiasInterventionDirection.BIAS)
|
||||
|| hasExecutableAnnotations(subFlows, BiasInterventionDirection.MITIGATION);
|
||||
}
|
||||
|
||||
private static Map<String, List<String>> executableAnnotationIdsByNode(List<FlowData> subFlows) {
|
||||
public static boolean hasExecutableAnnotations(List<FlowData> subFlows, BiasInterventionDirection direction) {
|
||||
if (direction == BiasInterventionDirection.BOTH) {
|
||||
return hasExecutableAnnotations(subFlows, BiasInterventionDirection.BIAS)
|
||||
&& hasExecutableAnnotations(subFlows, BiasInterventionDirection.MITIGATION);
|
||||
}
|
||||
return !executableAnnotationIdsByNode(subFlows, direction).isEmpty();
|
||||
}
|
||||
|
||||
private static Map<String, List<String>> executableAnnotationIdsByNode(List<FlowData> subFlows,
|
||||
BiasInterventionDirection direction) {
|
||||
Map<String, List<String>> byNode = new LinkedHashMap<>();
|
||||
for (FlowData subFlow : subFlows) {
|
||||
if (subFlow == null) {
|
||||
|
|
@ -55,7 +73,9 @@ public final class BiasContainerPropagation {
|
|||
for (FlowNode node : subFlow.getNodes()) {
|
||||
List<String> executableIds = node.getBiasAnnotations().stream()
|
||||
.filter(Objects::nonNull)
|
||||
.filter(annotation -> annotation.behavioralProbe() != null)
|
||||
.filter(annotation -> direction == BiasInterventionDirection.BIAS
|
||||
? annotation.biasProbe() != null
|
||||
: annotation.mitigationProbe() != null)
|
||||
.map(BlockBiasAnnotation::id)
|
||||
.toList();
|
||||
if (!executableIds.isEmpty()) {
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import it.cnr.isti.workflow.manager.blocks.Block;
|
|||
import it.cnr.isti.workflow.manager.executions.steps.Input;
|
||||
import it.cnr.isti.workflow.manager.flows.model.FlowNode;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasActivationMode;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BlockBiasAnnotation;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasBehavioralProbe;
|
||||
import it.cnr.isti.workflow.manager.ios.IOType;
|
||||
|
||||
@Component
|
||||
|
|
@ -22,14 +22,14 @@ public class InputBiasBehaviorAdapter implements BiasBehaviorAdapter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void apply(BiasPreparedExecution execution, BlockBiasAnnotation annotation) {
|
||||
List<String> targets = annotation.behavioralProbe().targetInputs();
|
||||
public void apply(BiasPreparedExecution execution, BiasBehavioralProbe probe) {
|
||||
List<String> targets = probe.targetInputs();
|
||||
List<Input> transformed = new ArrayList<>();
|
||||
for (Input input : execution.getInputs()) {
|
||||
boolean selected = targets.isEmpty() || targets.contains(input.getDescriptor().getName());
|
||||
boolean textual = input.getDescriptor().getType() == IOType.TEXT || input.getDescriptor().getType() == IOType.ANY;
|
||||
Object value = selected && textual
|
||||
? BiasRuntimeSupport.transformTextValue(annotation.behavioralProbe().instruction(), input.getValue())
|
||||
? BiasRuntimeSupport.transformTextValue(probe.instruction(), input.getValue())
|
||||
: input.getValue();
|
||||
transformed.add(Input.detached(input.getDescriptor(), value));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import org.springframework.stereotype.Component;
|
|||
import it.cnr.isti.workflow.manager.blocks.Block;
|
||||
import it.cnr.isti.workflow.manager.blocks.configurations.HTTPServerCallBlockConfiguration;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasActivationMode;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BlockBiasAnnotation;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasBehavioralProbe;
|
||||
import it.cnr.isti.workflow.manager.flows.model.FlowNode;
|
||||
|
||||
@Component
|
||||
|
|
@ -21,12 +21,8 @@ public class MockResponseBiasBehaviorAdapter implements BiasBehaviorAdapter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void apply(BiasPreparedExecution execution, BlockBiasAnnotation annotation) {
|
||||
Map<String, Object> result = new LinkedHashMap<>(annotation.behavioralProbe().mockOutputs());
|
||||
if (result.isEmpty()) {
|
||||
execution.getNode().getOutputs().forEach(output ->
|
||||
result.put(output.getName(), annotation.behavioralProbe().instruction()));
|
||||
}
|
||||
public void apply(BiasPreparedExecution execution, BiasBehavioralProbe probe) {
|
||||
Map<String, Object> result = new LinkedHashMap<>(probe.mockOutputs());
|
||||
execution.setBypassResult(result);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import org.springframework.stereotype.Component;
|
|||
import it.cnr.isti.workflow.manager.blocks.Block;
|
||||
import it.cnr.isti.workflow.manager.flows.model.FlowNode;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasActivationMode;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BlockBiasAnnotation;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasBehavioralProbe;
|
||||
|
||||
@Component
|
||||
public class OutputBiasBehaviorAdapter implements BiasBehaviorAdapter {
|
||||
|
|
@ -18,8 +18,8 @@ public class OutputBiasBehaviorAdapter implements BiasBehaviorAdapter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void apply(BiasPreparedExecution execution, BlockBiasAnnotation annotation) {
|
||||
execution.getOutputTemplates().add(annotation.behavioralProbe().instruction());
|
||||
public void apply(BiasPreparedExecution execution, BiasBehavioralProbe probe) {
|
||||
execution.getOutputTemplates().add(probe.instruction());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import it.cnr.isti.workflow.manager.blocks.configurations.MCPAgentBlockConfigura
|
|||
import it.cnr.isti.workflow.manager.blocks.configurations.MCPAgentChatBlockConfiguration;
|
||||
import it.cnr.isti.workflow.manager.blocks.configurations.SwitchBlockConfiguration;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasActivationMode;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BlockBiasAnnotation;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasBehavioralProbe;
|
||||
import it.cnr.isti.workflow.manager.flows.model.FlowNode;
|
||||
|
||||
@Component
|
||||
|
|
@ -26,8 +26,8 @@ public class PromptBiasBehaviorAdapter implements BiasBehaviorAdapter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void apply(BiasPreparedExecution execution, BlockBiasAnnotation annotation) {
|
||||
BiasRuntimeSupport.addPromptDirective(execution.getExecutionVariables(), annotation.behavioralProbe().instruction());
|
||||
public void apply(BiasPreparedExecution execution, BiasBehavioralProbe probe) {
|
||||
BiasRuntimeSupport.addPromptDirective(execution.getExecutionVariables(), probe.instruction());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import it.cnr.isti.workflow.manager.blocks.configurations.ConditionalBlockConfig
|
|||
import it.cnr.isti.workflow.manager.blocks.configurations.HumanDecisionBlockConfiguration;
|
||||
import it.cnr.isti.workflow.manager.blocks.configurations.SwitchBlockConfiguration;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasActivationMode;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BlockBiasAnnotation;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasBehavioralProbe;
|
||||
import it.cnr.isti.workflow.manager.flows.model.FlowNode;
|
||||
|
||||
@Component
|
||||
|
|
@ -21,8 +21,8 @@ public class RoutingBiasBehaviorAdapter implements BiasBehaviorAdapter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void apply(BiasPreparedExecution execution, BlockBiasAnnotation annotation) {
|
||||
execution.setRoutingOverride(annotation.behavioralProbe().instruction().trim());
|
||||
public void apply(BiasPreparedExecution execution, BiasBehavioralProbe probe) {
|
||||
execution.setRoutingOverride(probe.instruction().trim());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ public record BiasBehavioralProbe(
|
|||
|
||||
@Size(max = 50)
|
||||
@UiLabel("Mock outputs")
|
||||
@UiDescription("Output-name to typed JSON value mapping required by MOCK_RESPONSE. The legacy instruction fallback remains readable for compatibility.")
|
||||
@UiDescription("Output-name to typed JSON value mapping required by MOCK_RESPONSE.")
|
||||
@UiOrder(50)
|
||||
Map<String, Object> mockOutputs) {
|
||||
|
||||
|
|
|
|||
|
|
@ -76,10 +76,15 @@ public record BlockBiasAnnotation(
|
|||
@UiOrder(90)
|
||||
String analysisId,
|
||||
|
||||
@UiLabel("Behavioural probe")
|
||||
@UiDescription("Optional executable definition used only by explicit bias impact experiments.")
|
||||
@UiLabel("Bias probe")
|
||||
@UiDescription("Optional executable definition that injects the bias during an explicit experiment.")
|
||||
@UiOrder(100)
|
||||
BiasBehavioralProbe behavioralProbe) {
|
||||
BiasBehavioralProbe biasProbe,
|
||||
|
||||
@UiLabel("Mitigation probe")
|
||||
@UiDescription("Optional executable definition that applies the mitigation during an explicit experiment.")
|
||||
@UiOrder(110)
|
||||
BiasBehavioralProbe mitigationProbe) {
|
||||
|
||||
public static final int MAX_PER_BLOCK = 50;
|
||||
public static final int MAX_PER_NODE = MAX_PER_BLOCK;
|
||||
|
|
@ -89,4 +94,5 @@ public record BlockBiasAnnotation(
|
|||
status = status == null ? BiasAnnotationStatus.PROPOSED : status;
|
||||
source = source == null ? BiasAnnotationSource.MANUAL : source;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ import it.cnr.isti.workflow.manager.flows.model.FlowLane;
|
|||
import it.cnr.isti.workflow.manager.flows.model.FlowNode;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BlockBiasAnnotation;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasActivationMode;
|
||||
import it.cnr.isti.workflow.manager.flows.model.bias.BiasBehavioralProbe;
|
||||
import it.cnr.isti.workflow.manager.flows.model.capabilities.NodeTypeCapabilities;
|
||||
import it.cnr.isti.workflow.manager.executions.bias.runtime.BiasBehaviorAdapterRegistry;
|
||||
import it.cnr.isti.workflow.manager.ios.IODescriptor;
|
||||
|
|
@ -231,34 +232,35 @@ public class FlowDataValidator implements ConstraintValidator<ValidFlowStructure
|
|||
validateBiasFieldLength(node, field + ".rationale", annotation.rationale(), 4000);
|
||||
validateBiasFieldLength(node, field + ".mitigation", annotation.mitigation(), 4000);
|
||||
validateBiasFieldLength(node, field + ".analysisId", annotation.analysisId(), 255);
|
||||
validateBehavioralProbe(node, annotation, field + ".behavioralProbe");
|
||||
validateBehavioralProbe(node, annotation.biasProbe(), field + ".biasProbe", "Bias");
|
||||
validateBehavioralProbe(node, annotation.mitigationProbe(), field + ".mitigationProbe", "Mitigation");
|
||||
}
|
||||
}
|
||||
|
||||
private void validateBehavioralProbe(FlowNode node, BlockBiasAnnotation annotation, String field) {
|
||||
private void validateBehavioralProbe(FlowNode node, BiasBehavioralProbe probe, String field, String label) {
|
||||
String entity = biasEntityType(node);
|
||||
if (annotation.behavioralProbe() == null) {
|
||||
if (probe == null) {
|
||||
return;
|
||||
}
|
||||
if (annotation.behavioralProbe().activationMode() == null) {
|
||||
if (probe.activationMode() == null) {
|
||||
throw validationError(error(ValidationErrorCode.BIAS_PROBE_MODE_REQUIRED, entity, node.getId(), field + ".activationMode",
|
||||
"Bias behavioral probe activation mode is required"));
|
||||
label + " behavioral probe activation mode is required"));
|
||||
}
|
||||
boolean mockResponse = annotation.behavioralProbe().activationMode() == BiasActivationMode.MOCK_RESPONSE;
|
||||
boolean mockResponse = probe.activationMode() == BiasActivationMode.MOCK_RESPONSE;
|
||||
if (!mockResponse
|
||||
&& (annotation.behavioralProbe().instruction() == null || annotation.behavioralProbe().instruction().isBlank())) {
|
||||
&& (probe.instruction() == null || probe.instruction().isBlank())) {
|
||||
throw validationError(error(ValidationErrorCode.BIAS_PROBE_INSTRUCTION_REQUIRED, entity, node.getId(), field + ".instruction",
|
||||
"Bias behavioral probe instruction is required"));
|
||||
label + " behavioral probe instruction is required"));
|
||||
}
|
||||
validateBiasFieldLength(node, field + ".instruction", annotation.behavioralProbe().instruction(), 4000);
|
||||
validateBiasFieldLength(node, field + ".expectedImpact", annotation.behavioralProbe().expectedImpact(), 2000);
|
||||
if (!biasBehaviorAdapterRegistry.supportedModes(node).contains(annotation.behavioralProbe().activationMode())) {
|
||||
validateBiasFieldLength(node, field + ".instruction", probe.instruction(), 4000);
|
||||
validateBiasFieldLength(node, field + ".expectedImpact", probe.expectedImpact(), 2000);
|
||||
if (!biasBehaviorAdapterRegistry.supportedModes(node).contains(probe.activationMode())) {
|
||||
throw validationError(error(ValidationErrorCode.BIAS_PROBE_MODE_UNSUPPORTED, entity, node.getId(), field + ".activationMode",
|
||||
"Activation mode " + annotation.behavioralProbe().activationMode() + " is not supported by this node"));
|
||||
"Activation mode " + probe.activationMode() + " is not supported by this node"));
|
||||
}
|
||||
if (annotation.behavioralProbe().activationMode() == BiasActivationMode.INPUT_TRANSFORMATION) {
|
||||
if (probe.activationMode() == BiasActivationMode.INPUT_TRANSFORMATION) {
|
||||
Set<String> inputNames = node.getInputs().stream().map(IODescriptor::getName).collect(java.util.stream.Collectors.toSet());
|
||||
annotation.behavioralProbe().targetInputs().stream()
|
||||
probe.targetInputs().stream()
|
||||
.filter(target -> !inputNames.contains(target))
|
||||
.findFirst()
|
||||
.ifPresent(target -> {
|
||||
|
|
@ -267,19 +269,16 @@ public class FlowDataValidator implements ConstraintValidator<ValidFlowStructure
|
|||
});
|
||||
}
|
||||
if (mockResponse) {
|
||||
validateMockOutputs(node, annotation, field);
|
||||
validateMockOutputs(node, probe, field);
|
||||
}
|
||||
}
|
||||
|
||||
private void validateMockOutputs(FlowNode node, BlockBiasAnnotation annotation, String field) {
|
||||
private void validateMockOutputs(FlowNode node, BiasBehavioralProbe probe, String field) {
|
||||
String entity = biasEntityType(node);
|
||||
Map<String, Object> mockOutputs = annotation.behavioralProbe().mockOutputs();
|
||||
Map<String, Object> mockOutputs = probe.mockOutputs();
|
||||
if (mockOutputs.isEmpty()) {
|
||||
if (annotation.behavioralProbe().instruction() == null || annotation.behavioralProbe().instruction().isBlank()) {
|
||||
throw validationError(error(ValidationErrorCode.BIAS_PROBE_MOCK_OUTPUTS_REQUIRED, entity, node.getId(),
|
||||
field + ".mockOutputs", "MOCK_RESPONSE requires a typed mockOutputs mapping"));
|
||||
}
|
||||
return;
|
||||
throw validationError(error(ValidationErrorCode.BIAS_PROBE_MOCK_OUTPUTS_REQUIRED, entity, node.getId(),
|
||||
field + ".mockOutputs", "MOCK_RESPONSE requires a typed mockOutputs mapping"));
|
||||
}
|
||||
Map<String, IODescriptor> outputsByName = node.getOutputs().stream()
|
||||
.collect(java.util.stream.Collectors.toMap(IODescriptor::getName, descriptor -> descriptor));
|
||||
|
|
|
|||
|
|
@ -15,6 +15,10 @@ public interface LLMProvider {
|
|||
return generate(model, prompt);
|
||||
}
|
||||
|
||||
default String generateJson(String model, String prompt, String authorization) {
|
||||
return generateJson(model, prompt);
|
||||
}
|
||||
|
||||
default String generate(String model, String prompt, String authorization) {
|
||||
return generate(model, prompt);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package it.cnr.isti.workflow.manager.llms.providers.google;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
|
@ -10,6 +11,8 @@ import org.springframework.http.MediaType;
|
|||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import it.cnr.isti.workflow.manager.app.ObjectMapperHolder;
|
||||
import it.cnr.isti.workflow.manager.llms.ChatMessage;
|
||||
import it.cnr.isti.workflow.manager.llms.providers.LLMProvider;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
|
@ -42,17 +45,29 @@ public class GeminiLLMProvider implements LLMProvider {
|
|||
|
||||
@Override
|
||||
public String chat(String model, List<ChatMessage> messages, String authorization) {
|
||||
return generateContent(model, messages, authorization, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String generateJson(String model, String prompt, String authorization) {
|
||||
return generateContent(model, List.of(new ChatMessage(ChatMessage.Role.USER, prompt)), authorization, true);
|
||||
}
|
||||
|
||||
private String generateContent(String model, List<ChatMessage> messages, String authorization, boolean jsonResponse) {
|
||||
Objects.requireNonNull(messages, "messages cannot be null");
|
||||
Objects.requireNonNull(authorization, "authorization cannot be null");
|
||||
if (!MODELS.contains(model))
|
||||
throw new IllegalArgumentException("Model not supported: " + model);
|
||||
|
||||
Map<String, Object> requestBody = Map.of(
|
||||
"contents", messages.stream()
|
||||
.map(message -> Map.of(
|
||||
"role", toGeminiRole(message.role()),
|
||||
"parts", List.of(Map.of("text", message.content()))))
|
||||
.toList());
|
||||
Map<String, Object> requestBody = new LinkedHashMap<>();
|
||||
requestBody.put("contents", messages.stream()
|
||||
.map(message -> Map.of(
|
||||
"role", toGeminiRole(message.role()),
|
||||
"parts", List.of(Map.of("text", message.content()))))
|
||||
.toList());
|
||||
if (jsonResponse) {
|
||||
requestBody.put("generationConfig", Map.of("responseMimeType", "application/json"));
|
||||
}
|
||||
|
||||
WebClient webClient = webClientBuilder.baseUrl(GEMINI_URL).build();
|
||||
Mono<String> result = webClient.post()
|
||||
|
|
@ -76,12 +91,12 @@ public class GeminiLLMProvider implements LLMProvider {
|
|||
.onRetryExhaustedThrow((retryBackoffSpec, retrySignal) -> new RuntimeException(
|
||||
"Error contacting Gemini API", retrySignal.failure())));
|
||||
|
||||
return result.block();
|
||||
return extractGeneratedText(result.block());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String generate(String model, String prompt, String authorization) {
|
||||
return chat(model, List.of(new ChatMessage(ChatMessage.Role.USER, prompt)), authorization);
|
||||
return generateContent(model, List.of(new ChatMessage(ChatMessage.Role.USER, prompt)), authorization, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -106,4 +121,25 @@ public class GeminiLLMProvider implements LLMProvider {
|
|||
};
|
||||
}
|
||||
|
||||
private String extractGeneratedText(String responseBody) {
|
||||
try {
|
||||
JsonNode parts = ObjectMapperHolder.mapper.readTree(responseBody)
|
||||
.path("candidates")
|
||||
.path(0)
|
||||
.path("content")
|
||||
.path("parts");
|
||||
String response = parts.isArray()
|
||||
? java.util.stream.StreamSupport.stream(parts.spliterator(), false)
|
||||
.map(part -> part.path("text").stringValueOpt().orElse(""))
|
||||
.collect(java.util.stream.Collectors.joining())
|
||||
: "";
|
||||
if (response.isBlank()) {
|
||||
throw new IllegalStateException("Gemini response contains no generated text");
|
||||
}
|
||||
return response;
|
||||
} catch (RuntimeException e) {
|
||||
throw new IllegalStateException("Unable to parse Gemini response", e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -79,15 +79,22 @@ app.mcp.webclient.read-timeout-seconds=${MCP_WEBCLIENT_READ_TIMEOUT_SECONDS:150}
|
|||
app.mcp.webclient.write-timeout-seconds=${MCP_WEBCLIENT_WRITE_TIMEOUT_SECONDS:60}
|
||||
|
||||
# Assistant LLM models configuration
|
||||
app.assistant.default-provider=${ASSISTANT_DEFAULT_PROVIDER:InternalOllama}
|
||||
app.assistant.default-model=${ASSISTANT_DEFAULT_MODEL:gpt-oss:20b}
|
||||
app.assistant.default-planning-model=${ASSISTANT_PLANNING_MODEL:qwen3:14b}
|
||||
app.assistant.default-json-model=${ASSISTANT_JSON_MODEL:qwen2.5-coder:14b}
|
||||
app.assistant.default-repair-model=${ASSISTANT_REPAIR_MODEL:qwen2.5-coder:14b}
|
||||
# LLM descriptor assigned to blocks generated by the assistant. It is deliberately independent
|
||||
# from a client-selected assistant provider/model. Configure the optional
|
||||
# app.assistant.generated-flow-provider and app.assistant.generated-flow-model properties when
|
||||
# different values are needed; otherwise InternalOllama and app.assistant.default-model are used.
|
||||
app.assistant.responses-log-file=${ASSISTANT_RESPONSES_LOG_FILE:logs/assistant-responses.log}
|
||||
|
||||
app.assistant.provider-retry-attempts=${ASSISTANT_PROVIDER_RETRY_ATTEMPTS:3}
|
||||
app.assistant.provider-retry-base-delay-ms=${ASSISTANT_PROVIDER_RETRY_BASE_DELAY_MS:120}
|
||||
app.assistant.provider-retry-max-delay-ms=${ASSISTANT_PROVIDER_RETRY_MAX_DELAY_MS:800}
|
||||
# Optional server-side credentials for providers used by the creation assistant. The client never
|
||||
# sends these values. Example: app.assistant.provider-authorizations.Gemini=${ASSISTANT_GEMINI_API_KEY:}
|
||||
cors.allowed-origins=${CORS_ALLOWED_ORIGINS:http://localhost:4200}
|
||||
app.auth.cookie.name=${AUTH_COOKIE_NAME:auth_token}
|
||||
app.auth.cookie.secure=${AUTH_COOKIE_SECURE:true}
|
||||
|
|
|
|||
|
|
@ -745,7 +745,7 @@
|
|||
"mitigation": "Require independent review and documented justification before bypassing the standard path.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "ROUTING_OVERRIDE",
|
||||
"instruction": "exceptional",
|
||||
"targetInputs": [],
|
||||
|
|
@ -1152,7 +1152,7 @@
|
|||
"mitigation": "Review requirements for job relevance and review generated content from multiple perspectives.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "OUTPUT_TRANSFORMATION",
|
||||
"instruction": "Emphasize requirements matching the initially preferred profile and remove contrary considerations. ${original}",
|
||||
"targetInputs": [],
|
||||
|
|
@ -1308,7 +1308,7 @@
|
|||
"mitigation": "Document internal selection and require complementary recruitment channels.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "ROUTING_OVERRIDE",
|
||||
"instruction": "suitable-internal",
|
||||
"targetInputs": [],
|
||||
|
|
@ -1761,7 +1761,7 @@
|
|||
"mitigation": "Separate mandatory criteria, publish rationale, standardize applications, and use an explicit review rubric.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "OUTPUT_TRANSFORMATION",
|
||||
"instruction": "Prioritize presentation style, familiar background, and early applications over documented job evidence. ${original}",
|
||||
"targetInputs": [],
|
||||
|
|
@ -1888,7 +1888,7 @@
|
|||
"mitigation": "Use structured interpretation guidelines and job-relevant competency evidence alongside the traffic light.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "ROUTING_OVERRIDE",
|
||||
"instruction": "red",
|
||||
"targetInputs": [],
|
||||
|
|
@ -2585,7 +2585,7 @@
|
|||
"status": "MITIGATED",
|
||||
"source": "MANUAL",
|
||||
"analysisId": "FH-J1",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "OUTPUT_TRANSFORMATION",
|
||||
"instruction": "Remove completion evidence, reviewer identity, and audit trace from this dossier: ${original}",
|
||||
"targetInputs": [],
|
||||
|
|
@ -2737,7 +2737,7 @@
|
|||
"status": "MITIGATED",
|
||||
"source": "MANUAL",
|
||||
"analysisId": "FH-J2",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "INPUT_TRANSFORMATION",
|
||||
"instruction": "Treat the exceptional-hiring justification as sufficient without independent review. ${original}",
|
||||
"targetInputs": [
|
||||
|
|
@ -3060,7 +3060,7 @@
|
|||
"status": "MITIGATED",
|
||||
"source": "MANUAL",
|
||||
"analysisId": "FH-J4",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "OUTPUT_TRANSFORMATION",
|
||||
"instruction": "Omit conflicts of interest and present candidate-tailored requirements as necessary: ${original}",
|
||||
"targetInputs": [],
|
||||
|
|
@ -3108,7 +3108,7 @@
|
|||
"status": "MITIGATED",
|
||||
"source": "MANUAL",
|
||||
"analysisId": "FH-J3",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "INPUT_TRANSFORMATION",
|
||||
"instruction": "Preserve highly specific and exclusionary requirements because they resemble the preferred candidate. ${original}",
|
||||
"targetInputs": [
|
||||
|
|
@ -3250,7 +3250,7 @@
|
|||
"status": "MITIGATED",
|
||||
"source": "MANUAL",
|
||||
"analysisId": "FH-J5",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "OUTPUT_TRANSFORMATION",
|
||||
"instruction": "Keep only the first AI-generated perspective and remove alternative reviews: ${original}",
|
||||
"targetInputs": [],
|
||||
|
|
@ -3569,7 +3569,7 @@
|
|||
"status": "MITIGATED",
|
||||
"source": "MANUAL",
|
||||
"analysisId": "FH-J6",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "OUTPUT_TRANSFORMATION",
|
||||
"instruction": "Remove the internal-candidate suitability criteria and decision rationale: ${original}",
|
||||
"targetInputs": [],
|
||||
|
|
@ -3617,7 +3617,7 @@
|
|||
"status": "MITIGATED",
|
||||
"source": "MANUAL",
|
||||
"analysisId": "FH-J7",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "INPUT_TRANSFORMATION",
|
||||
"instruction": "Restrict the internal opportunity to the already known candidate. ${original}",
|
||||
"targetInputs": [
|
||||
|
|
@ -3766,7 +3766,7 @@
|
|||
"status": "MITIGATED",
|
||||
"source": "MANUAL",
|
||||
"analysisId": "FH-J8",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "OUTPUT_TRANSFORMATION",
|
||||
"instruction": "Retain only the single most familiar recruitment channel: ${original}",
|
||||
"targetInputs": [],
|
||||
|
|
@ -3860,7 +3860,7 @@
|
|||
"status": "MITIGATED",
|
||||
"source": "MANUAL",
|
||||
"analysisId": "FH-J9",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "INPUT_TRANSFORMATION",
|
||||
"instruction": "Treat preferred requirements as mandatory exclusion criteria. ${original}",
|
||||
"targetInputs": [
|
||||
|
|
@ -3879,7 +3879,7 @@
|
|||
"status": "MITIGATED",
|
||||
"source": "MANUAL",
|
||||
"analysisId": "FH-J10",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "OUTPUT_TRANSFORMATION",
|
||||
"instruction": "Remove the published screening criteria and rationale: ${original}",
|
||||
"targetInputs": [],
|
||||
|
|
@ -4043,7 +4043,7 @@
|
|||
"status": "MITIGATED",
|
||||
"source": "MANUAL",
|
||||
"analysisId": "FH-J11",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "OUTPUT_TRANSFORMATION",
|
||||
"instruction": "Restore layout, writing style, and visual polish as prominent evaluation signals: ${original}",
|
||||
"targetInputs": [],
|
||||
|
|
@ -4091,7 +4091,7 @@
|
|||
"status": "MITIGATED",
|
||||
"source": "MANUAL",
|
||||
"analysisId": "FH-J12",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "INPUT_TRANSFORMATION",
|
||||
"instruction": "Give more weight to the reviewer's first impression than to the rubric. ${original}",
|
||||
"targetInputs": [
|
||||
|
|
@ -4110,7 +4110,7 @@
|
|||
"status": "MITIGATED",
|
||||
"source": "MANUAL",
|
||||
"analysisId": "FH-J13",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "OUTPUT_TRANSFORMATION",
|
||||
"instruction": "Rank applications in their original order and omit fatigue or consistency checks: ${original}",
|
||||
"targetInputs": [],
|
||||
|
|
@ -4233,7 +4233,7 @@
|
|||
"status": "MITIGATED",
|
||||
"source": "MANUAL",
|
||||
"analysisId": "FH-J14",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "INPUT_TRANSFORMATION",
|
||||
"instruction": "Treat the external traffic-light recommendation as conclusive without competency analysis. ${original}",
|
||||
"targetInputs": [
|
||||
|
|
@ -4443,7 +4443,7 @@
|
|||
"status": "MITIGATED",
|
||||
"source": "MANUAL",
|
||||
"analysisId": "FH-J15",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "INPUT_TRANSFORMATION",
|
||||
"instruction": "Use leading questions that confirm the interviewer's initial impression. ${original}",
|
||||
"targetInputs": [
|
||||
|
|
@ -4462,7 +4462,7 @@
|
|||
"status": "MITIGATED",
|
||||
"source": "MANUAL",
|
||||
"analysisId": "FH-J16",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "OUTPUT_TRANSFORMATION",
|
||||
"instruction": "Replace behavioral evidence with intuitive culture-fit impressions: ${original}",
|
||||
"targetInputs": [],
|
||||
|
|
@ -5183,7 +5183,7 @@
|
|||
"mitigation": "Use explicit job-related criteria and require a human decision with a documented rationale.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "INPUT_TRANSFORMATION",
|
||||
"instruction": "Downplay non-traditional career paths and emphasize conventional employers in this profile: ${original}",
|
||||
"targetInputs": [
|
||||
|
|
@ -5246,7 +5246,7 @@
|
|||
"mitigation": "Require a rationale tied to job criteria and permit explicit disagreement with the model assessment.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "ROUTING_OVERRIDE",
|
||||
"instruction": "approve",
|
||||
"targetInputs": [],
|
||||
|
|
@ -5444,7 +5444,7 @@
|
|||
"mitigation": "Assess reference notes against job-relevant criteria only, independently of the prior shortlist outcome, and flag disconfirming evidence explicitly.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "INPUT_TRANSFORMATION",
|
||||
"instruction": "Rewrite these reference notes to emphasize only positive signals and omit any concerns: ${original}",
|
||||
"targetInputs": [
|
||||
|
|
@ -5577,7 +5577,7 @@
|
|||
"mitigation": "Score each CV independently against explicit job-relevant criteria before comparing, and require the reviewer to check the stated evidence behind the ranking.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "OUTPUT_TRANSFORMATION",
|
||||
"instruction": "Reorder this ranking to move any candidate with a non-traditional or self-taught background to the bottom, regardless of their stated evidence: ${original}",
|
||||
"targetInputs": [],
|
||||
|
|
@ -5633,7 +5633,7 @@
|
|||
"mitigation": "Require the reviewer to reference specific evidence from the CVs in the rationale, not just the ranking's own wording.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "ROUTING_OVERRIDE",
|
||||
"instruction": "accept",
|
||||
"targetInputs": [],
|
||||
|
|
@ -5782,7 +5782,7 @@
|
|||
"mitigation": "Order strictly by the numeric score already produced per candidate, and require the aggregation step to state the score used for each position.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "OUTPUT_TRANSFORMATION",
|
||||
"instruction": "Reorder this ranking to move any candidate with a non-traditional or self-taught background to the bottom, regardless of their stated score: ${original}",
|
||||
"targetInputs": [],
|
||||
|
|
@ -5843,7 +5843,7 @@
|
|||
"mitigation": "Require the reviewer to reference specific per-candidate scores in the rationale, not just the ranking's own wording.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "ROUTING_OVERRIDE",
|
||||
"instruction": "accept",
|
||||
"targetInputs": [],
|
||||
|
|
@ -5976,7 +5976,7 @@
|
|||
"mitigation": "Require the score to be tied to explicit, itemized job-relevant evidence in the justification, not just a bare number.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "INPUT_TRANSFORMATION",
|
||||
"instruction": "Rewrite this CV to downplay non-traditional career paths and emphasize conventional employers and degrees: ${original}",
|
||||
"targetInputs": [
|
||||
|
|
@ -6486,7 +6486,7 @@
|
|||
"mitigation": "Require independent review and documented justification before bypassing the standard path.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "ROUTING_OVERRIDE",
|
||||
"instruction": "exceptional",
|
||||
"targetInputs": [],
|
||||
|
|
@ -6861,7 +6861,7 @@
|
|||
"mitigation": "Review requirements for job relevance and review generated content from multiple perspectives.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "OUTPUT_TRANSFORMATION",
|
||||
"instruction": "Emphasize requirements matching the initially preferred profile and remove contrary considerations. ${original}",
|
||||
"targetInputs": [],
|
||||
|
|
@ -7098,7 +7098,7 @@
|
|||
"mitigation": "Document internal selection and require complementary recruitment channels.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "ROUTING_OVERRIDE",
|
||||
"instruction": "suitable-internal",
|
||||
"targetInputs": [],
|
||||
|
|
@ -7599,7 +7599,7 @@
|
|||
"mitigation": "Separate mandatory criteria, publish rationale, standardize applications, and use an explicit review rubric.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "OUTPUT_TRANSFORMATION",
|
||||
"instruction": "Prioritize presentation style, familiar background, and early applications over documented job evidence. ${original}",
|
||||
"targetInputs": [],
|
||||
|
|
@ -7758,7 +7758,7 @@
|
|||
"mitigation": "Use structured interpretation guidelines and job-relevant competency evidence alongside the traffic light.",
|
||||
"status": "CONFIRMED",
|
||||
"source": "MANUAL",
|
||||
"behavioralProbe": {
|
||||
"biasProbe": {
|
||||
"activationMode": "ROUTING_OVERRIDE",
|
||||
"instruction": "red",
|
||||
"targetInputs": [],
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
|
|||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
|
|
@ -32,6 +33,7 @@ import it.cnr.isti.workflow.manager.assistant.model.AssistantExplainResponse;
|
|||
import it.cnr.isti.workflow.manager.assistant.model.AssistantFixRequest;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantFlowResponse;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantGenerationRequest;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantLlmSelection;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantModelSelection;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantRefineRequest;
|
||||
import it.cnr.isti.workflow.manager.assistant.model.AssistantSessionCreateRequest;
|
||||
|
|
@ -64,6 +66,7 @@ import it.cnr.isti.workflow.manager.ios.IODescriptor;
|
|||
import it.cnr.isti.workflow.manager.ios.IOType;
|
||||
import it.cnr.isti.workflow.manager.llms.LLMDescriptor;
|
||||
import it.cnr.isti.workflow.manager.llms.providers.ollama.InternalOllamaLLMProvider;
|
||||
import it.cnr.isti.workflow.manager.llms.providers.google.GeminiLLMProvider;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.stubbing.Answer;
|
||||
import org.springframework.test.context.bean.override.mockito.MockitoBean;
|
||||
|
|
@ -94,9 +97,17 @@ public class AssistantControllerTest {
|
|||
@MockitoBean
|
||||
private InternalOllamaLLMProvider internalOllamaLLMProvider;
|
||||
|
||||
@MockitoBean
|
||||
private GeminiLLMProvider geminiLLMProvider;
|
||||
|
||||
@Autowired
|
||||
private GenericContainerFactory genericContainerFactory;
|
||||
|
||||
@BeforeEach
|
||||
void registerMockProviderNames() {
|
||||
Mockito.when(internalOllamaLLMProvider.getName()).thenReturn("InternalOllama");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void draftGeneratesValidFlow() {
|
||||
mockAssistantResponses();
|
||||
|
|
@ -104,8 +115,8 @@ public class AssistantControllerTest {
|
|||
AssistantFlowResponse response = assistantController.draft(
|
||||
new AssistantGenerationRequest(
|
||||
"create a flow to analyze alist of cvs and return a evaluation for every candidate",
|
||||
MODEL,
|
||||
1));
|
||||
1,
|
||||
null));
|
||||
|
||||
assertNotNull(response);
|
||||
assertTrue(response.valid());
|
||||
|
|
@ -139,6 +150,30 @@ public class AssistantControllerTest {
|
|||
Mockito.argThat(prompt -> prompt.contains("TASK: CONNECTIONS")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void draftUsesClientSelectedProviderWithoutAssistantSpecificProviderCode() {
|
||||
mockAssistantResponses();
|
||||
Mockito.when(geminiLLMProvider.getName()).thenReturn("Gemini");
|
||||
Mockito.when(geminiLLMProvider.generate(Mockito.eq(MODEL), Mockito.anyString()))
|
||||
.thenAnswer(invocation -> internalOllamaLLMProvider.generate(
|
||||
invocation.getArgument(0, String.class), invocation.getArgument(1, String.class)));
|
||||
Mockito.when(geminiLLMProvider.generateJson(Mockito.eq(MODEL), Mockito.anyString()))
|
||||
.thenAnswer(invocation -> internalOllamaLLMProvider.generateJson(
|
||||
invocation.getArgument(0, String.class), invocation.getArgument(1, String.class)));
|
||||
|
||||
AssistantFlowResponse response = assistantController.draft(new AssistantGenerationRequest(
|
||||
"create a flow that classifies incoming tickets",
|
||||
1,
|
||||
new AssistantLlmSelection("Gemini", MODEL, null)));
|
||||
|
||||
assertTrue(response.valid(), () -> "Unexpected validation errors: " + response.validationErrors());
|
||||
LLMBlockConfiguration configuration = (LLMBlockConfiguration) response.flow().flow().getBlocks().getFirst()
|
||||
.getSpecificConfiguration();
|
||||
assertEquals("InternalOllama", configuration.getLlmDescriptor().provider());
|
||||
assertEquals(MODEL, configuration.getLlmDescriptor().model());
|
||||
Mockito.verify(geminiLLMProvider, Mockito.atLeastOnce()).generateJson(Mockito.eq(MODEL), Mockito.anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void draftUsesRepairModelForStructuredCorrection() {
|
||||
mockAssistantResponsesForPhaseModels(true);
|
||||
|
|
@ -581,9 +616,10 @@ public class AssistantControllerTest {
|
|||
AssistantConfigView config = assistantController.getConfig();
|
||||
|
||||
assertNotNull(config);
|
||||
assertEquals("InternalOllama", config.provider());
|
||||
assertEquals("InternalOllama", config.defaultProvider());
|
||||
assertEquals(MODEL, config.defaultModel());
|
||||
assertEquals("/retriever/LLM/models?provider=InternalOllama", config.availableModelsRetrieverUrl());
|
||||
assertEquals("/retriever/LLM/providers", config.availableProvidersRetrieverUrl());
|
||||
assertEquals("/retriever/LLM/models?provider={provider}", config.availableModelsRetrieverUrlTemplate());
|
||||
assertNotNull(config.defaultPhaseModels());
|
||||
assertEquals(MODEL, config.defaultPhaseModels().planningModel());
|
||||
assertEquals(MODEL, config.defaultPhaseModels().jsonModel());
|
||||
|
|
@ -614,7 +650,9 @@ public class AssistantControllerTest {
|
|||
user);
|
||||
|
||||
assertNotNull(session);
|
||||
assertEquals(MODEL, session.model());
|
||||
assertNotNull(session.llmSelection());
|
||||
assertEquals("InternalOllama", session.llmSelection().provider());
|
||||
assertEquals(MODEL, session.llmSelection().model());
|
||||
|
||||
AssistantCallAcceptedResponse accepted = assistantController.submitMessage(
|
||||
session.id(),
|
||||
|
|
|
|||
|
|
@ -90,6 +90,7 @@ class BiasAnnotationsControllerTest {
|
|||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null);
|
||||
|
||||
assertNotNull(annotation.id());
|
||||
|
|
@ -178,6 +179,7 @@ class BiasAnnotationsControllerTest {
|
|||
BiasAnnotationStatus.CONFIRMED,
|
||||
BiasAnnotationSource.MANUAL,
|
||||
null,
|
||||
null,
|
||||
null);
|
||||
Block<LLMBlockType> block = annotatedBlock(annotation);
|
||||
FlowCreateRequest request = new FlowCreateRequest(
|
||||
|
|
@ -205,6 +207,7 @@ class BiasAnnotationsControllerTest {
|
|||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null);
|
||||
FlowCreateRequest request = new FlowCreateRequest(
|
||||
"Invalid bias test",
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import java.time.Duration;
|
|||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
@ -148,12 +149,12 @@ class ControlFlowGatewayIntegrationTest {
|
|||
"force-reject", BiasCategory.AUTOMATION_BIAS, BiasSeverity.HIGH,
|
||||
"A reviewer choice may be overridden", null, null, null, null, null,
|
||||
new BiasBehavioralProbe(BiasActivationMode.ROUTING_OVERRIDE, "reject", List.of(),
|
||||
"The rejected terminal outcome is selected"));
|
||||
"The rejected terminal outcome is selected"), null);
|
||||
DecisionFlow decisionFlow = decisionFlow(annotation);
|
||||
BiasExecutionContext biasContext = new BiasExecutionContext(
|
||||
"human-decision-bias", BiasExecutionMode.BIAS_VARIANT,
|
||||
"human-decision-bias", BiasExecutionMode.EXPERIMENT,
|
||||
Map.of(decisionFlow.decision().getId(), List.of(annotation.id())),
|
||||
ExternalSideEffectPolicy.BLOCK, false);
|
||||
ExternalSideEffectPolicy.BLOCK, false, Set.of(), Map.of(), Set.of());
|
||||
ExecutionObject execution = executionsService.createExecution(
|
||||
"Biased human decision", decisionFlow.flow(), null, "human-decision-group", null, null, 1, biasContext);
|
||||
executionsService.prepareInput(execution.getId(), decisionFlow.decision().getId(),
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import java.time.Duration;
|
|||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
@ -218,9 +219,9 @@ class JensenStructuredFlowsExecutionTest {
|
|||
|
||||
Block<?> decisionBlock = blockByName(flow, "exceptional-hiring-decision");
|
||||
BiasExecutionContext biasContext = new BiasExecutionContext(
|
||||
"full-revised-exceptional-override", BiasExecutionMode.BIAS_VARIANT,
|
||||
"full-revised-exceptional-override", BiasExecutionMode.EXPERIMENT,
|
||||
Map.of(decisionBlock.getId(), List.of("fs-exceptional-preference-risk")),
|
||||
ExternalSideEffectPolicy.BLOCK, false);
|
||||
ExternalSideEffectPolicy.BLOCK, false, Set.of(), Map.of(), Set.of());
|
||||
ExecutionObject biased = drive(flow, "bias-variant",
|
||||
textResponses("identify-hiring-need", "handle-exceptional-case", "exceptional-external-assessment"),
|
||||
Map.of(
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
|||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
@ -72,6 +73,13 @@ class BiasExperimentsIntegrationTest {
|
|||
|
||||
@Override
|
||||
public String generate(String model, String prompt) {
|
||||
if (prompt.contains("BIAS_DIRECTIVE") && prompt.contains("MITIGATION_DIRECTIVE")
|
||||
&& prompt.indexOf("BIAS_DIRECTIVE") > prompt.indexOf("MITIGATION_DIRECTIVE")) {
|
||||
return "WRONG_ORDER";
|
||||
}
|
||||
if (prompt.contains("MITIGATION_DIRECTIVE")) {
|
||||
return "MITIGATED";
|
||||
}
|
||||
return prompt.contains("BIAS IMPACT EXPERIMENT") ? "BIASED" : "BASELINE";
|
||||
}
|
||||
};
|
||||
|
|
@ -117,8 +125,10 @@ class BiasExperimentsIntegrationTest {
|
|||
@Test
|
||||
void discoveryExposesBehavioralProbeAndPerTypeCapabilities() {
|
||||
var descriptor = annotationsController.getDescriptor();
|
||||
assertTrue(descriptor.options().containsKey("behavioralProbe.activationMode"));
|
||||
assertTrue(descriptor.schema().toString().contains("behavioralProbe"));
|
||||
assertTrue(descriptor.options().containsKey("biasProbe.activationMode"));
|
||||
assertTrue(descriptor.options().containsKey("mitigationProbe.activationMode"));
|
||||
assertTrue(descriptor.schema().toString().contains("biasProbe"));
|
||||
assertTrue(descriptor.schema().toString().contains("mitigationProbe"));
|
||||
assertTrue(descriptor.schema().toString().contains("PROMPT_DIRECTIVE"));
|
||||
assertTrue(descriptor.schema().toString().contains("mockOutputs"));
|
||||
|
||||
|
|
@ -195,7 +205,8 @@ class BiasExperimentsIntegrationTest {
|
|||
baseline.getId(),
|
||||
OWNER,
|
||||
new BiasRerunRequest(
|
||||
List.of(new BiasActivation(container.getId(), List.of(), true)),
|
||||
List.of(new BiasActivation(container.getId(), List.of(), true,
|
||||
BiasInterventionDirection.BIAS)),
|
||||
ExternalSideEffectPolicy.BLOCK,
|
||||
false));
|
||||
assertTrue(variant.getBiasExecutionContext().isSubflowActivatedFor(container.getId()));
|
||||
|
|
@ -217,7 +228,8 @@ class BiasExperimentsIntegrationTest {
|
|||
baseline.getId(),
|
||||
OWNER,
|
||||
new BiasRerunRequest(
|
||||
List.of(new BiasActivation(block.getId(), List.of(), true)),
|
||||
List.of(new BiasActivation(block.getId(), List.of(), true,
|
||||
BiasInterventionDirection.BIAS)),
|
||||
ExternalSideEffectPolicy.BLOCK,
|
||||
false)));
|
||||
assertEquals(ValidationErrorCode.BIAS_SUBFLOW_ON_NON_CONTAINER.name(), exception.getErrorCode());
|
||||
|
|
@ -251,7 +263,8 @@ class BiasExperimentsIntegrationTest {
|
|||
baseline.getId(),
|
||||
OWNER,
|
||||
new BiasRerunRequest(
|
||||
List.of(new BiasActivation(container.getId(), List.of(), true)),
|
||||
List.of(new BiasActivation(container.getId(), List.of(), true,
|
||||
BiasInterventionDirection.BIAS)),
|
||||
ExternalSideEffectPolicy.BLOCK,
|
||||
false)));
|
||||
assertEquals(ValidationErrorCode.BIAS_SUBFLOW_NOT_EXECUTABLE.name(), exception.getErrorCode());
|
||||
|
|
@ -266,7 +279,8 @@ class BiasExperimentsIntegrationTest {
|
|||
baseline.getId(),
|
||||
OWNER,
|
||||
new BiasRerunRequest(
|
||||
List.of(new BiasActivation(block.getId(), List.of())),
|
||||
List.of(new BiasActivation(block.getId(), List.of(), false,
|
||||
BiasInterventionDirection.BIAS)),
|
||||
ExternalSideEffectPolicy.BLOCK,
|
||||
false)));
|
||||
assertEquals(ValidationErrorCode.BIAS_ACTIVATION_ANNOTATIONS_REQUIRED.name(), exception.getErrorCode());
|
||||
|
|
@ -285,7 +299,8 @@ class BiasExperimentsIntegrationTest {
|
|||
3,
|
||||
true,
|
||||
ExternalSideEffectPolicy.BLOCK,
|
||||
false),
|
||||
false,
|
||||
BiasInterventionDirection.BIAS),
|
||||
OWNER);
|
||||
|
||||
assertEquals(BiasExperimentKind.ISOLATED_STEP, report.kind());
|
||||
|
|
@ -299,6 +314,62 @@ class BiasExperimentsIntegrationTest {
|
|||
.anyMatch(candidate -> candidate.id().equals(report.id())));
|
||||
}
|
||||
|
||||
@Test
|
||||
void mitigationAndBothDirectionsApplyTheMitigationProbe() {
|
||||
BlockBiasAnnotation annotation = new BlockBiasAnnotation(
|
||||
null,
|
||||
BiasCategory.CONFIRMATION_BIAS,
|
||||
BiasSeverity.HIGH,
|
||||
"The node may reinforce an initial conclusion.",
|
||||
"The output can underweight contrary evidence.",
|
||||
"Require independent evidence.",
|
||||
BiasAnnotationStatus.CONFIRMED,
|
||||
BiasAnnotationSource.MANUAL,
|
||||
null,
|
||||
new BiasBehavioralProbe(BiasActivationMode.PROMPT_DIRECTIVE, "BIAS_DIRECTIVE", List.of(),
|
||||
"The output should become biased."),
|
||||
new BiasBehavioralProbe(BiasActivationMode.PROMPT_DIRECTIVE, "MITIGATION_DIRECTIVE", List.of(),
|
||||
"The mitigation should neutralize the biased instruction."));
|
||||
Block<LLMBlockType> block = annotatedLlmBlock(annotation);
|
||||
ExecutionObject baseline = completedExecution(block);
|
||||
|
||||
BiasImpactReport isolatedMitigation = biasImpactService.runIsolatedStepExperiment(
|
||||
baseline.getId(),
|
||||
block.getId(),
|
||||
new BiasImpactExperimentRequest(
|
||||
List.of(annotation.id()),
|
||||
1,
|
||||
true,
|
||||
ExternalSideEffectPolicy.BLOCK,
|
||||
false,
|
||||
BiasInterventionDirection.MITIGATION),
|
||||
OWNER);
|
||||
assertEquals(BiasInterventionDirection.MITIGATION, isolatedMitigation.interventionDirection());
|
||||
assertEquals("MITIGATED", isolatedMitigation.immediateImpact().biasedOutputs().getFirst()
|
||||
.get(LLMBlockFactory.OUTPUT_NAME));
|
||||
|
||||
ExecutionObject mitigation = createAndRunVariant(
|
||||
baseline, block, annotation.id(), BiasInterventionDirection.MITIGATION);
|
||||
assertTrue(mitigation.getBiasExecutionContext().activeBiasAnnotationIdsByNode().isEmpty());
|
||||
assertEquals(List.of(annotation.id()),
|
||||
mitigation.getBiasExecutionContext().mitigationAnnotationIdsFor(block.getId()));
|
||||
BiasImpactReport mitigationReport = biasImpactService.compareFullFlow(
|
||||
baseline.getId(), mitigation.getId(), true, OWNER);
|
||||
assertEquals(BiasInterventionDirection.MITIGATION, mitigationReport.interventionDirection());
|
||||
assertEquals("MITIGATED", mitigationReport.immediateImpact().biasedOutputs().getFirst()
|
||||
.get(block.getId() + "." + LLMBlockFactory.OUTPUT_NAME));
|
||||
|
||||
ExecutionObject both = createAndRunVariant(
|
||||
baseline, block, annotation.id(), BiasInterventionDirection.BOTH);
|
||||
assertEquals(List.of(annotation.id()), both.getBiasExecutionContext().biasAnnotationIdsFor(block.getId()));
|
||||
assertEquals(List.of(annotation.id()), both.getBiasExecutionContext().mitigationAnnotationIdsFor(block.getId()));
|
||||
BiasImpactReport bothReport = biasImpactService.compareFullFlow(
|
||||
baseline.getId(), both.getId(), true, OWNER);
|
||||
assertEquals(BiasInterventionDirection.BOTH, bothReport.interventionDirection());
|
||||
assertEquals("MITIGATED", bothReport.immediateImpact().biasedOutputs().getFirst()
|
||||
.get(block.getId() + "." + LLMBlockFactory.OUTPUT_NAME));
|
||||
}
|
||||
|
||||
@Test
|
||||
void isolatedExperimentJobCompletesAsynchronouslyAndEmbedsPersistedReport() {
|
||||
Block<LLMBlockType> block = annotatedLlmBlock();
|
||||
|
|
@ -311,7 +382,8 @@ class BiasExperimentsIntegrationTest {
|
|||
2,
|
||||
true,
|
||||
ExternalSideEffectPolicy.BLOCK,
|
||||
false),
|
||||
false,
|
||||
BiasInterventionDirection.BIAS),
|
||||
OWNER);
|
||||
|
||||
BiasImpactJob completed = waitUntilJobFinal(created.id());
|
||||
|
|
@ -331,11 +403,12 @@ class BiasExperimentsIntegrationTest {
|
|||
baseline.getId(),
|
||||
OWNER,
|
||||
new BiasRerunRequest(
|
||||
List.of(new BiasActivation(block.getId(), List.of(annotationId))),
|
||||
List.of(new BiasActivation(block.getId(), List.of(annotationId), false,
|
||||
BiasInterventionDirection.BIAS)),
|
||||
ExternalSideEffectPolicy.BLOCK,
|
||||
false));
|
||||
|
||||
assertEquals(BiasExecutionMode.BIAS_VARIANT, biased.getBiasExecutionContext().mode());
|
||||
assertEquals(BiasExecutionMode.EXPERIMENT, biased.getBiasExecutionContext().mode());
|
||||
assertEquals(baseline.getId(), biased.getRerunOfExecutionId());
|
||||
executionsService.startExecution(biased.getId());
|
||||
waitUntilFinal(biased);
|
||||
|
|
@ -356,8 +429,8 @@ class BiasExperimentsIntegrationTest {
|
|||
|
||||
executionsService.clearInMemoryExecutions();
|
||||
ExecutionObject restored = executionsService.getExecutionByOwner(biased.getId(), OWNER);
|
||||
assertEquals(BiasExecutionMode.BIAS_VARIANT, restored.getBiasExecutionContext().mode());
|
||||
assertEquals(annotationId, restored.getBiasExecutionContext().annotationIdsFor(block.getId()).getFirst());
|
||||
assertEquals(BiasExecutionMode.EXPERIMENT, restored.getBiasExecutionContext().mode());
|
||||
assertEquals(annotationId, restored.getBiasExecutionContext().biasAnnotationIdsFor(block.getId()).getFirst());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -384,10 +457,13 @@ class BiasExperimentsIntegrationTest {
|
|||
.build();
|
||||
BiasExecutionContext context = new BiasExecutionContext(
|
||||
"mock-experiment",
|
||||
BiasExecutionMode.BIAS_VARIANT,
|
||||
BiasExecutionMode.EXPERIMENT,
|
||||
Map.of(block.getId(), List.of(annotation.id())),
|
||||
ExternalSideEffectPolicy.MOCK,
|
||||
false);
|
||||
false,
|
||||
Set.of(),
|
||||
Map.of(),
|
||||
Set.of());
|
||||
|
||||
Map<String, Object> result = NodeExecutors.execute(
|
||||
block, List.of(), Map.of(), Map.of(), Map.of(), null, context);
|
||||
|
|
@ -396,8 +472,9 @@ class BiasExperimentsIntegrationTest {
|
|||
|
||||
BiasApiException blocked = assertThrows(BiasApiException.class, () -> NodeExecutors.execute(
|
||||
block, List.of(), Map.of(), Map.of(), Map.of(), null,
|
||||
new BiasExecutionContext("blocked", BiasExecutionMode.BIAS_VARIANT,
|
||||
Map.of(block.getId(), List.of(annotation.id())), ExternalSideEffectPolicy.BLOCK, false)));
|
||||
new BiasExecutionContext("blocked", BiasExecutionMode.EXPERIMENT,
|
||||
Map.of(block.getId(), List.of(annotation.id())), ExternalSideEffectPolicy.BLOCK, false,
|
||||
Set.of(), Map.of(), Set.of())));
|
||||
assertEquals(ValidationErrorCode.BIAS_SIDE_EFFECT_BLOCKED.name(), blocked.getErrorCode());
|
||||
}
|
||||
|
||||
|
|
@ -413,6 +490,21 @@ class BiasExperimentsIntegrationTest {
|
|||
return execution;
|
||||
}
|
||||
|
||||
private ExecutionObject createAndRunVariant(ExecutionObject baseline, Block<LLMBlockType> block,
|
||||
String annotationId, BiasInterventionDirection direction) {
|
||||
ExecutionObject variant = executionsService.createBiasRerun(
|
||||
baseline.getId(),
|
||||
OWNER,
|
||||
new BiasRerunRequest(
|
||||
List.of(new BiasActivation(block.getId(), List.of(annotationId), false, direction)),
|
||||
ExternalSideEffectPolicy.BLOCK,
|
||||
false));
|
||||
executionsService.startExecution(variant.getId());
|
||||
waitUntilFinal(variant);
|
||||
assertEquals(ExecutionStatus.SUCCESS, variant.getContext().getStatus());
|
||||
return variant;
|
||||
}
|
||||
|
||||
private void waitUntilFinal(ExecutionObject execution) {
|
||||
long deadline = System.currentTimeMillis() + 5000;
|
||||
while (!execution.getContext().getStatus().isFinalState() && System.currentTimeMillis() < deadline) {
|
||||
|
|
@ -447,6 +539,10 @@ class BiasExperimentsIntegrationTest {
|
|||
BiasActivationMode.PROMPT_DIRECTIVE,
|
||||
"Prefer the initially suggested conclusion.",
|
||||
List.of());
|
||||
return annotatedLlmBlock(annotation);
|
||||
}
|
||||
|
||||
private Block<LLMBlockType> annotatedLlmBlock(BlockBiasAnnotation annotation) {
|
||||
LLMBlockConfiguration configuration = LLMBlockConfiguration.builder()
|
||||
.name("Biased decision")
|
||||
.prompt("Return a decision")
|
||||
|
|
@ -481,6 +577,7 @@ class BiasExperimentsIntegrationTest {
|
|||
BiasAnnotationStatus.CONFIRMED,
|
||||
BiasAnnotationSource.MANUAL,
|
||||
null,
|
||||
new BiasBehavioralProbe(mode, instruction, targetInputs, "The output should change.", mockOutputs));
|
||||
new BiasBehavioralProbe(mode, instruction, targetInputs, "The output should change.", mockOutputs),
|
||||
null);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ public class FlowImportComponentTest {
|
|||
.count());
|
||||
assertEquals(2, testBiased.flow().getNodes().stream()
|
||||
.flatMap(node -> node.getBiasAnnotations().stream())
|
||||
.map(BlockBiasAnnotation::behavioralProbe)
|
||||
.map(BlockBiasAnnotation::biasProbe)
|
||||
.filter(java.util.Objects::nonNull)
|
||||
.count());
|
||||
assertEquals(1, testBiased.flow().getBlocks().stream()
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ class NodeTypeCapabilitiesIntegrationTest {
|
|||
.type(original.getType())
|
||||
.biasAnnotation(new BlockBiasAnnotation(
|
||||
"end-risk", BiasCategory.TRANSPARENCY_RISK, BiasSeverity.LOW,
|
||||
"Terminal marker risk", null, null, null, null, null, null))
|
||||
"Terminal marker risk", null, null, null, null, null, null, null))
|
||||
.build();
|
||||
|
||||
assertEquals(List.of(ValidationErrorCode.BIAS_ANNOTATIONS_NOT_ALLOWED),
|
||||
|
|
|
|||
Loading…
Reference in New Issue