fix: tolerant parsing of the simulator's HumanDecision choice
Simulated HumanDecision required the LLM's "CHOICE:" line to be exactly a configured option name. But the simulate prompt lists options as "- name: Label", so models - including strong instruct ones (qwen2.5:14b, etc.), not just gemma:7b - routinely echo the whole "name: Label" line, e.g. "red: Red light" or "existing: Existing position". That was rejected with HUMAN_DECISION_INVALID_CHOICE, breaking simulated runs of any flow with human decisions. resolveSimulatedChoice() now accepts, in order: an exact option name, the token before the first ':' (the echoed "name: Label" case), or the option label. Only ':' is treated as a separator - never '-' - so hyphenated option names like "not-red"/"assessment-not-required" are never truncated. Unknown choices still return null and raise the same error. Added a direct unit test (HumanDecisionSimulatedChoiceTest) covering exact name, echoed name:label, hyphenated names, label text and rejection. Also narrowed the flat-flow Jensen smoke test to exclude the new container-grouped variant (different top-level shape). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
4f0f52c4f1
commit
a5d94a2e55
|
|
@ -103,12 +103,11 @@ public class HumanDecisionExecutor implements BlockExecutor<HumanDecisionBlockTy
|
|||
"Simulator did not return a choice");
|
||||
}
|
||||
String resolvedChoice = rawChoice;
|
||||
String choice = configuration.getOptions().stream()
|
||||
.map(HumanDecisionOption::name)
|
||||
.filter(name -> name.equalsIgnoreCase(resolvedChoice))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new NodeDecisionException("HUMAN_DECISION_INVALID_CHOICE",
|
||||
"Simulator returned unsupported HumanDecision choice: " + resolvedChoice));
|
||||
String choice = resolveSimulatedChoice(configuration, rawChoice);
|
||||
if (choice == null) {
|
||||
throw new NodeDecisionException("HUMAN_DECISION_INVALID_CHOICE",
|
||||
"Simulator returned unsupported HumanDecision choice: " + resolvedChoice);
|
||||
}
|
||||
if (configuration.isRationaleRequired() && !StringUtils.hasText(rationale)) {
|
||||
throw new NodeDecisionException("HUMAN_DECISION_SIMULATION_FAILED",
|
||||
"Simulator did not provide a required rationale");
|
||||
|
|
@ -127,6 +126,42 @@ public class HumanDecisionExecutor implements BlockExecutor<HumanDecisionBlockTy
|
|||
return Map.of(choice, payload);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the simulator's raw CHOICE line to a configured option name, tolerantly.
|
||||
* <p>
|
||||
* The simulate prompt lists options as "- name: Label", so models (even strong ones)
|
||||
* routinely echo the whole "name: Label" line rather than the bare name. This accepts
|
||||
* an exact option name, the token before the first ':' (the echoed "name: Label" case),
|
||||
* or the option label. Returns null if nothing matches. Option names may contain '-'
|
||||
* (e.g. "not-red"), so only ':' is treated as a separator, never '-'.
|
||||
*/
|
||||
static String resolveSimulatedChoice(HumanDecisionBlockConfiguration configuration, String rawChoice) {
|
||||
String candidate = rawChoice == null ? "" : rawChoice.strip();
|
||||
if (candidate.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
String beforeColon = candidate.contains(":")
|
||||
? candidate.substring(0, candidate.indexOf(':')).strip()
|
||||
: candidate;
|
||||
for (HumanDecisionOption option : configuration.getOptions()) {
|
||||
if (option.name().equalsIgnoreCase(candidate)) {
|
||||
return option.name();
|
||||
}
|
||||
}
|
||||
for (HumanDecisionOption option : configuration.getOptions()) {
|
||||
if (option.name().equalsIgnoreCase(beforeColon)) {
|
||||
return option.name();
|
||||
}
|
||||
}
|
||||
for (HumanDecisionOption option : configuration.getOptions()) {
|
||||
if (option.label() != null
|
||||
&& (option.label().equalsIgnoreCase(candidate) || option.label().equalsIgnoreCase(beforeColon))) {
|
||||
return option.name();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public InteractionResult interact(Block<HumanDecisionBlockType> block, List<Input> inputs,
|
||||
Map<String, Object> interaction, Map<String, Object> partialResults, Map<String, Object> authorizations,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,55 @@
|
|||
package it.cnr.isti.workflow.manager.executions.executors.blocks;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import it.cnr.isti.workflow.manager.blocks.configurations.HumanDecisionBlockConfiguration;
|
||||
import it.cnr.isti.workflow.manager.blocks.configurations.HumanDecisionOption;
|
||||
|
||||
class HumanDecisionSimulatedChoiceTest {
|
||||
|
||||
private HumanDecisionBlockConfiguration config() {
|
||||
return HumanDecisionBlockConfiguration.builder()
|
||||
.name("decision")
|
||||
.question("Pick a light")
|
||||
.options(List.of(
|
||||
new HumanDecisionOption("red", "Red light"),
|
||||
new HumanDecisionOption("not-red", "No red flag")))
|
||||
.build();
|
||||
}
|
||||
|
||||
@Test
|
||||
void acceptsExactOptionName() {
|
||||
assertEquals("red", HumanDecisionExecutor.resolveSimulatedChoice(config(), "red"));
|
||||
assertEquals("not-red", HumanDecisionExecutor.resolveSimulatedChoice(config(), "NOT-RED"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void acceptsEchoedNameColonLabel() {
|
||||
// Models routinely echo the whole "- name: Label" prompt line.
|
||||
assertEquals("red", HumanDecisionExecutor.resolveSimulatedChoice(config(), "red: Red light"));
|
||||
assertEquals("not-red", HumanDecisionExecutor.resolveSimulatedChoice(config(), "not-red: No red flag"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotSplitHyphenatedNames() {
|
||||
// "not-red" must not be truncated to "not" by treating '-' as a separator.
|
||||
assertEquals("not-red", HumanDecisionExecutor.resolveSimulatedChoice(config(), "not-red"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void acceptsLabelText() {
|
||||
assertEquals("red", HumanDecisionExecutor.resolveSimulatedChoice(config(), "Red light"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsUnrelatedChoice() {
|
||||
assertNull(HumanDecisionExecutor.resolveSimulatedChoice(config(), "maybe"));
|
||||
assertNull(HumanDecisionExecutor.resolveSimulatedChoice(config(), ""));
|
||||
assertNull(HumanDecisionExecutor.resolveSimulatedChoice(config(), null));
|
||||
}
|
||||
}
|
||||
|
|
@ -101,8 +101,12 @@ public class FlowImportComponentTest {
|
|||
List<ImportedFlow> flows = ObjectMapperHolder.mapper.readValue(
|
||||
bundledFlows.toFile(),
|
||||
ObjectMapperHolder.mapper.getTypeFactory().constructCollectionType(List.class, ImportedFlow.class));
|
||||
// The container-grouped variant has a different top-level shape (a GenericContainer
|
||||
// first node that suspends into WAITING_FOR_SUBFLOW, not a top-level HumanInteractionBlock),
|
||||
// so it is covered by its own tests, not this flat-flow smoke test.
|
||||
List<ImportedFlow> jensenFlows = flows.stream()
|
||||
.filter(flow -> flow.name().startsWith("Jensen Recruitment Process - "))
|
||||
.filter(flow -> !flow.name().contains("(Containerized)"))
|
||||
.toList();
|
||||
assertEquals(2, jensenFlows.size());
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue