Require the simulator's credential upfront, not on first use

startSimulationExecution resolved the simulator's provider authorization
lazily, inside whichever step happened to reach it first - a credential-
requiring simulator would start a RUNNING execution that then failed deep
inside a step, instead of being refused outright. ExecutionSimulationRequest
now carries an optional credentialId (mirroring AssistantLlmSelection), and
the service validates or falls back to an already-provided credential for
the same provider before starting simulation at all.

Fixing this surfaced a second, previously silent gap: a simulated
container's child never inherited the simulator's credential, since it is
never part of any execution's requiredAuthorizations and so the ordinary
per-container authorization propagation loop never touched it. Every
simulated container subflow would have started failing the same upfront
check once it went in, so propagateSimulatorAuthorizationToChild copies the
already-validated credential down to the child before it starts simulating.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Lucio Lelii 2026-09-17 12:54:26 +02:00
parent dec3295d6d
commit 058918f4d7
7 changed files with 261 additions and 13 deletions

View File

@ -272,7 +272,8 @@ public class ExecutionsController {
@RequestBody @jakarta.validation.Valid ExecutionSimulationRequest request,
@AuthenticationPrincipal LoginEntity userDetails) {
return ExecutionView.fromExecution(
executionService.startSimulationExecution(visibleExecution(id, userDetails).getId(), request.simulator()));
executionService.startSimulationExecution(visibleExecution(id, userDetails).getId(), request.simulator(),
request.credentialId()));
}
@PutMapping(path = "{id}/resume")

View File

@ -9,5 +9,11 @@ import jakarta.validation.Valid;
import jakarta.validation.constraints.NotNull;
public record ExecutionSimulationRequest(
@Valid @NotNull LLMDescriptor simulator) {
@Valid @NotNull LLMDescriptor simulator,
/**
* A vault secret id, required only when the simulator's provider needs one. Lets a
* credential-requiring provider be chosen as simulator on its own terms, rather than only
* working when the flow's own steps happen to already use the same provider.
*/
String credentialId) {
}

View File

@ -732,11 +732,36 @@ public class ExecutionsService {
}
if (executionContext.simulationEnabled() && executionContext.simulationDescriptor() != null
&& child.isSimulationAvailable()) {
return startSimulationExecution(childId, executionContext.simulationDescriptor());
propagateSimulatorAuthorizationToChild(child, executionContext);
return startSimulationExecution(childId, executionContext.simulationDescriptor(), null);
}
return startExecution(childId);
}
/**
* The simulator's credential, once resolved for the top-level execution, is not among the
* child's own {@code requiredAuthorizations} - the ordinary per-container propagation loop in
* {@link #createAndStartSubflowChild} never copies it - so without this, every simulated
* container subflow would fail the same "no saved credential" check {@link
* #startSimulationExecution} now performs, on every single container, for a credential the
* parent already proved it has.
*/
private void propagateSimulatorAuthorizationToChild(ExecutionObject child, ContainerExecutionContext executionContext) {
LLMProvider provider = AuthorizationRequirementResolver.resolveProvider(llmProviders,
executionContext.simulationDescriptor().provider());
if (provider == null || !provider.requiresAuthorization()) {
return;
}
String key = provider.authorizationKey();
if (child.getProvidedAuthorizations().containsKey(key)) {
return;
}
Object value = getExecution(executionContext.parentExecutionId()).getProvidedAuthorizations().get(key);
if (value != null) {
child.setAuthorization(key, value);
}
}
/** The container's own name where it can be resolved, so the message points at the diagram. */
private String containerLabel(ContainerExecutionContext executionContext) {
try {
@ -1391,8 +1416,16 @@ public class ExecutionsService {
return eo;
}
/**
* @param credentialId a vault secret id for the simulator's own provider, or null to fall back
* to whatever the flow's own steps already provided for that exact
* provider. Validated - and, if given, stored - before simulation starts,
* so a provider that needs a credential nobody has supplied fails here,
* on this call, rather than later as a failed step once a human-interaction
* node is reached and {@link LLMCredentialResolver} finds nothing to resolve.
*/
@Transactional
public ExecutionObject startSimulationExecution(String id, LLMDescriptor simulatorDescriptor) {
public ExecutionObject startSimulationExecution(String id, LLMDescriptor simulatorDescriptor, String credentialId) {
ExecutionObject eo = getExecution(id);
if (!eo.isSimulationAvailable()) {
throw new ResponseStatusException(HttpStatus.BAD_REQUEST,
@ -1402,12 +1435,49 @@ public class ExecutionsService {
throw new ResponseStatusException(HttpStatus.BAD_REQUEST,
"Simulation requires a simulator descriptor");
}
resolveSimulatorAuthorization(eo, simulatorDescriptor, credentialId);
eo.setInteractionSimulationDescriptor(simulatorDescriptor);
eo.startSimulation();
touchExecution(id);
return eo;
}
/**
* Mirrors {@link #validateCredentialReference}, for a provider that is not among the execution's
* own {@code requiredAuthorizations} - the simulator's provider is chosen after the execution
* already exists, so it was never walked by {@link AuthorizationRequirementResolver}.
*/
private void resolveSimulatorAuthorization(ExecutionObject eo, LLMDescriptor descriptor, String credentialId) {
LLMProvider provider = AuthorizationRequirementResolver.resolveProvider(llmProviders, descriptor.provider());
if (provider == null || !provider.requiresAuthorization()) {
return;
}
String key = provider.authorizationKey();
String trimmedCredentialId = credentialId == null ? "" : credentialId.trim();
if (!trimmedCredentialId.isEmpty()) {
if (eo.getOwner() == null || eo.getOwner().isBlank()) {
throw new ResponseStatusException(HttpStatus.BAD_REQUEST,
"A user-owned execution is required to select a credential for provider " + provider.getName());
}
try {
userSecretService.requireUsableSecret(eo.getOwner(), trimmedCredentialId, provider.getName());
} catch (ResponseStatusException e) {
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getReason(), e);
}
eo.setAuthorization(key, trimmedCredentialId);
return;
}
// No credential supplied for the simulator specifically: fall back to whatever the flow's
// own steps already provided for this exact provider - the coincidental-reuse case that has
// always worked, left unchanged and not re-validated (it resolved once already, the same
// way a container's child execution inherits its parent's without a second vault round trip).
Object existing = eo.getProvidedAuthorizations().get(key);
if (existing == null || String.valueOf(existing).isBlank()) {
throw new ResponseStatusException(HttpStatus.BAD_REQUEST,
"Simulating with " + provider.getName() + " requires a saved credential");
}
}
@Transactional
public ExecutionObject resumeExecution(String id) {
ExecutionObject eo = getExecution(id);

View File

@ -926,7 +926,7 @@ public class ExecutionControllerTest {
chatBlock.getInputs().getFirst().getName(), "Ada Lovelace", testUser());
executionObject = executionsController.simulate(executionObject.getId(),
new ExecutionSimulationRequest(simulatorDescriptor), testUser());
new ExecutionSimulationRequest(simulatorDescriptor, null), testUser());
org.junit.jupiter.api.Assertions.assertTrue(executionObject.isInteractionSimulationEnabled());
waitForExecutionStatus(executionObject, ExecutionStatus.SUCCESS);
}
@ -964,7 +964,7 @@ public class ExecutionControllerTest {
new ExecutionSimulationRequest(LLMDescriptor.builder()
.provider("testProvider")
.model("simulateModel")
.build()), testUser()));
.build(), null), testUser()));
org.junit.jupiter.api.Assertions.assertEquals(HttpStatus.BAD_REQUEST, exception.getStatusCode());
org.junit.jupiter.api.Assertions.assertEquals(ExecutionStatus.CREATED, executionObject.getContext().getStatus());
}

View File

@ -4,6 +4,7 @@
package it.cnr.isti.workflow.manager.executions;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
@ -25,11 +26,16 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.TestConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.http.HttpStatus;
import org.springframework.test.context.bean.override.mockito.MockitoBean;
import org.springframework.test.context.TestPropertySource;
import org.springframework.util.ResourceUtils;
import org.springframework.web.server.ResponseStatusException;
import it.cnr.isti.workflow.manager.vault.UserSecretService;
import it.cnr.isti.workflow.manager.vault.model.VaultSecretCreateRequest;
import it.cnr.isti.workflow.manager.vault.model.VaultSecretView;
import tools.jackson.core.type.TypeReference;
import tools.jackson.databind.ObjectMapper;
@ -101,10 +107,53 @@ public class ExecutionTest {
.provider("testProvider")
.model("simulateModel")
.build();
/** A simulator provider that requires a credential, unlike testProvider - see the tests below. */
private static final LLMDescriptor CREDENTIAL_SIMULATOR_DESCRIPTOR = LLMDescriptor.builder()
.provider("credentialTestProvider")
.model("simulateModel")
.build();
private static final String CREDENTIAL_TEST_PROVIDER_KEY = "LLMProvider::credentialTestProvider::authorization";
@TestConfiguration
static class TestConfig {
@Bean
public LLMProvider credentialTestProvider() {
return new LLMProvider() {
@Override
public String getName() {
return "credentialTestProvider";
}
@Override
public List<String> getRegisteredModels() {
return List.of();
}
@Override
public boolean requiresAuthorization() {
return true;
}
@Override
public String generate(String model, String prompt) {
throw new UnsupportedOperationException("not exercised without a credential");
}
@Override
public String generate(String model, String prompt, String authorization) {
if (prompt.contains("###SIMULATED_CHAT_MESSAGE###")) {
return "MESSAGE: Please continue the interview";
}
if (prompt.contains("###SIMULATED_CHAT_FINAL###")) {
return "FINAL: done using " + authorization;
}
return "used " + authorization;
}
};
}
@Bean
public LLMProvider testProvider() {
return new LLMProvider() {
@ -208,6 +257,9 @@ public class ExecutionTest {
@MockitoBean
MCPAgentService mcpAgentService;
@Autowired
UserSecretService userSecretService;
LLMDescriptor llmBrick = LLMDescriptor.builder()
.provider("testProvider")
.model("testModel")
@ -427,7 +479,7 @@ public class ExecutionTest {
ExecutionObject execObject = executionsService.createExecution(flow);
execObject = executionsService.prepareInput(execObject.getId(), chatBlock.getId(), "cand", "John Doe");
execObject = executionsService.startSimulationExecution(execObject.getId(), SIMULATOR_DESCRIPTOR);
execObject = executionsService.startSimulationExecution(execObject.getId(), SIMULATOR_DESCRIPTOR, null);
while (execObject.getContext().getStatus() == ExecutionStatus.RUNNING) {
execObject = executionsService.getExecution(execObject.getId());
}
@ -441,6 +493,125 @@ public class ExecutionTest {
assertEquals(10, ((List<?>) history).size());
}
@Test
public void startSimulationExecutionFailsImmediatelyWithNoCredentialAnywhere() {
// The gap this closes: before, this failure only ever surfaced later, inside the step that
// reached the simulator - the RUNNING execution would then quietly turn into a failed step,
// rather than the request that started simulation being refused outright.
Block<ChatInteractionBlockType> chatBlock = chatInteractionBlockFactory.create(ChatInteractionBlockConfiguration.builder()
.name("Recruiter Chat")
.llmDescriptor(llmBrick)
.goalDescription("Assess ${{cand}} and reach a final decision")
.inputs(List.of(new ChatInteractionInput("cand", IOType.TEXT, false)))
.build());
ExecutionObject execObject = executionsService.createExecution("No credential anywhere flow",
FlowData.builder().block(chatBlock).build(), "sim-owner-none");
executionsService.prepareInput(execObject.getId(), chatBlock.getId(), "cand", "John Doe");
ResponseStatusException error = assertThrows(ResponseStatusException.class,
() -> executionsService.startSimulationExecution(execObject.getId(), CREDENTIAL_SIMULATOR_DESCRIPTOR, null));
assertEquals(HttpStatus.BAD_REQUEST, error.getStatusCode());
assertFalse(executionsService.getExecution(execObject.getId()).isInteractionSimulationEnabled());
}
@Test
public void startSimulationExecutionAcceptsAnExplicitCredentialForTheSimulator() {
VaultSecretView secret = userSecretService.create("sim-owner-explicit", new VaultSecretCreateRequest(
"sim-cred-" + java.util.UUID.randomUUID(), "credentialTestProvider", null, "secret-value", null));
Block<ChatInteractionBlockType> chatBlock = chatInteractionBlockFactory.create(ChatInteractionBlockConfiguration.builder()
.name("Recruiter Chat")
.llmDescriptor(llmBrick)
.goalDescription("Assess ${{cand}} and reach a final decision")
.inputs(List.of(new ChatInteractionInput("cand", IOType.TEXT, false)))
.build());
ExecutionObject execObject = executionsService.createExecution("Explicit simulator credential flow",
FlowData.builder().block(chatBlock).build(), "sim-owner-explicit");
executionsService.prepareInput(execObject.getId(), chatBlock.getId(), "cand", "John Doe");
execObject = executionsService.startSimulationExecution(execObject.getId(), CREDENTIAL_SIMULATOR_DESCRIPTOR, secret.id());
execObject = waitForExecutionStatus(execObject.getId(), ExecutionStatus.SUCCESS);
// Proves the credential's own value reached the provider, not just that no exception was
// thrown - the fake provider echoes back whatever "authorization" it was called with.
assertEquals("done using secret-value",
execObject.getContext().getResult().get(new FieldKey(chatBlock.getId(), ChatInteractionBlockFactory.RESPONSE_OUTPUT)));
}
@Test
public void startSimulationExecutionRejectsAnUnusableExplicitCredential() {
VaultSecretView secret = userSecretService.create("someone-else", new VaultSecretCreateRequest(
"not-mine-" + java.util.UUID.randomUUID(), "credentialTestProvider", null, "secret-value", null));
Block<ChatInteractionBlockType> chatBlock = chatInteractionBlockFactory.create(ChatInteractionBlockConfiguration.builder()
.name("Recruiter Chat")
.llmDescriptor(llmBrick)
.goalDescription("Assess ${{cand}} and reach a final decision")
.inputs(List.of(new ChatInteractionInput("cand", IOType.TEXT, false)))
.build());
ExecutionObject execObject = executionsService.createExecution("Unusable simulator credential flow",
FlowData.builder().block(chatBlock).build(), "sim-owner-wrong");
executionsService.prepareInput(execObject.getId(), chatBlock.getId(), "cand", "John Doe");
ResponseStatusException error = assertThrows(ResponseStatusException.class,
() -> executionsService.startSimulationExecution(execObject.getId(), CREDENTIAL_SIMULATOR_DESCRIPTOR, secret.id()));
assertEquals(HttpStatus.BAD_REQUEST, error.getStatusCode());
}
@Test
public void startSimulationExecutionFallsBackToAnAlreadyProvidedCredentialForTheSameProvider() {
// The coincidental-reuse case that has always worked: the flow's own step already needed
// credentialTestProvider and the user already provided one for it, so choosing the same
// provider as simulator - with no separate credentialId - still resolves, unchanged.
VaultSecretView secret = userSecretService.create("sim-owner-reuse", new VaultSecretCreateRequest(
"flow-cred-" + java.util.UUID.randomUUID(), "credentialTestProvider", null, "flow-secret", null));
Block<LLMBlockType> ownBlock = llmBlockFactory.create(LLMBlockConfiguration.builder()
.name("Uses the same provider")
.llmDescriptor(LLMDescriptor.builder().provider("credentialTestProvider").model("m").build())
.prompt("static prompt, no placeholders")
.build());
Block<HumanInteractionBlockType> innerInteraction = humanInteractiveBlockFactory.create(
HumanInteractiveBlockConfiguration.builder()
.name("Review")
.actionDescription("Review the submitted value")
.build());
ExecutionObject execObject = executionsService.createExecution("Reuse flow",
FlowData.builder().block(ownBlock).block(innerInteraction).build(), "sim-owner-reuse");
executionsService.prepareInput(execObject.getId(), innerInteraction.getId(), "input", "submission");
executionsService.setAuthorizationValue(execObject.getId(), CREDENTIAL_TEST_PROVIDER_KEY, secret.id());
assertDoesNotThrow(() -> executionsService.startSimulationExecution(execObject.getId(),
CREDENTIAL_SIMULATOR_DESCRIPTOR, null));
}
@Test
public void genericContainerPropagatesTheSimulatorsCredentialToItsChild() {
// Without propagation this fails exactly like
// startSimulationExecutionFailsImmediatelyWithNoCredentialAnywhere above, except inside the
// child's own step - the container would end FAILED, not SUCCESS, since the credential the
// parent already validated was never copied down to the child's own providedAuthorizations.
VaultSecretView secret = userSecretService.create("container-sim-owner", new VaultSecretCreateRequest(
"container-sim-cred-" + java.util.UUID.randomUUID(), "credentialTestProvider", null, "container-secret", null));
Block<HumanInteractionBlockType> innerInteraction = humanInteractiveBlockFactory.create(
HumanInteractiveBlockConfiguration.builder()
.name("Inner review")
.actionDescription("Review the submitted value")
.build());
Container<GenericContainerType> container = genericContainerFactory.create(GenericContainerConfiguration.builder()
.name("Simulated container needing a credential")
.subFlow(FlowData.builder().block(innerInteraction).build())
.build());
ExecutionObject parent = executionsService.createExecution("Container credential propagation flow",
FlowData.builder().container(container).build(), "container-sim-owner");
executionsService.prepareInput(parent.getId(), container.getId(), "input", "submission");
executionsService.startSimulationExecution(parent.getId(), CREDENTIAL_SIMULATOR_DESCRIPTOR, secret.id());
parent = waitForExecutionStatus(parent.getId(), ExecutionStatus.SUCCESS);
Step<?> containerStep = parent.getContext().getSteps().get(container.getId());
assertEquals(StepStatus.COMPLETED, containerStep.getStatus());
}
@Test
public void chatInteractionExecutionKeepsConversationHistoryAcrossMessages() {
Block<ChatInteractionBlockType> chatBlock = chatInteractionBlockFactory.create(ChatInteractionBlockConfiguration.builder()
@ -572,7 +743,7 @@ public class ExecutionTest {
ExecutionObject execution = executionsService.createExecution("MCP params",
FlowData.builder().block(chatBlock).build());
executionsService.prepareInput(execution.getId(), chatBlock.getId(), "cand", "Jordan");
execution = executionsService.startSimulationExecution(execution.getId(), llmBrick);
execution = executionsService.startSimulationExecution(execution.getId(), llmBrick, null);
for (int attempt = 0; attempt < 200 && !execution.getContext().getStatus().isFinalState(); attempt++) {
try {
Thread.sleep(50);
@ -718,7 +889,7 @@ public class ExecutionTest {
ExecutionObject execObject = executionsService.createExecution(flow);
execObject = executionsService.prepareInput(execObject.getId(), chatBlock.getId(), "cand", "John Doe");
execObject = executionsService.startSimulationExecution(execObject.getId(), SIMULATOR_DESCRIPTOR);
execObject = executionsService.startSimulationExecution(execObject.getId(), SIMULATOR_DESCRIPTOR, null);
while (execObject.getContext().getStatus() == ExecutionStatus.RUNNING) {
execObject = executionsService.getExecution(execObject.getId());
}
@ -1956,7 +2127,7 @@ public class ExecutionTest {
executionsService.prepareInput(parent.getId(), container.getId(), "input", "submission");
assertTrue(parent.isSimulationAvailable());
executionsService.startSimulationExecution(parent.getId(), SIMULATOR_DESCRIPTOR);
executionsService.startSimulationExecution(parent.getId(), SIMULATOR_DESCRIPTOR, null);
parent = waitForExecutionStatus(parent.getId(), ExecutionStatus.SUCCESS);
Step<?> containerStep = parent.getContext().getSteps().get(container.getId());

View File

@ -134,7 +134,7 @@ public class ExecutionWithContainer {
}
assertEquals(ExecutionStatus.READY, execObject.getContext().getStatus());
execObject = executionsService.startSimulationExecution(execObject.getId(), llmBrick);
execObject = executionsService.startSimulationExecution(execObject.getId(), llmBrick, null);
assertEquals(ExecutionStatus.RUNNING, execObject.getContext().getStatus());
while (execObject.getContext().getStatus() == ExecutionStatus.RUNNING) {
try {

View File

@ -920,7 +920,7 @@ class BiasExperimentsIntegrationTest {
OWNER);
executionsService.prepareInput(execution.getId(), decision.getId(),
HumanDecisionBlockFactory.INPUT_NAME, "the answer under review");
executionsService.startSimulationExecution(execution.getId(), simulator);
executionsService.startSimulationExecution(execution.getId(), simulator, null);
waitUntilFinal(execution);
assertEquals(ExecutionStatus.SUCCESS, execution.getContext().getStatus());
assertTrue(execution.isInteractionSimulationEnabled());
@ -937,7 +937,7 @@ class BiasExperimentsIntegrationTest {
List.of(baseline.annotationId()), false, BiasInterventionDirection.BIAS)),
ExternalSideEffectPolicy.BLOCK,
false));
executionsService.startSimulationExecution(variant.getId(), simulator);
executionsService.startSimulationExecution(variant.getId(), simulator, null);
waitUntilFinal(variant);
assertEquals(ExecutionStatus.SUCCESS, variant.getContext().getStatus());
return variant;