diff --git a/src/main/java/it/cnr/isti/workflow/manager/executions/AuthorizationRequirementResolver.java b/src/main/java/it/cnr/isti/workflow/manager/executions/AuthorizationRequirementResolver.java index ba88457..03bd1fe 100644 --- a/src/main/java/it/cnr/isti/workflow/manager/executions/AuthorizationRequirementResolver.java +++ b/src/main/java/it/cnr/isti/workflow/manager/executions/AuthorizationRequirementResolver.java @@ -127,6 +127,39 @@ final class AuthorizationRequirementResolver { .addStepReference(block.getId(), block.getName()); } + /** + * Every {@link LLMDescriptor} in the flow, wherever one appears - the same walk + * {@link #collectRequirements} does, without the authorization bookkeeping. Used to check a + * model's availability against its provider's catalogue at execution start, before any step + * that would otherwise be the first to find out the model is wrong. + */ + static List resolveAllDescriptors(FlowData flow) { + List references = new ArrayList<>(); + collectDescriptors(flow, references); + return references; + } + + private static void collectDescriptors(FlowData flow, List references) { + if (flow == null || flow.getNodes().isEmpty()) { + return; + } + for (Block block : flow.getBlocks() == null ? List.>of() : flow.getBlocks()) { + resolveDescriptors(block).forEach(descriptor -> + references.add(new LLMDescriptorReference(descriptor, block == null ? null : block.getName()))); + } + for (Container container : flow.getContainers() == null ? List.>of() : flow.getContainers()) { + if (container != null && container.getSpecificConfiguration() instanceof ContainerConfiguration containerConfiguration) { + collectDescriptors(containerConfiguration.getSubFlow(), references); + if (containerConfiguration instanceof LoopContainerConfiguration loopConfiguration) { + collectDescriptors(loopConfiguration.getGuardSubFlow(), references); + } + } + } + } + + record LLMDescriptorReference(LLMDescriptor descriptor, String blockName) { + } + static LLMProvider resolveProvider(Map llmProviders, String providerName) { LLMProvider provider = llmProviders.get(providerName); if (provider != null) { diff --git a/src/main/java/it/cnr/isti/workflow/manager/executions/ExecutionsService.java b/src/main/java/it/cnr/isti/workflow/manager/executions/ExecutionsService.java index 1af7057..162f1d6 100644 --- a/src/main/java/it/cnr/isti/workflow/manager/executions/ExecutionsService.java +++ b/src/main/java/it/cnr/isti/workflow/manager/executions/ExecutionsService.java @@ -1411,11 +1411,58 @@ public class ExecutionsService { @Transactional public ExecutionObject startExecution(String id) { ExecutionObject eo = getExecution(id); + verifyModelAvailability(eo); eo.start(); touchExecution(id); return eo; } + /** + * Best-effort: a model the flow names but its provider does not actually serve fails on the + * very first call, which can be minutes into a run. Endpoint and credential have just been + * supplied, and the machinery to enumerate every {@code (provider, model)} pair already exists + * for authorization requirements - this reuses the same walk to ask each provider that can + * answer, before the run starts, rather than after. + * + *

Only a provider whose {@link LLMProvider#canListModels()} is true is asked at all: every + * hosted provider says false there precisely because it cannot be asked without a credential + * the editor does not have, so this only ever fires for our own Ollama today. A provider that + * can be asked but fails to answer just now - the network, not the model - does not block the + * run either: this is a defense in depth, not a gate that a transient failure should be able to + * close. + */ + private void verifyModelAvailability(ExecutionObject eo) { + for (AuthorizationRequirementResolver.LLMDescriptorReference reference + : AuthorizationRequirementResolver.resolveAllDescriptors(eo.getFlow())) { + LLMDescriptor descriptor = reference.descriptor(); + String model = descriptor.model() == null ? "" : descriptor.model().trim(); + // A template placeholder resolves only at call time - global inputs cannot reach this + // field through a port, so writing one here is the only way to make the model depend on + // one, and its actual value is unknowable this early. + if (model.isEmpty() || model.contains("${{")) { + continue; + } + LLMProvider provider = AuthorizationRequirementResolver.resolveProvider(llmProviders, descriptor.provider()); + if (provider == null || !provider.canListModels()) { + continue; + } + List registered; + try { + registered = provider.getRegisteredModels(); + } catch (RuntimeException exception) { + logger.warn("Could not verify model availability for provider {}: {}", provider.getName(), + exception.getMessage()); + continue; + } + if (registered != null && !registered.isEmpty() && !registered.contains(model)) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, + "Model \"" + model + "\" is not available from provider " + provider.getName() + + (reference.blockName() == null || reference.blockName().isBlank() + ? "" : " (used by " + reference.blockName() + ")")); + } + } + } + /** * @param credentialId a vault secret id for the simulator's own provider, or null to fall back * to whatever the flow's own steps already provided for that exact @@ -1436,6 +1483,7 @@ public class ExecutionsService { "Simulation requires a simulator descriptor"); } resolveSimulatorAuthorization(eo, simulatorDescriptor, credentialId); + verifyModelAvailability(eo); eo.setInteractionSimulationDescriptor(simulatorDescriptor); eo.startSimulation(); touchExecution(id); diff --git a/src/test/java/it/cnr/isti/workflow/manager/executions/ExecutionTest.java b/src/test/java/it/cnr/isti/workflow/manager/executions/ExecutionTest.java index 9afeca8..21f1433 100644 --- a/src/test/java/it/cnr/isti/workflow/manager/executions/ExecutionTest.java +++ b/src/test/java/it/cnr/isti/workflow/manager/executions/ExecutionTest.java @@ -164,7 +164,7 @@ public class ExecutionTest { } @Override public List getRegisteredModels() { - return List.of("testModel"); + return List.of("testModel", "configuredModel"); } @Override @@ -207,8 +207,35 @@ public class ExecutionTest { } }; } + + /** Declares a real, non-empty catalogue - unlike every other provider stub in this file. */ + @Bean + public LLMProvider modelCheckedProvider() { + return new LLMProvider() { + @Override + public String getName() { + return "modelCheckedProvider"; + } + + @Override + public List getRegisteredModels() { + if (modelListingShouldFail.get()) { + throw new IllegalStateException("catalogue temporarily unreachable"); + } + return List.of("known-model"); + } + + @Override + public String generate(String model, String prompt) { + return "answered by " + model; + } + }; + } } + private static final java.util.concurrent.atomic.AtomicBoolean modelListingShouldFail = + new java.util.concurrent.atomic.AtomicBoolean(false); + @Autowired ExecutionsService executionsService; @@ -1079,6 +1106,74 @@ public class ExecutionTest { .get(new FieldKey(block.getId(), LLMBlockFactory.OUTPUT_NAME))); } + @Test + public void startExecutionFailsFastWhenTheConfiguredModelIsNotInTheProvidersCatalogue() { + // The gap this closes: before, a model the flow names but its provider does not actually + // serve failed only on the first real call, which can be minutes into a run for a step deep + // in a flow. modelCheckedProvider is the one stub in this file whose catalogue is both real + // (non-empty) and narrow, so a mismatch here is caught before the execution ever starts. + modelListingShouldFail.set(false); + Block llmBlock = llmBlockFactory.create(LLMBlockConfiguration.builder() + .name("Ask the model") + .prompt("Which model answered?") + .llmDescriptor(LLMDescriptor.builder().provider("modelCheckedProvider").model("nonexistent-model").build()) + .build()); + ExecutionObject execObject = executionsService.createExecution( + Flow.builder().name("Unknown model flow").description("").block(llmBlock).build()); + + ResponseStatusException error = assertThrows(ResponseStatusException.class, + () -> executionsService.startExecution(execObject.getId())); + + assertEquals(HttpStatus.BAD_REQUEST, error.getStatusCode()); + assertTrue(error.getReason().contains("nonexistent-model")); + assertNotEquals(ExecutionStatus.RUNNING, + executionsService.getExecution(execObject.getId()).getContext().getStatus()); + } + + @Test + public void startExecutionProceedsWhenTheConfiguredModelIsInTheProvidersCatalogue() { + modelListingShouldFail.set(false); + Block llmBlock = llmBlockFactory.create(LLMBlockConfiguration.builder() + .name("Ask the model") + .prompt("Which model answered?") + .llmDescriptor(LLMDescriptor.builder().provider("modelCheckedProvider").model("known-model").build()) + .build()); + ExecutionObject execObject = executionsService.createExecution( + Flow.builder().name("Known model flow").description("").block(llmBlock).build()); + + execObject = executionsService.startExecution(execObject.getId()); + while (execObject.getContext().getStatus() == ExecutionStatus.RUNNING) { + execObject = executionsService.getExecution(execObject.getId()); + } + + assertEquals(ExecutionStatus.SUCCESS, execObject.getContext().getStatus()); + } + + @Test + public void startExecutionProceedsWhenTheProvidersCatalogueCannotBeListedJustNow() { + // Best-effort: a provider that can normally be asked, but fails to answer right now, must not + // turn a transient network problem into every execution refusing to start. + modelListingShouldFail.set(true); + try { + Block llmBlock = llmBlockFactory.create(LLMBlockConfiguration.builder() + .name("Ask the model") + .prompt("Which model answered?") + .llmDescriptor(LLMDescriptor.builder().provider("modelCheckedProvider").model("known-model").build()) + .build()); + ExecutionObject execObject = executionsService.createExecution( + Flow.builder().name("Unreachable catalogue flow").description("").block(llmBlock).build()); + + execObject = executionsService.startExecution(execObject.getId()); + while (execObject.getContext().getStatus() == ExecutionStatus.RUNNING) { + execObject = executionsService.getExecution(execObject.getId()); + } + + assertEquals(ExecutionStatus.SUCCESS, execObject.getContext().getStatus()); + } finally { + modelListingShouldFail.set(false); + } + } + @Test public void llmExecutionPrependsSelectedSkillsToPrompt() { Block llmBlock = llmBlockFactory.create(LLMBlockConfiguration.builder()