Intra-vendor model routing feature enhancements

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/APIAdmin.java

@@ -675,7 +675,7 @@ List<KeyManagerConfigurationDTO> getGlobalKeyManagerConfigurations(String organi
      * @param organization  The organization name
      * @param llmProviderId The ID of the LLM provider
      * @return list of models registered under the LLM provider
-     * @throws APIManagementException
+     * @throws APIManagementException If an error occurs while retrieving the models
      */
     List<String> getLLMProviderModels(String organization, String llmProviderId) throws APIManagementException;
 

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/APIConstants.java

@@ -44,14 +44,17 @@ public class APIConstants {
     public static final String ENDPOINT_SECURITY_PRODUCTION = "production";
     public static final String ENDPOINT_SECURITY_SANDBOX = "sandbox";
     public static final String ENDPOINT_CONFIG_SESSION_TIMEOUT = "sessionTimeOut";
+    public static final String OM_ELEMENT_NAME = "name";
 
     public static class AIAPIConstants {
+        public static final int MILLISECONDS_IN_SECOND = 1000;
         public static final String LLM_PROVIDERS = "llmProviders";
         public static final String API_KEY_IDENTIFIER_TYPE_HEADER = "HEADER";
         public static final String API_KEY_IDENTIFIER_TYPE_QUERY_PARAMETER = "QUERY_PARAMETER";
         public static final String AI_API_REQUEST_METADATA = "AI_API_REQUEST_METADATA";
         public static final String AI_API_RESPONSE_METADATA = "AI_API_RESPONSE_METADATA";
         public static final String INPUT_SOURCE_PAYLOAD = "payload";
+        public static final String INPUT_SOURCE_HEADER = "header";
         public static final String CONNECTOR_TYPE = "connectorType";
         public static final String ID = "id";
         public static final String NAME = "name";
@@ -82,12 +85,14 @@ public static class AIAPIConstants {
         public static final String LLM_PROVIDER_SERVICE_METADATA_PROMPT_TOKEN_COUNT = "promptTokenCount";
         public static final String LLM_PROVIDER_SERVICE_METADATA_COMPLETION_TOKEN_COUNT = "completionTokenCount";
         public static final String LLM_PROVIDER_SERVICE_METADATA_TOTAL_TOKEN_COUNT = "totalTokenCount";
+        public static final String LLM_PROVIDER_SERVICE_METADATA_REMAINING_TOKEN_COUNT = "remainingTokenCount";
         public static final String LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL = "$.model";
         public static final String LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_PROMPT_TOKEN_COUNT = "$.usage" +
                 ".prompt_tokens";
         public static final String LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_COMPLETION_TOKEN_COUNT = "$.usage" +
                 ".completion_tokens";
         public static final String LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_TOTAL_TOKEN_COUNT = "$.usage.total_tokens";
+        public static final String LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_REMAINING_TOKEN_COUNT = "x-ratelimit-remaining-tokens";
         public static final String LLM_PROVIDER_SERVICE_DEFAULT = "default";
         public static final String NULL = "null";
         public static final String LLM_PROVIDER = "LLM_PROVIDER";
@@ -96,12 +101,24 @@ public static class AIAPIConstants {
         public static final String TRAFFIC_FLOW_DIRECTION_IN = "IN";
         public static final String TRAFFIC_FLOW_DIRECTION_OUT = "OUT";
         public static final String API_LLM_ENDPOINT = "_API_LLMEndpoint_";
-        public static final String TARGET_MODEL = "targetModel";
-        public static final String TARGET_ENDPOINT = "targetEndpoint";
-        public static final String SUSPEND_DURATION = "suspendDuration";
+        public static final String TARGET_MODEL = "TARGET_MODEL";
+        public static final String TARGET_ENDPOINT = "TARGET_ENDPOINT";
+        public static final String FAILOVER_TARGET_MODEL = "FAILOVER_TARGET_MODEL";
+        public static final String FAILOVER_TARGET_ENDPOINT = "FAILOVER_TARGET_ENDPOINT";
+        public static final String SUSPEND_DURATION = "SUSPEND_DURATION";
+        public static final String ENDPOINT_TIMEOUT = "ENDPOINT_TIMEOUT";
+        public static final String FAILOVER_ENDPOINTS = "FAILOVER_ENDPOINTS";
         public static final String REJECT_ENDPOINT = "REJECT";
         public static final String DEFAULT_ENDPOINT = "DEFAULT";
-        public static final String REMAINING_TOKEN_COUNT_HEADER = "x-ratelimit-remaining-tokens";
-        public static final String REMAINING_REQUEST_COUNT_HEADER = "x-ratelimit-remaining-requests";
+        public static final String EXIT_ENDPOINT = "EXIT";
+        public static final String REQUEST_PAYLOAD = "REQUEST_PAYLOAD";
+        public static final String REQUEST_HEADERS = "REQUEST_HEADERS";
+        public static final String REQUEST_HTTP_METHOD = "REQUEST_HTTP_METHOD";
+        public static final String REQUEST_REST_URL_POSTFIX = "REQUEST_REST_URL_POSTFIX";
+        public static final String REQUEST_MODEL = "REQUEST_MODEL";
+        public static final String CURRENT_ENDPOINT_INDEX = "CURRENT_ENDPOINT_INDEX";
+        public static final String DEFAULT_PRODUCTION_ENDPOINT_NAME = "DEFAULT PRODUCTION ENDPOINT";
+        public static final String DEFAULT_SANDBOX_ENDPOINT_NAME = "DEFAULT SANDBOX ENDPOINT";
+        public static final String ENDPOINT_SEQUENCE = "_EndpointsSeq";
     }
 }

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/APIProvider.java

@@ -1603,16 +1603,16 @@ APIEndpointInfo getAPIEndpointByUUID(String apiUUID, String endpointUUID, String
      * @param apiUUID      Unique identifier of API
      * @param apiEndpoint  API endpoint details
      * @param organization Organization name
-     * @return created endpoint UUID
-     * @throws APIManagementException if an error occurs while inserting endpoint detail
+     * @return UUID of the added endpoint
+     * @throws APIManagementException if an error occurs while adding the endpoint
      */
     String addAPIEndpoint(String apiUUID, APIEndpointInfo apiEndpoint, String organization)
             throws APIManagementException;
 
     /**
-     * Delete an API endpoint by providing the endpoint UUID.
+     * Delete endpoint by providing the endpoint UUID.
      *
-     * @param endpointUUID Unique identifier of Endpoint
+     * @param endpointUUID Unique identifier of endpoint
      * @throws APIManagementException if an error occurs while deleting the endpoint
      */
     void deleteAPIEndpointById(String endpointUUID) throws APIManagementException;

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/AzureOpenAiLLMProviderService.java

@@ -65,22 +65,29 @@ public LLMProvider getLLMProvider()
 
             List<LLMProviderMetadata> llmProviderMetadata = new ArrayList<>();
             llmProviderMetadata.add(new LLMProviderMetadata(
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_MODEL,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_REQUEST_MODEL,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL, false));
+            llmProviderMetadata.add(new LLMProviderMetadata(
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_RESPONSE_MODEL,
+                    APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_PROMPT_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_PROMPT_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_PROMPT_TOKEN_COUNT, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_COMPLETION_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants
-                            .LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_COMPLETION_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_COMPLETION_TOKEN_COUNT, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_TOTAL_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_TOTAL_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_TOTAL_TOKEN_COUNT, true));
+            llmProviderMetadata.add(new LLMProviderMetadata(
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_REMAINING_TOKEN_COUNT,
+                    APIConstants.AIAPIConstants.INPUT_SOURCE_HEADER,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_REMAINING_TOKEN_COUNT, false));
             llmProviderConfiguration.setMetadata(llmProviderMetadata);
 
             llmProvider.setConfigurations(llmProviderConfiguration.toJsonString());

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/ExceptionCodes.java

@@ -784,20 +784,27 @@ public enum ExceptionCodes implements ErrorHandler {
             "Workflow status is not defined"),
     RESOURCE_URI_TEMPLATE_NOT_DEFINED(902032, "Resource URI template value not defined", 400,
             "Resource URI template value (target) not defined", false),
-    API_ENDPOINT_NOT_FOUND(902040, "Cannot find the required API Endpoint details.", 404,
+    API_ENDPOINT_NOT_FOUND(902040, "Cannot find the required API endpoint details.", 404,
             "Requested API endpoint with id '%s' not found."),
-    ERROR_UPDATING_API_ENDPOINT_API(902041, "Error has occurred. Cannot update an API endpoint.", 500,
-            "Error when updating the API Endpoint."),
-    ERROR_INSERTING_API_ENDPOINT_API(902042, "Error has occurred. Fail to add an API endpoint to API.", 500,
-            "Error has occurred while inserting an API endpoint."),
-    ERROR_MISSING_ENDPOINT_CONFIG_OF_API_ENDPOINT_API(902043, "Missing mandatory API endpoint's endpoint config", 500,
-            "Required attributes %s for an API endpoint config specification %s are either missing or empty"),
-    ERROR_READING_API_ENDPOINTS_FILE(902044, "Error while reading API Endpoints from the endpoints file",
-            400, "Error while reading API Endpoints from the endpoints file"),
-    ERROR_ADDING_API_ENDPOINT(902045, "Error while adding API Endpoint to the API", 500,
-            "Error while adding API Endpoint with ID: %s to the API"),
+    ERROR_UPDATING_API_ENDPOINT(902041, "Error while updating the API endpoint.", 500,
+            "Error while updating the API endpoint."),
+    ENDPOINT_READONLY(902042, "API endpoint is read only", 400,
+            "API endpoint with UUID %s is read only"),
+    ERROR_ADDING_API_ENDPOINT(902043, "Failed to add endpoint to API.", 500,
+            "Error while adding API endpoint."),
+    ERROR_MISSING_ENDPOINT_CONFIG_OF_API_ENDPOINT_API(902044, "Mandatory endpoint config is missing " +
+            "in endpoint", 500, "Mandatory endpoint config is either missing or empty"),
+    ERROR_READING_API_ENDPOINTS_FILE(902045, "Error while reading API endpoints from the endpoints file",
+            400, "Error while reading API endpoints from the endpoints file"),
     ERROR_ADDING_API_ENDPOINTS(902046, "Error while adding API Endpoints to the API", 500,
-            "Error while adding API Endpoints to the API");
+            "Error while adding API Endpoint to the API"),
+    ERROR_DELETING_API_ENDPOINT(902047, "Error while deleting API endpoint", 500,
+            "Error while deleting API endpoint with UUID '%s'."),
+    ERROR_DELETING_PRIMARY_API_ENDPOINT(902048, "Failed to delete API endpoint since endpoint is " +
+            "defined as a primary endpoint", 400,
+            "Failed to delete API endpoint with UUID '%s' since it is defined as a primary endpoint."),
+    API_ENDPOINT_URL_INVALID(902049, "Endpoint URL is invalid", 400,
+            "Endpoint URL is invalid");
     private final long errorCode;
     private final String errorMessage;
     private final int httpStatusCode;

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/LLMProviderMetadata.java

@@ -31,8 +31,16 @@ public class LLMProviderMetadata {
     @JsonProperty("attributeIdentifier")
     private String attributeIdentifier;
 
+    @JsonProperty("required")
+    private boolean required = true;
+
     public LLMProviderMetadata() {}
 
+    /**
+     * @deprecated This constructor is deprecated. Use {@link #LLMProviderMetadata(String, String, String, boolean)}
+     *         instead.
+     */
+    @Deprecated
     public LLMProviderMetadata(@JsonProperty("attributeName") String attributeName,
                                @JsonProperty("inputSource") String inputSource,
                                @JsonProperty("attributeIdentifier") String attributeIdentifier) {
@@ -42,6 +50,16 @@ public LLMProviderMetadata(@JsonProperty("attributeName") String attributeName,
         this.attributeIdentifier = attributeIdentifier;
     }
 
+    public LLMProviderMetadata(@JsonProperty("attributeName") String attributeName,
+            @JsonProperty("inputSource") String inputSource,
+            @JsonProperty("attributeIdentifier") String attributeIdentifier,
+            @JsonProperty("required") boolean required) {
+        this.attributeName = attributeName;
+        this.inputSource = inputSource;
+        this.attributeIdentifier = attributeIdentifier;
+        this.required = required;
+    }
+
     public String getAttributeName() {
 
         return attributeName;
@@ -71,4 +89,12 @@ public void setAttributeIdentifier(String attributeIdentifier) {
 
         this.attributeIdentifier = attributeIdentifier;
     }
+
+    public boolean isRequired() {
+        return required;
+    }
+
+    public void setRequired(boolean required) {
+        this.required = required;
+    }
 }

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/MistralAiLLMProviderService.java

@@ -64,22 +64,29 @@ public LLMProvider getLLMProvider()
 
             List<LLMProviderMetadata> llmProviderMetadata = new ArrayList<>();
             llmProviderMetadata.add(new LLMProviderMetadata(
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_MODEL,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_REQUEST_MODEL,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL, false));
+            llmProviderMetadata.add(new LLMProviderMetadata(
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_RESPONSE_MODEL,
+                    APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_PROMPT_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_PROMPT_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_PROMPT_TOKEN_COUNT, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_COMPLETION_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants
-                            .LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_COMPLETION_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_COMPLETION_TOKEN_COUNT, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_TOTAL_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_TOTAL_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_TOTAL_TOKEN_COUNT, true));
+            llmProviderMetadata.add(new LLMProviderMetadata(
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_REMAINING_TOKEN_COUNT,
+                    APIConstants.AIAPIConstants.INPUT_SOURCE_HEADER,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_REMAINING_TOKEN_COUNT, false));
             llmProviderConfiguration.setMetadata(llmProviderMetadata);
 
             llmProvider.setConfigurations(llmProviderConfiguration.toJsonString());

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/OpenAiLLMProviderService.java

@@ -64,22 +64,29 @@ public LLMProvider getLLMProvider()
 
             List<LLMProviderMetadata> llmProviderMetadata = new ArrayList<>();
             llmProviderMetadata.add(new LLMProviderMetadata(
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_MODEL,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_REQUEST_MODEL,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL, false));
+            llmProviderMetadata.add(new LLMProviderMetadata(
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_RESPONSE_MODEL,
+                    APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_PROMPT_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_PROMPT_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_PROMPT_TOKEN_COUNT, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_COMPLETION_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants
-                            .LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_COMPLETION_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_COMPLETION_TOKEN_COUNT, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_TOTAL_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_TOTAL_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_TOTAL_TOKEN_COUNT, true));
+            llmProviderMetadata.add(new LLMProviderMetadata(
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_REMAINING_TOKEN_COUNT,
+                    APIConstants.AIAPIConstants.INPUT_SOURCE_HEADER,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_REMAINING_TOKEN_COUNT, false));
             llmProviderConfiguration.setMetadata(llmProviderMetadata);
 
             llmProvider.setConfigurations(llmProviderConfiguration.toJsonString());