Merge pull request #12947 from ashera96/intra-vendor-model-routing-po…

…st-alpha

Intra-vendor model routing feature enhancements

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/APIAdmin.java

@@ -675,7 +675,7 @@ List<KeyManagerConfigurationDTO> getGlobalKeyManagerConfigurations(String organi
      * @param organization  The organization name
      * @param llmProviderId The ID of the LLM provider
      * @return list of models registered under the LLM provider
-     * @throws APIManagementException
+     * @throws APIManagementException If an error occurs while retrieving the models
      */
     List<String> getLLMProviderModels(String organization, String llmProviderId) throws APIManagementException;
 

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/APIConstants.java

@@ -52,6 +52,7 @@ public static class AIAPIConstants {
         public static final String AI_API_REQUEST_METADATA = "AI_API_REQUEST_METADATA";
         public static final String AI_API_RESPONSE_METADATA = "AI_API_RESPONSE_METADATA";
         public static final String INPUT_SOURCE_PAYLOAD = "payload";
+        public static final String INPUT_SOURCE_HEADER = "header";
         public static final String CONNECTOR_TYPE = "connectorType";
         public static final String ID = "id";
         public static final String NAME = "name";
@@ -82,12 +83,14 @@ public static class AIAPIConstants {
         public static final String LLM_PROVIDER_SERVICE_METADATA_PROMPT_TOKEN_COUNT = "promptTokenCount";
         public static final String LLM_PROVIDER_SERVICE_METADATA_COMPLETION_TOKEN_COUNT = "completionTokenCount";
         public static final String LLM_PROVIDER_SERVICE_METADATA_TOTAL_TOKEN_COUNT = "totalTokenCount";
+        public static final String LLM_PROVIDER_SERVICE_METADATA_REMAINING_TOKEN_COUNT = "remainingTokenCount";
         public static final String LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL = "$.model";
         public static final String LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_PROMPT_TOKEN_COUNT = "$.usage" +
                 ".prompt_tokens";
         public static final String LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_COMPLETION_TOKEN_COUNT = "$.usage" +
                 ".completion_tokens";
         public static final String LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_TOTAL_TOKEN_COUNT = "$.usage.total_tokens";
+        public static final String LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_REMAINING_TOKEN_COUNT = "x-ratelimit-remaining-tokens";
         public static final String LLM_PROVIDER_SERVICE_DEFAULT = "default";
         public static final String NULL = "null";
         public static final String LLM_PROVIDER = "LLM_PROVIDER";

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/APIProvider.java

@@ -1603,16 +1603,16 @@ APIEndpointInfo getAPIEndpointByUUID(String apiUUID, String endpointUUID, String
      * @param apiUUID      Unique identifier of API
      * @param apiEndpoint  API endpoint details
      * @param organization Organization name
-     * @return created endpoint UUID
-     * @throws APIManagementException if an error occurs while inserting endpoint detail
+     * @return UUID of the added endpoint
+     * @throws APIManagementException if an error occurs while adding the endpoint
      */
     String addAPIEndpoint(String apiUUID, APIEndpointInfo apiEndpoint, String organization)
             throws APIManagementException;
 
     /**
-     * Delete an API endpoint by providing the endpoint UUID.
+     * Delete endpoint by providing the endpoint UUID.
      *
-     * @param endpointUUID Unique identifier of Endpoint
+     * @param endpointUUID Unique identifier of endpoint
      * @throws APIManagementException if an error occurs while deleting the endpoint
      */
     void deleteAPIEndpointById(String endpointUUID) throws APIManagementException;

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/AzureOpenAiLLMProviderService.java

@@ -65,22 +65,29 @@ public LLMProvider getLLMProvider()
 
             List<LLMProviderMetadata> llmProviderMetadata = new ArrayList<>();
             llmProviderMetadata.add(new LLMProviderMetadata(
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_MODEL,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_REQUEST_MODEL,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL, false));
+            llmProviderMetadata.add(new LLMProviderMetadata(
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_RESPONSE_MODEL,
+                    APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_PROMPT_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_PROMPT_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_PROMPT_TOKEN_COUNT, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_COMPLETION_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants
-                            .LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_COMPLETION_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_COMPLETION_TOKEN_COUNT, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_TOTAL_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_TOTAL_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_TOTAL_TOKEN_COUNT, true));
+            llmProviderMetadata.add(new LLMProviderMetadata(
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_REMAINING_TOKEN_COUNT,
+                    APIConstants.AIAPIConstants.INPUT_SOURCE_HEADER,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_REMAINING_TOKEN_COUNT, false));
             llmProviderConfiguration.setMetadata(llmProviderMetadata);
 
             llmProvider.setConfigurations(llmProviderConfiguration.toJsonString());

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/ExceptionCodes.java

@@ -784,20 +784,20 @@ public enum ExceptionCodes implements ErrorHandler {
             "Workflow status is not defined"),
     RESOURCE_URI_TEMPLATE_NOT_DEFINED(902032, "Resource URI template value not defined", 400,
             "Resource URI template value (target) not defined", false),
-    API_ENDPOINT_NOT_FOUND(902040, "Cannot find the required API Endpoint details.", 404,
+    API_ENDPOINT_NOT_FOUND(902040, "Cannot find the required API endpoint details.", 404,
             "Requested API endpoint with id '%s' not found."),
-    ERROR_UPDATING_API_ENDPOINT_API(902041, "Error has occurred. Cannot update an API endpoint.", 500,
-            "Error when updating the API Endpoint."),
-    ERROR_INSERTING_API_ENDPOINT_API(902042, "Error has occurred. Fail to add an API endpoint to API.", 500,
-            "Error has occurred while inserting an API endpoint."),
-    ERROR_MISSING_ENDPOINT_CONFIG_OF_API_ENDPOINT_API(902043, "Missing mandatory API endpoint's endpoint config", 500,
-            "Required attributes %s for an API endpoint config specification %s are either missing or empty"),
-    ERROR_READING_API_ENDPOINTS_FILE(902044, "Error while reading API Endpoints from the endpoints file",
-            400, "Error while reading API Endpoints from the endpoints file"),
-    ERROR_ADDING_API_ENDPOINT(902045, "Error while adding API Endpoint to the API", 500,
-            "Error while adding API Endpoint with ID: %s to the API"),
-    ERROR_ADDING_API_ENDPOINTS(902046, "Error while adding API Endpoints to the API", 500,
-            "Error while adding API Endpoints to the API");
+    ERROR_UPDATING_API_ENDPOINT(902041, "Error while updating the API endpoint.", 500,
+            "Error while updating the API endpoint."),
+    ERROR_ADDING_API_ENDPOINT(902042, "Failed to add endpoint to API.", 500,
+            "Error while adding API endpoint."),
+    ERROR_MISSING_ENDPOINT_CONFIG_OF_API_ENDPOINT_API(902043, "Mandatory endpoint config is missing in endpoint",
+            500, "Mandatory endpoint config is either missing or empty"),
+    ERROR_READING_API_ENDPOINTS_FILE(902044, "Error while reading API endpoints from the endpoints file",
+            400, "Error while reading API endpoints from the endpoints file"),
+    ERROR_ADDING_API_ENDPOINTS(902045, "Error while adding API Endpoints to the API", 500,
+            "Error while adding API Endpoint to the API"),
+    ERROR_DELETING_API_ENDPOINT(902046, "Error while deleting API endpoint", 500,
+            "Error while deleting API endpoint with UUID '%s'."),;
     private final long errorCode;
     private final String errorMessage;
     private final int httpStatusCode;

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/LLMProviderMetadata.java

@@ -31,8 +31,16 @@ public class LLMProviderMetadata {
     @JsonProperty("attributeIdentifier")
     private String attributeIdentifier;
 
+    @JsonProperty("required")
+    private boolean required = true;
+
     public LLMProviderMetadata() {}
 
+    /**
+     * @deprecated This constructor is deprecated. Use {@link #LLMProviderMetadata(String, String, String, boolean)}
+     *         instead.
+     */
+    @Deprecated
     public LLMProviderMetadata(@JsonProperty("attributeName") String attributeName,
                                @JsonProperty("inputSource") String inputSource,
                                @JsonProperty("attributeIdentifier") String attributeIdentifier) {
@@ -42,6 +50,16 @@ public LLMProviderMetadata(@JsonProperty("attributeName") String attributeName,
         this.attributeIdentifier = attributeIdentifier;
     }
 
+    public LLMProviderMetadata(@JsonProperty("attributeName") String attributeName,
+            @JsonProperty("inputSource") String inputSource,
+            @JsonProperty("attributeIdentifier") String attributeIdentifier,
+            @JsonProperty("required") boolean required) {
+        this.attributeName = attributeName;
+        this.inputSource = inputSource;
+        this.attributeIdentifier = attributeIdentifier;
+        this.required = required;
+    }
+
     public String getAttributeName() {
 
         return attributeName;
@@ -71,4 +89,12 @@ public void setAttributeIdentifier(String attributeIdentifier) {
 
         this.attributeIdentifier = attributeIdentifier;
     }
+
+    public boolean isRequired() {
+        return required;
+    }
+
+    public void setRequired(boolean required) {
+        this.required = required;
+    }
 }

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/MistralAiLLMProviderService.java

@@ -64,22 +64,29 @@ public LLMProvider getLLMProvider()
 
             List<LLMProviderMetadata> llmProviderMetadata = new ArrayList<>();
             llmProviderMetadata.add(new LLMProviderMetadata(
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_MODEL,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_REQUEST_MODEL,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL, false));
+            llmProviderMetadata.add(new LLMProviderMetadata(
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_RESPONSE_MODEL,
+                    APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_PROMPT_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_PROMPT_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_PROMPT_TOKEN_COUNT, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_COMPLETION_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants
-                            .LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_COMPLETION_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_COMPLETION_TOKEN_COUNT, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_TOTAL_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_TOTAL_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_TOTAL_TOKEN_COUNT, true));
+            llmProviderMetadata.add(new LLMProviderMetadata(
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_REMAINING_TOKEN_COUNT,
+                    APIConstants.AIAPIConstants.INPUT_SOURCE_HEADER,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_REMAINING_TOKEN_COUNT, false));
             llmProviderConfiguration.setMetadata(llmProviderMetadata);
 
             llmProvider.setConfigurations(llmProviderConfiguration.toJsonString());

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/OpenAiLLMProviderService.java

@@ -64,22 +64,29 @@ public LLMProvider getLLMProvider()
 
             List<LLMProviderMetadata> llmProviderMetadata = new ArrayList<>();
             llmProviderMetadata.add(new LLMProviderMetadata(
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_MODEL,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_REQUEST_MODEL,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL, false));
+            llmProviderMetadata.add(new LLMProviderMetadata(
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_RESPONSE_MODEL,
+                    APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_MODEL, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_PROMPT_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_PROMPT_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_PROMPT_TOKEN_COUNT, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_COMPLETION_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants
-                            .LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_COMPLETION_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_COMPLETION_TOKEN_COUNT, true));
             llmProviderMetadata.add(new LLMProviderMetadata(
                     APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_TOTAL_TOKEN_COUNT,
                     APIConstants.AIAPIConstants.INPUT_SOURCE_PAYLOAD,
-                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_TOTAL_TOKEN_COUNT));
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_TOTAL_TOKEN_COUNT, true));
+            llmProviderMetadata.add(new LLMProviderMetadata(
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_REMAINING_TOKEN_COUNT,
+                    APIConstants.AIAPIConstants.INPUT_SOURCE_HEADER,
+                    APIConstants.AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_IDENTIFIER_REMAINING_TOKEN_COUNT, false));
             llmProviderConfiguration.setMetadata(llmProviderMetadata);
 
             llmProvider.setConfigurations(llmProviderConfiguration.toJsonString());

components/apimgt/org.wso2.carbon.apimgt.api/src/main/java/org/wso2/carbon/apimgt/api/model/API.java

@@ -262,16 +262,6 @@ public void setBackendThrottlingConfiguration(BackendThrottlingConfiguration bac
         this.backendThrottlingConfiguration = backendThrottlingConfiguration;
     }
 
-    public String getPrimaryProductionEndpointId() {
-
-        return primaryProductionEndpointId;
-    }
-
-    public void setPrimaryProductionEndpointId(String primaryProductionEndpointId) {
-
-        this.primaryProductionEndpointId = primaryProductionEndpointId;
-    }
-
     public String getAudience() {
         return audience;
     }
@@ -1607,6 +1597,16 @@ public void setEgress(int egress) {
         isEgress = egress;
     }
 
+    public String getPrimaryProductionEndpointId() {
+
+        return primaryProductionEndpointId;
+    }
+
+    public void setPrimaryProductionEndpointId(String primaryProductionEndpointId) {
+
+        this.primaryProductionEndpointId = primaryProductionEndpointId;
+    }
+
     public String getPrimarySandboxEndpointId() {
         return primarySandboxEndpointId;
     }

components/apimgt/org.wso2.carbon.apimgt.gateway/src/main/java/org/wso2/carbon/apimgt/gateway/handlers/analytics/SynapseAnalyticsDataProvider.java

@@ -464,7 +464,7 @@ private void getAiAnalyticsData(Map aiApiResponseMetadata, int requestStartHour,
         aiMetadata.put(
                 Constants.AI_MODEL,
                 Optional.ofNullable(aiApiResponseMetadata.get(
-                            AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_MODEL)
+                            AIAPIConstants.LLM_PROVIDER_SERVICE_METADATA_RESPONSE_MODEL)
                         )
                         .map(Object::toString)
                         .orElse(null)

components/apimgt/org.wso2.carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIConstants.java

@@ -3308,7 +3308,7 @@ public static class APIEndpoint {
         public static final String PRODUCTION = "PRODUCTION";
         public static final String SANDBOX = "SANDBOX";
         public static final String PRIMARY_ENDPOINT_ID_SEPARATOR = "--";
-        public static final String DEFAULT_PROD_ENDPOINT = "Default Production Endpoint";
-        public static final String DEFAULT_SANDBOX_ENDPOINT = "Default Sandbox Endpoint";
+        public static final String DEFAULT_PROD_ENDPOINT_NAME = "Default Production Endpoint";
+        public static final String DEFAULT_SANDBOX_ENDPOINT_NAME = "Default Sandbox Endpoint";
     }
 }

components/apimgt/org.wso2.carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/dao/ApiMgtDAO.java

@@ -24404,7 +24404,8 @@ public List<APIEndpointInfo> getAPIEndpoints(String uuid, String organization) t
             currentApiUuid = uuid;
         }
 
-        try (Connection conn = APIMgtDBUtil.getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) {
+        try (Connection conn = APIMgtDBUtil.getConnection();
+                PreparedStatement ps = conn.prepareStatement(sql)) {
             ps.setString(1, currentApiUuid);
             ps.setString(2, organization);
             if (isRevision) {