Merge pull request #137 from rootcodelabs/dev

Getting update to 132

DSL/Liquibase/changelog/classifier-script-v9-models-metadata.sql

@@ -0,0 +1,52 @@
+-- liquibase formatted sql
+
+-- changeset kalsara Magamage:classifier-script-v9-changeset1
+CREATE TYPE Maturity_Label AS ENUM ('development', 'staging', 'production');
+
+-- changeset kalsara Magamage:classifier-script-v9-changeset2
+CREATE TYPE Deployment_Env AS ENUM ('jira', 'outlook', 'pinal', 'testing', 'undeployed');
+
+-- changeset kalsara Magamage:classifier-script-v9-changeset3
+CREATE TYPE Training_Status AS ENUM ('not trained', 'training in progress', 'trained', 'retraining needed', 'untrainable');
+
+-- changeset kalsara Magamage:classifier-script-v9-changeset4
+CREATE TYPE Base_Models AS ENUM ('xlnet', 'roberta', 'albert');
+
+-- changeset kalsara Magamage:classifier-script-v9-changeset5
+CREATE TABLE models_metadata (
+    id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY,
+    model_group_key TEXT NOT NULL,
+    model_name TEXT NOT NULL,
+    major_version INT NOT NULL DEFAULT 0,
+    minor_version INT NOT NULL DEFAULT 0,
+    latest BOOLEAN DEFAULT false,
+    maturity_label Maturity_Label,
+    deployment_env Deployment_Env,
+    training_status Training_Status,
+    base_models Base_Models[],
+    last_trained_timestamp TIMESTAMP WITH TIME ZONE,
+    created_timestamp TIMESTAMP WITH TIME ZONE,
+    connected_dg_id INT,
+    connected_dg_name TEXT,
+    model_s3_location TEXT,
+    inference_routes JSONB,
+    training_results JSONB,
+    CONSTRAINT models_metadata_pkey PRIMARY KEY (id)
+);
+
+-- changeset kalsara Magamage:classifier-script-v9-changeset6
+CREATE TABLE model_configurations (
+    id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY,
+    base_models Base_Models[],
+    deployment_platforms Deployment_Env[],
+    maturity_labels Maturity_Label[],
+    CONSTRAINT model_configurations_pkey PRIMARY KEY (id)
+);
+
+-- changeset kalsara Magamage:classifier-script-v9-changeset7
+INSERT INTO model_configurations (base_models, deployment_platforms, maturity_labels) VALUES
+(
+    ARRAY['xlnet', 'roberta', 'albert']::Base_Models[],
+    ARRAY['jira', 'outlook', 'pinal', 'testing', 'undeployed']::Deployment_Env[],
+    ARRAY['development', 'staging', 'production']::Maturity_Label[]
+);

DSL/Resql/get-data-model-filters.sql

@@ -0,0 +1,23 @@
+SELECT json_build_object(
+    'modelNames', modelNames,
+    'modelVersions', modelVersions,
+    'datasetGroups', datasetGroups,
+    'deploymentsEnvs', deploymentsEnvs,
+    'trainingStatuses', trainingStatuses,
+    'maturityLabels', maturityLabels
+)
+FROM (
+    SELECT
+        array_agg(DISTINCT model_name) AS modelNames,
+        array_agg(DISTINCT
+            major_version::TEXT || '.x'
+        ) FILTER (WHERE major_version > 0) ||
+        array_agg(DISTINCT
+            'x.' || minor_version::TEXT
+        ) FILTER (WHERE minor_version > 0) AS modelVersions,
+        array_agg(DISTINCT connected_dg_name) AS datasetGroups,
+        array_agg(DISTINCT deployment_env) AS deploymentsEnvs,
+        array_agg(DISTINCT training_status) AS trainingStatuses,
+        array_agg(DISTINCT maturity_label) AS maturityLabels
+    FROM models_metadata
+) AS subquery;

DSL/Resql/get-data-model-metadata-by-id.sql

@@ -0,0 +1,14 @@
+SELECT
+    id AS model_id,
+    model_name,
+    major_version,
+    minor_version,
+    latest,
+    maturity_label,
+    deployment_env,
+    training_status,
+    base_models,
+    connected_dg_id,
+    connected_dg_name
+FROM models_metadata
+WHERE id = :id;

DSL/Resql/get-data-model-options.sql

@@ -0,0 +1,2 @@
+SELECT base_models, deployment_platforms, maturity_labels
+FROM model_configurations;

DSL/Resql/get-paginated-data-model-metadata.sql

@@ -0,0 +1,32 @@
+SELECT
+    dt.id,
+    dt.model_group_key,
+    dt.model_name,
+    dt.major_version,
+    dt.minor_version,
+    dt.latest,
+    dt.maturity_label,
+    dt.deployment_env,
+    dt.training_status,
+    dt.base_models,
+    dt.last_trained_timestamp,
+    dt.created_timestamp,
+    dt.connected_dg_id,
+    dt.connected_dg_name,
+    jsonb_pretty(dt.training_results) AS training_results,
+    CEIL(COUNT(*) OVER() / :page_size::DECIMAL) AS total_pages
+FROM
+    models_metadata dt
+WHERE
+    (:major_version = -1 OR dt.major_version = :major_version)
+    AND (:minor_version = -1 OR dt.minor_version = :minor_version)
+    AND (:model_name = 'all' OR dt.model_name = :model_name)
+    AND (:deployment_maturity = 'all' OR dt.maturity_label = :deployment_maturity::Maturity_Label)
+    AND (:training_status = 'all' OR dt.training_status = :training_status::Training_Status)
+    AND (:platform = 'all' OR dt.deployment_env = :platform::Deployment_Env)
+    AND (:dataset_group = 'all' OR dt.connected_dg_name = :dataset_group)
+ORDER BY
+    CASE WHEN :sort_type = 'asc' THEN dt.model_name END ASC,
+    CASE WHEN :sort_type = 'desc' THEN dt.model_name END DESC
+OFFSET ((GREATEST(:page, 1) - 1) * :page_size) LIMIT :page_size;
+

DSL/Resql/get-validated-all-dataset-groups.sql

@@ -0,0 +1,3 @@
+SELECT id as dg_id, group_name, major_version, minor_version, patch_version
+FROM dataset_group_metadata
+WHERE is_enabled = true AND validation_status = 'success';

DSL/Resql/insert-model-metadata.sql

@@ -0,0 +1,27 @@
+INSERT INTO models_metadata (
+    model_group_key,
+    model_name,
+    major_version,
+    minor_version,
+    latest,
+    maturity_label,
+    deployment_env,
+    training_status,
+    base_models,
+    created_timestamp,
+    connected_dg_id,
+    connected_dg_name
+) VALUES (
+    :model_group_key,
+    :model_name,
+    :major_version,
+    :minor_version,
+    :latest,
+    :maturity_label::Maturity_Label,
+    :deployment_env::Deployment_Env,
+    :training_status::Training_Status,
+     ARRAY [:base_models]::Base_Models[],
+    :created_timestamp::timestamp with time zone,
+    :connected_dg_id,
+    :connected_dg_name
+) RETURNING id;

DSL/Resql/update-patch-version-dataset-group.sql

@@ -7,11 +7,7 @@ WITH update_latest AS (
 update_specific AS (
     UPDATE dataset_group_metadata
     SET
-        patch_version = (
-            SELECT COALESCE(MAX(patch_version), 0) + 1
-            FROM dataset_group_metadata
-            WHERE group_key = :group_key AND major_version = :major_version AND minor_version = :minor_version
-        ),
+        patch_version = patch_version + 1,
         enable_allowed = false,
         validation_status = 'in-progress'::Validation_Status,
         is_enabled = false,

DSL/Ruuter.private/DSL/GET/classifier/datamodel/create/options.yml

@@ -0,0 +1,71 @@
+declaration:
+  call: declare
+  version: 0.1
+  description: "Description placeholder for 'OPTIONS'"
+  method: get
+  accepts: json
+  returns: json
+  namespace: classifier
+
+get_data_model_options:
+  call: http.post
+  args:
+    url: "[#CLASSIFIER_RESQL]/get-data-model-options"
+  result: res_options
+  next: check_status
+
+check_status:
+  switch:
+    - condition: ${200 <= res_options.response.statusCodeValue && res_options.response.statusCodeValue < 300}
+      next: get_dataset_group_data
+  next: return_bad_request
+
+get_dataset_group_data:
+  call: http.post
+  args:
+    url: "[#CLASSIFIER_RESQL]/get-validated-all-dataset-groups"
+  result: res_dataset
+  next: check_dataset_group_status
+
+check_dataset_group_status:
+  switch:
+    - condition: ${200 <= res_dataset.response.statusCodeValue && res_dataset.response.statusCodeValue < 300}
+      next: check_data_exist
+  next: return_bad_request
+
+check_data_exist:
+  switch:
+    - condition: ${res_dataset.response.body.length>0}
+      next: assign_dataset
+  next: assign_empty
+
+assign_dataset:
+  assign:
+    dataset_group: ${res_dataset.response.body}
+  next: assign_success_response
+
+assign_empty:
+  assign:
+    dataset_group: []
+  next: assign_success_response
+
+assign_success_response:
+  assign:
+    format_res: {
+      baseModels: '${res_options.response.body[0].baseModels}',
+      deploymentPlatforms: '${res_options.response.body[0].deploymentPlatforms}',
+      maturityLabels: '${res_options.response.body[0].maturityLabels}',
+      datasetGroups: '${dataset_group}'
+    }
+  next: return_ok
+
+return_ok:
+  status: 200
+  return: ${format_res}
+  next: end
+
+return_bad_request:
+  status: 400
+  return: "Bad Request"
+  next: end
+

DSL/Ruuter.private/DSL/GET/classifier/datamodel/metadata.yml

@@ -0,0 +1,65 @@
+declaration:
+  call: declare
+  version: 0.1
+  description: "Description placeholder for 'METADATA'"
+  method: get
+  accepts: json
+  returns: json
+  namespace: classifier
+  allowlist:
+    params:
+      - field: modelId
+        type: number
+        description: "Parameter 'modelId'"
+
+extract_data:
+  assign:
+    model_id: ${Number(incoming.params.modelId)}
+  next: get_data_model_meta_data_by_id
+
+get_data_model_meta_data_by_id:
+  call: http.post
+  args:
+    url: "[#CLASSIFIER_RESQL]/get-data-model-metadata-by-id"
+    body:
+      id: ${model_id}
+  result: res_model
+  next: check_status
+
+check_status:
+  switch:
+    - condition: ${200 <= res_model.response.statusCodeValue && res_model.response.statusCodeValue < 300}
+      next: check_data_exist
+  next: assign_fail_response
+
+check_data_exist:
+  switch:
+    - condition: ${res_model.response.body.length>0}
+      next: assign_success_response
+  next: assign_fail_response
+
+assign_success_response:
+  assign:
+    format_res: {
+      operationSuccessful: true,
+      data: '${res_model.response.body}'
+    }
+  next: return_ok
+
+assign_fail_response:
+  assign:
+    format_res: {
+      operationSuccessful: false,
+      data: '${[]}'
+    }
+  next: return_bad_request
+
+return_ok:
+  status: 200
+  return: ${format_res}
+  next: end
+
+return_bad_request:
+  status: 400
+  return: ${format_res}
+  next: end