From d4f6b5d48c2b60d9c72db4566c7d16120733492c Mon Sep 17 00:00:00 2001 From: "kalsara.magamage_roo" Date: Wed, 31 Jul 2024 21:05:12 +0530 Subject: [PATCH 1/3] ESCLASS-159- create model API implementation and get model overview and filters --- .../classifier-script-v9-models-metadata.sql | 35 ++++++ DSL/Resql/get-data-model-filters.sql | 23 ++++ .../get-paginated-data-model-metadata.sql | 34 ++++++ DSL/Resql/insert-model-metadata.sql | 27 +++++ .../DSL/GET/classifier/datamodel/overview.yml | 104 ++++++++++++++++ .../classifier/datamodel/overview/filters.yml | 36 ++++++ .../DSL/POST/classifier/datamodel/create.yml | 111 ++++++++++++++++++ .../classifier/datasetgroup/update/status.yml | 2 +- 8 files changed, 371 insertions(+), 1 deletion(-) create mode 100644 DSL/Liquibase/changelog/classifier-script-v9-models-metadata.sql create mode 100644 DSL/Resql/get-data-model-filters.sql create mode 100644 DSL/Resql/get-paginated-data-model-metadata.sql create mode 100644 DSL/Resql/insert-model-metadata.sql create mode 100644 DSL/Ruuter.private/DSL/GET/classifier/datamodel/overview.yml create mode 100644 DSL/Ruuter.private/DSL/GET/classifier/datamodel/overview/filters.yml create mode 100644 DSL/Ruuter.private/DSL/POST/classifier/datamodel/create.yml diff --git a/DSL/Liquibase/changelog/classifier-script-v9-models-metadata.sql b/DSL/Liquibase/changelog/classifier-script-v9-models-metadata.sql new file mode 100644 index 00000000..a56747df --- /dev/null +++ b/DSL/Liquibase/changelog/classifier-script-v9-models-metadata.sql @@ -0,0 +1,35 @@ +-- liquibase formatted sql + +-- changeset kalsara Magamage:classifier-script-v9-changeset1 +CREATE TYPE Maturity_Label AS ENUM ('development', 'staging', 'production-ready'); + +-- changeset kalsara Magamage:classifier-script-v9-changeset2 +CREATE TYPE Deployment_Env AS ENUM ('outlook', 'testing', 'inival', 'undeployed'); + +-- changeset kalsara Magamage:classifier-script-v9-changeset3 +CREATE TYPE Training_Status AS ENUM ('not trained', 'training in progress', 'trained', 'retraining needed', 'untrainable'); + +-- changeset kalsara Magamage:classifier-script-v9-changeset4 +CREATE TYPE Base_Models AS ENUM ('xlnet', 'bert', 'roberta'); + +-- changeset kalsara Magamage:classifier-script-v9-changeset5 +CREATE TABLE models_metadata ( + id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, + model_group_key TEXT NOT NULL, + model_name TEXT NOT NULL, + major_version INT NOT NULL DEFAULT 0, + minor_version INT NOT NULL DEFAULT 0, + latest BOOLEAN DEFAULT false, + maturity_label Maturity_Label, + deployment_env Deployment_Env, + training_status Training_Status, + base_models Base_Models[], + last_trained_timestamp TIMESTAMP WITH TIME ZONE, + created_timestamp TIMESTAMP WITH TIME ZONE, + connected_dg_id INT, + connected_dg_name TEXT, + model_s3_location TEXT, + inference_routes JSONB, + training_results JSONB, + CONSTRAINT models_metadata_pkey PRIMARY KEY (id) +); \ No newline at end of file diff --git a/DSL/Resql/get-data-model-filters.sql b/DSL/Resql/get-data-model-filters.sql new file mode 100644 index 00000000..2d663837 --- /dev/null +++ b/DSL/Resql/get-data-model-filters.sql @@ -0,0 +1,23 @@ +SELECT json_build_object( + 'modelNames', modelNames, + 'modelVersions', modelVersions, + 'datasetGroups', datasetGroups, + 'deploymentsEnvs', deploymentsEnvs, + 'trainingStatuses', trainingStatuses, + 'maturityLabels', maturityLabels +) +FROM ( + SELECT + array_agg(DISTINCT model_name) AS modelNames, + array_agg(DISTINCT + major_version::TEXT || '.x' + ) FILTER (WHERE major_version > 0) || + array_agg(DISTINCT + 'x.' || minor_version::TEXT + ) FILTER (WHERE minor_version > 0) AS modelVersions, + array_agg(DISTINCT connected_dg_name) AS datasetGroups, + array_agg(DISTINCT deployment_env) AS deploymentsEnvs, + array_agg(DISTINCT training_status) AS trainingStatuses, + array_agg(DISTINCT maturity_label) AS maturityLabels + FROM models_metadata +) AS subquery; \ No newline at end of file diff --git a/DSL/Resql/get-paginated-data-model-metadata.sql b/DSL/Resql/get-paginated-data-model-metadata.sql new file mode 100644 index 00000000..7083ccc0 --- /dev/null +++ b/DSL/Resql/get-paginated-data-model-metadata.sql @@ -0,0 +1,34 @@ +SELECT + dt.id, + dt.model_group_key, + dt.model_name, + dt.major_version, + dt.minor_version, + dt.latest, + dt.maturity_label, + dt.deployment_env, + dt.training_status, + dt.base_models, + dt.last_trained_timestamp, + dt.created_timestamp, + dt.connected_dg_id, + dt.connected_dg_name, + dt.model_s3_location, + dt.inference_routes, + dt.training_results, + CEIL(COUNT(*) OVER() / :page_size::DECIMAL) AS total_pages +FROM + models_metadata dt +WHERE + (:major_version = -1 OR dt.major_version = :major_version) + AND (:minor_version = -1 OR dt.minor_version = :minor_version) + AND (:model_name = 'all' OR dt.model_name = :model_name) + AND (:deployment_maturity = 'all' OR dt.maturity_label = :deployment_maturity::Maturity_Label) + AND (:training_status = 'all' OR dt.training_status = :training_status::Training_Status) + AND (:platform = 'all' OR dt.deployment_env = :platform::Deployment_Env) + AND (:dataset_group = 'all' OR dt.connected_dg_name = :dataset_group) +ORDER BY + CASE WHEN :sort_type = 'asc' THEN dt.model_name END ASC, + CASE WHEN :sort_type = 'desc' THEN dt.model_name END DESC +OFFSET ((GREATEST(:page, 1) - 1) * :page_size) LIMIT :page_size; + diff --git a/DSL/Resql/insert-model-metadata.sql b/DSL/Resql/insert-model-metadata.sql new file mode 100644 index 00000000..93e66bb6 --- /dev/null +++ b/DSL/Resql/insert-model-metadata.sql @@ -0,0 +1,27 @@ +INSERT INTO models_metadata ( + model_group_key, + model_name, + major_version, + minor_version, + latest, + maturity_label, + deployment_env, + training_status, + base_models, + created_timestamp, + connected_dg_id, + connected_dg_name +) VALUES ( + :model_group_key, + :model_name, + :major_version, + :minor_version, + :latest, + :maturity_label::Maturity_Label, + :deployment_env::Deployment_Env, + :training_status::Training_Status, + ARRAY [:base_models]::Base_Models[], + :created_timestamp::timestamp with time zone, + :connected_dg_id, + :connected_dg_name +) RETURNING id; diff --git a/DSL/Ruuter.private/DSL/GET/classifier/datamodel/overview.yml b/DSL/Ruuter.private/DSL/GET/classifier/datamodel/overview.yml new file mode 100644 index 00000000..d529ec7b --- /dev/null +++ b/DSL/Ruuter.private/DSL/GET/classifier/datamodel/overview.yml @@ -0,0 +1,104 @@ +declaration: + call: declare + version: 0.1 + description: "Description placeholder for 'OVERVIEW'" + method: get + accepts: json + returns: json + namespace: classifier + allowlist: + params: + - field: page + type: number + description: "Parameter 'page'" + - field: pageSize + type: number + description: "Parameter 'pageSize'" + - field: sortType + type: string + description: "Parameter 'sortType'" + - field: modelName + type: string + description: "Parameter 'modelName'" + - field: majorVersion + type: string + description: "Parameter 'majorVersion'" + - field: minorVersion + type: string + description: "Parameter 'minorVersion'" + - field: platform + type: string + description: "Parameter 'platform'" + - field: datasetGroup + type: string + description: "Parameter 'datasetGroup'" + - field: trainingStatus + type: string + description: "Parameter 'trainingStatus'" + - field: deploymentMaturity + type: string + description: "Parameter 'deploymentMaturity'" + +extract_data: + assign: + page: ${Number(incoming.params.page)} + page_size: ${Number(incoming.params.pageSize)} + sort_type: ${incoming.params.sortType} + model_name: ${incoming.params.modelName} + major_version: ${Number(incoming.params.majorVersion)} + minor_version: ${Number(incoming.params.minorVersion)} + platform: ${incoming.params.platform} + dataset_group: ${incoming.params.datasetGroup} + training_status: ${incoming.params.trainingStatus} + deployment_maturity: ${incoming.params.deploymentMaturity} + next: get_dataset_meta_data_overview + +get_data_model_meta_data_overview: + call: http.post + args: + url: "[#CLASSIFIER_RESQL]/get-paginated-data-model-metadata" + body: + page: ${page} + page_size: ${page_size} + sorting: ${sort_type} + model_name: ${model_name} + major_version: ${major_version} + minor_version: ${minor_version} + platform: ${platform} + dataset_group: ${dataset_group} + training_status: ${training_status} + deployment_maturity: ${deployment_maturity} + result: res_model + next: check_status + +check_status: + switch: + - condition: ${200 <= res_model.response.statusCodeValue && res_model.response.statusCodeValue < 300} + next: assign_success_response + next: assign_fail_response + +assign_success_response: + assign: + format_res: { + operationSuccessful: true, + data: '${res_model.response.body}' + } + next: return_ok + +assign_fail_response: + assign: + format_res: { + operationSuccessful: false, + data: '${[]}' + } + next: return_bad_request + +return_ok: + status: 200 + return: ${format_res} + next: end + +return_bad_request: + status: 400 + return: ${format_res} + next: end \ No newline at end of file diff --git a/DSL/Ruuter.private/DSL/GET/classifier/datamodel/overview/filters.yml b/DSL/Ruuter.private/DSL/GET/classifier/datamodel/overview/filters.yml new file mode 100644 index 00000000..278e6bf2 --- /dev/null +++ b/DSL/Ruuter.private/DSL/GET/classifier/datamodel/overview/filters.yml @@ -0,0 +1,36 @@ +declaration: + call: declare + version: 0.1 + description: "Description placeholder for 'FILTERS'" + method: get + accepts: json + returns: json + namespace: classifier + +get_data_model_filters: + call: http.post + args: + url: "[#CLASSIFIER_RESQL]/get-data-model-filters" + result: res_filters + next: check_status + +check_status: + switch: + - condition: ${200 <= res_filters.response.statusCodeValue && res_filters.response.statusCodeValue < 300} + next: assign_Json_format + next: return_bad_request + +assign_Json_format: + assign: + data: ${JSON.parse(res_filters.response.body[0].jsonBuildObject.value)} + next: return_ok + +return_ok: + status: 200 + return: ${data} + next: end + +return_bad_request: + status: 400 + return: "Bad Request" + next: end \ No newline at end of file diff --git a/DSL/Ruuter.private/DSL/POST/classifier/datamodel/create.yml b/DSL/Ruuter.private/DSL/POST/classifier/datamodel/create.yml new file mode 100644 index 00000000..4df5e8fa --- /dev/null +++ b/DSL/Ruuter.private/DSL/POST/classifier/datamodel/create.yml @@ -0,0 +1,111 @@ +declaration: + call: declare + version: 0.1 + description: "Insert model metadata" + method: post + accepts: json + returns: json + namespace: classifier + allowlist: + body: + - field: modelName + type: string + description: "Body field 'modelName'" + - field: datasetGroupName + type: string + description: "Body field 'datasetGroupName'" + - field: dgId + type: number + description: "Body field 'dgId'" + - field: baseModels + type: array + description: "Body field 'baseModels'" + - field: deploymentPlatform + type: string + description: "Body field 'deploymentPlatform'" + - field: maturityLabel + type: string + description: "Body field 'maturityLabel'" + headers: + - field: cookie + type: string + description: "Cookie field" + +extract_request_data: + assign: + model_name: ${incoming.body.modelName} + dataset_group_name: ${incoming.body.datasetGroupName} + dg_id: ${incoming.body.dgId} + base_models: ${incoming.body.baseModels} + deployment_platform: ${incoming.body.deploymentPlatform} + maturity_label: ${incoming.body.maturityLabel} + next: check_for_request_data + +check_for_request_data: + switch: + - condition: ${model_name !== null && dataset_group_name !== null && dg_id !== null && base_models !== null && deployment_platform !== null && maturity_label !== null} + next: get_epoch_date + next: return_incorrect_request + +get_epoch_date: + assign: + current_epoch: ${Date.now()} + random_num: ${Math.floor(Math.random() * 100000)} + next: create_model_metadata + +create_model_metadata: + call: http.post + args: + url: "[#CLASSIFIER_RESQL]/insert-model-metadata" + body: + model_name: ${model_name} + model_group_key: "${random_num+ '_'+current_epoch}" + connected_dg_name: ${dataset_group_name} + connected_dg_id: ${dg_id} + base_models: ${base_models} + deployment_env: ${deployment_platform} + maturity_label: ${maturity_label} + training_status: not trained + major_version: 1 + minor_version: 0 + latest: true + created_timestamp: ${new Date(current_epoch).toISOString()} + result: res_model + next: check_status + +check_status: + switch: + - condition: ${200 <= res_model.response.statusCodeValue && res_model.response.statusCodeValue < 300} + next: assign_success_response + next: assign_fail_response + +assign_success_response: + assign: + format_res: { + modelId: '${res_model.response.body[0].id}', + operationSuccessful: true + } + next: return_ok + +assign_fail_response: + assign: + format_res: { + modelId: '', + operationSuccessful: false + } + next: return_bad_request + +return_ok: + status: 200 + return: ${format_res} + next: end + +return_bad_request: + status: 400 + return: ${format_res} + next: end + +return_incorrect_request: + status: 400 + return: 'Missing Required Fields' + next: end diff --git a/DSL/Ruuter.private/DSL/POST/classifier/datasetgroup/update/status.yml b/DSL/Ruuter.private/DSL/POST/classifier/datasetgroup/update/status.yml index 5c9a95ed..f487e49d 100644 --- a/DSL/Ruuter.private/DSL/POST/classifier/datasetgroup/update/status.yml +++ b/DSL/Ruuter.private/DSL/POST/classifier/datasetgroup/update/status.yml @@ -9,7 +9,7 @@ declaration: allowlist: body: - field: dgId - type: string + type: number description: "Body field 'dgId'" - field: operationType type: string From 9b8839342f9499fa08a4bd07955e9d1143350582 Mon Sep 17 00:00:00 2001 From: "kalsara.magamage_roo" Date: Thu, 1 Aug 2024 17:16:56 +0530 Subject: [PATCH 2/3] ESCLASS-159- model implementation API's and validation status issue fixed --- .../classifier-script-v9-models-metadata.sql | 23 +++++- DSL/Resql/get-data-model-metadata-by-id.sql | 14 ++++ DSL/Resql/get-data-model-options.sql | 1 + .../get-paginated-data-model-metadata.sql | 4 +- .../get-validated-all-dataset-groups.sql | 3 + .../update-patch-version-dataset-group.sql | 6 +- .../classifier/datamodel/create/options.yml | 71 +++++++++++++++++++ .../DSL/GET/classifier/datamodel/metadata.yml | 65 +++++++++++++++++ .../DSL/GET/classifier/datamodel/overview.yml | 3 +- .../classifier/datasetgroup/update/patch.yml | 5 +- .../datasetgroup/update/validation/status.yml | 36 ++++++++-- 11 files changed, 212 insertions(+), 19 deletions(-) create mode 100644 DSL/Resql/get-data-model-metadata-by-id.sql create mode 100644 DSL/Resql/get-data-model-options.sql create mode 100644 DSL/Resql/get-validated-all-dataset-groups.sql create mode 100644 DSL/Ruuter.private/DSL/GET/classifier/datamodel/create/options.yml create mode 100644 DSL/Ruuter.private/DSL/GET/classifier/datamodel/metadata.yml diff --git a/DSL/Liquibase/changelog/classifier-script-v9-models-metadata.sql b/DSL/Liquibase/changelog/classifier-script-v9-models-metadata.sql index a56747df..e3a04054 100644 --- a/DSL/Liquibase/changelog/classifier-script-v9-models-metadata.sql +++ b/DSL/Liquibase/changelog/classifier-script-v9-models-metadata.sql @@ -1,16 +1,16 @@ -- liquibase formatted sql -- changeset kalsara Magamage:classifier-script-v9-changeset1 -CREATE TYPE Maturity_Label AS ENUM ('development', 'staging', 'production-ready'); +CREATE TYPE Maturity_Label AS ENUM ('development', 'staging', 'production'); -- changeset kalsara Magamage:classifier-script-v9-changeset2 -CREATE TYPE Deployment_Env AS ENUM ('outlook', 'testing', 'inival', 'undeployed'); +CREATE TYPE Deployment_Env AS ENUM ('jira', 'outlook', 'pinal', 'testing', 'undeployed'); -- changeset kalsara Magamage:classifier-script-v9-changeset3 CREATE TYPE Training_Status AS ENUM ('not trained', 'training in progress', 'trained', 'retraining needed', 'untrainable'); -- changeset kalsara Magamage:classifier-script-v9-changeset4 -CREATE TYPE Base_Models AS ENUM ('xlnet', 'bert', 'roberta'); +CREATE TYPE Base_Models AS ENUM ('xlnet', 'roberta', 'albert'); -- changeset kalsara Magamage:classifier-script-v9-changeset5 CREATE TABLE models_metadata ( @@ -32,4 +32,21 @@ CREATE TABLE models_metadata ( inference_routes JSONB, training_results JSONB, CONSTRAINT models_metadata_pkey PRIMARY KEY (id) +); + +-- changeset kalsara Magamage:classifier-script-v9-changeset6 +CREATE TABLE model_configurations ( + id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, + base_models Base_Models[], + deployment_platforms Deployment_Env[], + maturity_labels Maturity_Label[], + CONSTRAINT model_configurations_pkey PRIMARY KEY (id) +); + +-- changeset kalsara Magamage:classifier-script-v9-changeset7 +INSERT INTO model_configurations (base_models, deployment_platforms, maturity_labels) VALUES +( + ARRAY['xlnet', 'roberta', 'albert']::Base_Models[], + ARRAY['jira', 'outlook', 'pinal', 'testing', 'undeployed']::Deployment_Env[], + ARRAY['development', 'staging', 'production']::Maturity_Label[] ); \ No newline at end of file diff --git a/DSL/Resql/get-data-model-metadata-by-id.sql b/DSL/Resql/get-data-model-metadata-by-id.sql new file mode 100644 index 00000000..2f88d651 --- /dev/null +++ b/DSL/Resql/get-data-model-metadata-by-id.sql @@ -0,0 +1,14 @@ +SELECT + id AS model_id, + model_name, + major_version, + minor_version, + latest, + maturity_label, + deployment_env, + training_status, + base_models, + connected_dg_id, + connected_dg_name +FROM models_metadata +WHERE id = :id; diff --git a/DSL/Resql/get-data-model-options.sql b/DSL/Resql/get-data-model-options.sql new file mode 100644 index 00000000..682c4629 --- /dev/null +++ b/DSL/Resql/get-data-model-options.sql @@ -0,0 +1 @@ +SELECT * from model_configurations; \ No newline at end of file diff --git a/DSL/Resql/get-paginated-data-model-metadata.sql b/DSL/Resql/get-paginated-data-model-metadata.sql index 7083ccc0..13d30b12 100644 --- a/DSL/Resql/get-paginated-data-model-metadata.sql +++ b/DSL/Resql/get-paginated-data-model-metadata.sql @@ -13,9 +13,7 @@ SELECT dt.created_timestamp, dt.connected_dg_id, dt.connected_dg_name, - dt.model_s3_location, - dt.inference_routes, - dt.training_results, + jsonb_pretty(dt.training_results) AS training_results, CEIL(COUNT(*) OVER() / :page_size::DECIMAL) AS total_pages FROM models_metadata dt diff --git a/DSL/Resql/get-validated-all-dataset-groups.sql b/DSL/Resql/get-validated-all-dataset-groups.sql new file mode 100644 index 00000000..f5f5854b --- /dev/null +++ b/DSL/Resql/get-validated-all-dataset-groups.sql @@ -0,0 +1,3 @@ +SELECT id as dg_id, group_name, major_version, minor_version, patch_version +FROM dataset_group_metadata +WHERE is_enabled = TRUE AND validation_status = 'success'; diff --git a/DSL/Resql/update-patch-version-dataset-group.sql b/DSL/Resql/update-patch-version-dataset-group.sql index 2c589db8..9d5f4784 100644 --- a/DSL/Resql/update-patch-version-dataset-group.sql +++ b/DSL/Resql/update-patch-version-dataset-group.sql @@ -7,11 +7,7 @@ WITH update_latest AS ( update_specific AS ( UPDATE dataset_group_metadata SET - patch_version = ( - SELECT COALESCE(MAX(patch_version), 0) + 1 - FROM dataset_group_metadata - WHERE group_key = :group_key AND major_version = :major_version AND minor_version = :minor_version - ), + patch_version = patch_version + 1, enable_allowed = false, validation_status = 'in-progress'::Validation_Status, is_enabled = false, diff --git a/DSL/Ruuter.private/DSL/GET/classifier/datamodel/create/options.yml b/DSL/Ruuter.private/DSL/GET/classifier/datamodel/create/options.yml new file mode 100644 index 00000000..60d23b8e --- /dev/null +++ b/DSL/Ruuter.private/DSL/GET/classifier/datamodel/create/options.yml @@ -0,0 +1,71 @@ +declaration: + call: declare + version: 0.1 + description: "Description placeholder for 'OPTIONS'" + method: get + accepts: json + returns: json + namespace: classifier + +get_data_model_options: + call: http.post + args: + url: "[#CLASSIFIER_RESQL]/get-data-model-options" + result: res_options + next: check_status + +check_status: + switch: + - condition: ${200 <= res_options.response.statusCodeValue && res_options.response.statusCodeValue < 300} + next: get_dataset_group_data + next: return_bad_request + +get_dataset_group_data: + call: http.post + args: + url: "[#CLASSIFIER_RESQL]/get-validated-all-dataset-groups" + result: res_dataset + next: check_dataset_group_status + +check_dataset_group_status: + switch: + - condition: ${200 <= res_dataset.response.statusCodeValue && res_dataset.response.statusCodeValue < 300} + next: check_data_exist + next: return_bad_request + +check_data_exist: + switch: + - condition: ${res_dataset.response.body.length>0} + next: assign_dataset + next: assign_empty + +assign_dataset: + assign: + dataset_group: ${res_dataset.response.body} + next: assign_success_response + +assign_empty: + assign: + dataset_group: [] + next: assign_success_response + +assign_success_response: + assign: + format_res: { + baseModels: '${res_options.response.body[0].baseModels}', + deploymentPlatforms: '${res_options.response.body[0].deploymentPlatforms}', + maturityLabels: '${res_options.response.body[0].maturityLabels}', + datasetGroups: '${dataset_group}' + } + next: return_ok + +return_ok: + status: 200 + return: ${format_res} + next: end + +return_bad_request: + status: 400 + return: "Bad Request" + next: end + diff --git a/DSL/Ruuter.private/DSL/GET/classifier/datamodel/metadata.yml b/DSL/Ruuter.private/DSL/GET/classifier/datamodel/metadata.yml new file mode 100644 index 00000000..30780d05 --- /dev/null +++ b/DSL/Ruuter.private/DSL/GET/classifier/datamodel/metadata.yml @@ -0,0 +1,65 @@ +declaration: + call: declare + version: 0.1 + description: "Description placeholder for 'METADATA'" + method: get + accepts: json + returns: json + namespace: classifier + allowlist: + params: + - field: modelId + type: number + description: "Parameter 'modelId'" + +extract_data: + assign: + model_id: ${Number(incoming.params.modelId)} + next: get_data_model_meta_data_by_id + +get_data_model_meta_data_by_id: + call: http.post + args: + url: "[#CLASSIFIER_RESQL]/get-data-model-metadata-by-id" + body: + id: ${model_id} + result: res_model + next: check_status + +check_status: + switch: + - condition: ${200 <= res_model.response.statusCodeValue && res_model.response.statusCodeValue < 300} + next: check_data_exist + next: assign_fail_response + +check_data_exist: + switch: + - condition: ${res_model.response.body.length>0} + next: assign_success_response + next: assign_fail_response + +assign_success_response: + assign: + format_res: { + operationSuccessful: true, + data: '${res_model.response.body}' + } + next: return_ok + +assign_fail_response: + assign: + format_res: { + operationSuccessful: false, + data: '${[]}' + } + next: return_bad_request + +return_ok: + status: 200 + return: ${format_res} + next: end + +return_bad_request: + status: 400 + return: ${format_res} + next: end \ No newline at end of file diff --git a/DSL/Ruuter.private/DSL/GET/classifier/datamodel/overview.yml b/DSL/Ruuter.private/DSL/GET/classifier/datamodel/overview.yml index d529ec7b..e6acc0a0 100644 --- a/DSL/Ruuter.private/DSL/GET/classifier/datamodel/overview.yml +++ b/DSL/Ruuter.private/DSL/GET/classifier/datamodel/overview.yml @@ -51,7 +51,7 @@ extract_data: dataset_group: ${incoming.params.datasetGroup} training_status: ${incoming.params.trainingStatus} deployment_maturity: ${incoming.params.deploymentMaturity} - next: get_dataset_meta_data_overview + next: get_data_model_meta_data_overview get_data_model_meta_data_overview: call: http.post @@ -68,6 +68,7 @@ get_data_model_meta_data_overview: dataset_group: ${dataset_group} training_status: ${training_status} deployment_maturity: ${deployment_maturity} + sort_type: ${sort_type} result: res_model next: check_status diff --git a/DSL/Ruuter.private/DSL/POST/classifier/datasetgroup/update/patch.yml b/DSL/Ruuter.private/DSL/POST/classifier/datasetgroup/update/patch.yml index 319caa2f..c7ac0b5d 100644 --- a/DSL/Ruuter.private/DSL/POST/classifier/datasetgroup/update/patch.yml +++ b/DSL/Ruuter.private/DSL/POST/classifier/datasetgroup/update/patch.yml @@ -60,10 +60,8 @@ update_old_dataset_group: url: "[#CLASSIFIER_RESQL]/update-patch-version-dataset-group" body: id: ${dg_id} - group_key: ${group_key} - major_version: ${major_version} - minor_version: ${minor_version} last_updated_timestamp: ${new Date().toISOString()} + group_key: ${group_key} result: res next: check_old_dataset_status @@ -80,6 +78,7 @@ execute_cron_manager: query: cookie: ${incoming.headers.cookie} dgId: ${dg_id} + newDgId: 0 updateType: 'patch' savedFilePath: 'None' patchPayload: ${encodeURIComponent(JSON.stringify(update_data_payload))} diff --git a/DSL/Ruuter.private/DSL/POST/classifier/datasetgroup/update/validation/status.yml b/DSL/Ruuter.private/DSL/POST/classifier/datasetgroup/update/validation/status.yml index 8fc9a70e..e594350d 100644 --- a/DSL/Ruuter.private/DSL/POST/classifier/datasetgroup/update/validation/status.yml +++ b/DSL/Ruuter.private/DSL/POST/classifier/datasetgroup/update/validation/status.yml @@ -11,6 +11,9 @@ declaration: - field: dgId type: string description: "Body field 'dgId'" + - field: newDgId + type: string + description: "Body field 'newDgId'" - field: updateType type: string description: "Body field 'updateType'" @@ -30,11 +33,30 @@ declaration: extract_request_data: assign: dg_id: ${incoming.body.dgId} + new_dg_id: ${incoming.body.newDgId} update_type: ${incoming.body.updateType} patch_payload: ${incoming.body.patchPayload} save_file_path: ${incoming.body.savedFilePath} validation_status: ${incoming.body.validationStatus} validation_errors: ${incoming.body.validationErrors} + next: check_request_type + +check_request_type: + switch: + - condition: ${update_type == 'minor'} + next: assign_minor_type + - condition: ${update_type == 'patch'} + next: assign_patch_type + next: update_type_not_found + +assign_patch_type: + assign: + active_dg_id: ${incoming.body.dgId} + next: update_dataset_group_validation + +assign_minor_type: + assign: + active_dg_id: ${incoming.body.newDgId} next: update_dataset_group_validation update_dataset_group_validation: @@ -42,7 +64,7 @@ update_dataset_group_validation: args: url: "[#CLASSIFIER_RESQL]/update-dataset-group-validation-data" body: - id: ${dg_id} + id: ${active_dg_id} validation_status: ${validation_status} validation_errors: ${JSON.stringify(validation_errors)} result: res @@ -67,6 +89,7 @@ execute_cron_manager: query: cookie: ${incoming.headers.cookie} dgId: ${dg_id} + newDgId: ${new_dg_id} updateType: ${update_type} savedFilePath: ${save_file_path} patchPayload: ${patch_payload} @@ -76,7 +99,7 @@ execute_cron_manager: assign_success_response: assign: format_res: { - dgId: '${dg_id}', + dgId: '${active_dg_id}', operationSuccessful: true, } next: return_ok @@ -84,7 +107,7 @@ assign_success_response: assign_fail_response: assign: format_res: { - dgId: '${dg_id}', + dgId: '${active_dg_id}', operationSuccessful: false, } next: return_bad_request @@ -94,7 +117,12 @@ return_ok: return: ${format_res} next: end +update_type_not_found: + status: 400 + return: "Update type not found" + next: end + return_bad_request: status: 400 return: ${format_res} - next: end + next: end \ No newline at end of file From 1538305732dfa8b1ef2e9bd8e651b60839397aa8 Mon Sep 17 00:00:00 2001 From: "kalsara.magamage_roo" Date: Thu, 1 Aug 2024 17:48:27 +0530 Subject: [PATCH 3/3] ESCLASS-159- sonar issues fixed --- DSL/Resql/get-data-model-options.sql | 3 ++- DSL/Resql/get-validated-all-dataset-groups.sql | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/DSL/Resql/get-data-model-options.sql b/DSL/Resql/get-data-model-options.sql index 682c4629..91df4715 100644 --- a/DSL/Resql/get-data-model-options.sql +++ b/DSL/Resql/get-data-model-options.sql @@ -1 +1,2 @@ -SELECT * from model_configurations; \ No newline at end of file +SELECT base_models, deployment_platforms, maturity_labels +FROM model_configurations; \ No newline at end of file diff --git a/DSL/Resql/get-validated-all-dataset-groups.sql b/DSL/Resql/get-validated-all-dataset-groups.sql index f5f5854b..2d0c0ccc 100644 --- a/DSL/Resql/get-validated-all-dataset-groups.sql +++ b/DSL/Resql/get-validated-all-dataset-groups.sql @@ -1,3 +1,3 @@ SELECT id as dg_id, group_name, major_version, minor_version, patch_version FROM dataset_group_metadata -WHERE is_enabled = TRUE AND validation_status = 'success'; +WHERE is_enabled = true AND validation_status = 'success';