diff --git a/tests/mockServer/data/modelEndpoints.json b/tests/mockServer/data/modelEndpoints.json index d2df996d1..e30aa9646 100644 --- a/tests/mockServer/data/modelEndpoints.json +++ b/tests/mockServer/data/modelEndpoints.json @@ -9,12 +9,21 @@ "owner": "admin", "v3io_user": "admin" }, + "endpoint_type": 3, + "name": "RandomForestClassifier", + "created": "2024-11-03T09:18:59.079000", + "updated": "2024-11-03T09:40:58.317000", "uid": "a7c95783e6a726a1a233e581ea898ba33fa7e342" }, "spec": { - "function_uri": "fraud-demo-admin/transaction-fraud", - "model": "RandomForestClassifier:latest", + "function_name": "transaction-fraud", + "function_tag": "latest", + "function_uid": "unversioned-latest", + "function_uri": "fraud-demo-admin/transaction-fraud@a98a5f1e34f9c56c3615f5601ee02a375169cbed", + "model_name": "RandomForestClassifier", "model_class": "ClassifierModel", + "model_uid": "898baa010eb35b914558182ad487bc582defde34", + "model_tag": null, "model_uri": "store://artifacts/fraud-demo-admin/training_model#1:latest", "feature_names": [ "amount_max_2h", @@ -28,13 +37,8 @@ "label", "__index_level_0__" ], - "stream_path": "v3io:///users/pipelines/fraud-demo-admin/model-endpoints/stream", - "algorithm": null, - "monitor_configuration": null, - "active": true - }, - "status": { - "state": null, + "children": null, + "children_uids": null, "feature_stats": { "amount_max_2h": { "count": 7446, @@ -361,6 +365,11 @@ ] } }, + "monitoring_feature_set_uri": "store://feature-sets/mm-app-project-n-v1/monitoring-serving-3-RandomForestClassifier:464ab6128bea4f27b8c2d52b91e8fba6_" + }, + "status": { + "state": null, + "monitoring_mode": "enabled", "current_stats": { "amount_avg_2h": { "count": 11, @@ -635,9 +644,9 @@ }, "first_request": "2022-05-02 11:14:21.082236+00:00", "last_request": "2022-05-19 09:43:49.502045+00:00", - "accuracy": null, "error_count": null, "drift_status": "DRIFT_DETECTED", + "avg_latency": null, "drift_measures": { "amount_max_2h": { "tvd": 0.9006299904768882, @@ -670,610 +679,7 @@ "hellinger": 0.6174075801868897, "kld": 3.8755852058078455 } - }, - "metrics": null, - "features": [ - { - "name": "amount_max_2h", - "weight": -1, - "expected": { - "min": 0, - "mean": 41.994970453935, - "max": 4759.79, - "histogram": { - "buckets": [ - 0, - 237.9895, - 475.979, - 713.9685, - 951.958, - 1189.9475, - 1427.937, - 1665.9265, - 1903.916, - 2141.9055, - 2379.895, - 2617.8845, - 2855.874, - 3093.8635, - 3331.853, - 3569.8424999999997, - 3807.832, - 4045.8215, - 4283.811, - 4521.8005, - 4759.79 - ], - "counts": [ - 7352, - 57, - 19, - 6, - 2, - 2, - 0, - 1, - 3, - 1, - 1, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 1 - ] - } - }, - "actual": { - "min": 12.19, - "mean": 29.96273243391514, - "max": 42.738011354613306, - "histogram": { - "buckets": [ - 12.19, - 13.717400567730666, - 15.24480113546133, - 16.772201703191996, - 18.299602270922662, - 19.82700283865333, - 21.354403406383994, - 22.881803974114657, - 24.409204541845323, - 25.93660510957599, - 27.464005677306652, - 28.991406245037318, - 30.518806812767984, - 32.04620738049865, - 33.573607948229316, - 35.10100851595998, - 36.62840908369065, - 38.155809651421315, - 39.68321021915198, - 41.21061078688264, - 42.738011354613306 - ], - "counts": [ - 1, - 2, - 0, - 0, - 1, - 0, - 0, - 0, - 1, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 5 - ] - } - } - }, - { - "name": "amount_sum_2h", - "weight": -1, - "expected": { - "min": -147.49, - "mean": 46.912805533172175, - "max": 4770.01, - "histogram": { - "buckets": [ - -147.49, - 98.38499999999999, - 344.26, - 590.135, - 836.01, - 1081.885, - 1327.76, - 1573.635, - 1819.51, - 2065.385, - 2311.26, - 2557.135, - 2803.01, - 3048.885, - 3294.76, - 3540.635, - 3786.51, - 4032.385, - 4278.26, - 4524.135, - 4770.01 - ], - "counts": [ - 6920, - 467, - 32, - 13, - 2, - 3, - 1, - 0, - 4, - 0, - 2, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 1 - ] - } - }, - "actual": { - "min": 0, - "mean": 10.536363636363637, - "max": 30.42, - "histogram": { - "buckets": [ - 0, - 1.5210000000000001, - 3.0420000000000003, - 4.563000000000001, - 6.0840000000000005, - 7.605, - 9.126000000000001, - 10.647, - 12.168000000000001, - 13.689000000000002, - 15.21, - 16.731, - 18.252000000000002, - 19.773000000000003, - 21.294, - 22.815, - 24.336000000000002, - 25.857000000000003, - 27.378000000000004, - 28.899, - 30.42 - ], - "counts": [ - 5, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 2, - 0, - 0, - 1, - 0, - 0, - 0, - 1, - 0, - 0, - 1 - ] - } - } - }, - { - "name": "amount_count_2h", - "weight": -1, - "expected": { - "min": -1, - "mean": 1.2539618587160892, - "max": 6, - "histogram": { - "buckets": [ - -1, - -0.65, - -0.30000000000000004, - 0.04999999999999982, - 0.3999999999999999, - 0.75, - 1.0999999999999996, - 1.4499999999999997, - 1.7999999999999998, - 2.15, - 2.5, - 2.8499999999999996, - 3.1999999999999993, - 3.55, - 3.8999999999999995, - 4.25, - 4.6, - 4.949999999999999, - 5.3, - 5.6499999999999995, - 6 - ], - "counts": [ - 6, - 0, - 0, - 0, - 0, - 5860, - 0, - 0, - 1306, - 0, - 0, - 229, - 0, - 0, - 42, - 0, - 0, - 2, - 0, - 1 - ] - } - }, - "actual": { - "min": 0, - "mean": 0.5454545454545454, - "max": 1, - "histogram": { - "buckets": [ - 0, - 0.05, - 0.1, - 0.15000000000000002, - 0.2, - 0.25, - 0.30000000000000004, - 0.35000000000000003, - 0.4, - 0.45, - 0.5, - 0.55, - 0.6000000000000001, - 0.65, - 0.7000000000000001, - 0.75, - 0.8, - 0.8500000000000001, - 0.9, - 0.9500000000000001, - 1 - ], - "counts": [ - 5, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 6 - ] - } - } - }, - { - "name": "amount_avg_2h", - "weight": -1, - "expected": { - "min": -107.44000000000001, - "mean": 38.96664405945027, - "max": 6405.95, - "histogram": { - "buckets": [ - -107.44000000000001, - 218.22949999999997, - 543.8989999999999, - 869.5684999999999, - 1195.2379999999998, - 1520.9074999999998, - 1846.5769999999998, - 2172.2464999999997, - 2497.9159999999997, - 2823.5854999999997, - 3149.2549999999997, - 3474.9244999999996, - 3800.5939999999996, - 4126.2635, - 4451.933, - 4777.6025, - 5103.272, - 5428.9415, - 5754.611, - 6080.2805, - 6405.95 - ], - "counts": [ - 7366, - 49, - 19, - 4, - 0, - 0, - 3, - 1, - 0, - 1, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 2 - ] - } - }, - "actual": { - "min": 12.19, - "mean": 28.0887273020327, - "max": 38.615200064471935, - "histogram": { - "buckets": [ - 12.19, - 13.511260003223596, - 14.832520006447194, - 16.15378000967079, - 17.475040012894386, - 18.796300016117982, - 20.117560019341582, - 21.438820022565178, - 22.760080025788774, - 24.08134002901237, - 25.402600032235966, - 26.723860035459566, - 28.04512003868316, - 29.36638004190676, - 30.687640045130358, - 32.00890004835395, - 33.33016005157755, - 34.65142005480114, - 35.97268005802474, - 37.29394006124834, - 38.615200064471935 - ], - "counts": [ - 1, - 2, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 5 - ] - } - } - }, - { - "name": "amount_max_12h", - "weight": -1, - "expected": { - "min": 0, - "mean": 53.890123556271824, - "max": 4759.79, - "histogram": { - "buckets": [ - 0, - 237.9895, - 475.979, - 713.9685, - 951.958, - 1189.9475, - 1427.937, - 1665.9265, - 1903.916, - 2141.9055, - 2379.895, - 2617.8845, - 2855.874, - 3093.8635, - 3331.853, - 3569.8424999999997, - 3807.832, - 4045.8215, - 4283.811, - 4521.8005, - 4759.79 - ], - "counts": [ - 7300, - 92, - 24, - 13, - 3, - 4, - 0, - 2, - 3, - 1, - 1, - 0, - 1, - 0, - 0, - 1, - 0, - 0, - 0, - 1 - ] - } - }, - "actual": { - "min": 20.92, - "mean": 55.03636363636363, - "max": 224.11, - "histogram": { - "buckets": [ - 20.92, - 31.079500000000003, - 41.239000000000004, - 51.3985, - 61.558, - 71.7175, - 81.877, - 92.0365, - 102.196, - 112.35549999999999, - 122.515, - 132.6745, - 142.834, - 152.99349999999998, - 163.15300000000002, - 173.3125, - 183.47199999999998, - 193.63150000000002, - 203.791, - 213.95049999999998, - 224.11 - ], - "counts": [ - 3, - 4, - 2, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1 - ] - } - } - }, - { - "name": "label", - "weight": -1, - "expected": { - "min": 0, - "mean": 0.014907332796132152, - "max": 1, - "histogram": { - "buckets": [ - 0, - 0.05, - 0.1, - 0.15000000000000002, - 0.2, - 0.25, - 0.30000000000000004, - 0.35000000000000003, - 0.4, - 0.45, - 0.5, - 0.55, - 0.6000000000000001, - 0.65, - 0.7000000000000001, - 0.75, - 0.8, - 0.8500000000000001, - 0.9, - 0.9500000000000001, - 1 - ], - "counts": [ - 7335, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 111 - ] - } - }, - "actual": null - } - ], - "children": null, - "children_uids": null, - "endpoint_type": 3 + } } }, { @@ -1281,11 +687,20 @@ "metadata": { "project": "default", "labels": {}, + "endpoint_type": 3, + "name": "GradientBoostingClassifier", + "created": "2024-12-03T09:18:59.079000", + "updated": "2024-12-03T09:40:58.317000", "uid": "d6df9915e64e9ba9441a82983d9fc7f874550b31" }, "spec": { - "function_uri": "fraud-demo-admin/transaction-fraud", - "model": "GradientBoostingClassifier:latest", + "function_name": "transaction-fraud", + "function_tag": "latest", + "function_uid": "unversioned-latest", + "function_uri": "fraud-demo-admin/transaction-fraud@a98a5f1e34f9c56c3615f5601ee02a375169cbed", + "model_name": "GradientBoostingClassifier", + "model_uid": "898baa010eb35b914558182ad487bc582defde34", + "model_tag": null, "model_class": "ClassifierModel", "model_uri": "store://artifacts/fraud-demo-admin/training_model#2:latest", "feature_names": [ @@ -1300,13 +715,8 @@ "label", "__index_level_0__" ], - "stream_path": "v3io:///users/pipelines/fraud-demo-admin/model-endpoints/stream", - "algorithm": null, - "monitor_configuration": null, - "active": true - }, - "status": { - "state": null, + "children": null, + "children_uids": null, "feature_stats": { "amount_max_2h": { "count": 7446, @@ -1633,6 +1043,11 @@ ] } }, + "monitoring_feature_set_uri": "store://feature-sets/mm-app-project-n-v1/monitoring-serving-3-RandomForestClassifier:464ab6128bea4f27b8c2d52b91e8fba6_" + }, + "status": { + "state": null, + "monitoring_mode": "enabled", "current_stats": { "amount_avg_2h": { "count": 11, @@ -1907,8 +1322,8 @@ }, "first_request": "2022-05-02 11:14:21.082452+00:00", "last_request": "2022-05-19 09:43:49.502382+00:00", - "accuracy": null, "error_count": null, + "avg_latency": null, "drift_status": "DRIFT_DETECTED", "drift_measures": { "amount_max_2h": { @@ -1942,610 +1357,7 @@ "hellinger": 0.6174075801868897, "kld": 3.8755852058078455 } - }, - "metrics": null, - "features": [ - { - "name": "amount_max_2h", - "weight": -1, - "expected": { - "min": 0, - "mean": 41.994970453935, - "max": 4759.79, - "histogram": { - "buckets": [ - 0, - 237.9895, - 475.979, - 713.9685, - 951.958, - 1189.9475, - 1427.937, - 1665.9265, - 1903.916, - 2141.9055, - 2379.895, - 2617.8845, - 2855.874, - 3093.8635, - 3331.853, - 3569.8424999999997, - 3807.832, - 4045.8215, - 4283.811, - 4521.8005, - 4759.79 - ], - "counts": [ - 7352, - 57, - 19, - 6, - 2, - 2, - 0, - 1, - 3, - 1, - 1, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 1 - ] - } - }, - "actual": { - "min": 12.19, - "mean": 29.96273243391514, - "max": 42.738011354613306, - "histogram": { - "buckets": [ - 12.19, - 13.717400567730666, - 15.24480113546133, - 16.772201703191996, - 18.299602270922662, - 19.82700283865333, - 21.354403406383994, - 22.881803974114657, - 24.409204541845323, - 25.93660510957599, - 27.464005677306652, - 28.991406245037318, - 30.518806812767984, - 32.04620738049865, - 33.573607948229316, - 35.10100851595998, - 36.62840908369065, - 38.155809651421315, - 39.68321021915198, - 41.21061078688264, - 42.738011354613306 - ], - "counts": [ - 1, - 2, - 0, - 0, - 1, - 0, - 0, - 0, - 1, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 5 - ] - } - } - }, - { - "name": "amount_sum_2h", - "weight": -1, - "expected": { - "min": -147.49, - "mean": 46.912805533172175, - "max": 4770.01, - "histogram": { - "buckets": [ - -147.49, - 98.38499999999999, - 344.26, - 590.135, - 836.01, - 1081.885, - 1327.76, - 1573.635, - 1819.51, - 2065.385, - 2311.26, - 2557.135, - 2803.01, - 3048.885, - 3294.76, - 3540.635, - 3786.51, - 4032.385, - 4278.26, - 4524.135, - 4770.01 - ], - "counts": [ - 6920, - 467, - 32, - 13, - 2, - 3, - 1, - 0, - 4, - 0, - 2, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 1 - ] - } - }, - "actual": { - "min": 0, - "mean": 10.536363636363637, - "max": 30.42, - "histogram": { - "buckets": [ - 0, - 1.5210000000000001, - 3.0420000000000003, - 4.563000000000001, - 6.0840000000000005, - 7.605, - 9.126000000000001, - 10.647, - 12.168000000000001, - 13.689000000000002, - 15.21, - 16.731, - 18.252000000000002, - 19.773000000000003, - 21.294, - 22.815, - 24.336000000000002, - 25.857000000000003, - 27.378000000000004, - 28.899, - 30.42 - ], - "counts": [ - 5, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 2, - 0, - 0, - 1, - 0, - 0, - 0, - 1, - 0, - 0, - 1 - ] - } - } - }, - { - "name": "amount_count_2h", - "weight": -1, - "expected": { - "min": -1, - "mean": 1.2539618587160892, - "max": 6, - "histogram": { - "buckets": [ - -1, - -0.65, - -0.30000000000000004, - 0.04999999999999982, - 0.3999999999999999, - 0.75, - 1.0999999999999996, - 1.4499999999999997, - 1.7999999999999998, - 2.15, - 2.5, - 2.8499999999999996, - 3.1999999999999993, - 3.55, - 3.8999999999999995, - 4.25, - 4.6, - 4.949999999999999, - 5.3, - 5.6499999999999995, - 6 - ], - "counts": [ - 6, - 0, - 0, - 0, - 0, - 5860, - 0, - 0, - 1306, - 0, - 0, - 229, - 0, - 0, - 42, - 0, - 0, - 2, - 0, - 1 - ] - } - }, - "actual": { - "min": 0, - "mean": 0.5454545454545454, - "max": 1, - "histogram": { - "buckets": [ - 0, - 0.05, - 0.1, - 0.15000000000000002, - 0.2, - 0.25, - 0.30000000000000004, - 0.35000000000000003, - 0.4, - 0.45, - 0.5, - 0.55, - 0.6000000000000001, - 0.65, - 0.7000000000000001, - 0.75, - 0.8, - 0.8500000000000001, - 0.9, - 0.9500000000000001, - 1 - ], - "counts": [ - 5, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 6 - ] - } - } - }, - { - "name": "amount_avg_2h", - "weight": -1, - "expected": { - "min": -107.44000000000001, - "mean": 38.96664405945027, - "max": 6405.95, - "histogram": { - "buckets": [ - -107.44000000000001, - 218.22949999999997, - 543.8989999999999, - 869.5684999999999, - 1195.2379999999998, - 1520.9074999999998, - 1846.5769999999998, - 2172.2464999999997, - 2497.9159999999997, - 2823.5854999999997, - 3149.2549999999997, - 3474.9244999999996, - 3800.5939999999996, - 4126.2635, - 4451.933, - 4777.6025, - 5103.272, - 5428.9415, - 5754.611, - 6080.2805, - 6405.95 - ], - "counts": [ - 7366, - 49, - 19, - 4, - 0, - 0, - 3, - 1, - 0, - 1, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 2 - ] - } - }, - "actual": { - "min": 12.19, - "mean": 28.0887273020327, - "max": 38.615200064471935, - "histogram": { - "buckets": [ - 12.19, - 13.511260003223596, - 14.832520006447194, - 16.15378000967079, - 17.475040012894386, - 18.796300016117982, - 20.117560019341582, - 21.438820022565178, - 22.760080025788774, - 24.08134002901237, - 25.402600032235966, - 26.723860035459566, - 28.04512003868316, - 29.36638004190676, - 30.687640045130358, - 32.00890004835395, - 33.33016005157755, - 34.65142005480114, - 35.97268005802474, - 37.29394006124834, - 38.615200064471935 - ], - "counts": [ - 1, - 2, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 5 - ] - } - } - }, - { - "name": "amount_max_12h", - "weight": -1, - "expected": { - "min": 0, - "mean": 53.890123556271824, - "max": 4759.79, - "histogram": { - "buckets": [ - 0, - 237.9895, - 475.979, - 713.9685, - 951.958, - 1189.9475, - 1427.937, - 1665.9265, - 1903.916, - 2141.9055, - 2379.895, - 2617.8845, - 2855.874, - 3093.8635, - 3331.853, - 3569.8424999999997, - 3807.832, - 4045.8215, - 4283.811, - 4521.8005, - 4759.79 - ], - "counts": [ - 7300, - 92, - 24, - 13, - 3, - 4, - 0, - 2, - 3, - 1, - 1, - 0, - 1, - 0, - 0, - 1, - 0, - 0, - 0, - 1 - ] - } - }, - "actual": { - "min": 20.92, - "mean": 55.03636363636363, - "max": 224.11, - "histogram": { - "buckets": [ - 20.92, - 31.079500000000003, - 41.239000000000004, - 51.3985, - 61.558, - 71.7175, - 81.877, - 92.0365, - 102.196, - 112.35549999999999, - 122.515, - 132.6745, - 142.834, - 152.99349999999998, - 163.15300000000002, - 173.3125, - 183.47199999999998, - 193.63150000000002, - 203.791, - 213.95049999999998, - 224.11 - ], - "counts": [ - 3, - 4, - 2, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1 - ] - } - } - }, - { - "name": "label", - "weight": -1, - "expected": { - "min": 0, - "mean": 0.014907332796132152, - "max": 1, - "histogram": { - "buckets": [ - 0, - 0.05, - 0.1, - 0.15000000000000002, - 0.2, - 0.25, - 0.30000000000000004, - 0.35000000000000003, - 0.4, - 0.45, - 0.5, - 0.55, - 0.6000000000000001, - 0.65, - 0.7000000000000001, - 0.75, - 0.8, - 0.8500000000000001, - 0.9, - 0.9500000000000001, - 1 - ], - "counts": [ - 7335, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 111 - ] - } - }, - "actual": null - } - ], - "children": null, - "children_uids": null, - "endpoint_type": 3 + } } } ] diff --git a/tests/mockServer/mock.js b/tests/mockServer/mock.js index a9efbfc51..f09ab5121 100644 --- a/tests/mockServer/mock.js +++ b/tests/mockServer/mock.js @@ -2259,6 +2259,21 @@ function getModelEndpoints(req, res) { features: null } })) + + if (req.query['name'] && req.query['function_name']) { + collectedEndpoints = collectedEndpoints.filter( + endpoint => + endpoint.metadata.name === req.query['name'] && + endpoint.spec.function_name === req.query['function_name'] + ) + } + + if (req.query['latest_only']) { + collectedEndpoints = collectedEndpoints.filter( + endpoint => endpoint.spec.function_tag === 'latest' + ) + } + if (req.query['label']) { collectedEndpoints = collectedEndpoints.filter(endpoint => filterByLabels(endpoint.metadata.labels, req.query['label']) @@ -2268,14 +2283,6 @@ function getModelEndpoints(req, res) { res.send({ endpoints: collectedEndpoints }) } -function getModelEndpoint(req, res) { - const endpoint = modelEndpoints.endpoints.find( - item => item.metadata.project === req.params.project && item.metadata.uid === req.params.uid - ) - - res.send(endpoint) -} - function getMetrics(req, res) { let metricsOptions = metricsData.metrics.find( @@ -2706,7 +2713,6 @@ app.get(`${mlrunAPIIngress}/log/:project/:uid`, getLog) app.get(`${mlrunAPIIngress}/projects/:project/runtime-resources`, getRuntimeResources) app.get(`${mlrunAPIIngress}/projects/:project/model-endpoints`, getModelEndpoints) -app.get(`${mlrunAPIIngress}/projects/:project/model-endpoints/:uid`, getModelEndpoint) app.get(`${mlrunAPIIngress}/projects/:project/model-endpoints/:uid/metrics`, getMetrics) app.get( `${mlrunAPIIngress}/projects/:project/model-endpoints/:uid/metrics-values`,