From 4bfeeb27ce67d3f523004577bd0fd864eb58b943 Mon Sep 17 00:00:00 2001 From: benchuang11046 Date: Thu, 14 May 2020 15:18:20 +0800 Subject: [PATCH 1/4] Back to development: 3.0.2 --- CHANGELOG.md | 6 ++++++ VERSION | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7ab39d..28716cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +3.0.2 (unreleased) +------------------ + +- Nothing changed yet. + + 3.0.1 (2020-04-27) ------------------ diff --git a/VERSION b/VERSION index cb2b00e..f189878 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.1 +3.0.2.dev0 From b2fa20d8ecf38ee05756b4f7457cf8e6d028e5a3 Mon Sep 17 00:00:00 2001 From: benchuang11046 Date: Thu, 14 May 2020 16:56:47 +0800 Subject: [PATCH 2/4] Fix boto3 to botocore upload model --- afs/utils.py | 58 +++++++++++++++++++++++++++++++----------------- requirements.txt | 1 + 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/afs/utils.py b/afs/utils.py index f117b6e..3e2bd70 100644 --- a/afs/utils.py +++ b/afs/utils.py @@ -1,8 +1,9 @@ import json import requests import logging -import botocore.session +# import botocore.session from botocore.client import Config +import boto3 _logger = logging.getLogger(__name__) @@ -43,46 +44,63 @@ def upload_file_to_blob( blob_endpoint, blob_accessKey, blob_secretKey, bucket_name, key, filename ): try: - config = Config(signature_version="s3") - session = botocore.session.get_session() - blob_client = session.create_client( + # config = Config(signature_version="s3") + # session = botocore.session.get_session() + # blob_client = session.create_client( + # "s3", + # region_name="", + # endpoint_url=blob_endpoint, + # aws_access_key_id=blob_accessKey, + # aws_secret_access_key=blob_secretKey, + # verify=False, + # config=config, + # ) + + blob_client = boto3.client( "s3", - region_name="", endpoint_url=blob_endpoint, - aws_access_key_id=blob_accessKey, aws_secret_access_key=blob_secretKey, + aws_access_key_id=blob_accessKey, verify=False, - config=config, + config=Config(signature_version="s3"), ) except Exception as e: - raise ConnectionError("Connect to blob {} error, exception: {}".foramt(blob_endpoint, e)) + raise ConnectionError("Connect to blob {} error, exception: {}".format(blob_endpoint, e)) retry = 0 while retry < 3: try: - resp = blob_client.put_object( - Bucket=bucket_name, Key=key, Body=open(filename, "rb").read() - ) + # resp = blob_client.put_object( + # Bucket=bucket_name, Key=key, Body=open(filename, "rb").read() + # ) + with open(filename, 'rb') as data: + blob_client.upload_fileobj(Fileobj=data, Bucket=bucket_name, Key=key) + except Exception as e: - resp = {} print( ConnectionError( "[ConnectionError] Put object error {} time, exeception: {}".format(retry, e) ) ) - - if not resp or resp["ResponseMetadata"]["HTTPStatusCode"] != 200: - retry = retry + 1 + retry += 1 if retry == 3: raise ConnectionError( - "[ConnectionError] Put object error after retry 3 times, check response {}".format(resp) + "[ConnectionError] Put object error after retry 3 times." ) - else: - break + + # if not resp or resp["ResponseMetadata"]["HTTPStatusCode"] != 200: + # if resp is None: + # retry += 1 + # if retry == 3: + # raise ConnectionError( + # "[ConnectionError] Put object error after retry 3 times, check response {}".format(resp) + # ) + # else: + # break resp_get = blob_client.list_objects(Bucket=bucket_name, Prefix=key) - if not resp_get or resp["ResponseMetadata"]["HTTPStatusCode"] != 200: - raise ConnectionError("List blob key has some error, check response {}".format(resp)) + if not resp_get or resp_get["ResponseMetadata"]["HTTPStatusCode"] != 200: + raise ConnectionError("List blob key has some error, check response {}".format(resp_get)) object_size = resp_get["Contents"][0]["Size"] return object_size diff --git a/requirements.txt b/requirements.txt index 8d0deb8..a80d1e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ requests urllib3 botocore +boto3 # For client pyyaml From bfca8a0cf1341e4385e40ebd9e15a6119014575e Mon Sep 17 00:00:00 2001 From: benchuang11046 Date: Thu, 14 May 2020 17:05:37 +0800 Subject: [PATCH 3/4] Fix upload success in while loop --- afs/utils.py | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/afs/utils.py b/afs/utils.py index 3e2bd70..308c178 100644 --- a/afs/utils.py +++ b/afs/utils.py @@ -44,18 +44,6 @@ def upload_file_to_blob( blob_endpoint, blob_accessKey, blob_secretKey, bucket_name, key, filename ): try: - # config = Config(signature_version="s3") - # session = botocore.session.get_session() - # blob_client = session.create_client( - # "s3", - # region_name="", - # endpoint_url=blob_endpoint, - # aws_access_key_id=blob_accessKey, - # aws_secret_access_key=blob_secretKey, - # verify=False, - # config=config, - # ) - blob_client = boto3.client( "s3", endpoint_url=blob_endpoint, @@ -70,12 +58,10 @@ def upload_file_to_blob( retry = 0 while retry < 3: try: - # resp = blob_client.put_object( - # Bucket=bucket_name, Key=key, Body=open(filename, "rb").read() - # ) with open(filename, 'rb') as data: blob_client.upload_fileobj(Fileobj=data, Bucket=bucket_name, Key=key) - + # Upload uccess + break except Exception as e: print( ConnectionError( @@ -88,16 +74,6 @@ def upload_file_to_blob( "[ConnectionError] Put object error after retry 3 times." ) - # if not resp or resp["ResponseMetadata"]["HTTPStatusCode"] != 200: - # if resp is None: - # retry += 1 - # if retry == 3: - # raise ConnectionError( - # "[ConnectionError] Put object error after retry 3 times, check response {}".format(resp) - # ) - # else: - # break - resp_get = blob_client.list_objects(Bucket=bucket_name, Prefix=key) if not resp_get or resp_get["ResponseMetadata"]["HTTPStatusCode"] != 200: raise ConnectionError("List blob key has some error, check response {}".format(resp_get)) From b64839146abf8ed7409b002ea244d5080a3f9222 Mon Sep 17 00:00:00 2001 From: benchuang11046 Date: Thu, 14 May 2020 18:15:13 +0800 Subject: [PATCH 4/4] Add Requirement #13878 - Support Coefficient parameter --- CHANGELOG.md | 4 +++- afs/models.py | 7 ++++++- docs/Examples.md | 29 ++++++++++++++++++++++++++--- tests/v2/test_models.py | 41 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28716cd..6691e37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ 3.0.2 (unreleased) ------------------ -- Nothing changed yet. +- Add Requirement #13878 - Support Coefficient parameter + +- Fix Bug #13864 - Fix botocore upload incomplete model, and use boto3 instead. 3.0.1 (2020-04-27) diff --git a/afs/models.py b/afs/models.py index d77fd4b..33e21c1 100644 --- a/afs/models.py +++ b/afs/models.py @@ -148,6 +148,7 @@ def upload_model( tags={}, extra_evaluation={}, feature_importance=None, + coefficient=None, model_repository_name=None, model_name=None, blob_mode=True, @@ -225,8 +226,12 @@ def upload_model( data = dict( tags=json.dumps(tags), evaluation_result=json.dumps(evaluation_result), - feature_importance=json.dumps(feature_importance), ) + if feature_importance: + data.update({'feature_importance': json.dumps(feature_importance)}) + if coefficient: + data.update({'coefficient': json.dumps(coefficient)}) + if self.afs_version >= '3.1.3': data.update( { diff --git a/docs/Examples.md b/docs/Examples.md index cdb8ac6..b51eb0c 100644 --- a/docs/Examples.md +++ b/docs/Examples.md @@ -36,6 +36,18 @@ feature_importance = [ {'feature': 'sepal_width', 'importance': 0.0033974420712357825} ] +coefficient = [ + {'feature': 'B-0070-0068-1-FN66F_strength', 'coefficient': -4.730741400252476}, + {'feature': 'B-0070-0068-1-FN66F_vendor', 'coefficient': -0.9335123601234512}, + {'feature': 'B-0070-0068-1-FN66F_tensile','coefficient': 0.16411707246054036}, + {'feature': 'B-0070-0068-1-FN66F_lot','coefficient': -0.08745686004816221}, + {'feature': 'Machine','coefficient': 0.015048547152059243}, + {'feature': 'Lot','coefficient': -0.010971975766858174}, + {'feature': 'RPM','coefficient': 0.0003730247816832932}, + {'feature': 'record_purpose','coefficient': 0.0} +] + + # Model object afs_models = models() @@ -45,6 +57,7 @@ afs_models = models() # 3. (optional) extra_evaluation is for other evaluations for the model, you can put them to this parameter. # 4. (optional) tags is the label for the model, like the time of data or the type of the algorithm. # 5. (optional) feature_importance is the record how the features important in the model. +# 6. (optional) coefficient indicates the direction of the relationship between a predictor variable and the response variable. afs_models.upload_model( model_path='model.h5', model_repository_name='model.h5', @@ -52,8 +65,9 @@ afs_models.upload_model( loss=0.3, extra_evaluation=extra_evaluation, tags=tags, - feature_importance=feature_importance - ) + feature_importance=feature_importance, + coefficient=coefficient, +) # Get the latest model info model_info = afs_models.get_latest_model_info(model_repository_name='model.h5') @@ -94,7 +108,16 @@ print(model_info) 'feature': 'sepal_width', 'importance': 0.0033974421 }], - 'coefficient': [], + 'coefficient': [ + {'feature': 'B-0070-0068-1-FN66F_strength', 'coefficient': -4.730741400252476}, + {'feature': 'B-0070-0068-1-FN66F_vendor', 'coefficient': -0.9335123601234512}, + {'feature': 'B-0070-0068-1-FN66F_tensile','coefficient': 0.16411707246054036}, + {'feature': 'B-0070-0068-1-FN66F_lot','coefficient': -0.08745686004816221}, + {'feature': 'Machine','coefficient': 0.015048547152059243}, + {'feature': 'Lot','coefficient': -0.010971975766858174}, + {'feature': 'RPM','coefficient': 0.0003730247816832932}, + {'feature': 'record_purpose','coefficient': 0.0} + ], 'size': 11, 'created_at': '2020-04-06T10:23:56.228000+00:00' } diff --git a/tests/v2/test_models.py b/tests/v2/test_models.py index 5f6da1b..01477c1 100644 --- a/tests/v2/test_models.py +++ b/tests/v2/test_models.py @@ -187,3 +187,44 @@ def test_create_model_with_datasetid_target(test_env, afs_models, clean_mr, dele assert "dataset_id" in resp assert "afs_target" in resp print(resp) + +def test_create_model_with_ft_and_cofficient(test_env, afs_models, clean_mr, delete_mr_and_model, model_file, model_repository_name): + + feature_importance = [ + {'feature': 'petal_length', 'importance': 0.9473576807512394}, + {'feature': 'petal_width', 'importance': 0.038191635936882906}, + {'feature': 'sepal_length', 'importance': 0.011053241240641932}, + {'feature': 'sepal_width', 'importance': 0.0033974420712357825} + ] + + coefficient = [ + {'feature': 'B-0070-0068-1-FN66F_strength', 'coefficient': -4.730741400252476}, + {'feature': 'B-0070-0068-1-FN66F_vendor', 'coefficient': -0.9335123601234512}, + {'feature': 'B-0070-0068-1-FN66F_tensile','coefficient': 0.16411707246054036}, + {'feature': 'B-0070-0068-1-FN66F_lot','coefficient': -0.08745686004816221}, + {'feature': 'Machine','coefficient': 0.015048547152059243}, + {'feature': 'Lot','coefficient': -0.010971975766858174}, + {'feature': 'RPM','coefficient': 0.0003730247816832932}, + {'feature': 'record_purpose','coefficient': 0.0} + ] + + resp = afs_models.upload_model( + model_path="unit_test_model", + accuracy=1.0, + loss=1.0, + tags={"tag_key": "tag_value"}, + extra_evaluation={"extra_loss": 1.23}, + feature_importance=feature_importance, + coefficient=coefficient, + model_repository_name=model_repository_name, + model_name="test_model", + ) + assert isinstance(resp, dict) + assert "uuid" in resp + assert "name" in resp + assert "created_at" in resp + assert "tags" in resp + assert "evaluation_result" in resp + assert "feature_importance" in resp + assert "coefficient" in resp + print(resp)