Merge pull request #52 from benchuang11046/dev.3.0.2

Dev.3.0.2

CHANGELOG.md

@@ -1,3 +1,11 @@
+3.0.2 (unreleased)
+------------------
+
+- Add Requirement #13878 - Support Coefficient parameter
+
+- Fix Bug #13864 - Fix botocore upload incomplete model, and use boto3 instead.
+
+
 3.0.1 (2020-04-27)
 ------------------
 

VERSION

@@ -1 +1 @@
-3.0.1
+3.0.2.dev0

afs/models.py

@@ -148,6 +148,7 @@ def upload_model(
         tags={},
         extra_evaluation={},
         feature_importance=None,
+        coefficient=None,
         model_repository_name=None,
         model_name=None,
         blob_mode=True,
@@ -225,8 +226,12 @@ def upload_model(
         data = dict(
             tags=json.dumps(tags), 
             evaluation_result=json.dumps(evaluation_result),
-            feature_importance=json.dumps(feature_importance),
         )
+        if feature_importance:
+            data.update({'feature_importance': json.dumps(feature_importance)})
+        if coefficient:
+            data.update({'coefficient': json.dumps(coefficient)})
+
         if self.afs_version >= '3.1.3':
             data.update(
                 {

afs/utils.py

@@ -1,8 +1,9 @@
 import json
 import requests
 import logging
-import botocore.session
+# import botocore.session
 from botocore.client import Config
+import boto3
 
 _logger = logging.getLogger(__name__)
 
@@ -43,46 +44,39 @@ def upload_file_to_blob(
     blob_endpoint, blob_accessKey, blob_secretKey, bucket_name, key, filename
 ):
     try:
-        config = Config(signature_version="s3")
-        session = botocore.session.get_session()
-        blob_client = session.create_client(
+        blob_client = boto3.client(
             "s3",
-            region_name="",
             endpoint_url=blob_endpoint,
-            aws_access_key_id=blob_accessKey,
             aws_secret_access_key=blob_secretKey,
+            aws_access_key_id=blob_accessKey,
             verify=False,
-            config=config,
+            config=Config(signature_version="s3"),
         )
     except Exception as e:
-        raise ConnectionError("Connect to blob {} error, exception: {}".foramt(blob_endpoint, e))
+        raise ConnectionError("Connect to blob {} error, exception: {}".format(blob_endpoint, e))
 
     retry = 0
     while retry < 3:
         try:
-            resp = blob_client.put_object(
-                Bucket=bucket_name, Key=key, Body=open(filename, "rb").read()
-            )
+            with open(filename, 'rb') as data:
+                blob_client.upload_fileobj(Fileobj=data, Bucket=bucket_name, Key=key)
+            # Upload uccess
+            break
         except Exception as e:
-            resp = {}
             print(
                 ConnectionError(
                     "[ConnectionError] Put object error {} time, exeception: {}".format(retry, e)
                 )
             )
-
-        if not resp or resp["ResponseMetadata"]["HTTPStatusCode"] != 200:
-            retry = retry + 1
+            retry += 1
             if retry == 3:
                 raise ConnectionError(
-                    "[ConnectionError] Put object error after retry 3 times, check response {}".format(resp)
+                    "[ConnectionError] Put object error after retry 3 times."
                 )
-        else:
-            break
 
     resp_get = blob_client.list_objects(Bucket=bucket_name, Prefix=key)
-    if not resp_get or resp["ResponseMetadata"]["HTTPStatusCode"] != 200:
-        raise ConnectionError("List blob key has some error, check response {}".format(resp))
+    if not resp_get or resp_get["ResponseMetadata"]["HTTPStatusCode"] != 200:
+        raise ConnectionError("List blob key has some error, check response {}".format(resp_get))
 
     object_size = resp_get["Contents"][0]["Size"]
     return object_size

docs/Examples.md

@@ -36,6 +36,18 @@ feature_importance = [
 	{'feature': 'sepal_width',  'importance': 0.0033974420712357825}
 ]
 
+coefficient = [
+	{'feature': 'B-0070-0068-1-FN66F_strength', 'coefficient': -4.730741400252476}, 
+	{'feature': 'B-0070-0068-1-FN66F_vendor', 'coefficient': -0.9335123601234512}, 
+	{'feature': 'B-0070-0068-1-FN66F_tensile','coefficient': 0.16411707246054036}, 
+	{'feature': 'B-0070-0068-1-FN66F_lot','coefficient': -0.08745686004816221}, 
+	{'feature': 'Machine','coefficient': 0.015048547152059243}, 
+	{'feature': 'Lot','coefficient': -0.010971975766858174}, 
+	{'feature': 'RPM','coefficient': 0.0003730247816832932}, 
+	{'feature': 'record_purpose','coefficient': 0.0}
+]
+
+
 # Model object
 afs_models = models()
 
@@ -45,15 +57,17 @@ afs_models = models()
 #   3. (optional) extra_evaluation is for other evaluations for the model, you can put them to this parameter.
 #   4. (optional) tags is the label for the model, like the time of data or the type of the algorithm.
 #   5. (optional) feature_importance is the record how the features important in the model.
+#	6. (optional) coefficient indicates the direction of the relationship between a predictor variable and the response variable.
 afs_models.upload_model(
     model_path='model.h5',
 	model_repository_name='model.h5',
 	accuracy=0.4,
 	loss=0.3, 
 	extra_evaluation=extra_evaluation, 
 	tags=tags,
-	feature_importance=feature_importance
-	)
+	feature_importance=feature_importance,
+	coefficient=coefficient,
+)
 
 # Get the latest model info 
 model_info = afs_models.get_latest_model_info(model_repository_name='model.h5')
@@ -94,7 +108,16 @@ print(model_info)
 		'feature': 'sepal_width',
 		'importance': 0.0033974421
 	}],
-	'coefficient': [],
+	'coefficient': [
+		{'feature': 'B-0070-0068-1-FN66F_strength', 'coefficient': -4.730741400252476}, 
+		{'feature': 'B-0070-0068-1-FN66F_vendor', 'coefficient': -0.9335123601234512}, 
+		{'feature': 'B-0070-0068-1-FN66F_tensile','coefficient': 0.16411707246054036}, 
+		{'feature': 'B-0070-0068-1-FN66F_lot','coefficient': -0.08745686004816221}, 
+		{'feature': 'Machine','coefficient': 0.015048547152059243}, 
+		{'feature': 'Lot','coefficient': -0.010971975766858174}, 
+		{'feature': 'RPM','coefficient': 0.0003730247816832932}, 
+		{'feature': 'record_purpose','coefficient': 0.0}
+	],
 	'size': 11,
 	'created_at': '2020-04-06T10:23:56.228000+00:00'
 }

requirements.txt

@@ -1,6 +1,7 @@
 requests
 urllib3
 botocore
+boto3
 
 # For client
 pyyaml

tests/v2/test_models.py

@@ -187,3 +187,44 @@ def test_create_model_with_datasetid_target(test_env, afs_models, clean_mr, dele
     assert "dataset_id" in resp
     assert "afs_target" in resp
     print(resp)
+
+def test_create_model_with_ft_and_cofficient(test_env, afs_models, clean_mr, delete_mr_and_model, model_file, model_repository_name):
+
+    feature_importance = [
+        {'feature': 'petal_length', 'importance': 0.9473576807512394}, 
+        {'feature': 'petal_width',  'importance': 0.038191635936882906}, 
+        {'feature': 'sepal_length', 'importance': 0.011053241240641932}, 
+        {'feature': 'sepal_width',  'importance': 0.0033974420712357825}
+    ]
+
+    coefficient = [
+        {'feature': 'B-0070-0068-1-FN66F_strength', 'coefficient': -4.730741400252476}, 
+        {'feature': 'B-0070-0068-1-FN66F_vendor', 'coefficient': -0.9335123601234512}, 
+        {'feature': 'B-0070-0068-1-FN66F_tensile','coefficient': 0.16411707246054036}, 
+        {'feature': 'B-0070-0068-1-FN66F_lot','coefficient': -0.08745686004816221}, 
+        {'feature': 'Machine','coefficient': 0.015048547152059243}, 
+        {'feature': 'Lot','coefficient': -0.010971975766858174}, 
+        {'feature': 'RPM','coefficient': 0.0003730247816832932}, 
+        {'feature': 'record_purpose','coefficient': 0.0}
+    ]
+
+    resp = afs_models.upload_model(
+        model_path="unit_test_model",
+        accuracy=1.0,
+        loss=1.0,
+        tags={"tag_key": "tag_value"},
+        extra_evaluation={"extra_loss": 1.23},
+        feature_importance=feature_importance,
+        coefficient=coefficient,
+        model_repository_name=model_repository_name,
+        model_name="test_model",
+    )
+    assert isinstance(resp, dict)
+    assert "uuid" in resp
+    assert "name" in resp
+    assert "created_at" in resp
+    assert "tags" in resp
+    assert "evaluation_result" in resp
+    assert "feature_importance" in resp
+    assert "coefficient" in resp
+    print(resp)