MCT 1.0 release

Specifically, this commit... * updates RELEASE.md and verison.py to 1.0.0 * updates setup.py with explicitly-defined submodule paths (the `find_packages` function does not find the modoel_card_toolkit.proto submodule) * updates go/mct-release instructions * updates notebooks to work with schema v2 PiperOrigin-RevId: 388333574
tensorflow · Aug 2, 2021 · 0c6ee2b · 0c6ee2b
1 parent 101bac0
commit 0c6ee2b
Show file tree

Hide file tree

Showing 7 changed files with 71 additions and 66 deletions.
diff --git a/RELEASE.md b/RELEASE.md
@@ -1,6 +1,6 @@
 <!-- mdlint off(HEADERS_TOO_MANY_H1) -->
 
-# Current Version(Still in Development)
+# Release 1.0.0
 
 ## Major Features and Improvements
 

diff --git a/model_card_toolkit/documentation/examples/MLMD_Model_Card_Toolkit_Demo.ipynb b/model_card_toolkit/documentation/examples/MLMD_Model_Card_Toolkit_Demo.ipynb
@@ -1080,9 +1080,9 @@
       },
       "outputs": [],
       "source": [
-        "from model_card_toolkit import ModelCardToolkit\n",
+        "import model_card_toolkit as mctlib\n",
         "\n",
-        "mct = ModelCardToolkit(mlmd_store=mlmd_store, model_uri=model_uri)"
+        "mct = mctlib.ModelCardToolkit(mlmd_store=mlmd_store, model_uri=model_uri)"
       ]
     },
     {
@@ -1135,7 +1135,8 @@
         "    'about the model’s intended uses, limitations, and ethical considerations.'\n",
         ")\n",
         "model_card.model_details.owners = [\n",
-        "  {'name': 'Model Cards Team', 'contact': 'model-cards@google.com'}\n",
+        "                                   mctlib.Owner(name='Model Cards Team',\n",
+        "                                         contact='model-cards@google.com')\n",
         "]"
       ]
     },
@@ -1147,14 +1148,14 @@
       },
       "outputs": [],
       "source": [
-        "model_card.considerations.use_cases = [\n",
+        "model_card.considerations.use_cases = [mctlib.UseCase(description=\n",
         "    'This dataset that this model was trained on was originally created to '\n",
         "    'support the machine learning community in conducting empirical analysis '\n",
         "    'of ML algorithms. The Adult Data Set can be used in fairness-related '\n",
         "    'studies that compare inequalities across sex and race, based on '\n",
-        "    'people’s annual incomes.'\n",
+        "    'people’s annual incomes.')\n",
         "]\n",
-        "model_card.considerations.limitations = [\n",
+        "model_card.considerations.limitations = [mctlib.Limitation(description=\n",
         "    'This is a class-imbalanced dataset across a variety of sensitive classes.'\n",
         "    ' The ratio of male-to-female examples is about 2:1 and there are far more'\n",
         "    ' examples with the “white” attribute than every other race combined. '\n",
@@ -1166,17 +1167,16 @@
         "    'fewer female examples in the $50,000+ earner group, causing our model to '\n",
         "    'overfit these examples. To avoid this, we can try various remediation '\n",
         "    'strategies in future iterations (e.g. undersampling, hyperparameter '\n",
-        "    'tuning, etc), but we may not be able to fix all of the fairness issues.'\n",
+        "    'tuning, etc), but we may not be able to fix all of the fairness issues.')\n",
         "]\n",
-        "model_card.considerations.ethical_considerations = [{\n",
-        "    'name':\n",
-        "        'We risk expressing the viewpoint that the attributes in this dataset '\n",
-        "        'are the only ones that are predictive of someone’s income, even '\n",
-        "        'though we know this is not the case.',\n",
-        "    'mitigation_strategy':\n",
-        "        'As mentioned, some interventions may need to be performed to address '\n",
-        "        'the class imbalances in the dataset.'\n",
-        "}]"
+        "model_card.considerations.ethical_considerations = [mctlib.Risk(\n",
+        "    name= 'We risk expressing the viewpoint that the attributes in this dataset '\n",
+        "          'are the only ones that are predictive of someone’s income, even '\n",
+        "          'though we know this is not the case.',\n",
+        "    mitigation_strategy= 'As mentioned, some interventions may need to be '\n",
+        "                         'performed to address the class imbalances in the dataset.'\n",
+        "        )\n",
+        "]"
       ]
     },
     {

diff --git a/model_card_toolkit/documentation/examples/Scikit_Learn_Model_Card_Toolkit_Demo.ipynb b/model_card_toolkit/documentation/examples/Scikit_Learn_Model_Card_Toolkit_Demo.ipynb
@@ -122,7 +122,7 @@
         "from datetime import date\n",
         "from io import BytesIO\n",
         "from IPython import display\n",
-        "from model_card_toolkit import ModelCardToolkit\n",
+        "import model_card_toolkit as mctlib\n",
         "from sklearn.datasets import load_breast_cancer\n",
         "from sklearn.ensemble import GradientBoostingClassifier\n",
         "from sklearn.model_selection import train_test_split\n",
@@ -330,7 +330,7 @@
       },
       "outputs": [],
       "source": [
-        "mct = ModelCardToolkit()\n",
+        "mct = mctlib.ModelCardToolkit()\n",
         "\n",
         "model_card = mct.scaffold_assets()"
       ]
@@ -357,45 +357,46 @@
         "    'This model predicts whether breast cancer is benign or malignant based on '\n",
         "    'image measurements.')\n",
         "model_card.model_details.owners = [\n",
-        "    {'name': 'Model Cards Team', 'contact': 'model-cards@google.com'}\n",
+        "    mctlib.Owner(name= 'Model Cards Team', contact='model-cards@google.com')\n",
         "]\n",
         "model_card.model_details.references = [\n",
-        "    'https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+(Diagnostic)',\n",
-        "    'https://minds.wisconsin.edu/bitstream/handle/1793/59692/TR1131.pdf'\n",
+        "    mctlib.Reference(reference='https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+(Diagnostic)'),\n",
+        "    mctlib.Reference(reference='https://minds.wisconsin.edu/bitstream/handle/1793/59692/TR1131.pdf')\n",
         "]\n",
         "model_card.model_details.version.name = str(uuid.uuid4())\n",
         "model_card.model_details.version.date = str(date.today())\n",
         "\n",
-        "model_card.considerations.ethical_considerations = [{\n",
-        "    'name': ('Manual selection of image sections to digitize could create '\n",
+        "model_card.considerations.ethical_considerations = [mctlib.Risk(\n",
+        "    name=('Manual selection of image sections to digitize could create '\n",
         "            'selection bias'),\n",
-        "    'mitigation_strategy': 'Automate the selection process'\n",
-        "}]\n",
-        "model_card.considerations.limitations = ['Breast cancer diagnosis']\n",
-        "model_card.considerations.use_cases = ['Breast cancer diagnosis']\n",
-        "model_card.considerations.users = ['Medical professionals', 'ML researchers']\n",
+        "    mitigation_strategy='Automate the selection process'\n",
+        ")]\n",
+        "model_card.considerations.limitations = [mctlib.Limitation(description='Breast cancer diagnosis')]\n",
+        "model_card.considerations.use_cases = [mctlib.UseCase(description='Breast cancer diagnosis')]\n",
+        "model_card.considerations.users = [mctlib.User(description='Medical professionals'), User(description='ML researchers')]\n",
         "\n",
-        "\n",
-        "model_card.model_parameters.data.train.graphics.description = (\n",
+        "model_card.model_parameters.data.append(mctlib.Dataset())\n",
+        "model_card.model_parameters.data[0].graphics.description = (\n",
         "  f'{len(X_train)} rows with {len(X_train.columns)} features')\n",
-        "model_card.model_parameters.data.train.graphics.collection = [\n",
-        "    {'image': mean_radius_train},\n",
-        "    {'image': mean_texture_train}\n",
+        "model_card.model_parameters.data[0].graphics.collection = [\n",
+        "    mctlib.Graphic(image=mean_radius_train),\n",
+        "    mctlib.Graphic(image=mean_texture_train)\n",
         "]\n",
-        "model_card.model_parameters.data.eval.graphics.description = (\n",
+        "model_card.model_parameters.data.append(mctlib.Dataset())\n",
+        "model_card.model_parameters.data[1].graphics.description = (\n",
         "  f'{len(X_test)} rows with {len(X_test.columns)} features')\n",
-        "model_card.model_parameters.data.eval.graphics.collection = [\n",
-        "    {'image': mean_radius_test},\n",
-        "    {'image': mean_texture_test}\n",
+        "model_card.model_parameters.data[1].graphics.collection = [\n",
+        "    mctlib.Graphic(image=mean_radius_test),\n",
+        "    mctlib.Graphic(image=mean_texture_test)\n",
         "]\n",
         "model_card.quantitative_analysis.graphics.description = (\n",
         "  'ROC curve and confusion matrix')\n",
         "model_card.quantitative_analysis.graphics.collection = [\n",
-        "    {'image': roc_curve},\n",
-        "    {'image': confusion_matrix}\n",
+        "    mctlib.Graphic(image=roc_curve),\n",
+        "    mctlib.Graphic(image=confusion_matrix)\n",
         "]\n",
         "\n",
-        "mct.update_model_card_json(model_card)"
+        "mct.update_model_card(model_card)"
       ]
     },
     {

diff --git a/model_card_toolkit/documentation/examples/Standalone_Model_Card_Toolkit_Demo.ipynb b/model_card_toolkit/documentation/examples/Standalone_Model_Card_Toolkit_Demo.ipynb
@@ -140,7 +140,7 @@
       "source": [
         "import tensorflow as tf\n",
         "import numpy as np\n",
-        "from model_card_toolkit import ModelCardToolkit\n",
+        "import model_card_toolkit as mctlib\n",
         "from model_card_toolkit.documentation.examples import cats_vs_dogs\n",
         "from model_card_toolkit.utils.graphics import figure_to_base64str\n",
         "import tempfile\n",
@@ -335,15 +335,15 @@
         "    'performed with high accuracy on both Cat and Dog images.'\n",
         ")\n",
         "model_card.model_details.owners = [\n",
-        "  {'name': 'Model Cards Team', 'contact': 'model-cards@google.com'}\n",
+        "  mctlib.Owner(name='Model Cards Team', contact='model-cards@google.com')\n",
         "]\n",
-        "model_card.model_details.version = {'name': 'v1.0', 'date': '08/28/2020'}\n",
+        "model_card.model_details.version = mctlib.Version(name='v1.0', date='08/28/2020')\n",
         "model_card.model_details.references = [\n",
-        "    'https://www.tensorflow.org/guide/keras/transfer_learning',\n",
-        "    'https://arxiv.org/abs/1801.04381',\n",
+        "    mctlib.Reference(reference='https://www.tensorflow.org/guide/keras/transfer_learning'),\n",
+        "    mctlib.Reference(reference='https://arxiv.org/abs/1801.04381'),\n",
         "]\n",
-        "model_card.model_details.license = 'Apache-2.0'\n",
-        "model_card.model_details.citation = 'https://github.com/tensorflow/model-card-toolkit/blob/master/model_card_toolkit/documentation/examples/Standalone_Model_Card_Toolkit_Demo.ipynb'"
+        "model_card.model_details.licenses = [mctlib.License(identifier='Apache-2.0')]\n",
+        "model_card.model_details.citations = [mctlib.Citation(citation='https://github.com/tensorflow/model-card-toolkit/blob/master/model_card_toolkit/documentation/examples/Standalone_Model_Card_Toolkit_Demo.ipynb')]"
       ]
     },
     {
@@ -368,9 +368,9 @@
       "outputs": [],
       "source": [
         "model_card.quantitative_analysis.performance_metrics = [\n",
-        "  {'type': 'accuracy', 'value': accuracy},\n",
-        "  {'type': 'accuracy', 'value': cat_accuracy, 'slice': 'cat'},\n",
-        "  {'type': 'accuracy', 'value': dog_accuracy, 'slice': 'Dog'},\n",
+        "  mctlib.PerformanceMetric(type='accuracy', value=str(accuracy)),\n",
+        "  mctlib.PerformanceMetric(type='accuracy', value=str(cat_accuracy), slice='cat'),\n",
+        "  mctlib.PerformanceMetric(type='accuracy', value=str(dog_accuracy), slice='Dog'),\n",
         "]"
       ]
     },
@@ -394,20 +394,20 @@
       "outputs": [],
       "source": [
         "model_card.considerations.use_cases = [\n",
-        "    'This model classifies images of cats and dogs.'\n",
+        "    mctlib.UseCase(description='This model classifies images of cats and dogs.')\n",
         "]\n",
         "model_card.considerations.limitations = [\n",
-        "    'This model is not able to classify images of other classes.'\n",
+        "    mctlib.Limitation(description='This model is not able to classify images of other classes.')\n",
         "]\n",
-        "model_card.considerations.ethical_considerations = [{\n",
-        "    'name':\n",
+        "model_card.considerations.ethical_considerations = [mctlib.Risk(\n",
+        "    name=\n",
         "        'While distinguishing between cats and dogs is generally agreed to be '\n",
         "        'a benign application of machine learning, harmful results can occur '\n",
         "        'when the model attempts to classify images that don’t contain cats or '\n",
         "        'dogs.',\n",
-        "    'mitigation_strategy':\n",
+        "    mitigation_strategy=\n",
         "        'Avoid application on non-dog and non-cat images.'\n",
-        "}]"
+        ")]"
       ]
     },
     {
@@ -486,11 +486,12 @@
       },
       "outputs": [],
       "source": [
-        "model_card.model_parameters.data.eval.graphics.collection = [\n",
-        "  {'name': 'Validation Set Size', 'image': validation_set_size_barchart},\n",
+        "model_card.model_parameters.data.append(mctlib.Dataset())\n",
+        "model_card.model_parameters.data[0].graphics.collection = [\n",
+        "  mctlib.Graphic(name='Validation Set Size', image=validation_set_size_barchart),\n",
         "]\n",
         "model_card.quantitative_analysis.graphics.collection = [\n",
-        "  {'name': 'Accuracy', 'image': accuracy_barchart},\n",
+        "  mctlib.Graphic(name='Accuracy', image=accuracy_barchart),\n",
         "]"
       ]
     },
@@ -514,7 +515,7 @@
       },
       "outputs": [],
       "source": [
-        "mct.update_model_card_json(model_card)"
+        "mct.update_model_card(model_card)"
       ]
     },
     {
@@ -560,7 +561,7 @@
       "source": [
         "# Generate a model card document in Markdown\n",
         "md_path = os.path.join(model_card_dir, 'template/md/default_template.md.jinja')\n",
-        "md_doc = mct.export_format(md_path, 'model_card.md')\n",
+        "md_doc = mct.export_format(template_path=md_path, 'model_card.md')\n",
         "\n",
         "# Display the model card document in Markdown\n",
         "display.display(display.Markdown(md_doc))"

diff --git a/model_card_toolkit/model_card.py b/model_card_toolkit/model_card.py
@@ -20,10 +20,10 @@
 ModelCardsToolkit serves as an API to read and write MC properties by the users.
 """
 
+import dataclasses
 import json as json_lib
 from typing import Any, Dict, List, Optional, Text
 
-import dataclasses
 from model_card_toolkit.base_model_card_field import BaseModelCardField
 from model_card_toolkit.proto import model_card_pb2
 from model_card_toolkit.utils import validation

diff --git a/model_card_toolkit/version.py b/model_card_toolkit/version.py
@@ -14,4 +14,4 @@
 """Contains the version string of the Model Card Toolkit."""
 
 # Note that setup.py uses this version.
-__version__ = '0.1.3.dev'
+__version__ = '1.0.0'
diff --git a/setup.py b/setup.py
@@ -25,7 +25,6 @@
 import subprocess
 
 from setuptools import Command
-from setuptools import find_packages
 from setuptools import setup
 
 REQUIRED_PACKAGES = [
@@ -110,7 +109,11 @@ def run(self):
     url='https://github.com/tensorflow/model-card-toolkit',
     author='Google LLC',
     author_email='tensorflow-extended-dev@googlegroups.com',
-    packages=find_packages(exclude=('bazel-model_card_toolkit*',)),
+    packages=[
+        'model_card_toolkit', 'model_card_toolkit.documentation',
+        'model_card_toolkit.documentation.examples', 'model_card_toolkit.proto',
+        'model_card_toolkit.utils'
+    ],
     package_data={
         'model_card_toolkit': ['schema/**/*.json', 'template/**/*.jinja']
     },