Merge pull request #37 from aws-deepracer-community/dev

Feature Development

.gitignore

@@ -56,7 +56,6 @@ coverage.xml
 *.pot
 
 # Django stuff:
-*.log
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
@@ -129,4 +128,5 @@ dmypy.json
 .pyre/
 
 # vscode
-settings.json
+.vscode/launch.json
+.vscode/settings.json

README.md

@@ -1,95 +1,53 @@
-# deepracer-utils
-A set of utilities to take your DeepRacer experience to the next level.
+# Deepracer Utilities - Analyzing Your DeepRacer Model
+This is a set of utilities that will take your DeepRacer experience to the next level by allowing you to analyze your model, step by step, episode by episode. Only through analyzing what your model does will you be able to write the right reward function, choose the right action space and to tune the hyperparameters!
 
-# Add deepracer to AWS CLI and boto3
-Make sure you have deepracer-utils >= 0.9 installed.
+## Installation
 
-Then run this to install:
-```
-python -m deepracer install-cli
-```
-
-If you want to ensure you have the latest deepracer service definition, you can run:
-```
-python -m deepracer install-cli --force
-```
-
-To remove deepracer support from aws-cli and boto3, run:
-```
-python -m deepracer remove-cli
-```
-
-## Development
-
-### Prepare the environment
-For pip/venv:
-```
-python3 -m venv env
-source env/bin/activate
-pip install -r requirements.txt
-```
-_(For newer systems python3 may be referred to as python)_
-
-For Anaconda:
-```
-conda create --name env pip
-conda activate pip
-pip install -r requirements.txt
+You can install the latest version of deepracer-utils via pip through
 ```
-
-### Install deepracer-utils for development
-```
-python setup.py develop
+pip install deepracer-utils
 ```
-Once finished working, run:
+Otherwise you can build your own version with 
 ```
-python setup.py develop --uninstall
+python3 setup.py build
+python3 setup.py install
 ```
 
-See [Python Packaging User Guide](https://packaging.python.org/guides/distributing-packages-using-setuptools/#id70) for more info.
-
-### Testing
-
-Run:
+### AWS CLI and boto3 extension
+This package contains an extension to the AWS CLI and Boto3 that allows you to interact
+with the Deepracer Console through commands starting with `aws deepracer`. For details run
 ```
-tox
+aws deepracer help
 ```
-This will package the project, install and run tests.
-
-### Verifying the style guide
 
-Run:
+Then run this to install:
 ```
-pycodestyle
+python -m deepracer install-cli
 ```
 
-### Releasing, Packaging, distribution
-
-Checking the current version:
+To remove deepracer support from aws-cli and boto3, run:
 ```
-python setup.py version
+python -m deepracer remove-cli
 ```
 
-Marking new release:
-```
-git tag deepracer-utils-version
-git push origin deepracer-utils-version
-```
+## About the Utilities
 
-Building the package:
-```
-python setup.py sdist bdist_wheel
-```
+The best reference on how to use the utilities can be found in the [deepracer-analysis](https://github.com/aws-deepracer-community/deepracer-analysis) Jupyter notebooks.
 
-Uploading to test.pypi.org:
-```
-python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/*
-```
+An overview of the different modules provided, and the key classes involved:
+| Module | Class | Description |
+|--------|-------|-------------|
+|`deepracer.logs` | DeepRacerLog | Class that is pointed to a Deepracer Model folder, locally or in an S3 bucket, and that reads in and processes trace files from simtrace or robomaker log files.|
+|`deepracer.logs` | AnalysisUtils | Class that processes the raw log input and summarizes by episode.|
+|`deepracer.logs` | PlottingUtils | Class that visualises the track and plots each step in an episode.|
+|`deepracer.logs` | TrainingMetrics | Class that reads in Metrics data and provides data similar to the training graph in the Console.|
+|`deepracer.console` | ConsoleHelper | Class that reads out logfiles directly from the console, and together with e.g. TrainingMetrics can be used to visualize training progress in real time.|
+|`deepracer.tracks` | TrackIO | Class that processes track routes (.npy files) and displays waypoints graphically.|
+|`deepracer.model` | n/a | Methods to run inference on individual images and to perform visual analysis.|
+## Other information
 
-Uploading to pypi.org:
-```
-python -m twine upload dist/*
-```
+* Refer to [development.md](docs/development.md) for instructions on coding standards, unit tests etc.
+* Refer to [examples.md](docs/examples.md) for usage guidance.
 
 ## License
 This project retains the license of the 

deepracer/__init__.py

@@ -1,4 +1,4 @@
-from . import logs, tracks, boto3_enhancer
+from . import logs, tracks, boto3_enhancer, console
 
 from ._version import get_versions
 __version__ = get_versions()['version']

deepracer/console/__init__.py

@@ -0,0 +1 @@
+from .helper import ConsoleHelper, LeaderboardSubmissionType

deepracer/console/helper.py

@@ -0,0 +1,201 @@
+"""
+Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2019-2020 AWS DeepRacer Community. All Rights Reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this
+software and associated documentation files (the "Software"), to deal in the Software
+without restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+
+import tarfile
+from enum import Enum
+from io import BytesIO, TextIOWrapper
+from urllib.request import urlopen
+
+import boto3
+import pandas as pd
+
+from deepracer import boto3_enhancer
+from deepracer.logs import SimulationLogsIO
+
+
+class LeaderboardSubmissionType(Enum):
+    RANKED = 1
+    LATEST = 2
+
+
+class ConsoleHelper:
+    """ Class used to retrieve information from AWS DeepRacer Console.
+    """
+    def __init__(self, profile: str = None, region: str = 'us-east-1'):
+
+        if profile is not None:
+            session = boto3.session.Session(profile_name=profile)
+        else:
+            session = boto3.session.Session()
+        self.dr = boto3_enhancer.deepracer_client(session=session, region_name=region)
+
+    def find_model(self, model_name: str) -> str:
+        """Finds the Model ARN based on a model display name
+
+        Arguments:
+        model_name - (str) String with the model name
+
+        Returns:
+        String with the model ARN. None if not found.
+        """
+
+        m_response = self.dr.list_models(ModelType="REINFORCEMENT_LEARNING", MaxResults=25)
+        model_dict = m_response["Models"]
+        models = pd.DataFrame.from_dict(model_dict)
+        my_model = models[models["ModelName"] == model_name]
+
+        if my_model.size > 0:
+            return my_model.loc[:, 'ModelArn'].values[0]
+
+        while "NextToken" in m_response:
+            m_response = self.dr.list_models(
+                ModelType="REINFORCEMENT_LEARNING",
+                MaxResults=50,
+                NextToken=m_response["NextToken"],
+            )
+            model_dict = m_response["Models"]
+
+            models = pd.DataFrame.from_dict(model_dict)
+            my_model = models[models["ModelName"] == model_name]
+            if my_model.size > 0:
+                return my_model.loc[:, 'ModelArn'].values[0]
+
+        return None
+
+    def find_leaderboard(self, leaderboard_guid: str) -> str:
+        """Finds the Leaderboard ARN based on a short GUID
+
+        Arguments:
+        leaderboard_guid - (str) String with the leaderboard GUID
+                Example: 8fab8c08-66cd-4170-a0b0-ab2b464e42d9
+
+        Returns:
+        String with the model ARN. None if not found.
+        """
+
+        leaderboard_arn = "arn:aws:deepracer:::leaderboard/{}".format(leaderboard_guid)
+
+        l_response = self.dr.list_leaderboards(MaxResults=25)
+        lboards_dict = l_response["Leaderboards"]
+        leaderboards = pd.DataFrame.from_dict(l_response["Leaderboards"])
+        if leaderboards[leaderboards["Arn"] == leaderboard_arn].size > 0:
+            return leaderboard_arn
+
+        while "NextToken" in l_response:
+            l_response = self.dr.list_leaderboards(
+                MaxResults=50, NextToken=l_response["NextToken"]
+            )
+            lboards_dict = l_response["Leaderboards"]
+
+            leaderboards = pd.DataFrame.from_dict(lboards_dict)
+            if leaderboards[leaderboards["Arn"] == leaderboard_arn].size > 0:
+                return leaderboard_arn
+
+        return None
+
+    def get_training_job(self, model_arn: str) -> dict:
+        """Retrieves Training Job information for a Model
+
+        Arguments:
+        model_arn - (str) String with the Model ARN
+
+        Returns:
+        Dict with the Training Job information.
+        """
+
+        m_response = self.dr.list_training_jobs(ModelArn=model_arn)
+        m_jobs = m_response["TrainingJobs"]
+        training_job_arn = m_jobs[0]['JobArn']
+        m_response = self.dr.get_training_job(TrainingJobArn=training_job_arn)
+        m_job = m_response['TrainingJob']
+        return m_job
+
+    def get_training_log_robomaker(self, model_arn: str, data: list = None) -> pd.DataFrame:
+        """Retrieves the Robomaker training log for a completed training
+
+        Arguments:
+        model_arn - (str) String with the Model ARN
+        data - (list) Optional pass in a pre-filled data object
+
+        Returns:
+        Pandas DataFrame with the parsed log.
+        """
+        if data is None:
+            data = []
+
+        training_job = self.get_training_job(model_arn)
+        training_job_arn = training_job['JobArn']
+        f_url = self.dr.get_asset_url(Arn=training_job_arn, AssetType="LOGS")["Url"]
+        bytes_io = BytesIO(urlopen(f_url).read())
+
+        with tarfile.open(fileobj=bytes_io, mode="r:gz") as tar_file:
+            for member in tar_file.getmembers():
+                if member.name.find("robomaker") > 0:
+                    log_buf = TextIOWrapper(tar_file.extractfile(member))
+                    data = SimulationLogsIO.load_buffer(log_buf)
+                    df = SimulationLogsIO.convert_to_pandas(data)
+                    return df
+
+        return None
+
+    def get_leaderboard_log_robomaker(self, leaderboard_guid: str,
+                                      select=LeaderboardSubmissionType.RANKED,
+                                      data: list = None) -> pd.DataFrame:
+        """Retrieves the Robomaker log for leaderboard submission
+
+        Arguments:
+        leaderboard_guid - (str) String with the Leaderboard GUID
+        select - (LeaderboardSubmissionType) Get ranked or latest submission
+        data - (list) Optional pass in a pre-filled data object
+
+        Returns:
+        Pandas DataFrame with the parsed log.
+        """
+        if data is None:
+            data = []
+
+        leaderboard_arn = self.find_leaderboard(leaderboard_guid)
+
+        if select == LeaderboardSubmissionType.RANKED:
+            response_m = self.dr.get_ranked_user_submission(LeaderboardArn=leaderboard_arn)
+            submission = response_m["LeaderboardSubmission"]
+            activity_arn = "arn:aws:deepracer:us-east-1:180406016328:leaderboard_evaluation_job/{}"\
+                .format(submission["SubmissionVideoS3path"].split("/")[7].split("-", 2)[2])
+
+        if leaderboard_arn is None:
+            return None
+
+        else:
+            response_m = self.dr.get_latest_user_submission(LeaderboardArn=leaderboard_arn)
+            submission = response_m["LeaderboardSubmission"]
+            activity_arn = submission["ActivityArn"]
+
+        if submission["LeaderboardSubmissionStatusType"] == "SUCCESS":
+            f_url = self.dr.get_asset_url(Arn=activity_arn, AssetType="LOGS")["Url"]
+            bytes_io = BytesIO(urlopen(f_url).read())
+
+            with tarfile.open(fileobj=bytes_io, mode="r:gz") as tar_file:
+                for member in tar_file.getmembers():
+                    if member.name.find("robomaker") > 0:
+                        log_buf = TextIOWrapper(tar_file.extractfile(member))
+                        data = SimulationLogsIO.load_buffer(log_buf)
+                        df = SimulationLogsIO.convert_to_pandas(data)
+                        df['stream'] = member.name.split("/")[-1].split("-")[0]
+                        return df
+        else:
+            return None

deepracer/logs/__init__.py

@@ -1,4 +1,6 @@
-from .log_utils import ActionBreakdownUtils, AnalysisUtils, EvaluationUtils, NewRewardUtils, \
-    PlottingUtils, SimulationLogsIO
-from .metrics import TrainingMetrics
+from .handler import FileHandler, FSFileHandler, S3FileHandler
 from .log import DeepRacerLog
+from .log_utils import (ActionBreakdownUtils, AnalysisUtils, EvaluationUtils,
+                        NewRewardUtils, PlottingUtils, SimulationLogsIO)
+from .metrics import TrainingMetrics
+from .misc import LogFolderType, LogType