From 818911c75c86116b6cc48c099f128151277a64f6 Mon Sep 17 00:00:00 2001 From: Owen Thomas <31292660+owen-t@users.noreply.github.com> Date: Tue, 24 Jul 2018 14:43:48 -0700 Subject: [PATCH] Upgrade Master to TF 1.9 (#12) * Upgrade master to TF 1.9 --- CHANGELOG.rst | 18 ++++++++++ README.rst | 23 ++++++++----- create_integ_test_docker_images.py | 18 +++++----- setup.py | 7 ++-- .../src/pipemode_dataset_op_kernel.cpp | 4 +-- test/functional/test_pipemode.py | 2 +- test/integ/Dockerfile | 33 +++++++++++++++++-- test/integ/test_pipemode.py | 2 +- 8 files changed, 80 insertions(+), 27 deletions(-) create mode 100644 CHANGELOG.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..2cf1694 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,18 @@ +========= +CHANGELOG +========= + +1.9.0.1.0.2 +=========== + +* enhancement: Updated README doc description of installing into container. + +1.9.0.1.0.1 +=========== + +* enhancement: Updated README doc to document PyPI install. + +1.9.0.1.0.0 +=========== + +* enhancement: Upgraded master to TF 1.9 diff --git a/README.rst b/README.rst index 357bc12..57660df 100644 --- a/README.rst +++ b/README.rst @@ -5,14 +5,21 @@ SageMaker TensorFlow .. role:: python(code) :language: python -SageMaker specific extensions to TensorFlow, for Python 2 and TensorFlow versions 1.7 and 1.8. This package includes the :python:`PipeModeDataset` class, that allows SageMaker Pipe Mode channels to be read using TensorFlow DataSets. - -Pre-Install -~~~~~~~~~~~ -SageMaker TensorFlow build requires :code:`cmake` to be installed. Please :code:`pip install cmake` before building SageMaker TensorFlow. +SageMaker specific extensions to TensorFlow, for Python 2 and TensorFlow versions 1.7, 1.8, and 1.9. This package includes the :python:`PipeModeDataset` class, that allows SageMaker Pipe Mode channels to be read using TensorFlow DataSets. Install ~~~~~~~ +You can build SageMaker TensorFlow into your docker images with the following command: + +:: + + pip install sagemaker-tensorflow + + +Build from source +~~~~~~~~~~~~~~~~~ +SageMaker TensorFlow build requires :code:`cmake` to be installed. Please :code:`pip install cmake` before building SageMaker TensorFlow. + SageMaker TensorFlow extensions is installed as a python package named :code:`sagemaker_tensorflow`. First, make sure you have cmake installed. If not: @@ -29,7 +36,7 @@ To install this package, run: in this directory. -To install in a SageMaker docker image, you can use the following RUN command in your Dockerfile, assuming you have Python 2.7 and pip already installed in your image: +To build in a SageMaker docker image, you can use the following RUN command in your Dockerfile, assuming you have Python 2.7 and pip already installed in your image: :: @@ -40,8 +47,8 @@ To install in a SageMaker docker image, you can use the following RUN command in cd .. && \ rm -rf sagemaker-tensorflow-extensions -Installing for a specific TensorFlow version -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Building for a specific TensorFlow version +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Release branching is used to track different versions of TensorFlow. Tensorflow versions 1.7 and 1.8 are supported. To build for a specific release of TensorFlow, checkout the release branch prior to running a pip install. For example, to build for TensorFlow 1.7, you can run the following command in your Dockerfile: :: diff --git a/create_integ_test_docker_images.py b/create_integ_test_docker_images.py index d474e14..1ca4198 100644 --- a/create_integ_test_docker_images.py +++ b/create_integ_test_docker_images.py @@ -2,15 +2,15 @@ import argparse import base64 -import subprocess -import docker import boto3 import botocore +import docker import os +import subprocess +import sys -TF_VERSION = "1.8.0" +TF_VERSION = "1.9.0" REGION = "us-west-2" -FROM_IMAGE = "520713654638.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow:1.6.0-gpu-py2" if __name__ == '__main__': parser = argparse.ArgumentParser() @@ -23,9 +23,10 @@ username, password = base64.b64decode(token['authorizationData'][0]['authorizationToken']).decode().split(':') registry = token['authorizationData'][0]['proxyEndpoint'] - sdist_path = 'dist/sagemaker_tensorflow-{}.1.0.0.tar.gz'.format(TF_VERSION) - if not os.path.exists(sdist_path): - subprocess.check_call(['python', 'setup.py', 'sdist']) + dist_path = 'dist/sagemaker_tensorflow-1.9.0.1.0.0-cp27-cp27mu-linux_x86_64.whl' + + if not os.path.exists(dist_path): + subprocess.check_call([sys.executable, 'setup.py', 'bdist_wheel']) try: ecr_client.create_repository(repositoryName='sagemaker_tensorflow_integ_test') except botocore.exceptions.ClientError as e: @@ -35,12 +36,11 @@ raise tag = '{}/sagemaker_tensorflow_integ_test:{}-{}'.format(registry, TF_VERSION, args.device)[8:] - client.images.pull(FROM_IMAGE, auth_config={'username': username, 'password': password}) client.images.build( path='.', dockerfile='test/integ/Dockerfile', tag=tag, - buildargs={'sagemaker_tensorflow': sdist_path, + buildargs={'sagemaker_tensorflow': dist_path, 'device': args.device, 'tensorflow_version': TF_VERSION, 'script': 'test/integ/scripts/estimator_script.py'}) diff --git a/setup.py b/setup.py index d4840e5..46d9f4f 100644 --- a/setup.py +++ b/setup.py @@ -94,9 +94,8 @@ def build_extension(self, ext): setup( name='sagemaker_tensorflow', - version='1.8.0.1.0.0', + version='1.9.0.1.0.2', description='Amazon Sagemaker specific TensorFlow extensions.', - packages=find_packages(where='src', exclude=('test',)), package_dir={'': 'src'}, py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], @@ -115,9 +114,9 @@ def build_extension(self, ext): "Programming Language :: Python", 'Programming Language :: Python :: 2.7', ], - install_requires=['cmake', 'tensorflow==1.8'], + install_requires=['cmake', 'tensorflow==1.9'], extras_require={ 'test': ['tox', 'flake8', 'pytest', 'pytest-cov', 'pytest-xdist', 'mock', - 'sagemaker', 'docker'] + 'sagemaker', 'docker', 'boto3'] }, ) diff --git a/src/pipemode_op/Dataset/src/pipemode_dataset_op_kernel.cpp b/src/pipemode_op/Dataset/src/pipemode_dataset_op_kernel.cpp index 07475eb..a3e7a5e 100644 --- a/src/pipemode_op/Dataset/src/pipemode_dataset_op_kernel.cpp +++ b/src/pipemode_op/Dataset/src/pipemode_dataset_op_kernel.cpp @@ -113,7 +113,7 @@ class PipeModeDatasetOp : public DatasetOpKernel { channel_(channel), benchmark_(benchmark) {} - std::unique_ptr MakeIterator(const std::string& prefix) const override { + std::unique_ptr MakeIteratorInternal(const std::string& prefix) const override { auto new_prefix = prefix + "::PipeMode-" + channel_ + "-" + std::to_string(pipe_state_manager_.GetPipeIndex()); auto ptr = std::unique_ptr( @@ -134,7 +134,7 @@ class PipeModeDatasetOp : public DatasetOpKernel { return *shapes; } - std::string DebugString() override { return "PipeModeDatasetOp::Dataset"; } + std::string DebugString() const override { return "PipeModeDatasetOp::Dataset"; } protected: Status AsGraphDefInternal(DatasetGraphDefBuilder* b, diff --git a/test/functional/test_pipemode.py b/test/functional/test_pipemode.py index fae20f8..2eb30eb 100644 --- a/test/functional/test_pipemode.py +++ b/test/functional/test_pipemode.py @@ -110,7 +110,7 @@ def input_fn(): estimator = make_estimator(model_dir=model_dir) estimator.train(input_fn=input_fn) - assert os.path.exists(os.path.join(estimator.model_dir, 'model.ckpt-1.index')) + assert os.path.exists(os.path.join(estimator.model_dir, 'model.ckpt-0.index')) def test_multi_channels(): diff --git a/test/integ/Dockerfile b/test/integ/Dockerfile index ee1aaca..a600711 100644 --- a/test/integ/Dockerfile +++ b/test/integ/Dockerfile @@ -1,9 +1,38 @@ -FROM 520713654638.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow:1.6.0-gpu-py2 +FROM ubuntu:16.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + git \ + libcurl3-dev \ + libfreetype6-dev \ + libpng12-dev \ + libzmq3-dev \ + pkg-config \ + python-dev \ + rsync \ + software-properties-common \ + unzip \ + zip \ + zlib1g-dev \ + openjdk-8-jdk \ + openjdk-8-jre-headless \ + wget \ + vim \ + iputils-ping \ + nginx \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py && \ + rm get-pip.py RUN pip install cmake ARG device=cpu -ARG tensorflow_version=1.4.0 +ARG tensorflow_version=1.9.0 ARG script RUN if [ "$device"="cpu" ] ; then \ diff --git a/test/integ/test_pipemode.py b/test/integ/test_pipemode.py index 9cb757c..bdb4184 100644 --- a/test/integ/test_pipemode.py +++ b/test/integ/test_pipemode.py @@ -31,7 +31,7 @@ def multi_records_test_data(): s3_url = integ_test_resources.make_test_data( directory=test_data, name='multi', - num_files=5, + num_files=1, num_records=1000, dimension=DIMENSION) yield s3_url