Upgrade Master to TF 1.9 (#12)

* Upgrade master to TF 1.9

CHANGELOG.rst

@@ -0,0 +1,18 @@
+=========
+CHANGELOG
+=========
+
+1.9.0.1.0.2
+===========
+
+* enhancement: Updated README doc description of installing into container.
+
+1.9.0.1.0.1
+===========
+
+* enhancement: Updated README doc to document PyPI install.
+
+1.9.0.1.0.0
+===========
+
+* enhancement: Upgraded master to TF 1.9

README.rst

@@ -5,14 +5,21 @@ SageMaker TensorFlow
 .. role:: python(code)
    :language: python
 
-SageMaker specific extensions to TensorFlow, for Python 2 and TensorFlow versions 1.7 and 1.8. This package includes the :python:`PipeModeDataset` class, that allows SageMaker Pipe Mode channels to be read using TensorFlow DataSets.
-
-Pre-Install
-~~~~~~~~~~~
-SageMaker TensorFlow build requires :code:`cmake` to be installed. Please :code:`pip install cmake` before building SageMaker TensorFlow.
+SageMaker specific extensions to TensorFlow, for Python 2 and TensorFlow versions 1.7, 1.8, and 1.9. This package includes the :python:`PipeModeDataset` class, that allows SageMaker Pipe Mode channels to be read using TensorFlow DataSets.
 
 Install
 ~~~~~~~
+You can build SageMaker TensorFlow into your docker images with the following command:
+
+::
+
+   pip install sagemaker-tensorflow
+
+
+Build from source
+~~~~~~~~~~~~~~~~~
+SageMaker TensorFlow build requires :code:`cmake` to be installed. Please :code:`pip install cmake` before building SageMaker TensorFlow.
+
 SageMaker TensorFlow extensions is installed as a python package named :code:`sagemaker_tensorflow`. 
 
 First, make sure you have cmake installed. If not:
@@ -29,7 +36,7 @@ To install this package, run:
 
 in this directory. 
 
-To install in a SageMaker docker image, you can use the following RUN command in your Dockerfile, assuming you have Python 2.7 and pip already installed in your image:
+To build in a SageMaker docker image, you can use the following RUN command in your Dockerfile, assuming you have Python 2.7 and pip already installed in your image:
 
 ::
 
@@ -40,8 +47,8 @@ To install in a SageMaker docker image, you can use the following RUN command in
         cd .. && \
         rm -rf sagemaker-tensorflow-extensions
 
-Installing for a specific TensorFlow version
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Building for a specific TensorFlow version
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Release branching is used to track different versions of TensorFlow. Tensorflow versions 1.7 and 1.8 are supported. To build for a specific release of TensorFlow, checkout the release branch prior to running a pip install. For example, to build for TensorFlow 1.7, you can run the following command in your Dockerfile:
 
 ::

create_integ_test_docker_images.py

@@ -2,15 +2,15 @@
 
 import argparse
 import base64
-import subprocess
-import docker
 import boto3
 import botocore
+import docker
 import os
+import subprocess
+import sys
 
-TF_VERSION = "1.8.0"
+TF_VERSION = "1.9.0"
 REGION = "us-west-2"
-FROM_IMAGE = "520713654638.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow:1.6.0-gpu-py2"
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
@@ -23,9 +23,10 @@
     username, password = base64.b64decode(token['authorizationData'][0]['authorizationToken']).decode().split(':')
     registry = token['authorizationData'][0]['proxyEndpoint']
 
-    sdist_path = 'dist/sagemaker_tensorflow-{}.1.0.0.tar.gz'.format(TF_VERSION)
-    if not os.path.exists(sdist_path):
-        subprocess.check_call(['python', 'setup.py', 'sdist'])
+    dist_path = 'dist/sagemaker_tensorflow-1.9.0.1.0.0-cp27-cp27mu-linux_x86_64.whl'
+
+    if not os.path.exists(dist_path):
+        subprocess.check_call([sys.executable, 'setup.py', 'bdist_wheel'])
     try:
         ecr_client.create_repository(repositoryName='sagemaker_tensorflow_integ_test')
     except botocore.exceptions.ClientError as e:
@@ -35,12 +36,11 @@
             raise
 
     tag = '{}/sagemaker_tensorflow_integ_test:{}-{}'.format(registry, TF_VERSION, args.device)[8:]
-    client.images.pull(FROM_IMAGE, auth_config={'username': username, 'password': password})
     client.images.build(
         path='.',
         dockerfile='test/integ/Dockerfile',
         tag=tag,
-        buildargs={'sagemaker_tensorflow': sdist_path,
+        buildargs={'sagemaker_tensorflow': dist_path,
                    'device': args.device,
                    'tensorflow_version': TF_VERSION,
                    'script': 'test/integ/scripts/estimator_script.py'})

setup.py

@@ -94,9 +94,8 @@ def build_extension(self, ext):
 
 setup(
     name='sagemaker_tensorflow',
-    version='1.8.0.1.0.0',
+    version='1.9.0.1.0.2',
     description='Amazon Sagemaker specific TensorFlow extensions.',
-
     packages=find_packages(where='src', exclude=('test',)),
     package_dir={'': 'src'},
     py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')],
@@ -115,9 +114,9 @@ def build_extension(self, ext):
         "Programming Language :: Python",
         'Programming Language :: Python :: 2.7',
     ],
-    install_requires=['cmake', 'tensorflow==1.8'],
+    install_requires=['cmake', 'tensorflow==1.9'],
     extras_require={
         'test': ['tox', 'flake8', 'pytest', 'pytest-cov', 'pytest-xdist', 'mock',
-                 'sagemaker', 'docker']
+                 'sagemaker', 'docker', 'boto3']
     },
 )

src/pipemode_op/Dataset/src/pipemode_dataset_op_kernel.cpp

@@ -113,7 +113,7 @@ class PipeModeDatasetOp : public DatasetOpKernel {
             channel_(channel),
             benchmark_(benchmark) {}
 
-        std::unique_ptr<IteratorBase> MakeIterator(const std::string& prefix) const override {
+        std::unique_ptr<IteratorBase> MakeIteratorInternal(const std::string& prefix) const override {
             auto new_prefix = prefix + "::PipeMode-" + channel_ + "-"
                 + std::to_string(pipe_state_manager_.GetPipeIndex());
             auto ptr = std::unique_ptr<IteratorBase>(
@@ -134,7 +134,7 @@ class PipeModeDatasetOp : public DatasetOpKernel {
             return *shapes;
         }
 
-        std::string DebugString() override { return "PipeModeDatasetOp::Dataset"; }
+        std::string DebugString() const override { return "PipeModeDatasetOp::Dataset"; }
 
      protected:
         Status AsGraphDefInternal(DatasetGraphDefBuilder* b,

test/functional/test_pipemode.py

@@ -110,7 +110,7 @@ def input_fn():
 
     estimator = make_estimator(model_dir=model_dir)
     estimator.train(input_fn=input_fn)
-    assert os.path.exists(os.path.join(estimator.model_dir, 'model.ckpt-1.index'))
+    assert os.path.exists(os.path.join(estimator.model_dir, 'model.ckpt-0.index'))
 
 
 def test_multi_channels():

test/integ/Dockerfile

@@ -1,9 +1,38 @@
-FROM 520713654638.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow:1.6.0-gpu-py2
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        curl \
+        git \
+        libcurl3-dev \
+        libfreetype6-dev \
+        libpng12-dev \
+        libzmq3-dev \
+        pkg-config \
+        python-dev \
+        rsync \
+        software-properties-common \
+        unzip \
+        zip \
+        zlib1g-dev \
+        openjdk-8-jdk \
+        openjdk-8-jre-headless \
+        wget \
+        vim \
+        iputils-ping \
+        nginx \
+        && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \
+    python get-pip.py && \
+    rm get-pip.py
 
 RUN pip install cmake
 
 ARG device=cpu
-ARG tensorflow_version=1.4.0
+ARG tensorflow_version=1.9.0
 ARG script
 
 RUN if [ "$device"="cpu" ] ; then \

test/integ/test_pipemode.py

@@ -31,7 +31,7 @@ def multi_records_test_data():
     s3_url = integ_test_resources.make_test_data(
         directory=test_data,
         name='multi',
-        num_files=5,
+        num_files=1,
         num_records=1000,
         dimension=DIMENSION)
     yield s3_url