Tutorials + Associated Deployment Issues

.gitignore

@@ -10,3 +10,4 @@
 *Untitled*.ipynb
 *$*
 *credentials*
+*.terraform*

.travis.yml

@@ -7,18 +7,19 @@ services:
 - docker
 env:
   global:
-  - IMAGE_NAME=675906086666.dkr.ecr.us-west-2.amazonaws.com/planet-snowcover:latest
-  - secure: FLs0oWya43hMJbuoLB5UKHDGYeYGpPhRKmkCfnDIc9PeLHhmS2zB8Sk8vogtlSxquFGnSC6aSOKceK/DOb1jkR/kQxc0rOb7yU11McGEUOrAQk+hXiEArvAwMonjT4kucldY6oWYXFfIp9Nt/FnMD3foe954dqQC2kBIme2OAjxtQVMb4P9uRpxq26CfzfT4SR/cx7AX1pfVbuDVnxS6qVxUZOw4sCT3UP4hpj99B6aWsoBklihwpsQomcU9/Ob3Icxum0h37WdGI1VT2uUJB3gHbOiwAx4ME3aft3eVGFjl+ba52++NOXlCUJoZc6fx81b5HVDRLYj+KSwzi2OiVwYZPaEd989bYG07MPludIPX3td6950q4qkj92JBpddzB83oUtUB6FxUk/TQ7fmHSTZN+xsfPeWwY/dOKDt3JZ8fGTdt1/i8W2d1doOPuOqt6rexfNuzGr2BbVHgkD24CEruICV3ELczoCXru72AcFimnAPI6Kmap/Fmkw+fKe2SbYkAQB/CDK0VIm3A7pYMSiUTHIW8NEVF6cMOoKzXngWK0CUnL7q0HEFMHe8cjsMTgOHqBq6ll/yj04JU+9pjIEPLWHtWctq6eqMn1VGWu+N9e/wBYx8GEggMD29v0yO9sROzdZIwtjYzTP48qJKBCYRFsUM6T/r+I1oP94lSSD0=
-  - secure: B4vRUBdA3nUfcQh0IB1PkDpFaSTFyXp3rjImAobNwFiQY1yqcqNqOSvRRkY2F8PdkKEPsHY2hsK058mFKeiTLHaZoNTRD4urs42wOih3M9RhopRsRNkAIdL/ymvirZPNNRSQAfTipMrlzguZFoiYkPqALNOHn9LT5fSZt40rbkrELSzo9k/GcRdZnk7JCM1HP4VNBv6mNKW6NVHkt+zd+blMVMYv3IQJSAI+5wDRJKrLWiErqYqV/Nzy7ESslBZpUDq4SSfk0yv4GMfgUnw0mEkH34KPpqYrngZDh0Odeuh0+MtVlyaVmfygp3Czqlkoa+WNXc6JB8kt/ygRizLIQBwABmQvyyOk+uUQmNCCjyzy3dBAnd5YFUvh9qOs6MT6u1WYMZKG/wlD8A+NknQ1wmCe1vsOgDJWLBz4YKoiHJ0ILydLrw7qT/QfHy0WjemKL8BMyMwTRl4NMASRLcIvxdHcy8IL6DBehlI7tDn/vncEZokBwb22dFgkxvDPmGf41V+JaunAcIlh3YMxygug+umQb95mTPwgMDWaTAv1jrkQU6mwxGozXw8/pHukSwLO7Uz10AkUKBhjLzcuLkDlfa4OLYxRZzNETL6PupeVz33PYvbM2zdDAYOnf4tBbuQhYg+emXk0OVtdJlezVfZdme7AxoYEFLYCYkKqxwrZEoc=
+  - TAG_SLUG_PR=$([ "$TRAVIS_PULL_REQUEST" != "false" ] && echo "$TRAVIS_PULL_REQUEST_BRANCH" || echo "latest")
+  - IMAGE_NAME="675906086666.dkr.ecr.us-west-2.amazonaws.com/planet-snowcover:$TAG_SLUG_PR"
+  - DOCKERHUB_IMAGE_NAME="tonycannistra/planet-snowcover:$TAG_SLUG_PR"
+  - secure: T/E2WhjWS13M2ruqpRJ5pSN46eJB4JSRwoXOcI88xFLKgqvVf/uf6cemOPzHz7WCF33usYNGtfkRXLTlC+i4Pt4uWLCCqDUgfsXj4aKvsV3HVMxcp9ONuozKaDzq8PGInRbCOKUWvJKlLq34WiN8J59lnBYWOcxvlWzbSxhvlHmPPj7xNs7rkSNp+YeOQ0w/2cklxMMFf2EkteJfK3ouf39UntYfnZyRvJNu7zGtePryZsllwMDJsOM1wc6SKB9IpTjdLsbTrj1SLz2RmZk1EnZNn/onjtshsYmTRF3OZHCMHiHPfxtj/xMUQXiUAU1T7eLtgGCXFuMZ3fQ4RKX/a5iHiKedlvjVngpIpJ5xPmXtn657arrJzQMUVCkh8PSDFyWe6vbimAn8KkckLH/dhBiW61YX56OlzqYsayOBonAEuUV7d/Tu9fX8cdCDwzaq56wd1/qo4k0DN7/2mYTfOspv/NKTQy2zV8XDxTHq3a5UeIgRd3t3vbZIrICj0e1FaPovOJId0vNaZGv6plU1WAWvm+Djw7DlOWWglY+BW5XvAW6di+Q4KIAcJVgzGBr4W0cBp3TUOWBkuutxIYHgYJFFlrZfHQafpDLsYqgQKjLwSIZUikU+/BxL17+BqFtLO/rqG0hf+oWYcTUZTBg3FfM6q4lYog2eKj/oTtcPenk=
+  - secure: K7HZ/d4OkXWfPKOsmhEn347K8nxKkRuQsNgZ5rKZRT3tXwrfsPQGibgYYIEtbreVGYUoTwP0PcD4a4adiatNpAayzG3uG4+w62eZyyvM8bR2a/9yi6e4YjYHv5mHszVpnD54RGLg8kvtD4/hRoMt64KqfaGalTq3HqLUpUnRYSu9QCVqUEgSPfZif6gg4DpysInAoZ8SbvTMpL5E05DG7HBiYSsnOpVry/z/BtDV8bJG4S5ba5OouzmgI9Ln9TPntEeSupOpemB/D4X5LcW6TEcLU2ouFL9G1qUbGHdWPgs0faFwuUTDFzUE5KxyvOYs2JIlqQLFIwLcZ1ZkueZfKts0Wm2jVXya5KrZLwc7L0yXjBBvDL6bqS98a0u/+zrbdNV6VeToOZLH0K0wgtWhUggXtd22XXhR83WFewSJymKN/nApeH3pQuKPjGkQWip9G1d4C/KL5jB9L5Aj3Z+uOZjS0MOTq11E9MbbYESMQXRJYVM6B/vXm6Zbrq3dCVJIdNMyw5YrdpkA4PUT1baoCFlx6PDkAZBU4thhT+7Yu4VUXK/0xC/tNheizknDcQvyKNREY6Ae2A2KBDiEXlsKhJzi4lSoqytf0tjHWM6NS0X8P24zKjLtQigiim8iyLBMsHlm23rd1YhiCabkEC7fSzEHIzJ6U3QJheZM42IYu6c=
+  - secure: DueBtc+i04vk/P++FhMZph3OYUXSnTCnVf/ACaMc1T3BfBcP69QYbPOCgct/4CwnhZNLtOuXnJA+G7QEIGoZTWv3hxtAhTijrDWY5VyiTNjoiLQEbLIEklt0FTeSjLqOvm7i2TPEuSV7nYLLVsZLZaVAa4fT5kFxqx6zAHCCZ4gbQ7YE+pOmdLCgr1oTM9TW0FpBCUQHaaRlByVv1S2AfaBx3Ob3FVRGpq8MFbmVrZ7irpLlPJ4okpxQ9WRJdY5runlRloXUAtpEYrEpxDP/sB1m1Lww5fB/X9F+IXlmppqA1gLkycUZrqug9weTw9/+ra71yGskdOgZiaqufkLn9fPx7v03SPm6AE9znu+OL9fdNpjx8Pc9TJ55TwSSd68kGTBFJt9KWo7tqxjEorouRzDf/x8zz+R8mF25vjDmybHWk1aXGeeDmNVgp33tBYxjq9an77riNEtUZZpjq4cZ8wyu4jR6A8DgG+eefdj4GUnYTR7mc8P99SFGdpXey40/swYWrHQkjPKLnnNG+erwRADWCH/8KI0QX7M4PrSw0JaRvrpc0+mzTi8ArD1jGm0P/scFLnWck39ja4rKu91iJbiaChv2y4G14SmKUG5IC2RaCC3I6IM1G8wVo2YVvEQStOU97FsKDKoF/i1597f1HtkCXuGgKH33N/gQRsl3DPg=
+  - secure: DEhA2alHfrqaF7Cfzkj1G/0PmNQsYzh/i7hAp0z6667aC3qqHWi51BSa76nheJEr8LoZ+wC4/5Nu7CyrELYdTkpP9uTeSp5viPuUeuuMknk0o1FFJ/hdb2BKQzrilIT5Nk8b+PF8Pj0wtFBsz1ztmhoE+kJ1hKCeMSIG380nSGGs5AHJaF/y6xysOlv/UIdicVCNy2aME41wqUnvrM7mTnPFsxPZpS/mGYUTW8ejwaway7xWg/V6HrDKe5drnNbi+OzsBag+YW2aLfU/ucvn5kJZLWz56eKox0zrxGFPNKX7UxCMTRD4WNrRQF5iSZIaEFpi2JOiUGFeC/rVgHBZt/t9PtCQC6in2aJjAnL7gy/nnBAF3sYyRC+IoH6X+10DjpRWgDIU68W1pfNMlUIMA5ftJM6H4GeGx7mwiQIyCbWCvNITZpL86+9Re9k6Rms65wm33xqeVlXppQTqlhLmUVtsFdXsPudvFMl4NjzAiirtO37FJWbWnHPnmjGHTrUvp14GhDM3ynpsjKlFXg+U6aPSLumq6Wrt/FZT9ZiKyx9g7nT2Md8s16xJusQLmwRmf5zDTUV4zeDqFmnHWTHUyLEanisve+THXrl5Nnz9Hx21UBq6NZDRgNaFHmA6YeCyBbE5eekXxg3fZ4WwqCnIEmXLJq9fZ5s/abwrPlQwYJU=
 before_script:
 - docker pull "$IMAGE_NAME" || true
 script:
 - if test -z "$AWS_ACCESS_KEY_ID"; then echo "AWS ACCESS KEY empty"; fi;
 - if test -z "$AWS_SECRET_ACCESS_KEY"; then echo "AWS SECRET empty"; fi;
-- bash ./environment/create-aws-creds.sh
-- stat ./environment/aws_config
-- docker build --pull --cache-from "$IMAGE_NAME" --tag "$IMAGE_NAME" --file environment/Dockerfile
-  .
+- docker build --pull --cache-from "$IMAGE_NAME" --tag "$IMAGE_NAME" --file environment/Dockerfile .
 after_script:
 - docker images
 after_success:
@@ -30,3 +31,7 @@ after_success:
 - export PATH=$PATH:$HOME/.local/bin
 - eval $(aws ecr get-login --no-include-email --region us-west-2)
 - docker push "$IMAGE_NAME"
+- docker logout
+- docker login --username=${DOCKERHUB_USER} --password=${DOCKERHUB_PASS}
+- docker tag $IMAGE_NAME $DOCKERHUB_IMAGE_NAME
+- docker push "$DOCKERHUB_IMAGE_NAME"

deployment/README.md

@@ -18,9 +18,11 @@ Before we deploy any infrastructure, we need to set up your local environment so
 
 ## 🔐 Configuring AWS + SSH
 
-There's some preparation that we need to do to be able to deploy our infrastructure. You'll need to have **configured AWS credentials** and **configured an SSH key**. These steps will detail how to do that.
+There's some preparation that we need to do to be able to deploy our infrastructure. You'll need to have **configured AWS credentials** with correct permissions and **configured an SSH key**. These steps will detail how to do that.
 
-### **Configure AWS**:
+### **Configure AWS (Personal Account)**
+
+Follow these steps if you've never created an AWS account before or if you are a qualified administrator of an AWS account.
 
 1. **Sign up for AWS.** To perform this configuration, be sure you have an AWS account. If you don't, [go here](https://aws.amazon.com/premiumsupport/knowledge-center/create-and-activate-aws-account/) and follow the instructions.
 1. **Install the AWS Command Line Tools**. For this tutorial, you'll need to install the AWS CLI tools. If you have `pip` installed, run the following command to install them:
@@ -30,6 +32,14 @@ There's some preparation that we need to do to be able to deploy our infrastruct
 3. **Create AWS Credentials**: To sign in to AWS from the CLI, you need an *AccessKey* and *SecretKey* pair. Follow the instructions listed on AWS's documentation [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) to configure access keys. ⚠️**Note**⚠️ that you can only download your secret key once –– if you misplace it, you'll need to create another.
 3. **Sign In with your AWS Credentials**. Once you've installed the AWS cli and configured keys in the AWS console, run `aws configure` to configure the CLI with your credentials.
 
+### **Configure AWS (Group/Shared Account)**
+
+Follow these steps if you're a delegated IAM user of a shared AWS account.
+
+1. **Ask your administrator to update your account policy**. Deploying the resources required to run the code in this project requires a somewhat broad set of permissions. In particular, we require access to the following services: EC2, SageMaker, and S3, as well as some specific IAM permissions. To facilitate this, we provide [`aws_userpolicy.json`](./aws_userpolicy.json), which contains a broad set of permissions that will allow for infrastructure deployment. Please send this policy to your administrator and ask for it to be attached to your account.
+1. **Follow steps 2-4 Above**. Now that you've been properly credentialed, you can install the AWS command line toole, create some credentials, and sign-in.
+
+
 
 
 ### Configure your SSH Keys
@@ -101,27 +111,53 @@ Finally, to deploy these resources, run
 
     terraform apply
 
-This process will ask you for confirmation (type `yes`), and will talk to AWS to configure the cloud compute resources we need to do this work. It should take less than 5 minutes.
+This process will ask you for confirmation (type `yes`), and will talk to AWS to configure the cloud compute resources we need to do this work. It should take about 20 minutes.
 
 When it's complete, you should see an output looking something like this:
 
     Outputs:
 
     public_instance_ip = [
-    "35.160.4.86",
-    ]
-    public_route_table_ids = [
-    "rtb-08ef3ffbecd7f07ae",
+    "XXX.XXX.XXX.XXX",
     ]
-    public_subnets = [
-    "subnet-08b663ab9282029fe",
-    ]
-    vpc_id = vpc-052b4f723bcd819ce
 
-The most important piece of information to note here is **`public_instance_ip`**, which is the address of our newly deployed cloud computer.
+    SSH_CMD = "ssh -i /path/to/key.pem ubuntu@$XXX.XXX.XXX.XXX"
+
+    sagemaker_role_arn = "arn:aws:sagemaker:xxxxx"
+
+Importantly, the `SSH_CMD` value is a command which will connect you to the newly-deployed EC2 instance. You'll also need the `sagemaker_role_arn` value for model training.
+
+## Login + Start JupyterLab
+
+The output of `terraform apply` (which is always available by running `terraform output`), is both an IP adress and a command which can be used to SSH into a newly-deployed EC2 instance. This instance has been provisioned with a Docker image containing JupyterLab, our tutorial and development environment. Follow these steps to launch JupyterLab.
+
+1. SSH into the instance: `eval $(terraform output SSH_CMD)` or copy/paste the SSH_CMD value above.
+1. Once logged-in, run `sudo docker run -ti -p 8888:8888 -v $(pwd):/host tonycannistra/planet-snowcover lab`. This should produce output with a URL like: `http://0.0.0.0:8888/?token=4715df79c9b1c97244618552e2198ba478cd42f3fa5ebbd8`.
+1. In your browser, paste the value of `public_instance_ip` from `terraform output` as follows: `XXX.XXX.XXX.XXX:8888`. In the box, copy and paste the value after `token` in the above URL and press Enter.
+
+**TODO: add smarter volume mounting + git pull of tutorials**
+
+## IMPORTANT: Turn off your instances *every time*.
+
+As mentioned above, this deployment comes with a monthly fee, as does every Amazon resource. To save money, **shut down your EC2 instance when you're done using it**. If you wish to return to your work, do not `terminate` the instance.
+
+If you don't have any saved work and would like to destroy all of the deployed resources, run `terraform destroy`. All unsaved work stored in any of the deployed resources (EC2, mainly) will be lost. 
 
-## Accessing Resources + Deploying Jupyter Lab
+## Appendix: Provisioned Resources
 
-To actually do some work on these computers, we need to get a little familiar with the `ssh` command line tool. Basically, `ssh` allows you to login to a remote computer (in this case, a cloud computer run by AWS) and perform some tasks, as if you were sitting on your local command line.
+This terraform configuration will create (and therefore can destroy) the following resources:
 
-***TODO: Finish this section***
+* AWS Elastic Container Repository repo (name: `ps-images`)
+* EC2 Virtual Private Cloud
+* EC2 Internet Gateway
+* EC2 subnet within the VPC
+* EC2 network routing table within the VPC for global access from all IPs
+* EC2 security group with ports 22/80/8888 ingress and * egress
+* EC2 public key
+* IAM role for EC2 access to services
+* IAMPo asfzlicyAttachment to above IAM role (policy specified in `variables.tf`)
+* **EC2 Instance** (name: `mainDevInstance`), provisioned with:
+  * local public SSH key for access (*via `file` provisioner*)
+  * Development docker image from DockerHub (specified in `variables.tf`), which is pushed to the newly-created `ps-images` ECR repo (*via `remote-exec` provisioner*)
+* IAM role for SageMaker to access AWS account resources (S3).
+* IAMPolicyAttachment to attach specific resource access to above role

deployment/aws_userpolicy.json

@@ -0,0 +1,66 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "Stmt1595008360505",
+            "Action": "s3:*",
+            "Effect": "Allow",
+            "Resource": "*"
+        },
+        {
+            "Sid": "Stmt1595008423533",
+            "Action": "sagemaker:*",
+            "Effect": "Allow",
+            "Resource": "*"
+        },
+        {
+            "Sid": "Stmt1595008489022",
+            "Action": [
+                "iam:AttachGroupPolicy",
+                "iam:AttachRolePolicy",
+                "iam:CreatePolicy",
+                "iam:CreateRole",
+                "iam:GetGroup",
+                "iam:GetPolicy",
+                "iam:GetRole",
+                "iam:GetRolePolicy",
+                "iam:GetUser",
+                "iam:ListPolicies",
+                "iam:ListRoles",
+                "iam:ListUserPolicies",
+                "iam:ListUsers",
+                "iam:PutRolePolicy",
+                "iam:PutUserPolicy",
+                "iam:UpdateRole",
+                "iam:CreateAccessKey",
+                "iam:CreateInstanceProfile",
+                "iam:ListAttachedRolePolicies",
+                "iam:GetPolicyVersion",
+                "iam:ListPolicyVersions",
+                "iam:DetachRolePolicy",
+                "iam:GetInstanceProfile",
+                "iam:DeletePolicy",
+                "iam:DeleteRole",
+                "iam:RemoveRoleFromInstanceProfile",
+                "iam:DeleteInstanceProfile",
+                "iam:AddRoleToInstanceProfile",
+                "iam:PassRole",
+                "iam:ListInstanceProfilesForRole"
+            ],
+            "Effect": "Allow",
+            "Resource": "*"
+        },
+        {
+            "Sid": "id",
+            "Action": "ecr:*",
+            "Effect": "Allow",
+            "Resource": "*"
+        },
+        {
+            "Sid": "Stmt1595008520486",
+            "Action": "ec2:*",
+            "Effect": "Allow",
+            "Resource": "*"
+        }
+    ]
+}