Merge pull request #1221 from iturgeon/iturgeon/polish-docker-scripts…

…-and-environ-for-deploys

polish docker scripts and clean up release builds

.travis.yml

@@ -1,11 +1,17 @@
 language: minimal
 services:
   - docker
-before_script: cd docker
+before_install:
+  # update docker
+  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+  - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+  - sudo apt-get update
+  - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+  - cd docker
 script:
   - bash run_tests_ci.sh
 after_success:
-  - bash run_build_release_package.sh
+  - bash run_build_github_release_package.sh
 deploy:
   skip_cleanup: true
   provider: releases

docker/.env

@@ -0,0 +1,22 @@
+# important: this tag must be updated
+# when theres a change to any dockerfile
+#
+# to update: use the next upcoming git tag
+# dockerub builds an image for every tag
+TAG=v5.0.0
+
+FUEL_ENV=development
+
+# Database settings
+DB_HOST=mysql
+MYSQL_ROOT_PASSWORD=drRoots
+MYSQL_USER=materia
+MYSQL_PASSWORD=odin
+MYSQL_DATABASE=materia
+
+# Fake S3
+INPUT_BUCKET=fakes3_uploads
+OUTPUT_BUCKET=fakes3_assets
+OUTPUT_MAX_DIMENSIONS=75x75, 0x0
+OUTPUT_BASE_KEY=media
+IS_FAKES3=True

docker/docker-compose.admin.yml

@@ -1,21 +1,28 @@
-version: '2'
+version: '3.2'
 # Docker Compose file meant for tasks like building assets and running tests
 services:
 
   phpfpm:
     environment:
       - FUEL_ENV=test
     volumes:
-      - static_files_test:/var/www/html/public/widget
+      - static_widget_files_test:/var/www/html/public/widget
       - uploaded_media_test:/var/www/html/fuel/packages/materia/media
     command: echo "yay"
 
   # We use a node container to compile js, css, etc
   node:
-    image: ucfopen/materia-node:latest
+    container_name: materia-node
+    image: ucfopen/materia-node:${TAG}
+    build:
+      context: ./dockerfiles
+      dockerfile: materia-node
+    networks:
+      - backend
     volumes:
       - ../:/var/www/html:rw
       - node_modules:/var/www/html/node_modules
+      - build_app_root:/build:rw
 
   mysql:
     environment:
@@ -28,6 +35,7 @@ services:
     #   - /var/lib/mysql
 
 volumes:
-  static_files_test: {}
-  uploaded_media_test: {}
+  build_app_root: {}
+  static_widget_files_test: {} # contain widgets installed in tests
+  uploaded_media_test: {} # contain files uploaded in tests
   node_modules: {} # node modules for npm

docker/docker-compose.yml

@@ -1,25 +1,36 @@
-version: '2'
+version: '3.2'
 
 services:
   nginx:
+    container_name: materia-nginx
     image: nginx:stable-alpine
     ports:
       - "80:80" # main materia
       - "8008:8008" # static files
+    networks:
+      - frontend
     volumes:
       - ./config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
       - ../public:/var/www/html/public:ro
     links:
       - phpfpm
 
   phpfpm:
-    image: ucfopen/materia-web-base:latest
+    container_name: materia-phpfpm
+    image: ucfopen/materia-web-base:${TAG} # TAG default value is in .env file
+    build:
+      context: ./dockerfiles
+      dockerfile: materia-web
     environment:
-      - FUEL_ENV=development
-      - DB_HOST=mysql
-      - DB_NAME=materia
-      - DB_USER=materia
-      - DB_PASS=odin
+      # See .env for default values
+      - FUEL_ENV
+      - DB_HOST
+      - DB_NAME=${MYSQL_DATABASE}
+      - DB_USER=${MYSQL_USER}
+      - DB_PASS=${MYSQL_DATABASE}
+    networks:
+      - frontend
+      - backend
     volumes:
       - ../:/var/www/html:rw
       - ./config/php/php.ini:/usr/local/etc/php/conf.d/php.ini:ro
@@ -30,33 +41,54 @@ services:
       - fakes3
 
   mysql:
+    container_name: materia-mysql
     image: mysql:5.7.18
     environment:
-      - MYSQL_ROOT_PASSWORD=drRoots
-      - MYSQL_USER=materia
-      - MYSQL_PASSWORD=odin
-      - MYSQL_DATABASE=materia
+      - MYSQL_ROOT_PASSWORD
+      - MYSQL_USER
+      - MYSQL_PASSWORD
+      - MYSQL_DATABASE
     ports:
       - "3306:3306" # allow mysql access from the host - use /etc/hosts to set mysql to your docker-machine ip
+    networks:
+      - backend
     volumes:
       - "./config/mysql/01_create_test.sql:/docker-entrypoint-initdb.d/01_create_test.sql"
 
   memcached:
+    container_name: materia-memcached
     image: memcached:1.4.27-alpine
+    networks:
+      - backend
 
   fakes3:
-    image: ucfopen/materia-fake-s3:latest
+    container_name: materia-fakes3
+    image: ucfopen/materia-fake-s3:${TAG} # TAG default value is in .env file
+    build:
+      context: .
+      dockerfile: ./dockerfiles/materia-fakes3
     environment:
-      - INPUT_BUCKET=fakes3_uploads
-      - OUTPUT_BUCKET=fakes3_assets
-      - OUTPUT_MAX_DIMENSIONS=75x75, 0x0
-      - OUTPUT_BASE_KEY=media
-      - IS_FAKES3=True
+      - INPUT_BUCKET
+      - OUTPUT_BUCKET
+      - OUTPUT_MAX_DIMENSIONS
+      - OUTPUT_BASE_KEY
+      - IS_FAKES3
     ports:
       - "10001:10001"
+    networks:
+      - frontend
+      - backend
     volumes:
       - uploaded_media:/s3mnt/fakes3_root/fakes3_uploads/media/
 
+networks:
+  frontend:
+    # ommited till we can eaily use composer v3.5 on travis
+    # name: materia_frontend
+  backend:
+    # ommited till we can eaily use composer v3.5 on travis
+    # name: materia_backend
+
 volumes:
   # static_files: {} # compiled js/css and uploaded widgets
   uploaded_media: {} # uploaded media files

docker/dockerfiles/materia-node

@@ -6,6 +6,7 @@ RUN apt-get update && apt-get install -y \
   fontconfig \
   git \
   python-dev \
+  zip \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 

docker/run_build_github_release_package.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+#######################################################
+# ABOUT THIS SCRIPT
+#
+# Install and build a base release package
+# This should try to include as many constructed
+# assets as possible to reduce the work needed
+# to deploy Materia. This build will not
+# disrupt the current files on disk -
+# ex: no need to install node # or npm packages
+# to build js - just include the js
+#
+# EX: ./run_build_release_package.sh
+#######################################################
+set -e
+
+# declare files that should have been created
+declare -a FILES_THAT_SHOULD_EXIST=(
+	"public/js/materia.enginecore.js"
+	"public/css/widget-play.css"
+)
+
+# declare files to omit from zip
+declare -a FILES_TO_EXCLUDE=(
+	".git*"
+	".gitignore"
+	"app.json"
+	"nginx_app.conf"
+	"Procfile"
+	"node_modules*"
+	"githooks"
+	"phpcs.xml"
+	"src*"
+	"fuel/app/config/development*"
+	"fuel/app/config/heroku*"
+	"fuel/app/config/test*"
+	"fuel/app/config/production*"
+	"public/widget*"
+	"githooks*"
+	"coverage.xml"
+	"coverage*"
+)
+
+# combine the files to exclude
+EXCLUDE=''
+for i in "${FILES_TO_EXCLUDE[@]}"
+do
+	EXCLUDE="$EXCLUDE --exclude=\"./$i\""
+done
+
+# store the docker compose command to shorten the following commands
+DC="docker-compose -f docker-compose.yml -f docker-compose.admin.yml"
+
+set -o xtrace
+
+# # stop and remove docker containers
+$DC down --volumes --remove-orphans
+
+$DC pull --ignore-pull-failures node
+
+# get rid of any left over package files
+rm -rf clean_build_clone || true
+rm -rf ../materia-pkg* || true
+git clone ../ ./clean_build_clone
+
+# gather build info
+DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+GITUSER=$(git config user.name)
+GITEMAIL=$(git config user.email)
+GITCOMMIT=$(cd clean_build_clone && git rev-parse HEAD)
+GITREMOTE=$(cd clean_build_clone&& git remote get-url origin)
+
+# remove .git dir for slightly faster copy
+rm -rf clean_build_clone/.git
+
+# start a build container
+$DC run --no-deps --detach --workdir /build/clean_build_clone --name materia-build node tail -f /dev/null
+
+# copy the clean build clone into the container
+docker cp ./clean_build_clone materia-build:/build
+
+# clean up
+rm -rf clean_build_clone || true
+
+# install production node_modules
+docker exec materia-build yarn install --frozen-lockfile --non-interactive --production
+
+# verify all files we expect to be created exist
+for i in "${FILES_THAT_SHOULD_EXIST[@]}"
+do
+	docker exec materia-build stat /build/clean_build_clone/$i
+done
+
+# zip, excluding some files
+docker exec materia-build  bash -c "zip -r $EXCLUDE ../materia-pkg.zip ./"
+
+# calulate hashes
+MD5=$(docker exec materia-build md5sum ../materia-pkg.zip)
+SHA1=$(docker exec materia-build sha1sum ../materia-pkg.zip)
+SHA256=$(docker exec materia-build sha256sum ../materia-pkg.zip)
+
+# copy zip file from container to host
+docker cp materia-build:/build/materia-pkg.zip ../materia-pkg.zip
+
+# write build info file
+echo "build_date: $DATE" > ../materia-pkg-build-info.yml
+echo "git: $GITREMOTE" >> ../materia-pkg-build-info.yml
+echo "git_version: $GITCOMMIT" >> ../materia-pkg-build-info.yml
+echo "git_user: $GITUSER" >> ../materia-pkg-build-info.yml
+echo "git_user_email: $GITEMAIL" >> ../materia-pkg-build-info.yml
+echo "sha1: $SHA1" >> ../materia-pkg-build-info.yml
+echo "sha256: $SHA256" >> ../materia-pkg-build-info.yml
+echo "md5: $MD5" >> ../materia-pkg-build-info.yml
+
+# clean environment and configs
+$DC down --volumes --remove-orphans