From 883e9590c2e94cc3d8a75057fa9ac5e81371e44f Mon Sep 17 00:00:00 2001 From: jzgo Date: Mon, 30 Aug 2021 10:16:09 -0700 Subject: [PATCH 001/150] Dockerize for local development --- .dockerignore | 10 + .env | 24 +++ .gitlab-ci.yml | 121 +++++++++++ .gitlab/issue_templates/Bug.md | 46 +++++ .gitlab/issue_templates/Feature.md | 41 ++++ .gitlab/issue_templates/Question.md | 18 ++ .gitlab/merge_request_templates/Bug.md | 36 ++++ .gitlab/merge_request_templates/Feature.md | 36 ++++ .sops.yaml | 3 + Dockerfile | 39 ++++ Gemfile | 8 +- Gemfile.lock | 14 +- README.md | 98 +++++++++ bin/decrypt-secrets | 22 ++ bin/encrypt-secrets | 19 ++ chart/.gitignore | 3 + chart/.helmignore | 23 +++ chart/Chart.yaml | 25 +++ chart/README.md | 223 +++++++++++++++++++++ chart/bin/check_sidekiq.rb | 0 chart/bin/decrypt | 17 ++ chart/bin/deploy | 14 ++ chart/bin/encrypt | 15 ++ chart/bin/remove | 15 ++ chart/templates/_helpers.tpl | 78 +++++++ chart/templates/fcrepo-deploy.yaml | 63 ++++++ chart/templates/fcrepo-env-cm.yaml | 8 + chart/templates/fcrepo-env-secret.yaml | 8 + chart/templates/fcrepo-pvc.yaml | 20 ++ chart/templates/fcrepo-svc.yaml | 19 ++ chart/templates/rails-env-cm.yaml | 28 +++ chart/templates/rails-env-secret.yaml | 10 + chart/templates/rails-pvc-shared.yml | 20 ++ chart/templates/setup-job.yaml | 73 +++++++ chart/templates/sidekiq-deploy.yaml | 80 ++++++++ chart/templates/web-deploy.yaml | 67 +++++++ chart/templates/web-ing-wildcard.yaml | 20 ++ chart/templates/web-ing.yaml | 20 ++ chart/templates/web-svc.yaml | 20 ++ config/database.yml | 43 ++-- config/environments/development.rb | 12 +- db/schema.rb | 114 +++++------ docker-compose.yml | 164 +++++++++++++++ ops/env.conf | 9 + ops/nginx.sh | 31 +++ ops/webapp.conf | 15 ++ ops/worker.sh | 8 + package.json | 2 +- 48 files changed, 1707 insertions(+), 95 deletions(-) create mode 100644 .dockerignore create mode 100644 .env create mode 100644 .gitlab-ci.yml create mode 100644 .gitlab/issue_templates/Bug.md create mode 100644 .gitlab/issue_templates/Feature.md create mode 100644 .gitlab/issue_templates/Question.md create mode 100644 .gitlab/merge_request_templates/Bug.md create mode 100644 .gitlab/merge_request_templates/Feature.md create mode 100644 .sops.yaml create mode 100644 Dockerfile create mode 100644 bin/decrypt-secrets create mode 100644 bin/encrypt-secrets create mode 100644 chart/.gitignore create mode 100644 chart/.helmignore create mode 100644 chart/Chart.yaml create mode 100644 chart/README.md create mode 100644 chart/bin/check_sidekiq.rb create mode 100644 chart/bin/decrypt create mode 100644 chart/bin/deploy create mode 100644 chart/bin/encrypt create mode 100644 chart/bin/remove create mode 100644 chart/templates/_helpers.tpl create mode 100644 chart/templates/fcrepo-deploy.yaml create mode 100644 chart/templates/fcrepo-env-cm.yaml create mode 100644 chart/templates/fcrepo-env-secret.yaml create mode 100644 chart/templates/fcrepo-pvc.yaml create mode 100644 chart/templates/fcrepo-svc.yaml create mode 100644 chart/templates/rails-env-cm.yaml create mode 100644 chart/templates/rails-env-secret.yaml create mode 100644 chart/templates/rails-pvc-shared.yml create mode 100644 chart/templates/setup-job.yaml create mode 100644 chart/templates/sidekiq-deploy.yaml create mode 100644 chart/templates/web-deploy.yaml create mode 100644 chart/templates/web-ing-wildcard.yaml create mode 100644 chart/templates/web-ing.yaml create mode 100644 chart/templates/web-svc.yaml create mode 100644 docker-compose.yml create mode 100644 ops/env.conf create mode 100644 ops/nginx.sh create mode 100644 ops/webapp.conf create mode 100644 ops/worker.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..bb0342712 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +.bundle +.convox +.env +.env.* +.ruby-* +db/*.sqlite3 +db/*.sqlite3-journal +log/* +node_modules +tmp diff --git a/.env b/.env new file mode 100644 index 000000000..c5b27695b --- /dev/null +++ b/.env @@ -0,0 +1,24 @@ +APP_NAME=GBH +PASSENGER_APP_ENV=development +REGISTRY_HOST=registry.gitlab.com +REGISTRY_URI=/notch8/gbh + +# SMTP Mailer variables +# To enable mailer: +# - Uncomment and edit SMTP vars +# - Uncomment SMTP Mailer section in respective config/environments file +# SMTP_USER_NAME=CHANGEME +# SMTP_PASSWORD=CHANGEME +# SMTP_ADDRESS=CHANGEME +# SMTP_DOMAIN=CHANGEME +# SMTP_PORT=CHANGEME +# SMTP_TYPE=CHA NGEME + +TAG=dev +MYSQL_DATABASE=gbh +MYSQL_USER=root +MYSQL_PASSWORD=DatabaseFTW +MYSQL_ROOT_PASSWORD=DatabaseFTW +MYSQL_HOST=db +DB_ADAPTER=mysql2 +TEST_DB=GBH_test diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..244972e0d --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,121 @@ +stages: +- build +- lint +- test +- deploy + +before_script: +- export TAG=${CI_COMMIT_SHORT_SHA} +- export BRANCH=${CI_COMMIT_REF_NAME} +- export REGISTRY_HOST=${CI_REGISTRY} +- export REGISTRY_URI="/${CI_PROJECT_PATH}" + +base: + stage: build + script: + - docker login -u "gitlab-ci-token" -p "$CI_JOB_TOKEN" $CI_REGISTRY + - sc build -s base + - sc push -s base + when: manual + tags: + - local + +build: + stage: build + script: + - docker login -u "gitlab-ci-token" -p "$CI_JOB_TOKEN" $CI_REGISTRY + - sc build + - sc push + - docker tag $CI_REGISTRY_IMAGE:$TAG $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME + tags: + - local + +lint: + stage: go + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA + variables: + PRONTO_GITLAB_API_ENDPOINT: "https://gitlab.com/api/v4" + before_script: + - export PRONTO_GITLAB_API_PRIVATE_TOKEN=$GITLAB_API_TOKEN + script: + - cd /home/app/webapp && bundle exec pronto run -f gitlab + tags: + - docker + +test: + stage: test + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA + services: + - mysql:5.7 + - name: solr:latest + entrypoint: ["docker-entrypoint.sh", "solr-precreate", "test"] + - redis:latest + - name: botimer/fcrepo:4.5.1 + + variables: + MYSQL_DATABASE: GBH + MYSQL_ROOT_PASSWORD: password + MYSQL_USER: GBH_test + MYSQL_PASSWORD: password + MYSQL_HOST: mysql + DATABASE_ADAPTER: mysql2 + DATABASE_HOST: mysql + DATABASE_NAME: GBH + DATABASE_PASSWORD: password + DATABASE_USER: GBH_test + TEST_DB: GBH + SITE_URI_BASE: TODO + script: + - cd /home/app/webapp && bundle exec rake db:create db:schema:load spec + tags: + - docker + variables: + GIT_STRATEGY: none + +review: + stage: review + type: deploy + environment: + name: $CI_COMMIT_REF_SLUG + url: http://web.$CI_PROJECT_NAME-$CI_COMMIT_REF_SLUG.staging.$SITE_URI_BASE + on_stop: stop_review + only: + - branches + except: + - main + script: + - ./chart/bin/deploy review $CI_COMMIT_REF_SLUG + tags: + - local + +stop_review: + stage: review + script: + - ./chart/bin/remove review $CI_COMMIT_REF_SLUG + + when: manual + environment: + name: $CI_PROJECT_NAME-$CI_BUILD_REF_SLUG + action: stop + +staging: + stage: staging + type: deploy + script: + - ./chart/bin/deploy staging $CI_COMMIT_REF_SLUG + only: + - main + tags: + - local + +production: + stage: production + type: deploy + script: + - ./chart/bin/deploy production $CI_COMMIT_REF_SLUG + when: manual + only: + - main + tags: + - local diff --git a/.gitlab/issue_templates/Bug.md b/.gitlab/issue_templates/Bug.md new file mode 100644 index 000000000..f2c8ce0d8 --- /dev/null +++ b/.gitlab/issue_templates/Bug.md @@ -0,0 +1,46 @@ + # Summary + +A short description of the bug. Use # to link to the parent issue or merge request + +# Observed Behavior + +The current, bugged behavior. + +# Expected Behavior + +How it should behave once it's fixed. What "done" looks like. + +# Screenshots or Video + +Include screenshots or videos that demonstrate the bug + +# Other Information + +Any other logs, related issues, merge requests, wireframes, documentation, slack conversations or anything else that would help the developer try to find and fix this issue, or provide context. + +# Acceptance Criteria + +This is what done looks like. A higher level checklist of the goals and functionality after this bug is fixed + +* [ ] I see these views +* [ ] When a user does this, that happens + +# Steps to Reproduce / Testing Instructions + +How did you trigger this bug? Include links to the page where the error occured, if applicable. + +* [ ] As a (logged in/admin/etc) User +* [ ] With these other conditions +* [ ] I did these things +* [ ] And observed this behavior + +## Further information + +Edit in any later issues or merge requests that are conencted to this issue, for future reference + +### Implementation suggestions + +Are nice, but should probably go in a comment. e.g. gems, unrelated issues that dealt with similar problems, stack overflow links. + +/label ~"Needs Rework", ~"Ready for Development" + diff --git a/.gitlab/issue_templates/Feature.md b/.gitlab/issue_templates/Feature.md new file mode 100644 index 000000000..5c48e6b1d --- /dev/null +++ b/.gitlab/issue_templates/Feature.md @@ -0,0 +1,41 @@ +# Summary + +A short description of the feature + +# User Stories + +What problem is this feature going to solve? + +# Screenshots or Video + +Include any relevant media such as wireframes, screenshots, or videos + +# Relationships + +Are there any other issues or merge requests that are related to this feature (use # to generate a quick link). In particular, is this part of a larger epic, or does it depend on any other tickets being resolved first? Do you need anything else or are you making any other assumptions for this feature? + +# Acceptance Criteria + +This is what done looks like. A higher level checklist of the goals and functionality of this feature + +* [ ] I see these views +* [ ] When a user does this, that happens + +# Testing Instructions + +Steps to demonstrate this feature is properly working. Include a video, if possible. The How to the Acceptance Criteria's What + +* [ ] As a (user/admin) +* [ ] Go here and do these things +* [ ] To see this happen + + +## Further information + +Edit in any later issues or merge requests that are conencted to this issue, for future reference + +### Implimentation Suggestions + +Any development details that would be helpful when working on this feature e.g. gems, unrelated issues that dealt with similar problems, stack overflow links. + +/label ~Icebox \ No newline at end of file diff --git a/.gitlab/issue_templates/Question.md b/.gitlab/issue_templates/Question.md new file mode 100644 index 000000000..91c0f978f --- /dev/null +++ b/.gitlab/issue_templates/Question.md @@ -0,0 +1,18 @@ +# Summary +A short description of the question. + +## Who has this information +Consider using @ to tag them, if they're in this repository + +# Additional information +More details about what needs to be answered +* [ ] Checkboxes are helpful +* [ ] To make sure all answers are enumerated + +# Other links +Pictures, videos, links to issues, or anything else that might be helpful to provide context for the question + +/label ~"Planning in Progress" + +/assign me + \ No newline at end of file diff --git a/.gitlab/merge_request_templates/Bug.md b/.gitlab/merge_request_templates/Bug.md new file mode 100644 index 000000000..24fde3f65 --- /dev/null +++ b/.gitlab/merge_request_templates/Bug.md @@ -0,0 +1,36 @@ +# Summary +A short description of the bug + +# Related Issue +Use # to add a quick link to the parent issue(s), or ! for parent merge requests. Also, go to the issue and edit it to mention this MR. + +# Expected Behavior + +Demo how this merge request is working in the app. Consult the steps for reproducing the bug in the issue. + +# Screenshots / Video +Include screenshots or video demonstrating how this bug fix works + +# Dependencies + +Are there any other outstanding issues which need to be resolved before this can be merged? + +# Special Attention / Side Effects + +Is there anything you would like to have special scrutiny given to during code review? Will the changes in this merge request potentially impact any thing else that isn't explicitly part of this bug fix? + +# Testing / Reproduction instructions + +Step by step instructions on how to test this bug fix. Include links and username/passswords if necessary. Videos are especially helpful. Also, name any spec files that should be run for this bug fix, or to test any side effects from this bug fix. + +1. Go here +2. Do this +3. Observe this behavior + +# Deployment + +Where is this deployed? Are there any additional steps to get this working on staging/production? (e.g. setting up enviromental variables) + +# Other Information + +Any other logs, related issues, merge requests, wireframes, documentation, slack conversations or anything else that would help the developer try to find and fix this issue, or provide context. \ No newline at end of file diff --git a/.gitlab/merge_request_templates/Feature.md b/.gitlab/merge_request_templates/Feature.md new file mode 100644 index 000000000..2644820c0 --- /dev/null +++ b/.gitlab/merge_request_templates/Feature.md @@ -0,0 +1,36 @@ +# Summary +A short description of the feature + +# Related Issue +Use # to add a quick link to the parent issue(s), or ! for parent merge requests. Also, go to that issue and edit the description to mention this MR. + +# Expected Behavior + +Demo how this merge request is working in the app. Consult the acceptance criteria in the issue. + +# Screenshots / Video +Include screenshots or video demonstrating how this feature works + +# Dependencies + +Are there any other outstanding issues which need to be resolved before this can be merged? + +# Special Attention / Side Effects + +Is there anything you would like to have special scrutiny given to during code review? Will the changes in this merge request potentially impact any thing else that isn't explicitly part of this feature? + +# Testing / Reproduction instructions + +Step by step instructions on how to test this feature. Include links and username/passswords if necessary. Videos are especially helpful. Also, name any spec files that should be run for this feature, or to test any side effects from this feature. + +1. Go here +2. Do this +3. Observe this behavior + +# Deployment + +Where is this deployed? Are there any additional steps to get this working on staging/production? (e.g. setting up enviromental variables) + +# Other Information + +Any other logs, related issues, merge requests, wireframes, documentation, slack conversations or anything else that would help the developer try to find and fix this issue, or provide context. \ No newline at end of file diff --git a/.sops.yaml b/.sops.yaml new file mode 100644 index 000000000..e536b9fdd --- /dev/null +++ b/.sops.yaml @@ -0,0 +1,3 @@ +--- +creation_rules: + - pgp: "CHANGEME" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..c24f2aeec --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +ARG HYRAX_IMAGE_VERSION=3.0.1 +FROM ghcr.io/samvera/hyrax/hyrax-base:$HYRAX_IMAGE_VERSION as hyku-base + +USER root + +ARG EXTRA_APK_PACKAGES="openjdk11-jre ffmpeg" +RUN apk --no-cache upgrade && \ + apk --no-cache add \ + curl \ + curl-dev \ + libcurl \ + libxml2-dev \ + mariadb-dev \ + mediainfo \ + # --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/main/ nodejs=12.9.0 npm \ + perl \ + cmake \ + $EXTRA_APK_PACKAGES + +USER app + +RUN mkdir -p /app/fits && \ + cd /app/fits && \ + wget https://github.com/harvard-lts/fits/releases/download/1.5.0/fits-1.5.0.zip -O fits.zip && \ + unzip fits.zip && \ + rm fits.zip && \ + chmod a+x /app/fits/fits.sh +ENV PATH="${PATH}:/app/fits" + +COPY --chown=1001:101 $APP_PATH/Gemfile* /app/samvera/hyrax-webapp/ +RUN bundle install --jobs "$(nproc)" + +COPY --chown=1001:101 $APP_PATH /app/samvera/hyrax-webapp + +RUN RAILS_ENV=production SECRET_KEY_BASE=`bin/rake secret` DB_ADAPTER=nulldb bundle exec rake assets:precompile + +FROM hyku-base as hyku-worker +ENV MALLOC_ARENA_MAX=2 +CMD bundle exec sidekiq \ No newline at end of file diff --git a/Gemfile b/Gemfile index 72faa9873..78d2525a6 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,5 @@ source 'https://rubygems.org' -ruby '2.5.3' - git_source(:github) do |repo_name| repo_name = "#{repo_name}/#{repo_name}" unless repo_name.include?("/") "https://github.com/#{repo_name}.git" @@ -36,12 +34,12 @@ gem 'hydra-role-management', '~> 1.0' # gem 'capistrano-rails', group: :development group :production do - gem 'mysql2', '~> 0.4.10' + gem 'mysql2', '~> 0.5.0' end group :development, :test do # Use sqlite3 as the database for Active Record - gem 'sqlite3', '1.3.13' + # gem 'sqlite3', '1.3.13' gem 'capybara-screenshot' gem 'rspec', "~> 3.7" gem 'rspec-rails', "~> 3.7" @@ -94,7 +92,6 @@ gem 'sony_ci_api', '~> 0.2.1' # gem 'hyrax-iiif_av', github: 'samvera-labs/hyrax-iiif_av', branch: 'hyrax_master' gem 'webpacker' gem 'react-rails' -gem 'faker' gem 'database_cleaner' gem 'redlock', '~> 1.0' gem 'httparty', '~> 0.18' @@ -102,3 +99,4 @@ gem 'httparty', '~> 0.18' # Adding pry to all environments, because it's very useful for debugging # production environments on demo instances. gem 'pry-byebug', platforms: [:mri, :mingw, :x64_mingw] +gem 'activerecord-nulldb-adapter' \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index 53292ee6b..ed99918e1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -67,6 +67,8 @@ GEM arel (~> 8.0) activerecord-import (1.0.7) activerecord (>= 3.2) + activerecord-nulldb-adapter (0.4.0) + activerecord (>= 2.0.0) activesupport (5.1.7) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) @@ -562,7 +564,7 @@ GEM multi_json (1.15.0) multi_xml (0.6.0) multipart-post (2.1.1) - mysql2 (0.4.10) + mysql2 (0.5.3) nest (3.2.0) redic net-http-persistent (3.1.0) @@ -873,7 +875,6 @@ GEM actionpack (>= 4.0) activesupport (>= 4.0) sprockets (>= 3.0.0) - sqlite3 (1.3.13) ssrf_filter (1.0.7) sxp (1.1.0) rdf (~> 3.1) @@ -930,6 +931,7 @@ PLATFORMS ruby DEPENDENCIES + activerecord-nulldb-adapter aws-sdk-codedeploy aws-sdk-s3 bixby @@ -955,7 +957,7 @@ DEPENDENCIES jquery-rails letter_opener listen (>= 3.0.5, < 3.2) - mysql2 (~> 0.4.10) + mysql2 (~> 0.5.0) nokogiri pbcore (~> 0.3.0) pry-byebug @@ -977,7 +979,6 @@ DEPENDENCIES simple_form (= 5.0.0) solr_wrapper (~> 2.1) sony_ci_api (~> 0.2.1) - sqlite3 (= 1.3.13) turbolinks (~> 5) uglifier (>= 1.3.0) web-console (>= 3.3.0) @@ -986,8 +987,5 @@ DEPENDENCIES webpacker xray-rails -RUBY VERSION - ruby 2.5.3p105 - BUNDLED WITH - 2.0.2 + 2.2.16 diff --git a/README.md b/README.md index e6c35d5fe..f163c6809 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,101 @@ +[Docker development setup](#docker-development-setup) + +[Bash into the container](#bash-into-the-container) + +[Handling Secrets with SOPS](#handling-secrets-with-sops) + +[Deploy a new release](#deploy-a-new-release) + +[Run import from admin page](#run-import-from-admin-page) + +# Docker development setup + +We recommend committing .env to your repo with good defaults. .env.development, .env.production etc can be used for local overrides and should not be in the repo. See [Handling Secrets with SOPS](#handling-secrets-with-sops) for how to manage secrets. + +1) Install Docker.app + +2) Install stack car + ``` bash + gem install stack_car + ``` + +3) Sign in with dory + ``` bash + dory up + ``` + +4) Install dependencies + ``` bash + yarn install + ``` + +5) Start the server + ``` bash + sc up + ``` + +6) Load and seed the database + ``` bash + sc be rake db:migrate db: seed + ``` +### Troubleshooting Docker Development Setup +Confirm or configure settings. Sub your information for the examples. +``` bash +git config --global user.name example +git config --global user.email example@example.com +docker login registry.gitlab.com +``` + +### While in the container you can do the following +- Run rspec + ``` bash + bundle exec rspec + ``` +- Access the rails console + ``` bash + bundle exec rails c + ``` + +### Handling Secrets with SOPS + +[**SOPS**](https://github.com/mozilla/sops) is used to handle this project's secrets. + +The secrets in this repository include: +- `.env*` files +- `*-values.yaml` files + +Scripts (`bin/decrypt-secrets` and `bin/encrypt-secrets`) are included in this project to help with managing secrets. + +**To decrypt secrets**: + +You will need to do this if you are new to the project or there have been changes to any secrets files that are required for development. + +In terminal: +```bash +bin/decrypt-secrets +``` + +This will find and decrypt files with the `.enc` extension. + +**To encrypt secrets**: + +You will need to do this when you have edited secrets and are ready to commit them. + +In terminal: +```bash +bin/encrypt-secrets +``` + +This will find and output an encrypted version of secret files with an `.enc` extension. + +# Deploy a new release + +``` bash +sc release {staging | production} # creates and pushes the correct tags +sc deploy {staging | production} # deployes those tags to the server +``` + +Release and Deployment are handled by the gitlab ci by default. See ops/deploy-app to deploy from locally, but note all Rancher install pull the currently tagged registry image # ams Archival Management System to support the American Archive of Public Broadcasting diff --git a/bin/decrypt-secrets b/bin/decrypt-secrets new file mode 100644 index 000000000..955415d3b --- /dev/null +++ b/bin/decrypt-secrets @@ -0,0 +1,22 @@ +#!/usr/bin/env ruby + +# require 'byebug' + +parent_dir = File.dirname(__dir__) +Dir.chdir(File.join(parent_dir)) +[ + ".env.*", + "chart/*-values.yaml", + "ops/kube_config.yml", + "ops/.backend", + "ops/k8s/*-values.yaml" +].each do |files| + Dir.glob(files).each do |file| + if file.match(/enc/) + next unless File.exists?(file) + cmd = "sops --decrypt #{file} > #{file.gsub(/.enc$/, '')}" + puts cmd + `#{cmd}` + end + end +end \ No newline at end of file diff --git a/bin/encrypt-secrets b/bin/encrypt-secrets new file mode 100644 index 000000000..f37487927 --- /dev/null +++ b/bin/encrypt-secrets @@ -0,0 +1,19 @@ +#!/usr/bin/env ruby + +# require 'byebug' + +parent_dir = File.dirname(__dir__) +[ + ".env.*", + "chart/*-values.yaml", + "ops/kube_config.yml", + "ops/.backend", + "ops/k8s/*-values.yaml" +].each do |files| + Dir.glob(files).each do |file| + next if /enc/.match?(file) + cmd = "sops --encrypt #{file} > #{file}.enc" + puts cmd + `#{cmd}` + end +end \ No newline at end of file diff --git a/chart/.gitignore b/chart/.gitignore new file mode 100644 index 000000000..5798eef83 --- /dev/null +++ b/chart/.gitignore @@ -0,0 +1,3 @@ +server.pem +server.key +*values.yaml \ No newline at end of file diff --git a/chart/.helmignore b/chart/.helmignore new file mode 100644 index 000000000..48f87622e --- /dev/null +++ b/chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +server.pem +server.key diff --git a/chart/Chart.yaml b/chart/Chart.yaml new file mode 100644 index 000000000..f5aec2afb --- /dev/null +++ b/chart/Chart.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +appVersion: "0.0.1" +description: A Helm chart for GBH +name: GBH +version: 0.0.1 +dependencies: + +- name: solr + version: 1.5.2 + repository: https://charts.helm.sh/incubator + condition: solr.enabled + + +- name: redis + version: 11.0.4 + repository: https://charts.bitnami.com/bitnami + condition: redis.enabled + + + +- name: mariadb + version: 7.3.14 + repository: https://charts.helm.sh/stable + condition: mariadb.enabled + diff --git a/chart/README.md b/chart/README.md new file mode 100644 index 000000000..053c03ecd --- /dev/null +++ b/chart/README.md @@ -0,0 +1,223 @@ +Helm Chart +========== + +This is a Rails Helm Chart which can be used to deploy a Rails instance to a Kubernetes cluster. + +# Requirements + +* helm +``` +brew install helm +``` + +* kubernetes +Kubectl is the command line tool for controlling Kubernetes clusters. It is available via (https://docs.docker.com/docker-for-mac/)[Docker for Mac] + +Alternatively: +``` +brew install kubectl +``` + +# Getting Started Locally using Docker for Mac + +## Setup + +Install Docker for Mac (DfM) + +Enable the Kubernetes Cluster in the DfM Settings + +In the menu bar item for DfM you'll 'Kubernetes', this will list the available clusters. For local deployment make sure docker-desktop is selected. + +## KubeConfig + +Kubernetetes creates a config file at `~/.kube/config`. When we come to setting up access to external clusters, we will be editing this file. That will add clusters to the DfM Kubernetes list. Remember that if you are running deployment actions using helm or kubectl they will use the cluster selected in that list, so if you were deploying to a production server yesterday, that will still be selected. It is a good practice to run `kubectl cluster-info` or `kubectl config current-context` before starting any deployment to make sure you are deploying to the right cluster. + +## GitLab Secret + +To pull images from a private registry, you'll need a secret + +For GitLab, create a Personal Access Token in GitLab with read access. + +Create your secret (called gitlab) in kubectl, substituting the items in {} with your data: +``` +create secret docker-registry gitlab --docker-server=https://registry.gitlab.com --docker-username={YOUR USERNAME} --docker-password={PERSONAL ACCESS TOKEN} --docker-email={YOUR EMAIL} --namespace {NAMESPACE eg. hyku-staging} +``` + +Reference the secret in `imagePullSecrets`, see the sample.yamnl file for an example. + +For other private registries, please consult their documentation on access tokens. + +## TLS Secret + +We also need to setup a secret for TLS certificates. + +``` +# this command will generate self signed server certificate and key: server.pem, server.key +# key and cert are stored in Secret object named `demoapp-puma-tls`. +# you can confirm this object by `kubectl describe secret demoapp-puma-tls` +export COMMON_NAME=localhost +openssl req -new -x509 -nodes -keyout server.key -days 3650 \ + -subj "/CN=${COMMON_NAME}" \ + -extensions v3_req \ + -config <(cat openssl.conf | sed s/\${COMMON_NAME}/$COMMON_NAME/) > server.pem +``` + +NOTE: you may need change openssl.conf to point to your local path, eg. /System/Library/OpenSSL/openssl.cnf + +``` +kubectl create secret tls demoapp-puma-tls --key server.key --cert server.pem +``` + +## Add Helm Chart Repository + +We are going to need to install a couple of things on our local cluster. For this we need to install charts from the Helm stable chart repository. + +One off installation of the repository: +``` +helm repo add stable https://kubernetes-charts.storage.googleapis.com +``` + +## Install NFS + +To run locally and use NFS file mounts we'll need an NFS server: + +Helm install to run the nfs server on kubernetes: +``` +helm install stable/nfs-server-provisioner --generate-name +``` + +NOTE: you can substitute --generate-name with --name followed by your chosen name for the resource + +NOTE: stop / remove it with helm uninstall {name} --namespace default + +### Ingress + +To run locally we'll need an Ingress controller - this provides us with the ability to access the application on the web: + +Helm install to run the ingress controller on kubernetes: +``` +helm install stable/nginx-ingress --generate-name +``` + +NOTE: you can substitute --generate-name with --name followed by your chosen name for the resource + +NOTE: stop / remove it with helm uninstall {name} --namespace default + +## Values + +When deploying the Helm chart we will provide a yaml file containing various configurations choices. + +A sample values file is provided to give defaults: `sample.yaml`. Copy this file (eg. to development-values.yamnl) and change values as appropriate. + +**Handling values files** + +Since values files are likely to contain sensitive information like API keys, they are included in `.gitignore` and MUST NOT be added to the repository. Encrypt the file before committing them to the repository, using the provided bin scripts in this directory. + +Example workflow (given values file is already created): +- Edit values file +- `chart/bin/encrypt staging ` + - This command will create `staging-values.yaml.enc` +- `git add staging-values.yaml.enc` +- Commit and push + +When pulling down a repo or branch, you will need to start by decrypting. + +Example: +- `chart/bin/decrypt staging` + +## Deploy using Helm + +From ./chart/ + +``` +./bin/deploy development latest +``` + +Open demoapp in browser +``` +open locaallhost +``` + +## Cleanup +helm uninstall development --namespace REPO_NAME + +eg. `helm uninstall development --namespace project-env` + +Tip: add the --dry-run to see what will be deleted + +## Kubernetes Dashboard + +Kubernetes provides a web-based dashboard for viewing and managing the deployed resources. + +# Install it: +``` +helm install stable/kubernetes-dashboard --generate-name +``` + +Make a note of the start command printed on install. It includes the release name (eg. kubernetes-dashboard-1579333192). + +Tip: You can replace --generate-name with --name and supply a name for the release to give you a stable name. + +Start it: +``` +(RELEASE_NAME will be the value from your installation - find it with helm ls) + +export POD_NAME=$(kubectl get pods -n default -l "app=kubernetes-dashboard,release=RELEASE_NAME" -o jsonpath="{.items[0].metadata.name}") +echo https://127.0.0.1:8443/ +kubectl -n default port-forward $POD_NAME 8443:8443 +``` + +Open it: +``` +https://127.0.0.1:8443/ +``` + +It will ask you to login by one of two methods. Opt for 'access token'. + +Print your access token in a console as follwos, and then copy paste it into the token box on the dashboard login: +``` +kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | awk '/^deployment-controller-token-/{print $1}') | awk '$1=="token:"{print $2}' +``` + +# Deploying to Staging and Production clusters + +Staging and Production deployment require the following steps: + +1. Add the necessary kube config for your remote cluster + +2. Switch kubernetes context + +Either using the list in DfM Kubernetes, or with the following: + +``` +# check the current context +kubectl config current-context +# find the context you want in the list +kubectl config get-contexts +# switch +kubectl config use-context CONTEXT_NAME +``` + +3. Setup the *-values.yaml for staging or production + +4. Deploy + +``` +# bin/deploy ENVIRONMENT TAG +bin/deploy staging latest +``` + +NOTE: the TAG will be used to pull the latest image from the GitLab repository. If the code has changed, make sure it's been pushed and the tagged image in the repository updated. + +The namespace will be set to the git repository name, eg. project-env. Make sure the namespace exists in your cluster. Create it with `kubectl create namespace project-env` + +# Troubleshooting + +The Kubernetes Dashboard (locally) allows you to view logs and access a shell session. If problems occur during deployment, there is an event history that can provide more information. + +There are equivalent kubectl commands for logs and accessing a shell, eg. + +``` +kubectl kubectl exec -it POD --namespace NAMESPACE -- /bin/bash +kubectl kubectl logs POD --namespace NAMESPACE +``` \ No newline at end of file diff --git a/chart/bin/check_sidekiq.rb b/chart/bin/check_sidekiq.rb new file mode 100644 index 000000000..e69de29bb diff --git a/chart/bin/decrypt b/chart/bin/decrypt new file mode 100644 index 000000000..97a3bfc8f --- /dev/null +++ b/chart/bin/decrypt @@ -0,0 +1,17 @@ +#!/bin/bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $DIR/../../chart + +echo $DIR + +REPO=$(basename $(git config --get remote.origin.url)) +NAMESPACE=${REPO%.git} + +if [ -z "$1" ] +then + echo './chart/bin/decrypt ENVIRONMENT' + exit 1 +fi + +keybase decrypt -i $1-values.yaml.enc -o $1-values.yaml + diff --git a/chart/bin/deploy b/chart/bin/deploy new file mode 100644 index 000000000..e60ecf124 --- /dev/null +++ b/chart/bin/deploy @@ -0,0 +1,14 @@ +#!/bin/bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $DIR/../../chart + +REPO=$(basename $(git config --get remote.origin.url)) +NAMESPACE=${REPO%.git} + +if [ -z "$1" ] || [ -z "$2" ] +then + echo './chart/bin/deploy ENVIRONMENT TAG' + exit 1 +fi + +helm upgrade --install --namespace $NAMESPACE-$1 $1 . -f $1-values.yaml --set rails.image.tag=$2 diff --git a/chart/bin/encrypt b/chart/bin/encrypt new file mode 100644 index 000000000..8da5420ce --- /dev/null +++ b/chart/bin/encrypt @@ -0,0 +1,15 @@ +#!/bin/bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $DIR/../../chart + +REPO=$(basename $(git config --get remote.origin.url)) +NAMESPACE=${REPO%.git} + +if [ -z "$1" ] || [ -z "$2" ] +then + echo './chart/bin/encrypt ENVIRONMENT TEAM' + exit 1 +fi + +keybase encrypt -i $1-values.yaml -o $1-values.yaml.enc --team $2 + diff --git a/chart/bin/remove b/chart/bin/remove new file mode 100644 index 000000000..42de547d3 --- /dev/null +++ b/chart/bin/remove @@ -0,0 +1,15 @@ +#!/bin/bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $DIR/../../chart + +REPO=$(basename $(git config --get remote.origin.url)) +NAMESPACE=${REPO%.git} + +if [ -z "$1" ] || [ -z "$2" ] +then + echo './chart/bin/remove ENVIRONMENT TAG' + exit 1 +fi + +raise 'refusing to remove production' if $1 == 'production' +helm uninstall --namespace $NAMESPACE-$1 $1 . -f $1-values.yaml --set rails.image.tag=$2 diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl new file mode 100644 index 000000000..74927e1a8 --- /dev/null +++ b/chart/templates/_helpers.tpl @@ -0,0 +1,78 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "app.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "app.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "app.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Shorthand for component names +*/}} + + +{{- define "app.redis.name" -}} +{{- .Release.Name -}}-redis-primary +{{- end -}} + + +{{- define "app.sidekiq.name" -}} +{{- include "app.fullname" . -}}-sidekiq +{{- end -}} + +{{- define "app.web.name" -}} +{{- include "app.fullname" . -}}-web +{{- end -}} +{{- define "app.rails-env.name" -}} +{{- include "app.fullname" . -}}-rails-env +{{- end -}} +{{- define "app.setup.name" -}} +{{- include "app.fullname" . -}}-setup +{{- end -}} + +{{- define "app.zookeeper.name" -}} +{{- include "solr.zookeeper-service-name" . -}} +{{- end -}} +{{- define "app.zookeeper-env.name" -}} +{{- include "app.fullname" . -}}-zookeeper-env +{{- end -}} +{{- define "app.solr.name" -}} +{{- .Release.Name -}}-solr-svc +{{- end -}} +{{- define "app.solr.collection" -}} +{{- if eq .Values.env.configmap.SETTINGS__MULTITENANCY__ENABLED false }}single{{- end -}} +{{- end -}} + + +{{- define "app.fcrepo.name" -}} +{{- include "app.fullname" . -}}-fcrepo +{{- end -}} +{{- define "app.fcrepo-env.name" -}} +{{- include "app.fullname" . -}}-fcrepo-env +{{- end -}} + diff --git a/chart/templates/fcrepo-deploy.yaml b/chart/templates/fcrepo-deploy.yaml new file mode 100644 index 000000000..4abf44160 --- /dev/null +++ b/chart/templates/fcrepo-deploy.yaml @@ -0,0 +1,63 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "app.fcrepo.name" . }} + labels: + app: {{ template "app.name" . }} + chart: {{ template "app.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + component: fcrepo +spec: + replicas: 1 + selector: + matchLabels: + app: {{ template "app.name" . }} + release: {{ .Release.Name }} + component: fcrepo + strategy: + type: Recreate + template: + metadata: + labels: + app: {{ template "app.name" . }} + release: {{ .Release.Name }} + component: fcrepo + spec: + restartPolicy: Always + {{- if .Values.fcrepo.imagePullSecrets }} + imagePullSecrets: + {{ toYaml .Values.fcrepo.imagePullSecrets }} + {{- end }} + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ template "app.fcrepo.name" . }} + initContainers: + - name: "remove-lost-found" + image: "busybox:1.25.0" + command: + - rm + - -fr + - /data/lost+found + volumeMounts: + - name: data + mountPath: /data + containers: + - name: fcrepo + image: {{ .Values.fcrepo.image.repository }}:{{ .Values.fcrepo.image.tag }} + livenessProbe: + tcpSocket: + port: 8080 + readinessProbe: + tcpSocket: + port: 8080 + volumeMounts: + - name: data + mountPath: /data + envFrom: + - configMapRef: + name: {{ template "app.fcrepo-env.name" . }} + - secretRef: + name: {{ template "app.fcrepo-env.name" . }} diff --git a/chart/templates/fcrepo-env-cm.yaml b/chart/templates/fcrepo-env-cm.yaml new file mode 100644 index 000000000..82a604b71 --- /dev/null +++ b/chart/templates/fcrepo-env-cm.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "app.fcrepo-env.name" . }} +data: + DATABASE_USER: {{ .Values.env.configmap.DATABASE_USER }} + DATABASE_NAME: {{ .Values.env.configmap.FC_DATABASE_NAME }} diff --git a/chart/templates/fcrepo-env-secret.yaml b/chart/templates/fcrepo-env-secret.yaml new file mode 100644 index 000000000..4a04bc65b --- /dev/null +++ b/chart/templates/fcrepo-env-secret.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "app.fcrepo-env.name" . }} +data: + DATABASE_PASSWORD: {{ .Values.env.secret.DATABASE_PASSWORD | b64enc }} + diff --git a/chart/templates/fcrepo-pvc.yaml b/chart/templates/fcrepo-pvc.yaml new file mode 100644 index 000000000..d78ef5ba5 --- /dev/null +++ b/chart/templates/fcrepo-pvc.yaml @@ -0,0 +1,20 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ template "app.fcrepo.name" . }} + labels: + app: {{ template "app.name" . }} + chart: {{ template "app.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + component: fcrepo +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.fcrepo.storage.size }} + {{- if .Values.fcrepo.storage.className }} + storageClassName: "{{ .Values.fcrepo.storage.ClassName }}" + {{- end }} diff --git a/chart/templates/fcrepo-svc.yaml b/chart/templates/fcrepo-svc.yaml new file mode 100644 index 000000000..78531c0b5 --- /dev/null +++ b/chart/templates/fcrepo-svc.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "app.fcrepo.name" . }} + labels: + app: {{ template "app.name" . }} + chart: {{ template "app.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + component: fcrepo +spec: + ports: + - protocol: TCP + port: 8080 + selector: + app: {{ template "app.name" . }} + release: {{ .Release.Name }} + component: fcrepo diff --git a/chart/templates/rails-env-cm.yaml b/chart/templates/rails-env-cm.yaml new file mode 100644 index 000000000..b7bc56649 --- /dev/null +++ b/chart/templates/rails-env-cm.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "app.rails-env.name" . }} +data: + + + FEDORA_URL: http://{{ template "app.fcrepo.name" . }}:8080/fcrepo/rest + + + RAILS_CACHE_STORE_URL: redis://{{ template "app.redis.name" . }}:6379/1 + REDIS_HOST: {{ template "app.redis.name" . }} + REDIS_URL: redis://{{ template "app.redis.name" . }}:6379/1 + + + SIDEKIQ_TIMEOUT: {{ .Values.sidekiq.timeout | quote }} + + SOLR_URL: http://{{ template "app.solr.name" . }}:8983/solr/ + SETTINGS__SOLR__URL: http://{{ template "app.solr.name" . }}:8983/solr/ + SETTINGS__ZOOKEEPER__CONNECTION_STR: {{ template "app.zookeeper.name" . }}:2181/configs + + +{{- range $key, $value := .Values.env.configmap }} + + {{ $key }}: {{ $value | quote }} + +{{- end }} diff --git a/chart/templates/rails-env-secret.yaml b/chart/templates/rails-env-secret.yaml new file mode 100644 index 000000000..a11bdb3ae --- /dev/null +++ b/chart/templates/rails-env-secret.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "app.rails-env.name" . }} +data: +{{- range $key, $value := .Values.env.secret }} + {{ $key }}: {{ $value |b64enc }} +{{- end }} + diff --git a/chart/templates/rails-pvc-shared.yml b/chart/templates/rails-pvc-shared.yml new file mode 100644 index 000000000..2be4e1254 --- /dev/null +++ b/chart/templates/rails-pvc-shared.yml @@ -0,0 +1,20 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ template "app.rails-env.name" . }}-shared + labels: + app: {{ template "app.name" . }} + chart: {{ template "app.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + component: rails +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: {{ .Values.rails.shared.storage.size }} + {{- if .Values.rails.shared.storage.className }} + storageClassName: "{{ .Values.rails.shared.storage.className }}" + {{- end }} diff --git a/chart/templates/setup-job.yaml b/chart/templates/setup-job.yaml new file mode 100644 index 000000000..09d1ec19b --- /dev/null +++ b/chart/templates/setup-job.yaml @@ -0,0 +1,73 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "app.setup.name" . }} + labels: + app: {{ template "app.name" . }} + chart: {{ template "app.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": post-install,pre-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": before-hook-creation +spec: + backoffLimit: 4 + template: + metadata: + labels: + app: {{ template "app.name" . }} + release: {{ .Release.Name }} + spec: + restartPolicy: Never + {{- if .Values.rails.imagePullSecrets }} + imagePullSecrets: + {{ toYaml .Values.rails.imagePullSecrets }} + {{- end }} + volumes: + - name: shared + persistentVolumeClaim: + claimName: {{ template "app.rails-env.name" . }}-shared +<% if options[:solr] %> + initContainers: + - name: check-solr-ready + image: {{ .Values.rails.image.repository }}:{{ .Values.rails.image.tag }} + command: ["/bin/bash"] + args: + - "-l" + - "-c" + - "response=0 && until [ $response == 200 ]; do response=$(curl --write-out %{http_code} --silent --output /dev/null $(echo $SETTINGS__SOLR__URL)admin/collections?action=LIST); echo Response is $response; sleep 5; done;" + envFrom: + - configMapRef: + name: {{ template "app.rails-env.name" . }} +<% end %> + containers: + - name: setup + image: {{ .Values.rails.image.repository }}:{{ .Values.rails.image.tag }} + volumeMounts: + - mountPath: /home/app/webapp/public/assets + name: shared + subPath: assets + - mountPath: /home/app/webapp/public/packs + name: shared + subPath: packs + - mountPath: /home/app/webapp/public/system + name: shared + subPath: system + command: ["/bin/bash"] + args: + - "-l" + - "-c" +<% if options[:solr] %> + - "(bundle check || bundle install) && bundle exec rails db:create zookeeper:upload db:migrate db:seed && RAILS_ENV=production bundle exec rails assets:precompile DATABASE_ADAPTER=nulldb && echo SETUP COMPLETE" +<% else %> + - "(bundle check || bundle install) && bundle exec rails db:create db:migrate db:seed && RAILS_ENV=production bundle exec rails assets:precompile DATABASE_ADAPTER=nulldb && echo SETUP COMPLETE" +<% end %> + envFrom: + - configMapRef: + name: {{ template "app.rails-env.name" . }} + - secretRef: + name: {{ template "app.rails-env.name" . }} diff --git a/chart/templates/sidekiq-deploy.yaml b/chart/templates/sidekiq-deploy.yaml new file mode 100644 index 000000000..0a986f94a --- /dev/null +++ b/chart/templates/sidekiq-deploy.yaml @@ -0,0 +1,80 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "app.sidekiq.name" . }} + labels: + app: {{ template "app.name" . }} + chart: {{ template "app.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + component: sidekiq +spec: + replicas: {{ .Values.sidekiq.replicas }} + selector: + matchLabels: + app: {{ template "app.name" . }} + release: {{ .Release.Name }} + component: sidekiq + template: + metadata: + labels: + app: {{ template "app.name" . }} + release: {{ .Release.Name }} + component: sidekiq + annotations: + checksum/rails-env-cm: {{ include (print $.Template.BasePath "/rails-env-cm.yaml") . | sha256sum }} + checksum/rails-env-secret: {{ include (print $.Template.BasePath "/rails-env-secret.yaml") . | sha256sum }} + spec: + restartPolicy: Always + terminationGracePeriodSeconds: {{ .Values.sidekiq.timeout | add 5 }} + {{- if .Values.rails.imagePullSecrets }} + imagePullSecrets: + {{ toYaml .Values.rails.imagePullSecrets }} + {{- end }} + volumes: + - name: shared + persistentVolumeClaim: + claimName: {{ template "app.rails-env.name" . }}-shared + containers: + - name: sidekiq + image: {{ .Values.rails.image.repository }}:{{ .Values.rails.image.tag }} + imagePullPolicy: Always + command: ["/bin/bash"] + args: + - "-l" + - "-c" + - "bundle exec sidekiq" + # livenessProbe: + # exec: + # command: + # - ./bin/rails runner ./chart/bin/check_sidekiq.rb + # initialDelaySeconds: 30 + # Use sub-path for individual folders + volumeMounts: + - mountPath: /home/app/webapp/tmp/imports + name: shared + subPath: import_path + - mountPath: /home/app/webapp/tmp/exports + name: shared + subPath: export_path + - mountPath: /home/app/webapp/tmp/derivatives_path + name: shared + subPath: derivatives_path + - mountPath: /home/app/webapp/tmp/uploads + name: shared + subPath: upload_path + - mountPath: /home/app/webapp/public/uploads + name: shared + subPath: uploads + - mountPath: /home/app/webapp/public/assets + name: shared + subPath: assets + - mountPath: /home/app/webapp/public/branding + name: shared + subPath: branding + envFrom: + - configMapRef: + name: {{ template "app.rails-env.name" . }} + - secretRef: + name: {{ template "app.rails-env.name" . }} diff --git a/chart/templates/web-deploy.yaml b/chart/templates/web-deploy.yaml new file mode 100644 index 000000000..0b0b8c552 --- /dev/null +++ b/chart/templates/web-deploy.yaml @@ -0,0 +1,67 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "app.web.name" . }} + labels: + app: {{ template "app.name" . }} + chart: {{ template "app.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + component: web +spec: + replicas: {{ .Values.web.replicas }} + selector: + matchLabels: + app: {{ template "app.name" . }} + release: {{ .Release.Name }} + component: web + template: + metadata: + labels: + app: {{ template "app.name" . }} + release: {{ .Release.Name }} + component: web + annotations: + checksum/rails-env-cm: {{ include (print $.Template.BasePath "/rails-env-cm.yaml") . | sha256sum }} + checksum/rails-env-secret: {{ include (print $.Template.BasePath "/rails-env-secret.yaml") . | sha256sum }} + spec: + restartPolicy: Always + {{- if .Values.rails.imagePullSecrets }} + imagePullSecrets: + {{ toYaml .Values.rails.imagePullSecrets }} + {{- end }} + volumes: + - name: shared + persistentVolumeClaim: + claimName: {{ template "app.rails-env.name" . }}-shared + containers: + - name: web + image: {{ .Values.rails.image.repository }}:{{ .Values.rails.image.tag }} + imagePullPolicy: IfNotPresent + # Use sub-path for individual folders + volumeMounts: + - mountPath: /home/app/webapp/public/assets + name: shared + subPath: assets + - mountPath: /home/app/webapp/public/packs + name: shared + subPath: packs + - mountPath: /home/app/webapp/public/system + name: shared + subPath: system + readinessProbe: + tcpSocket: + port: 80 + initialDelaySeconds: 30 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: 80 + initialDelaySeconds: 60 + periodSeconds: 120 + envFrom: + - configMapRef: + name: {{ template "app.rails-env.name" . }} + - secretRef: + name: {{ template "app.rails-env.name" . }} diff --git a/chart/templates/web-ing-wildcard.yaml b/chart/templates/web-ing-wildcard.yaml new file mode 100644 index 000000000..b610169cf --- /dev/null +++ b/chart/templates/web-ing-wildcard.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: apps/v1 +kind: Ingress +metadata: + name: {{ template "app.web.name" . }}-in-wildcard + annotations: + # kubernetes.io/ingress.allow-http: "false" + # for GKE + # kubernetes.io/ingress.global-static-ip-name: rails-k8s-app +spec: + # tls: + # - hosts: + # - {{ .Values.ingress.host }} + rules: + - host: "*.{{ .Values.ingress.host }}" + http: + paths: + - backend: + serviceName: {{ template "app.web.name" . }} + servicePort: 80 diff --git a/chart/templates/web-ing.yaml b/chart/templates/web-ing.yaml new file mode 100644 index 000000000..0dc685d07 --- /dev/null +++ b/chart/templates/web-ing.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: apps/v1 +kind: Ingress +metadata: + name: {{ template "app.web.name" . }}-in + annotations: + {{- range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + # tls: + # - hosts: + # - {{ .Values.ingress.host }} + rules: + - host: "{{ .Values.ingress.host }}" + http: + paths: + - backend: + serviceName: {{ template "app.web.name" . }} + servicePort: 80 diff --git a/chart/templates/web-svc.yaml b/chart/templates/web-svc.yaml new file mode 100644 index 000000000..f7d04bbc5 --- /dev/null +++ b/chart/templates/web-svc.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "app.web.name" . }} + labels: + app: {{ template "app.name" . }} + chart: {{ template "app.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + component: web +spec: + ports: + - protocol: TCP + port: 80 + targetPort: 80 + selector: + app: {{ template "app.name" . }} + release: {{ .Release.Name }} + component: web diff --git a/config/database.yml b/config/database.yml index ab21829a2..635acc596 100644 --- a/config/database.yml +++ b/config/database.yml @@ -1,28 +1,25 @@ -test: - adapter: sqlite3 - database: db/test.sqlite3 - pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 50 } %> +# Hi - please dont edit this file directly, instead make changes in your .env file + +login: &login + adapter: <%= ENV['DB_ADAPTER'] %> + host: <%= ENV['MYSQL_HOST'] %> + username: <%= ENV['MYSQL_USER'] %> + password: <%= ENV['MYSQL_PASSWORD'] %> + database: <%= ENV['MYSQL_DATABASE'] %> + pool: 5 timeout: 5000 + development: - adapter: mysql2 - encoding: utf8 - database: ams_development - username: ams_development - password: password - host: localhost - port: 3306 - # The time that MySQL2 lib will try to get a db connection. - # Default is 5. - # checkout_timeout: 10 - pool: 12 + <<: *login + +test: + <<: *login + database: <%= ENV['TEST_DB'] %> + +staging: + <<: *login + production: - adapter: mysql2 - encoding: utf8 - database: <%= ENV['DB_NAME'] %> - username: <%= ENV['DB_USER'] %> - password: <%= ENV['DB_PWD'] %> - host: <%= ENV['DB_HOST'] %> - port: 3306 - pool: 20 + <<: *login diff --git a/config/environments/development.rb b/config/environments/development.rb index ab580e90b..f5cc2d8ad 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -1,7 +1,12 @@ Rails.application.configure do + # In the development environment your application's code is reloaded on + # every request. This slows down response time but is perfect for development + # since you don't have to restart the web server when you make code changes. + config.cache_classes = !!Sidekiq.server? + # Verifies that versions and hashed value of the package contents in the project's package.json -config.webpacker.check_yarn_integrity = true + config.webpacker.check_yarn_integrity = true # Method for using environment variables for Booleans def truthy_env_var?(val) ['yes', 'true', '1'].include? val.to_s.downcase.strip @@ -76,6 +81,11 @@ def truthy_env_var?(val) # Raises error for missing translations # config.action_view.raise_on_missing_translations = true + if ENV["RAILS_LOG_TO_STDOUT"].present? + logger = ActiveSupport::Logger.new(STDOUT) + logger.formatter = config.log_formatter + config.logger = ActiveSupport::TaggedLogging.new(logger) + end # Use an evented file watcher to asynchronously detect changes in source code, # routes, locales, etc. This feature depends on the listen gem. diff --git a/db/schema.rb b/db/schema.rb index 92f9af8e3..b6a1f89e3 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -12,7 +12,7 @@ ActiveRecord::Schema.define(version: 20210718231531) do - create_table "admin_data", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "admin_data", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.text "sonyci_id" t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -23,7 +23,7 @@ t.index ["hyrax_batch_ingest_batch_id"], name: "index_admin_data_on_hyrax_batch_ingest_batch_id" end - create_table "annotations", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "annotations", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "annotation_type" t.string "ref" t.string "source" @@ -36,7 +36,7 @@ t.index ["admin_data_id"], name: "index_annotations_on_admin_data_id" end - create_table "bookmarks", id: :integer, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "bookmarks", id: :integer, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.integer "user_id", null: false t.string "user_type" t.string "document_id" @@ -48,7 +48,7 @@ t.index ["user_id"], name: "index_bookmarks_on_user_id" end - create_table "checksum_audit_logs", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "checksum_audit_logs", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "file_set_id" t.string "file_id" t.string "checked_uri" @@ -61,7 +61,7 @@ t.index ["file_set_id", "file_id"], name: "by_file_set_id_and_file_id" end - create_table "collection_branding_infos", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "collection_branding_infos", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "collection_id" t.string "role" t.string "local_path" @@ -73,7 +73,7 @@ t.datetime "updated_at", null: false end - create_table "collection_type_participants", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "collection_type_participants", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.bigint "hyrax_collection_type_id" t.string "agent_type" t.string "agent_id" @@ -83,7 +83,7 @@ t.index ["hyrax_collection_type_id"], name: "hyrax_collection_type_id" end - create_table "content_blocks", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "content_blocks", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "name" t.text "value" t.datetime "created_at", null: false @@ -91,7 +91,7 @@ t.string "external_key" end - create_table "curation_concerns_operations", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "curation_concerns_operations", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "status" t.string "operation_type" t.string "job_class" @@ -112,7 +112,7 @@ t.index ["user_id"], name: "index_curation_concerns_operations_on_user_id" end - create_table "featured_works", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "featured_works", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.integer "order", default: 5 t.string "work_id" t.datetime "created_at", null: false @@ -121,7 +121,7 @@ t.index ["work_id"], name: "index_featured_works_on_work_id" end - create_table "file_download_stats", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "file_download_stats", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.datetime "date" t.integer "downloads" t.string "file_id" @@ -132,7 +132,7 @@ t.index ["user_id"], name: "index_file_download_stats_on_user_id" end - create_table "file_view_stats", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "file_view_stats", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.datetime "date" t.integer "views" t.string "file_id" @@ -143,7 +143,7 @@ t.index ["user_id"], name: "index_file_view_stats_on_user_id" end - create_table "hyrax_batch_ingest_batch_items", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "hyrax_batch_ingest_batch_items", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.bigint "batch_id" t.string "id_within_batch" t.text "source_data" @@ -157,7 +157,7 @@ t.index ["batch_id"], name: "index_hyrax_batch_ingest_batch_items_on_batch_id" end - create_table "hyrax_batch_ingest_batches", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "hyrax_batch_ingest_batches", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "status" t.string "submitter_email" t.string "source_location" @@ -171,7 +171,7 @@ t.datetime "end_time" end - create_table "hyrax_collection_types", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "hyrax_collection_types", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "title" t.text "description" t.string "machine_id" @@ -188,14 +188,14 @@ t.index ["machine_id"], name: "index_hyrax_collection_types_on_machine_id", unique: true end - create_table "hyrax_features", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "hyrax_features", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "key", null: false t.boolean "enabled", default: false, null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false end - create_table "instantiation_admin_data", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "instantiation_admin_data", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "aapb_preservation_lto" t.string "aapb_preservation_disk" t.datetime "created_at", null: false @@ -203,7 +203,7 @@ t.string "md5" end - create_table "job_io_wrappers", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "job_io_wrappers", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.bigint "user_id" t.bigint "uploaded_file_id" t.string "file_set_id" @@ -217,7 +217,7 @@ t.index ["user_id"], name: "index_job_io_wrappers_on_user_id" end - create_table "mailboxer_conversation_opt_outs", id: :integer, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "mailboxer_conversation_opt_outs", id: :integer, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "unsubscriber_type" t.integer "unsubscriber_id" t.integer "conversation_id" @@ -225,13 +225,13 @@ t.index ["unsubscriber_id", "unsubscriber_type"], name: "index_mailboxer_conversation_opt_outs_on_unsubscriber_id_type" end - create_table "mailboxer_conversations", id: :integer, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "mailboxer_conversations", id: :integer, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "subject", default: "" t.datetime "created_at", null: false t.datetime "updated_at", null: false end - create_table "mailboxer_notifications", id: :integer, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "mailboxer_notifications", id: :integer, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "type" t.text "body" t.string "subject", default: "" @@ -254,7 +254,7 @@ t.index ["type"], name: "index_mailboxer_notifications_on_type" end - create_table "mailboxer_receipts", id: :integer, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "mailboxer_receipts", id: :integer, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "receiver_type" t.integer "receiver_id" t.integer "notification_id", null: false @@ -271,7 +271,7 @@ t.index ["receiver_id", "receiver_type"], name: "index_mailboxer_receipts_on_receiver_id_and_receiver_type" end - create_table "minter_states", id: :integer, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "minter_states", id: :integer, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "namespace", default: "default", null: false t.string "template", null: false t.text "counters" @@ -282,7 +282,7 @@ t.index ["namespace"], name: "index_minter_states_on_namespace", unique: true end - create_table "permission_template_accesses", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "permission_template_accesses", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.bigint "permission_template_id" t.string "agent_type" t.string "agent_id" @@ -293,7 +293,7 @@ t.index ["permission_template_id"], name: "index_permission_template_accesses_on_permission_template_id" end - create_table "permission_templates", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "permission_templates", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "source_id" t.string "visibility" t.datetime "created_at", null: false @@ -303,7 +303,7 @@ t.index ["source_id"], name: "index_permission_templates_on_source_id", unique: true end - create_table "proxy_deposit_requests", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "proxy_deposit_requests", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "work_id", null: false t.bigint "sending_user_id", null: false t.bigint "receiving_user_id", null: false @@ -317,7 +317,7 @@ t.index ["sending_user_id"], name: "index_proxy_deposit_requests_on_sending_user_id" end - create_table "proxy_deposit_rights", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "proxy_deposit_rights", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.bigint "grantor_id" t.bigint "grantee_id" t.datetime "created_at", null: false @@ -326,21 +326,21 @@ t.index ["grantor_id"], name: "index_proxy_deposit_rights_on_grantor_id" end - create_table "pushes", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "pushes", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.text "pushed_id_csv" t.datetime "created_at", null: false t.datetime "updated_at", null: false t.integer "user_id" end - create_table "qa_local_authorities", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "qa_local_authorities", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "name" t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["name"], name: "index_qa_local_authorities_on_name", unique: true end - create_table "qa_local_authority_entries", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "qa_local_authority_entries", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.bigint "local_authority_id" t.string "label" t.string "uri" @@ -350,11 +350,11 @@ t.index ["uri"], name: "index_qa_local_authority_entries_on_uri", unique: true end - create_table "roles", id: :integer, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "roles", id: :integer, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "name" end - create_table "roles_users", id: false, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "roles_users", id: false, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.integer "role_id" t.integer "user_id" t.index ["role_id", "user_id"], name: "index_roles_users_on_role_id_and_user_id" @@ -363,7 +363,7 @@ t.index ["user_id"], name: "index_roles_users_on_user_id" end - create_table "searches", id: :integer, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "searches", id: :integer, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.binary "query_params" t.integer "user_id" t.string "user_type" @@ -372,7 +372,7 @@ t.index ["user_id"], name: "index_searches_on_user_id" end - create_table "single_use_links", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "single_use_links", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "downloadKey" t.string "path" t.string "itemId" @@ -381,7 +381,7 @@ t.datetime "updated_at", null: false end - create_table "sipity_agents", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sipity_agents", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "proxy_for_id", null: false t.string "proxy_for_type", null: false t.datetime "created_at", null: false @@ -389,7 +389,7 @@ t.index ["proxy_for_id", "proxy_for_type"], name: "sipity_agents_proxy_for", unique: true end - create_table "sipity_comments", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sipity_comments", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.integer "entity_id", null: false t.integer "agent_id", null: false t.text "comment" @@ -400,7 +400,7 @@ t.index ["entity_id"], name: "index_sipity_comments_on_entity_id" end - create_table "sipity_entities", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sipity_entities", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "proxy_for_global_id", null: false t.integer "workflow_id", null: false t.integer "workflow_state_id" @@ -411,7 +411,7 @@ t.index ["workflow_state_id"], name: "index_sipity_entities_on_workflow_state_id" end - create_table "sipity_entity_specific_responsibilities", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sipity_entity_specific_responsibilities", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.integer "workflow_role_id", null: false t.string "entity_id", null: false t.integer "agent_id", null: false @@ -423,7 +423,7 @@ t.index ["workflow_role_id"], name: "sipity_entity_specific_responsibilities_role" end - create_table "sipity_notifiable_contexts", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sipity_notifiable_contexts", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.integer "scope_for_notification_id", null: false t.string "scope_for_notification_type", null: false t.string "reason_for_notification", null: false @@ -436,7 +436,7 @@ t.index ["scope_for_notification_id", "scope_for_notification_type"], name: "sipity_notifiable_contexts_concern" end - create_table "sipity_notification_recipients", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sipity_notification_recipients", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.integer "notification_id", null: false t.integer "role_id", null: false t.string "recipient_strategy", null: false @@ -448,7 +448,7 @@ t.index ["role_id"], name: "sipity_notification_recipients_role" end - create_table "sipity_notifications", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sipity_notifications", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "name", null: false t.string "notification_type", null: false t.datetime "created_at", null: false @@ -457,7 +457,7 @@ t.index ["notification_type"], name: "index_sipity_notifications_on_notification_type" end - create_table "sipity_roles", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sipity_roles", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "name", null: false t.text "description" t.datetime "created_at", null: false @@ -465,7 +465,7 @@ t.index ["name"], name: "index_sipity_roles_on_name", unique: true end - create_table "sipity_workflow_actions", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sipity_workflow_actions", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.integer "workflow_id", null: false t.integer "resulting_workflow_state_id" t.string "name", null: false @@ -476,7 +476,7 @@ t.index ["workflow_id"], name: "sipity_workflow_actions_workflow" end - create_table "sipity_workflow_methods", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sipity_workflow_methods", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "service_name", null: false t.integer "weight", null: false t.integer "workflow_action_id", null: false @@ -485,7 +485,7 @@ t.index ["workflow_action_id"], name: "index_sipity_workflow_methods_on_workflow_action_id" end - create_table "sipity_workflow_responsibilities", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sipity_workflow_responsibilities", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.integer "agent_id", null: false t.integer "workflow_role_id", null: false t.datetime "created_at", null: false @@ -493,7 +493,7 @@ t.index ["agent_id", "workflow_role_id"], name: "sipity_workflow_responsibilities_aggregate", unique: true end - create_table "sipity_workflow_roles", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sipity_workflow_roles", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.integer "workflow_id", null: false t.integer "role_id", null: false t.datetime "created_at", null: false @@ -501,7 +501,7 @@ t.index ["workflow_id", "role_id"], name: "sipity_workflow_roles_aggregate", unique: true end - create_table "sipity_workflow_state_action_permissions", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sipity_workflow_state_action_permissions", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.integer "workflow_role_id", null: false t.integer "workflow_state_action_id", null: false t.datetime "created_at", null: false @@ -509,7 +509,7 @@ t.index ["workflow_role_id", "workflow_state_action_id"], name: "sipity_workflow_state_action_permissions_aggregate", unique: true end - create_table "sipity_workflow_state_actions", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sipity_workflow_state_actions", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.integer "originating_workflow_state_id", null: false t.integer "workflow_action_id", null: false t.datetime "created_at", null: false @@ -517,7 +517,7 @@ t.index ["originating_workflow_state_id", "workflow_action_id"], name: "sipity_workflow_state_actions_aggregate", unique: true end - create_table "sipity_workflow_states", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sipity_workflow_states", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.integer "workflow_id", null: false t.string "name", null: false t.datetime "created_at", null: false @@ -526,7 +526,7 @@ t.index ["workflow_id", "name"], name: "sipity_type_state_aggregate", unique: true end - create_table "sipity_workflows", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sipity_workflows", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "name", null: false t.string "label" t.text "description" @@ -538,7 +538,7 @@ t.index ["permission_template_id", "name"], name: "index_sipity_workflows_on_permission_template_and_name", unique: true end - create_table "sony_ci_webhook_logs", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "sony_ci_webhook_logs", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "url" t.string "action" t.text "request_headers" @@ -551,20 +551,20 @@ t.datetime "updated_at", null: false end - create_table "tinymce_assets", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "tinymce_assets", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "file" t.datetime "created_at", null: false t.datetime "updated_at", null: false end - create_table "trophies", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "trophies", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.integer "user_id" t.string "work_id" t.datetime "created_at", null: false t.datetime "updated_at", null: false end - create_table "uploaded_files", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "uploaded_files", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "file" t.bigint "user_id" t.string "file_set_uri" @@ -574,7 +574,7 @@ t.index ["user_id"], name: "index_uploaded_files_on_user_id" end - create_table "user_stats", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "user_stats", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.integer "user_id" t.datetime "date" t.integer "file_views" @@ -585,7 +585,7 @@ t.index ["user_id"], name: "index_user_stats_on_user_id" end - create_table "users", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "users", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "email", default: "", null: false t.string "encrypted_password", default: "", null: false t.string "reset_password_token" @@ -629,7 +629,7 @@ t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true end - create_table "version_committers", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "version_committers", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.string "obj_id" t.string "datastream_id" t.string "version_id" @@ -638,7 +638,7 @@ t.datetime "updated_at", null: false end - create_table "work_view_stats", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" do |t| + create_table "work_view_stats", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1" do |t| t.datetime "date" t.integer "work_views" t.string "work_id" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..22e127068 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,164 @@ +version: '3.8' + + +x-app: &app + build: + context: . + dockerfile: Dockerfile + target: hyku-base + args: + - EXTRA_APK_PACKAGES=less vim bash openjdk11-jre ffmpeg rsync + - SETTINGS__BULKRAX__ENABLED=true + # command: sh -l -c "bundle && bundle exec puma -v -b tcp://0.0.0.0:3000" + # image: registry.gitlab.com/notch8/britishlibrary:${TAG:-latest} + env_file: + - .env + # NOTE: all common env variables moved to .env + volumes: + - uploads:/app/samvera/hyrax-webapp/public/uploads + - assets:/app/samvera/hyrax-webapp/public/assets + - cache:/app/samvera/hyrax-webapp/tmp/cache + - .:/app/samvera/hyrax-webapp + networks: + internal: + + +volumes: + fcrepo: + solr: + mysql: + redis: + zk: + uploads: + assets: + cache: + +networks: + internal: + +services: + db: + image: mysql:5.7 + environment: + - MYSQL_DATABASE=${MYSQL_DATABASE} + - MYSQL_USER=${MYSQL_DATABASE} + - MYSQL_PASSWORD=${MYSQL_PASSWORD} + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MYSQL_HOST=${MYSQL_HOST} + env_file: + - .env + - .env.development + ports: + - '3306' + volumes: + - 'mysql:/var/lib/mysql' + networks: + internal: + + + solr: + image: bitnami/solr:8 + environment: + - OOM=script + - SOLR_ADMIN_USERNAME=admin + - SOLR_ADMIN_PASSWORD=admin + - SOLR_COLLECTION=hydra-development + - SOLR_CLOUD_BOOTSTRAP=yes + - SOLR_ENABLE_CLOUD_MODE=yes + - SOLR_ENABLE_AUTHENTICATION=yes + - SOLR_PORT_NUMBER=8983 + - SOLR_ZK_HOSTS=zoo + - VIRTUAL_PORT=8983 + - VIRTUAL_HOST=solr.hyku.test + depends_on: + zoo: + condition: service_healthy + expose: + - 8983 + volumes: + - solr:/bitnami + networks: + internal: + + redis: + image: redis:5 + command: redis-server + volumes: + - redis:/data + networks: + internal: + + fcrepo: + image: ghcr.io/samvera/fcrepo4:4.7.5 + volumes: + - fcrepo:/data:cached + env_file: + - .env + expose: + - 8080 + environment: + - JAVA_OPTS=${JAVA_OPTS} -Dfcrepo.modeshape.configuration="classpath:/config/file-simple/repository.json" -Dfcrepo.object.directory="/data/objects" -Dfcrepo.binary.directory="/data/binaries" + networks: + internal: + + + web: + <<: *app + environment: + - VIRTUAL_PORT=3000 + - VIRTUAL_HOST=.hyku.test + # command: sh -l -c "bundle && bundle exec puma -v -b tcp://0.0.0.0:3000" + depends_on: + # initialize_app: + # condition: service_healthy + db: + condition: service_started + solr: + condition: service_started + fcrepo: + condition: service_started + redis: + condition: service_started + zoo: + condition: service_started + # check_volumes: + # condition: service_started + # chrome: + # condition: service_started + sidekiq: + condition: service_started + # initialize_app: + # condition: service_completed_successfully + + expose: + - 3000 + + sidekiq: + <<: *app + command: bundle exec sidekiq + depends_on: + - fcrepo + - db + - solr + - redis + + zoo: + image: bitnami/zookeeper:3.6 + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + - ZOO_4LW_COMMANDS_WHITELIST=mntr, srvr, ruok + - ZOO_SERVER_ID=1 + - ZOO_SERVERS=zoo:2888:3888 + expose: + - 2181 + - 2888 + - 3888 + volumes: + - zk:/bitnami/zookeeper + networks: + internal: + healthcheck: + test: ["CMD-SHELL", "echo 'ruok' | nc -w 2 -q 2 localhost 2181 | grep imok || exit 1"] + interval: "10s" + timeout: "8s" + diff --git a/ops/env.conf b/ops/env.conf new file mode 100644 index 000000000..8ae0a929e --- /dev/null +++ b/ops/env.conf @@ -0,0 +1,9 @@ +env DATABASE_ADAPTER; +env DATABASE_NAME; +env DATABASE_USER; +env DATABASE_PASSWORD; +env DATABASE_ROOT_PASSWORD; +env DATABASE_HOST; +env TEST_DB; +env TAG; +env VIRTUAL_PORT; diff --git a/ops/nginx.sh b/ops/nginx.sh new file mode 100644 index 000000000..4fee56c4b --- /dev/null +++ b/ops/nginx.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -e +if [[ ! -e /var/log/nginx/error.log ]]; then + # The Nginx log forwarder might be sleeping and waiting + # until the error log becomes available. We restart it in + # 1 second so that it picks up the new log file quickly. + (sleep 1 && sv restart /etc/service/nginx-log-forwarder) +fi + +if [ -z $PASSENGER_APP_ENV ] +then + export PASSENGER_APP_ENV=development +fi + +rm -rf /home/app/webapp/.ruby* + +declare -p | grep -Ev 'BASHOPTS|PWD|BASH_VERSINFO|EUID|PPID|SHELLOPTS|UID' > /container.env + +if [[ $PASSENGER_APP_ENV == "development" ]] || [[ $PASSENGER_APP_ENV == "test" ]] +then + /bin/bash -l -c 'chown -fR app:app /app/samvera/hyrax-webapp/tmp/cache' + /sbin/setuser app /bin/bash -l -c 'cd /app/samvera/hyrax-webapp && bundle exec rails db:test:prepare' +fi + +if [[ $PASSENGER_APP_ENV == "production" ]] || [[ $PASSENGER_APP_ENV == "staging" ]] +then + /bin/bash -l -c 'chown -fR app:app /app/samvera/hyrax-webapp/public/assets' # mounted volume may have wrong permissions + /sbin/setuser app /bin/bash -l -c 'cd /app/samvera/hyrax-webapp && rsync -a public/assets-new/ public/assets/' +fi + +exec /usr/sbin/nginx diff --git a/ops/webapp.conf b/ops/webapp.conf new file mode 100644 index 000000000..dd739d931 --- /dev/null +++ b/ops/webapp.conf @@ -0,0 +1,15 @@ +server { + listen 80; + server_name _; + root /home/app/webapp/public; + client_body_in_file_only clean; + client_body_buffer_size 32K; + + client_max_body_size 0; + + sendfile on; + send_timeout 300s; + # The following deploys your Ruby/Python/Node.js/Meteor app on Passenger. + passenger_enabled on; + passenger_user app; +} diff --git a/ops/worker.sh b/ops/worker.sh new file mode 100644 index 000000000..840d3a653 --- /dev/null +++ b/ops/worker.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# `/sbin/setuser memcache` runs the given command as the user `memcache`. +# If you omit that part, the command will be run as root. + +exec /sbin/setuser app /bin/bash -l -c 'cd /home/app/webapp && bundle exec sidekiq >>/var/log/worker.log 2>&1' + + diff --git a/package.json b/package.json index 29a94d184..524b94570 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "AMS", "private": true, "engines": { - "node": "12.9.0" + "node": "12.22.4" }, "dependencies": { "@rails/webpacker": "3.5", From 1d252ce55a5f4ba452a9d4f6b74f2a3d5e6b1a7a Mon Sep 17 00:00:00 2001 From: jzgo Date: Mon, 30 Aug 2021 10:42:02 -0700 Subject: [PATCH 002/150] Clean up dockerfile --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c24f2aeec..b5a3bb480 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,6 @@ RUN apk --no-cache upgrade && \ libxml2-dev \ mariadb-dev \ mediainfo \ - # --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/main/ nodejs=12.9.0 npm \ perl \ cmake \ $EXTRA_APK_PACKAGES From 034da2b881607c3687395fbc7a60b485a4cbd495 Mon Sep 17 00:00:00 2001 From: jzgo Date: Mon, 30 Aug 2021 11:06:32 -0700 Subject: [PATCH 003/150] WIP: get ci running --- .gitlab-ci.yml | 86 ++++++++++++++++++++++++++++---------------------- 1 file changed, 48 insertions(+), 38 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 244972e0d..6415e168e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,14 +1,18 @@ stages: -- build -- lint -- test -- deploy + - build + - lint + - test + - deploy + +variables: + REGISTRY_HOST: registry.gitlab.com + REGISTRY_URI: /notch8/ngao before_script: -- export TAG=${CI_COMMIT_SHORT_SHA} -- export BRANCH=${CI_COMMIT_REF_NAME} -- export REGISTRY_HOST=${CI_REGISTRY} -- export REGISTRY_URI="/${CI_PROJECT_PATH}" + - export TAG=${CI_COMMIT_SHORT_SHA} + - export BRANCH=${CI_COMMIT_REF_NAME} + - export REGISTRY_HOST=${CI_REGISTRY} + - export REGISTRY_URI="/${CI_PROJECT_PATH}" base: stage: build @@ -47,14 +51,21 @@ test: stage: test image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA services: - - mysql:5.7 - - name: solr:latest + - name: mysql:5.7 + - alias: db + - name: bitnami/solr:8 + - alias: solr entrypoint: ["docker-entrypoint.sh", "solr-precreate", "test"] - - redis:latest - - name: botimer/fcrepo:4.5.1 + - name: redis:5-alpine + - alias: redis + - name: samvera/fcrepo4:4.7.5 + - alias: fcrepo + - name: bitnami/zookeeper:3 + alias: zk + variables: - MYSQL_DATABASE: GBH + MYSQL_DATABASE: GBH MYSQL_ROOT_PASSWORD: password MYSQL_USER: GBH_test MYSQL_PASSWORD: password @@ -64,8 +75,7 @@ test: DATABASE_NAME: GBH DATABASE_PASSWORD: password DATABASE_USER: GBH_test - TEST_DB: GBH - SITE_URI_BASE: TODO + TEST_DB: GBH script: - cd /home/app/webapp && bundle exec rake db:create db:schema:load spec tags: @@ -73,31 +83,31 @@ test: variables: GIT_STRATEGY: none -review: - stage: review - type: deploy - environment: - name: $CI_COMMIT_REF_SLUG - url: http://web.$CI_PROJECT_NAME-$CI_COMMIT_REF_SLUG.staging.$SITE_URI_BASE - on_stop: stop_review - only: - - branches - except: - - main - script: - - ./chart/bin/deploy review $CI_COMMIT_REF_SLUG - tags: - - local +# review: +# stage: review +# type: deploy +# environment: +# name: $CI_COMMIT_REF_SLUG +# url: http://web.$CI_PROJECT_NAME-$CI_COMMIT_REF_SLUG.staging.$SITE_URI_BASE +# on_stop: stop_review +# only: +# - branches +# except: +# - main +# script: +# - ./chart/bin/deploy review $CI_COMMIT_REF_SLUG +# tags: +# - local -stop_review: - stage: review - script: - - ./chart/bin/remove review $CI_COMMIT_REF_SLUG +# stop_review: +# stage: review +# script: +# - ./chart/bin/remove review $CI_COMMIT_REF_SLUG - when: manual - environment: - name: $CI_PROJECT_NAME-$CI_BUILD_REF_SLUG - action: stop +# when: manual +# environment: +# name: $CI_PROJECT_NAME-$CI_BUILD_REF_SLUG +# action: stop staging: stage: staging From f075baaf55d3bd003db57e168ecbe99269b13831 Mon Sep 17 00:00:00 2001 From: jzgo Date: Mon, 30 Aug 2021 11:11:28 -0700 Subject: [PATCH 004/150] Correct yaml --- .gitlab-ci.yml | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6415e168e..43c72ab0f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,7 +36,7 @@ build: - local lint: - stage: go + stage: lint image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA variables: PRONTO_GITLAB_API_ENDPOINT: "https://gitlab.com/api/v4" @@ -52,14 +52,14 @@ test: image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA services: - name: mysql:5.7 - - alias: db + alias: db - name: bitnami/solr:8 - - alias: solr + alias: solr entrypoint: ["docker-entrypoint.sh", "solr-precreate", "test"] - name: redis:5-alpine - - alias: redis + alias: redis - name: samvera/fcrepo4:4.7.5 - - alias: fcrepo + alias: fcrepo - name: bitnami/zookeeper:3 alias: zk @@ -109,23 +109,23 @@ test: # name: $CI_PROJECT_NAME-$CI_BUILD_REF_SLUG # action: stop -staging: - stage: staging - type: deploy - script: - - ./chart/bin/deploy staging $CI_COMMIT_REF_SLUG - only: - - main - tags: - - local +# staging: +# stage: deploy +# type: deploy +# script: +# - ./chart/bin/deploy staging $CI_COMMIT_REF_SLUG +# only: +# - main +# tags: +# - local -production: - stage: production - type: deploy - script: - - ./chart/bin/deploy production $CI_COMMIT_REF_SLUG - when: manual - only: - - main - tags: - - local +# production: +# stage: deploy +# type: deploy +# script: +# - ./chart/bin/deploy production $CI_COMMIT_REF_SLUG +# when: manual +# only: +# - main +# tags: +# - local From d7b25b9cc6f17209cb7e45ec7f3d2613900cfb5b Mon Sep 17 00:00:00 2001 From: jzgo Date: Mon, 30 Aug 2021 11:14:20 -0700 Subject: [PATCH 005/150] Remove base build --- .gitlab-ci.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 43c72ab0f..56f199d8e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,16 +14,6 @@ before_script: - export REGISTRY_HOST=${CI_REGISTRY} - export REGISTRY_URI="/${CI_PROJECT_PATH}" -base: - stage: build - script: - - docker login -u "gitlab-ci-token" -p "$CI_JOB_TOKEN" $CI_REGISTRY - - sc build -s base - - sc push -s base - when: manual - tags: - - local - build: stage: build script: From 48dec33623f29fcc84e68c1b1cf763a94e651892 Mon Sep 17 00:00:00 2001 From: jzgo Date: Mon, 30 Aug 2021 11:34:24 -0700 Subject: [PATCH 006/150] Bump node version for specs --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 524b94570..eb5106ef0 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "AMS", "private": true, "engines": { - "node": "12.22.4" + "node": "12.22.5" }, "dependencies": { "@rails/webpacker": "3.5", From 730716405897b6a18c8770067a4ec4f148baeb29 Mon Sep 17 00:00:00 2001 From: jzgo Date: Mon, 30 Aug 2021 12:13:21 -0700 Subject: [PATCH 007/150] Revert node version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index eb5106ef0..524b94570 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "AMS", "private": true, "engines": { - "node": "12.22.5" + "node": "12.22.4" }, "dependencies": { "@rails/webpacker": "3.5", From 852b11fb58579abf7e98a894063bf9e7459b283d Mon Sep 17 00:00:00 2001 From: jzgo Date: Tue, 31 Aug 2021 09:03:48 -0700 Subject: [PATCH 008/150] Troubleshoot incompatible node --- Dockerfile | 2 +- docker-compose.yml | 2 +- package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b5a3bb480..2dad63ef3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM ghcr.io/samvera/hyrax/hyrax-base:$HYRAX_IMAGE_VERSION as hyku-base USER root -ARG EXTRA_APK_PACKAGES="openjdk11-jre ffmpeg" +ARG EXTRA_APK_PACKAGES="openjdk11-jre ffmpeg yarn" RUN apk --no-cache upgrade && \ apk --no-cache add \ curl \ diff --git a/docker-compose.yml b/docker-compose.yml index 22e127068..555072948 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,6 @@ version: '3.8' x-app: &app build: context: . - dockerfile: Dockerfile target: hyku-base args: - EXTRA_APK_PACKAGES=less vim bash openjdk11-jre ffmpeg rsync @@ -14,6 +13,7 @@ x-app: &app env_file: - .env # NOTE: all common env variables moved to .env + image: registry.gitlab.com/notch8/gbh:${TAG:-latest} volumes: - uploads:/app/samvera/hyrax-webapp/public/uploads - assets:/app/samvera/hyrax-webapp/public/assets diff --git a/package.json b/package.json index 524b94570..eb5106ef0 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "AMS", "private": true, "engines": { - "node": "12.22.4" + "node": "12.22.5" }, "dependencies": { "@rails/webpacker": "3.5", From 92c6b9ba05deec59cf5ea6aa94f94b19adef9f82 Mon Sep 17 00:00:00 2001 From: jzgo Date: Tue, 31 Aug 2021 09:25:28 -0700 Subject: [PATCH 009/150] Correct path --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 56f199d8e..8fe3db390 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,7 +33,7 @@ lint: before_script: - export PRONTO_GITLAB_API_PRIVATE_TOKEN=$GITLAB_API_TOKEN script: - - cd /home/app/webapp && bundle exec pronto run -f gitlab + - cd /app/samvera/hyrax-webapp && bundle exec pronto run -f gitlab tags: - docker @@ -67,7 +67,7 @@ test: DATABASE_USER: GBH_test TEST_DB: GBH script: - - cd /home/app/webapp && bundle exec rake db:create db:schema:load spec + - cd /app/samvera/hyrax-webapp && bundle exec rake db:create db:schema:load spec tags: - docker variables: From b6f56037392718242eb5ff2ab71fa9c6892cceff Mon Sep 17 00:00:00 2001 From: jzgo Date: Tue, 31 Aug 2021 10:14:47 -0700 Subject: [PATCH 010/150] Add pronto --- Dockerfile | 2 +- Gemfile | 13 ++++++----- Gemfile.lock | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2dad63ef3..16c7dd695 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM ghcr.io/samvera/hyrax/hyrax-base:$HYRAX_IMAGE_VERSION as hyku-base USER root -ARG EXTRA_APK_PACKAGES="openjdk11-jre ffmpeg yarn" +ARG EXTRA_APK_PACKAGES="openjdk11-jre ffmpeg pkg-config yarn" RUN apk --no-cache upgrade && \ apk --no-cache add \ curl \ diff --git a/Gemfile b/Gemfile index 78d2525a6..5a43df7fa 100644 --- a/Gemfile +++ b/Gemfile @@ -33,10 +33,6 @@ gem 'hydra-role-management', '~> 1.0' # Use Capistrano for deployment # gem 'capistrano-rails', group: :development -group :production do - gem 'mysql2', '~> 0.5.0' -end - group :development, :test do # Use sqlite3 as the database for Active Record # gem 'sqlite3', '1.3.13' @@ -82,6 +78,7 @@ gem 'simple_form', '5.0.0' gem 'aws-sdk-s3' gem 'aws-sdk-codedeploy' gem 'carrierwave', '~> 1.3' +gem 'mysql2', '~> 0.5.3' gem 'nokogiri' gem 'bootstrap-multiselect-rails' gem 'hyrax-batch_ingest', git: 'https://github.com/samvera-labs/hyrax-batch_ingest' @@ -99,4 +96,10 @@ gem 'httparty', '~> 0.18' # Adding pry to all environments, because it's very useful for debugging # production environments on demo instances. gem 'pry-byebug', platforms: [:mri, :mingw, :x64_mingw] -gem 'activerecord-nulldb-adapter' \ No newline at end of file +gem 'activerecord-nulldb-adapter' +gem 'pronto' +gem 'pronto-brakeman', require: false +gem 'pronto-flay', require: false +gem 'pronto-rails_best_practices', require: false +gem 'pronto-rails_schema', require: false +gem 'pronto-rubocop', require: false \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index ed99918e1..a8b995eaf 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -147,6 +147,7 @@ GEM bootstrap_form (4.4.0) actionpack (>= 5.0) activemodel (>= 5.0) + brakeman (5.1.1) breadcrumbs_on_rails (3.0.1) browse-everything (1.1.0) addressable (~> 2.5) @@ -181,6 +182,8 @@ GEM ssrf_filter (~> 1.0) childprocess (3.0.0) clipboard-rails (1.7.1) + code_analyzer (0.5.2) + sexp_processor coderay (1.1.3) coffee-rails (4.2.2) coffee-script (>= 2.2.0) @@ -287,6 +290,7 @@ GEM equivalent-xml (0.6.0) nokogiri (>= 1.4.3) erubi (1.9.0) + erubis (2.7.0) ethon (0.12.0) ffi (>= 1.3.0) execjs (2.7.0) @@ -304,12 +308,20 @@ GEM fcrepo_wrapper (0.9.0) ruby-progressbar ffi (1.13.1) + flay (2.12.1) + erubis (~> 2.7.0) + path_expander (~> 1.0) + ruby_parser (~> 3.0) + sexp_processor (~> 4.0) flipflop (2.6.0) activesupport (>= 4.0) flot-rails (0.0.7) jquery-rails font-awesome-rails (4.7.0.5) railties (>= 3.2, < 6.1) + gitlab (4.17.0) + httparty (~> 0.18) + terminal-table (~> 1.5, >= 1.5.1) globalid (0.4.2) activesupport (>= 4.2.0) google-api-client (0.48.0) @@ -586,6 +598,9 @@ GEM multi_json (~> 1.3) multi_xml (~> 0.5) rack (>= 1.2, < 3) + octokit (4.21.0) + faraday (>= 0.9) + sawyer (~> 0.8.0, >= 0.5.3) openseadragon (0.5.0) rails (> 3.2.0) orm_adapter (0.5.0) @@ -594,6 +609,7 @@ GEM parser (2.7.2.0) ast (~> 2.4.1) parslet (2.0.0) + path_expander (1.1.0) pbcore (0.3.0) factory_bot (~> 4.11) faker (~> 1.9) @@ -601,6 +617,27 @@ GEM sax-machine (~> 1.3) posix-spawn (0.3.15) power_converter (0.1.2) + pronto (0.10.0) + gitlab (~> 4.0, >= 4.0.0) + httparty (>= 0.13.7) + octokit (~> 4.7, >= 4.7.0) + rainbow (>= 2.2, < 4.0) + rugged (~> 0.24, >= 0.23.0) + thor (~> 0.20.0) + pronto-brakeman (0.10.0) + brakeman (>= 3.2.0) + pronto (~> 0.10.0) + pronto-flay (0.10.0) + flay (~> 2.8) + pronto (~> 0.10.0) + pronto-rails_best_practices (0.10.0) + pronto (~> 0.10.0) + rails_best_practices (~> 1.16, >= 1.15.0) + pronto-rails_schema (0.10.0) + pronto (~> 0.10.0) + pronto-rubocop (0.10.0) + pronto (~> 0.10.0) + rubocop (~> 0.50, >= 0.49.1) pry (0.13.1) coderay (~> 1.1) method_source (~> 1.0) @@ -646,6 +683,14 @@ GEM loofah (~> 2.3) rails_autolink (1.1.6) rails (> 3.1) + rails_best_practices (1.21.0) + activesupport + code_analyzer (>= 0.5.2) + erubis + i18n + json + require_all (~> 3.0) + ruby-progressbar railties (5.1.7) actionpack (= 5.1.7) activesupport (= 5.1.7) @@ -732,6 +777,7 @@ GEM uber (< 0.2.0) request_store (1.5.0) rack (>= 1.4) + require_all (3.0.0) responders (3.0.1) actionpack (>= 5.0) railties (>= 5.0) @@ -797,7 +843,10 @@ GEM oauth2 ruby-progressbar (1.10.1) ruby_dep (1.5.0) + ruby_parser (3.17.0) + sexp_processor (~> 4.15, >= 4.15.1) rubyzip (1.3.0) + rugged (0.99.0) samvera-nesting_indexer (2.0.0) dry-equalizer sass (3.7.4) @@ -813,6 +862,9 @@ GEM tilt (>= 1.1, < 3) sassc (2.4.0) ffi (~> 1.9) + sawyer (0.8.2) + addressable (>= 2.3.5) + faraday (> 0.8, < 2.0) sax-machine (1.3.2) scanf (1.0.0) select2-rails (3.5.10) @@ -820,6 +872,7 @@ GEM selenium-webdriver (3.142.7) childprocess (>= 0.5, < 4.0) rubyzip (>= 1.2.2) + sexp_processor (4.15.3) shex (0.6.1) ebnf (~> 2.0) json-ld (~> 3.1) @@ -879,6 +932,8 @@ GEM sxp (1.1.0) rdf (~> 3.1) temple (0.8.2) + terminal-table (1.8.0) + unicode-display_width (~> 1.1, >= 1.1.1) thor (0.20.3) thread_safe (0.3.6) tilt (2.0.10) @@ -957,9 +1012,15 @@ DEPENDENCIES jquery-rails letter_opener listen (>= 3.0.5, < 3.2) - mysql2 (~> 0.5.0) + mysql2 (~> 0.5.3) nokogiri pbcore (~> 0.3.0) + pronto + pronto-brakeman + pronto-flay + pronto-rails_best_practices + pronto-rails_schema + pronto-rubocop pry-byebug puma (~> 3.12) rails (~> 5.1.5) From ad0bcc239ae8401269f7754f1bb9018ca9d1655d Mon Sep 17 00:00:00 2001 From: jzgo Date: Tue, 14 Sep 2021 08:54:39 -0700 Subject: [PATCH 011/150] Updates to readme for n8 development --- README.md | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index f163c6809..f9be76312 100644 --- a/README.md +++ b/README.md @@ -22,29 +22,18 @@ We recommend committing .env to your repo with good defaults. .env.development, 3) Sign in with dory ``` bash dory up - ``` -4) Install dependencies - ``` bash - yarn install - ``` - -5) Start the server +4) Start the server ``` bash sc up ``` -6) Load and seed the database +5) Load and seed the database ``` bash sc be rake db:migrate db: seed ``` -### Troubleshooting Docker Development Setup -Confirm or configure settings. Sub your information for the examples. -``` bash -git config --global user.name example -git config --global user.email example@example.com -docker login registry.gitlab.com -``` + +6) The app should be visible at in the browser at `hyku.test` ### While in the container you can do the following - Run rspec @@ -88,13 +77,6 @@ bin/encrypt-secrets This will find and output an encrypted version of secret files with an `.enc` extension. -# Deploy a new release - -``` bash -sc release {staging | production} # creates and pushes the correct tags -sc deploy {staging | production} # deployes those tags to the server -``` - Release and Deployment are handled by the gitlab ci by default. See ops/deploy-app to deploy from locally, but note all Rancher install pull the currently tagged registry image # ams From 63077dc9c87f219b3238553c2c747309fee6e0b1 Mon Sep 17 00:00:00 2001 From: jzgo Date: Tue, 14 Sep 2021 09:28:51 -0700 Subject: [PATCH 012/150] Adds notes on n8 infra and adding remotes --- README.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/README.md b/README.md index f9be76312..88f7b2222 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,40 @@ bin/encrypt-secrets This will find and output an encrypted version of secret files with an `.enc` extension. Release and Deployment are handled by the gitlab ci by default. See ops/deploy-app to deploy from locally, but note all Rancher install pull the currently tagged registry image + +## Staging Deploys: N8 Architecture + +Staging builds and deploys to Notch8 infrastructure are handled by Gitlab CI. + +**Setup your `gitlab` git remote** + +You'll only need to do this once. You need to set this remote to push, build and deploy your work. +- Run `git remote add gitlab +## Staging Deploys: N8 Architecture + +Staging builds and deploys to Notch8 infrastructure are handled by Gitlab CI. + +**Setup your `gitlab` git remote** + +You'll only need to do this once. You need to set this remote to push, build and deploy your work. +- Run `git remote add gitlab git@gitlab.com:notch8/ngao.git` +- Run `git remote`. You've successfully added the **gitlab** remote if your output lists it. It will look like: +``` +> git remote # Run git remote +gitlab # New gitlab remote +origin +``` + +- Run `git remote`. You've successfully added the **gitlab** remote if your output lists it. It will look like: +``` +> git remote # Run git remote +gitlab # New gitlab remote +origin +``` + + + + # ams Archival Management System to support the American Archive of Public Broadcasting From 9c0482bea536db4741abb915072e909aea9a2d93 Mon Sep 17 00:00:00 2001 From: jzgo Date: Tue, 14 Sep 2021 09:32:17 -0700 Subject: [PATCH 013/150] Clean up readme --- README.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/README.md b/README.md index 88f7b2222..870db6da7 100644 --- a/README.md +++ b/README.md @@ -85,14 +85,6 @@ Staging builds and deploys to Notch8 infrastructure are handled by Gitlab CI. **Setup your `gitlab` git remote** -You'll only need to do this once. You need to set this remote to push, build and deploy your work. -- Run `git remote add gitlab -## Staging Deploys: N8 Architecture - -Staging builds and deploys to Notch8 infrastructure are handled by Gitlab CI. - -**Setup your `gitlab` git remote** - You'll only need to do this once. You need to set this remote to push, build and deploy your work. - Run `git remote add gitlab git@gitlab.com:notch8/ngao.git` - Run `git remote`. You've successfully added the **gitlab** remote if your output lists it. It will look like: From 1d7b5aea5de9c61ecca5c8570e6332e1421d89ef Mon Sep 17 00:00:00 2001 From: jzgo Date: Tue, 14 Sep 2021 09:34:08 -0700 Subject: [PATCH 014/150] Clean up readme --- README.md | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/README.md b/README.md index 870db6da7..ed358e7c9 100644 --- a/README.md +++ b/README.md @@ -86,14 +86,8 @@ Staging builds and deploys to Notch8 infrastructure are handled by Gitlab CI. **Setup your `gitlab` git remote** You'll only need to do this once. You need to set this remote to push, build and deploy your work. -- Run `git remote add gitlab git@gitlab.com:notch8/ngao.git` -- Run `git remote`. You've successfully added the **gitlab** remote if your output lists it. It will look like: -``` -> git remote # Run git remote -gitlab # New gitlab remote -origin -``` +- Run `git remote add gitlab git@gitlab.com:notch8/GBH.git` - Run `git remote`. You've successfully added the **gitlab** remote if your output lists it. It will look like: ``` > git remote # Run git remote @@ -101,9 +95,6 @@ gitlab # New gitlab remote origin ``` - - - # ams Archival Management System to support the American Archive of Public Broadcasting From 08dcfa99c764710a3760dfb0e2347272fa03f141 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Tue, 14 Sep 2021 15:08:48 -0700 Subject: [PATCH 015/150] fedora url --- .env | 1 + config/fedora.yml | 4 ++-- package.json | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.env b/.env index c5b27695b..e80ce41d4 100644 --- a/.env +++ b/.env @@ -22,3 +22,4 @@ MYSQL_ROOT_PASSWORD=DatabaseFTW MYSQL_HOST=db DB_ADAPTER=mysql2 TEST_DB=GBH_test +FCREPO_URL=http://fcrepo:8080/rest diff --git a/config/fedora.yml b/config/fedora.yml index e01a25eb3..ce69c2f5c 100644 --- a/config/fedora.yml +++ b/config/fedora.yml @@ -1,12 +1,12 @@ development: user: fedoraAdmin password: fedoraAdmin - url: http://127.0.0.1:<%= ENV['FCREPO_DEVELOPMENT_PORT'] || 8984 %>/rest + url: <%= ENV['FCREPO_URL'] || "http://127.0.0.1:#{ENV['FCREPO_DEVELOPMENT_PORT'] || 8984}/rest" %> base_path: /dev test: user: fedoraAdmin password: fedoraAdmin - url: http://127.0.0.1:<%= ENV['FCREPO_TEST_PORT'] || 8986 %>/rest + url: <%= ENV['FCREPO_URL'] || "http://127.0.0.1:#{ENV['FCREPO_TEST_PORT'] || 8986}/rest" %> base_path: /test production: user: fedoraAdmin diff --git a/package.json b/package.json index eb5106ef0..ee543f2dd 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "AMS", "private": true, "engines": { - "node": "12.22.5" + "node": "12.22.6" }, "dependencies": { "@rails/webpacker": "3.5", From e4a43726e295a1b9a743e5a4f7734ccfc49662be Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Tue, 14 Sep 2021 17:03:27 -0700 Subject: [PATCH 016/150] up and running --- .env | 8 ++++++ bin/solrcloud-assign-configset.sh | 35 ++++++++++++++++++++++++++ bin/solrcloud-upload-configset.sh | 42 +++++++++++++++++++++++++++++++ config/redis.yml | 4 +-- config/solr.yml | 4 +-- docker-compose.yml | 32 ++++++++++++++++------- solr/config/schema.xml | 42 +++++++++++++++---------------- 7 files changed, 133 insertions(+), 34 deletions(-) create mode 100755 bin/solrcloud-assign-configset.sh create mode 100755 bin/solrcloud-upload-configset.sh diff --git a/.env b/.env index e80ce41d4..57186f4cf 100644 --- a/.env +++ b/.env @@ -23,3 +23,11 @@ MYSQL_HOST=db DB_ADAPTER=mysql2 TEST_DB=GBH_test FCREPO_URL=http://fcrepo:8080/rest +SOLR_URL=http://admin:admin@solr:8983/solr/hydra-development +SOLR_ADMIN_PASSWORD=admin +SOLR_ADMIN_USER=admin +SOLR_COLLECTION_NAME=hydra-development +SOLR_CONFIGSET_NAME=hyku +SOLR_HOST=solr +SOLR_PORT=8983 +REDIS_SERVER=redis diff --git a/bin/solrcloud-assign-configset.sh b/bin/solrcloud-assign-configset.sh new file mode 100755 index 000000000..597d42170 --- /dev/null +++ b/bin/solrcloud-assign-configset.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env sh +COUNTER=0; + +if [ "$SOLR_ADMIN_USER" ]; then + solr_user_settings="--user $SOLR_ADMIN_USER:$SOLR_ADMIN_PASSWORD" +fi + +solr_config_name="${SOLR_CONFIGSET_NAME:-solrconfig}" +solr_collection_name="${SOLR_COLLECTION_NAME:-hyrax}" + +# Solr Cloud Collection API URLs +solr_collection_list_url="$SOLR_HOST:$SOLR_PORT/solr/admin/collections?action=LIST" +solr_collection_modify_url="$SOLR_HOST:$SOLR_PORT/solr/admin/collections?action=MODIFYCOLLECTION&collection=${solr_collection_name}&collection.configName=${solr_config_name}" + +while [ $COUNTER -lt 30 ]; do + if nc -z "${SOLR_HOST}" "${SOLR_PORT}"; then + if curl --silent $solr_user_settings "$solr_collection_list_url" | grep -q "$solr_collection_name"; then + echo "-- Collection ${solr_collection_name} exists; setting ${solr_config_name} ConfigSet ..." + echo $solr_collection_modify_url + curl $solr_user_settings "$solr_collection_modify_url" + exit + else + echo "-- Collection ${solr_collection_name} does not exist; creating and setting ${solr_config_name} ConfigSet ..." + solr_collection_create_url="$SOLR_HOST:$SOLR_PORT/solr/admin/collections?action=CREATE&name=${solr_collection_name}&collection.configName=${solr_config_name}&numShards=1" + curl $solr_user_settings "$solr_collection_create_url" + exit + fi + fi + echo "-- Looking for Solr (${SOLR_HOST}:${SOLR_PORT})..." + COUNTER=$(( COUNTER+1 )); + sleep 5s +done + +echo "--- ERROR: failed to create/update Solr collection after 5 minutes"; +exit 1 diff --git a/bin/solrcloud-upload-configset.sh b/bin/solrcloud-upload-configset.sh new file mode 100755 index 000000000..a5aa5a370 --- /dev/null +++ b/bin/solrcloud-upload-configset.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env sh + +COUNTER=0; +# /app/samvera/hyrax-webapp/solr/conf +CONFDIR="${1}" + +if [ "$SOLR_ADMIN_USER" ]; then + solr_user_settings="--user $SOLR_ADMIN_USER:$SOLR_ADMIN_PASSWORD" +fi + +solr_config_name="${SOLR_CONFIGSET_NAME:-solrconfig}" + +# Solr Cloud ConfigSet API URLs +solr_config_list_url="http://$SOLR_HOST:$SOLR_PORT/api/cluster/configs?omitHeader=true" +solr_config_upload_url="http://$SOLR_HOST:$SOLR_PORT/solr/admin/configs?action=UPLOAD&name=${solr_config_name}" + +while [ $COUNTER -lt 30 ]; do + echo "-- Looking for Solr (${SOLR_HOST}:${SOLR_PORT})..." + if nc -z "${SOLR_HOST}" "${SOLR_PORT}"; then + # shellcheck disable=SC2143,SC2086 + if curl --silent --user 'fake:fake' "$solr_config_list_url" | grep -q '401'; then + # the solr pods come up and report available before they are ready to accept trusted configs + # only try to upload the config if auth is on. + if curl --silent $solr_user_settings "$solr_config_list_url" | grep -q "$solr_config_name"; then + echo "-- ConfigSet already exists; skipping creation ..."; + else + echo "-- ConfigSet for ${CONFDIR} does not exist; creating ..." + (cd "$CONFDIR" && zip -r - *) | curl -X POST $solr_user_settings --header "Content-Type:application/octet-stream" --data-binary @- "$solr_config_upload_url" + fi + exit + else + echo "-- Solr at $solr_config_list_url is accepting unauthorized connections; we can't upload a trusted ConfigSet." + echo "-- It's possible SolrCloud is bootstrapping its configuration, so this process will retry." + echo "-- see: https://solr.apache.org/guide/8_6/configsets-api.html#configsets-upload" + fi + fi + COUNTER=$(( COUNTER+1 )); + sleep 5s +done + +echo "--- ERROR: failed to create Solr ConfigSet after 5 minutes"; +exit 1 diff --git a/config/redis.yml b/config/redis.yml index 78549de53..06869d749 100644 --- a/config/redis.yml +++ b/config/redis.yml @@ -1,8 +1,8 @@ development: - host: localhost + host: <%= ENV['REDIS_SERVER'] || 'localhost' %> port: 6379 test: - host: localhost + host: <%= ENV['REDIS_SERVER'] || 'localhost' %> port: 6379 production: host: <%= ENV['REDIS_SERVER'] %> diff --git a/config/solr.yml b/config/solr.yml index 606693763..be96fbde6 100644 --- a/config/solr.yml +++ b/config/solr.yml @@ -1,7 +1,7 @@ # This is a sample config file that points to a solr server for each environment development: - url: http://127.0.0.1:<%= ENV['SOLR_TEST_PORT'] || 8983 %>/solr/hydra-development + url: <%= ENV['SOLR_URL'] || "http://127.0.0.1:#{ENV['SOLR_TEST_PORT'] || 8983 }/solr/hydra-development" %> test: - url: http://127.0.0.1:<%= ENV['SOLR_TEST_PORT'] || 8985 %>/solr/hydra-test + url: <%= ENV['SOLR_URL'] || "http://127.0.0.1:#{ENV['SOLR_TEST_PORT'] || 8985 }/solr/hydra-test" %> production: url: <%= ENV['SOLR_URL'] %> diff --git a/docker-compose.yml b/docker-compose.yml index 555072948..5b8e42364 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,14 +62,13 @@ services: - OOM=script - SOLR_ADMIN_USERNAME=admin - SOLR_ADMIN_PASSWORD=admin - - SOLR_COLLECTION=hydra-development - SOLR_CLOUD_BOOTSTRAP=yes - SOLR_ENABLE_CLOUD_MODE=yes - SOLR_ENABLE_AUTHENTICATION=yes - SOLR_PORT_NUMBER=8983 - SOLR_ZK_HOSTS=zoo - VIRTUAL_PORT=8983 - - VIRTUAL_HOST=solr.hyku.test + - VIRTUAL_HOST=solr.ams.test depends_on: zoo: condition: service_healthy @@ -100,17 +99,33 @@ services: - JAVA_OPTS=${JAVA_OPTS} -Dfcrepo.modeshape.configuration="classpath:/config/file-simple/repository.json" -Dfcrepo.object.directory="/data/objects" -Dfcrepo.binary.directory="/data/binaries" networks: internal: - + + initialize_app: + <<: *app + environment: + - CONFDIR=/app/samvera/hyrax-webapp/solr/config + entrypoint: ["sh", "-c"] + command: + - > + if [ ! -f "/app/samvera/hyrax-webapp/solr_db_initialized" ]; then + solrcloud-upload-configset.sh /app/samvera/hyrax-webapp/solr/config && + solrcloud-assign-configset.sh && + SOLR_COLLECTION_NAME=hydra-test solrcloud-assign-configset.sh && + rails db:create db:migrate db:seed && + touch /app/samvera/hyrax-webapp/solr_db_initialized + fi + depends_on: + - db + - solr + - fcrepo web: <<: *app environment: - VIRTUAL_PORT=3000 - - VIRTUAL_HOST=.hyku.test + - VIRTUAL_HOST=.ams.test # command: sh -l -c "bundle && bundle exec puma -v -b tcp://0.0.0.0:3000" depends_on: - # initialize_app: - # condition: service_healthy db: condition: service_started solr: @@ -127,8 +142,8 @@ services: # condition: service_started sidekiq: condition: service_started - # initialize_app: - # condition: service_completed_successfully + initialize_app: + condition: service_completed_successfully expose: - 3000 @@ -161,4 +176,3 @@ services: test: ["CMD-SHELL", "echo 'ruok' | nc -w 2 -q 2 localhost 2181 | grep imok || exit 1"] interval: "10s" timeout: "8s" - diff --git a/solr/config/schema.xml b/solr/config/schema.xml index e7e4cefca..76e721261 100644 --- a/solr/config/schema.xml +++ b/solr/config/schema.xml @@ -63,25 +63,25 @@ - - - - + + + + - - - - - + + + + + - - - + + + - + - @@ -233,7 +233,7 @@ - + @@ -251,7 +251,7 @@ - + @@ -263,10 +263,10 @@ - - + + @@ -276,7 +276,7 @@ - + @@ -292,7 +292,7 @@ - + @@ -308,7 +308,7 @@ - + From 71510cf4964fb9e71e6a2e5140a7027c2a3047df Mon Sep 17 00:00:00 2001 From: Jezreel Go Date: Thu, 23 Sep 2021 18:26:16 +0000 Subject: [PATCH 017/150] Helm staging setup --- .env | 2 +- .env.development.enc | 21 ++ .env.enc | 29 ++ .env.production.enc | 21 ++ .gitignore | 3 + .gitlab-ci.yml | 175 ++++++------ .sops.yaml | 2 +- bin/decrypt-secrets | 5 +- bin/encrypt-secrets | 5 +- bin/helm_deploy | 45 +++ chart/Chart.yaml | 12 +- chart/bin/deploy | 2 +- chart/bin/remove | 2 +- chart/charts/mariadb-7.3.14.tgz | Bin 0 -> 22931 bytes chart/charts/redis-11.0.4.tgz | Bin 0 -> 63008 bytes chart/requirements.lock | 9 + chart/staging-values.yaml.enc | 140 +++++++++ chart/templates/_helpers.tpl | 6 - chart/templates/fcrepo-env-cm.yaml | 2 +- chart/templates/fcrepo-env-secret.yaml | 2 +- chart/templates/rails-env-cm.yaml | 19 +- chart/templates/setup-job.yaml | 14 +- chart/templates/sidekiq-deploy.yaml | 16 +- chart/templates/web-deploy.yaml | 24 +- chart/templates/web-ing-wildcard.yaml | 2 +- chart/templates/web-ing.yaml | 2 +- config/database.yml | 2 +- config/fedora.yml | 2 +- config/solr.yml | 3 +- hyrax/.helmignore | 23 ++ hyrax/Chart.lock | 18 ++ hyrax/Chart.yaml | 30 ++ hyrax/README.md | 152 ++++++++++ hyrax/charts/fcrepo-0.8.0.tgz | Bin 0 -> 49927 bytes hyrax/charts/mariadb-9.5.1.tgz | Bin 0 -> 38260 bytes hyrax/charts/memcached-4.2.21.tgz | Bin 0 -> 11714 bytes hyrax/charts/minio-6.7.2.tgz | Bin 0 -> 37662 bytes hyrax/charts/redis-10.7.16.tgz | Bin 0 -> 59094 bytes hyrax/charts/solr-1.0.1.tgz | Bin 0 -> 65470 bytes hyrax/templates/NOTES.txt | 21 ++ hyrax/templates/_helpers.tpl | 190 +++++++++++++ hyrax/templates/branding-pvc.yaml | 14 + hyrax/templates/configmap-env.yaml | 42 +++ hyrax/templates/cron-embargo.yaml | 24 ++ hyrax/templates/cron-lease.yaml | 24 ++ hyrax/templates/deployment-worker.yaml | 129 +++++++++ hyrax/templates/deployment.yaml | 200 +++++++++++++ hyrax/templates/derivatives-pvc.yaml | 14 + hyrax/templates/hpa.yaml | 28 ++ hyrax/templates/ingress.yaml | 50 ++++ hyrax/templates/secrets.yaml | 22 ++ hyrax/templates/service.yaml | 15 + hyrax/templates/serviceaccount.yaml | 12 + hyrax/templates/tests/test-connection.yaml | 15 + hyrax/templates/uploads-pvc.yaml | 14 + hyrax/values.yaml | 283 +++++++++++++++++++ ops/env.conf | 18 +- ops/staging-deploy.tmpl.yaml | 295 +++++++++++++++++++ ops/staging-deploy.tmpl.yaml.enc | 312 +++++++++++++++++++++ ops/webapp.conf | 3 +- solr_db_initialized | 0 61 files changed, 2354 insertions(+), 161 deletions(-) create mode 100644 .env.development.enc create mode 100644 .env.enc create mode 100644 .env.production.enc mode change 100644 => 100755 bin/decrypt-secrets mode change 100644 => 100755 bin/encrypt-secrets create mode 100755 bin/helm_deploy mode change 100644 => 100755 chart/bin/deploy mode change 100644 => 100755 chart/bin/remove create mode 100644 chart/charts/mariadb-7.3.14.tgz create mode 100644 chart/charts/redis-11.0.4.tgz create mode 100644 chart/requirements.lock create mode 100644 chart/staging-values.yaml.enc create mode 100644 hyrax/.helmignore create mode 100644 hyrax/Chart.lock create mode 100644 hyrax/Chart.yaml create mode 100644 hyrax/README.md create mode 100644 hyrax/charts/fcrepo-0.8.0.tgz create mode 100644 hyrax/charts/mariadb-9.5.1.tgz create mode 100644 hyrax/charts/memcached-4.2.21.tgz create mode 100644 hyrax/charts/minio-6.7.2.tgz create mode 100644 hyrax/charts/redis-10.7.16.tgz create mode 100644 hyrax/charts/solr-1.0.1.tgz create mode 100644 hyrax/templates/NOTES.txt create mode 100644 hyrax/templates/_helpers.tpl create mode 100644 hyrax/templates/branding-pvc.yaml create mode 100644 hyrax/templates/configmap-env.yaml create mode 100644 hyrax/templates/cron-embargo.yaml create mode 100644 hyrax/templates/cron-lease.yaml create mode 100644 hyrax/templates/deployment-worker.yaml create mode 100644 hyrax/templates/deployment.yaml create mode 100644 hyrax/templates/derivatives-pvc.yaml create mode 100644 hyrax/templates/hpa.yaml create mode 100644 hyrax/templates/ingress.yaml create mode 100644 hyrax/templates/secrets.yaml create mode 100644 hyrax/templates/service.yaml create mode 100644 hyrax/templates/serviceaccount.yaml create mode 100644 hyrax/templates/tests/test-connection.yaml create mode 100644 hyrax/templates/uploads-pvc.yaml create mode 100644 hyrax/values.yaml create mode 100644 ops/staging-deploy.tmpl.yaml create mode 100644 ops/staging-deploy.tmpl.yaml.enc create mode 100644 solr_db_initialized diff --git a/.env b/.env index 57186f4cf..4047eb878 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -APP_NAME=GBH +APP_NAME=gbh-hyrax PASSENGER_APP_ENV=development REGISTRY_HOST=registry.gitlab.com REGISTRY_URI=/notch8/gbh diff --git a/.env.development.enc b/.env.development.enc new file mode 100644 index 000000000..cc7a28aef --- /dev/null +++ b/.env.development.enc @@ -0,0 +1,21 @@ +{ + "data": "ENC[AES256_GCM,data:/QV46n6aYq3DbVF9dIRE9rnjme03aRW0clulhlNfvLl9aqWFIzDgUA2ubT9Zu+SE9P/8z0pzlRHcLsHl9WrKy0nVBZeP5Gii0qMbBBNSiFLnREcEG+oAz6K465O3Is9QI6bMkXMKrE5hzoBaB9K/xUJaVUvw//k6rC4smsHy12AKzdY4HLocXg6JvC6BkqdFlyljbbRKbDxp3/JDVv8jFw==,iv:PImElXiALSCWqPSXBp3glD8ukQtxV4e1pRAjekiUgtM=,tag:0XS/X2E2gGEU/AneSsOzxQ==,type:str]", + "sops": { + "kms": null, + "gcp_kms": null, + "azure_kv": null, + "hc_vault": null, + "age": null, + "lastmodified": "2021-09-07T19:03:04Z", + "mac": "ENC[AES256_GCM,data:3Tg4mRiqyloOlOsgR+rxD6+9UNYZqMDyJ/GNnqsJCqVFfekA/jh0fkKYdYm3R2beCuwohPnjvBci5uVNrA3p26LOUoa/oIWtqbaAnNzfQllFDO2w1xZ722Jii4GidH6YHHiAO1DzkC5mgrMpAoiQQRZqa5rMaNxEnPxVe5/Ha3Y=,iv:WRqT9a9Q4wqxBCOZ6x5BOaP+e4Cvwry4MyERgYOEPNM=,tag:uJvNaXBzifyfEWRvG1dbCQ==,type:str]", + "pgp": [ + { + "created_at": "2021-09-07T19:03:03Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhQIMAx1u4ocvSXxJARAAzXKg3pOsYJXSS5Vch2egHvAxenS+HcufS3NsEvbT5J7Q\ny2hYYtA089hj7jG2J+u2oU317W74xiGrtmsMtNmqoyuiZI/dW8arVePlIPwtSlR1\nomIQjIdRRQ6T7qa8UKQmjmIWtQ7UImyLElI8J6jPvfca2EW0/Du8SGo+89DkP0gl\n5X9+79zMJQkUj1vmpoY4FKKjGDDKzMmS07KNUh+BzYnnlruSKWyrpBe9ZA6jclQP\nLuvlTQsGdbFcommLwg/o7CbJAgXOGEpUCXK+ahdTWuup/YnFZcmamNAzQkNMWIkV\n8d2QMqN5DHrCtYOpnsqdtj4L8WvfZYd8KS8BM3CaOk3mJi/9WbbkpfEkOGfcGoV4\n1dQH2h2LRyh4L6bIs5UiXRC1PjRGM1PkqJSiaqcWOF61OqUyfjTzggHPetP+NtSx\nzBeFBYMjaiylUF+5iyPYTIwNqBr4QUAugu19VYa2ro/e5wuh49T18cJ/6XsAhD41\nMGbr1JSRqi0pcBIaNdgaq5U1Rx7TzR7hkvf7sf55MCGxoH9CDguQqt94QCOZ2SyI\n51MQox72uaD7JLHIN4TjK8KFv0xayaAXN3afUti+rMzT4uRGQHoW3RWAOFSL3IO3\ntrCFJw/rbcwkEp+bMLb51TCYS7FHoAbckcdJMJhnlPWsFaAPx5aj6z7r/xKAnGLS\nXgHsnyf/n10R9eTLkA/K2WUmVQn+ZiOTf9nvjXFB/ztWge5PuE+T3feMT/dYkJck\nHJhyvhchCRqGnvuaPm/l8KwvnplSJ1Yjs9EcQ+oOem9jpCRemNgd8VJmwDEzhm8=\n=sAaC\n-----END PGP MESSAGE-----\n", + "fp": "B6125B16B0DD59F34D6975FBF885927FDA9C48E2" + } + ], + "unencrypted_suffix": "_unencrypted", + "version": "3.7.1" + } +} \ No newline at end of file diff --git a/.env.enc b/.env.enc new file mode 100644 index 000000000..8b771c75d --- /dev/null +++ b/.env.enc @@ -0,0 +1,29 @@ +APP_NAME=ENC[AES256_GCM,data:362ZhN7vUMr7,iv:FuUFP9lWdOeBwQK2E8N+UgXnmfH0mVDQkQenFQ5IdUc=,tag:dtwGivM24zJDtHHcajUQ2A==,type:str] +PASSENGER_APP_ENV=ENC[AES256_GCM,data:RUmOJggK6S5H6T4=,iv:n4oSgwNlwjnkpzvnmT+wUUKxfYLLN2L6rOi73kfoT0c=,tag:LwwrPd1iOlrHoDR275NeOw==,type:str] +REGISTRY_HOST=ENC[AES256_GCM,data:Ee4l3tS1+6Xv8R9zP22NvI/GTg==,iv:cLWms6WoytHZ5DAirXfQq6LHrAoAnQgK6BOc7a79TNE=,tag:ZZF3SXRfCzJp3eiLwETu8g==,type:str] +REGISTRY_URI=ENC[AES256_GCM,data:1rPTdUEs6q84ZBg=,iv:/hWmn2NsXSnPBBD1Bsx184leK+NSzA1xpXRjctqjqA8=,tag:2WrHfeg0fGkxKN0UeqMRYg==,type:str] +#ENC[AES256_GCM,data:WjFwhy2zUexA1IlIRWzRsh9p/k+cng==,iv:Lef2FnXFpe7KM9SOKo0CIIbACN+RQiU10svcMVO9iWU=,tag:5lMQQxvtf9T4UP1EkERGuA==,type:comment] +#ENC[AES256_GCM,data:RgMnweZSw/nyd1bjywTYAJf1,iv:w2iUt+ouwb4ymDYX146gsmYlZmNyWlZuw+UiaJXeklM=,tag:UYVOsmyUz60OvLozZCWtaQ==,type:comment] +#ENC[AES256_GCM,data:WsyCbyMFJPoS9s3Z7mdJlPTKiCs6XcPbOUDJHJJC+g==,iv:+o1z8RSakj7x7GDZBNWX+H9A/jCcHDc8TQPZYmWMNC0=,tag:ez7mbrVTzq/05x337VSgCA==,type:comment] +#ENC[AES256_GCM,data:16JjRKHZkEA1Xh4bfT8y4QzIXgHj8omwi0f25KcMu6rkqsK9hFD+wUJgqNpzkiYhqghiWS7f9sqDmFy2l7RgvWya9WumOw4=,iv:CNZ+tr0qDQt6aiLay1kZ9d3YMx5IZ4yzl8jkOBKVXIg=,tag:rhG9628RMjUZrxh23tW07g==,type:comment] +#ENC[AES256_GCM,data:jVkqHqK/sx1auWrRIiCbEIn8ig0buLIZ,iv:HKUCwwxJauHhqjSRRJ/gqkcIyFPKn1rWg36JcLmNNFQ=,tag:TYqja9gpBNCtv4YMDbBd9w==,type:comment] +#ENC[AES256_GCM,data:lzeYQydWcQCCJorjEZuZcIcPT5Akeqk=,iv:EgCT1wEHAOe7mIxoQu5Rg80GUCmq7dsKvVSSnlKOzJk=,tag:LCUaTm/yuNj1lbNyYnXEqg==,type:comment] +#ENC[AES256_GCM,data:1i2/YgnWdm3gIPRFQklqkm/1S6Essw==,iv:gX+8haUM3Og/LpTndN7/s0wPTK6O2Dx5lXOCry5gc2c=,tag:9Tb50DP2bCRQQuFrR2lnNw==,type:comment] +#ENC[AES256_GCM,data:FDAt/3xZ2ArMWzNdPe4Ko5fYCaz9,iv:V7Va+53+SJ5mtVzh5fFec7G8UVBcidfQg7n6a9SaKgs=,tag:xh2GnouH+sQsTntiR0LYJw==,type:comment] +#ENC[AES256_GCM,data:f5L7qKnRmuZt6JDZ47jM0EGkcw==,iv:/IVuHXq3J6wk54IrOtJSW/4eGs3T9OmsVnb9Lpo9ms0=,tag:glUhR5J9CR5/2/OaXFOAqQ==,type:comment] +#ENC[AES256_GCM,data:TXBXs2Rgmj4stzc5Bo0vCHM+SlY=,iv:UIUpeQ1d48ZTavutJjP4PX7IK2i5O8jTno/8d1PuL9c=,tag:cfUVQrFA2LPcUqYTGonwIg==,type:comment] +TAG=ENC[AES256_GCM,data:jZyI,iv:eQ6tIU/IbTsaHYbAM34NscgtTos8MJhnT6MKAh9KYdc=,tag:hzjKcaL8DYE4v7IaeQCGgw==,type:str] +MYSQL_DATABASE=ENC[AES256_GCM,data:UhrC,iv:rLVjQayrVIhT1mdI4Yfu3KGnWEvNNntw1sLsX4Bcqas=,tag:6VzFa4nx3Ig0fh/MV9kWiA==,type:str] +MYSQL_USER=ENC[AES256_GCM,data:gteXqw==,iv:mK2zP/ViZz2c8pftvIx1NPDHWWhvJIyuzxLCxc5aGQs=,tag:AEa8eLQ4Usbm6UtWzL9ChA==,type:str] +MYSQL_PASSWORD=ENC[AES256_GCM,data:VnXTVU9YETOQ2VI=,iv:6fRDRdefxIjkJn+axDf+bSxwVfyvPqzj9h0xBORW9as=,tag:bWLuzAYRy+KKEJEu763pCA==,type:str] +MYSQL_ROOT_PASSWORD=ENC[AES256_GCM,data:9sRKIkzrWYLXtIg=,iv:M6Ub+lArlLnJOg4Ypn9utJMbSjLtDjtm0U3jhkLjp0c=,tag:tOtI9noInjRlKMES7zQ8bQ==,type:str] +MYSQL_HOST=ENC[AES256_GCM,data:f/g=,iv:l3RVhdsBMGTtL7UgNrRouTK0fKTZBJzEUCx+ZNmpCuY=,tag:GPHN3m0Vl7y7WfgUQANWzw==,type:str] +DB_ADAPTER=ENC[AES256_GCM,data:IoB1HNlA,iv:2ONmsqTqmbOoA7kY7xV7HRxwtoBL/eqZugZr3NGl07c=,tag:bE9/SgCUxsYfWxToQKFplg==,type:str] +TEST_DB=ENC[AES256_GCM,data:pnA4V/trO2o=,iv:NSWohoQLuAy+rTikGBCBOsD0rX6eUlefVVNEq8Ad4M0=,tag:GWVKZie1FjuLDU3gQI9Bmw==,type:str] +sops_pgp__list_0__map_fp=B6125B16B0DD59F34D6975FBF885927FDA9C48E2 +sops_unencrypted_suffix=_unencrypted +sops_pgp__list_0__map_created_at=2021-09-07T19:03:01Z +sops_pgp__list_0__map_enc=-----BEGIN PGP MESSAGE-----\n\nhQIMAx1u4ocvSXxJARAAogJrncDOvrnQjtNQw4aqKayykk4sYippOuZXVKaGyrYP\nAqZmrsC0/ra830O+GHI8DuWJCUTndy1busQmAxIWNL4EZ30wbPeWd3KPrWXGrZ6E\nnof6TVXoXFmHMcgJcrsVknKVbhZaAyOmBOk9vnExzjwVQJXhM/xN0W9n2c5x0F0k\ns7iea2QjLghLBOg7xZp/ardNDl9K56XpIXRKQnuA4skWoGUL626PTy48vE2Gh5+l\nYgc3RLagDHTEr1zK5V6VwIEfiu6ODPzdlInJSUaGp1PtaL8gUDxOao8ke3McBidC\nAuwDTXTnIb4oeoqp32i7o/+2XymGUe8qXP1b0PoZ4gd2c5b66OxWgLmSaTulOday\nYtHWR26N+XhzP19mHFfZKIbjUATB6A0vawPQIzn3SGqyo8Y5Pmlu/cu3I3iBHzdo\nm0TO5b+6LZ3fIFthKLcelbKwEFiAdKVGW7CtUaIelZ/aT9c5poDVq8ewvT3v5Y89\nnUU1QqULRIKoQ18EJ9OQen9WC3VQnfk5XSYDc/DBzt12UjSdwdKUXlNKYDrgjKdi\n901Aab10VsjY8Eb+02R3Nvp+4bewdxegsUaA2U2F07Ahqzkby7M8GfbGYX28rqSR\ngU4dlfG/TDrIB/oiWIRVOXQY+u9jbKoFelc4n378Zu0Sv5uXrahgIKsnpQSySkjS\nXAGhOBQAhNA9no554zSwLdHT+XUitfHMjUw4TJsEu8P4t9cY/EGuaEDofx6OIkf1\nbfQPIJ/kZkD0OhJ5iMxkMBWcZhJjMkTi/qO0yTf/XfIrVLyrTc/u6G35hLYH\n=dSrj\n-----END PGP MESSAGE-----\n +sops_version=3.7.1 +sops_lastmodified=2021-09-07T19:03:03Z +sops_mac=ENC[AES256_GCM,data:LutzEHOdycJ+RUgztgQ53b8J+CRnxtoQv19744sXO75t4ZpEHhL/qYOKv3qXoRb6cuk5xTiCTKZrTA8wT8fyzQnzXh0HR/8wdiQLFfBcn5nbIW8gsjA+uoNaFk0hdqF2eHozpam3bL0m9SaXm4jh4mEHpZkVV6OxSaOUi5mmM1s=,iv:0uPfG/7QXuSyUuITUEnsy9fawyLR+QwrLtt/+3gM1NM=,tag:CNi5HzEqA9UACWEaqQ7IRA==,type:str] diff --git a/.env.production.enc b/.env.production.enc new file mode 100644 index 000000000..8b8a158b4 --- /dev/null +++ b/.env.production.enc @@ -0,0 +1,21 @@ +{ + "data": "ENC[AES256_GCM,data:R62C3mMXMCjBkQfcavFXjKOEeKSEEP5JD0X4UDq+KMfCi6uBbQim83EZF6sVm8z0rvLmufpxUo/9dpUknqBswz4XDLG5rGdOqh+FWN2yi9QIlsh9eoDQxc9B4VP8xqgzdTQNykTWd7Zz527TAZ2uOoCRC31Fd1nhk3vP/Hip2d9HZoOpaskeY/GkF9OdDSBwx/QxsaLBcWJ+046gmfN9iMkoeKIAZdEQ0TAiOQ/TZtV0TTUGbWFD3SLHSpw9oDvkkUHbwN27E9W2wFWtBkOtJCE/7sTiTCb0i0fSCpoJFF5XJyGfwQkSmTlmIU889K1ORPdXaSuuruO864dJXpQEDRAt2Ju5ia6PPk+KG3kwleTiT9bpD0VAZ8Inr4GgkH0PHIbcnTtrd9stAfyLWlFFMDspyWPtRgPe2nJh09nCIe/VrbFwb8Pku7v+ftg4Cbd5LUSIHcg5Cx+hVp6ZNqO61zC2EFwPUaPAZZ9nJ0fnFZzFw91DNixUbYCKKJImuO/nWcu0jUhyPkm+ui9P5qqmFXu/BjFvkwXmFXn41O+yFmwOdVrnxQGLxWJgGjTW8RVddkmdtQI7WMIPSJ23Mp5g32VWON8y0pcGsoWiIkXbrQ3V6lleAEApbkt4G84TEsgynkfAbGcXG7scY9oqoeZdQfxBJUmJlFmRlUpDVFENxd/MKSK2KdpHF0e7Xa5zVq1W39bUgnrrX5oFIGwAkNXrtbxhkmg276GyI6C6b4Q3LU3AWWkuRe82tSY=,iv:DjbQmYKBHHX6tMszFDA3xTVcSXVTMKxC0H2MOOLFjkI=,tag:YSIv2dCY8poMH3gWWiuUng==,type:str]", + "sops": { + "kms": null, + "gcp_kms": null, + "azure_kv": null, + "hc_vault": null, + "age": null, + "lastmodified": "2021-09-07T19:03:03Z", + "mac": "ENC[AES256_GCM,data:KhMpieYbrCPZI4ITYkvVjsMx9muSoKSR8JsDiGZhGsdDhkRTKruKWAClFbZbdVn/k79NtQ33mc6Cljt/Acwrh6pAVOG+k5hYH1DyWS4bcfi4gX6rNQhhWftd0MokkVXz1rYFHKEejOpdqlv3/ViCGUCSwrnIcLTFiLVU1L4lGW4=,iv:vp39lvrtWdQeqaXynVBt309kJD2mL46CQSz2EsCNnKI=,tag:1vbhg0MojtmrzMF5J8bdEg==,type:str]", + "pgp": [ + { + "created_at": "2021-09-07T19:03:03Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhQIMAx1u4ocvSXxJARAAhmC9vilg3C8XUhFvRBjaZ81HcIAbyBsM7Ufq8JrX4pLz\nnRC7TEc1ODERpiAqvRPGosMMGrAivwDeGcmwC3ftrvWlm9eYZjktx8zpC4FCGvyh\nClpZW9mpUG1VdTGibPlP90BRvdbJnVZpGVbkq4lri3InA0uES6Oe9WqGX7ybqMwR\nQCe9JpnguH6+XAlWFitptDD7Uq5Iy4eZhTO8tm/c5JOSpyGJ6xix10LIH7F9iq0K\nRajmkYkVRU5Shh521QksCAMoz7fk86ozfXgmiqxDpjsTkgwawAEypjqrup48Xp4e\nnLMZBUaZztaYYWVmrbZmxx7mJ/8XUxTiGdEujzmkUo4qRJKgRsVh90bKTXxJeFD1\nD5VnIDc8t7TNAyvTFVoRXNDtvBqp8uZKpYGKn0o3Y5aWwnHJAwoJR7ORlnYgaaHf\nuRdDec63BTraY6NTtLbXb4CRtfC1Db/J3gCqEZDv//MfWgV40yTx+AJ2RE+bbxpx\niX+RVaWmIwtBn2lebzAIdjLCGeXDd7Q2hYksgvGghIjoKdWm9rCXyFEP3r4wD0PH\nXHGhH5vjiTgxcfTgaEv07U1IYJOBbeSvYdRf6F6Etbf/PHfPD2WQYqLQ2XuHb4sz\nbqiSaVAZffdjph/tEfQYAH1DgzmwKHzZ77BE/0NlrlZdAaQ8REEgAS2yDi9lMoXS\nXAE+fR53GxgrZDNRuBeGGX32151XdByIJXTiXJa58tqf3pSL1VubRcfQW7rjaK0U\nN2A1kpyFp5mSlaYy5wXqvpA+y8sgETVOE8ER5LYtx7NSNPkHkyU1vcaSIkLg\n=PPZl\n-----END PGP MESSAGE-----\n", + "fp": "B6125B16B0DD59F34D6975FBF885927FDA9C48E2" + } + ], + "unencrypted_suffix": "_unencrypted", + "version": "3.7.1" + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index bf61a9173..90f359321 100644 --- a/.gitignore +++ b/.gitignore @@ -31,12 +31,15 @@ # Redis dumps dump.rdb +.env .env.development .env.test .env.production /public/packs /public/packs-test /node_modules +/ops/*-deploy.tmpl.yaml +/ops/*-deploy.yaml yarn-debug.log* .yarn-integrity fits.log diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8fe3db390..ce61de88a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,11 +2,11 @@ stages: - build - lint - test - - deploy + - staging variables: REGISTRY_HOST: registry.gitlab.com - REGISTRY_URI: /notch8/ngao + REGISTRY_URI: /notch8/gbh before_script: - export TAG=${CI_COMMIT_SHORT_SHA} @@ -25,97 +25,96 @@ build: tags: - local -lint: - stage: lint - image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA - variables: - PRONTO_GITLAB_API_ENDPOINT: "https://gitlab.com/api/v4" - before_script: - - export PRONTO_GITLAB_API_PRIVATE_TOKEN=$GITLAB_API_TOKEN - script: - - cd /app/samvera/hyrax-webapp && bundle exec pronto run -f gitlab - tags: - - docker +# lint: +# stage: lint +# image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA +# variables: +# PRONTO_GITLAB_API_ENDPOINT: "https://gitlab.com/api/v4" +# before_script: +# - export PRONTO_GITLAB_API_PRIVATE_TOKEN=$GITLAB_API_TOKEN +# script: +# - cd /app/samvera/hyrax-webapp && bundle exec pronto run -f gitlab +# tags: +# - docker -test: - stage: test - image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA - services: - - name: mysql:5.7 - alias: db - - name: bitnami/solr:8 - alias: solr - entrypoint: ["docker-entrypoint.sh", "solr-precreate", "test"] - - name: redis:5-alpine - alias: redis - - name: samvera/fcrepo4:4.7.5 - alias: fcrepo - - name: bitnami/zookeeper:3 - alias: zk +# test: +# stage: test +# image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA +# services: +# - name: mysql:5.7 +# alias: db +# - name: bitnami/solr:8 +# alias: solr +# entrypoint: ["docker-entrypoint.sh", "solr-precreate", "test"] +# - name: redis:5-alpine +# alias: redis +# - name: samvera/fcrepo4:4.7.5 +# alias: fcrepo +# - name: bitnami/zookeeper:3 +# alias: zk +# variables: +# MYSQL_DATABASE: GBH +# MYSQL_ROOT_PASSWORD: password +# MYSQL_USER: GBH_test +# MYSQL_PASSWORD: password +# MYSQL_HOST: mysql +# DATABASE_ADAPTER: mysql2 +# DATABASE_HOST: mysql +# DATABASE_NAME: GBH +# DATABASE_PASSWORD: password +# DATABASE_USER: GBH_test +# TEST_DB: GBH +# script: +# - cd /app/samvera/hyrax-webapp && bundle exec rake db:create db:schema:load spec +# tags: +# - docker +# variables: +# GIT_STRATEGY: none + +gbh.staging: + stage: staging + extends: + - .deploy + environment: + name: gbh.staging + url: http://gbh-staging.$KUBE_INGRESS_BASE_DOMAIN + on_stop: gbh.staging.stop + only: + refs: + - n8-staging variables: - MYSQL_DATABASE: GBH - MYSQL_ROOT_PASSWORD: password - MYSQL_USER: GBH_test - MYSQL_PASSWORD: password - MYSQL_HOST: mysql - DATABASE_ADAPTER: mysql2 - DATABASE_HOST: mysql - DATABASE_NAME: GBH - DATABASE_PASSWORD: password - DATABASE_USER: GBH_test - TEST_DB: GBH + DEPLOY_IMAGE: $CI_REGISTRY_IMAGE + DEPLOY_TAG: $CI_COMMIT_SHORT_SHA + WORKER_IMAGE: $CI_REGISTRY_IMAGE + HELM_EXPERIMENTAL_OCI: 1 + HELM_RELEASE_NAME: gbh-staging + KUBE_NAMESPACE: gbh-staging + HELM_EXTRA_ARGS: > + --values ops/staging-deploy.yaml script: - - cd /app/samvera/hyrax-webapp && bundle exec rake db:create db:schema:load spec + - export KUBECONFIG=$KUBECONFIG_R2 + - envsubst < ops/staging-deploy.tmpl.yaml > ops/staging-deploy.yaml + - ./bin/helm_deploy gbh-staging gbh-staging tags: - - docker - variables: - GIT_STRATEGY: none - -# review: -# stage: review -# type: deploy -# environment: -# name: $CI_COMMIT_REF_SLUG -# url: http://web.$CI_PROJECT_NAME-$CI_COMMIT_REF_SLUG.staging.$SITE_URI_BASE -# on_stop: stop_review -# only: -# - branches -# except: -# - main -# script: -# - ./chart/bin/deploy review $CI_COMMIT_REF_SLUG -# tags: -# - local - -# stop_review: -# stage: review -# script: -# - ./chart/bin/remove review $CI_COMMIT_REF_SLUG - -# when: manual -# environment: -# name: $CI_PROJECT_NAME-$CI_BUILD_REF_SLUG -# action: stop + - local -# staging: -# stage: deploy -# type: deploy -# script: -# - ./chart/bin/deploy staging $CI_COMMIT_REF_SLUG -# only: -# - main -# tags: -# - local +gbh.staging.stop: + stage: staging + extends: + - .deploy + environment: + name: gbh.staging + url: http://gbh-staging.$KUBE_INGRESS_BASE_DOMAIN + action: stop + when: manual + allow_failure: true + script: + - export KUBECONFIG=$KUBECONFIG_GBH + - ./bin/helm_delete gbh-staging gbh-staging + tags: + - local -# production: -# stage: deploy -# type: deploy -# script: -# - ./chart/bin/deploy production $CI_COMMIT_REF_SLUG -# when: manual -# only: -# - main -# tags: -# - local +.deploy: + image: dtzar/helm-kubectl:3.5.3 \ No newline at end of file diff --git a/.sops.yaml b/.sops.yaml index e536b9fdd..31c502a70 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -1,3 +1,3 @@ --- creation_rules: - - pgp: "CHANGEME" + - pgp: "B6125B16B0DD59F34D6975FBF885927FDA9C48E2" diff --git a/bin/decrypt-secrets b/bin/decrypt-secrets old mode 100644 new mode 100755 index 955415d3b..8f0f46cb9 --- a/bin/decrypt-secrets +++ b/bin/decrypt-secrets @@ -5,10 +5,13 @@ parent_dir = File.dirname(__dir__) Dir.chdir(File.join(parent_dir)) [ - ".env.*", + # TODO: Troubleshoot local env encrypt/decrypt + # ".env", + # ".env.*", "chart/*-values.yaml", "ops/kube_config.yml", "ops/.backend", + "ops/*-deploy.tmpl.yaml", "ops/k8s/*-values.yaml" ].each do |files| Dir.glob(files).each do |file| diff --git a/bin/encrypt-secrets b/bin/encrypt-secrets old mode 100644 new mode 100755 index f37487927..8d57f46b2 --- a/bin/encrypt-secrets +++ b/bin/encrypt-secrets @@ -4,10 +4,13 @@ parent_dir = File.dirname(__dir__) [ - ".env.*", + # TODO: Troubleshoot local env encrypt/decrypt + # ".env", + # ".env.*", "chart/*-values.yaml", "ops/kube_config.yml", "ops/.backend", + "ops/*-deploy.tmpl.yaml", "ops/k8s/*-values.yaml" ].each do |files| Dir.glob(files).each do |file| diff --git a/bin/helm_deploy b/bin/helm_deploy new file mode 100755 index 000000000..f2d58e4ed --- /dev/null +++ b/bin/helm_deploy @@ -0,0 +1,45 @@ +#!/bin/sh + +# This script wraps up helm deployment. It is meant as a clear starting point for +# commandline deployment or CI based deployment. It requires the following ENV vars be set +# +# CHART_VERSION: this is the version of the hyrax chart you want to deploy. default - 0.22.0 +# DEPLOY_IMAGE: this is the build image that runs the rails application. Typically this would run puma or passenger. eg: samvera/hyrax or ghcr.io/samvera/hyku. Defaults to gcrh.io/samvera/hyku +# WORKER_IMAGE: this is the worker target, usually built from the same Dockerfile as DEPLOY_IMAGE. eg: samvera/hyrax/worker or ghcr.io/samvera/hyku/worker. Defaults to gcrh.io/samvera/hyku/worker +# DEPLOY_TAG: name of of the tag you want to deploy for deploy image. eg: "latest" or "v3.0.1" or "f123asdf1". Defaults to latest +# WORKER_TAG: name of of the tag you want to deploy for deploy image. eg: "latest" or "v3.0.1" or "f123asdf1". Defaults to DEPLOY_TAG +# HELM_EXTRA_ARGS: any additional arguments you'd like passed to helm upgrade directly. can be blank. + +if [ -z "$1" ] || [ -z "$2" ] +then + echo './bin/deploy RELEASE_NAME NAMESPACE' + exit 1 +fi +release_name="${1}" +namespace="${2}" + +CHART_VERSION="${CHART_VERSION:-0.22.0}" + +DEPLOY_IMAGE="${DEPLOY_IMAGE:-ghcr.io/samvera/hyku}" +WORKER_IMAGE="${WORKER_IMAGE:-ghcr.io/samvera/hyku/worker}" +DEPLOY_TAG="${DEPLOY_TAG:-latest}" +WORKER_TAG="${WORKER_TAG:-$DEPLOY_TAG}" + +# helm chart pull ghcr.io/samvera/hyrax/hyrax-helm:$CHART_VERSION +# helm chart export ghcr.io/samvera/hyrax/hyrax-helm:$CHART_VERSION + +helm repo update + +helm upgrade \ + --install \ + --atomic \ + --timeout 15m0s \ + --set image.repository="$DEPLOY_IMAGE" \ + --set image.tag="$DEPLOY_TAG" \ + --set worker.image.repository="$WORKER_IMAGE" \ + --set worker.image.tag="$DEPLOY_TAG" \ + $HELM_EXTRA_ARGS \ + --namespace="$namespace" \ + --create-namespace \ + "$release_name" \ + hyrax diff --git a/chart/Chart.yaml b/chart/Chart.yaml index f5aec2afb..5b690ecb4 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -1,23 +1,13 @@ apiVersion: v1 appVersion: "0.0.1" description: A Helm chart for GBH -name: GBH +name: gbh-hyrax version: 0.0.1 dependencies: - -- name: solr - version: 1.5.2 - repository: https://charts.helm.sh/incubator - condition: solr.enabled - - - name: redis version: 11.0.4 repository: https://charts.bitnami.com/bitnami condition: redis.enabled - - - - name: mariadb version: 7.3.14 repository: https://charts.helm.sh/stable diff --git a/chart/bin/deploy b/chart/bin/deploy old mode 100644 new mode 100755 index e60ecf124..5288e7a0d --- a/chart/bin/deploy +++ b/chart/bin/deploy @@ -2,7 +2,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd $DIR/../../chart -REPO=$(basename $(git config --get remote.origin.url)) +REPO="$(echo $(basename $(git config --get remote.n8.url)) | tr '[:upper:]' '[:lower:]')" NAMESPACE=${REPO%.git} if [ -z "$1" ] || [ -z "$2" ] diff --git a/chart/bin/remove b/chart/bin/remove old mode 100644 new mode 100755 index 42de547d3..5fbc06347 --- a/chart/bin/remove +++ b/chart/bin/remove @@ -12,4 +12,4 @@ then fi raise 'refusing to remove production' if $1 == 'production' -helm uninstall --namespace $NAMESPACE-$1 $1 . -f $1-values.yaml --set rails.image.tag=$2 +helm uninstall --namespace $NAMESPACE-$1 $1 . #-f $1-values.yaml --set rails.image.tag=$2 diff --git a/chart/charts/mariadb-7.3.14.tgz b/chart/charts/mariadb-7.3.14.tgz new file mode 100644 index 0000000000000000000000000000000000000000..ba253b88a562bddb5ce87002e20bc227fc29c712 GIT binary patch literal 22931 zcmV)(K#RX0iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMa0cH20TFgkzhDQe~3jXh&Zw)1O7JDKx2KRWH(ox~fbyTARi zeQbz?B#bG74S;skan8A~abNE~$#+lyNRT2a%2pD0kGVS?i^PvY0ad6f6sqP#FcOW2 z&t^oZ!Gg@=KW+Tk+uPfF{O}?EySKMj{dfOh?~(i4yx%`~@bLbhVDpgFJegEP{Aq9R zrtMS*_d9t=%6_6kGMa81R~**#UKHIiIhP^)0i)41d>nGB@uK0-?8Z|UI4iu z42R_4>H&mtCKVNf9!;h!p(B`bmBzgppVJY{luG4jIJ9Io;9}Zip~jp0GG;0vb2bS1 zd`P6EN)DwW;kk5mhHO4{e+T#}I5^n*aB%;@AWf#dbGo?XB9fzC0JfC=YaxG)dqIxR zUI6ArD)^kL8O>xbAsRwg$IJ#c<3O7-HOs~bnT68*8$>)jry?jitf^CUHAHZtQ~Dw0 zLQ&DXaBJ-G;QnC$LGNo*!N2a~7TE=fGb%sN2GGv`AMD@XdtBxJkM|$n@&7OJY;D2I zn2$*e&yl2o&B>HPN(7lxMTP8bZNYI&iKM%ba797QhypcZ5-wRB1HYg`ut>kBOOOwu5q4?sYs}z za=`d7;-MTWBG2WJM01ufy&(m5V;bhD4(3^`SQ^tH??(;|I`S79_z?p4)Uf(|2Syw$ z0iv0`*$j~9i+rbWj75>D=>+SKdH^EFw5`ZEzD>DgitG24HhKVxOml%Gi0GJ+B-q~z z#KGepz}6O=q%>rc1(0H@j(N<&h57uPPDmCjsn^=Q!}yXc