#61: Let all webapps run under single Tomcat container

Notes:
- the new db-init scripts for userdetails service are copied over from ala-cas-5 project. Ideally we checkout the project (however the sources have not changed since 2017)
- living-atlas-1 service depends_on attribute is commented out as it did not seem to pick up. To be looked at.
- mysql-*-init services can and should be joined into one service. In principe they can also be moved to mysql's initdb.d procedure. We would have to ensure the execution order though.

.gitignore

@@ -7,3 +7,5 @@ node_modules
 *.log
 
 generated
+
+.DS_Store

config/collectory/collectory.yml

@@ -5,7 +5,6 @@ security:
     clientId: collections
   oidc:
     clientId: collections
-    discoveryUri: ${common.protocol}://auth.${common.domain}/cas/oidc/.well-known
 webservice:
   client-id: collections
   client-secret: xxx
@@ -32,7 +31,7 @@ collectionsMap:
   centreMapLon: 4.4
   defaultZoom: 9
 dataSource:
-  url: jdbc:mysql://root:my-super-secret-password@mysql:3306/collectory?autoReconnect=true&connectTimeout=0&useUnicode=true&characterEncoding=UTF-8&allowPublicKeyRetrieval=true&useSSL=False&serverTimezone=Europe/Paris
+  url: jdbc:mysql://collectory:my-super-secret-password@mysql:3306/collectory?autoReconnect=true&connectTimeout=0&useUnicode=true&characterEncoding=UTF-8&allowPublicKeyRetrieval=true&useSSL=False&serverTimezone=Europe/Paris
 defaultLocale: en
 disableAlertLinks: true
 disableCAS: false

config/image-service/image-service.yml

@@ -63,3 +63,31 @@ collectory:
   baseURL: "${common.protocol}://collections.${common.domain}"
 biocache:
   baseURL: "${common.protocol}://biocache.${common.domain}"
+
+dataSource:
+  driverClassName: org.postgresql.Driver
+  dialect: org.hibernate.spatial.dialect.postgis.PostgisPG9Dialect
+  url: 'jdbc:postgresql://postgis:5432/images'
+  username: 'images'
+  password: 'my-super-secret-password'
+  pooled: true
+  jmxExport: true
+  dbCreate: update
+  properties:
+    jmxEnabled: true
+    initialSize: 5
+    maxActive: 50
+    minIdle: 5
+    maxIdle: 25
+    maxWait: 10000
+    maxAge: 600000
+    timeBetweenEvictionRunsMillis: 5000
+    minEvictableIdleTimeMillis: 60000
+    validationQuery: SELECT 1
+    validationQueryTimeout: 3
+    validationInterval: 15000
+    testOnBorrow: true
+    testWhileIdle: true
+    testOnReturn: false
+    jdbcInterceptors: ConnectionState
+    defaultTransactionIsolation: 2 # TRANSACTION_READ_COMMITTED

config/logger/logger.yml

@@ -11,4 +11,7 @@ webservice:
   client-secret: xxx
 
 grails:
-  serverURL: ${common.protocol}://logger.${common.domain}
+  serverURL: ${common.protocol}://logger.${common.domain}
+
+dataSource:
+  url: jdbc:mysql://logger:my-super-secret-password@mysql:3306/logger?autoReconnect=true&connectTimeout=0&useUnicode=true&characterEncoding=UTF-8&allowPublicKeyRetrieval=true&useSSL=False&serverTimezone=Europe/Paris

config/spatial-service/spatial-service.yml

@@ -44,6 +44,8 @@ dataSource:
   driverClassName: org.postgresql.Driver
   dialect: org.hibernate.spatial.dialect.postgis.PostgisPG9Dialect
   url: 'jdbc:postgresql://postgis:5432/layersdb'
+  username: 'spatial'
+  password: 'my-super-secret-password'
   pooled: true
   jmxExport: true
   dbCreate: update

config/userdetails/db-init/V1__init.sql

@@ -0,0 +1,212 @@
+-- MySQL dump 10.13  Distrib 5.5.55, for debian-linux-gnu (x86_64)
+--
+-- Host: localhost    Database: emmet
+-- ------------------------------------------------------
+-- Server version	5.5.55-0ubuntu0.14.04.1
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `auth_key`
+--
+
+DROP TABLE IF EXISTS `auth_key`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `auth_key` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `version` bigint(20) NOT NULL,
+  `auth_key` varchar(255) NOT NULL,
+  `mobile_user_id` bigint(20) NOT NULL,
+  PRIMARY KEY (`id`),
+  KEY `FK5563974818294D56` (`mobile_user_id`),
+  CONSTRAINT `FK5563974818294D56` FOREIGN KEY (`mobile_user_id`) REFERENCES `mobile_user` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `authorised_system`
+--
+
+DROP TABLE IF EXISTS `authorised_system`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `authorised_system` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `version` bigint(20) NOT NULL,
+  `host` varchar(255) NOT NULL,
+  `description` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=192 DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `authorities`
+--
+
+DROP TABLE IF EXISTS `authorities`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `authorities` (
+  `userid` int(11) NOT NULL,
+  `authority` varchar(30) NOT NULL,
+  PRIMARY KEY (`userid`,`authority`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `c3p0TestTable`
+--
+
+DROP TABLE IF EXISTS `c3p0TestTable`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `c3p0TestTable` (
+  `a` char(1) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `identities`
+--
+
+DROP TABLE IF EXISTS `identities`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `identities` (
+  `userid` int(11) NOT NULL,
+  `identityuri` varchar(255) NOT NULL,
+  `domain` varchar(255) NOT NULL,
+  PRIMARY KEY (`userid`,`identityuri`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `mobile_user`
+--
+
+DROP TABLE IF EXISTS `mobile_user`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `mobile_user` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `version` bigint(20) NOT NULL,
+  `user_name` varchar(255) NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `passwords`
+--
+
+DROP TABLE IF EXISTS `passwords`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `passwords` (
+  `userid` int(11) NOT NULL,
+  `password` varchar(255) NOT NULL,
+  `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  `expiry` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
+  `status` varchar(10) NOT NULL,
+  PRIMARY KEY (`userid`,`password`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `profiles`
+--
+
+DROP TABLE IF EXISTS `profiles`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `profiles` (
+  `userid` int(11) NOT NULL,
+  `property` varchar(255) NOT NULL,
+  `value` text NOT NULL,
+  PRIMARY KEY (`userid`,`property`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `role`
+--
+
+DROP TABLE IF EXISTS `role`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `role` (
+  `role` varchar(255) NOT NULL,
+  `description` varchar(255) NOT NULL,
+  PRIMARY KEY (`role`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `role`
+--
+
+LOCK TABLES `role` WRITE;
+/*!40000 ALTER TABLE `role` DISABLE KEYS */;
+INSERT INTO `role` VALUES ('ROLE_ABRS_ADMIN',''),('ROLE_ABRS_INSTITUTION',''),('ROLE_ADMIN','Admin role for ALA staff'),('ROLE_API_EDITOR','Enables a user to update the online web service API'),('ROLE_APPD_USER','APPD user'),('ROLE_AVH_ADMIN',''),('ROLE_AVH_CLUB',''),('ROLE_COLLECTION_ADMIN',''),('ROLE_COLLECTION_EDITOR',''),('ROLE_COLLECTORS_ADMIN',''),('ROLE_FC_ADMIN','Admin role for the Field Capture webapp'),('ROLE_FC_OFFICER','Field Capture officer role'),('ROLE_FC_READ_ONLY','Provides read only access to all projects in the field capture system.'),('ROLE_IMAGE_ADMIN',''),('ROLE_SPATIAL_ADMIN',''),('ROLE_SYSTEM_ADMIN',''),('ROLE_USER',''),('ROLE_VP_ADMIN',''),('ROLE_VP_TEST_ADMIN','The admin role for BVP Test server'),('ROLE_VP_VALIDATOR','');
+/*!40000 ALTER TABLE `role` ENABLE KEYS */;
+UNLOCK TABLES;
+
+--
+-- Table structure for table `user_role`
+--
+
+DROP TABLE IF EXISTS `user_role`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `user_role` (
+  `user_id` bigint(20) NOT NULL,
+  `role_id` varchar(255) NOT NULL,
+  PRIMARY KEY (`user_id`,`role_id`),
+  KEY `FK143BF46AF129182D` (`role_id`),
+  CONSTRAINT `FK143BF46AF129182D` FOREIGN KEY (`role_id`) REFERENCES `role` (`role`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `users`
+--
+
+DROP TABLE IF EXISTS `users`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `users` (
+  `userid` int(11) NOT NULL AUTO_INCREMENT,
+  `username` varchar(255) DEFAULT NULL,
+  `firstname` varchar(255) DEFAULT NULL,
+  `lastname` varchar(255) DEFAULT NULL,
+  `email` varchar(255) DEFAULT NULL,
+  `activated` char(1) NOT NULL,
+  `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  `expiry` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
+  `locked` char(1) NOT NULL,
+  `temp_auth_key` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`userid`),
+  UNIQUE KEY `username` (`username`)
+) ENGINE=InnoDB AUTO_INCREMENT=43954 DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2017-05-28  6:17:37

config/userdetails/db-init/V2__password_type.sql

@@ -0,0 +1,27 @@
+
+
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+ALTER TABLE `passwords`
+  ALTER COLUMN expiry SET DEFAULT '2038-01-01 00:00:00',
+  ADD COLUMN type VARCHAR(255)
+    AFTER password
+;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+UPDATE `passwords` SET `type` = 'legacy';
+/*!40101 SET character_set_client = utf8 */;
+
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+ALTER TABLE `passwords`
+  ALTER COLUMN type SET DEFAULT 'bycrypt';
+/*!40101 SET character_set_client = utf8 */;
+
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+ALTER TABLE `passwords`
+  MODIFY COLUMN type VARCHAR(255) NOT NULL;
+/*!40101 SET character_set_client = utf8 */;

config/userdetails/db-init/V3__nullable_expiry.sql

@@ -0,0 +1,7 @@
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+ALTER TABLE `users`
+  MODIFY COLUMN expiry TIMESTAMP NULL DEFAULT NULL;
+/*!40101 SET character_set_client = utf8 */;
+
+UPDATE `users` SET expiry = null WHERE expiry = 0;

config/userdetails/db-init/V4__user_auto_timestamp.sql

@@ -0,0 +1,6 @@
+ALTER TABLE `users` CHANGE `created` `created` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP;
+ALTER TABLE `users` ADD `date_created` TIMESTAMP NULL;
+UPDATE `users` SET `date_created` = `created`;
+ALTER TABLE `users` CHANGE `date_created` `date_created` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP;
+ALTER TABLE `users` CHANGE `created` `last_updated` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP;
+ALTER TABLE `users` ADD `last_login` TIMESTAMP NULL;

config/userdetails/db-init/V5__make_userdetails_domain_validatable.sql

@@ -0,0 +1,14 @@
+/* Disable created on update */
+ALTER TABLE `passwords` CHANGE `created` `created` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP;
+/* update passwords to nullable expiry and remove all invalid expiryies */
+ALTER TABLE `passwords`
+MODIFY COLUMN expiry TIMESTAMP NULL DEFAULT NULL;
+
+UPDATE `passwords` SET expiry = null WHERE expiry = 0 or expiry = '2038-01-01 00:00:00';
+
+/** switch userid from int(11) to bigint(20) for GORM default id size */
+ALTER TABLE authorities MODIFY userid BIGINT(20);
+ALTER TABLE identities MODIFY userid BIGINT(20);
+ALTER TABLE passwords MODIFY userid BIGINT(20);
+ALTER TABLE profiles MODIFY userid BIGINT(20);
+ALTER TABLE users MODIFY userid BIGINT(20) AUTO_INCREMENT;

config/userdetails/db-init/V6__normalise_states.sql

@@ -0,0 +1,15 @@
+update profiles set value = 'ACT' where property = 'state' and value = 'Australian Capital Territory';
+update profiles set value = 'NSW' where property = 'state' and value = 'New South Wales';
+update profiles set value = 'NT' where property = 'state' and value = 'Northern Territory';
+update profiles set value = 'QLD' where property = 'state' and value = 'Queensland';
+update profiles set value = 'SA' where property = 'state' and value = 'South Australia';
+update profiles set value = 'TAS' where property = 'state' and value = 'Tasmania';
+update profiles set value = 'VIC' where property = 'state' and value = 'Victoria';
+update profiles set value = 'WA' where property = 'state' and value = 'Western Australia';
+update profiles set value = '' where property = 'state' and value = 'N/A';
+update profiles set value = '' where property = 'state' and value = 'Select State';
+
+insert into profiles (userid, property, value) select userid, 'country', 'AU' from profiles where property = 'state' and `value` IN ('ACT', 'NSW', 'NT', 'QLD', 'SA', 'TAS', 'VIC', 'WA');
+insert into profiles (userid, property, value) select userid, 'country', 'US' from profiles where property = 'state' and `value` IN ('CT', 'PA');
+insert into profiles (userid, property, value) select userid, 'country', '' from profiles where property = 'state' and `value` NOT IN ('ACT', 'NSW', 'NT', 'QLD', 'SA', 'TAS', 'VIC', 'WA', 'CT', 'PA');
+

config/userdetails/db-init/V7__fix_user_role_fk.sql

@@ -0,0 +1,2 @@
+DELETE ur FROM user_role ur WHERE NOT EXISTS (SELECT u.userid FROM users u WHERE u.userid = ur.user_id);
+ALTER TABLE user_role ADD CONSTRAINT fk_users_userid FOREIGN KEY (user_id) REFERENCES users(userid) ON DELETE CASCADE;

config/userdetails/db-init/V8__update_charsets_for_latin1_tables.sql

@@ -0,0 +1,7 @@
+SET FOREIGN_KEY_CHECKS=0;
+ALTER TABLE `user_role` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `role` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `mobile_user` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `authorised_system` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `auth_key` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+SET FOREIGN_KEY_CHECKS=1;

config/userdetails/db-init/V9__update_charsets_for_remaining_tables.sql

@@ -0,0 +1,9 @@
+-- Update all tables to utf8mb4 for good measure
+SET FOREIGN_KEY_CHECKS=0;
+ALTER TABLE `authorities` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `c3p0TestTable` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `identities` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `passwords` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `profiles` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `users` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+SET FOREIGN_KEY_CHECKS=1;