From b8019254381a7b8c341e722b5e59da0ce4f6496b Mon Sep 17 00:00:00 2001 From: Prasad Talasila Date: Tue, 7 Nov 2023 10:34:38 +0100 Subject: [PATCH] Update services installation (#240) - Adds MQTT service to installation scripts. The MQTT service is directly installed on the OS using apt-get package manager - Updates documentation --- deploy/services/README.md | 42 +++++ deploy/services/mqtt-default.conf | 2 + deploy/services/package.json | 13 ++ .../two-machine => services}/services.js | 30 +++- .../two-machine => services}/services.yml | 3 + deploy/vagrant/two-machine/README.md | 38 ++--- deploy/vagrant/two-machine/package.json | 8 - docs/FAQ.md | 47 +++++- docs/admin/services.md | 143 ++++++------------ docs/admin/vagrant/two-machines.md | 31 +--- docs/thanks.md | 6 +- 11 files changed, 194 insertions(+), 169 deletions(-) create mode 100644 deploy/services/README.md create mode 100644 deploy/services/mqtt-default.conf create mode 100644 deploy/services/package.json rename deploy/{vagrant/two-machine => services}/services.js (76%) rename deploy/{vagrant/two-machine => services}/services.yml (79%) delete mode 100644 deploy/vagrant/two-machine/package.json diff --git a/deploy/services/README.md b/deploy/services/README.md new file mode 100644 index 000000000..62ce64329 --- /dev/null +++ b/deploy/services/README.md @@ -0,0 +1,42 @@ +# Platform Services + +It is recommended to install certain third-party software for use +by digital twins running inside the DTaaS software. + +The installation scripts in this directory install: + +* **Influx** time-series database and dashboard service +* **Grafana** visualization and dashboard service +* **RabbitMQ** AMQP broker and its' management interface +* Eclipse Mosquitto **MQTT** broker + +## Configure and Install + +The first step in installation is to specify the config of the services. +There are two configuration files. The __services.yml__ contains most +of configuration settings. The __mqtt-default.conf__ file contains +the MQTT listening port. Update these two config files before proceeding +with the installation of the services. + +```bash +yarn install +node services.js +``` + +## Use + +After the installation is complete, you can see the following services active +at the following ports / URLs. + +| service | external url | +|:---|:---| +| Influx | services.foo.com | +| Grafana | services.foo.com:3000 | +| RabbitMQ Broker | services.foo.com:5672 | +| RabbitMQ Broker Management Website | services.foo.com:15672 | +| MQTT Broker | services.foo.com:1883 | +|| + +The firewall and network access settings of corporate / cloud network need to be +configured to allow external access to the services. Otherwise the users of DTaaS +will not be able to utilize these services from their user workspaces. diff --git a/deploy/services/mqtt-default.conf b/deploy/services/mqtt-default.conf new file mode 100644 index 000000000..d1e2a74d3 --- /dev/null +++ b/deploy/services/mqtt-default.conf @@ -0,0 +1,2 @@ +listener 1883 +password_file /etc/mosquitto/passwd \ No newline at end of file diff --git a/deploy/services/package.json b/deploy/services/package.json new file mode 100644 index 000000000..7378a6aad --- /dev/null +++ b/deploy/services/package.json @@ -0,0 +1,13 @@ +{ + "name": "@into-cps-association/dtaas-install-services", + "version": "0.0.1", + "description": "Install platform services for the DTaaS software", + "author": "Prasad Talasila", + "private": false, + "type": "module", + "dependencies": { + "chalk": "^5.3.0", + "execa": "^8.0.1", + "js-yaml": "^4.1.0" + } +} diff --git a/deploy/vagrant/two-machine/services.js b/deploy/services/services.js similarity index 76% rename from deploy/vagrant/two-machine/services.js rename to deploy/services/services.js index a4284bcd4..4fc960b55 100755 --- a/deploy/vagrant/two-machine/services.js +++ b/deploy/services/services.js @@ -15,7 +15,7 @@ const sleep = (ms) => }); try { - console.log(chalk.blue("Load services configuration")); + log(chalk.blue("Load services configuration")); config = await yaml.load(fs.readFileSync('services.yml', 'utf8')); log(chalk.green("configuration loading is successful and config is a valid yaml file")); } catch (e) { @@ -84,7 +84,7 @@ await $$`docker run -d \ grafana/grafana:10.1.4`; log(chalk.green("Grafana server docker container started successfully")); -console.log(chalk.blue("Wait one minute for Grafana server to bootstrap")); +log(chalk.blue("Wait one minute for Grafana server to bootstrap")); await sleep(60000); //60 seconds await $$`docker exec grafana grafana-cli admin reset-admin-password ${grafanaConfig.password}`; @@ -102,17 +102,37 @@ try { await $$`docker rm rabbitmq-server`; } catch (e) { } -//await $$`docker run -d --name rabbitmq-server -p 5672:5672 -p 15672:15672 rabbitmq:3-management`; + log(chalk.green("Start RabbitMQ server docker container")); await $$`docker run -d --name rabbitmq-server \ -p ${rabbitmqConfig.ports.main}:5672 \ -p ${rabbitmqConfig.ports.management}:15672 rabbitmq:3-management`; log(chalk.green("RabbitMQ server docker container started successfully\n")); -console.log(chalk.blue("Wait 2 minutes for RabbitMQ server to bootstrap")); +log(chalk.blue("Wait 2 minutes for RabbitMQ server to bootstrap")); await sleep(120000); //120 seconds let args = [rabbitmqConfig.username, rabbitmqConfig.password]; -//console.log(chalk.blue("Add ${rabbitmqConfig.username} user and give permission to ${rabbitmqConfig.vhost} vhost")); +log(chalk.blue( + "Add %s user and give permission to %s vhost"), + rabbitmqConfig.username, rabbitmqConfig.vhost); await $$`docker exec rabbitmq-server rabbitmqctl add_user ${args}`; await $$`docker exec rabbitmq-server rabbitmqctl set_permissions -p ${rabbitmqConfig.vhost} ${rabbitmqConfig.username} ".*" ".*" ".*"`; + +//--------------- +log(chalk.blue("Install and start MQTT server")); +const mqttConfig = config.services.mqtt; + +log(chalk.blue("Attempt to install mosquitto MQTT server using apt-get package manager")); +await $$`sudo apt-get install -y mosquitto mosquitto-clients`; +log(chalk.blue("Create user account for %s in MQTT server"), mqttConfig.username); +await $$`sudo sudo mosquitto_passwd -c -b /etc/mosquitto/passwd ${mqttConfig.username} ${mqttConfig.password}`; +await $$`sudo chown root:mosquitto /etc/mosquitto/passwd`; +await $$`sudo chmod 660 /etc/mosquitto/passwd`; + +log(chalk.blue("Set MQTT listening port configuration")); +await $$`sudo cp mqtt-default.conf /etc/mosquitto/conf.d/default.conf`; +await $$`sudo chmod 664 /etc/mosquitto/conf.d/default.conf`; +await $$`sudo chown root:mosquitto /etc/mosquitto/conf.d/default.conf`; +await $$`sudo systemctl restart mosquitto`; +await $$`sudo systemctl status mosquitto`; diff --git a/deploy/vagrant/two-machine/services.yml b/deploy/services/services.yml similarity index 79% rename from deploy/vagrant/two-machine/services.yml rename to deploy/services/services.yml index 89fa3d3d6..1ce7191f9 100644 --- a/deploy/vagrant/two-machine/services.yml +++ b/deploy/services/services.yml @@ -17,3 +17,6 @@ services: password: "dtaas1357" # need to have letters and numbers datapath: "/home/prasad/git/prasadtalasila/DTaaS/deploy/vagrant/two-machine/influx" #no spaces in the path port: 80 + mqtt: # usernames and passwords are in deploy/config/services/mqtt/config/password + username: "dtaas" + password: "dtaas" diff --git a/deploy/vagrant/two-machine/README.md b/deploy/vagrant/two-machine/README.md index 2bfb4cb32..393189e04 100644 --- a/deploy/vagrant/two-machine/README.md +++ b/deploy/vagrant/two-machine/README.md @@ -15,10 +15,11 @@ server1. The _workspaces.sh_ contains installation commands for provisioning user workspaces. If you desire to have more users, you need to modify this shell script. -The default installation setup also installs InfluxDB, Grafana and RabbitMQ -services on server2. If you would like to install more services, you can create -shell scripts to install the same on server2. If you have these scripts ready, -you can place them in this directory and invoke them from _services.sh_ script. +The default installation setup also installs InfluxDB, Grafana, RabbitMQ +and MQTT services on server2. If you would like to install more services, +you can create +scripts to install the same on server2. If you have these scripts ready, +you can place them in this directory and invoke them from _services.js_ script. ## Create Base Vagrant Box @@ -60,37 +61,19 @@ The fields to update are: ## Launch platform default services -RabbitMQ, Grafana and InfluxDB services are provisioned on this server. -InfluxDB webUI will be available at: _services.foo.com_. -The RabbitMQ service and its management interface shall be available -at 5672 and 15672 TCP ports respectively. -The Grafana service shall be available at TCP port 3000. - -The firewall and network access settings of corporate / cloud network need to be -configured to allow external access to the services. Otherwise the users of DTaaS -will not be able to utilize these services from their user workspaces. +RabbitMQ, Grafana, InfluxDB and MQTT services are provisioned on this server. Execute the following commands from terminal to start the machine. ```bash vagrant up --provision services vagrant ssh services -wget https://raw.githubusercontent.com/INTO-CPS-Association/DTaaS/feature/distributed-demo/deploy/vagrant/two-machine/services.sh -bash services.sh wget https://raw.githubusercontent.com/INTO-CPS-Association/DTaaS/feature/distributed-demo/deploy/vagrant/route.sh sudo bash route.sh ``` -After the server is up and running, you can see the following services active -within server2. - -| service | external url | -|:---|:---| -| Influx visualization service | services.foo.com | -| Grafana visualization service | services.foo.com:3000 | -| RabbitMQ communication service | services.foo.com:5672 | -| RabbitMQ management service | services.foo.com:15672 | -|| +Follow the instructions in [services](../../services/README.md) to install +the platform default services on this vagrant machine. ## Launch DTaaS application @@ -99,6 +82,11 @@ Execute the following commands from terminal ```bash vagrant up --provision dtaas vagrant ssh dtaas +``` + +Once inside the vagrant box, execute + +```bash wget https://raw.githubusercontent.com/INTO-CPS-Association/DTaaS/feature/distributed-demo/deploy/vagrant/route.sh sudo bash route.sh ``` diff --git a/deploy/vagrant/two-machine/package.json b/deploy/vagrant/two-machine/package.json deleted file mode 100644 index 67281a69c..000000000 --- a/deploy/vagrant/two-machine/package.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "type": "module", - "dependencies": { - "chalk": "^5.3.0", - "execa": "^8.0.1", - "js-yaml": "^4.1.0" - } -} diff --git a/docs/FAQ.md b/docs/FAQ.md index b22d56fa1..94c45cc87 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -10,7 +10,7 @@ ## General Questions - + ??? Question "What is DTaaS?" @@ -216,9 +216,7 @@ The core feature of DTaaS software is to help users create DTs from assets already available in the library. - ![Create Library Assets](./user/servers/lib/author.png) - However, it is possible for users to take advantage of services available in their workspace to install asset authoring tools in their own workspace. @@ -227,4 +225,45 @@ Thus any licensed software tools installed in their workspace is only available to them. - +## GDPR Concerns + +??? Question "Does your platform adhere to GDPR compliance standards? If so, how?" + + The DTaaS software platform does not store any personal information + of users. It only stores username to identify users and these + usernames do not contain enough information to deduce the true + identify of users. + +??? Question "Which security measures are deployed? How is data encrypted (if exists)?" + + The default installation requires a HTTPS terminating reverse proxy server + from user to the DTaaS software installation. The administrators of DTaaS + software can also install HTTPS certificates into the application. + The codebase can generate HTTPS application and the users also have + the option of installing their own certificates obtained from + certification agencies such as LetsEncrypt. + +??? Question "What security measures does your cloud provider offer?" + + The current installation of DTaaS software runs on Aarhus University + servers. The university network offers firewall access control to servers + so that only permitted user groups have access to the network and + physical access to the server. + +??? Question "How is user access controlled and authenticated?" + + There is a two-level authentication mechanism in place in each default + installation of DTaaS. The first-level is HTTP basic authentication + over secure HTTPS connection. The second-level is the OAuth PKCE + authentication flow for each user. The OAuth authentication is provider + by a Gitlab instance. The DTaaS does not store the account and + authentication information of users. + +??? Question "Does you platform manage personal data? How is data classified and tagged based on the sensitivity? Who has access to the critical data?" + + The platform does not store personal data of users. + +??? Question "How are identities and roles managed within the platform?" + + There are two roles for users on the platform. One is the administrator + and the other one is user. The user roles are managed by the administrator. diff --git a/docs/admin/services.md b/docs/admin/services.md index 96362b054..9af78dc1b 100644 --- a/docs/admin/services.md +++ b/docs/admin/services.md @@ -3,125 +3,68 @@ The DTaaS software platform uses third-party software services to provide enhanced value to users. -InfluxDB, RabbitMQ and Grafana are default services +InfluxDB, Grafana, RabbitMQ and Mosquitto are default services integrated into the DTaaS software platform. -_The InfluxDB service requires a dedicated hostname. The management -interface of RabbitMQ service requires a dedicated hostname as well._ - -Thus successful installation of these services -is dependent on your ability to use -multiple hostnames for different services. You can download the required -services using the docker commands. - -```sh -docker pull grafana/grafana:10.1.4 -docker pull influxdb:2.7 -docker pull telegraf:1.28.2 -docker pull rabbitmq:3-management -docker pull eclipse-mosquitto:2 -``` - -The two-machine vagrant deployment scenario installs the RabbitMQ, Grafana, and -InfluxDB services on the second vagrant machine. - -If you would like to install some of these services for native OS -installation or single vagrant machine, you can do this as well. - -## RabbitMQ - -Start the RabbitMQ service with - -```bash -docker run -d \ - --name rabbitmq-server \ - -p 15672:15672 -p 5672:5672 \ - rabbitmq:3-management -``` +## Pre-requisites -Users and the vhosts need to be setup on the server. Sample commands to do so are: +All these services run on raw TCP/UDP ports. Thus a direct network +access to these services is required for both the DTs running inside +the DTaaS software and the PT located outside the DTaaS software. -```bash -docker exec rabbitmq-server rabbitmqctl add_user -docker exec rabbitmq-server rabbitmqctl set_permissions -p "/" ".*" ".*" ".*" -``` - -The RabbitMQ service requires raw TCP/UDP protocol access to network. -The default Traefik configuration of DTaaS does not permit -TCP/UDP traffic. There are two possible choices here: +There are two possible choices here: * Configure Traefik gateway to permit TCP/UDP traffic -* Bypass Traefik altogether for RabbitMQ service +* Bypass Traefik altogether Unless you are an informed user of Traefik, we recommend bypassing traefik -for RabbitMQ service. +and provide raw TCP/UDP access to these services from the Internet. -## Grafana - -Grafana service can run well behind Traefik gateway. Here is a sample docker -command to run Grafana service at port 3000: - -```bash -docker run -d \ - -p 3000:3000 \ - --name=grafana \ - -e "GF_SERVER_SERVE_FROM_SUB_PATH=true" \ - -e "GF_SERVER_DOMAIN=localhost" \ - -e "GF_SERVER_ROOT_URL=%(protocol)s://%(domain)s:%(http_port)s" \ - -e "GF_AUTH_BASIC_ENABLED=false" \ - -e "GF_AUTH_PROXY_ENABLED=false" \ - -e "GF_SECURITY_ADMIN_PASSWORD=DTaaSGrafana" \ - -e "GF_SECURITY_ALLOW_EMBEDDING=true" \ - -e "GF_SECURITY_ALLOW_EMBEDDING=true" \ - -e "GF_AUTH_ANONYMOUS_ENABLED=true" \ - -e "GF_AUTH_ANONYMOUS_ORG_NAME=Main" \ - -e "GF_AUTH_ANONYMOUS_ORG_ROLE=Editor" \ - -e "GF_USERS_ALLOW_SIGN_UP=false" \ - -e "GF_FEATURE_TOGGLES_ENABLE=publicDashboards" \ - -e "GF_PATHS_CONFIG=/etc/grafana/grafana.ini" \ - -e "GF_PATHS_DATA=/var/lib/grafana" \ - -e "GF_PATHS_HOME=/usr/share/grafana" \ - -e "GF_PATHS_LOGS=/var/log/grafana" \ - -e "GF_PATHS_PLUGINS=/var/lib/grafana/plugins" \ - -e "GF_PATHS_PROVISIONING=/etc/grafana/provisioning" \ - -e "HOME=/home/grafana" \ - grafana/grafana -printf "Complete the setup from GUI" -``` +_The InfluxDB service requires a dedicated hostname. The management +interface of RabbitMQ service requires a dedicated hostname as well._ -The user credentials have also been set in the command as: +Grafana service can run well behind Traefik gateway. The default Traefik +configuration makes permits access to Grafana at URL: http(s): _foo.com/vis_. -**username**: admin +## Configure and Install -**password**: DTaaSGrafana +If you have not cloned the DTaaS git repository, cloning would be +the first step. +In case you already have the codebase, you can skip the cloning step. +To clone, do: -Remember to change these credentials before starting the docker container. +```bash +git clone https://github.com/into-cps-association/DTaaS.git +cd DTaaS/deploy/services +``` -## InfluxDB +The next step in installation is to specify the config of the services. +There are two configuration files. The __services.yml__ contains most +of configuration settings. The __mqtt-default.conf__ file contains +the MQTT listening port. Update these two config files before proceeding +with the installation of the services. -The barebones InfluxDB service can be installed using: +Now continue with the installation of services. ```bash -INFLUXDB_DATA="${PWD}/data/influxdb2" -mkdir -p "$INFLUXDB_DATA" - -# Remember to change the settings -docker run -d -p 80:8086 \ - --name influxdb24 \ - -v "$INFLUXDB_DATA/data":/var/lib/influxdb2 \ - -v "$INFLUXDB_DATA/config":/etc/influxdb2 \ - -e DOCKER_INFLUXDB_INIT_MODE=setup \ - -e DOCKER_INFLUXDB_INIT_USERNAME=dtaas \ - -e DOCKER_INFLUXDB_INIT_PASSWORD=dtaas1357 \ - -e DOCKER_INFLUXDB_INIT_ORG=dtaas \ - -e DOCKER_INFLUXDB_INIT_BUCKET=dtaas \ - influxdb:2.4 +yarn install +node services.js ``` -The user credentials have also been set in the command as: +## Use -**username**: dtaas +After the installation is complete, you can see the following services active +at the following ports / URLs. -**password**: dtaas1357 +| service | external url | +|:---|:---| +| Influx | services.foo.com | +| Grafana | services.foo.com:3000 | +| RabbitMQ Broker | services.foo.com:5672 | +| RabbitMQ Broker Management Website | services.foo.com:15672 | +| MQTT Broker | services.foo.com:1883 | +|| -Remember to change these credentials before starting the docker container. +The firewall and network access settings of corporate / cloud network need to be +configured to allow external access to the services. Otherwise the users of DTaaS +will not be able to utilize these services from their user workspaces. diff --git a/docs/admin/vagrant/two-machines.md b/docs/admin/vagrant/two-machines.md index f9332b89d..b8ce4eabb 100644 --- a/docs/admin/vagrant/two-machines.md +++ b/docs/admin/vagrant/two-machines.md @@ -82,33 +82,15 @@ The fields to update are: ## Installation Steps -### Launch DTaaS Platform Default Services - -RabbitMQ, Grafana, InfluxDB and MQTT services are provisioned on this server. -InfluxDB and visualization service will be available at: _services.foo.com_. -The Grafana service shall be available at TCP port 3000. -The MQTT service shall be available at TCP port 1833. -The RabbitMQ service and its management interface shall be available at -5672 and 15672 TCP ports respectively. +The installation instructions are given separately for each vagrant machine. -The firewall and network access settings of corporate / cloud network -need to be configured to allow external access to the services. -Otherwise the users of DTaaS will not be able to utilize these -services from their user workspaces. - -Execute the following commands from terminal to start the machine. +### Launch DTaaS Platform Default Services -```bash -vagrant up --provision services -vagrant ssh services -wget https://raw.githubusercontent.com/INTO-CPS-Association/DTaaS/feature/distributed-demo/deploy/vagrant/two-machine/services.sh -bash services.sh -wget https://raw.githubusercontent.com/INTO-CPS-Association/DTaaS/feature/distributed-demo/deploy/vagrant/route.sh -sudo bash route.sh -``` +Follow the installation guide for [services](../services.md) +to install the DTaaS platform services. -After the server is up and running, -you can see the following services active within server2. +After the services are up and running, +you can see the following services active within server2 (_services.foo.com_). | service | external url | |:---|:---| @@ -117,6 +99,7 @@ you can see the following services active within server2. | MQTT communication service | services.foo.com:1883 | | RabbitMQ communication service | services.foo.com:5672 | | RabbitMQ management service | services.foo.com:15672 | +|| ### Install DTaaS Application diff --git a/docs/thanks.md b/docs/thanks.md index f6872380b..0ca23f27d 100644 --- a/docs/thanks.md +++ b/docs/thanks.md @@ -15,6 +15,6 @@ Henrik Ejersbo, Tanusree Roy, Farshid Naseri Digital Twin as a Service (DTaaS): A Platform for Digital Twin Developers and Users [arXiv preprint arXiv:2305.07244](https://arxiv.org/abs/2305.07244). -1. Astitva Sehgal -1. Tanusree Roy -1. Farshid Naseri +1. Astitva Sehgal for developer and example documentation. +1. Tanusree Roy and Farshid Naseri for asking interesting questions + that ended up in FAQs.