Skip to content

Commit

Permalink
feat: update doc following core API migration and 2.0.0 release (#14)
Browse files Browse the repository at this point in the history
- feat: describe missing endpoints used to set / remove rights on entities
- feat: update contract for get authorized entities endpoint
- feat: global update for new architecture in 2.0.0
- feat: add migration procedure to upgrade to 2.0.0

---------

Co-authored-by: Gaël Poujol <76944966+gpoujol@users.noreply.github.com>
Co-authored-by: gpoujol <gael.poujol@egm.io>
  • Loading branch information
3 people authored Jan 30, 2023
1 parent bdc2e43 commit c513a74
Show file tree
Hide file tree
Showing 13 changed files with 589 additions and 200 deletions.
57 changes: 13 additions & 44 deletions docs/admin/backup_and_restore.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,42 +25,24 @@ echo "Performing a backup of PostgreSQL databases (search and subscription)"
# Following instructions from https://docs.timescale.com/latest/using-timescaledb/backup#pg_dump-pg_restore
# It display warnings ... that can be ignored
# Cf https://github.com/timescale/timescaledb/issues/1581
docker exec postgres /bin/bash -c "export PGPASSWORD=$POSTGRES_PASSWORD && /usr/local/bin/pg_dump -Fc -U postgres stellio_search" | gzip -9 > /tmp/postgres_search_$now.gz
docker exec postgres /bin/bash -c "export PGPASSWORD=$POSTGRES_PASSWORD && /usr/local/bin/pg_dump -Fc -U postgres stellio_subscription" | gzip -9 > /tmp/postgres_subscription_$now.gz
docker exec postgres /bin/bash -c "su - postgres -c 'pg_dump -Fc stellio_search | gzip -c'" > /tmp/postgres_search_$now.gz
docker exec postgres /bin/bash -c "su - postgres -c 'pg_dump -Fc stellio_subscription | gzip -c'" > /tmp/postgres_subscription_$now.gz

mv /tmp/postgres_search_$now.gz $BACKUP_DIR/.
mv /tmp/postgres_subscription_$now.gz $BACKUP_DIR/.

echo
echo "Stopping Stellio containers (to backup Neo4j and Grafana)"
echo "Stopping Stellio containers (to backup Kafka)"

cd $STELLIO_COMPOSE_DIR
/usr/local/bin/docker-compose -f docker-compose.yml stop

echo
echo "Performing a backup of Neo4j"

# It needs to be writable by the user inside the neo4j container run just after
# TODO it should be improved, this is currently a brute force solution
chmod 777 $BACKUP_DIR

# Be sure to change the neo4j version if you are not yet on 4.4
# Also check the name of the volume for neo4j, you can find it with the "docker volume ls" command
neo4j_data_volume=$(docker volume inspect --format '{{ .Mountpoint }}' stellio-launcher_neo4j-storage)
docker run --rm --publish=7474:7474 --publish=7687:7687 --volume=$neo4j_data_volume:/data --volume=$BACKUP_DIR:/backups neo4j:4.4 neo4j-admin dump --database=stellio --to=/backups/neo4j_$now.dump

echo
echo "Performing a backup of Kafka"

docker run --rm --volumes-from kafka -v $BACKUP_DIR:/backup ubuntu tar cvf /backup/kafka_$now.tar /var/lib/kafka/data
gzip -f $BACKUP_DIR/kafka_$now.tar

echo
echo "Performing a backup of Zookeeper"

docker run --rm --volumes-from zookeeper -v $BACKUP_DIR:/backup ubuntu tar cvf /backup/zookeeper_$now.tar /var/lib/zookeeper/data
gzip -f $BACKUP_DIR/zookeeper_$now.tar

echo
echo "Restarting Stellio containers"

Expand Down Expand Up @@ -88,7 +70,7 @@ You can call it in a cron job like this:
WARNING : backup date has to be set up manually in other places below, review them carefully!

```shell
backup_date=2021-05-13
backup_date=2023-01-27
export BACKUP_DIR=$HOME/backup/stellio
export STELLIO_COMPOSE_DIR=$HOME/stellio-launcher
```
Expand All @@ -110,7 +92,7 @@ docker exec -it stellio-postgres bash
* Once in the container, restore the databases

```shell
backup_date=2021-05-13 # need to be set again in the container
backup_date=2023-01-27 # need to be set again in the container
gunzip /tmp/postgres_search_$backup_date.gz
gunzip /tmp/postgres_subscription_$backup_date.gz

Expand All @@ -121,13 +103,13 @@ psql
\c stellio_search
CREATE EXTENSION IF NOT EXISTS timescaledb;
SELECT timescaledb_pre_restore();
\! pg_restore -Fc -d stellio_search /tmp/postgres_search_2021-05-13 -- change the date!
\! pg_restore -Fc -d stellio_search /tmp/postgres_search_2023-01-27 -- change the date!
SELECT timescaledb_post_restore();

\c stellio_subscription
CREATE EXTENSION IF NOT EXISTS timescaledb;
SELECT timescaledb_pre_restore();
\! pg_restore -Fc -d stellio_subscription /tmp/postgres_subscription_2021-05-13 -- change the date!
\! pg_restore -Fc -d stellio_subscription /tmp/postgres_subscription_2023-01-27-- change the date!
SELECT timescaledb_post_restore();

exit # from psql
Expand All @@ -145,32 +127,19 @@ exit # from the container
docker-compose stop postgres
```

## Step 2 - Restore neo4j

```shell
cd $STELLIO_COMPOSE_DIR
# Start and stop neo4j to create the container and volume if they do not yet exist
docker-compose up -d neo4j
docker-compose logs -f neo4j # wait for neo4j to finish starting
docker-compose stop neo4j

docker run --interactive --tty --rm --publish=7474:7474 --publish=7687:7687 --volumes-from neo4j --volume=$BACKUP_DIR:/backups neo4j:4.4 neo4j-admin load --from=/backups/neo4j_$backup_date.dump --database=stellio --force
```

## Step 3 - Restore Kafka and Zookeeper
## Step 2 - Restore Kafka

```shell
cd $STELLIO_COMPOSE_DIR
# Start and stop kafka and zookeeper to create the containers and volumes if they do not yet exist
docker-compose up -d zookeeper kafka
docker-compose logs -f zookeeper kafka # wait for zookeeper and kafka to finish starting
docker-compose stop zookeeper kafka
# Start and stop kafka to create the container and volume if it does not yet exist
docker-compose up -d kafka
docker-compose logs -f kafka # wait for kafka to finish starting
docker-compose stop kafka

docker run --rm --volumes-from zookeeper -v $BACKUP_DIR:/backup ubuntu tar -C /var/lib/zookeeper/data -xzf /backup/zookeeper_$backup_date.tar.gz --strip-components 4
docker run --rm --volumes-from kafka -v $BACKUP_DIR:/backup ubuntu tar -C /var/lib/kafka/data -xzf /backup/kafka_$backup_date.tar.gz --strip-components 4
```

## Step 4 - Restart Stellio
## Step 3 - Restart Stellio

```shell
cd $STELLIO_COMPOSE_DIR
Expand Down
17 changes: 5 additions & 12 deletions docs/admin/monitoring_and_logging.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ Setting up of Prometheus is of course beyond the scope of this documentation, th

However, it is recommended to monitor the VMs with [node_exporter]("https://github.com/prometheus/node_exporter") and the Docker containers with [cAdvisor]("https://github.com/google/cadvisor").

The Stellio services can also be configured to expose an [health endpoint]("https://docs.spring.io/spring-boot/docs/2.3.4.RELEASE/reference/html/production-ready-features.html#production-ready-health") and [Prometheus metrics]("https://docs.spring.io/spring-boot/docs/2.3.4.RELEASE/reference/html/production-ready-features.html#production-ready-metrics-export-prometheus") (in parenthesis, the name of the environement variable to use when injecting the values into a Docker container):
The Stellio services can also be configured to expose an [health endpoint]("https://docs.spring.io/spring-boot/docs/current/reference/html/production-ready-features.html#production-ready-health") and [Prometheus metrics]("https://docs.spring.io/spring-boot/docs/current/reference/html/production-ready-features.html#production-ready-metrics-export-prometheus") (in parenthesis, the name of the environement variable to use when injecting the values into a Docker container):

- `management.endpoint.prometheus.enabled` (`MANAGEMENT_ENDPOINT_PROMETHEUS_ENABLED`): `true`
- `management.endpoints.web.exposure.include` (`MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE`): `health,prometheus`
Expand All @@ -17,13 +17,13 @@ The Stellio services can also be configured to expose an [health endpoint]("http
In a docker-compose or Docker Swarm based deployment, the environement variables can be declared by adding the following in the `environment` section:

```
entity-service:
container_name: entity-service
image: stellio/stellio-entity-service:latest
search-service:
container_name: search-service
image: stellio/stellio-search-service:latest-dev
environment:
- MANAGEMENT_ENDPOINT_PROMETHEUS_ENABLED=${MANAGEMENT_ENDPOINT_PROMETHEUS_ENABLED}
- MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE=${MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE}
- MANAGEMENT_METRICS_TAGS_APPLICATION=Entity Service - ${MANAGEMENT_METRICS_PF_TAG}
- MANAGEMENT_METRICS_TAGS_APPLICATION=Search Service - ${MANAGEMENT_METRICS_PF_TAG}
```

### Example Prometheus configurations
Expand All @@ -40,9 +40,6 @@ An example Prometheus configuration to get health information from Stellio servi
- targets: ['http://stellio-host:8080/actuator/health']
labels:
name: 'API Gateway'
- targets: ['http://stellio-host:8082/actuator/health']
labels:
name: 'Entity Service'
- targets: ['http://stellio-host:8083/actuator/health']
labels:
name: 'Search Service'
Expand Down Expand Up @@ -82,7 +79,6 @@ An example Prometheus configuration to get metrics information from Stellio serv
metrics_path: '/actuator/prometheus'
scrape_interval: 30s
static_configs:
- targets: ['stellio-host:8082'] # 8082 : Entity service
- targets: ['stellio-host:8083'] # 8083 : Search service
- targets: ['stellio-host:8084'] # 8084 : Subscription service
```
Expand Down Expand Up @@ -117,12 +113,9 @@ Some classic alerts that are generally recommended:
```
- alert: container_down_stellio
expr: |
absent(container_memory_usage_bytes{name="entity-service",job="Stellio Docker"}) or
absent(container_memory_usage_bytes{name="api-gateway",job="Stellio Docker"}) or
absent(container_memory_usage_bytes{name="postgres",job="Stellio Docker"}) or
absent(container_memory_usage_bytes{name="zookeeper",job="Stellio Docker"}) or
absent(container_memory_usage_bytes{name="kafka",job="Stellio Docker"}) or
absent(container_memory_usage_bytes{name="neo4j",job="Stellio Docker"}) or
absent(container_memory_usage_bytes{name="subscription-service",job="Stellio Docker"}) or
absent(container_memory_usage_bytes{name="search-service",job="Stellio Docker"})
for: 30s
Expand Down
20 changes: 20 additions & 0 deletions docs/admin/scripts/export_subjects_infos.cypher
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
WITH "MATCH (n:`https://ontology.eglobalmark.com/authorization#Client`)-[:HAS_VALUE]->(pSid:Property {name: 'https://ontology.eglobalmark.com/authorization#serviceAccountId'})
MATCH (n)-[:HAS_VALUE]->(pClientId:Property {name: 'https://ontology.eglobalmark.com/authorization#clientId'})
RETURN substring(n.id, 19), substring(pSid.value, 17), pClientId.value" AS query
CALL apoc.export.csv.query(query, "/subject_export/export_clients.csv", {})
YIELD file, source, format, nodes, relationships, properties, time, rows, batchSize, batches, done, data
RETURN file, source, format, nodes, relationships, properties, time, rows, batchSize, batches, done, data;

WITH "MATCH (n:`https://ontology.eglobalmark.com/authorization#User`)-[:HAS_VALUE]->(pUsername:Property {name: 'https://ontology.eglobalmark.com/authorization#username'})
OPTIONAL MATCH (n)-[:HAS_VALUE]->(pGivenName:Property {name: 'https://ontology.eglobalmark.com/authorization#givenName'})
OPTIONAL MATCH (n)-[:HAS_VALUE]->(pFamilyName:Property {name: 'https://ontology.eglobalmark.com/authorization#familyName'})
RETURN substring(n.id, 17), pUsername.value, pGivenName.value, pFamilyName.value" AS query
CALL apoc.export.csv.query(query, "/subject_export/export_users.csv", {})
YIELD file, source, format, nodes, relationships, properties, time, rows, batchSize, batches, done, data
RETURN file, source, format, nodes, relationships, properties, time, rows, batchSize, batches, done, data;

WITH "MATCH (n:`https://ontology.eglobalmark.com/authorization#Group`)-[:HAS_VALUE]->(pName:Property {name: 'https://schema.org/name'})
RETURN substring(n.id, 18), pName.value" AS query
CALL apoc.export.csv.query(query, "/subject_export/export_groups.csv", {})
YIELD file, source, format, nodes, relationships, properties, time, rows, batchSize, batches, done, data
RETURN file, source, format, nodes, relationships, properties, time, rows, batchSize, batches, done, data;
87 changes: 87 additions & 0 deletions docs/admin/scripts/import_subjects_infos.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
-- Create temp table for groups and import data

create table temp_info_groups (sub text, name text, subject_info jsonb);
copy temp_info_groups(sub, name) from '/tmp/export_groups.csv' csv header;
update temp_info_groups
set subject_info = jsonb_build_object(
'type', 'Property',
'value', jsonb_build_object('kind', 'Group', 'name', name)
);
update subject_referential
set subject_info = (
select subject_info
from temp_info_groups
where sub = subject_referential.subject_id
)
where subject_type = 'GROUP';
update subject_referential
set subject_info = jsonb_build_object(
'type', 'Property',
'value', jsonb_build_object('kind', 'Group')
)
where subject_type = 'GROUP'
and subject_info is null;

-- Create temp table for users and import data

create table temp_info_users (sub text, username text, given_name text, family_name text, subject_info jsonb);
copy temp_info_users(sub, username, given_name, family_name) from '/tmp/export_users.csv' csv header;
update temp_info_users
set given_name = null
where given_name = 'null';
update temp_info_users
set family_name = null
where family_name = 'null';
update temp_info_users
set subject_info = jsonb_build_object(
'type', 'Property',
'value', jsonb_strip_nulls(
jsonb_build_object('kind', 'User', 'username', username, 'givenName', given_name, 'familyName', family_name)
)
);
update subject_referential
set subject_info = (
select subject_info
from temp_info_users
where sub = subject_referential.subject_id
)
where subject_type = 'USER';

update subject_referential
set subject_info = jsonb_build_object(
'type', 'Property',
'value', jsonb_build_object('kind', 'User')
)
where subject_type = 'USER'
and subject_info is null;

-- Create temp table for clients and import data

create table temp_info_clients (sub text, sid text, client_id text, subject_info jsonb);
copy temp_info_clients(sub, sid, client_id) from '/tmp/export_clients.csv' csv header;
update temp_info_clients
set subject_info = jsonb_build_object(
'type', 'Property',
'value', jsonb_build_object('kind', 'Client', 'clientId', client_id)
);
update subject_referential
set subject_info = (
select subject_info
from temp_info_clients
where sub = subject_referential.subject_id
)
where subject_type = 'CLIENT';

update subject_referential
set subject_info = jsonb_build_object(
'type', 'Property',
'value', jsonb_build_object('kind', 'Client')
)
where subject_type = 'CLIENT'
and subject_info is null;

-- Delete temp tables

drop table temp_info_groups;
drop table temp_info_users;
drop table temp_info_clients;
Loading

0 comments on commit c513a74

Please sign in to comment.