-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathMakefile
174 lines (145 loc) · 8.13 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
SHELL=/bin/bash -o pipefail
BUILD_PRINT = \e[1;34mSTEP:
END_BUILD_PRINT = \e[0m
CURRENT_UID := $(shell id -u)
export CURRENT_UID
# These are constants used for make targets so we can start prod and staging services on the same machine
ENV_FILE := .env
# include .env files if they exist
-include .env
PROJECT_PATH = $(shell pwd)
AIRFLOW_INFRA_FOLDER ?= ${PROJECT_PATH}/.airflow
RML_MAPPER_PATH = ${PROJECT_PATH}/.rmlmapper/rmlmapper.jar
XML_PROCESSOR_PATH = ${PROJECT_PATH}/.saxon/saxon-he-10.6.jar
LIMES_ALIGNMENT_PATH = $(PROJECT_PATH)/.limes/limes.jar
HOSTNAME = $(shell hostname)
#-----------------------------------------------------------------------------
# Dev commands
#-----------------------------------------------------------------------------
setup: install init-saxon init-rml-mapper init-limes
install:
@ echo -e "$(BUILD_PRINT)Installing the requirements$(END_BUILD_PRINT)"
@ python -m pip install --upgrade pip
@ python -m pip install --no-cache-dir -r requirements.txt
install-dev:
@ echo -e "$(BUILD_PRINT)Installing the dev requirements$(END_BUILD_PRINT)"
@ python -m pip install --upgrade pip
@ python -m pip install --no-cache-dir -r requirements.dev.txt
test: test-unit
test-unit:
@ echo -e "$(BUILD_PRINT)Unit Testing ...$(END_BUILD_PRINT)"
@ tox -e unit
test-features:
@ echo -e "$(BUILD_PRINT)Gherkin Features Testing ...$(END_BUILD_PRINT)"
@ tox -e features
test-e2e:
@ echo -e "$(BUILD_PRINT)End to End Testing ...$(END_BUILD_PRINT)"
@ tox -e e2e
test-all-parallel:
@ echo -e "$(BUILD_PRINT)Complete Testing ...$(END_BUILD_PRINT)"
@ tox -p
test-all:
@ echo -e "$(BUILD_PRINT)Complete Testing ...$(END_BUILD_PRINT)"
@ tox
#-----------------------------------------------------------------------------
# VAULT SERVICES
#-----------------------------------------------------------------------------
# Testing whether an env variable is set or not
guard-%:
@ if [ "${${*}}" = "" ]; then \
echo -e "$(BUILD_PRINT)Environment variable $* not set $(END_BUILD_PRINT)"; \
exit 1; \
fi
# Testing that vault is installed
vault-installed: #; @which vault1 > /dev/null
@ if ! hash vault 2>/dev/null; then \
echo -e "$(BUILD_PRINT)Vault is not installed, refer to https://www.vaultproject.io/downloads $(END_BUILD_PRINT)"; \
exit 1; \
fi
# Get secrets in dotenv format
staging-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
@ echo -e "$(BUILD_PRINT)Creating .env.staging file $(END_BUILD_PRINT)"
@ echo VAULT_ADDR=${VAULT_ADDR} > .env
@ echo VAULT_TOKEN=${VAULT_TOKEN} >> .env
@ echo DOMAIN=ontobuilder.com >> .env
@ echo ENVIRONMENT=staging >> .env
@ echo SUBDOMAIN=staging. >> .env
@ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> .env
@ echo LIMES_ALIGNMENT_PATH=${LIMES_ALIGNMENT_PATH} >> .env
@ echo XML_PROCESSOR_PATH=${XML_PROCESSOR_PATH} >> .env
@ echo AIRFLOW_INFRA_FOLDER=~/airflow-infra/staging >> .env
@ echo AIRFLOW_WORKER_HOSTNAME=${HOSTNAME} >> .env
@ vault kv get -format="json" ted-staging/airflow | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-staging/mongo-db | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-staging/metabase | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-staging/ted-sws | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-staging/agraph | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-staging/fuseki | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-staging/github | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-staging/minio | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
dev-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
@ echo -e "$(BUILD_PRINT)Create .env file $(END_BUILD_PRINT)"
@ echo VAULT_ADDR=${VAULT_ADDR} > .env
@ echo VAULT_TOKEN=${VAULT_TOKEN} >> .env
@ echo DOMAIN=localhost >> .env
@ echo ENVIRONMENT=dev >> .env
@ echo SUBDOMAIN= >> .env
@ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> .env
@ echo LIMES_ALIGNMENT_PATH=${LIMES_ALIGNMENT_PATH} >> .env
@ echo XML_PROCESSOR_PATH=${XML_PROCESSOR_PATH} >> .env
@ echo AIRFLOW_INFRA_FOLDER=${AIRFLOW_INFRA_FOLDER} >> .env
@ echo AIRFLOW_WORKER_HOSTNAME=${HOSTNAME} >> .env
@ vault kv get -format="json" ted-dev/airflow | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-dev/mongo-db | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-dev/metabase | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-dev/agraph | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-dev/fuseki | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-dev/ted-sws | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-dev/github | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-dev/minio | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
prod-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
@ echo -e "$(BUILD_PRINT)Create .env file $(END_BUILD_PRINT)"
@ echo VAULT_ADDR=${VAULT_ADDR} > .env
@ echo VAULT_TOKEN=${VAULT_TOKEN} >> .env
@ echo DOMAIN=ted-data.eu >> .env
@ echo ENVIRONMENT=prod >> .env
@ echo SUBDOMAIN= >> .env
@ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> .env
@ echo LIMES_ALIGNMENT_PATH=${LIMES_ALIGNMENT_PATH} >> .env
@ echo XML_PROCESSOR_PATH=${XML_PROCESSOR_PATH} >> .env
@ echo AIRFLOW_INFRA_FOLDER=~/airflow-infra/prod >> .env
@ echo AIRFLOW_WORKER_HOSTNAME=${HOSTNAME} >> .env
@ echo AIRFLOW_CELERY_WORKER_CONCURRENCY=32 >> .env
@ vault kv get -format="json" ted-prod/airflow | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-prod/mongo-db | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-prod/metabase | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-prod/agraph | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-prod/fuseki | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-prod/ted-sws | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-prod/github | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
@ vault kv get -format="json" ted-prod/minio | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
local-dotenv-file: rml-mapper-path-add-dotenv-file
rml-mapper-path-add-dotenv-file:
@ echo -e "$(BUILD_PRINT)Add rml-mapper path to local .env file $(END_BUILD_PRINT)"
@ test -e .env || touch .env
@ sed -i '/^RML_MAPPER_PATH/d' .env
@ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> .env
refresh-mapping-files:
@ python -m ted_sws.data_manager.entrypoints.cli.cmd_generate_mapping_resources
install-local:
@ echo -e "$(BUILD_PRINT)Installing the local requirements$(END_BUILD_PRINT)"
@ python -m pip install --upgrade --force-reinstall --no-deps ../ted-rdf-conversion-pipeline
init-rml-mapper:
@ echo -e "RMLMapper folder initialisation!"
@ mkdir -p ./.rmlmapper
@ wget https://github.com/RMLio/rmlmapper-java/releases/download/v6.2.2/rmlmapper-6.2.2-r371-all.jar -O ./.rmlmapper/rmlmapper.jar
init-limes:
@ echo -e "Limes folder initialisation!"
@ mkdir -p ./.limes
@ wget -c https://github.com/dice-group/LIMES/releases/download/1.7.9/limes.jar -P ./.limes
init-saxon:
@ echo -e "$(BUILD_PRINT)Saxon folder initialization $(END_BUILD_PRINT)"
@ wget -c https://downloads.sourceforge.net/project/saxon/Saxon-HE/10/Java/SaxonHE10-6J.zip -P .saxon/
@ cd .saxon && unzip SaxonHE10-6J.zip
clean:
@ test -d .saxon && rm -rfv SaxonHE10-6J.zip