-
Notifications
You must be signed in to change notification settings - Fork 204
/
Makefile
245 lines (190 loc) · 8.32 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
# Makefile for OBOFoundry.github.io
#
# This file contains commands for validating and generating
# configuration files for the OBO Foundry website.
# The `util/` directory contains additional scripts
# called from this Makefile.
# It is intended to be run on a Unix system (Linux, Mac OS X)
# **before** the site is built using
# [Jekyll](http://jekyllrb.com).
#
# Software requirements:
#
# - [GNU Make](http://www.gnu.org/software/make/)
# - [Apache Jena](https://jena.apache.org/download/)
# - [Python 3](https://www.python.org/downloads/)
# - [PyYAML](http://pyyaml.org/wiki/PyYAML)
# - [SPARQLWrapper](https://pypi.python.org/pypi/SPARQLWrapper)
#
# Run `make` in this directory to update all generated files
# (i.e. the default `all` task).
# Make does its best to detect changes and run only the required tasks,
# but sometimes it helps to delete the target files first by running `make clean`
#
# WARNING: Makefiles contain significant tab characters!
# Before editing this file, ensure that your editor is not set up to convert tabs
# to spaces, and then use tabs to indent recipe lines.
### Configuration
# All ontology .md files
ONTS := $(wildcard ontology/*.md)
# All principles .md files
PRINCIPLES := $(wildcard principles/*.md)
### Main Tasks
.PHONY: all pull_and_build test pull clean
all: _config.yml registry/ontologies.ttl registry/obo_context.jsonld registry/obo_prefixes.ttl
pull:
git pull
pull_and_build: pull all
test: reports/metadata-grid.html _config.yml tox
integration-test: test valid-purl-report.txt
# Remove and/or revert all targets to their repository versions:
clean:
rm -Rf registry/ontologies.nt registry/ontologies.ttl registry/ontologies.yml sparql-consistency-report.txt jenkins-output.txt valid-purl-report.txt valid-purl-report.txt.tmp _site/ tmp/ reports/
git checkout _config.yml registry/ontologies.jsonld registry/ontologies.ttl registry/ontologies.yml
### Directories:
tmp:
mkdir -p $@
reports:
mkdir -p $@
reports/robot:
mkdir -p $@
reports/principles:
mkdir -p $@
### Build Configuration Files
# Create the site-wide config file by combining all metadata on ontologies + principles
# and combining with site-wide metadata.
#
# Note that anything in _config.yml is accessible to any liquid template via the
# `sites` object - think of it like the global database
#
# (this is somewhat hacky, but concatenating these yamls is safe)
_config.yml: _config_header.yml registry/ontologies.yml principles/all.yml
cat $^ > $@.tmp && mv $@.tmp $@
# Sort ontologies based on the validation (metadata-grid)
registry/ontologies.yml: reports/metadata-grid.csv
./util/sort-ontologies.py tmp/unsorted-ontologies.yml $< $@ && rm -rf tmp
# Extract the metadata from each principle in the principles/ directory, and concatenate
# into a single yaml file in that directory
principles/all.yml: $(PRINCIPLES)
./util/extract-metadata.py concat-principles -o $@.tmp $^ && mv $@.tmp $@
# Use a generic yaml->json conversion, but adding a @content
registry/ontologies.jsonld: registry/ontologies.yml
./util/yaml2json.py $< > $@.tmp && mv $@.tmp $@
registry/obo_context.jsonld: registry/ontologies.yml
./util/processor.py -i $< extract-context > $@.tmp && mv $@.tmp $@
# generate triples mapping prefixes to their corresponding PURLs.
# we use the SHACL vocabulary for this
registry/obo_prefixes.ttl: registry/ontologies.yml
./util/make-shacl-prefixes.py $< > $@.tmp && mv $@.tmp $@
# Use Apache-Jena RIOT to convert jsonld to n-triples
# NOTE: UGLY HACK. If there is a problem then Jena will write WARN message (to stdout!!!), there appears to
# be no way to get it to flag this even with strict and check options, so we do a check with grep, ugh.
# see: http://stackoverflow.com/questions/20860222/why-do-i-have-these-warnings-with-jena-2-11-0
registry/ontologies.nt: registry/ontologies.jsonld
riot --base=http://purl.obolibrary.org/obo/ --strict --check -q registry/context.jsonld $< > $@.tmp && mv $@.tmp $@ && egrep '(WARN|ERROR)' $@ && exit 1 || echo ok
registry/ontologies.ttl: registry/ontologies.nt
riot --base=http://purl.obolibrary.org/obo/ --out=ttl $< > $@.tmp && mv $@.tmp $@
### Validate Configuration Files
# generate both a report of the violations and a grid of all results
# the grid is later used to sort the ontologies on the home page
RESULTS = reports/metadata-violations.tsv reports/metadata-grid.csv
reports/metadata-grid.csv: tmp/unsorted-ontologies.yml | extract-metadata reports
./util/validate-metadata.py $< $(RESULTS)
# generate an HTML output of the metadata grid
# TODO: determine where this output belongs
reports/metadata-grid.html: reports/metadata-grid.csv
./util/create-html-grid.py $< $@
# Extract metadata from each ontology .md file and combine into single yaml
tmp/unsorted-ontologies.yml: $(ONTS) | tmp
./util/extract-metadata.py concat -o $@.tmp $^ && mv $@.tmp $@
extract-metadata: $(ONTS)
./util/extract-metadata.py validate $^
prettify: $(ONTS)
./util/extract-metadata.py prettify $^
# Run tox tests (requires `pip install tox`)
tox:
tox -e py
##########################
## Metadata Maintenance ##
##########################
# Use SPARQL queries to Wikidata to enrich the
# OBO operations committee metadata file
.PHONY: update-operations-metadata
update-operations-metadata:
python -m obofoundry update-operations-metadata
### OBO Dashboard
# This is the Jenkins job
# The reports will be archived
dashboard: build/dashboard.zip
# Build directories
build:
mkdir -p $@
build/ontologies:
mkdir -p $@
# reboot the JVM for Py4J
reboot:
bash ./util/reboot.sh
# This version of ROBOT includes features for starting Py4J
# This will be changed to ROBOT release once feature is released
#.PHONY: build/robot.jar
build/robot.jar: | build
curl -o $@ -Lk \
https://build.obolibrary.io/job/ontodev/job/robot/job/py4j/lastSuccessfulBuild/artifact/bin/robot.jar
# This version of ROBOT includes features for removing external axioms to create 'base' artefacts
# This will be removed once this feature is released
#.PHONY: build/robot-foreign.jar
build/robot-foreign.jar: | build
curl -o $@ -Lk \
https://build.obolibrary.io/job/ontodev/job/robot/job/562-feature/lastSuccessfulBuild/artifact/bin/robot.jar
# Generate the initial dashboard results file
# ALWAYS make sure nothing is running on port 25333
# Then boot Py4J gateway to ROBOT on that port
reports/dashboard.csv: registry/ontologies.yml | \
reports/robot reports/principles build/ontologies build/robot.jar build/robot-foreign.jar
make reboot
./util/principles/dashboard.py $< $@ --big false
reports/big-dashboard.csv: reports/dashboard.csv
make reboot
./util/principles/dashboard.py registry/ontologies.yml $@ --big true
# Combine the dashboard files
reports/dashboard-full.csv: reports/dashboard.csv reports/big-dashboard.csv registry/ontologies.yml
./util/principles/sort_tables.py $^ $@
# Generate the HTML grid output for dashboard
reports/dashboard.html: reports/dashboard-full.csv
./util/create-html-grid.py $< $@
# Move all important results to a dashboard directory
build/dashboard: reports/dashboard.html
mkdir -p $@
mkdir -p $@/assets
mkdir -p $@/reports
cp $< $@
cp -r reports/robot $@/reports
cp -r reports/principles $@/reports
cp -r assets/svg $@/assets
rm -rf build/dashboard.zip
zip -r $@.zip $@
# Clean up, removing ontology files
# We don't want to keep them because we will download new ones each time to stay up-to-date
# Reports are all archived in build/dashboard.zip
clean-dashboard: build/dashboard
rm -rf build/ontologies
rm -rf reports/robot
rm -rf reports/principles
rm -rf build/dashboard
# Note this should *not* be run as part of general travis jobs, it is expensive
# and may be prone to false positives as it is inherently network-based
#
# TODO: Other non-travis CI job. Nightly?
# TODO: Integrate this with some kind of OCLC query check
#
# See: https://github.com/OBOFoundry/OBOFoundry.github.io/issues/18
valid-purl-report.txt: registry/ontologies.yml
./util/processor.py -i $< check-urls > $@.tmp && mv $@.tmp $@
sparql-consistency-report.txt: registry/ontologies.yml
./util/processor.py -i $< sparql-compare > $@.tmp && mv $@.tmp $@
# output of central OBO build
# See FAQ for more details, and also README.md
jenkins-output.txt:
wget http://build.berkeleybop.org/job/simple-build-obo-all/lastBuild/consoleFull -O $@
reports/%.csv: registry/ontologies.ttl sparql/%.sparql
arq --data $< --query sparql/$*.sparql --results csv > $@.tmp && mv $@.tmp $@