Skip to content

Commit

Permalink
Merge pull request #8 from VirtualFlyBrain/odk-update
Browse files Browse the repository at this point in the history 
Odk update
  • Loading branch information
@Clare72
Clare72 authored Jun 30, 2022
2 parents f9869ef + 0c06529 commit 1606d71
Show file tree
Hide file tree
Showing 5 changed files with 81 additions and 53 deletions.
88 changes: 49 additions & 39 deletions src/ontology/Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# ----------------------------------------
# Makefile for vfbext
# Generated using ontology-development-kit
# ODK Version: v1.3.0
# ODK Version: v1.3.1
# ----------------------------------------
# IMPORTANT: DO NOT EDIT THIS FILE. To override default make goals, use vfbext.Makefile instead

Expand Down Expand Up @@ -40,7 +40,7 @@ REPORT_PROFILE_OPTS =
OBO_FORMAT_OPTIONS =
SPARQL_VALIDATION_CHECKS = owldef-self-reference iri-range label-with-iri
SPARQL_EXPORTS = basic-report class-count-by-prefix edges xrefs obsoletes synonyms
ODK_VERSION_MAKEFILE = v1.3.0
ODK_VERSION_MAKEFILE = v1.3.1

TODAY ?= $(shell date +%Y-%m-%d)
OBODATE ?= $(shell date +'%d:%m:%Y %H:%M')
Expand All @@ -60,15 +60,21 @@ RELEASE_ARTEFACTS = $(sort $(ONT)-base $(ONT)-simple $(ONT)-non-classified $(ONT
.PHONY: .FORCE

.PHONY: all
all: odkversion test all_assets
all: all_odk

.PHONY: all_odk
all_odk: odkversion test all_assets

.PHONY: test
test: odkversion sparql_test robot_reports $(REPORTDIR)/validate_profile_owl2dl_$(ONT).owl.txt
$(ROBOT) reason --input $(SRC) --reasoner ELK --equivalent-classes-allowed all --exclude-tautologies structural --output test.owl && rm test.owl && echo "Success"
$(ROBOT) reason --input $(SRC) --reasoner ELK --equivalent-classes-allowed all \
--exclude-tautologies structural --output test.owl && rm test.owl &&\
echo "Success"

.PHONY: odkversion
odkversion:
echo "ODK Makefile version: $(ODK_VERSION_MAKEFILE) (this is the version of the ODK with which this Makefile was generated, not the version of the ODK you are running)" &&\
echo "ODK Makefile version: $(ODK_VERSION_MAKEFILE) (this is the version of the ODK with which this Makefile was generated, \
not the version of the ODK you are running)" &&\
echo "ROBOT version (ODK): " && $(ROBOT) --version

$(TMPDIR) $(REPORTDIR) $(MIRRORDIR) $(IMPORTDIR) $(COMPONENTSDIR) $(SUBSETDIR):
Expand Down Expand Up @@ -160,10 +166,10 @@ endif
# ----------------------------------------

$(REPORTDIR)/$(SRC)-obo-report.tsv: $(SRCMERGED) | $(REPORTDIR)
$(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --print 5 -o $@
$(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --base-iri http://purl.obolibrary.org/obo/fbbt/vfb/VFBext_ --base-iri http://purl.obolibrary.org/obo/fbbt/vfb/VFB_ --print 5 -o $@

$(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR)
$(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --print 5 -o $@
$(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --base-iri http://purl.obolibrary.org/obo/fbbt/vfb/VFBext_ --base-iri http://purl.obolibrary.org/obo/fbbt/vfb/VFB_ --print 5 -o $@

# ----------------------------------------
# Release assets
Expand Down Expand Up @@ -196,17 +202,22 @@ CLEANFILES=$(MAIN_FILES) $(SRCMERGED)
# It will ensure that all assets/files are fresh, and will copy to release folder

.PHONY: prepare_release
prepare_release: all
prepare_release: all_odk
rsync -R $(RELEASE_ASSETS) $(RELEASEDIR) &&\
rm -f $(CLEANFILES) &&\
echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release on your git hosting site such as GitHub or GitLab"
echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release \
on your git hosting site such as GitHub or GitLab"

.PHONY: prepare_initial_release
prepare_initial_release: all_assets
rsync -R $(RELEASE_ASSETS) $(RELEASEDIR) &&\
rm -f $(patsubst %, ./%, $(CLEANFILES)) &&\
cd $(RELEASEDIR) && git add $(RELEASE_ASSETS)

.PHONY: prepare_release_fast
prepare_release_fast:
$(MAKE) prepare_release IMP=false PAT=false MIR=false COMP=false

# ------------------------
# Imports: Seeding system
# ------------------------
Expand Down Expand Up @@ -257,7 +268,7 @@ $(IMPORTDIR)/%_terms_combined.txt: $(IMPORTSEED) $(IMPORTDIR)/%_terms.txt
$(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms_combined.txt
if [ $(IMP) = true ]; then $(ROBOT) query -i $< --update ../sparql/preprocess-module.ru \
extract -T $(IMPORTDIR)/$*_terms_combined.txt --force true --copy-ontology-annotations true --individuals include --method BOT \
query --update ../sparql/inject-subset-declaration.ru --update ../sparql/postprocess-module.ru \
query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru --update ../sparql/postprocess-module.ru \
annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output $@.tmp.owl && mv $@.tmp.owl $@; fi

.PRECIOUS: $(IMPORTDIR)/%_import.owl
Expand All @@ -269,32 +280,32 @@ $(IMPORTDIR)/caro_import.owl: $(MIRRORDIR)/caro.owl
## Module for ontology: obi

$(IMPORTDIR)/obi_import.owl: $(MIRRORDIR)/obi.owl $(IMPORTDIR)/obi_terms_combined.txt
if [ $(IMP) = true ]; then $(ROBOT) extract -i $< -T $(IMPORTDIR)/obi_terms_combined.txt --force true --method BOT \
if [ $(IMP) = true ]; then $(ROBOT) extract -i $< -T $(IMPORTDIR)/obi_terms_combined.txt --copy-ontology-annotations true --force true --method BOT \
remove --base-iri $(URIBASE)/OBI --axioms external --preserve-structure false --trim false \
query --update ../sparql/inject-subset-declaration.ru \
query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru \
remove $(patsubst %, --term %, $(ANNOTATION_PROPERTIES)) -T $(IMPORTDIR)/obi_terms_combined.txt --select complement --select "classes individuals annotation-properties" \
annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output $@.tmp.owl && mv $@.tmp.owl $@; fi


.PHONY: refresh-imports
refresh-imports:
make IMP=true MIR=true PAT=false IMP_LARGE=true all_imports -B
$(MAKE) IMP=true MIR=true PAT=false IMP_LARGE=true all_imports -B

.PHONY: no-mirror-refresh-imports
no-mirror-refresh-imports:
make IMP=true MIR=false PAT=false IMP_LARGE=true all_imports -B
$(MAKE) IMP=true MIR=false PAT=false IMP_LARGE=true all_imports -B

.PHONY: refresh-imports-excluding-large
refresh-imports-excluding-large:
make IMP=true MIR=true PAT=false IMP_LARGE=false all_imports -B
$(MAKE) IMP=true MIR=true PAT=false IMP_LARGE=false all_imports -B

.PHONY: refresh-%
refresh-%:
make IMP=true IMP_LARGE=true MIR=true PAT=false $(IMPORTDIR)/$*_import.owl -B
$(MAKE) IMP=true IMP_LARGE=true MIR=true PAT=false $(IMPORTDIR)/$*_import.owl -B

.PHONY: no-mirror-refresh-%
no-mirror-refresh-%:
make IMP=true IMP_LARGE=true MIR=false PAT=false $(IMPORTDIR)/$*_import.owl -B

$(MAKE) IMP=true IMP_LARGE=true MIR=false PAT=false $(IMPORTDIR)/$*_import.owl -B

# ----------------------------------------
# Mirroring upstream ontologies
Expand All @@ -310,32 +321,41 @@ IMP_LARGE=true # Global parameter to bypass handling of large imports
.PHONY: mirror-ro
.PRECIOUS: $(MIRRORDIR)/ro.owl
mirror-ro: | $(TMPDIR)
if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/ro.owl --create-dirs -o $(MIRRORDIR)/ro.owl --retry 4 --max-time 200 && $(ROBOT) convert -i $(MIRRORDIR)/ro.owl -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi
if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/ro.owl --create-dirs -o $(MIRRORDIR)/ro.owl --retry 4 --max-time 200 &&\
$(ROBOT) convert -i $(MIRRORDIR)/ro.owl -o $@.tmp.owl &&\
mv $@.tmp.owl $(TMPDIR)/$@.owl; fi


## ONTOLOGY: caro
.PHONY: mirror-caro
.PRECIOUS: $(MIRRORDIR)/caro.owl
mirror-caro: | $(TMPDIR)
if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/caro.owl --create-dirs -o $(MIRRORDIR)/caro.owl --retry 4 --max-time 200 && $(ROBOT) convert -i $(MIRRORDIR)/caro.owl -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi
if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/caro.owl --create-dirs -o $(MIRRORDIR)/caro.owl --retry 4 --max-time 200 &&\
$(ROBOT) convert -i $(MIRRORDIR)/caro.owl -o $@.tmp.owl &&\
mv $@.tmp.owl $(TMPDIR)/$@.owl; fi


## ONTOLOGY: fbbt
.PHONY: mirror-fbbt
.PRECIOUS: $(MIRRORDIR)/fbbt.owl
mirror-fbbt: | $(TMPDIR)
if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/fbbt.owl --create-dirs -o $(MIRRORDIR)/fbbt.owl --retry 4 --max-time 200 && $(ROBOT) convert -i $(MIRRORDIR)/fbbt.owl -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi
if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/fbbt.owl --create-dirs -o $(MIRRORDIR)/fbbt.owl --retry 4 --max-time 200 &&\
$(ROBOT) convert -i $(MIRRORDIR)/fbbt.owl -o $@.tmp.owl &&\
mv $@.tmp.owl $(TMPDIR)/$@.owl; fi


## ONTOLOGY: obi
.PHONY: mirror-obi
.PRECIOUS: $(MIRRORDIR)/obi.owl
mirror-obi: | $(TMPDIR)
if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/obi.owl --create-dirs -o $(MIRRORDIR)/obi.owl --retry 4 --max-time 200 && $(ROBOT) convert -i $(MIRRORDIR)/obi.owl -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi
if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/obi.owl --create-dirs -o $(MIRRORDIR)/obi.owl --retry 4 --max-time 200 &&\
$(ROBOT) convert -i $(MIRRORDIR)/obi.owl -o $@.tmp.owl &&\
mv $@.tmp.owl $(TMPDIR)/$@.owl; fi


$(MIRRORDIR)/%.owl: mirror-% | $(MIRRORDIR)
if [ $(IMP) = true ] && [ $(MIR) = true ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." && cp $(TMPDIR)/mirror-$*.owl $@; fi; fi
if [ $(IMP) = true ] && [ $(MIR) = true ] && [ -f $(TMPDIR)/mirror-$*.owl ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." &&\
cp $(TMPDIR)/mirror-$*.owl $@; fi; fi



Expand Down Expand Up @@ -371,7 +391,7 @@ SPARQL_EXPORTS_ARGS = $(foreach V,$(SPARQL_EXPORTS),-s $(SPARQLDIR)/$(V).sparql
.PHONY: custom_reports
custom_reports: $(SRC) | $(REPORTDIR)
ifneq ($(SPARQL_EXPORTS_ARGS),)
$(ROBOT) query -f tsv -i $< $(SPARQL_EXPORTS_ARGS)
$(ROBOT) query -f tsv --use-graphs true -i $< $(SPARQL_EXPORTS_ARGS)
endif

# ----------------------------------------
Expand Down Expand Up @@ -403,8 +423,6 @@ $(ONT)-full.json: $(ONT)-full.owl
$(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
convert --check false -f json -o $@.tmp.json &&\
jq -S 'walk(if type == "array" then sort else . end)' $@.tmp.json > $@ && rm $@.tmp.json
# We always want a base - even if it is not explicitly configured..
# We always want a full release - even if it is not explicitly configured..
# ----------------------------------------
# Release artefacts: main release artefacts
# ----------------------------------------
Expand All @@ -428,15 +446,6 @@ $(ONTOLOGYTERMS): $(SRC) $(OTHER_SRC)
touch $(ONTOLOGYTERMS) && \
$(ROBOT) query --use-graphs true -f csv -i $< --query ../sparql/vfbext_terms.sparql $@










# base: OTHER sources of interest, such as definitions owl
$(ONT)-base.owl: $(SRC) $(OTHER_SRC)
$(ROBOT) remove --input $< --select imports --trim false \
Expand All @@ -459,8 +468,7 @@ $(ONT)-non-classified.owl: $(SRC) $(OTHER_SRC) $(IMPORT_FILES)
$(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --annotation oboInOwl:date "$(OBODATE)" --output $@.tmp.owl && mv $@.tmp.owl $@
# foo-simple: (edit->reason,relax,reduce,drop imports, drop every axiom which contains an entity outside the "namespaces of interest")
# drop every axiom: filter --term-file keep_terms.txt --trim true
# remove --select imports --trim false \
# remove --select imports --trim false
$(ONT)-simple.owl: $(SRC) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES)
$(ROBOT) merge --input $< $(patsubst %, -i %, $(OTHER_SRC)) \
reason --reasoner ELK --equivalent-classes-allowed all --exclude-tautologies structural \
Expand All @@ -469,7 +477,7 @@ $(ONT)-simple.owl: $(SRC) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES)
relax \
filter --term-file $(SIMPLESEED) --select "annotations ontology anonymous self" --trim true --signature true \
reduce -r ELK \
query --update ../sparql/inject-subset-declaration.ru \
query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru \
$(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --annotation oboInOwl:date "$(OBODATE)" --output $@.tmp.owl && mv $@.tmp.owl $@
# ----------------------------------------
# Debugging Tools
Expand All @@ -493,7 +501,7 @@ validate-tsv: $(TSV) | $(TMPDIR)
done

validate-all-tsv: $(ALL_TSV_FILES)
make validate-tsv TSV="$^"
$(MAKE) validate-tsv TSV="$^"

# ----------------------------------------
# Editors Utilities
Expand Down Expand Up @@ -536,6 +544,7 @@ Usage: [IMAGE=(odklite|odkfull)] [ODK_DEBUG=yes] sh run.sh make [(IMP|MIR|IMP_LA

Core commands:
* prepare_release: Run the entire release pipeline. Use make IMP=false prepare_release to avoid rerunning the imports
* prepare_release_fast: Run the entire release pipeline without refreshing imports, recreating components or recompiling patterns.
* update_repo: Update the ODK repository setup using the config file vfbext-odk.yaml
* test: Running all validation tests
* odkversion: Show the current version of the ODK Makefile and ROBOT.
Expand All @@ -545,6 +554,7 @@ Core commands:

Imports management:
* refresh-imports: Refresh all imports and mirrors.
* refresh-components: Refresh all components.
* no-mirror-refresh-imports: Refresh all imports without downloading mirrors.
* refresh-imports-excluding-large: Refresh all imports and mirrors, but skipping the ones labelled as 'is_large'.
* refresh-%: Refresh a single import, i.e. refresh-go will refresh 'imports/go_import.owl'.
Expand Down
11 changes: 9 additions & 2 deletions src/ontology/run.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,22 @@
# See README-editors.md for more details.

IMAGE=${IMAGE:-odkfull}

ODK_JAVA_OPTS=-Xmx8G
ODK_DEBUG=${ODK_DEBUG:-no}

TIMECMD=
if [ x$ODK_DEBUG = xyes ]; then
# If you wish to change the format string, take care of using
# non-breaking spaces (U+00A0) instead of normal spaces, to
# prevent the shell from tokenizing the format string.
echo "Running ${IMAGE} with ${ODK_JAVA_OPTS} of memory for ROBOT and Java-based pipeline steps."
TIMECMD="/usr/bin/time -f ### DEBUG STATS ###\nElapsed time: %E\nPeak memory: %M kb"
fi

docker run -v $PWD/../../:/work -w /work/src/ontology -e ROBOT_JAVA_ARGS='-Xmx8G' -e JAVA_OPTS='-Xmx8G' --rm -ti obolibrary/$IMAGE $TIMECMD "$@"
docker run -v $PWD/../../:/work -w /work/src/ontology -e ROBOT_JAVA_ARGS="$ODK_JAVA_OPTS" -e JAVA_OPTS="$ODK_JAVA_OPTS" --rm -ti obolibrary/$IMAGE $TIMECMD "$@"

case "$@" in
*update_repo*|*release*)
echo "Please remember to update your ODK image from time to time: https://oboacademy.github.io/obook/howto/odk-update/."
;;
esac
20 changes: 9 additions & 11 deletions src/ontology/vfbext.Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,16 @@
## Customize Makefile settings for vfbext
##
##
## If you need to customize your Makefile, make
## changes here rather than in the main Makefile

# Include copying import files
.PHONY: prepare_release
prepare_release: $(ASSETS)
rsync -R $(RELEASE_ASSETS) $(RELEASEDIR) &&\
rsync -R $(IMPORT_FILES) $(RELEASEDIR) &&\
rm -f $(CLEANFILES) &&\
rm -f imports/*_terms_combined.txt &&\
echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release on your git hosting site such as GitHub or GitLab"

# Add VFB iri
$(ONT).owl: $(ONT)-full.owl
Expand All @@ -16,13 +24,3 @@ $(IMPORTDIR)/caro_import.owl: $(MIRRORDIR)/caro.owl
extract --method MIREOT --force true --copy-ontology-annotations true \
--lower-term "obo:CARO_0030002" \
annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output $@.tmp.owl && mv $@.tmp.owl $@; fi


# Include copying import files
.PHONY: prepare_release
prepare_release: $(ASSETS)
rsync -R $(RELEASE_ASSETS) $(RELEASEDIR) &&\
rsync -R $(IMPORT_FILES) $(RELEASEDIR) &&\
rm -f $(CLEANFILES) &&\
rm -f imports/*_terms_combined.txt &&\
echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release on your git hosting site such as GitHub or GitLab"
4 changes: 3 additions & 1 deletion src/scripts/update_repo.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,7 @@ mkdir -p $ROOTDIR/.github/workflows
cp -n target/$OID/.github/workflows/qc.yml $ROOTDIR/.github/workflows/qc.yml



echo "WARNING: These files should be manually migrated: mkdocs.yaml, .gitignore, src/ontology/catalog.xml (if you added a new import or component)"
echo "Update successfully completed."
echo "WARNING: Your QC workflows have not been updated automatically. Please update the ODK version number(s) in .github/workflows/qc.yml."
echo "Ontology repository update successfully completed."
11 changes: 11 additions & 0 deletions src/sparql/inject-synonymtype-declaration.ru
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
PREFIX : <http://www.test.com/ns/test#>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>

INSERT { ?y rdfs:subPropertyOf <http://www.geneontology.org/formats/oboInOwl#SynonymTypeProperty> . }

WHERE {
?x <http://www.geneontology.org/formats/oboInOwl#hasSynonymType> ?y .
FILTER(isIRI(?y))
FILTER(regex(str(?y),"^(http://purl.obolibrary.org/obo/)") || regex(str(?y),"^(http://www.ebi.ac.uk/efo/)") || regex(str(?y),"^(https://w3id.org/biolink/)"))
}

0 comments on commit 1606d71

Please sign in to comment.