diff --git a/src/ontology/Makefile b/src/ontology/Makefile index 5abd0af..11b0963 100644 --- a/src/ontology/Makefile +++ b/src/ontology/Makefile @@ -1,7 +1,7 @@ # ---------------------------------------- # Makefile for vfbext # Generated using ontology-development-kit -# ODK Version: v1.3.0 +# ODK Version: v1.3.1 # ---------------------------------------- # IMPORTANT: DO NOT EDIT THIS FILE. To override default make goals, use vfbext.Makefile instead @@ -40,7 +40,7 @@ REPORT_PROFILE_OPTS = OBO_FORMAT_OPTIONS = SPARQL_VALIDATION_CHECKS = owldef-self-reference iri-range label-with-iri SPARQL_EXPORTS = basic-report class-count-by-prefix edges xrefs obsoletes synonyms -ODK_VERSION_MAKEFILE = v1.3.0 +ODK_VERSION_MAKEFILE = v1.3.1 TODAY ?= $(shell date +%Y-%m-%d) OBODATE ?= $(shell date +'%d:%m:%Y %H:%M') @@ -60,15 +60,21 @@ RELEASE_ARTEFACTS = $(sort $(ONT)-base $(ONT)-simple $(ONT)-non-classified $(ONT .PHONY: .FORCE .PHONY: all -all: odkversion test all_assets +all: all_odk + +.PHONY: all_odk +all_odk: odkversion test all_assets .PHONY: test test: odkversion sparql_test robot_reports $(REPORTDIR)/validate_profile_owl2dl_$(ONT).owl.txt - $(ROBOT) reason --input $(SRC) --reasoner ELK --equivalent-classes-allowed all --exclude-tautologies structural --output test.owl && rm test.owl && echo "Success" + $(ROBOT) reason --input $(SRC) --reasoner ELK --equivalent-classes-allowed all \ + --exclude-tautologies structural --output test.owl && rm test.owl &&\ + echo "Success" .PHONY: odkversion odkversion: - echo "ODK Makefile version: $(ODK_VERSION_MAKEFILE) (this is the version of the ODK with which this Makefile was generated, not the version of the ODK you are running)" &&\ + echo "ODK Makefile version: $(ODK_VERSION_MAKEFILE) (this is the version of the ODK with which this Makefile was generated, \ + not the version of the ODK you are running)" &&\ echo "ROBOT version (ODK): " && $(ROBOT) --version $(TMPDIR) $(REPORTDIR) $(MIRRORDIR) $(IMPORTDIR) $(COMPONENTSDIR) $(SUBSETDIR): @@ -160,10 +166,10 @@ endif # ---------------------------------------- $(REPORTDIR)/$(SRC)-obo-report.tsv: $(SRCMERGED) | $(REPORTDIR) - $(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --print 5 -o $@ + $(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --base-iri http://purl.obolibrary.org/obo/fbbt/vfb/VFBext_ --base-iri http://purl.obolibrary.org/obo/fbbt/vfb/VFB_ --print 5 -o $@ $(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR) - $(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --print 5 -o $@ + $(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --base-iri http://purl.obolibrary.org/obo/fbbt/vfb/VFBext_ --base-iri http://purl.obolibrary.org/obo/fbbt/vfb/VFB_ --print 5 -o $@ # ---------------------------------------- # Release assets @@ -196,10 +202,11 @@ CLEANFILES=$(MAIN_FILES) $(SRCMERGED) # It will ensure that all assets/files are fresh, and will copy to release folder .PHONY: prepare_release -prepare_release: all +prepare_release: all_odk rsync -R $(RELEASE_ASSETS) $(RELEASEDIR) &&\ rm -f $(CLEANFILES) &&\ - echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release on your git hosting site such as GitHub or GitLab" + echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release \ + on your git hosting site such as GitHub or GitLab" .PHONY: prepare_initial_release prepare_initial_release: all_assets @@ -207,6 +214,10 @@ prepare_initial_release: all_assets rm -f $(patsubst %, ./%, $(CLEANFILES)) &&\ cd $(RELEASEDIR) && git add $(RELEASE_ASSETS) +.PHONY: prepare_release_fast +prepare_release_fast: + $(MAKE) prepare_release IMP=false PAT=false MIR=false COMP=false + # ------------------------ # Imports: Seeding system # ------------------------ @@ -257,7 +268,7 @@ $(IMPORTDIR)/%_terms_combined.txt: $(IMPORTSEED) $(IMPORTDIR)/%_terms.txt $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms_combined.txt if [ $(IMP) = true ]; then $(ROBOT) query -i $< --update ../sparql/preprocess-module.ru \ extract -T $(IMPORTDIR)/$*_terms_combined.txt --force true --copy-ontology-annotations true --individuals include --method BOT \ - query --update ../sparql/inject-subset-declaration.ru --update ../sparql/postprocess-module.ru \ + query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru --update ../sparql/postprocess-module.ru \ annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output $@.tmp.owl && mv $@.tmp.owl $@; fi .PRECIOUS: $(IMPORTDIR)/%_import.owl @@ -269,32 +280,32 @@ $(IMPORTDIR)/caro_import.owl: $(MIRRORDIR)/caro.owl ## Module for ontology: obi $(IMPORTDIR)/obi_import.owl: $(MIRRORDIR)/obi.owl $(IMPORTDIR)/obi_terms_combined.txt - if [ $(IMP) = true ]; then $(ROBOT) extract -i $< -T $(IMPORTDIR)/obi_terms_combined.txt --force true --method BOT \ + if [ $(IMP) = true ]; then $(ROBOT) extract -i $< -T $(IMPORTDIR)/obi_terms_combined.txt --copy-ontology-annotations true --force true --method BOT \ remove --base-iri $(URIBASE)/OBI --axioms external --preserve-structure false --trim false \ - query --update ../sparql/inject-subset-declaration.ru \ + query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru \ remove $(patsubst %, --term %, $(ANNOTATION_PROPERTIES)) -T $(IMPORTDIR)/obi_terms_combined.txt --select complement --select "classes individuals annotation-properties" \ annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output $@.tmp.owl && mv $@.tmp.owl $@; fi + .PHONY: refresh-imports refresh-imports: - make IMP=true MIR=true PAT=false IMP_LARGE=true all_imports -B + $(MAKE) IMP=true MIR=true PAT=false IMP_LARGE=true all_imports -B .PHONY: no-mirror-refresh-imports no-mirror-refresh-imports: - make IMP=true MIR=false PAT=false IMP_LARGE=true all_imports -B + $(MAKE) IMP=true MIR=false PAT=false IMP_LARGE=true all_imports -B .PHONY: refresh-imports-excluding-large refresh-imports-excluding-large: - make IMP=true MIR=true PAT=false IMP_LARGE=false all_imports -B + $(MAKE) IMP=true MIR=true PAT=false IMP_LARGE=false all_imports -B .PHONY: refresh-% refresh-%: - make IMP=true IMP_LARGE=true MIR=true PAT=false $(IMPORTDIR)/$*_import.owl -B + $(MAKE) IMP=true IMP_LARGE=true MIR=true PAT=false $(IMPORTDIR)/$*_import.owl -B .PHONY: no-mirror-refresh-% no-mirror-refresh-%: - make IMP=true IMP_LARGE=true MIR=false PAT=false $(IMPORTDIR)/$*_import.owl -B - + $(MAKE) IMP=true IMP_LARGE=true MIR=false PAT=false $(IMPORTDIR)/$*_import.owl -B # ---------------------------------------- # Mirroring upstream ontologies @@ -310,32 +321,41 @@ IMP_LARGE=true # Global parameter to bypass handling of large imports .PHONY: mirror-ro .PRECIOUS: $(MIRRORDIR)/ro.owl mirror-ro: | $(TMPDIR) - if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/ro.owl --create-dirs -o $(MIRRORDIR)/ro.owl --retry 4 --max-time 200 && $(ROBOT) convert -i $(MIRRORDIR)/ro.owl -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi + if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/ro.owl --create-dirs -o $(MIRRORDIR)/ro.owl --retry 4 --max-time 200 &&\ + $(ROBOT) convert -i $(MIRRORDIR)/ro.owl -o $@.tmp.owl &&\ + mv $@.tmp.owl $(TMPDIR)/$@.owl; fi ## ONTOLOGY: caro .PHONY: mirror-caro .PRECIOUS: $(MIRRORDIR)/caro.owl mirror-caro: | $(TMPDIR) - if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/caro.owl --create-dirs -o $(MIRRORDIR)/caro.owl --retry 4 --max-time 200 && $(ROBOT) convert -i $(MIRRORDIR)/caro.owl -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi + if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/caro.owl --create-dirs -o $(MIRRORDIR)/caro.owl --retry 4 --max-time 200 &&\ + $(ROBOT) convert -i $(MIRRORDIR)/caro.owl -o $@.tmp.owl &&\ + mv $@.tmp.owl $(TMPDIR)/$@.owl; fi ## ONTOLOGY: fbbt .PHONY: mirror-fbbt .PRECIOUS: $(MIRRORDIR)/fbbt.owl mirror-fbbt: | $(TMPDIR) - if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/fbbt.owl --create-dirs -o $(MIRRORDIR)/fbbt.owl --retry 4 --max-time 200 && $(ROBOT) convert -i $(MIRRORDIR)/fbbt.owl -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi + if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/fbbt.owl --create-dirs -o $(MIRRORDIR)/fbbt.owl --retry 4 --max-time 200 &&\ + $(ROBOT) convert -i $(MIRRORDIR)/fbbt.owl -o $@.tmp.owl &&\ + mv $@.tmp.owl $(TMPDIR)/$@.owl; fi ## ONTOLOGY: obi .PHONY: mirror-obi .PRECIOUS: $(MIRRORDIR)/obi.owl mirror-obi: | $(TMPDIR) - if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/obi.owl --create-dirs -o $(MIRRORDIR)/obi.owl --retry 4 --max-time 200 && $(ROBOT) convert -i $(MIRRORDIR)/obi.owl -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi + if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/obi.owl --create-dirs -o $(MIRRORDIR)/obi.owl --retry 4 --max-time 200 &&\ + $(ROBOT) convert -i $(MIRRORDIR)/obi.owl -o $@.tmp.owl &&\ + mv $@.tmp.owl $(TMPDIR)/$@.owl; fi $(MIRRORDIR)/%.owl: mirror-% | $(MIRRORDIR) - if [ $(IMP) = true ] && [ $(MIR) = true ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." && cp $(TMPDIR)/mirror-$*.owl $@; fi; fi + if [ $(IMP) = true ] && [ $(MIR) = true ] && [ -f $(TMPDIR)/mirror-$*.owl ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." &&\ + cp $(TMPDIR)/mirror-$*.owl $@; fi; fi @@ -371,7 +391,7 @@ SPARQL_EXPORTS_ARGS = $(foreach V,$(SPARQL_EXPORTS),-s $(SPARQLDIR)/$(V).sparql .PHONY: custom_reports custom_reports: $(SRC) | $(REPORTDIR) ifneq ($(SPARQL_EXPORTS_ARGS),) - $(ROBOT) query -f tsv -i $< $(SPARQL_EXPORTS_ARGS) + $(ROBOT) query -f tsv --use-graphs true -i $< $(SPARQL_EXPORTS_ARGS) endif # ---------------------------------------- @@ -403,8 +423,6 @@ $(ONT)-full.json: $(ONT)-full.owl $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ convert --check false -f json -o $@.tmp.json &&\ jq -S 'walk(if type == "array" then sort else . end)' $@.tmp.json > $@ && rm $@.tmp.json -# We always want a base - even if it is not explicitly configured.. -# We always want a full release - even if it is not explicitly configured.. # ---------------------------------------- # Release artefacts: main release artefacts # ---------------------------------------- @@ -428,15 +446,6 @@ $(ONTOLOGYTERMS): $(SRC) $(OTHER_SRC) touch $(ONTOLOGYTERMS) && \ $(ROBOT) query --use-graphs true -f csv -i $< --query ../sparql/vfbext_terms.sparql $@ - - - - - - - - - # base: OTHER sources of interest, such as definitions owl $(ONT)-base.owl: $(SRC) $(OTHER_SRC) $(ROBOT) remove --input $< --select imports --trim false \ @@ -459,8 +468,7 @@ $(ONT)-non-classified.owl: $(SRC) $(OTHER_SRC) $(IMPORT_FILES) $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --annotation oboInOwl:date "$(OBODATE)" --output $@.tmp.owl && mv $@.tmp.owl $@ # foo-simple: (edit->reason,relax,reduce,drop imports, drop every axiom which contains an entity outside the "namespaces of interest") # drop every axiom: filter --term-file keep_terms.txt --trim true -# remove --select imports --trim false \ - +# remove --select imports --trim false $(ONT)-simple.owl: $(SRC) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES) $(ROBOT) merge --input $< $(patsubst %, -i %, $(OTHER_SRC)) \ reason --reasoner ELK --equivalent-classes-allowed all --exclude-tautologies structural \ @@ -469,7 +477,7 @@ $(ONT)-simple.owl: $(SRC) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES) relax \ filter --term-file $(SIMPLESEED) --select "annotations ontology anonymous self" --trim true --signature true \ reduce -r ELK \ - query --update ../sparql/inject-subset-declaration.ru \ + query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru \ $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --annotation oboInOwl:date "$(OBODATE)" --output $@.tmp.owl && mv $@.tmp.owl $@ # ---------------------------------------- # Debugging Tools @@ -493,7 +501,7 @@ validate-tsv: $(TSV) | $(TMPDIR) done validate-all-tsv: $(ALL_TSV_FILES) - make validate-tsv TSV="$^" + $(MAKE) validate-tsv TSV="$^" # ---------------------------------------- # Editors Utilities @@ -536,6 +544,7 @@ Usage: [IMAGE=(odklite|odkfull)] [ODK_DEBUG=yes] sh run.sh make [(IMP|MIR|IMP_LA Core commands: * prepare_release: Run the entire release pipeline. Use make IMP=false prepare_release to avoid rerunning the imports +* prepare_release_fast: Run the entire release pipeline without refreshing imports, recreating components or recompiling patterns. * update_repo: Update the ODK repository setup using the config file vfbext-odk.yaml * test: Running all validation tests * odkversion: Show the current version of the ODK Makefile and ROBOT. @@ -545,6 +554,7 @@ Core commands: Imports management: * refresh-imports: Refresh all imports and mirrors. +* refresh-components: Refresh all components. * no-mirror-refresh-imports: Refresh all imports without downloading mirrors. * refresh-imports-excluding-large: Refresh all imports and mirrors, but skipping the ones labelled as 'is_large'. * refresh-%: Refresh a single import, i.e. refresh-go will refresh 'imports/go_import.owl'. diff --git a/src/ontology/run.sh b/src/ontology/run.sh index e9c4f03..9a498dd 100755 --- a/src/ontology/run.sh +++ b/src/ontology/run.sh @@ -11,7 +11,7 @@ # See README-editors.md for more details. IMAGE=${IMAGE:-odkfull} - +ODK_JAVA_OPTS=-Xmx8G ODK_DEBUG=${ODK_DEBUG:-no} TIMECMD= @@ -19,7 +19,14 @@ if [ x$ODK_DEBUG = xyes ]; then # If you wish to change the format string, take care of using # non-breaking spaces (U+00A0) instead of normal spaces, to # prevent the shell from tokenizing the format string. + echo "Running ${IMAGE} with ${ODK_JAVA_OPTS} of memory for ROBOT and Java-based pipeline steps." TIMECMD="/usr/bin/time -f ### DEBUG STATS ###\nElapsed time: %E\nPeak memory: %M kb" fi -docker run -v $PWD/../../:/work -w /work/src/ontology -e ROBOT_JAVA_ARGS='-Xmx8G' -e JAVA_OPTS='-Xmx8G' --rm -ti obolibrary/$IMAGE $TIMECMD "$@" \ No newline at end of file +docker run -v $PWD/../../:/work -w /work/src/ontology -e ROBOT_JAVA_ARGS="$ODK_JAVA_OPTS" -e JAVA_OPTS="$ODK_JAVA_OPTS" --rm -ti obolibrary/$IMAGE $TIMECMD "$@" + +case "$@" in +*update_repo*|*release*) + echo "Please remember to update your ODK image from time to time: https://oboacademy.github.io/obook/howto/odk-update/." + ;; +esac \ No newline at end of file diff --git a/src/ontology/vfbext.Makefile b/src/ontology/vfbext.Makefile index d2b0c61..9bb52c0 100644 --- a/src/ontology/vfbext.Makefile +++ b/src/ontology/vfbext.Makefile @@ -1,8 +1,16 @@ ## Customize Makefile settings for vfbext -## +## ## If you need to customize your Makefile, make ## changes here rather than in the main Makefile +# Include copying import files +.PHONY: prepare_release +prepare_release: $(ASSETS) + rsync -R $(RELEASE_ASSETS) $(RELEASEDIR) &&\ + rsync -R $(IMPORT_FILES) $(RELEASEDIR) &&\ + rm -f $(CLEANFILES) &&\ + rm -f imports/*_terms_combined.txt &&\ + echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release on your git hosting site such as GitHub or GitLab" # Add VFB iri $(ONT).owl: $(ONT)-full.owl @@ -16,13 +24,3 @@ $(IMPORTDIR)/caro_import.owl: $(MIRRORDIR)/caro.owl extract --method MIREOT --force true --copy-ontology-annotations true \ --lower-term "obo:CARO_0030002" \ annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output $@.tmp.owl && mv $@.tmp.owl $@; fi - - -# Include copying import files -.PHONY: prepare_release -prepare_release: $(ASSETS) - rsync -R $(RELEASE_ASSETS) $(RELEASEDIR) &&\ - rsync -R $(IMPORT_FILES) $(RELEASEDIR) &&\ - rm -f $(CLEANFILES) &&\ - rm -f imports/*_terms_combined.txt &&\ - echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release on your git hosting site such as GitHub or GitLab" diff --git a/src/scripts/update_repo.sh b/src/scripts/update_repo.sh index f50e9c8..987b81c 100644 --- a/src/scripts/update_repo.sh +++ b/src/scripts/update_repo.sh @@ -24,5 +24,7 @@ mkdir -p $ROOTDIR/.github/workflows cp -n target/$OID/.github/workflows/qc.yml $ROOTDIR/.github/workflows/qc.yml + echo "WARNING: These files should be manually migrated: mkdocs.yaml, .gitignore, src/ontology/catalog.xml (if you added a new import or component)" -echo "Update successfully completed." \ No newline at end of file +echo "WARNING: Your QC workflows have not been updated automatically. Please update the ODK version number(s) in .github/workflows/qc.yml." +echo "Ontology repository update successfully completed." \ No newline at end of file diff --git a/src/sparql/inject-synonymtype-declaration.ru b/src/sparql/inject-synonymtype-declaration.ru new file mode 100644 index 0000000..9906089 --- /dev/null +++ b/src/sparql/inject-synonymtype-declaration.ru @@ -0,0 +1,11 @@ +PREFIX : +PREFIX rdf: +PREFIX rdfs: + +INSERT { ?y rdfs:subPropertyOf . } + +WHERE { + ?x ?y . + FILTER(isIRI(?y)) + FILTER(regex(str(?y),"^(http://purl.obolibrary.org/obo/)") || regex(str(?y),"^(http://www.ebi.ac.uk/efo/)") || regex(str(?y),"^(https://w3id.org/biolink/)")) +} \ No newline at end of file