From 2e28502972ec2536576dad98689b068676f112b0 Mon Sep 17 00:00:00 2001
From: Manuel Holtgrewe <manuel.holtgrewe@bih-charite.de>
Date: Fri, 24 Nov 2023 08:29:44 +0100
Subject: [PATCH] docs: major update of all data to fixed protobufs
---
README.md | 329 +++++++++++++++++++++++++++++++++---------------------
1 file changed, 200 insertions(+), 129 deletions(-)
diff --git a/README.md b/README.md
index b2e440e..a29f6ac 100644
--- a/README.md
+++ b/README.md
@@ -73,32 +73,61 @@ For this, we have prepared strongly reduced data sets (overall less than 2GB rat
Obtain the annonars data:
```bash session
-mkdir -p .dev/volumes/varfish-static/data/download
-SRC_DST="
-full/annonars/gnomad-mtdna-grch37-3.1+0.12.8/*:annonars/gnomad-mtdna-grch37-3.1+0.12.8
-full/annonars/gnomad-mtdna-grch38-3.1+0.12.8/*:annonars/gnomad-mtdna-grch38-3.1+0.12.8
-full/annonars/helixmtdb-grch37-20200327+0.12.8/*:annonars/helixmtdb-grch37-20200327+0.12.8
-full/annonars/helixmtdb-grch38-20200327+0.12.8/*:annonars/helixmtdb-grch38-20200327+0.12.8
-full/annonars/genes-3.1+2.1.1+4.4+20230606+10.1+20230828+0.19.0/*:annonars/genes-3.1+2.1.1+4.4+20230606+10.1+20230828+0.19.0
-full/mehari/genes-txs-grch37-0.3.0/*:mehari/genes-txs-grch37-0.3.0
-full/mehari/genes-txs-grch38-0.3.0/*:mehari/genes-txs-grch38-0.3.0
-full/mehari/genes-xlink-20230624/*:mehari/genes-xlink-20230624
-full/tracks/*:tracks
-full/worker/*:worker
-reduced-dev/annonars/*:annonars
-reduced-dev/mehari/*:mehari
-reduced-dev/viguno/*:viguno
-"
-(set -x; for src_dst in $SRC_DST; do \
- src=$(echo $src_dst | cut -d : -f 1); \
- dst=$(echo $src_dst | cut -d : -f 2); \
- mkdir -p .dev/volumes/varfish-static/data/download/$dst; \
+cat <<"EOF" >/tmp/tokens.txt
+## -- full data -------------------------------------------------------------
+## uncomment the lines and use instead of reduced data for a full installation
+# full/annonars/cadd-grch37-1.6+0.29.1
+# full/annonars/cadd-grch38-1.6+0.29.1
+# full/annonars/cons-grch37-20161007+0.29.1
+# full/annonars/cons-grch38-20190906+0.29.1
+# full/annonars/dbnsfp-grch37-4.4a+0.29.1
+# full/annonars/dbnsfp-grch37-4.4c+0.29.1
+# full/annonars/dbnsfp-grch38-4.4a+0.29.1
+# full/annonars/dbnsfp-grch38-4.4c+0.29.1
+# full/annonars/dbscsnv-grch37-1.1+0.29.1
+# full/annonars/dbscsnv-grch38-1.1+0.29.1
+# full/annonars/dbsnp-grch37-b151+0.29.1
+# full/annonars/dbsnp-grch38-b151+0.29.1
+full/annonars/functional-grch37-105.20201022+0.29.1
+full/annonars/functional-grch38-110+0.29.1
+full/annonars/genes-3.1+2.1.1+4.4+20230606+10.1+20231123+0.29.3
+# full/annonars/gnomad-exomes-grch37-2.1.1+0.29.1
+# full/annonars/gnomad-exomes-grch38-2.1.1+0.29.1
+# full/annonars/gnomad-genomes-grch37-2.1.1+0.29.1
+# full/annonars/gnomad-genomes-grch38-3.1.2+0.29.1
+full/annonars/gnomad-mtdna-grch37-3.1+0.29.1
+full/annonars/gnomad-mtdna-grch38-3.1+0.29.1
+full/annonars/gnomad-sv-exomes-grch37-0.3.1+0.29.1
+full/annonars/gnomad-sv-exomes-grch38-4.0+0.29.1
+full/annonars/gnomad-sv-genomes-grch37-2.1.1+0.29.1
+full/annonars/gnomad-sv-genomes-grch38-4.0+0.29.1
+full/annonars/helixmtdb-grch37-20200327+0.29.1
+full/annonars/helixmtdb-grch38-20200327+0.29.1
+full/annonars/regions-grch37-20231122+0.29.3
+full/annonars/regions-grch38-20231122+0.29.3
+# full/mehari/freqs-grch37-2.1.1+2.1.1+3.1+20200327+0.29.1
+# full/mehari/freqs-grch37-2.1.1+2.1.1+3.1+20200327+0.29.1
+full/mehari/genes-xlink-20231122
+full/tracks/*
+full/worker/*
+# full/viguno/hpo-20230606+0.1.6
+## -- reduced data for dev ---------------------------------------------------
+reduced-dev/annonars/*
+reduced-dev/mehari/*
+reduced-dev/viguno/*
+EOF
+STATIC=varfish-static
+mkdir -p .dev/volumes/$STATIC/data/download
+(set -x; for token in $(grep -v ^# /tmp/tokens.txt); do \
+ src=$token; \
+ dst=$(echo $token | perl -p -e 's|/\*||' | perl -p -e 's#^(full/|reduced-dev/)##'); \
+ mkdir -p .dev/volumes/$STATIC/data/download/$dst; \
s5cmd \
--endpoint-url=https://ceph-s3-public.cubi.bihealth.org \
--no-sign-request \
sync \
"s3://varfish-public/$src" \
- ".dev/volumes/varfish-static/data/download/$dst"; \
+ ".dev/volumes/$STATIC/data/download/$dst"; \
done)
```
@@ -109,21 +138,22 @@ Setup symlink structure so the data is at the expected location.
## annonars
##
-mkdir -p .dev/volumes/varfish-static/data/annonars
+STATIC=varfish-static
+mkdir -p .dev/volumes/$STATIC/data/annonars
-ln -sr .dev/volumes/varfish-static/data/download/annonars/genes-* \
- .dev/volumes/varfish-static/data/annonars/genes
+ln -sr .dev/volumes/$STATIC/data/download/annonars/genes-* \
+ .dev/volumes/$STATIC/data/annonars/genes
names="cadd dbsnp dbnsfp dbscsnv gnomad-mtdna gnomad-genomes gnomad-exomes helixmtdb cons"; \
for genome in grch37 grch38; do \
for name in $names; do \
- mkdir -p .dev/volumes/varfish-static/data/annonars/$genome; \
- test -e .dev/volumes/varfish-static/data/$genome/$name || \
+ mkdir -p .dev/volumes/$STATIC/data/annonars/$genome; \
+ test -e .dev/volumes/$STATIC/data/$genome/$name || \
ln -sr \
- $(echo .dev/volumes/varfish-static/data/download/annonars/$name-$genome-* \
+ $(echo .dev/volumes/$STATIC/data/download/annonars/$name-$genome-* \
| tr ' ' '\n' \
| tail -n 1) \
- .dev/volumes/varfish-static/data/annonars/$genome/$name; \
+ .dev/volumes/$STATIC/data/annonars/$genome/$name; \
done; \
done
@@ -131,127 +161,166 @@ done
## mehari
##
-mkdir -p .dev/volumes/varfish-static/data/mehari/grch3{7,8}
+STATIC=varfish-static
+mkdir -p .dev/volumes/$STATIC/data/mehari/grch3{7,8}
-ln -sr .dev/volumes/varfish-static/data/download/mehari/genes-txs-grch37-*/mehari-data-txs-grch37-*in.zst \
- .dev/volumes/varfish-static/data/mehari/grch37/txs.bin.zst
-ln -sr .dev/volumes/varfish-static/data/download/mehari/freqs-grch37-* \
- .dev/volumes/varfish-static/data/mehari/grch37/freqs
-ln -sr .dev/volumes/varfish-static/data/download/mehari/genes-txs-grch38-*/mehari-data-txs-grch38-*.bin.zst \
- .dev/volumes/varfish-static/data/mehari/grch38/txs.bin.zst
-ln -sr .dev/volumes/varfish-static/data/download/mehari/freqs-grch38-* \
- .dev/volumes/varfish-static/data/mehari/grch38/freqs
+rm -f .dev/volumes/$STATIC/data/mehari/grch3?/freqs
+
+ln -sr .dev/volumes/$STATIC/data/download/mehari/freqs-grch37-* \
+ .dev/volumes/$STATIC/data/mehari/grch37/freqs
+ln -sr .dev/volumes/$STATIC/data/download/mehari/freqs-grch38-* \
+ .dev/volumes/$STATIC/data/mehari/grch38/freqs
##
## viguno
##
-ln -sr .dev/volumes/varfish-static/data/download/mehari/genes-xlink-20230624/genes-xlink.tsv \
- .dev/volumes/varfish-static/data/hgnc_xlink.tsv
-ln -sr .dev/volumes/varfish-static/data/download/viguno/hpo-20230606+0.1.6 \
- .dev/volumes/varfish-static/data/hpo
+STATIC=varfish-static
+
+rm -f .dev/volumes/$STATIC/data/{hgnc_xlink.tsv,hpo}
+
+ln -sr .dev/volumes/$STATIC/data/download/mehari/genes-xlink-20231122/genes-xlink.tsv \
+ .dev/volumes/$STATIC/data/hgnc_xlink.tsv
+ln -sr .dev/volumes/$STATIC/data/download/viguno/hpo-20230606+0.1.6 \
+ .dev/volumes/$STATIC/data/hpo
##
## worker
##
-mkdir -p .dev/volumes/varfish-static/data/worker/{grch3{7,8}/strucvars/bgdbs,noref/genes}
-
-ln -sr .dev/volumes/varfish-static/data/download/worker/bgdb-exac-grch37-*/bgdb-exac.bin \
- .dev/volumes/varfish-static/data/worker/grch37/strucvars/bgdbs/exac.bin
-ln -sr .dev/volumes/varfish-static/data/download/worker/bgdb-g1k-grch37-phase3v2+0.9.0/bgdb-g1k.bin \
- .dev/volumes/varfish-static/data/worker/grch37/strucvars/bgdbs/g1k.bin
-ln -sr .dev/volumes/varfish-static/data/download/worker/bgdb-gnomad-grch37-*/bgdb-gnomad.bin \
- .dev/volumes/varfish-static/data/worker/grch37/strucvars/bgdbs/gnomad.bin
-ln -sr .dev/volumes/varfish-static/data/download/worker/bgdb-dbvar-grch37-*/bgdb-dbvar.bin \
- .dev/volumes/varfish-static/data/worker/grch37/strucvars/bgdbs/dbvar.bin
-ln -sr .dev/volumes/varfish-static/data/download/worker/bgdb-dbvar-grch38-*/bgdb-dbvar.bin \
- .dev/volumes/varfish-static/data/worker/grch38/strucvars/bgdbs/dbvar.bin
-ln -sr .dev/volumes/varfish-static/data/download/worker/bgdb-dgv-grch37-*/bgdb-dgv.bin \
- .dev/volumes/varfish-static/data/worker/grch37/strucvars/bgdbs/dgv.bin
-ln -sr .dev/volumes/varfish-static/data/download/worker/bgdb-dgv-grch38-*/bgdb-dgv.bin \
- .dev/volumes/varfish-static/data/worker/grch38/strucvars/bgdbs/dgv.bin
-ln -sr .dev/volumes/varfish-static/data/download/worker/bgdb-dgv-gs-grch37-*/bgdb-dgv-gs.bin \
- .dev/volumes/varfish-static/data/worker/grch37/strucvars/bgdbs/dgv-gs.bin
-ln -sr .dev/volumes/varfish-static/data/download/worker/bgdb-dgv-gs-grch38-*/bgdb-dgv-gs.bin \
- .dev/volumes/varfish-static/data/worker/grch38/strucvars/bgdbs/dgv-gs.bin
-
-ln -sr .dev/volumes/varfish-static/data/download/worker/clinvar-strucvars-grch37-*/clinvar-strucvars.bin \
- .dev/volumes/varfish-static/data/worker/grch37/strucvars/clinvar.bin
-ln -sr .dev/volumes/varfish-static/data/download/worker/clinvar-strucvars-grch38-*/clinvar-strucvars.bin \
- .dev/volumes/varfish-static/data/worker/grch38/strucvars/clinvar.bin
-
-ln -sr .dev/volumes/varfish-static/data/download/worker/patho-mms-grch37-*/patho-mms.bed \
- .dev/volumes/varfish-static/data/worker/grch37/strucvars/patho-mms.bed
-ln -sr .dev/volumes/varfish-static/data/download/worker/patho-mms-grch38-*/patho-mms.bed \
- .dev/volumes/varfish-static/data/worker/grch38/strucvars/patho-mms.bed
-
-mkdir -p .dev/volumes/varfish-static/data/worker/grch3{7,8}/tads
-
-ln -sr .dev/volumes/varfish-static/data/download/worker/tads-grch37-dixon2015/hesc.bed \
- .dev/volumes/varfish-static/data/worker/grch37/tads/hesc.bed
-ln -sr .dev/volumes/varfish-static/data/download/worker/tads-grch38-dixon2015/hesc.bed \
- .dev/volumes/varfish-static/data/worker/grch38/tads/hesc.bed
-
-ln -sr .dev/volumes/varfish-static/data/download/worker/genes-xlink-*/genes-xlink.bin \
- .dev/volumes/varfish-static/data/worker/noref/genes/xlink.bin
-ln -sr .dev/volumes/varfish-static/data/download/worker/acmg-sf-*/acmg_sf.tsv \
- .dev/volumes/varfish-static/data/worker/noref/genes/acmg.tsv
-ln -sr .dev/volumes/varfish-static/data/download/worker/mim2gene-*/mim2gene.tsv \
- .dev/volumes/varfish-static/data/worker/noref/genes/mime2gene.tsv
-
-mkdir -p .dev/volumes/varfish-static/data/worker/grch3{7,8}/genes
-
-ln -sr .dev/volumes/varfish-static/data/download/worker/genes-regions-grch37-*/ensembl_genes.bin \
- .dev/volumes/varfish-static/data/worker/grch37/genes/ensembl_regions.bin
-ln -sr .dev/volumes/varfish-static/data/download/worker/genes-regions-grch38-*/ensembl_genes.bin \
- .dev/volumes/varfish-static/data/worker/grch38/genes/ensembl_regions.bin
-
-ln -sr .dev/volumes/varfish-static/data/download/worker/genes-regions-grch37-*/refseq_genes.bin \
- .dev/volumes/varfish-static/data/worker/grch37/genes/refseq_regions.bin
-ln -sr .dev/volumes/varfish-static/data/download/worker/genes-regions-grch38-*/refseq_genes.bin \
- .dev/volumes/varfish-static/data/worker/grch38/genes/refseq_regions.bin
-
-mkdir -p .dev/volumes/varfish-static/data/worker/grch3{7,8}/features
-
-ln -sr .dev/volumes/varfish-static/data/download/worker/masked-repeat-grch37-*/masked-repeat.bin \
- .dev/volumes/varfish-static/data/worker/grch37/features/masked_repeat.bin
-ln -sr .dev/volumes/varfish-static/data/download/worker/masked-segdup-grch37-*/masked-segdup.bin \
- .dev/volumes/varfish-static/data/worker/grch37/features/masked_seqdup.bin
-
-ln -sr .dev/volumes/varfish-static/data/download/worker/masked-repeat-grch38-*/masked-repeat.bin \
- .dev/volumes/varfish-static/data/worker/grch38/features/masked_repeat.bin
-ln -sr .dev/volumes/varfish-static/data/download/worker/masked-segdup-grch38-*/masked-segdup.bin \
- .dev/volumes/varfish-static/data/worker/grch38/features/masked_seqdup.bin
+STATIC=varfish-static
+mkdir -p .dev/volumes/$STATIC/data/worker/{grch3{7,8}/strucvars/bgdbs,noref/genes}
+
+rm -f .dev/volumes/$STATIC/data/worker/grch3?/strucvars/bgdbs/{exac,g1k,gnomad,dbvar,dgv,dgv-gs}.bin
+
+ln -sr .dev/volumes/$STATIC/data/download/worker/bgdb-exac-grch37-*/bgdb-exac.bin \
+ .dev/volumes/$STATIC/data/worker/grch37/strucvars/bgdbs/exac.bin
+ln -sr .dev/volumes/$STATIC/data/download/worker/bgdb-g1k-grch37-phase3v2+0.9.0/bgdb-g1k.bin \
+ .dev/volumes/$STATIC/data/worker/grch37/strucvars/bgdbs/g1k.bin
+ln -sr .dev/volumes/$STATIC/data/download/worker/bgdb-gnomad-grch37-*/bgdb-gnomad.bin \
+ .dev/volumes/$STATIC/data/worker/grch37/strucvars/bgdbs/gnomad.bin
+ln -sr .dev/volumes/$STATIC/data/download/worker/bgdb-dbvar-grch37-*/bgdb-dbvar.bin \
+ .dev/volumes/$STATIC/data/worker/grch37/strucvars/bgdbs/dbvar.bin
+ln -sr .dev/volumes/$STATIC/data/download/worker/bgdb-dbvar-grch38-*/bgdb-dbvar.bin \
+ .dev/volumes/$STATIC/data/worker/grch38/strucvars/bgdbs/dbvar.bin
+ln -sr .dev/volumes/$STATIC/data/download/worker/bgdb-dgv-grch37-*/bgdb-dgv.bin \
+ .dev/volumes/$STATIC/data/worker/grch37/strucvars/bgdbs/dgv.bin
+ln -sr .dev/volumes/$STATIC/data/download/worker/bgdb-dgv-grch38-*/bgdb-dgv.bin \
+ .dev/volumes/$STATIC/data/worker/grch38/strucvars/bgdbs/dgv.bin
+ln -sr .dev/volumes/$STATIC/data/download/worker/bgdb-dgv-gs-grch37-*/bgdb-dgv-gs.bin \
+ .dev/volumes/$STATIC/data/worker/grch37/strucvars/bgdbs/dgv-gs.bin
+ln -sr .dev/volumes/$STATIC/data/download/worker/bgdb-dgv-gs-grch38-*/bgdb-dgv-gs.bin \
+ .dev/volumes/$STATIC/data/worker/grch38/strucvars/bgdbs/dgv-gs.bin
+
+rm -f .dev/volumes/$STATIC/data/worker/grch3?/strucvars/clinvar.bin
+
+ln -sr .dev/volumes/$STATIC/data/download/worker/clinvar-strucvars-grch37-*/clinvar-strucvars.bin \
+ .dev/volumes/$STATIC/data/worker/grch37/strucvars/clinvar.bin
+ln -sr .dev/volumes/$STATIC/data/download/worker/clinvar-strucvars-grch38-*/clinvar-strucvars.bin \
+ .dev/volumes/$STATIC/data/worker/grch38/strucvars/clinvar.bin
+
+rm -f .dev/volumes/$STATIC/data/worker/grch3?/strucvars/patho-mms.bin
+
+ln -sr .dev/volumes/$STATIC/data/download/worker/patho-mms-grch37-*/patho-mms.bed \
+ .dev/volumes/$STATIC/data/worker/grch37/strucvars/patho-mms.bed
+ln -sr .dev/volumes/$STATIC/data/download/worker/patho-mms-grch38-*/patho-mms.bed \
+ .dev/volumes/$STATIC/data/worker/grch38/strucvars/patho-mms.bed
+
+mkdir -p .dev/volumes/$STATIC/data/worker/grch3{7,8}/tads
+rm -f .dev/volumes/$STATIC/data/worker/grch3?/tads/hesc.bed
+
+ln -sr .dev/volumes/$STATIC/data/download/worker/tads-grch37-dixon2015/hesc.bed \
+ .dev/volumes/$STATIC/data/worker/grch37/tads/hesc.bed
+ln -sr .dev/volumes/$STATIC/data/download/worker/tads-grch38-dixon2015/hesc.bed \
+ .dev/volumes/$STATIC/data/worker/grch38/tads/hesc.bed
+
+rm -f .dev/volumes/$STATIC/data/worker/noref/genes/{xlink.bin,acmg.tsv,mime2gene.tsv}
+
+ln -sr .dev/volumes/$STATIC/data/download/worker/genes-xlink-*/genes-xlink.bin \
+ .dev/volumes/$STATIC/data/worker/noref/genes/xlink.bin
+ln -sr .dev/volumes/$STATIC/data/download/worker/acmg-sf-*/acmg_sf.tsv \
+ .dev/volumes/$STATIC/data/worker/noref/genes/acmg.tsv
+ln -sr .dev/volumes/$STATIC/data/download/worker/mim2gene-*/mim2gene.tsv \
+ .dev/volumes/$STATIC/data/worker/noref/genes/mime2gene.tsv
+
+mkdir -p .dev/volumes/$STATIC/data/worker/grch3{7,8}/genes
+rm -f .dev/volumes/$STATIC/data/worker/grch3?/genes/{ensembl_genes.bin,refseq_genes.bin}
+
+ln -sr .dev/volumes/$STATIC/data/download/worker/genes-regions-grch37-*/ensembl_genes.bin \
+ .dev/volumes/$STATIC/data/worker/grch37/genes/ensembl_regions.bin
+ln -sr .dev/volumes/$STATIC/data/download/worker/genes-regions-grch38-*/ensembl_genes.bin \
+ .dev/volumes/$STATIC/data/worker/grch38/genes/ensembl_regions.bin
+
+ln -sr .dev/volumes/$STATIC/data/download/worker/genes-regions-grch37-*/refseq_genes.bin \
+ .dev/volumes/$STATIC/data/worker/grch37/genes/refseq_regions.bin
+ln -sr .dev/volumes/$STATIC/data/download/worker/genes-regions-grch38-*/refseq_genes.bin \
+ .dev/volumes/$STATIC/data/worker/grch38/genes/refseq_regions.bin
+
+mkdir -p .dev/volumes/$STATIC/data/worker/grch3{7,8}/features
+rm -f .dev/volumes/$STATIC/data/worker/grch3?/features/{masked_repeat.bin,masked_seqdup.bin}
+
+ln -sr .dev/volumes/$STATIC/data/download/worker/masked-repeat-grch37-*/masked-repeat.bin \
+ .dev/volumes/$STATIC/data/worker/grch37/features/masked_repeat.bin
+ln -sr .dev/volumes/$STATIC/data/download/worker/masked-segdup-grch37-*/masked-segdup.bin \
+ .dev/volumes/$STATIC/data/worker/grch37/features/masked_seqdup.bin
+
+ln -sr .dev/volumes/$STATIC/data/download/worker/masked-repeat-grch38-*/masked-repeat.bin \
+ .dev/volumes/$STATIC/data/worker/grch38/features/masked_repeat.bin
+ln -sr .dev/volumes/$STATIC/data/download/worker/masked-segdup-grch38-*/masked-segdup.bin \
+ .dev/volumes/$STATIC/data/worker/grch38/features/masked_seqdup.bin
##
## tracks
##
-mkdir -p .dev/volumes/varfish-static/data/nginx/grch3{7,8}
+STATIC=varfish-static
+mkdir -p .dev/volumes/$STATIC/data/nginx/grch3{7,8}
+rm -f .dev/volumes/$STATIC/data/nginx/grch3?/*
-paths_37=$(find .dev/volumes/varfish-static/data/download/tracks/ -type f -name '*.bed' -or -name '*.bed.gz' | sort | grep grch37)
+paths_37=$(find .dev/volumes/$STATIC/data/download/tracks/ -type f -name '*.bed' -or -name '*.bed.gz' | sort | grep grch37)
for path in $paths_37; do
if [[ -e ${path}.tbi ]]; then
- ln -sr $path ${path}.tbi .dev/volumes/varfish-static/data/nginx/grch37
+ ln -sr $path ${path}.tbi .dev/volumes/$STATIC/data/nginx/grch37
else
- ln -sr $path .dev/volumes/varfish-static/data/nginx/grch37
+ ln -sr $path .dev/volumes/$STATIC/data/nginx/grch37
fi
done
-paths_38=$(find .dev/volumes/varfish-static/data/download/tracks/ -type f -name '*.bed' -or -name '*.bed.gz' | sort | grep grch38)
+paths_38=$(find .dev/volumes/$STATIC/data/download/tracks/ -type f -name '*.bed' -or -name '*.bed.gz' | sort | grep grch38)
for path in $paths_38; do
if [[ -e ${path}.tbi ]]; then
- ln -sr $path ${path}.tbi .dev/volumes/varfish-static/data/nginx/grch38
+ ln -sr $path ${path}.tbi .dev/volumes/$STATIC/data/nginx/grch38
else
- ln -sr $path .dev/volumes/varfish-static/data/nginx/grch38
+ ln -sr $path .dev/volumes/$STATIC/data/nginx/grch38
fi
done
```
+To download Mehari transcripts
+
+```bash session
+STATIC=varfish-static
+mkdir -p .dev/volumes/$STATIC/data/download/mehari-data-txs-grch3{7,8}
+
+for ext in .zst .zst.sha256 .zst.report .zst.report.sha256; do
+ wget -O .dev/volumes/$STATIC/data/download/mehari-data-txs-grch37/mehari-data-txs-grch37-0.4.1.bin$ext \
+ https://github.com/bihealth/mehari-data-tx/releases/download/v0.4.1/mehari-data-txs-grch37-0.4.1.bin$ext
+ wget -O .dev/volumes/$STATIC/data/download/mehari-data-txs-grch38/mehari-data-txs-grch38-0.4.1.bin$ext \
+ https://github.com/bihealth/mehari-data-tx/releases/download/v0.4.1/mehari-data-txs-grch37-0.4.1.bin$ext
+done
+
+rm -f .dev/volumes/$STATIC/data/mehari/grch3?/txs.bin.zst
+ln -sr .dev/volumes/$STATIC/data/download/mehari-data-txs-grch37/mehari-data-txs-grch37-0.4.1.bin.zst \
+ .dev/volumes/$STATIC/data/mehari/grch37/txs.bin.zst
+ln -sr .dev/volumes/$STATIC/data/download/mehari-data-txs-grch38/mehari-data-txs-grch38-0.4.1.bin.zst \
+ .dev/volumes/$STATIC/data/mehari/grch38/txs.bin.zst
+```
+
To create an in-house database:
```bash session
+STATIC=varfish-static
+
varfish-server-worker db mk-inhouse \
--path-output-tsv /tmp/inhouse.tsv \
$(find YOUR_PATH -name '*.gts.tsv.gz' | sort)
@@ -259,38 +328,40 @@ varfish-server-worker db mk-inhouse \
varfish-server-worker db to-bin \
--input-type strucvar-inhouse \
--path-input /tmp/inhouse.tsv \
- --path-output-bin .dev/volumes/varfish-static/data/worker/grch37/strucvars/inhouse.bin
+ --path-output-bin .dev/volumes/$STATIC/data/worker/grch37/strucvars/inhouse.bin
```
To obtain ClinVar
Note that this will install the data from November 12, 2023 and you might want to [look here for the latest release](https://github.com/bihealth/annonars-data-clinvar/releases).
```bash session
+STATIC=varfish-static
+
wget -O /tmp/annonars-clinvar-minimal-grch37-20231112+0.25.0.tar.gz \
https://github.com/bihealth/annonars-data-clinvar/releases/download/annonars-data-clinvar-20231112/annonars-clinvar-minimal-grch37-20231112+0.25.0.tar.gz
wget -O /tmp/annonars-clinvar-minimal-grch38-20231112+0.25.0.tar.gz \
https://github.com/bihealth/annonars-data-clinvar/releases/download/annonars-data-clinvar-20231112/annonars-clinvar-minimal-grch38-20231112+0.25.0.tar.gz
-tar -C .dev/volumes/varfish-static/data/download/annonars/ \
+tar -C .dev/volumes/$STATIC/data/download/annonars/ \
-xf /tmp/annonars-clinvar-minimal-grch37-20231112+0.25.0.tar.gz
-tar -C .dev/volumes/varfish-static/data/download/annonars \
+tar -C .dev/volumes/$STATIC/data/download/annonars \
-xf /tmp/annonars-clinvar-minimal-grch38-20231112+0.25.0.tar.gz
-rm -f .dev/volumes/varfish-static/data/annonars/grch37/clinvar
-ln -sr .dev/volumes/varfish-static/data/download/annonars/annonars-clinvar-minimal-grch37-20231112+0.25.0 \
- .dev/volumes/varfish-static/data/annonars/grch37/clinvar
-rm -f .dev/volumes/varfish-static/data/annonars/grch38/clinvar
-ln -sr .dev/volumes/varfish-static/data/download/annonars/annonars-clinvar-minimal-grch38-20231112+0.25.0 \
- .dev/volumes/varfish-static/data/annonars/grch38/clinvar
+rm -f .dev/volumes/$STATIC/data/annonars/grch37/clinvar
+ln -sr .dev/volumes/$STATIC/data/download/annonars/annonars-clinvar-minimal-grch37-20231112+0.25.0 \
+ .dev/volumes/$STATIC/data/annonars/grch37/clinvar
+rm -f .dev/volumes/$STATIC/data/annonars/grch38/clinvar
+ln -sr .dev/volumes/$STATIC/data/download/annonars/annonars-clinvar-minimal-grch38-20231112+0.25.0 \
+ .dev/volumes/$STATIC/data/annonars/grch38/clinvar
wget -O /tmp/annonars-clinvar-genes-20231112+0.25.0.tar.gz \
https://github.com/bihealth/annonars-data-clinvar/releases/download/annonars-data-clinvar-20231112/annonars-clinvar-genes-20231112+0.25.0.tar.gz
-tar -C .dev/volumes/varfish-static/data/download/annonars \
+tar -C .dev/volumes/$STATIC/data/download/annonars \
-xf /tmp/annonars-clinvar-genes-20231112+0.25.0.tar.gz
-rm -f .dev/volumes/varfish-static/data/annonars/clinvar-genes
-ln -sr .dev/volumes/varfish-static/data/download/annonars/annonars-clinvar-genes-20231112+0.25.0 \
- .dev/volumes/varfish-static/data/annonars/clinvar-genes
+rm -f .dev/volumes/$STATIC/data/annonars/clinvar-genes
+ln -sr .dev/volumes/$STATIC/data/download/annonars/annonars-clinvar-genes-20231112+0.25.0 \
+ .dev/volumes/$STATIC/data/annonars/clinvar-genes
```