add xopen threading to db build scripts

db-scripts/annotate-swissprot.py

@@ -70,7 +70,7 @@ def is_taxon_child(child, LCA, taxonomy):
     conn.execute('PRAGMA threads = 2;')
     conn.commit()
     conn.row_factory = sqlite3.Row
-    with xopen(str(xml_path), mode="rb") as fh, alive_bar() as bar:
+    with xopen(str(xml_path), mode="rb", threads=2) as fh, alive_bar() as bar:
         ups_entries = []
         i = 0
         for event, elem in et.iterparse(fh, tag='{*}entry'):

db-scripts/extract-pfam.py

@@ -30,7 +30,7 @@
 
 
 non_families = 0
-with xopen(str(pfam_path)) as fh_pfam, family_path.open('w') as fh_family, non_family_path.open('w') as fh_non_family:
+with xopen(str(pfam_path), threads=2) as fh_pfam, family_path.open('w') as fh_family, non_family_path.open('w') as fh_non_family:
     entries = fh_pfam.read().split('//')
     for entry_text in entries:
         id = None

db-scripts/init-psc.py

@@ -75,7 +75,7 @@ def is_taxon_child(child, LCA, taxonomy):
 psc_seqs = 0
 psc_sorf_seqs = 0
 print('parse & store PSC information...')
-with sqlite3.connect(str(db_path), isolation_level='EXCLUSIVE') as conn, xopen(str(uniref90_path), mode='rb') as fh_xml, psc_path.open(mode='wt') as fh_fasta_psc, psc_sorf_path.open(mode='wt') as fh_fasta_psc_sorf, alive_bar() as bar:
+with sqlite3.connect(str(db_path), isolation_level='EXCLUSIVE') as conn, xopen(str(uniref90_path), mode='rb', threads=2) as fh_xml, psc_path.open(mode='wt') as fh_fasta_psc, psc_sorf_path.open(mode='wt') as fh_fasta_psc_sorf, alive_bar() as bar:
     conn.execute('PRAGMA page_size = 4096;')
     conn.execute('PRAGMA cache_size = 100000;')
     conn.execute('PRAGMA locking_mode = EXCLUSIVE;')
@@ -169,7 +169,7 @@ def is_taxon_child(child, LCA, taxonomy):
 
 print(f'UniParc ({len(uniref90_uniparc_ids)})...')
 log_psc.debug('lookup non-representative UniParc seed sequences: %s', len(uniref90_uniparc_ids))
-with xopen(str(uniparc_path), mode='rt') as fh_uniparc, psc_path.open(mode='at') as fh_fasta_psc, psc_sorf_path.open(mode='at') as fh_fasta_sorf, alive_bar() as bar:
+with xopen(str(uniparc_path), mode='rt', threads=2) as fh_uniparc, psc_path.open(mode='at') as fh_fasta_psc, psc_sorf_path.open(mode='at') as fh_fasta_sorf, alive_bar() as bar:
     for record in SeqIO.parse(fh_uniparc, 'fasta'):
         uniref90_id = uniref90_uniparc_ids.get(record.id, None)
         if(uniref90_id):

db-scripts/init-pscc.py

@@ -75,7 +75,7 @@ def is_taxon_child(child, LCA, taxonomy):
 pscc_total = 0
 uniref50_uniparc_ids = {}
 print('parse & store PSCC information...')
-with sqlite3.connect(str(db_path), isolation_level='EXCLUSIVE') as conn, xopen(str(uniref50_path), mode='rb') as fh_xml, pscc_path.open(mode='wt') as fh_fasta_pscc, pscc_sorf_path.open(mode='wt') as fh_fasta_pscc_sorf, alive_bar() as bar:
+with sqlite3.connect(str(db_path), isolation_level='EXCLUSIVE') as conn, xopen(str(uniref50_path), mode='rb', threads=2) as fh_xml, pscc_path.open(mode='wt') as fh_fasta_pscc, pscc_sorf_path.open(mode='wt') as fh_fasta_pscc_sorf, alive_bar() as bar:
     conn.execute('PRAGMA page_size = 4096;')
     conn.execute('PRAGMA cache_size = 100000;')
     conn.execute('PRAGMA locking_mode = EXCLUSIVE;')
@@ -169,7 +169,7 @@ def is_taxon_child(child, LCA, taxonomy):
 
 print(f'UniParc ({len(uniref50_uniparc_ids)})...')
 log_pscc.debug('lookup non-representative UniParc seed sequences: %s', len(uniref50_uniparc_ids))
-with xopen(str(uniparc_path), mode='rt') as fh_uniparc, pscc_path.open(mode='at') as fh_fasta_psc, pscc_sorf_path.open(mode='at') as fh_fasta_sorf, alive_bar() as bar:
+with xopen(str(uniparc_path), mode='rt', threads=2) as fh_uniparc, pscc_path.open(mode='at') as fh_fasta_psc, pscc_sorf_path.open(mode='at') as fh_fasta_sorf, alive_bar() as bar:
     for record in SeqIO.parse(fh_uniparc, 'fasta'):
         uniref50_id = uniref50_uniparc_ids.get(record.id, None)
         if(uniref50_id):

db-scripts/init-ups-ips.py

@@ -83,7 +83,7 @@ def is_taxon_child(child, LCA, taxonomy):
     db_updates = 0
     ups_seqs = 0
     ips_seqs = 0
-    with xopen(str(uniref100_path), mode='rb') as fh_xml, ips_path.open(mode='wt') as fh_fasta_ips, alive_bar() as bar:
+    with xopen(str(uniref100_path), mode='rb', threads=2) as fh_xml, ips_path.open(mode='wt') as fh_fasta_ips, alive_bar() as bar:
         for event, elem in et.iterparse(fh_xml, tag='{*}entry'):
             if('Fragment' not in elem.find('./{*}name').text):  # skip protein fragments
                 common_tax_id = elem.find('./{*}property[@type="common taxon ID"]')
@@ -170,7 +170,7 @@ def is_taxon_child(child, LCA, taxonomy):
     print(f'UniParc ({len(uniparc_to_uniref100)})...')
     log_ups.debug('lookup non-representative UniParc member sequences: %s', len(uniparc_to_uniref100))
     db_updates = 0
-    with xopen(str(uniparc_path), mode='rt') as fh_uniparc, alive_bar() as bar:
+    with xopen(str(uniparc_path), mode='rt', threads=2) as fh_uniparc, alive_bar() as bar:
         for record in SeqIO.parse(fh_uniparc, 'fasta'):
             uniparc_id = record.id
             uniref100_id = uniparc_to_uniref100.get(uniparc_id, None)