fix: adding data specific p-value filters

src/gentropy/config.py

@@ -122,10 +122,16 @@ class GWASCatalogSumstatsPreprocessConfig(StepConfig):
 class EqtlCatalogueConfig(StepConfig):
     """eQTL Catalogue step configuration."""
 
+    session: Any = field(
+        default_factory=lambda: {
+            "start_hail": True,
+        }
+    )
     eqtl_catalogue_paths_imported: str = MISSING
     eqtl_catalogue_study_index_out: str = MISSING
     eqtl_catalogue_credible_sets_out: str = MISSING
     mqtl_quantification_methods_blacklist: list[str] = field(default_factory=lambda: [])
+    eqtl_lead_pvalue_threshold: float = 1e-3
     _target_: str = "gentropy.eqtl_catalogue.EqtlCatalogueStep"
 
 
@@ -169,6 +175,7 @@ class FinngenFinemappingConfig(StepConfig):
     _target_: str = (
         "gentropy.finngen_finemapping_ingestion.FinnGenFinemappingIngestionStep"
     )
+    finngen_finemapping_lead_pvalue_threshold: float = 1e-5
 
 
 @dataclass

src/gentropy/eqtl_catalogue.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from gentropy.common.session import Session
+from gentropy.config import EqtlCatalogueConfig
 from gentropy.datasource.eqtl_catalogue.finemapping import EqtlCatalogueFinemapping
 from gentropy.datasource.eqtl_catalogue.study_index import EqtlCatalogueStudyIndex
 
@@ -58,13 +59,21 @@ def __init__(
         processed_susie_df = EqtlCatalogueFinemapping.parse_susie_results(
             credible_sets_df, lbf_df, studies_metadata
         )
-        credible_sets = EqtlCatalogueFinemapping.from_susie_results(processed_susie_df)
-        study_index = EqtlCatalogueStudyIndex.from_susie_results(processed_susie_df)
 
-        # Load
-        study_index.df.write.mode(session.write_mode).parquet(
-            eqtl_catalogue_study_index_out
+        (
+            EqtlCatalogueStudyIndex.from_susie_results(processed_susie_df)
+            # Writing the output:
+            .df.write.mode(session.write_mode)
+            .parquet(eqtl_catalogue_study_index_out)
         )
-        credible_sets.df.write.mode(session.write_mode).parquet(
-            eqtl_catalogue_credible_sets_out
+
+        (
+            EqtlCatalogueFinemapping.from_susie_results(processed_susie_df)
+            # Flagging sub-significnat loci:
+            .validate_lead_pvalue(
+                pvalue_cutoff=EqtlCatalogueConfig().eqtl_lead_pvalue_threshold
+            )
+            # Writing the output:
+            .df.write.mode(session.write_mode)
+            .parquet(eqtl_catalogue_credible_sets_out)
         )

src/gentropy/finngen_finemapping_ingestion.py

@@ -31,13 +31,17 @@ def __init__(
         """
         # Read finemapping outputs from the input paths.
 
-        finngen_finemapping_df = FinnGenFinemapping.from_finngen_susie_finemapping(
-            spark=session.spark,
-            finngen_susie_finemapping_snp_files=finngen_susie_finemapping_snp_files,
-            finngen_susie_finemapping_cs_summary_files=finngen_susie_finemapping_cs_summary_files,
-        )
-
-        # Write the output.
-        finngen_finemapping_df.df.write.mode(session.write_mode).parquet(
-            finngen_finemapping_out
+        (
+            FinnGenFinemapping.from_finngen_susie_finemapping(
+                spark=session.spark,
+                finngen_susie_finemapping_snp_files=finngen_susie_finemapping_snp_files,
+                finngen_susie_finemapping_cs_summary_files=finngen_susie_finemapping_cs_summary_files,
+            )
+            # Flagging sub-significnat loci:
+            .validate_lead_pvalue(
+                pvalue_cutoff=FinngenFinemappingConfig().finngen_finemapping_lead_pvalue_threshold
+            )
+            # Writing the output:
+            .df.write.mode(session.write_mode)
+            .parquet(finngen_finemapping_out)
         )

src/gentropy/pics.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from gentropy.common.session import Session
+from gentropy.config import WindowBasedClumpingStepConfig
 from gentropy.dataset.study_locus import CredibleInterval, StudyLocus
 from gentropy.method.pics import PICS
 
@@ -28,8 +29,14 @@ def __init__(
             session, study_locus_ld_annotated_in
         )
         # PICS
-        picsed_sl = PICS.finemap(study_locus_ld_annotated).filter_credible_set(
-            credible_interval=CredibleInterval.IS99
+        (
+            PICS.finemap(study_locus_ld_annotated)
+            .filter_credible_set(credible_interval=CredibleInterval.IS99)
+            # Flagging sub-significnat loci:
+            .validate_lead_pvalue(
+                pvalue_cutoff=WindowBasedClumpingStepConfig().gwas_significance
+            )
+            # Writing the output:
+            .df.write.mode(session.write_mode)
+            .parquet(picsed_study_locus_out)
         )
-        # Write
-        picsed_sl.df.write.mode(session.write_mode).parquet(picsed_study_locus_out)