feat: enhance variant index partitioning (#834)

opentargets · Oct 12, 2024 · c4520aa · c4520aa
1 parent 9f446e8
commit c4520aa
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/src/gentropy/variant_index.py b/src/gentropy/variant_index.py
@@ -59,8 +59,9 @@ def __init__(
                     col("variantId"), col("chromosome"), col("position")
                 ),
             )
-            .write.partitionBy("chromosome")
-            .mode(session.write_mode)
+            .repartitionByRange("chromosome", "position")
+            .sortWithinPartitions("chromosome", "position")
+            .write.mode(session.write_mode)
             .parquet(variant_index_path)
         )