From 6363e03a2d4e8ce4c21102ba1814515fe672029d Mon Sep 17 00:00:00 2001 From: Ralph Rassweiler Date: Thu, 20 Jun 2024 10:46:15 -0300 Subject: [PATCH] fix: cassandra configs (#364) * fix: to lower case * pin numpy --- butterfree/configs/db/cassandra_config.py | 30 ++++++++++++----------- requirements.txt | 1 + 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/butterfree/configs/db/cassandra_config.py b/butterfree/configs/db/cassandra_config.py index d60bb697..d576359c 100644 --- a/butterfree/configs/db/cassandra_config.py +++ b/butterfree/configs/db/cassandra_config.py @@ -228,26 +228,28 @@ def translate(self, schema: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """ cassandra_mapping = { - "TimestampType": "timestamp", - "BinaryType": "boolean", - "BooleanType": "boolean", - "DateType": "timestamp", - "DecimalType": "decimal", - "DoubleType": "double", - "FloatType": "float", - "IntegerType": "int", - "LongType": "bigint", - "StringType": "text", - "ArrayType(LongType,true)": "frozen>", - "ArrayType(StringType,true)": "frozen>", - "ArrayType(FloatType,true)": "frozen>", + "timestamptype": "timestamp", + "binarytype": "boolean", + "booleantype": "boolean", + "datetype": "timestamp", + "decimaltype": "decimal", + "doubletype": "double", + "floattype": "float", + "integertype": "int", + "longtype": "bigint", + "stringtype": "text", + "arraytype(longtype,true)": "frozen>", + "arraytype(stringtype,true)": "frozen>", + "arraytype(floattype,true)": "frozen>", } cassandra_schema = [] for features in schema: cassandra_schema.append( { "column_name": features["column_name"], - "type": cassandra_mapping[str(features["type"]).replace("()", "")], + "type": cassandra_mapping[ + str(features["type"]).replace("()", "").lower() + ], "primary_key": features["primary_key"], } ) diff --git a/requirements.txt b/requirements.txt index 0af8a62a..f3968c60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ pyspark==3.5.1 typer==0.4.2 typing-extensions>3.7.4,<5 boto3==1.17.* +numpy==1.26.4