diff --git a/dataset-processor/dataset_processor.py b/dataset-processor/dataset_processor.py index 113de3b6..090dff46 100644 --- a/dataset-processor/dataset_processor.py +++ b/dataset-processor/dataset_processor.py @@ -2,7 +2,7 @@ import os import json import requests -from data_enrichment.data_enrichment import DataEnrichment +# from data_enrichment.data_enrichment import DataEnrichment from constants import * from s3_mock import S3FileCounter @@ -19,7 +19,7 @@ class DatasetProcessor: def __init__(self): - self.data_enricher = DataEnrichment() + # self.data_enricher = DataEnrichment() self.s3_file_counter = S3FileCounter() def check_and_convert(self, data): @@ -83,8 +83,8 @@ def enrich_data(self, data, selected_fields, record_count): enriched_entry = {} for key, value in entry.items(): if isinstance(value, str) and (key in selected_fields): - enriched_value = self.data_enricher.enrich_data(value, num_return_sequences=1, language_id='en') - # enriched_value = ["enrichupdate"] + # enriched_value = self.data_enricher.enrich_data(value, num_return_sequences=1, language_id='en') + enriched_value = ["enrichupdate"] enriched_entry[key] = enriched_value[0] if enriched_value else value else: enriched_entry[key] = value