Skip to content

Commit

Permalink
CDK: ref to csv.DictReader
Browse files Browse the repository at this point in the history
Signed-off-by: Artem Inzhyyants <artem.inzhyyants@gmail.com>
  • Loading branch information
artem1205 committed Dec 26, 2024
1 parent 46e80d3 commit 61a3919
Showing 1 changed file with 5 additions and 10 deletions.
15 changes: 5 additions & 10 deletions airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
import csv
import gzip
import json
import logging
from abc import ABC, abstractmethod
from dataclasses import dataclass
from io import BufferedIOBase
from io import BufferedIOBase, TextIOWrapper
from typing import Any, Generator, MutableMapping, Optional

import pandas as pd
import requests
from numpy import nan

from airbyte_cdk.sources.declarative.decoders.decoder import Decoder

Expand Down Expand Up @@ -71,13 +70,9 @@ def parse(
"""
Parse CSV data from decompressed bytes.
"""
reader = pd.read_csv( # type: ignore
data, sep=self.delimiter, iterator=True, dtype=object, encoding=self.encoding
)
for chunk in reader:
chunk = chunk.replace({nan: None}).to_dict(orient="records")
for row in chunk:
yield row
text_data = TextIOWrapper(data, encoding=self.encoding)
reader = csv.DictReader(text_data, delimiter=self.delimiter)
yield from reader


@dataclass
Expand Down

0 comments on commit 61a3919

Please sign in to comment.