From c451ce61874b8b00ea72327a9e88beb0014f70f8 Mon Sep 17 00:00:00 2001 From: Perdana Hadi Date: Thu, 22 Aug 2024 21:11:10 +0700 Subject: [PATCH] revised uploader --- .github/workflows/scrape.yml | 2 +- upload_to_sheets.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scrape.yml b/.github/workflows/scrape.yml index a53db16..ddf6e7e 100644 --- a/.github/workflows/scrape.yml +++ b/.github/workflows/scrape.yml @@ -23,7 +23,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install -r requirements.txt pandas google-auth google-auth-oauthlib google-auth-httplib2 google-api-python-client - name: Install Playwright browsers run: playwright install --with-deps chromium firefox webkit diff --git a/upload_to_sheets.py b/upload_to_sheets.py index 0222775..fa3bcfa 100644 --- a/upload_to_sheets.py +++ b/upload_to_sheets.py @@ -40,6 +40,10 @@ def setup_credentials(): def read_csv(file_path): try: df = pd.read_csv(file_path) + # Replace NaN values with empty strings + df = df.fillna('') + # Convert all values to strings and strip whitespace + df = df.applymap(lambda x: str(x).strip() if isinstance(x, str) else x) return [df.columns.tolist()] + df.values.tolist() except FileNotFoundError: print(f"Error: CSV file not found at {file_path}") @@ -52,6 +56,9 @@ def validate_data(data): # Add more validation as needed return True +def clean_data(data): + return [[str(cell).replace('\n', ' ').strip() for cell in row] for row in data] + @contextmanager def get_sheets_service(creds): service = build("sheets", "v4", credentials=creds) @@ -146,8 +153,10 @@ def main(): if not validate_data(csv_content): sys.exit(1) + cleaned_content = clean_data(csv_content) + with get_sheets_service(creds) as service: - upload_to_sheets(service, spreadsheet_id, csv_content) + upload_to_sheets(service, spreadsheet_id, cleaned_content) if __name__ == "__main__": main()