chunk_gen_from_uniq.py

import wikipediaapi
import logging
import pandas as pd
from tqdm import tqdm
# import wikipedia
import requests
import json
import sys
import base64

LINKS_CSV_FILE = "./data/pages/pages.csv"

WIKI_REQUEST = 'http://ru.wikipedia.org/w/api.php?action=query&prop=pageimages&format=json&piprop=original&titles='

# def get_wiki_image(search_term):
#     try:
#         result = wikipedia.search(search_term, results = 1)
#         wikipedia.set_lang('ru')
#         wkpage = wikipedia.WikipediaPage(title = result[0])
#         title = wkpage.title
#         response  = requests.get(WIKI_REQUEST+title)
#         json_data = json.loads(response.text)
#         img_link = list(json_data['query']['pages'].values())[0]['original']['source']
#         return img_link        
#     except:
#         return 0
    

def get_as_base64(url):
    if url == 0:
        return 0 
    return base64.b64encode(requests.get(url).content)

def get_module_logger(mod_name):
    """
    To use this, do logger = get_module_logger(__name__)
    """
    logger = logging.getLogger(mod_name)
    handler = logging.StreamHandler()
    formatter = logging.Formatter(
        '%(asctime)s [%(name)-12s] %(levelname)-8s %(message)s')
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    logger.setLevel(logging.DEBUG)
    return logger

def safe_clear(df): 
    for col in df:
        if col not in ['source_name','dest_name','dest_url','source_url']:
            df = df.drop(col,axis=1)
            print("dropped ", col)
    return df

def load(): 
    links_df = pd.read_csv(LINKS_CSV_FILE)
    return links_df


def write(ldf):
    ldf.to_csv(LINKS_CSV_FILE)
    
def main():
    ldf = load()
    wiki_wiki = wikipediaapi.Wikipedia('ru')

    data  = []
    counter = 0
    chunk_num = 1
    for page in tqdm(ldf["page_name"]):
        counter+=1
        data.append(page)
        if (counter % 11711) == 0:
            dfff = pd.DataFrame(data)
            dfff.to_csv(f'data/pages/pages_chunk_{chunk_num}.csv')
            data = []
            chunk_num+=1
        
        
    
    
        # print(page_py.categorymembers)
    
    

if __name__ == "__main__":
    get_module_logger(__name__).info("Started parsing script")
    main()