Merge pull request #35 from RecandChat/fixed_bulk_data_retrival

Fixed bulk data retrival

monkelib/API/get_bulk_data.py

@@ -12,12 +12,12 @@
 def get_users(user_amount: int = 100) -> (list, bool):
     """
     This functions gets the users from the GitHub API based on the query parameters.
-    This are them having at least 1000 followers and 1000 repos.
+    These are them having at least 1000 followers and 1000 repos.
     :return: A list of users and a boolean indicating if the request was successful.
     """
-    url: str = 'https://api.github.com/users'
+    url: str = 'https://api.github.com/search/users'
     query_parameters: dict = {
-        'q': 'followers:>1000 repos:>1000',
+        'q': 'repos:>1000 followers:>1000',
         'per_page': user_amount,
     }
 
@@ -27,7 +27,7 @@ def get_users(user_amount: int = 100) -> (list, bool):
 
         users_data: list = []
 
-        for user in response.json():
+        for user in response.json()['items']:
             users_data.append(user['login'])
 
         return users_data, True
@@ -241,5 +241,5 @@ def get_bulk_data(user_amount: int = 100) -> bool:
 
     df = DataFrame(users_repos)
     df.drop_duplicates(subset='id', keep='first', inplace=True)
-    save_to_csv(df, 'bulkData.csv')
+    save_to_csv(df, 'original/bulkDataNew.csv')
     return True

monkelib/API/helper_functions.py

@@ -53,8 +53,6 @@ def get_repo_fields(repo: dict) -> dict:
         'is_disabled': repo['disabled'],
         'is_template': repo['is_template'],
         'license': repo['license']['name'] if repo['license'] else "No license",
-        'allows_forking': repo['allow_forking'],
-        'open_issues_count': repo['open_issues_count'],
         'open_issues': repo['open_issues'],
         'topics': repo['topics'],
     }

monkelib/Tests/test_functional.py

@@ -2,58 +2,120 @@
 from monkelib.API.get_bulk_data import *
 
 
-def test_get_users(sample_size):
+def test_get_users(sample_size) -> None:
+    """
+    This function tests the get_users function.
+    :param sample_size: How many users to get.
+    :return: None
+    """
+    users_list: list
+    users_flag: bool
     users_list, users_flag = get_users(sample_size)
-    assert users_flag
     assert len(users_list) == 1
+    assert users_flag
 
 
-def test_get_followers(sample_user):
+def test_get_followers(sample_user) -> None:
+    """
+    This function tests the get_followers function.
+    :param sample_user: A sample user (Lukasaurus11)
+    :return: None
+    """
+    followers_list: list
+    followers_flag: bool
     followers_list, followers_flag = get_followers(sample_user)
-    assert followers_flag
     assert len(followers_list) > 0
+    assert followers_flag
 
 
-def test_get_followers_fail(non_existent_user):
+def test_get_followers_fail(non_existent_user) -> None:
+    """
+    This function tests the get_followers function with a non-existent user.
+    :param non_existent_user: A non-existent user (ThisUserDoesNotExist1234567890)
+    :return: None
+    """
+    followers_list: list
+    followers_flag: bool
     followers_list, followers_flag = get_followers(non_existent_user)
-    assert followers_flag is False
     assert len(followers_list) == 0
+    assert followers_flag is False
 
 
-def test_get_following(sample_user):
+def test_get_following(sample_user) -> None:
+    """
+    This function tests the get_following function.
+    :param sample_user: A sample user (Lukasaurus11)
+    :return: None
+    """
+    following_list: list
+    following_flag: bool
     following_list, following_flag = get_following(sample_user)
-    assert following_flag
     assert len(following_list) > 0
+    assert following_flag
 
 
-def test_get_following_fail(non_existent_user):
+def test_get_following_fail(non_existent_user) -> None:
+    """
+    This function tests the get_following function with a non-existent user.
+    :param non_existent_user: A non-existent user (ThisUserDoesNotExist1234567890)
+    :return: None
+    """
+    following_list: list
+    following_flag: bool
     following_list, following_flag = get_following(non_existent_user)
-    assert following_flag is False
     assert len(following_list) == 0
+    assert following_flag is False
 
 
-def test_get_user_repos(sample_user):
+def test_get_user_repos(sample_user) -> None:
+    """
+    This function tests the get_user_repos function.
+    :param sample_user: A sample user (Lukasaurus11)
+    :return: None
+    """
+    user_repos: list
+    user_repos_flag: bool
     user_repos, user_repos_flag = get_user_repos(sample_user)
-    assert user_repos_flag
     assert len(user_repos) > 0
+    assert user_repos_flag
 
 
-def test_get_user_repos_fail(non_existent_user):
+def test_get_user_repos_fail(non_existent_user) -> None:
+    """
+    This function tests the get_user_repos function with a non-existent user.
+    :param non_existent_user: A non-existent user (ThisUserDoesNotExist1234567890)
+    :return: None
+    """
+    user_repos: list
+    user_repos_flag: bool
     user_repos, user_repos_flag = get_user_repos(non_existent_user)
-    assert user_repos_flag is False
     assert len(user_repos) == 0
+    assert user_repos_flag is False
 
 
-def test_get_misc_data():
-    misc_data_flag = get_misc_data(['language'])
+def test_get_misc_data() -> None:
+    """
+    This function tests the get_misc_data function, with an accepted field (language).
+    :return: None
+    """
+    misc_data_flag: bool = get_misc_data(['language'])
     assert misc_data_flag
 
 
-def test_get_misc_data_fail():
-    misc_data_flag = get_misc_data(['ThisFieldDoesNotExist'])
+def test_get_misc_data_fail() -> None:
+    """
+    This function tests the get_misc_data function, with a non-accepted field (ThisFieldDoesNotExist).
+    :return: None
+    """
+    misc_data_flag: bool = get_misc_data(['ThisFieldDoesNotExist'])
     assert misc_data_flag is False
 
 
-def test_get_bulk_data(sample_size):
-    bulk_data_flag = get_bulk_data(sample_size)
+def test_get_bulk_data(sample_size) -> None:
+    """
+    This function tests the get_bulk_data function.
+    :param sample_size: How many users to get.
+    :return: None
+    """
+    bulk_data_flag: bool = get_bulk_data(sample_size)
     assert bulk_data_flag