diff --git a/.github/workflows/flake8_test_coverage.yml b/.github/workflows/flake8_test_coverage.yml index 3823ca4..8b3cc85 100644 --- a/.github/workflows/flake8_test_coverage.yml +++ b/.github/workflows/flake8_test_coverage.yml @@ -22,10 +22,7 @@ jobs: run: poetry install - name: Lint with flake8 run: | - # stop the build if there are Python syntax errors or undefined names - poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings - poetry run flake8 . --count --exit-zero --max-line-length=119 --statistics + poetry run flake8 . --count --max-line-length=119 --statistics - name: Coverage with pytest run: | poetry run coverage run --omit "tests/*.py" -m pytest ./tests/ --junitxml=./report.xml -s diff --git a/badges/tests.svg b/badges/tests.svg index 9b71f92..ae13d85 100644 --- a/badges/tests.svg +++ b/badges/tests.svg @@ -1 +1 @@ -tests: 48tests48 \ No newline at end of file +tests: 55tests55 \ No newline at end of file diff --git a/paradox_localization_utils/update_paratranz.py b/paradox_localization_utils/update_paratranz.py index 1ba442b..e5bc095 100644 --- a/paradox_localization_utils/update_paratranz.py +++ b/paradox_localization_utils/update_paratranz.py @@ -19,7 +19,9 @@ def get_args(): return parser.parse_args() -def create_or_update_files(project_id: int, token: str, loc_dir: str, language: str, parallel_nb: int): +def create_or_update_files( + project_id: int, token: str, loc_dir: str, language: str, parallel_nb: int +) -> dict[str, int]: __assert_localisation_directory_format(loc_dir, language) start = time.time() try: @@ -28,11 +30,12 @@ def create_or_update_files(project_id: int, token: str, loc_dir: str, language: print("WARNING: Fail to get the list of files from Paratranz") print("Files can be created but not updated") current_files = dict() - print(f"Update_paratranz on {os.path.join(loc_dir, language)}") + print(f"Update_paratranz on {loc_dir}") all_files = [] - for root, _, files in os.walk(os.path.join(loc_dir, language)): + for root, _, files in os.walk(loc_dir): for file in files: - all_files.append(os.path.join(root, file)) + if file.endswith(f"{language}.yml"): + all_files.append(os.path.join(root, file)) files_with_errors = [] Parallel(n_jobs=parallel_nb, backend="threading")( delayed(create_or_update_file)( @@ -46,6 +49,7 @@ def create_or_update_files(project_id: int, token: str, loc_dir: str, language: for file in files_with_errors: print(file) print(f"Total time of the execution: {compute_time(start)}") + return current_files def __assert_localisation_directory_format(loc_dir: str, language: str): @@ -71,7 +75,14 @@ def create_or_update_file( files_with_errors: list, sleeping_before_retry: int = 2, ): - file_relative_path = file_path.replace(f"{loc_dir}\\{language}\\", "").replace(f"{loc_dir}/{language}/", "") + file_relative_path = ( + file_path.replace(f"{loc_dir}\\", "") + .replace(f"{loc_dir}/", "") + .replace(f"{language}\\", "") + .replace(f"{language}/", "") + .replace(f"replace\\{language}\\", "replace\\") + .replace(f"replace/{language}/", "replace/") + ) paratranz_path = os.path.dirname(file_relative_path) if file_path.endswith(f"{language}.yml"): try: @@ -83,7 +94,7 @@ def create_or_update_file( file_path, paratranz_path, sleeping_before_retry, - current_files[file_relative_path.replace("\\", "/")], + current_files.pop(file_relative_path.replace("\\", "/")), ) else: print(f"Create file {file_relative_path}") @@ -94,6 +105,47 @@ def create_or_update_file( files_with_errors.append(file_relative_path) +def delete_files_if_wanted( + token: str, + project_id: int, + loc_dir: str, + parallel_nb: int, + files_to_delete: dict[str, int], + sleeping_before_retry: int = 2, +): + if len(files_to_delete) > 0: + print(f"{len(files_to_delete)} files are in Paratranz but not in {loc_dir}:") + for file in files_to_delete: + print(file) + will_delete_files = input("Do you want to delete these files from Paratranz? [y/N]") + if will_delete_files == "y": + Parallel(n_jobs=parallel_nb, backend="threading")( + delayed(__delete_file_with_retry)(token, project_id, file_name, file_id, sleeping_before_retry) + for file_name, file_id in files_to_delete.items() + ) + else: + print("Files NOT deleted") + + +def __delete_file_with_retry(token: str, project_id: int, file_name: str, file_id: int, sleeping_before_retry: int): + headers = {"Authorization": token} + url = f"https://paratranz.cn/api/projects/{project_id}/files/{file_id}" + try: + __delete_file(url, headers) + except requests.HTTPError: + print(f"Fail to delete {file_name}, retry in {sleeping_before_retry} seconds") + time.sleep(sleeping_before_retry) + try: + __delete_file(url, headers) + except requests.HTTPError: + pass + + +def __delete_file(url: str, headers: dict): + r = requests.delete(url, headers=headers) + manage_request_error(r) + + def __post_file_to_paratranz_with_retry( token: str, project_id: int, @@ -125,6 +177,9 @@ def __post_file_to_paratranz(url: str, headers: dict, filepath: str, paratranz_p if __name__ == "__main__": - print("Version of the software: 1.1 (27th September 2024)") + print("Version of the software: 1.1 (28th September 2024)") args = get_args() - create_or_update_files(args.project_id, args.token, args.loc_dir, args.language, args.parallel_nb) + files_to_delete = create_or_update_files( + args.project_id, args.token, args.loc_dir, args.language, args.parallel_nb + ) + delete_files_if_wanted(args.token, args.project_id, args.loc_dir, args.parallel_nb, files_to_delete) diff --git a/tests/test_update_paratranz.py b/tests/test_update_paratranz.py index 93c6b12..82c9cb3 100644 --- a/tests/test_update_paratranz.py +++ b/tests/test_update_paratranz.py @@ -8,6 +8,7 @@ from paradox_localization_utils.update_paratranz import ( create_or_update_file, create_or_update_files, + delete_files_if_wanted, get_project_files, ) from tests.utils import generate_random_str @@ -46,6 +47,24 @@ def empty_file_with_subdir(language: str, tmp_path: Path): yield file +@pytest.fixture(scope="function") +def empty_file_in_replace_with_language_subdir(language: str, tmp_path: Path): + file_name = f"file_{language}.yml" + file = tmp_path / "replace" / language / file_name + file.parent.mkdir(parents=True) + file.touch() + yield file + + +@pytest.fixture(scope="function") +def empty_file_in_replace_in_root(language: str, tmp_path: Path): + file_name = f"file_{language}.yml" + file = tmp_path / "replace" / file_name + file.parent.mkdir(parents=True) + file.touch() + yield file + + @responses.activate def test_get_project_files(project_id: int): responses.get( @@ -132,6 +151,100 @@ def test_update_file_with_subdir( assert files_with_errors == [] +@responses.activate +def test_create_file_in_replace_with_language_subdir( + project_id: int, token: str, language: str, empty_file_in_replace_with_language_subdir: Path, tmp_path: Path +): + current_files = dict() + files_with_errors = [] + mock = responses.post(f"https://paratranz.cn/api/projects/{project_id}/files") + create_or_update_file( + token, + project_id, + tmp_path, + language, + str(empty_file_in_replace_with_language_subdir), + current_files, + files_with_errors, + ) + assert mock.call_count == 1 + assert "replace" in mock.calls[0].request.body.decode() + assert f"\\{language}" not in mock.calls[0].request.body.decode() + assert f"/{language}" not in mock.calls[0].request.body.decode() + assert files_with_errors == [] + + +@responses.activate +def test_update_file_in_replace_with_language_subdir( + project_id: int, token: str, language: str, empty_file_in_replace_with_language_subdir: Path, tmp_path: Path +): + file_id = 24 + current_files = { + f"replace/{empty_file_in_replace_with_language_subdir.name}": file_id + } + files_with_errors = [] + mock = responses.post(f"https://paratranz.cn/api/projects/{project_id}/files/{file_id}") + create_or_update_file( + token, + project_id, + tmp_path, + language, + str(empty_file_in_replace_with_language_subdir), + current_files, + files_with_errors, + ) + assert mock.call_count == 1 + assert "replace" in mock.calls[0].request.body.decode() + assert f"\\{language}" not in mock.calls[0].request.body.decode() + assert f"/{language}" not in mock.calls[0].request.body.decode() + assert files_with_errors == [] + + +@responses.activate +def test_create_file_in_replace_in_root( + project_id: int, token: str, language: str, empty_file_in_replace_in_root: Path, tmp_path: Path +): + current_files = dict() + files_with_errors = [] + mock = responses.post(f"https://paratranz.cn/api/projects/{project_id}/files") + create_or_update_file( + token, + project_id, + tmp_path, + language, + str(empty_file_in_replace_in_root), + current_files, + files_with_errors, + ) + assert mock.call_count == 1 + assert "replace" in mock.calls[0].request.body.decode() + assert files_with_errors == [] + + +@responses.activate +def test_update_file_in_replace_in_root( + project_id: int, token: str, language: str, empty_file_in_replace_in_root: Path, tmp_path: Path +): + file_id = 24 + current_files = { + f"replace/{empty_file_in_replace_in_root.name}": file_id + } + files_with_errors = [] + mock = responses.post(f"https://paratranz.cn/api/projects/{project_id}/files/{file_id}") + create_or_update_file( + token, + project_id, + tmp_path, + language, + str(empty_file_in_replace_in_root), + current_files, + files_with_errors, + ) + assert mock.call_count == 1 + assert "replace" in mock.calls[0].request.body.decode() + assert files_with_errors == [] + + first_call = True @@ -152,42 +265,76 @@ def raise_error_first_call(url: str, headers: dict, filepath: str, paratranz_pat @responses.activate def test_create_or_update_files(project_id: int, token: str, language: str, tmp_path: Path): + # Files in root file_name1 = f"file1_{language}.yml" file1 = tmp_path / language / file_name1 file_name2 = f"file2_{language}.yml" file2 = tmp_path / language / file_name2 + + # FIles in subdir file_name3 = f"file3_{language}.yml" subdir = "subdir" file3 = tmp_path / language / subdir / file_name3 file_name4 = f"file4_{language}.yml" file4 = tmp_path / language / subdir / file_name4 + + # Files without f"{language}.yml" file_name5 = "file5.yml" file5 = tmp_path / "other_language" / file_name5 - file_name6 = "file5.yml" + file_name6 = "file6.yml" file6 = tmp_path / language / file_name6 - for file in [file1, file2, file3, file4, file5, file6]: + + # Files in replace/language/ + file_name7 = f"file7_{language}.yml" + file7 = tmp_path / "replace" / language / file_name7 + file_name9 = f"file9_{language}.yml" + file9 = tmp_path / "replace" / language / file_name9 + + # Files in replace/ + file_name8 = f"file8_{language}.yml" + file8 = tmp_path / "replace" / file_name8 + file_name10 = f"file10_{language}.yml" + file10 = tmp_path / "replace" / file_name10 + + for file in [file1, file2, file3, file4, file5, file6, file7, file8, file9, file10]: file.parent.mkdir(parents=True, exist_ok=True) file.touch() - file_id = 24 + file4_id = 24 + file9_id = 25 + file10_id = 26 responses.get( f"https://paratranz.cn/api/projects/{project_id}/files", - json=[{"id": file_id, "name": f"{subdir}/{file_name4}"}], + json=[ + {"id": file4_id, "name": f"{subdir}/{file_name4}"}, + {"id": file9_id, "name": f"replace/{file_name9}"}, + {"id": file10_id, "name": f"replace/{file_name10}"}, + ], ) create_mock = responses.post(f"https://paratranz.cn/api/projects/{project_id}/files") - update_mock = responses.post(f"https://paratranz.cn/api/projects/{project_id}/files/{file_id}") - create_or_update_files(project_id, token, tmp_path, language, 1) - assert create_mock.call_count == 3 + update_mock_4 = responses.post(f"https://paratranz.cn/api/projects/{project_id}/files/{file4_id}") + update_mock_9 = responses.post(f"https://paratranz.cn/api/projects/{project_id}/files/{file9_id}") + update_mock_10 = responses.post(f"https://paratranz.cn/api/projects/{project_id}/files/{file10_id}") + res = create_or_update_files(project_id, token, tmp_path, language, 1) + assert create_mock.call_count == 5 for call in create_mock.calls: found = False - for file_name in [file_name1, file_name2, file_name3]: + for file_name in [file_name1, file_name2, file_name3, file_name7, file_name8]: if file_name in call.request.body.decode(): + if file_name == file_name3: + assert subdir in call.request.body.decode() found = True break assert found - assert subdir in create_mock.calls[-1].request.body.decode() - assert update_mock.call_count == 1 - assert file_name4 in update_mock.calls[0].request.body.decode() - assert subdir in update_mock.calls[0].request.body.decode() + assert update_mock_4.call_count == 1 + assert file_name4 in update_mock_4.calls[0].request.body.decode() + assert subdir in update_mock_4.calls[0].request.body.decode() + assert update_mock_9.call_count == 1 + assert file_name9 in update_mock_9.calls[0].request.body.decode() + assert "replace" in update_mock_9.calls[0].request.body.decode() + assert update_mock_10.call_count == 1 + assert file_name10 in update_mock_10.calls[0].request.body.decode() + assert "replace" in update_mock_10.calls[0].request.body.decode() + assert res == dict() @responses.activate @@ -196,13 +343,14 @@ def test_create_or_update_files_get_paratranz_files_error( ): responses.get(f"https://paratranz.cn/api/projects/{project_id}/files", status=500) create_mock = responses.post(f"https://paratranz.cn/api/projects/{project_id}/files") - create_or_update_files(project_id, token, tmp_path, language, 1) + res = create_or_update_files(project_id, token, tmp_path, language, 1) assert create_mock.call_count == 1 assert empty_file.name in create_mock.calls[0].request.body.decode() captured = capsys.readouterr() logs = captured.out.split("\n") assert "WARNING: Fail to get the list of files from Paratranz" in logs assert "Files can be created but not updated" in logs + assert res == dict() @responses.activate @@ -211,7 +359,7 @@ def test_create_or_update_files_create_error( ): responses.get(f"https://paratranz.cn/api/projects/{project_id}/files", json=[]) mock = responses.post(f"https://paratranz.cn/api/projects/{project_id}/files", status=500) - create_or_update_files(project_id, token, tmp_path, language, 1) + res = create_or_update_files(project_id, token, tmp_path, language, 1) assert mock.call_count == 2 # Retry for i in range(mock.call_count): assert empty_file.name in mock.calls[i].request.body.decode() @@ -222,6 +370,7 @@ def test_create_or_update_files_create_error( if logs[i] == "ERROR: Non updated files:": break assert empty_file.name in logs[i:] + assert res == dict() def test_create_or_update_files_non_existing_dir(project_id: int, token: str, language: str): @@ -237,3 +386,32 @@ def test_create_or_update_files_non_existing_language_dir(project_id: int, token responses.post(f"https://paratranz.cn/api/projects/{project_id}/files") with pytest.raises(ValueError, match=re.escape(f"Directory {tmp_path/language} does not exist")): create_or_update_files(project_id, token, tmp_path, language, 1) + + +@responses.activate +def test_delete_files(project_id: int, token: str, mocker: MockerFixture): + mocker.patch("paradox_localization_utils.update_paratranz.input", return_value="y") + file_name = generate_random_str() + file_id = 24 + loc_dir = generate_random_str() + mock = responses.delete(f"https://paratranz.cn/api/projects/{project_id}/files/{file_id}") + delete_files_if_wanted(token, project_id, loc_dir, 1, {file_name: file_id}, 0) + assert mock.call_count == 1 + + +@responses.activate +def test_delete_files_no_files(project_id: int, token: str, mocker: MockerFixture): + mocker.patch("paradox_localization_utils.update_paratranz.input", return_value="y") + loc_dir = generate_random_str() + delete_files_if_wanted(token, project_id, loc_dir, 1, dict(), 0) + + +@responses.activate +def test_delete_files_user_refuses(project_id: int, token: str, mocker: MockerFixture): + mocker.patch("paradox_localization_utils.update_paratranz.input", return_value="n") + file_name = generate_random_str() + file_id = 24 + loc_dir = generate_random_str() + mock = responses.delete(f"https://paratranz.cn/api/projects/{project_id}/files/{file_id}") + delete_files_if_wanted(token, project_id, loc_dir, 1, {file_name: file_id}, 0) + assert mock.call_count == 0 diff --git a/tests/utils.py b/tests/utils.py index e5518ab..ab66dec 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -15,6 +15,7 @@ def get_data_dir() -> str: else: raise DataDirNotFoundException() + def generate_random_str(length: int = 10) -> str: - letters = string.ascii_lowercase - return ''.join(random.choice(letters) for _ in range(length)) + letters = string.ascii_lowercase + return "".join(random.choice(letters) for _ in range(length))