From ccaf7b0b316a23363952b9e81b07ddd4bd777e46 Mon Sep 17 00:00:00 2001 From: Dibyam Kumar Date: Sat, 5 Aug 2023 21:04:50 +0530 Subject: [PATCH 1/3] Update fetch_tutorials script to use Github API --- .../utils/tutorials/fetch_tutorials.py | 85 +- .../tutorials/mocks/github-response-mock.html | 1942 ----------------- .../tutorials/mocks/github-response-mock.json | 98 + .../utils/tutorials/test_fetch_tutorials.py | 4 +- 4 files changed, 152 insertions(+), 1977 deletions(-) delete mode 100644 new-website/utils/tutorials/mocks/github-response-mock.html create mode 100644 new-website/utils/tutorials/mocks/github-response-mock.json diff --git a/new-website/utils/tutorials/fetch_tutorials.py b/new-website/utils/tutorials/fetch_tutorials.py index 62f02373..716a7a1c 100644 --- a/new-website/utils/tutorials/fetch_tutorials.py +++ b/new-website/utils/tutorials/fetch_tutorials.py @@ -12,64 +12,74 @@ import os import shutil import subprocess -import re import requests -from bs4 import BeautifulSoup from utils import clean +DEEPCHEM_REPO_OWNER = "deepchem" +DEEPCHEM_REPO_NAME = "deepchem" +TUTORIALS_PATH = "examples/tutorials" +TUTORIAL_RENDER_ORDER_PATH = "examples/tutorials/website-render-order" -def fetch_file_list_from_repo(path_to_directory): + +def fetch_file_list_from_repo(repo_owner, repo_name, path): """ - Fetches the names of all the files from a given directory in a Github repository. + Fetches the names of all the files in the given path from the given Github repository. Parameters ---------- - path_to_directory: str - The URL of the directory in the Github repository. + repo_owner: str + The owner of the Github repository. + repo_name: str + The name of the Github repository. + path: str + The path in the Github repository from which the file names are to be fetched. Returns ------- - files: list - A list of strings, where each string represents the name of a file present in the directory. - """ - files = [] - response = requests.get(path_to_directory) - soup = BeautifulSoup(response.content, 'html.parser') - - fileNames = soup.find_all( - 'a', attrs={'class': 'js-navigation-open Link--primary'}) - for fileName in fileNames: - fileName = fileName.text - files.append(fileName) - - return files + file_names: list + A list of strings, where each string represents the name of a file. - -def fetch_tutorial_data(): + Raises + ------ + Exception + If the response status code is not 200. """ - Fetches the names of all the tutorials from the given Github URL. - + url = "https://api.github.com/repos/{}/{}/contents/{}".format( + repo_owner, repo_name, path) + response = requests.get(url) + if response.status_code == 200: + data = response.json() + return [file["name"] for file in data] + else: + raise Exception( + "Error fetching file names: {}".format(response.status_code)) + + +def get_tutorial_list(): + """ + Fetches the names of all the tutorials from the Deepchem Github repository. Returns ------- tutorials: list A list of strings, where each string represents the name of a tutorial. """ - tutorials_url = 'https://github.com/deepchem/deepchem/tree/master/examples/tutorials' - tutorials = fetch_file_list_from_repo(tutorials_url) + tutorials = fetch_file_list_from_repo( + DEEPCHEM_REPO_OWNER, DEEPCHEM_REPO_NAME, TUTORIALS_PATH) # Filter only the ipynb files tutorials = [ tutorial for tutorial in tutorials if tutorial.endswith('.ipynb')] return tutorials + def fetch_tutorial_render_order(): """ Downloads the CSV files containing the tutorial order from the Deepchem repository. """ raw_path = 'https://raw.githubusercontent.com/deepchem/deepchem/master/examples/tutorials/website-render-order/' - csv_directory = 'https://github.com/deepchem/deepchem/tree/master/examples/tutorials/website-render-order' - tutorial_order = fetch_file_list_from_repo(csv_directory) + tutorial_order = fetch_file_list_from_repo( + DEEPCHEM_REPO_OWNER, DEEPCHEM_REPO_NAME, TUTORIAL_RENDER_ORDER_PATH) # Filter only the csv files tutorial_order = [ @@ -77,18 +87,23 @@ def fetch_tutorial_render_order(): for tutorial_group in tutorial_order: response = requests.get(raw_path + tutorial_group) - with open(f"./website-render-order/{tutorial_group}", "wb") as tutorial_file: - tutorial_file.write(response.content) + if response.status_code == 200: + with open(f"./website-render-order/{tutorial_group}", "wb") as tutorial_file: + tutorial_file.write(response.content) + else: + raise Exception( + "Error fetching tutorial render order: {}".format(response.status_code)) def create_directories(): """ Creates the required directories """ - os.makedirs('./html-notebooks',exist_ok=True) + os.makedirs('./html-notebooks', exist_ok=True) os.makedirs('./ipynb-notebooks', exist_ok=True) os.makedirs('./website-render-order', exist_ok=True) + def convert_to_html(tutorials): """ Converts the Jupyter notebooks in the './ipynb-notebooks' directory to HTML files and stores them in the './html-notebooks' directory. @@ -125,7 +140,11 @@ def convert_to_html(tutorials): if __name__ == "__main__": - tutorials = fetch_tutorial_data() create_directories() - convert_to_html(tutorials) + tutorials = get_tutorial_list() + +# The script throws an AssertionError if no tutorials are fetched. This is to prevent website deployment if no tutorials are fetched. + assert len(tutorials) > 0 + fetch_tutorial_render_order() + convert_to_html(tutorials) diff --git a/new-website/utils/tutorials/mocks/github-response-mock.html b/new-website/utils/tutorials/mocks/github-response-mock.html deleted file mode 100644 index 0793e109..00000000 --- a/new-website/utils/tutorials/mocks/github-response-mock.html +++ /dev/null @@ -1,1942 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - deepchem/examples/tutorials at master · deepchem/deepchem · GitHub - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - -
- Skip to content - - - - - - - - -
- - -
-
-
- - - - - - -
- - -
-
- - -
- -
-
- - - master - - - - -
-
-
- Switch branches/tags - -
- - - -
- -
- -
- - -
- -
- - - - - - - - - - - - - - - - -
- - - - -
-
- -
- -
- - -
- -
-
-
-

Name already in use

-
-
- -
-
-
-
- -
- A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch? -
- -
-
- - -
- -
- - -
-
- - - - deepchem - - - - / - - - examples - - - / - tutorials - / -
-
- -
- - Go to file - -
-
- -
- - - - deepchem - - - - / - - - examples - - - / - tutorials - / -
- - -
-
-

Latest commit

-
- -
- -
-
- - @paupaiz - -
-
- -
-
-
- paupaiz - - - - adding my bibtex - -
- - - -
- - - 6248fee - - - Aug 12, 2022 - -
-
-
-
- adding my bibtex -
-
- 6248fee -
-
-
-

Git stats

- -
-
-
-

Files

- - - Permalink - -
- - Failed to load latest commit information. - - -
-
-
-
-
Type
-
Name
-
Latest commit message
-
Commit time
-
-
-
- - . . - -
-
-
-
- -
-
- -
- -
- - assets - -
- -
- - Add Trident Chemwidgets tutorial - -
- -
- July 14, 2022 16:33 -
- -
-
-
- -
- -
- - About_nODE_Using_Torchdiffeq_in_Deepchem.ipynb - -
- -
- - minor fixes - -
- -
- February 25, 2022 16:43 -
- -
-
-
- -
- -
- - Advanced_Model_Training.ipynb - -
- -
- - bibtex tutorial 9 ( - #2926 - ) - -
- -
- July 26, 2022 09:03 -
- -
-
-
- -
- -
- - Advanced_model_training_using_hyperopt.ipynb - -
- -
- - adding hyperopt tutorials - -
- -
- February 11, 2022 19:30 -
- -
-
-
- -
- -
- - An_Introduction_To_MoleculeNet.ipynb - -
- -
- - updated installation procedure for colab [skip ci] - -
- -
- October 31, 2021 22:00 -
- -
-
-
- -
- -
- - Atomic_Contributions_for_Molecules.ipynb - -
- -
- - Delete the checkpoint files - -
- -
- December 29, 2021 11:28 -
- -
-
-
- -
- -
- - Conditional_Generative_Adversarial_Networks.ipynb - -
- -
- - minor fix to tutorial - -
- -
- November 6, 2021 20:41 -
- -
- -
-
- - -
- - - - -
- -
-
-
- - - - - - -
- -
- -
- -
- DeepChem Step-by-Step Tutorial - Why do the DeepChem Tutorial? - Introduction to DeepChem - Molecular Machine Learning - Modeling Proteins - Protein Ligand Modeling - Quantum Chemistry - Bioinformatics - Material Science - Machine Learning Methods - Deep Differential Equations -
-
- -
- -

- README.md -

-
-
- -
-
-

- DeepChem Step-by-Step Tutorial

-

In this tutorial series, you'll learn how to use DeepChem to solve interesting - and challenging problems in the life sciences. The tutorial acts as a introduction - to DeepChem as well as application of DeepChem to a wide array of problems - across domains like molecular machine learning, - quantum chemistry, bioinformatics and material science. This tutorial series is - continually updated with new DeepChem features and models as implemented and is - designed to be accessible to beginners.

-

- Why do the DeepChem Tutorial?

-

- 1) Career Advancement: - Applying AI in the life sciences is a booming - industry at present. There are a host of newly funded startups and initiatives - at large pharmaceutical and biotech companies centered around AI. Learning and - mastering DeepChem will bring you to the forefront of this field and will - prepare you to enter a career in this field.

-

- 2) Humanitarian Considerations: - Disease is the oldest cause of human - suffering. From the dawn of human civilization, humans have suffered from pathogens, - cancers, and neurological conditions. One of the greatest achievements of - the last few centuries has been the development of effective treatments for - many diseases. By mastering the skills in this tutorial, you will be able to - stand on the shoulders of the giants of the past to help develop new - medicine.

-

- 3) Lowering the Cost of Medicine: - The art of developing new medicine is - currently an elite skill that can only be practiced by a small core of expert - practitioners. By enabling the growth of open source tools for drug discovery, - you can help democratize these skills and open up drug discovery to more - competition. Increased competition can help drive down the cost of medicine.

-

The tutorial is organized as follows:

-

- Introduction to DeepChem

-

The following tutorials covers the core features of DeepChem. Covering these - tutorials will help you in getting started with DeepChem for machine learning. The later - tutorials discuss about using DeepChem for specific applications.

- -

- Molecular Machine Learning

- -

- Modeling Proteins

- -

- Protein Ligand Modeling

- -

- Quantum Chemistry

- -

- Bioinformatics

- -

- Material Science

- -

- Machine Learning Methods

- -

- Deep Differential Equations

- -
-
-
- - - - -
- -
- - -
- - - - -
-
diff --git a/new-website/utils/tutorials/mocks/github-response-mock.json b/new-website/utils/tutorials/mocks/github-response-mock.json new file mode 100644 index 00000000..72127082 --- /dev/null +++ b/new-website/utils/tutorials/mocks/github-response-mock.json @@ -0,0 +1,98 @@ +[ + { + "name": "About_nODE_Using_Torchdiffeq_in_Deepchem.ipynb", + "path": "examples/tutorials/About_nODE_Using_Torchdiffeq_in_Deepchem.ipynb", + "sha": "e04715d9ae2cafe1feddf518170631109389258d", + "size": 58509, + "url": "https://api.github.com/repos/deepchem/deepchem/contents/examples/tutorials/About_nODE_Using_Torchdiffeq_in_Deepchem.ipynb?ref=master", + "html_url": "https://github.com/deepchem/deepchem/blob/master/examples/tutorials/About_nODE_Using_Torchdiffeq_in_Deepchem.ipynb", + "git_url": "https://api.github.com/repos/deepchem/deepchem/git/blobs/e04715d9ae2cafe1feddf518170631109389258d", + "download_url": "https://raw.githubusercontent.com/deepchem/deepchem/master/examples/tutorials/About_nODE_Using_Torchdiffeq_in_Deepchem.ipynb", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/deepchem/deepchem/contents/examples/tutorials/About_nODE_Using_Torchdiffeq_in_Deepchem.ipynb?ref=master", + "git": "https://api.github.com/repos/deepchem/deepchem/git/blobs/e04715d9ae2cafe1feddf518170631109389258d", + "html": "https://github.com/deepchem/deepchem/blob/master/examples/tutorials/About_nODE_Using_Torchdiffeq_in_Deepchem.ipynb" + } + }, + { + "name": "Advanced_Model_Training.ipynb", + "path": "examples/tutorials/Advanced_Model_Training.ipynb", + "sha": "6c05b041ff6643d6e17ee4752928e2b40120926b", + "size": 14737, + "url": "https://api.github.com/repos/deepchem/deepchem/contents/examples/tutorials/Advanced_Model_Training.ipynb?ref=master", + "html_url": "https://github.com/deepchem/deepchem/blob/master/examples/tutorials/Advanced_Model_Training.ipynb", + "git_url": "https://api.github.com/repos/deepchem/deepchem/git/blobs/6c05b041ff6643d6e17ee4752928e2b40120926b", + "download_url": "https://raw.githubusercontent.com/deepchem/deepchem/master/examples/tutorials/Advanced_Model_Training.ipynb", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/deepchem/deepchem/contents/examples/tutorials/Advanced_Model_Training.ipynb?ref=master", + "git": "https://api.github.com/repos/deepchem/deepchem/git/blobs/6c05b041ff6643d6e17ee4752928e2b40120926b", + "html": "https://github.com/deepchem/deepchem/blob/master/examples/tutorials/Advanced_Model_Training.ipynb" + } + }, + { + "name": "Advanced_model_training_using_hyperopt.ipynb", + "path": "examples/tutorials/Advanced_model_training_using_hyperopt.ipynb", + "sha": "9eec5a84bd58292474cf74d61a26d03d08f24b00", + "size": 30688, + "url": "https://api.github.com/repos/deepchem/deepchem/contents/examples/tutorials/Advanced_model_training_using_hyperopt.ipynb?ref=master", + "html_url": "https://github.com/deepchem/deepchem/blob/master/examples/tutorials/Advanced_model_training_using_hyperopt.ipynb", + "git_url": "https://api.github.com/repos/deepchem/deepchem/git/blobs/9eec5a84bd58292474cf74d61a26d03d08f24b00", + "download_url": "https://raw.githubusercontent.com/deepchem/deepchem/master/examples/tutorials/Advanced_model_training_using_hyperopt.ipynb", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/deepchem/deepchem/contents/examples/tutorials/Advanced_model_training_using_hyperopt.ipynb?ref=master", + "git": "https://api.github.com/repos/deepchem/deepchem/git/blobs/9eec5a84bd58292474cf74d61a26d03d08f24b00", + "html": "https://github.com/deepchem/deepchem/blob/master/examples/tutorials/Advanced_model_training_using_hyperopt.ipynb" + } + }, + { + "name": "An_Introduction_To_MoleculeNet.ipynb", + "path": "examples/tutorials/An_Introduction_To_MoleculeNet.ipynb", + "sha": "f1c8723fbefef7f739ab6d126729175993ec414e", + "size": 24984, + "url": "https://api.github.com/repos/deepchem/deepchem/contents/examples/tutorials/An_Introduction_To_MoleculeNet.ipynb?ref=master", + "html_url": "https://github.com/deepchem/deepchem/blob/master/examples/tutorials/An_Introduction_To_MoleculeNet.ipynb", + "git_url": "https://api.github.com/repos/deepchem/deepchem/git/blobs/f1c8723fbefef7f739ab6d126729175993ec414e", + "download_url": "https://raw.githubusercontent.com/deepchem/deepchem/master/examples/tutorials/An_Introduction_To_MoleculeNet.ipynb", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/deepchem/deepchem/contents/examples/tutorials/An_Introduction_To_MoleculeNet.ipynb?ref=master", + "git": "https://api.github.com/repos/deepchem/deepchem/git/blobs/f1c8723fbefef7f739ab6d126729175993ec414e", + "html": "https://github.com/deepchem/deepchem/blob/master/examples/tutorials/An_Introduction_To_MoleculeNet.ipynb" + } + }, + { + "name": "Atomic_Contributions_for_Molecules.ipynb", + "path": "examples/tutorials/Atomic_Contributions_for_Molecules.ipynb", + "sha": "6d5356e49b8292f70e0d38f27f678a28a404fc59", + "size": 1774157, + "url": "https://api.github.com/repos/deepchem/deepchem/contents/examples/tutorials/Atomic_Contributions_for_Molecules.ipynb?ref=master", + "html_url": "https://github.com/deepchem/deepchem/blob/master/examples/tutorials/Atomic_Contributions_for_Molecules.ipynb", + "git_url": "https://api.github.com/repos/deepchem/deepchem/git/blobs/6d5356e49b8292f70e0d38f27f678a28a404fc59", + "download_url": "https://raw.githubusercontent.com/deepchem/deepchem/master/examples/tutorials/Atomic_Contributions_for_Molecules.ipynb", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/deepchem/deepchem/contents/examples/tutorials/Atomic_Contributions_for_Molecules.ipynb?ref=master", + "git": "https://api.github.com/repos/deepchem/deepchem/git/blobs/6d5356e49b8292f70e0d38f27f678a28a404fc59", + "html": "https://github.com/deepchem/deepchem/blob/master/examples/tutorials/Atomic_Contributions_for_Molecules.ipynb" + } + }, + { + "name": "Conditional_Generative_Adversarial_Networks.ipynb", + "path": "examples/tutorials/Conditional_Generative_Adversarial_Networks.ipynb", + "sha": "e87142e3cee9dfb461c195e4772bc730d596fff9", + "size": 83015, + "url": "https://api.github.com/repos/deepchem/deepchem/contents/examples/tutorials/Conditional_Generative_Adversarial_Networks.ipynb?ref=master", + "html_url": "https://github.com/deepchem/deepchem/blob/master/examples/tutorials/Conditional_Generative_Adversarial_Networks.ipynb", + "git_url": "https://api.github.com/repos/deepchem/deepchem/git/blobs/e87142e3cee9dfb461c195e4772bc730d596fff9", + "download_url": "https://raw.githubusercontent.com/deepchem/deepchem/master/examples/tutorials/Conditional_Generative_Adversarial_Networks.ipynb", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/deepchem/deepchem/contents/examples/tutorials/Conditional_Generative_Adversarial_Networks.ipynb?ref=master", + "git": "https://api.github.com/repos/deepchem/deepchem/git/blobs/e87142e3cee9dfb461c195e4772bc730d596fff9", + "html": "https://github.com/deepchem/deepchem/blob/master/examples/tutorials/Conditional_Generative_Adversarial_Networks.ipynb" + } + } +] \ No newline at end of file diff --git a/new-website/utils/tutorials/test_fetch_tutorials.py b/new-website/utils/tutorials/test_fetch_tutorials.py index cec6aed5..860391c6 100644 --- a/new-website/utils/tutorials/test_fetch_tutorials.py +++ b/new-website/utils/tutorials/test_fetch_tutorials.py @@ -52,11 +52,11 @@ def test_fetch_tutorial_data(self, mock_get): mock_response = Response() mock_response.status_code = 200 - with open('./mocks/github-response-mock.html', 'rb') as f: + with open('./mocks/github-response-mock.json', 'rb') as f: mock_response._content = f.read() mock_get.return_value = mock_response - tutorials = fetch_tutorials.fetch_tutorial_data() + tutorials = fetch_tutorials.get_tutorial_list() expected_tutorials = [ 'About_nODE_Using_Torchdiffeq_in_Deepchem.ipynb', From 899ed8d5ab7f10dc03e7b78a0b06050f6051447e Mon Sep 17 00:00:00 2001 From: Dibyam Kumar Date: Sun, 6 Aug 2023 02:12:33 +0530 Subject: [PATCH 2/3] Remove redundant code --- .../utils/tutorials/fetch_tutorials.py | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/new-website/utils/tutorials/fetch_tutorials.py b/new-website/utils/tutorials/fetch_tutorials.py index 716a7a1c..6059bf1c 100644 --- a/new-website/utils/tutorials/fetch_tutorials.py +++ b/new-website/utils/tutorials/fetch_tutorials.py @@ -49,7 +49,7 @@ def fetch_file_list_from_repo(repo_owner, repo_name, path): response = requests.get(url) if response.status_code == 200: data = response.json() - return [file["name"] for file in data] + return data else: raise Exception( "Error fetching file names: {}".format(response.status_code)) @@ -66,10 +66,12 @@ def get_tutorial_list(): tutorials = fetch_file_list_from_repo( DEEPCHEM_REPO_OWNER, DEEPCHEM_REPO_NAME, TUTORIALS_PATH) + tutorial_names = [tutorial['name'] for tutorial in tutorials] + # Filter only the ipynb files - tutorials = [ - tutorial for tutorial in tutorials if tutorial.endswith('.ipynb')] - return tutorials + tutorial_names = [ + tutorial for tutorial in tutorial_names if tutorial.endswith('.ipynb')] + return tutorial_names def fetch_tutorial_render_order(): @@ -77,18 +79,20 @@ def fetch_tutorial_render_order(): Downloads the CSV files containing the tutorial order from the Deepchem repository. """ - raw_path = 'https://raw.githubusercontent.com/deepchem/deepchem/master/examples/tutorials/website-render-order/' - tutorial_order = fetch_file_list_from_repo( + tutorial_order_csv_data = fetch_file_list_from_repo( DEEPCHEM_REPO_OWNER, DEEPCHEM_REPO_NAME, TUTORIAL_RENDER_ORDER_PATH) + tutorial_order_csv_names = [tutorial['name'] + for tutorial in tutorial_order_csv_data] + # Filter only the csv files - tutorial_order = [ - tutorial for tutorial in tutorial_order if tutorial.endswith('.csv')] + tutorial_order_csv_names = [ + tutorial for tutorial in tutorial_order_csv_names if tutorial.endswith('.csv')] - for tutorial_group in tutorial_order: - response = requests.get(raw_path + tutorial_group) + for tutorial_group in tutorial_order_csv_data: + response = requests.get(tutorial_group.get('download_url')) if response.status_code == 200: - with open(f"./website-render-order/{tutorial_group}", "wb") as tutorial_file: + with open(f"./website-render-order/{tutorial_group.get('name')}", "wb") as tutorial_file: tutorial_file.write(response.content) else: raise Exception( @@ -104,7 +108,7 @@ def create_directories(): os.makedirs('./website-render-order', exist_ok=True) -def convert_to_html(tutorials): +def convert_to_html(tutorial): """ Converts the Jupyter notebooks in the './ipynb-notebooks' directory to HTML files and stores them in the './html-notebooks' directory. @@ -116,19 +120,26 @@ def convert_to_html(tutorials): fromPath = "./ipynb-notebooks/" toPath = "./html-notebooks/" - tutorialURL = 'https://raw.githubusercontent.com/deepchem/deepchem/master/examples/tutorials/' - for tutorial in tutorials: try: - file_name_html = f'{tutorial.rsplit(".")[0]}.html' - response = requests.get(tutorialURL + tutorial) - with open(f"./ipynb-notebooks/{tutorial}", "wb") as tutorial_file: + tutorial_file_name = tutorial["name"] + tutorial_download_link = tutorial["download_url"] + + if not tutorial_file_name.endswith('.ipynb'): + continue + + print() + + file_name_html = f'{tutorial_file_name.rsplit(".")[0]}.html' + response = requests.get(tutorial_download_link) + with open(f"./ipynb-notebooks/{tutorial_file_name}", "wb") as tutorial_file: tutorial_file.write(response.content) subprocess.call( - f'jq -M "del(.metadata.widgets)" ./ipynb-notebooks/{tutorial} > ./ipynb-notebooks/fixed-{tutorial}', shell=True) + f'jq -M "del(.metadata.widgets)" ./ipynb-notebooks/{tutorial_file_name} > ./ipynb-notebooks/fixed-{tutorial_file_name}', shell=True + ) subprocess.call( - f'python -m nbconvert --to html ./ipynb-notebooks/fixed-{tutorial}', shell=True) + f'python -m nbconvert --to html ./ipynb-notebooks/fixed-{tutorial_file_name}', shell=True) shutil.copyfile(f'{fromPath}fixed-{file_name_html}', toPath + file_name_html) @@ -136,12 +147,14 @@ def convert_to_html(tutorials): notebook_list.write(file_name_html + '\n') except Exception as exception: print(exception) - print(f"Could not process {tutorial}") + print(f"Could not process {tutorial_file_name}") if __name__ == "__main__": create_directories() - tutorials = get_tutorial_list() + tutorials = fetch_file_list_from_repo(DEEPCHEM_REPO_OWNER, DEEPCHEM_REPO_NAME, TUTORIALS_PATH) + + print(tutorials) # The script throws an AssertionError if no tutorials are fetched. This is to prevent website deployment if no tutorials are fetched. assert len(tutorials) > 0 From 7b52bf2d2aec6d1d2d8f5a7d0bdd1e86e6d2d01b Mon Sep 17 00:00:00 2001 From: Dibyam Kumar Date: Sun, 6 Aug 2023 02:45:39 +0530 Subject: [PATCH 3/3] Update tests and comments for code clarity. --- .../utils/tutorials/fetch_tutorials.py | 15 +++---- .../utils/tutorials/test_fetch_tutorials.py | 41 ++++++++++--------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/new-website/utils/tutorials/fetch_tutorials.py b/new-website/utils/tutorials/fetch_tutorials.py index 6059bf1c..ddc33033 100644 --- a/new-website/utils/tutorials/fetch_tutorials.py +++ b/new-website/utils/tutorials/fetch_tutorials.py @@ -23,7 +23,7 @@ def fetch_file_list_from_repo(repo_owner, repo_name, path): """ - Fetches the names of all the files in the given path from the given Github repository. + Fetches the metadata like name, download_url etc of all the files in a given path in a Github repository . Parameters ---------- @@ -36,8 +36,8 @@ def fetch_file_list_from_repo(repo_owner, repo_name, path): Returns ------- - file_names: list - A list of strings, where each string represents the name of a file. + data: list + A list of dictionaries, where each dictionary contains various metadata like the name and download_url of a file. Raises ------ @@ -108,7 +108,7 @@ def create_directories(): os.makedirs('./website-render-order', exist_ok=True) -def convert_to_html(tutorial): +def convert_to_html(tutorials): """ Converts the Jupyter notebooks in the './ipynb-notebooks' directory to HTML files and stores them in the './html-notebooks' directory. @@ -128,8 +128,6 @@ def convert_to_html(tutorial): if not tutorial_file_name.endswith('.ipynb'): continue - print() - file_name_html = f'{tutorial_file_name.rsplit(".")[0]}.html' response = requests.get(tutorial_download_link) with open(f"./ipynb-notebooks/{tutorial_file_name}", "wb") as tutorial_file: @@ -152,9 +150,8 @@ def convert_to_html(tutorial): if __name__ == "__main__": create_directories() - tutorials = fetch_file_list_from_repo(DEEPCHEM_REPO_OWNER, DEEPCHEM_REPO_NAME, TUTORIALS_PATH) - - print(tutorials) + tutorials = fetch_file_list_from_repo( + DEEPCHEM_REPO_OWNER, DEEPCHEM_REPO_NAME, TUTORIALS_PATH) # The script throws an AssertionError if no tutorials are fetched. This is to prevent website deployment if no tutorials are fetched. assert len(tutorials) > 0 diff --git a/new-website/utils/tutorials/test_fetch_tutorials.py b/new-website/utils/tutorials/test_fetch_tutorials.py index 860391c6..5caa182f 100644 --- a/new-website/utils/tutorials/test_fetch_tutorials.py +++ b/new-website/utils/tutorials/test_fetch_tutorials.py @@ -15,34 +15,37 @@ from requests import Response import fetch_tutorials import os - +import json class TestConvertToHTML(unittest.TestCase): def test_convert_to_html(self): - tutorials = ['About_nODE_Using_Torchdiffeq_in_Deepchem.ipynb', - 'Advanced_model_training_using_hyperopt.ipynb'] - try: - os.makedirs('./html-notebooks') - os.makedirs('./ipynb-notebooks') - - except Exception as exception: - print("Directories already exist, or could not create directories. ") - print(exception) + with open('./mocks/github-response-mock.json', 'rb') as f: + tutorials = json.load(f) + + try: + os.makedirs('./html-notebooks') + os.makedirs('./ipynb-notebooks') - fetch_tutorials.convert_to_html(tutorials) + except Exception as exception: + print("Directories already exist, or could not create directories. ") + print(exception) - for tutorial in tutorials: - file_name_html = f'{tutorial.rsplit(".")[0]}.html' - self.assertTrue(os.path.isfile( - f"./html-notebooks/{file_name_html}")) + fetch_tutorials.convert_to_html(tutorials) - with open('./notebooks.txt', "r") as notebook_list: - content = notebook_list.read() for tutorial in tutorials: - file_name_html = f'{tutorial.rsplit(".")[0]}.html' - self.assertIn(file_name_html, content) + tutorial_file_name = tutorial["name"] + file_name_html = f'{tutorial_file_name.rsplit(".")[0]}.html' + self.assertTrue(os.path.isfile( + f"./html-notebooks/{file_name_html}")) + + with open('./notebooks.txt', "r") as notebook_list: + content = notebook_list.read() + for tutorial in tutorials: + tutorial_file_name = tutorial["name"] + file_name_html = f'{tutorial_file_name.rsplit(".")[0]}.html' + self.assertIn(file_name_html, content) class TestFetchTutorialData(unittest.TestCase):