links and docker

zevisvei · zevisvei · commit 5051a5c58f64 · 2025-01-24T08:55:27.000+02:00
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,23 @@
+FROM ubuntu
+
+RUN apt-get update && apt-get install -y sudo wget python3 python3-pip git
+
+
+RUN git clone https://github.com/zevisvei/sefaria_ebooks.git
+
+
+WORKDIR /sefaria_ebooks
+
+
+RUN sudo apt install python3.12-venv -y
+RUN python3 -m venv venv
+RUN ./venv/bin/pip install -r requirements.txt
+
+
+RUN sudo apt install libegl1 libopengl0 libxcb-cursor0 libfreetype6 xz-utils libqt6core6 libqt6gui6t64 libqt6widgets6t64 -y
+
+
+RUN sudo -v && wget -nv -O- https://download.calibre-ebook.com/linux-installer.sh | sudo sh /dev/stdin
+
+
+CMD ["./venv/bin/python", "main.py"]
diff --git a/main.py b/main.py
@@ -5,7 +5,6 @@
 
 app = Flask(__name__)
 
-# טוען את כל הספרים
 sefaria_api_instance = sefaria_api.SefariaApi()
 all_books = sefaria_api_instance.table_of_contents()
 list_all_books = utils.recursive_register_categories(all_books)
@@ -34,17 +33,16 @@ def run_script():
         book_dir = ' dir="rtl"' if lang == "hebrew" else ""
         book_content = result.process_book()
         book_content = f'<html lang={lang[:2]}><head><title></title></head><body{book_dir}>{"".join(book_content)}</body></html>'
+        if "footnote-marker" in book_content:
+            book_content = utils.footnotes_to_epub(book_content)
         metadata = result.get_metadata()
-        
-        # יצירת קובץ HTML
+
         with open(html_file, "w", encoding="utf-8") as f:
             f.write(book_content)
-        
-        # יצירת קובץ EPUB
+
         utils.to_ebook(html_file, epub_file, metadata)
         epub_file = os.path.abspath(f"{file_name}.epub")
-        
-        # החזרת הקובץ להורדה
+
         return send_file(
             epub_file,
             as_attachment=True,
@@ -62,4 +60,4 @@ def run_script():
 
 
 if __name__ == "__main__":
-    app.run(debug=True)
+    app.run(host='0.0.0.0', port=8000, debug=True)
diff --git a/main_from_export.py b/main_from_export.py
@@ -1,33 +1,6 @@
 from sefaria.get_from_export import Book
-from sefaria.utils import sanitize_filename
+from sefaria.utils import sanitize_filename, to_ebook, footnotes_to_epub
 import os
-import subprocess
-
-
-def to_ebook(
-    input_file: str,
-    output_file: str,
-    dict_args: dict[str, str],
-    level1_toc: str = "//h:h1",
-    level2_toc: str = "//h:h2",
-    level3_toc: str = "//h:h3",
-):
-    args = [
-        "ebook-convert",
-        input_file,
-        output_file,
-        f"--level1-toc={level1_toc}",
-        f"--level2-toc={level2_toc}",
-        f"--level3-toc={level3_toc}",
-    ]
-    for key, value in dict_args.items():
-        args.append(f"--{key}={value}")
-    subprocess.run(
-        args,
-        stdout=subprocess.DEVNULL,  # משתיק את הפלט
-        stderr=subprocess.DEVNULL,  # משתיק את השגיאות
-        check=True,
-    )
 
 
 def get_book(book_title: str, text_file_path: str, schema_file_path: str, lang: str):
@@ -41,7 +14,6 @@ def get_book(book_title: str, text_file_path: str, schema_file_path: str, lang:
 
 
 def main(json_folder, schemas_folder, output_folder, lang: str):
-    eroor_list = []
     """
     Process all books in the given folder whose path ends with 'Hebrew/Merged.json'.
     It finds the corresponding schema file in the schemas folder by matching the
@@ -50,7 +22,7 @@ def main(json_folder, schemas_folder, output_folder, lang: str):
     :param folder_path: Path to the folder containing the book files.
     :param schemas_folder: Path to the folder containing the schema files.
     """
-    for root, _,files in os.walk(json_folder):
+    for root, _, files in os.walk(json_folder):
         for file in files:
             file_path = os.path.join(root, file)
             if file_path.lower().endswith(f'{lang}{os.sep}merged.json'):
@@ -66,14 +38,15 @@ def main(json_folder, schemas_folder, output_folder, lang: str):
                     print(output_file_name)
                     book_dir = ' dir="rtl"' if lang == "hebrew" else ""
                     book_content = f'<html lang={lang[:2]}><head><title></title></head><body{book_dir}>{"".join(book_content)}</body></html>'
+                    if "footnote-marker" in book_content:
+                        book_content = footnotes_to_epub(book_content)
                     with open(f'{output_file_name}.html', 'w', encoding='utf-8') as file:
                         file.write(book_content)
                     to_ebook(f"{output_file_name}.html", f"{output_file_name}.epub", metadata)
                     os.remove(f"{output_file_name}.html")
                 except Exception as e:
-                    eroor_list.append(f"{file_path} {e}")
-    with open("error.txt", "w", encoding="utf-8") as f:
-        f.write("\n".join(eroor_list))
+                    with open("error.txt", "a", encoding="utf-8") as f:
+                        f.write(f"{file_path} {e}\n")
 
 
 json_folder = "json"
@@ -82,4 +55,3 @@ def main(json_folder, schemas_folder, output_folder, lang: str):
 lang = "hebrew"
 main(json_folder=json_folder, schemas_folder=schemas_folder,
      output_folder=output_folder, lang=lang)
-                
diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,4 @@
 python_hebrew_numbers
 requests
-flask
+flask
+bs4
diff --git a/sefaria/get_from_export.py b/sefaria/get_from_export.py
@@ -119,10 +119,12 @@ def process_simple_book(self) -> None:
 
     def process_node(self, node: dict, text: list, level: int = 0) -> None:
         node_title = node['heTitle'] if self.section_names_lang == "he" else node["title"]
-        self.book_content.append(f"<h{min(level, 6)}>{node_title}</h{min(level, 6)}>\n")
+        if node_title:
+            self.book_content.append(f"<h{min(level, 6)}>{node_title}</h{min(level, 6)}>\n")
+            level += 1
         if node.get("nodes"):
             for sub_node in node['nodes']:
-                self.process_node(sub_node, text[sub_node['title']] if sub_node['key'] != 'default' else text[''], level=level+1)
+                self.process_node(sub_node, text[sub_node['title']] if sub_node['key'] != 'default' else text[''], level=level)
         else:  # Process nested arrays
             if self.section_names_lang == "he":
                 section_names = node.get(
@@ -133,7 +135,7 @@ def process_node(self, node: dict, text: list, level: int = 0) -> None:
                     "sectionNames"
                 )
             depth = node.get('depth', 1)
-            self.recursive_sections(section_names, text, depth, level+1)
+            self.recursive_sections(section_names, text, depth, level)
 
     def recursive_sections(
         self,
diff --git a/sefaria/utils.py b/sefaria/utils.py
@@ -2,6 +2,7 @@
 from hebrew_numbers import int_to_gematria
 import json
 import subprocess
+from bs4 import BeautifulSoup
 
 
 def recursive_register_categories(
@@ -99,3 +100,37 @@ def to_ebook(
         stderr=subprocess.DEVNULL,  # משתיק את השגיאות
         check=True,
     )
+
+
+def footnotes_to_epub(html_content: str) -> str:
+    soup = BeautifulSoup(html_content, 'html.parser')
+    notes = []
+    for sup_tag in soup.find_all('sup', class_='footnote-marker'):
+        next_tag = sup_tag.find_next_sibling() 
+        if next_tag and next_tag.name == 'i' and 'footnote' in next_tag.get('class', []):
+            note_id = f"note_{len(notes) + 1}"
+            back_note_id = f"back_note_{len(notes) + 1}"
+            sup_a = soup.new_tag("a", id=back_note_id, href=f"#{note_id}", title=sup_tag.text, class_="noteref", role="doc-noteref")
+            sup_a.string = sup_tag.text
+            sup_tag.string = "" 
+            sup_tag.append(sup_a)
+            note_a = soup.new_tag("a", href=f"#{back_note_id}", title=sup_tag.text)
+            note_a.string = f"←{sup_tag.text}"
+            note_span = soup.new_tag("span", id=note_id)
+            note_span.string = next_tag.text
+            note_p = soup.new_tag("p")
+            note_p.append(note_a)
+            note_p.append(note_span)
+            notes.append(note_p)
+            next_tag.extract()
+
+    if notes:
+        h1 = soup.new_tag("h1")
+        h1.string = "הערות שוליים"
+    body_tag = soup.body
+    if body_tag:
+        body_tag.append(h1)
+        for note in notes:
+            body_tag.append(note)
+
+    return str(soup)
