Convert into main + sub packages instead of multiple main packages

.github/workflows/tex2pdf.yaml

@@ -19,15 +19,16 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: "3.11"
+    - name: Setup TeX Live
+      uses: teatimeguest/setup-texlive-action@v3
+      with:
+        packages: scheme-basic
     - name: Install dependencies for tex2pdf
       run: |
-        cd tex2pdf_service
         python -m pip install --upgrade pip
         pip install pytest poetry
-        poetry install
-        poetry run pip install types-requests
+        poetry install --extras service
     - name: pytest for tex2pdf
       run: |
-        cd tex2pdf_service
         export PYTHONPATH=$PWD
         poetry run pytest -k "not integration" tests

.gitignore

@@ -165,3 +165,4 @@ cython_debug/
 # Emacs backups
 *~
 \#*
+.python-version

.pre-commit-config.yaml

@@ -1,5 +1,3 @@
-files: preflight_parser|zerozeroreadme|tex_inspection
-
 default_language_version:
   python: python3.11
 

tex2pdf_service/Appliance.Dockerfile → Appliance.Dockerfile

@@ -31,9 +31,9 @@ COPY tex2pdf/ ./tex2pdf/
 COPY poetry.lock pyproject.toml ./
 # poetry is BROKEN wrt to installing multiple packages from same git repo
 # see https://github.com/python-poetry/poetry/issues/6958
-RUN poetry config installer.parallel false
+# RUN poetry config installer.parallel false
 # install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
-RUN poetry install --without=dev
+RUN poetry install --extras service --without=dev
 
 
 # application specific changes

tex2pdf_service/Makefile → Makefile

@@ -13,11 +13,11 @@ TEX2PDF_CPUS := $(shell echo $${TEX2PDF_CPUS:-4})
 TEX2PDF_WORKERS := $(shell echo $${TEX2PDF_WORKERS:-8})
 TEX2PDF_DOCKER_PLATFORM := $(shell echo $${TEX2PDF_DOCKER_PLATFORM:-linux/amd64})
 
-APP_DOCKER_RUN := docker run --cpus ${TEX2PDF_CPUS} --rm -p ${app_port}:${dockerport} -e PORT=${dockerport} -e WORKERS=${TEX2PDF_WORKERS} --name ${app_name} --security-opt="no-new-privileges=true" 
+APP_DOCKER_RUN := docker run --cpus ${TEX2PDF_CPUS} --rm -p ${app_port}:${dockerport} -e PORT=${dockerport} -e WORKERS=${TEX2PDF_WORKERS} --name ${app_name} --security-opt="no-new-privileges=true"
 
 .PHONY: HELLO app.docker app.run app.stop
 
-default: HELLO venv/lib/python3.11/site-packages/fastapi
+default: HELLO
 
 HELLO:
 	@echo To see the README of this Makefile, type "make help"
@@ -53,7 +53,7 @@ app.docker:
 #-# Command: app.run
 #-#   runs the appliance container with the terminal attached (for test)
 app.run: app.stop
-	${APP_DOCKER_RUN} -it ${app_tag}-${base_tag_version}:latest 
+	${APP_DOCKER_RUN} -it ${app_tag}-${base_tag_version}:latest
 
 #-#
 #-# Command: app.stop
@@ -68,3 +68,10 @@ app.stop:
 app.sh: app.stop
 	${APP_DOCKER_RUN}  -v ${HOME}/Downloads:/home/worker/Downloads -w /home/worker -it ${app_tag}-${base_tag_version}:latest  /bin/bash
 
+
+
+##### Makefile entries for preflight parser
+
+regenerate_test_json:
+	rm -f tests/*.json
+	for i in tests/* ; do echo "=== $$i ===" ; python -m tex2pdf.preflight_parser $$i | json_pp > $$i.json; done

README.md

@@ -1,6 +1,35 @@
 # submission-tools
-arXiv submission related tool repository
 
-- **pdf_profile**: Read PDF file and outputs a digst.
-- **tex2pdf_service**: Compile arXiv TeX submission and generates PDF.
-- **tex_inspection**: A library to inspect the arXiv TeX submission files
+arXiv submission related tool repository, mainly collection `tex2pdf` with
+its subpackages (`preflight_parser`, `zerozeroreadme`, `tex_inspection`, and `service`)
+and the `pdf_profile` package.
+
+# PDF Profile
+
+TBD.
+
+# TeX Inspection
+
+Package: `tex2pdf.tex_inspection`
+
+A library that inspects the arXiv submission
+
+# Preflight Parser
+
+Package: `tex2pdf.preflight_parser`
+
+Full specification: [GenPDF Preflight & 00README Spec Proposal](https://docs.google.com/document/d/1X1ZVvNX75GtFqejcOI1AHb7PkRbkT-syJlpsU4v_-o4/edit?usp=sharing)
+
+# ZeroZeroReadme
+
+Package: `tex2pdf.zerozeroreadme`
+
+Full specification: [00README for authors](https://docs.google.com/document/d/1WbAh2atcDLG0-yJEP9WDKqujHtwH1Kp_TojykeC2HFQ/edit?usp=sharing)
+
+# TeX2PDF Service
+
+Package: `tex2pdf.service`
+
+arXiv's next generation TeX compilation based on TexLive Docker container.
+
+See detailed [README.md](tex2pdf/service/README.md)

tex2pdf_service/app-logging.conf → app-logging.conf

@@ -8,7 +8,7 @@ keys=access_handler
 keys=json
 
 [formatter_json]
-class=tex2pdf.CustomJsonFormatter
+class=tex2pdf.service.CustomJsonFormatter
 format=%(asctime)s %(name)s %(levelname)s %(message)s %(process)d %(threadName)s
 
 [handler_access_handler]

tex2pdf_service/app-logging.json → app-logging.json

@@ -3,7 +3,7 @@
   "disable_existing_loggers": false,
   "formatters": {
     "json": {
-      "class": "tex2pdf.CustomJsonFormatter",
+      "class": "tex2pdf.service.CustomJsonFormatter",
       "format": "%(asctime)s %(name)s %(levelname)s %(message)s %(process)d %(threadName)s"
     }
   },

tex2pdf_service/app.sh → app.sh

@@ -10,4 +10,4 @@ fi
 export PORT
 #export TEXMFHOME=/usr/local/texlive/2023
 #. /home/worker/venv/bin/activate
-hypercorn --config hypercorn-config.toml --bind 0.0.0.0:$PORT --log-config app-logging.conf --workers $WORKERS tex2pdf.tex2pdf_api:app
+hypercorn --config hypercorn-config.toml --bind 0.0.0.0:$PORT --log-config app-logging.conf --workers $WORKERS tex2pdf.service.tex2pdf_api:app

preflight_parser/check_all_submissions.sh → bin/check_all_submissions.sh

@@ -43,7 +43,7 @@ do_one () {
   rm -rf __tmp__/*
   tar -C __tmp__ -xf $1
   bn=$(basename $1 .tar.gz)
-  (python -m preflight_parser __tmp__ | json_pp) > json/$bn.json 2> json/$bn.log
+  (python -m tex2pdf.preflight_parser __tmp__ | json_pp) > json/$bn.json 2> json/$bn.log
   # remove .log file if it has 0 size
   if ! [ -s json/$bn.log ] ; then
     rm json/$bn.log

tex2pdf_service/bin/compile_submissions.py → bin/compile_submissions.py

@@ -34,7 +34,7 @@
 from sqlite3 import Connection
 
 import click
-from tex2pdf.remote_call import service_process_tarball
+from tex2pdf.service.remote_call import service_process_tarball
 from tqdm import tqdm
 
 logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s: %(message)s')

tex2pdf_service/tests/conftest.py → conftest.py

@@ -1,6 +1,8 @@
+"""Pytest configuration file."""
+
 
 def pytest_addoption(parser):
+    """Add command line options to pytest."""
     parser.addoption("--keep-docker-running", action="store_true", help="keep docker image running")
     parser.addoption("--no-docker-setup", action="store_true", help="do not run docker setup, expect a running docker")
     parser.addoption("--docker-port", action="store", default="33031", help="outside docker port to use")
-

tex2pdf_service/mypy.ini → mypy.ini

@@ -1,6 +1,8 @@
 [mypy]
 mypy_path = tex2pdf
 
+exclude = tests
+
 #
 # Covered by --strict, with some turned off:
 #
@@ -18,9 +20,10 @@ warn_unused_configs=True
 #may be worth reconsidering this one:
 no_implicit_optional=True
 strict_optional=True
+disallow_untyped_calls=True
 
 #
-# Other: 
+# Other:
 #
 ignore_missing_imports=True
 follow_imports=silent
@@ -30,5 +33,7 @@ follow_imports=silent
 #
 plugins = pydantic.mypy
 
-[tex2pdf.*]
-disallow_untyped_calls=True
+#[tex2pdf.*]
+#disallow_untyped_calls=True
+
+