Merge pull request #26 from TareHimself/dev

Batch processing

.gitignore

@@ -31,4 +31,5 @@ yarn-error.log*
 manga-translator-service-account-data.json
 exa/*
 local/*
-trans-test/*
+trans-test/*
+*.env

Dockerfile

@@ -1,57 +1,108 @@
 # Build ui using node
-FROM node:18.12.1
+# FROM node:18.12.1
 
-COPY package.json .
-COPY package-lock.json .
-COPY public public
-COPY src src
-COPY tsconfig.json .
-COPY .eslintrc.json .
+# WORKDIR /app
 
-RUN npm install
+# COPY ui ui
 
-RUN npm run build
+# WORKDIR /app/ui
+
+# RUN npm install
+
+# RUN npm run build
 
 
 # Use the NVIDIA CUDA base image
 FROM nvidia/cuda:11.7.1-runtime-ubuntu20.04
 
 
-COPY --from=0 build build
+WORKDIR /app
+
+# COPY --from=0 /app/ui/build /ui/build
 # Set the working directory to /app
 #WORKDIR /app
 
 # Update package lists and install required packages
-RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
-    apt-get install -y python3.9 python3-pip
 
-# # Make sure Python 3.9 is the default python3 version
-# RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1
+# RUN apt-get update
+# RUN apt-get remove python
+# RUN apt-get remove python-pip
+# RUN apt-get -y install software-properties-common
+# RUN add-apt-repository ppa:deadsnakes/ppa
+# RUN apt-get -y install python3.9
+# RUN apt-get -y install python3-pip
 
-# Create a symbolic link for pip (optional)
-RUN ln -s /usr/bin/pip3 /usr/bin/pip
+COPY translator translator
+COPY server.py .
+COPY fonts fonts
+COPY models models
+COPY requirements.txt .
+
+# # Install base utilities
+# RUN apt-get update \
+#     && apt-get install -y build-essential \
+#     && apt-get install -y wget \
+#     && apt-get clean \
+#     && rm -rf /var/lib/apt/lists/*
+
+# # Install miniconda
+# ENV CONDA_DIR /opt/conda
+# RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \
+#     /bin/bash ~/miniconda.sh -b -p /opt/conda
 
-# Verify Python and pip versions
-RUN python3 --version && pip --version
+# # Put conda in path so we can use conda activate
+# ENV PATH=$CONDA_DIR/bin:$PATH
 
 # # Create symbolic links to set Python 3.9 as the default
 # RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 && \
 #     update-alternatives --config python3
 # RUN apt-get update && apt-get install -y python3.9 python3.9-dev
+# RUN conda create -n translator python=3.9 -y
 
-COPY translator translator
-COPY server.py .
-COPY fonts fonts
-COPY models models
-COPY requirements.txt .
+# SHELL ["conda", "run", "-n", "translator", "/bin/bash", "-c"]
+
+# RUN pip install -r requirements.txt
+# RUN pip uninstall -y torch torchvision torchaudio
+# RUN pip install opencv-python
+# RUN pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu117
+
+# RUN apt-get update && apt-get install -y --no-install-recommends \
+#     software-properties-common \
+#     libsm6 libxext6 ffmpeg libfontconfig1 libxrender1 libgl1-mesa-glx \
+#     curl python3-pip
+
+# RUN pip3 install --upgrade pip
+
+# RUN pip3 install -r requirements.txt
+# RUN pip3 uninstall -y torch torchvision torchaudio
+# RUN pip3 install opencv-python
+# RUN pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu117
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    software-properties-common \
+    libsm6 libxext6 ffmpeg libfontconfig1 libxrender1 libgl1-mesa-glx \
+    curl python3-pip
+
+RUN curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o ~/miniconda.sh \
+    && sh ~/miniconda.sh -b -p /opt/conda \
+    && rm ~/miniconda.sh
+
+ENV PATH /opt/conda/bin:$PATH
+
+RUN conda update -n base -c defaults conda
+
+COPY conda.yml conda.yml
+
+RUN conda env create -f conda.yml --name translator
 
+RUN conda activate translator
 
-RUN pip install -r requirements.txt
+# RUN python3.9 -m pip install -r requirements.txt
 
-RUN pip uninstall -y torch torchvision torchaudio
+# RUN python3.9 -m 
 
-RUN pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu117
+# RUN python3.9 -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu117
 
-CMD ["python","server.py"]
+CMD ["python3","server.py"]
 
 

extractor.py

@@ -4,11 +4,13 @@
 from typing import Any
 import os
 import re
+import sys
+from importlib.resources import is_resource
 from typing import Union
 import pkg_resources
 from functools import cmp_to_key
 INSTALLED_PACKAGES = {pkg.key for pkg in pkg_resources.working_set}
-base_dir = "./"
+BASE_DIR = os.getcwd()
 
 # This file is an attempt to extract a given function or name from a module with all its depencencies. it has trouble with relative imports and needs more work
 
@@ -213,43 +215,20 @@ def get_other_refs(tree: ast.AST,exclude: list[str]) -> dict[str,ast.AST]:
             deps[node_name] = node
 
     return deps
-
-def try_build_module_path(current: str,remaining: list[str],max_lookaheads = 1) -> Union[str,None]:
-
-    if len(remaining) == 0:
-        if os.path.exists(current) and os.path.isfile(current):
-            return current
-        return None
-
-    new_path = os.path.join(current,remaining[0])
-
-    if os.path.exists(new_path):
-        return try_build_module_path(new_path,remaining[1:])
-    else:
-        for i in range(max_lookaheads):
-            delta = i + 1
-            if len(remaining) <= delta:
-                break
-
-            test_path = os.path.join(current,remaining[delta])
-            if os.path.exists(delta):
-                return try_build_module_path(test_path,remaining[delta:])
-
-        return None
 
 
 paths_cache: dict[str,str] = {}
 
 def module_to_file_path(cur_file_path: str,module: str) -> str:
-    global base_dir
+    global BASE_DIR
     global paths_cache
     global INSTALLED_PACKAGES
 
     cache_key = module
     if cache_key in paths_cache:
         return paths_cache[cache_key]
 
-    start_path = base_dir
+    start_path = BASE_DIR
 
     if module.split('.')[0] in INSTALLED_PACKAGES:
         return None
@@ -443,17 +422,11 @@ def comp_a_b(a:str,b: str):
             print(key)
             file_parts.append(content)
         import_parts.update(col_info.imports)
-            
-    return list(import_parts),file_parts
-
+
+
+    return [f"# This file was created using extractor.py\n"] + list(import_parts) + [""],file_parts
 
-    # # imported_refs = filter(lambda a: a in parsed_imports.keys(),map(lambda a: parsed_imports_aliases.get(a,a),related_refs))
 
-# print(extract_from_file(filename="module_test/a.py",names=["A"]))
-# print(extract_from_file(filename="translator/core/pipelines.py",names=["FullConversion"]))
-# "D:\\Github\\manga-translator\\deepfillv2-pytorch\\test.py"
-# with open("translator/core/pipelines.py",'r') as f:
-# base_dir = './lama'
 with open("out.py",'w',encoding='utf8') as out_file:
-    file_imports,file_content = extract_from_file(filename="D:\\Github\\manga-translator\\extractor.py",names=["extract_from_file"])
+    file_imports,file_content = extract_from_file(filename="D:\\Github\\manga-translator\\d.py",names=["non_max_suppression","scale_boxes","process_mask"])
     out_file.write("\n".join(file_imports + file_content))