Skip to content

Commit afe9eea

Browse files
pkarwpkarw
authored
Merge pull request #76 from CatchTheTornado/feature/59-63-multiformat-reorganize-and-converters
#59 #63 multiformat, reorganize and converters
2 parents 167e42d + 6959bd7 commit afe9eea

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+1380
-472
lines changed

‎.dockerignore

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Virtual environments
2+
.venv
3+
4+
# Documentation and examples
5+
docs/
6+
examples/
7+
screenshots/
8+
9+
# Scripts and local storage
10+
scripts/
11+
storage/
12+
13+
# Python cache files
14+
__pycache__/
15+
*.py[cod]
16+
*.pyo
17+
*.pyd
18+
19+
# Editor/IDE settings
20+
.idea/
21+
.vscode/
22+
23+
# Build artifacts
24+
build/
25+
dist/
26+
*.egg-info/
27+
28+
# Version control
29+
.git/
30+
.gitignore

‎.env.example

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#APP_ENV=production # sets the app into prod mode, othervise dev mode with auto-reload on code changes
22
REDIS_CACHE_URL=redis://redis:6379/1
33
OLLAMA_HOST=http://ollama:11434
4-
STORAGE_PROFILE_PATH=/storage_profiles
4+
STORAGE_PROFILE_PATH=./storage_profiles
55
LLAMA_VISION_PROMPT="You are OCR. Convert image to markdown."
66

77
# CLI settings

‎.env.localhost.example

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ CELERY_RESULT_BACKEND=redis://localhost:6379/0
1616
OLLAMA_HOST=http://localhost:11434
1717
APP_ENV=development # Default to development mode
1818

19-
STORAGE_PROFILE_PATH=../storage_profiles
19+
STORAGE_PROFILE_PATH=./storage_profiles
2020

2121
# AWS S3
2222
#AWS_ACCESS_KEY_ID=your-access-key-id

‎.gitignore

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
1-
app/__pycache__/*
1+
text_extract_api/__pycache__/*
22
.env
33
.env.local
44
*.pyc
55
.venv
6+
.dvenv
67
.DS_Store
78
storage/*
89
client_secret*.json
910
.env.localhost
10-
11+
.idea
12+
# Python good practice ignore
13+
*.egg-info/
14+
/build/
15+
.pyproject.hash

‎Makefile

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
SHELL := /bin/bash
2+
3+
export DISABLE_VENV ?= 0
4+
export DISABLE_LOCAL_OLLAMA ?= 0
5+
6+
.PHONY: help
7+
help:
8+
@echo "Available commands:"
9+
@echo " make install (recommended) - Automatic setup for local or Docker"
10+
@echo " make run - Start the local application server"
11+
@echo " make run-docker - Run Docker containers with CPU support"
12+
@echo " make run-docker-gpu - Run Docker containers with GPU support"
13+
@echo " make clean - Clean the project environment"
14+
@echo " make clear-cache - Clear application cache"
15+
16+
.PHONY: install
17+
install:
18+
@width=$$(tput cols || echo 100); \
19+
[ "$$width" -gt "100" ] && width=100; \
20+
padding=$$(printf "%$${width}s" "" | tr ' ' '#'); \
21+
printf "\n\e[1;34m%s\e[0m\n" "$$padding"; \
22+
printf "\e[1;34m###%*sCatchTheTornado/text-extract-api%*s###\e[0m\n" $$(($$width / 2 - 21)) "" $$(($$width / 2 - 17)) ""; \
23+
printf "\e[1;34m###%*sAUTOMATIC SETUP%*s###\e[0m\n" $$(($$width / 2 - 10)) "" $$(($$width / 2 - 11)) ""; \
24+
printf "%s\n" "$$padding"; \
25+
printf "\e[1;34m Do you want to run the application locally or with Docker?\e[0m\n"; \
26+
printf "\e[1;33m [L] \e[0m Local - Run the application locally\n"; \
27+
printf "\e[1;33m [D] \e[0m Docker - Run the axpplication in Docker\n"; \
28+
read -p " > " choice; \
29+
case "$$choice" in \
30+
[lL]) echo -e "\033[1;32m ✔ You chose: Local Setup\033[0m"; $(MAKE) setup-local ;; \
31+
[dD]) echo -e "\033[1;32m ✔ You chose: Docker\033[0m"; $(MAKE) setup-docker ;; \
32+
*) echo "Invalid option. Exiting."; exit 1 ;; \
33+
esac
34+
35+
.PHONY: setup-local
36+
setup-local:
37+
@rm -f .pyproject.hash
38+
@if [ ! -f .env ]; then \
39+
printf "\n\e[1;34m Copy .env.localhost.example to .env.localhost \e[0m"; \
40+
cp .env.localhost.example .env.localhost; \
41+
fi
42+
@while true; do \
43+
printf "\n\e[1;34m Python setup environment...\e[0m"; \
44+
printf "\e[1;34m\n Do you want to install requirements?\e[0m\n"; \
45+
printf "\e[1;33m [y] \e[0m Yes - Install and then run application locally\n"; \
46+
printf "\e[1;33m [n] \e[0m No - Skip and run application locally \n"; \
47+
read -p " > " choice; \
48+
case "$$choice" in \
49+
[yY]) \
50+
echo -e "\033[1;32m ✔ Installing Python dependencies...\033[0m"; \
51+
$(MAKE) install-requirements; \
52+
$(MAKE) run; \
53+
break; \
54+
;; \
55+
[nN]|[sS]) \
56+
echo -e "\033[1;33m Skipping requirement installation. Starting the local server instead...\033[0m"; \
57+
$(MAKE) run; \
58+
break; \
59+
;; \
60+
*) \
61+
echo -e "\033[1;31m Invalid input: Please enter 'y', 'n', or 's' to proceed.\033[0m"; \
62+
;; \
63+
esac; \
64+
done
65+
66+
.PHONY: install-linux
67+
install-linux:
68+
@echo -e "\033[1;34m Installing Linux dependencies...\033[0m"; \
69+
sudo apt update && sudo apt install -y libmagic1 tesseract-ocr poppler-utils pkg-config
70+
71+
.PHONY: install-macos
72+
install-macos:
73+
@echo -e "\033[1;34m Installing macOS dependencies...\033[0m"; \
74+
brew update && brew install libmagic tesseract poppler pkg-config ghostscript ffmpeg automake autoconf
75+
76+
.PHONY: install-requirements
77+
install-requirements:
78+
@if [ "$$(uname)" = "Linux" ]; then $(MAKE) install-linux; \
79+
elif [ "$$(uname)" = "Darwin" ]; then $(MAKE) install-macos; \
80+
else echo "Unsupported OS. Exiting."; exit 1; fi; \
81+
82+
.PHONY: run
83+
run:
84+
@echo "Starting the local application server..."; \
85+
DISABLE_VENV=$(DISABLE_VENV) DISABLE_LOCAL_OLLAMA=$(DISABLE_LOCAL_OLLAMA) ./run.sh
86+
87+
.PHONY: setup-docker
88+
setup-docker:
89+
@rm -f .pyproject.hash
90+
@if [ ! -f .env ]; then \
91+
printf "\n\e[1;34m Copy .env.example to .env \e[0m"; \
92+
cp .env.example .env; \
93+
fi
94+
@echo -e "\033[1;34m Available Docker options:\033[0m"; \
95+
echo -e "\033[1;33m 1:\033[0m Run Docker containers with CPU support"; \
96+
echo -e "\033[1;33m 2:\033[0m Run Docker containers with GPU support"; \
97+
read -p "Enter your choice (1 = CPU, 2 = GPU, any other key to exit): " docker_choice; \
98+
case "$$docker_choice" in \
99+
1) $(MAKE) run-docker ;; \
100+
2) $(MAKE) run-docker-gpu ;; \
101+
*) echo -e "\033[1;34m Exiting without starting Docker.\033[0m" ;; \
102+
esac
103+
104+
.PHONY: run-docker
105+
run-docker:
106+
@echo -e "\033[1;34m Starting Docker container with CPU support...\033[0m";
107+
@docker-compose -f docker-compose.yml up --build
108+
109+
.PHONY: run-docker-gpu
110+
run-docker-gpu:
111+
@echo -e "\033[1;34m Starting Docker container with GPU support...\033[0m";
112+
@docker-compose -f docker-compose.gpu.yml -p text-extract-api-gpu up --build
113+
114+
.PHONY: clean
115+
clean:
116+
@echo "Cleaning project..."; \
117+
docker-compose down -v; \
118+
$(MAKE) clean-cache
119+
120+
.PHONY: clean-python-cache
121+
clean-cache:
122+
find . -type d -name '__pycache__' -exec rm -rf {} + && find . -type f -name '*.pyc' -delete
123+

‎README.md

Lines changed: 56 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -45,19 +45,54 @@ Before running the example see [getting started](#getting-started)
4545

4646
You might want to run the app directly on your machine for development purposes OR to use for example Apple GPUs (which are not supported by Docker at the moment).
4747

48+
### Prerequisites
49+
4850
To have it up and running please execute the following steps:
4951

5052
[Download and install Ollama](https://ollama.com/download)
5153
[Download and install Docker](https://www.docker.com/products/docker-desktop/)
5254

53-
If you are on Mac or just need to have your dependencies well organized, create a [virtual python env](https://docs.python.org/3/library/venv.html):
55+
> ### Setting Up Ollama on a Remote Host
56+
>
57+
> To connect to an external Ollama instance, set the environment variable: `OLLAMA_HOST=http://address:port`, e.g.:
58+
> ```bash
59+
> OLLAMA_HOST=http(s)://127.0.0.1:5000
60+
> ```
61+
>
62+
> If you want to disable the local Ollama model, use env `DISABLE_LOCAL_OLLAMA=1`, e.g.
63+
> ```bash
64+
> DISABLE_LOCAL_OLLAMA=1 make install
65+
> ```
66+
> **Note**: When local Ollama is disabled, ensure the required model is downloaded on the external instance.
67+
>
68+
> Currently, the `DISABLE_LOCAL_OLLAMA` variable cannot be used to disable Ollama in Docker. As a workaround, remove the `ollama` service from `docker-compose.yml` or `docker-compose.gpu.yml`.
69+
>
70+
> Support for using the variable in Docker environments will be added in a future release.
71+
72+
73+
### Clone the Repository
74+
75+
First, clone the repository and change current directory to it:
76+
77+
```sh
78+
git clone https://github.com/CatchTheTornado/text-extract-api.git
79+
cd text-extract-api
80+
```
81+
82+
### Setup with `Makefile`
83+
84+
Be default application create [virtual python env](https://docs.python.org/3/library/venv.html): `.venv`. You can disable this functionality on local setup by adding `DISABLE_VENV=1` before running script:
5485
5586
```bash
56-
python3 -m venv .venv
57-
source .venv/bin/activate
58-
# now you've got access to `python` and `pip` commands
87+
DISABLE_VENV=1 make install
88+
```
89+
90+
```bash
91+
DISABLE_VENV=1 make run
5992
```
6093
94+
### Manual setup
95+
6196
Configure environment variables:
6297
6398
```bash
@@ -67,6 +102,9 @@ cp .env.localhost.example .env.localhost
67102
You might want to just use the defaults - should be fine. After ENV variables are set, just execute:
68103
69104
```bash
105+
python3 -m venv .venv
106+
source .venv/bin/activate
107+
pip install -e .
70108
chmod +x run.sh
71109
run.sh
72110
```
@@ -84,7 +122,7 @@ python client/cli.py ocr_upload --file examples/example-mri.pdf --ocr_cache --pr
84122
To have multiple tasks runing at once - for concurrent processing please run the following command to start single worker process:
85123
86124
```bash
87-
celery -A main.celery worker --loglevel=info --pool=solo & # to scale by concurrent processing please run this line as many times as many concurrent processess you want to have running
125+
celery -A text_extract_api.tasks worker --loglevel=info --pool=solo & # to scale by concurrent processing please run this line as many times as many concurrent processess you want to have running
88126
```
89127
90128
## Online demo
@@ -98,7 +136,7 @@ Open in the browser: <a href="https://demo.doctractor.com/">demo.doctractor.com<
98136
```bash
99137
python3 -m venv .venv
100138
source .venv/bin/activate
101-
pip install -r app/requirements.txt
139+
pip install -e .
102140
export OCR_UPLOAD_URL=https://doctractor:Aekie2ao@api.doctractor.com/ocr/upload
103141
export RESULT_URL=https://doctractor:Aekie2ao@api.doctractor.com/ocr/result/
104142
@@ -129,7 +167,11 @@ git clone https://github.com/CatchTheTornado/text-extract-api.git
129167
cd text-extract-api
130168
```
131169
132-
### Setup environmental variables
170+
### Using `Makefile`
171+
You can use the `make install` and `make run` command to setup the Docker environment for `text-extract-api`. You can find the manual steps required to do so described below.
172+
173+
174+
### Manual setup
133175
134176
Create `.env` file in the root directory and set the necessary environment variables. You can use the `.env.example` file as a template:
135177
@@ -148,9 +190,9 @@ cp .env.example.localhost .env
148190
Then modify the variables inside the file:
149191
150192
```bash
151-
#APP_ENV=production # sets the app into prod mode, othervise dev mode with auto-reload on code changes
193+
#APP_ENV=production # sets the app into prod mode, otherwise dev mode with auto-reload on code changes
152194
REDIS_CACHE_URL=redis://localhost:6379/1
153-
STORAGE_PROFILE_PATH=/storage_profiles
195+
STORAGE_PROFILE_PATH=./storage_profiles
154196
LLAMA_VISION_PROMPT="You are OCR. Convert image to markdown."
155197
156198
# CLI settings
@@ -182,7 +224,7 @@ docker-compose up --build
182224
... for GPU support run:
183225
184226
```bash
185-
docker-compose -f docker-compose.gpu.yml up --build
227+
docker-compose -f docker-compose.gpu.yml -p text-extract-api-gpu up --build
186228
```
187229
188230
**Note:** While on Mac - Docker does not support Apple GPUs. In this case you might want to run the application natively without the Docker Compose please check [how to run it natively with GPU support](#getting-started)
@@ -206,15 +248,15 @@ If the on-prem is too much hassle [ask us about the hosted/cloud edition](mailto
206248
python3 -m venv .venv
207249
source .venv/bin/activate
208250
# now you've got access to `python` and `pip` within your virutal env.
209-
pip install -r app/requirements.txt # install main project requirements
251+
pip install -e . # install main project requirements
210252
```
211253
212254
213255
The project includes a CLI for interacting with the API. To make it work first run:
214256
215257
```bash
216258
cd client
217-
pip install -r requirements.txt
259+
pip install -e .
218260
```
219261
220262
@@ -263,7 +305,7 @@ python client/cli.py ocr_upload --file examples/example-mri.pdf --ocr_cache --pr
263305
```
264306
265307
The `ocr` command can store the results using the `storage_profiles`:
266-
- **storage_profile**: Used to save the result - the `default` profile (`/storage_profiles/default.yaml`) is used by default; if empty file is not saved
308+
- **storage_profile**: Used to save the result - the `default` profile (`./storage_profiles/default.yaml`) is used by default; if empty file is not saved
267309
- **storage_filename**: Outputting filename - relative path of the `root_path` set in the storage profile - by default a relative path to `/storage` folder; can use placeholders for dynamic formatting: `{file_name}`, `{file_extension}`, `{Y}`, `{mm}`, `{dd}` - for date formatting, `{HH}`, `{MM}`, `{SS}` - for time formatting
268310
269311
@@ -361,7 +403,7 @@ apiClient.uploadFile(formData).then(response => {
361403
- **ocr_cache**: Whether to cache the OCR result (true or false).
362404
- **prompt**: When provided, will be used for Ollama processing the OCR result
363405
- **model**: When provided along with the prompt - this model will be used for LLM processing
364-
- **storage_profile**: Used to save the result - the `default` profile (`/storage_profiles/default.yaml`) is used by default; if empty file is not saved
406+
- **storage_profile**: Used to save the result - the `default` profile (`./storage_profiles/default.yaml`) is used by default; if empty file is not saved
365407
- **storage_filename**: Outputting filename - relative path of the `root_path` set in the storage profile - by default a relative path to `/storage` folder; can use placeholders for dynamic formatting: `{file_name}`, `{file_extension}`, `{Y}`, `{mm}`, `{dd}` - for date formatting, `{HH}`, `{MM}`, `{SS}` - for time formatting
366408
367409
Example:

‎app/Dockerfile

Lines changed: 0 additions & 35 deletions
This file was deleted.

0 commit comments

Comments
 (0)