diff --git a/.github/workflows/backend_build.yml b/.github/workflows/backend_build.yml index 9744c901..ff506399 100644 --- a/.github/workflows/backend_build.yml +++ b/.github/workflows/backend_build.yml @@ -51,15 +51,17 @@ jobs: - name: Install gdal run: | - sudo apt-get -y install gdal-bin libgdal-dev python3-gdal && sudo apt-get -y autoremove && sudo apt-get clean - pip install GDAL==$(gdal-config --version) --global-option=build_ext --global-option="-I/usr/include/gdal" - + sudo apt-get update && sudo apt-get -y install gdal-bin libgdal-dev python3-gdal && sudo apt-get -y autoremove && sudo apt-get clean + pip install GDAL==$(gdal-config --version) - name: Install ramp dependecies run: | cd ramp-code && cd colab && make install + - name: Install tensorflow + run: pip install tensorflow==2.9.2 + - name: Install fair utilities - run: pip install hot-fair-utilities==1.0.41 + run: pip install hot-fair-utilities - name: Install Psycopg2 run: | diff --git a/Readme.md b/Readme.md index 93dfe360..a8c6db0c 100644 --- a/Readme.md +++ b/Readme.md @@ -25,29 +25,3 @@ To eliminate model biases, fAIr is built to work with the local communities and See below a suggested product roadmap [subject to change] that provides high-level overview for planned work. ![image](https://user-images.githubusercontent.com/98902727/218769416-b3c71d3b-7c20-4d40-ab1e-88442d06445d.png) -# General Workflow of fAIr - -![fAIr1](https://github.com/hotosm/fAIr/assets/97789856/01c0e3b6-a00c-439d-a2ed-1c14b62e6364) - -1. Project Area by tha project manager and imagery from Open Areal Map is submitted to the task manager which then is sent to Open Street Map after undergoing the process of manual mapping and validation. -2. Local dataset(created using the imagery and raw API data as inputs) is trained and local model is created and trained. -3. It is then validated , published , mapped and pushed back into Open Street Map. -4. Finally according to the feedback , the published model is sent for improvement and training . -
- -# fAIr Architecture -![fAIr2](https://github.com/hotosm/fAIr/assets/97789856/63394f65-ce0d-4a3d-8683-7455f14fb366) - -1. Third party extensions are sent to fAIr backend which then generates data for OSM raw data API , osmconflator and geoson2osm xml . -2. Data from fAIr backend is sent to fAIr utilities . The backend is using s separate library we call it fAIr-utilities to handle: - - 1. Data preparation for the models - 2. models trainings - 3. inference process - 4. post processing (converting the predicted features to geo data) -3. The public API is then sent to the fAIr frontend. - - - - - diff --git a/backend/Dockerfile b/backend/Dockerfile index c8f1cbbb..b7e40872 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -28,6 +28,6 @@ RUN pip install setuptools --upgrade COPY requirements.txt requirements.txt RUN pip install -r requirements.txt -RUN mkdir /app + WORKDIR /app COPY . /app \ No newline at end of file diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml deleted file mode 100644 index 34f254a4..00000000 --- a/backend/docker-compose.yml +++ /dev/null @@ -1,71 +0,0 @@ -version: '3.8' - -services: - - postgres: - restart: always - image: postgis/postgis - container_name: pgsql - environment: - - POSTGRES_DB=ai - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=admin - ports: - - '5434:5432' - volumes: - - ./postgres-data:/var/lib/postgresql/data - - redis: - image: redis - container_name: redis - ports: - - "6379:6379" - - app: - build: . - container_name: api - command: python manage.py runserver 0.0.0.0:8000 - deploy: - resources: - reservations: - devices: - - driver: nvidia - capabilities: [gpu] - ports: - - 8000:8000 - volumes: - - .:/app - # - C:/Users/ommrn/Documents/HOT/fAIr-utilities:/RAMP_HOME - depends_on: - - redis - - postgres - - worker: - build: . - container_name: worker - command: celery -A aiproject worker --loglevel=INFO - deploy: - resources: - reservations: - devices: - - driver: nvidia - capabilities: [gpu] - volumes: - - .:/app - # - C:/Users/ommrn/Documents/HOT/fAIr-utilities:/RAMP_HOME - depends_on: - - app - - redis - - postgres - - - worker-dashboard: - build: . - container_name: flower - command: celery --broker=redis://redis:6379// flower --address=0.0.0.0 --port=5000 - ports: - - 5500:5000 - depends_on: - - app - - redis - - worker \ No newline at end of file diff --git a/backend/docker_sample_env b/backend/docker_sample_env index 21bf789c..c57331a2 100644 --- a/backend/docker_sample_env +++ b/backend/docker_sample_env @@ -1,17 +1,19 @@ +DEBUG=True SECRET_KEY=yl2w)c0boi_ma-1v5)935^2#&m*r!1s9z9^*9e5co^08_ixzo6 DATABASE_URL=postgis://postgres:admin@pgsql:5432/ai -EXPORT_TOOL_API_URL=http://44.203.33.53:8000/raw-data/current-snapshot/ -CORS_ALLOWED_ORIGINS=http://localhost:3000 -GDAL_LIBRARY_PATH='' -MAXAR_CONNECT_ID= +EXPORT_TOOL_API_URL=https://galaxy-api.hotosm.org/v1/raw-data/current-snapshot/ +CORS_ALLOWED_ORIGINS=http://127.0.0.1:3000 OSM_CLIENT_ID= OSM_CLIENT_SECRET= OSM_URL=https://www.openstreetmap.org OSM_SCOPE=read_prefs -OSM_LOGIN_REDIRECT_URI=http://127.0.0.1:8000/api/v1/auth/callback/ +OSM_LOGIN_REDIRECT_URI=http://127.0.0.1:3000/authenticate/ OSM_SECRET_KEY= CELERY_BROKER_URL="redis://redis:6379/0" CELERY_RESULT_BACKEND="redis://redis:6379/0" -RAMP_HOME="/home/kshitij/hotosm/fAIr-utilities" -TRAINING_WORKSPACE="/home/kshitij/hotosm/fAIr/backend/training" -TESTING_TOKEN= \ No newline at end of file +RAMP_HOME="/RAMP_HOME" +TRAINING_WORKSPACE="/TRAINING_WORKSPACE" + +TESTING_TOKEN= +GDAL_LIBRARY_PATH='' +MAXAR_CONNECT_ID= \ No newline at end of file diff --git a/backend/requirements.txt b/backend/requirements.txt index b270b66f..6cae52f9 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -17,5 +17,5 @@ django_celery_results==2.4.0 flower==1.2.0 validators==0.20.0 gpxpy==1.5.0 -hot-fair-utilities==1.0.51 +hot-fair-utilities geojson2osm==0.0.1 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..263647eb --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,84 @@ +version: "3.8" + +services: + postgres: + restart: always + image: postgis/postgis + container_name: pgsql + environment: + - POSTGRES_DB=ai + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=admin + ports: + - "5434:5432" + + redis: + image: redis + container_name: redis + ports: + - "6379:6379" + + backend-api: + build: + context: ./backend + dockerfile: Dockerfile + container_name: api + command: python manage.py runserver 0.0.0.0:8000 + deploy: + resources: + reservations: + devices: + - driver: nvidia + capabilities: [gpu] + ports: + - 8000:8000 + volumes: + - ./backend:/app + - ${RAMP_HOME}:/RAMP_HOME + - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE + depends_on: + - redis + - postgres + + backend-worker: + build: + context: ./backend + dockerfile: Dockerfile + container_name: worker + command: celery -A aiproject worker --loglevel=INFO + deploy: + resources: + reservations: + devices: + - driver: nvidia + capabilities: [gpu] + volumes: + - ./backend:/app + - ${RAMP_HOME}:/RAMP_HOME + - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE + depends_on: + - backend-api + - redis + - postgres + + worker-dashboard: + image: mher/flower + container_name: flower + command: celery --broker=redis://redis:6379// flower --address=0.0.0.0 --port=5000 + ports: + - 5500:5000 + depends_on: + - backend-api + - redis + - backend-worker + + frontend: + build: + context: ./frontend + dockerfile: Dockerfile.frontend + container_name: frontend + command: npm start -- --host 0.0.0.0 --port 3000 + ports: + - 3000:3000 + depends_on: + - backend-api diff --git a/docs/Docker-installation.md b/docs/Docker-installation.md new file mode 100644 index 00000000..7fdd8d01 --- /dev/null +++ b/docs/Docker-installation.md @@ -0,0 +1,126 @@ +Docker Compose is created with redis , worker , postgis database , api and frontend all in one making it easy for development . For production it is not recommended + +## [DEV] Installation With Docker + +1. Clone Repo + + ``` + git clone https://github.com/hotosm/fAIr.git + ``` + +2. Get Docker Compose Installed + + If docker is not installed , Install it from [here](https://docs.docker.com/engine/install/) + ``` + docker compose version + ``` + +3. Check your Graphics + + fAIr works best with graphics card. It is highly recommended to use graphics card . It might not work with CPU only . Nvidia Graphics cards are tested + + You need to make sure you can see your graphics card details and can be accessed through docker by installing necessary drivers + + By following command you can see your graphics and graphics driver details + ``` + nvidia-smi + ``` + +4. Clonse Base Model and Create RAMP_HOME + + - Create a new folder called RAMP , outside fAIr + + ``` + mkdir ramp + ``` + - Download BaseModel Checkpoint from [here](https://drive.google.com/file/d/1wvJhkiOrSlHmmvJ0avkAdu9sslFf5_I0/view?usp=sharing) + + ``` + pip install gdown + gdown --fuzzy https://drive.google.com/file/d/1wvJhkiOrSlHmmvJ0avkAdu9sslFf5_I0/view?usp=sharing + ``` + - Clone Ramp Code + + ``` + git clone https://github.com/kshitijrajsharma/ramp-code-fAIr.git ramp-code + ``` + - Unzip downloaded basemodel and move inside ramp-code/ramp + + ``` + unzip checkpoint.tf.zip -d ramp-code/ramp + ``` + - Export Env variable for RAMP_HOME + Grab the file path of folder we created earlier ```ramp``` and export it as env variable + ``` + export RAMP_HOME=/home/YOUR_RAMP_LOCATION + ``` + eg : export RAMP_HOME=/home/kshitij/ramp + + - Export ```TRAINING_WORKSPACE``` Env + Training workspace is the folder where fAIr will store its training files + for eg : + ``` + export TRAINING_WORKSPACE=/home/kshitij/hotosm/fAIr/trainings + ``` + +5. Register your Local setup to OSM + + - Go to [OpenStreetMap](https://www.openstreetmap.org/) , Login/Create Account + - Click on your Profile and Hit ```My Settings``` + - Navigate to ```Oauth2 Applications``` + - Register new application + - Check permissions for ```Read user preferences``` and Redirect URI to be ```http://127.0.0.1:3000/authenticate/``` , Give it name as ```fAIr Dev Local``` + - You will get ```OSM_CLIENT_ID``` , ```OSM_CLIENT_SECRET``` Copy them + +6. Create Env variables + - Create a file ```.env``` in backend with [docker_sample_env](../backend/docker_sample_env) content + ``` + cp docker_sample_env .env + ``` + - Fill out the details of ```OSM_CLIENT_ID``` &```OSM_CLIENT_SECRET``` in .env file and generate a unique key & paste it to ```OSM_SECRET_KEY``` (It can be random for dev setup) + + Leave rest of the items as it is unless you know what you are doing + + - Create ```.env``` in /frontend + ``` + cp .env_sample .env + ``` + You can leave it as it is for dev setup + +7. Build & Run containers + + ``` + docker compose build + ``` + + ``` + docker compose up + ``` + +8. Run Migrations + + See Running containers grab their ID and launch bash to make migrations (This is needed for the first time to set database) + + docker container ps + + Grab Container ID & Open Bash + + docker exec -it CONTAINER_ID bash + + + Once Bash is promoted hit following commands + + python manage.py makemigrations + python manage.py makemigrations login + python manage.py makemigrations core + python manage.py migrate + +9. Play and Develop + + Restart containers + + ``` + docker compose restart + ``` + + Frontend will be available on 5000 port , Backend will be on 8000 , Flower will be on 5500 \ No newline at end of file diff --git a/frontend/.env sample b/frontend/.env_sample similarity index 75% rename from frontend/.env sample rename to frontend/.env_sample index 5964a03f..c0447728 100644 --- a/frontend/.env sample +++ b/frontend/.env_sample @@ -1,4 +1,4 @@ REACT_APP_CONNECT_ID= REACT_APP_TM_API=https://tasking-manager-tm4-production-api.hotosm.org/api/v2/projects/PROJECT_ID/tasks/ REACT_APP_ENV=Dev -REACT_APP_API_BASE=http://localhost:8000/api/v1 \ No newline at end of file +REACT_APP_API_BASE=http://127.0.0.1:8000/api/v1 \ No newline at end of file diff --git a/frontend/Dockerfile.frontend b/frontend/Dockerfile.frontend new file mode 100644 index 00000000..c852d1e4 --- /dev/null +++ b/frontend/Dockerfile.frontend @@ -0,0 +1,15 @@ + +FROM node:16.14.2 + +WORKDIR /app + + +COPY . /app + + +RUN npm install --legacy-peer-deps + + +# RUN npm run build + +# EXPOSE 3000 diff --git a/frontend/src/components/Layout/TrainingDS/DatasetEditor/TileServerList.js b/frontend/src/components/Layout/TrainingDS/DatasetEditor/TileServerList.js index 3f1846cf..f6889205 100644 --- a/frontend/src/components/Layout/TrainingDS/DatasetEditor/TileServerList.js +++ b/frontend/src/components/Layout/TrainingDS/DatasetEditor/TileServerList.js @@ -63,6 +63,10 @@ const TileServerList = (props) => { setInputError(false); const res = await axios.get(url.replace("/{z}/{x}/{y}", "")); + if (!res) { + setInputError("Invalid OAM Link"); + return; + } if (res.error) setInputError(res.error.response.statusText); props.addImagery(res.data, url); @@ -74,7 +78,8 @@ const TileServerList = (props) => { return res.data; } catch (e) { console.log("isError"); - setInputError(e); + // setInputError(e); + setInputError("Invalid OAM Link"); } finally { setLoading(false); } diff --git a/frontend/src/components/Layout/TrainingDS/DatasetNew/DatasetNew.js b/frontend/src/components/Layout/TrainingDS/DatasetNew/DatasetNew.js index d914bb24..de4aeaf5 100644 --- a/frontend/src/components/Layout/TrainingDS/DatasetNew/DatasetNew.js +++ b/frontend/src/components/Layout/TrainingDS/DatasetNew/DatasetNew.js @@ -12,6 +12,7 @@ const DatasetNew = (props) => { const navigate = useNavigate(); const [oamURL, setOAMURL] = useState(); const { accessToken } = useContext(AuthContext); + const regUrl = /^(https?|chrome):\/\/[^\s$.?#].[^\s]*$/; const saveDataset = async () => { try { const body = { @@ -118,7 +119,13 @@ const DatasetNew = (props) => { console.log("save"); mutate(); }} - disabled={DSName.trim() === "" || !oamURL || isLoading} + disabled={ + !oamURL || + !regUrl.test(oamURL) || + DSName.trim() === "" || + !oamURL.endsWith("{z}/{x}/{y}") || + isLoading + } > Create Training Dataset