Merge pull request hotosm#151 from hotosm/feature/apply_feedback

Implemented API for apply/feedback
jeafreezy · Aug 14, 2023 · 17f974c · 17f974c
2 parents 3c99fa4 + 0940537
commit 17f974c
Show file tree

Hide file tree

Showing 4 changed files with 70 additions and 78 deletions.
diff --git a/backend/core/serializers.py b/backend/core/serializers.py
@@ -120,12 +120,18 @@ class Meta:
         # read_only_fields = ("created_at", "osm_id")
 
 
-class LabelFileSerializer(
-    GeoFeatureModelSerializer
-):  # serializers are used to translate models objects to api
+class LabelFileSerializer(GeoFeatureModelSerializer):
     class Meta:
         model = Label
-        geo_field = "geom"  # this will be used as geometry in order to create geojson api , geofeatureserializer will let you create api in geojson
+        geo_field = "geom"
+        # auto_bbox = True
+        fields = ("osm_id",)
+
+
+class FeedbackLabelFileSerializer(GeoFeatureModelSerializer):
+    class Meta:
+        model = FeedbackLabel
+        geo_field = "geom"
         # auto_bbox = True
         fields = ("osm_id",)
 
@@ -158,7 +164,19 @@ class FeedbackParamSerializer(serializers.Serializer):
     training_id = serializers.IntegerField(required=True)
     epochs = serializers.IntegerField(required=False)
     batch_size = serializers.IntegerField(required=False)
-    freeze_layers = serializers.BooleanField(required=False)
+    zoom_level = serializers.ListField(required=False)
+
+    def validate(self, data):
+        """
+        Check supplied data
+        """
+        if "zoom_level" in data:
+            for i in data["zoom_level"]:
+                if int(i) < 19 or int(i) > 21:
+                    raise serializers.ValidationError(
+                        "Zoom level Supported between 19-21"
+                    )
+        return data
 
 
 class PredictionParamSerializer(serializers.Serializer):

diff --git a/backend/core/tasks.py b/backend/core/tasks.py
@@ -10,8 +10,12 @@
 import ramp.utils
 import tensorflow as tf
 from celery import shared_task
-from core.models import AOI, Feedback, Label, Training
-from core.serializers import FeedbackFileSerializer, LabelFileSerializer
+from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training
+from core.serializers import (
+    FeedbackFileSerializer,
+    FeedbackLabelFileSerializer,
+    LabelFileSerializer,
+)
 from core.utils import bbox, download_imagery, get_start_end_download_coords
 from django.conf import settings
 from django.contrib.gis.db.models.aggregates import Extent
@@ -65,34 +69,32 @@ def train_model(
                 shutil.rmtree(training_input_image_source)
             os.makedirs(training_input_image_source)
             if feedback:
-                feedback_objects = Feedback.objects.filter(
-                    training__id=feedback,
-                    validated=True,
-                )
-                bbox_feedback = feedback_objects.aggregate(Extent("geom"))[
-                    "geom__extent"
-                ]
-                bbox_geo = GEOSGeometry(
-                    f"POLYGON(({bbox_feedback[0]} {bbox_feedback[1]},{bbox_feedback[2]} {bbox_feedback[1]},{bbox_feedback[2]} {bbox_feedback[3]},{bbox_feedback[0]} {bbox_feedback[3]},{bbox_feedback[0]} {bbox_feedback[1]}))"
-                )
-                print(training_input_image_source)
-                print(bbox_feedback)
-                with open(
-                    os.path.join(training_input_image_source, "labels_bbox.geojson"),
-                    "w",
-                    encoding="utf-8",
-                ) as f:
-                    f.write(bbox_geo.geojson)
+                try:
+                    aois = FeedbackAOI.objects.filter(training=feedback)
+                except FeedbackAOI.DoesNotExist:
+                    raise ValueError(
+                        f"No Feedback AOI is attached with supplied training id:{dataset_id}, Create AOI first",
+                    )
+
+            else:
+                try:
+                    aois = AOI.objects.filter(dataset=dataset_id)
+                except AOI.DoesNotExist:
+                    raise ValueError(
+                        f"No AOI is attached with supplied dataset id:{dataset_id}, Create AOI first",
+                    )
 
+            for obj in aois:
+                bbox_coords = bbox(obj.geom.coords[0])
                 for z in zoom_level:
                     zm_level = z
                     print(
                         f"""Running Download process for
-                                feedback {training_id} - dataset : {dataset_id} , zoom : {zm_level}"""
+                            aoi : {obj.id} - dataset : {dataset_id} , zoom : {zm_level}"""
                     )
                     try:
                         tile_size = DEFAULT_TILE_SIZE  # by default
-                        bbox_coords = list(bbox_feedback)
+
                         start, end = get_start_end_download_coords(
                             bbox_coords, zm_level, tile_size
                         )
@@ -107,49 +109,17 @@ def train_model(
                     except Exception as ex:
                         raise ex
 
-            else:
-                try:
-                    aois = AOI.objects.filter(dataset=dataset_id)
-                except AOI.DoesNotExist:
-                    raise ValueError(
-                        f"No AOI is attached with supplied dataset id:{dataset_id}, Create AOI first",
-                    )
-
-                for obj in aois:
-                    bbox_coords = bbox(obj.geom.coords[0])
-                    for z in zoom_level:
-                        zm_level = z
-                        print(
-                            f"""Running Download process for
-                                aoi : {obj.id} - dataset : {dataset_id} , zoom : {zm_level}"""
-                        )
-                        try:
-                            tile_size = DEFAULT_TILE_SIZE  # by default
-
-                            start, end = get_start_end_download_coords(
-                                bbox_coords, zm_level, tile_size
-                            )
-                            # start downloading
-                            download_imagery(
-                                start,
-                                end,
-                                zm_level,
-                                base_path=training_input_image_source,
-                                source=source_imagery,
-                            )
-                        except Exception as ex:
-                            raise ex
-
             ## -----------LABEL GENERATOR---------
-            logging.debug("Label Generator started")
+            logging.info("Label Generator started")
+            aoi_list = [r.id for r in aois]
+            logging.info(aoi_list)
+
             if feedback:
-                feedback_objects = Feedback.objects.filter(
-                    training__id=feedback,
-                    validated=True,
-                )
-                serialized_field = FeedbackFileSerializer(feedback_objects, many=True)
+                label = FeedbackLabel.objects.filter(feedback_aoi__in=aoi_list)
+                logging.info(label)
+
+                serialized_field = FeedbackLabelFileSerializer(label, many=True)
             else:
-                aoi_list = [r.id for r in aois]
                 label = Label.objects.filter(aoi__in=aoi_list)
                 serialized_field = LabelFileSerializer(label, many=True)
 

diff --git a/backend/core/views.py b/backend/core/views.py
@@ -244,8 +244,8 @@ class LabelViewSet(viewsets.ModelViewSet):
 
 
 class RawdataApiFeedbackView(APIView):
-    # authentication_classes = [OsmAuthentication]
-    # permission_classes = [IsOsmAuthenticated]
+    authentication_classes = [OsmAuthentication]
+    permission_classes = [IsOsmAuthenticated]
 
     def post(self, request, feedbackaoi_id, *args, **kwargs):
         """Downloads available osm data as labels within given feedback aoi
@@ -408,6 +408,15 @@ def run_task_status(request, run_id: str):
 
 
 class FeedbackView(APIView):
+    """Applies Associated feedback to Training Published Checkpoint
+
+    Args:
+        APIView (_type_): _description_
+
+    Returns:
+        _type_: _description_
+    """
+
     authentication_classes = [OsmAuthentication]
     permission_classes = [IsOsmAuthenticated]
 
@@ -421,12 +430,7 @@ def post(self, request, *args, **kwargs):
             training_id = deserialized_data["training_id"]
             training_instance = Training.objects.get(id=training_id)
 
-            unique_zoom_levels = (
-                Feedback.objects.filter(training__id=training_id, validated=True)
-                .values("zoom_level")
-                .distinct()
-            )
-            zoom_level = [z["zoom_level"] for z in unique_zoom_levels]
+            zoom_level = deserialized_data.get("zoom_level", [19, 20])
             epochs = deserialized_data.get("epochs", 20)
             batch_size = deserialized_data.get("batch_size", 8)
             instance = Training.objects.create(
@@ -448,7 +452,7 @@ def post(self, request, *args, **kwargs):
                 zoom_level=instance.zoom_level,
                 source_imagery=instance.source_imagery,
                 feedback=training_id,
-                freeze_layers=instance.freeze_layers,
+                freeze_layers=True,  # True by default for feedback
             )
             if not instance.source_imagery:
                 instance.source_imagery = instance.model.dataset.source_imagery

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -35,7 +35,7 @@ services:
     volumes:
       - ./backend:/app
       - ${RAMP_HOME}:/RAMP_HOME
-      # - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE
+      - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE
     depends_on:
       - redis
       - postgres
@@ -55,7 +55,7 @@ services:
     volumes:
       - ./backend:/app
       - ${RAMP_HOME}:/RAMP_HOME
-      # - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE
+      - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE
     depends_on:
       - backend-api
       - redis