Receptionist fixes (#260)

Co-authored-by: Zoe <ezoe013@gmail.com>

Author: haiwei-luo

common/vision/lasr_vision_clip/nodes/vqa

@@ -31,7 +31,7 @@ class VqaService:
         """
         possible_answers = request.possible_answers
         answer, cos_score, annotated_img = query_image(
-            self._model, request.image_raw, possible_answers, annotate=True
+            request.image_raw, self._model, possible_answers, annotate=True
         )
 
         self._debug_pub.publish(annotated_img)

common/vision/lasr_vision_feature_extraction/nodes/service

@@ -48,9 +48,7 @@ def detect(
         head_frame,
         torso_frame,
         full_frame,
-        head_mask,
-        torso_mask,
-        request.image_raw,
+        image_raw=request.image_raw,
         cloth_predictor=cloth_predictor,
     )
     response = TorchFaceFeatureDetectionDescriptionResponse()

common/vision/lasr_vision_feature_extraction/src/lasr_vision_feature_extraction/__init__.py

@@ -545,8 +545,6 @@ def predict_frame(
     head_frame,
     torso_frame,
     full_frame,
-    head_mask,
-    torso_mask,
     cloth_predictor,
     image_raw,
     clip_service: rospy.ServiceProxy = rospy.ServiceProxy(
@@ -568,10 +566,7 @@ def predict_frame(
     ).describe()
 
     # 0.5 for True, -0.5 for False
-    rst_person = {
-        "glasses": -0.5,
-        "hat": -0.5,
-    }
+    rst_person = {"glasses": -0.5, "hat": -0.5, "hair_shape": "short hair"}
 
     glasses_query = VqaRequest(
         possible_answers=["A person wearing glasses", "A person not wearing glasses"],
@@ -583,14 +578,24 @@ def predict_frame(
         image_raw=image_raw,
     )
 
+    hair_query = VqaRequest(
+        possible_answers=["A person with short hair", "A person with long hair"],
+        image_raw=image_raw,
+    )
+
     glasses_response = clip_service(glasses_query)
-    hat_response = clip_service(hat_query)
 
     if glasses_response.answer == "A person wearing glasses":
         rst_person["glasses"] = 0.5
+    hat_response = clip_service(hat_query)
     if hat_response.answer == "A person wearing a hat":
         rst_person["hat"] = 0.5
 
+    hair_response = clip_service(hair_query)
+
+    if hair_response.answer == "A person with long hair":
+        rst_person["hair_shape"] = "long hair"
+
     result = {
         **rst_person,
         **rst_cloth,

tasks/receptionist/launch/setup.launch

@@ -19,8 +19,7 @@
     <node pkg="lasr_vision_feature_extraction" type="service" name="torch_service" output="screen"/>
     <node pkg="lasr_vision_clip" name="clip_service" type="vqa" output="screen"/>
     <arg name="debug" default="true" />
-    <node pkg="lasr_vision_bodypix" type="mask_service.py" name="bodypix_service" output="screen" args="--debug $(arg debug)"/>
-    <node pkg="lasr_vision_bodypix" type="keypoint_service.py" name="bodypix__keypoint_service" output="screen" args="--debug $(arg debug)"/>
+    <node pkg="lasr_vision_bodypix" type="bodypix_services.py" name="bodypix_service" output="screen" args="--debug $(arg debug)"/>
     <node pkg="lasr_vision_deepface" type="service" name="deepface_service" output="screen"/>
 
 </launch>

tasks/receptionist/scripts/main.py

@@ -55,14 +55,14 @@
     rospy.loginfo(sofa_area.area)
     rospy.loginfo(list(sofa_area.exterior.coords))
     sofa_area_publisher.publish(
-    PolygonStamped(
-        polygon=Polygon(
-            points=[Point(x=x, y=y, z=0.0) for (x, y) in sofa_area.exterior.coords]
-        ),
-        header=Header(frame_id="map"),
-    )
+        PolygonStamped(
+            polygon=Polygon(
+                points=[Point(x=x, y=y, z=0.0) for (x, y) in sofa_area.exterior.coords]
+            ),
+            header=Header(frame_id="map"),
+        )
     )
-    #rospy.spin()
+    # rospy.spin()
     # assert sofa_area.is_valid, "Sofa area is not valid"
 
     sofa_point = Point(**sofa_point_param)

tasks/receptionist/src/receptionist/states/introduce.py

@@ -48,17 +48,19 @@ def stringify_guest_data(
         },
     )
 
+    relevant_guest_data["attributes"]["has_hair"] = 0.5
+
     guest_str = f"{relevant_guest_data['name']}, their favourite drink is {relevant_guest_data['drink']}. "
 
     if not relevant_guest_data["detection"] or not describe_features:
         return guest_str
 
     filtered_attributes = {}
-    # filtered_attributes["hair"] = {
-    #     "confidence": relevant_guest_data["attributes"]["has_hair"],
-    #     "hair_shape": relevant_guest_data["attributes"]["hair_shape"],
-    #     "hair_colour": relevant_guest_data["attributes"]["hair_colour"],
-    # }
+    filtered_attributes["hair"] = {
+        "confidence": relevant_guest_data["attributes"]["has_hair"],
+        "hair_shape": relevant_guest_data["attributes"]["hair_shape"],
+        # "hair_colour": relevant_guest_data["attributes"]["hair_colour"],
+    }
 
     most_confident_clothes = find_most_confident_clothes(
         relevant_guest_data,
@@ -117,8 +119,8 @@ def stringify_guest_data(
 
         if attribute_name == "hair":
             hair_shape = attribute_value["hair_shape"]
-            hair_colour = attribute_value["hair_colour"]
-            guest_str += f"They have {hair_shape} and {hair_colour}. "
+            # hair_colour = attribute_value["hair_colour"]
+            guest_str += f"They have {hair_shape}. "
         elif attribute_name == "facial_hair":
             if confidence < 0:
                 guest_str += "They don't have facial hair. "