Merge pull request #147 from Avanade/143-Refactor-/-expand-Realsense-…

…module

143 refactor / expand realsense module

control-agent/text-to-speech/realsense.py

@@ -6,41 +6,43 @@
 import pyrealsense2 as rs
 
 
-def take_photo():
-    pipeline = rs.pipeline()
-    config = rs.config()
-
-    config.enable_stream(rs.stream.color, 1280, 720, rs.format.bgr8, 30)
-
-    profile = pipeline.start(config)
-    # Get the sensor once at the beginning. (Sensor index: 1)
-    sensor = pipeline.get_active_profile().get_device().query_sensors()[1]
-    # Set the exposure anytime during the operation
-    sensor.set_option(rs.option.exposure, 80.000)
-
-    # We will be removing the background of objects more than
-    #  clipping_distance_in_meters meters away
-    clipping_distance_in_meters = 1.5
-
-    time.sleep(2)  # or the picture will look green
-
-    align_to = rs.stream.color
-    align = rs.align(align_to)
-
-    frames = pipeline.wait_for_frames()
-
-    aligned_frames = align.process(frames)
-    color_frame = aligned_frames.get_color_frame()
-
-    color_image = np.asanyarray(color_frame.get_data())
-    r_color_image = np.rot90(color_image, 3)
-
-    bimg = Image.fromarray(r_color_image, "RGB")
-    b, g, r = bimg.split()
-    img = Image.merge("RGB", (r, g, b))
-    img.save("test.png")
-
-    img_byte = io.BytesIO()
-    img.save(img_byte, format="PNG")
-
-    return img_byte.getvalue()
+class Realsense:
+    def __init__(self):
+        """initiates the camera for use"""
+        self.pipeline = rs.pipeline()
+        config = rs.config()
+        config.enable_stream(rs.stream.color, 1280, 720, rs.format.bgr8, 30)
+        config.enable_stream(rs.stream.depth, 1280, 720, rs.format.z16, 30)
+
+        self.pipeline.start(config)
+        self.align = rs.align(rs.stream.color)
+
+    def get_frame(self):
+        """Take a depth and colour frame"""
+        frames = self.pipeline.wait_for_frames()
+        aligned_frames = self.align.process(frames)
+        depth_frame = aligned_frames.get_depth_frame()
+        colour_frame = aligned_frames.get_color_frame()
+
+        if not depth_frame or not colour_frame:
+            return False, None, None
+
+        depth_image = np.asanyarray(depth_frame.get_data())
+        colour_image = np.asanyarray(colour_frame.get_data())
+
+        return True, colour_image, depth_image
+
+    def take_colour_photo(self):
+        status, colour_image, depth_image = self.get_frame()
+        r_colour_image = np.rot90(colour_image, 3)
+
+        # TODO: check why image layers come back incorrectly
+        bimg = Image.fromarray(r_colour_image, "RGB")
+        b, g, r = bimg.split()
+        img = Image.merge("RGB", (r, g, b))
+        img.save("test.png")
+
+        img_byte = io.BytesIO()
+        img.save(img_byte, format="PNG")
+
+        return img_byte.getvalue()

control-agent/text-to-speech/speech-detection.py

@@ -8,7 +8,7 @@
 
 import speech
 
-import realsense
+from realsense import *
 
 LUIS_CONFIDENCE_LIMIT = 0.7
 PATH_TO_COMMANDS = "/home/hello-robot/Chatbot"
@@ -207,9 +207,9 @@ def selfie_intent():
     """Intent response takes a photo on realsense and uplaod
     to azure blob"""
     speech.speak("Smile. 3, 2, 1.")
-    image = realsense.take_photo()
+    image = realsense.take_colour_photo()
     speech.speak("click")
-    speech.uploadBlob(image)
+    speech.upload_blob(image)
     speech.speak("I've saved that to Azure for you, check it out in my blob storage")
 
 
@@ -274,7 +274,7 @@ def vision_intent():
     using Azure computer vision services"""
     speech.speak("I'm looking")
 
-    image = realsense.take_photo()
+    image = realsense.take_colour_photo()
 
     INDIVIDUALS_LIST = [
         "person",
@@ -334,19 +334,6 @@ def vision_intent():
         speech.speak("I can see" + str(result["description"]["captions"][0]["text"]))
 
 
-def joke_intent():
-    """Reads aloud a joke, as an example of a data
-    lookup with an external API"""
-    speech.speak("ok - let me think of one")
-
-    url = "https://v2.jokeapi.dev/joke/Programming"
-    query = {"type": "twopart", "blacklistFlags": "nsfw"}
-    response = requests.get(url, params=query)
-
-    speech.speak(response.json()["setup"])
-    speech.speak(response.json()["delivery"])
-
-
 def intent_handler(intent):
     """Handles the intent responces and calls the associated fucntions"""
 
@@ -380,8 +367,6 @@ def intent_handler(intent):
         arm_intent(intent)
     elif intent.intent_id == "Wrist":
         wrist_intent()
-    elif intent.intent_id == "Joke":
-        joke_intent()
 
     return True
 
@@ -390,9 +375,13 @@ def intent_handler(intent):
 run = True
 # warm up confirmation
 speech.speak("starting up")
+# start camera
+realsense = Realsense()
 
 while run == True:
 
     intent = recognize_intent()
 
     run = intent_handler(intent)
+
+    continue

control-agent/text-to-speech/speech.py

@@ -260,7 +260,7 @@ def identify_face(face_id):
         return "a stranger"
 
 
-def uploadBlob(blob_bytes):
+def upload_blob(blob_bytes):
     """upload a blob to the Azure storage account named as a timestamp
     Keyword arguments:
     blobBytes -- bytes data of a file to uplaod