Merge pull request #70 from CS3219-AY2425S1/titus/add-message-queue

feat: add message queue
CS3219-AY2425S1 · Nov 11, 2024 · 7d931ee · 7d931ee
2 parents 439b410 + d75215c
commit 7d931ee
Show file tree

Hide file tree

Showing 19 changed files with 412 additions and 121 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -19,7 +19,7 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
-      
+
       - name: Set up .env
         env:
           QUESTION_FIREBASE_CREDENTIAL_PATH: ${{ vars.QUESTION_SERVICE_FIREBASE_CREDENTIAL_PATH }}
@@ -30,7 +30,7 @@ jobs:
           echo "FIREBASE_CREDENTIAL_PATH=$QUESTION_FIREBASE_CREDENTIAL_PATH" >> .env
           echo "JWT_SECRET=$JWT_SECRET" >> .env
           echo "EXECUTION_SERVICE_URL=$EXECUTION_SERVICE_URL" >> .env
-      
+
       - name: Set up credentials
         env:
           QUESTION_FIREBASE_JSON: ${{ secrets.QUESTION_SERVICE_FIREBASE_CREDENTIAL }}
@@ -41,8 +41,8 @@ jobs:
 
       - name: Setup Go
         uses: actions/setup-go@v5
-        with: 
-          go-version: '1.23.x'
+        with:
+          go-version: "1.23.x"
 
       - name: Install Go dependencies
         run: |
@@ -51,7 +51,7 @@ jobs:
 
       - name: Install firebase tools
         run: curl -sL firebase.tools | bash
-      
+
       - name: Run Go tests with Firebase emulator
         run: firebase emulators:exec --only firestore 'cd ./apps/question-service; go test -v ./tests'
 
@@ -66,11 +66,11 @@ jobs:
         run: |
           cd ./apps/frontend
           cp .env.example  .env
-      
+
       - name: Set up Node.js
         uses: actions/setup-node@v2
         with:
-          node-version: '22'
+          node-version: "22"
 
       - name: Install pnpm
         run: npm i -g pnpm
@@ -117,6 +117,7 @@ jobs:
           EXECUTION_SERVICE_PORT: ${{ vars.EXECUTION_SERVICE_PORT }}
           MATCHING_SERVICE_TIMEOUT: ${{ vars.MATCHING_SERVICE_TIMEOUT }}
           REDIS_URL: ${{ vars.REDIS_URL }}
+          RABBITMQ_URL: ${{ vars.RABBITMQ_URL }}
           QUESTION_SERVICE_GRPC_URL: ${{ vars.QUESTION_SERVICE_GPRC_URL }}
         run: |
           cd ./apps/frontend
@@ -147,11 +148,13 @@ jobs:
           cd ../history-service
           echo "FIREBASE_CREDENTIAL_PATH=$HISTORY_FIREBASE_CREDENTIAL_PATH" >> .env
           echo "PORT=$HISTORY_SERVICE_PORT" >> .env
-          
+          echo "RABBITMQ_URL=$RABBITMQ_URL" >> .env
+
           cd ../execution-service
           echo "FIREBASE_CREDENTIAL_PATH=$EXECUTION_FIREBASE_CREDENTIAL_PATH" >> .env
           echo "PORT=$EXECUTION_SERVICE_PORT" >> .env
           echo "HISTORY_SERVICE_URL=$HISTORY_SERVICE_URL" >> .env
+          echo "RABBITMQ_URL=$RABBITMQ_URL" >> .env
 
           cd ../signalling-service
           echo "PORT=$SIGNALLING_SERVICE_PORT" >> .env
@@ -170,7 +173,7 @@ jobs:
 
           cd ../history-service
           echo "$HISTORY_FIREBASE_JSON" > "./$HISTORY_FIREBASE_CREDENTIAL_PATH"
-          
+
           cd ../execution-service
           echo "$EXECUTION_FIREBASE_JSON" > "./$EXECUTION_FIREBASE_CREDENTIAL_PATH"
 

diff --git a/apps/docker-compose.yml b/apps/docker-compose.yml
@@ -68,7 +68,9 @@ services:
       - apps_network
     volumes:
       - ./history-service:/history-service
-
+    depends_on:
+      - rabbitmq
+
   signalling-service:
     build:
       context: ./signalling-service
@@ -95,6 +97,8 @@ services:
     volumes:
       - ./execution-service:/execution-service
       - /var/run/docker.sock:/var/run/docker.sock
+    depends_on:
+      - rabbitmq
 
   redis:
     image: redis:latest
@@ -104,6 +108,19 @@ services:
       - 6379:6379
     container_name: redis-container
 
+  rabbitmq:
+    image: rabbitmq:3-management
+    networks:
+      - apps_network
+    ports:
+      - 5672:5672 # Port for RabbitMQ message broker
+      - 15672:15672 # Port for RabbitMQ Management UI
+    environment:
+      RABBITMQ_DEFAULT_USER: guest
+      RABBITMQ_DEFAULT_PASS: guest
+    volumes:
+      - rabbitmq_data:/var/lib/rabbitmq
+
   python-sandbox:
     build:
       context: ./execution-service/execution/python
@@ -114,3 +131,8 @@ services:
 
 networks:
   apps_network:
+
+volumes:
+  # Mounts a volume for RabbitMQ data persistence.
+  # This ensures that data is not lost when the container is restarted or removed.
+  rabbitmq_data:
diff --git a/apps/execution-service/.env.example b/apps/execution-service/.env.example
@@ -3,6 +3,8 @@ PORT=8083
 
 # If you are NOT USING docker, use the below variables
 # HISTORY_SERVICE_URL=http://localhost:8082/
+# RABBITMQ_URL=amqp://guest:guest@localhost:5672/
 
 # If you are USING docker, use the below variables
 HISTORY_SERVICE_URL=http://history-service:8082/
+RABBITMQ_URL=amqp://guest:guest@rabbitmq:5672/
diff --git a/apps/execution-service/README.md b/apps/execution-service/README.md
@@ -20,7 +20,32 @@ go run main.go
 
 The server will be available at http://localhost:8083.
 
-## Running the Application via Docker
+### Setting up message queue with RabbitMQ
+
+A message queue is used to pass submission results asynchronously from the execution-service to the history-service.
+
+1. In order to do so, we can run the following command to set up a docker container for RabbitMQ:
+
+```bash
+docker run -d --name rabbitmq -p 5672:5672 -p 15672:15672 rabbitmq:3-management
+```
+
+2. Then we can run the execution-service:
+
+```bash
+go run main.go
+```
+
+3. We can run the history-service by changing our directory and running the same command:
+
+```bash
+cd ../history-service
+go run main.go
+```
+
+To view more details on the RabbitMQ queue, we can go to `localhost:15672`, and login using `guest` as the username and password.
+
+### Running the Application via Docker
 
 To run the application via Docker, run the following command:
 
@@ -74,10 +99,10 @@ The following json format will be returned:
 
 ```json
 [
-   {
-      "input":"hello",
-      "expected":"olleh"
-   }
+  {
+    "input": "hello",
+    "expected": "olleh"
+  }
 ]
 ```
 
@@ -98,16 +123,16 @@ The following json format will be returned:
 
 ```json
 {
-  "visibleTestResults":[
+  "visibleTestResults": [
     {
-      "input":"hello",
-      "expected":"olleh",
-      "actual":"olleh",
-      "passed":true,
-      "error":""
+      "input": "hello",
+      "expected": "olleh",
+      "actual": "olleh",
+      "passed": true,
+      "error": ""
     }
   ],
-  "customTestResults":null
+  "customTestResults": null
 }
 ```
 
@@ -127,29 +152,29 @@ The following json format will be returned:
 
 ```json
 {
-  "visibleTestResults":[
+  "visibleTestResults": [
     {
-      "input":"hello",
-      "expected":"olleh",
-      "actual":"olleh",
-      "passed":true,
-      "error":""
+      "input": "hello",
+      "expected": "olleh",
+      "actual": "olleh",
+      "passed": true,
+      "error": ""
     }
   ],
-  "customTestResults":[
+  "customTestResults": [
     {
-      "input":"Hannah",
-      "expected":"hannaH",
-      "actual":"hannaH",
-      "passed":true,
-      "error":""
+      "input": "Hannah",
+      "expected": "hannaH",
+      "actual": "hannaH",
+      "passed": true,
+      "error": ""
     },
     {
-      "input":"abcdefg",
-      "expected":"gfedcba",
-      "actual":"gfedcba",
-      "passed":true,
-      "error":""
+      "input": "abcdefg",
+      "expected": "gfedcba",
+      "actual": "gfedcba",
+      "passed": true,
+      "error": ""
     }
   ]
 }
@@ -178,40 +203,40 @@ The following json format will be returned:
 
 ```json
 {
-  "visibleTestResults":[
+  "visibleTestResults": [
     {
-      "input":"hello",
-      "expected":"olleh",
-      "actual":"olleh",
-      "passed":true,
-      "error":""
+      "input": "hello",
+      "expected": "olleh",
+      "actual": "olleh",
+      "passed": true,
+      "error": ""
     }
   ],
-  "hiddenTestResults":{
-    "passed":2,
-    "total":2
+  "hiddenTestResults": {
+    "passed": 2,
+    "total": 2
   },
-  "status":"Accepted"
+  "status": "Accepted"
 }
 ```
 
 If compilation error exists or any of the tests (visible and hidden) fails, status "Attempted" will be returned:
 
 ```json
 {
-  "visibleTestResults":[
+  "visibleTestResults": [
     {
-      "input":"hello",
-      "expected":"olleh",
-      "actual":"",
-      "passed":false,
-      "error":"Command execution failed: Traceback (most recent call last):\n  File \"/tmp/4149249165.py\", line 2, in \u003cmodule\u003e\n    prit(name[::-1])\n    ^^^^\nNameError: name 'prit' is not defined. Did you mean: 'print'?\n: %!w(*exec.ExitError=\u0026{0x4000364678 []})"
+      "input": "hello",
+      "expected": "olleh",
+      "actual": "",
+      "passed": false,
+      "error": "Command execution failed: Traceback (most recent call last):\n  File \"/tmp/4149249165.py\", line 2, in \u003cmodule\u003e\n    prit(name[::-1])\n    ^^^^\nNameError: name 'prit' is not defined. Did you mean: 'print'?\n: %!w(*exec.ExitError=\u0026{0x4000364678 []})"
     }
   ],
-  "hiddenTestResults":{
-    "passed":0,
-    "total":2
+  "hiddenTestResults": {
+    "passed": 0,
+    "total": 2
   },
-  "status":"Attempted"
+  "status": "Attempted"
 }
 ```
diff --git a/apps/execution-service/go.mod b/apps/execution-service/go.mod
@@ -31,6 +31,7 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
 	github.com/googleapis/gax-go/v2 v2.13.0 // indirect
+	github.com/rabbitmq/amqp091-go v1.10.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect

diff --git a/apps/execution-service/go.sum b/apps/execution-service/go.sum
@@ -85,6 +85,8 @@ github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwA
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
+github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=