add full spring-boot-starter-docker-compose setup and integration test setup

Author: NiklasTiede

README.md

@@ -61,12 +61,10 @@ Here's a diagram of the setup:
 
 ## How to Run this Project Locally
 
-The app can be built in 3 steps:
+The app can be built in 2 steps:
 
-1. Run `docker-compose` to set up preloaded backend services (MySQL, Elasticsearch and
-  MinIO)
-2. Run the Spring Boot Backend with `./gradlew bootRun`
-3. Run the React Frontend with `yarn install` & `yarn start`
+1. Run the Spring Boot Backend with `./gradlew bootRun` (this will automatically spin up mysql/elastic/minio containers)
+2. Run the React Frontend with `yarn install` & `yarn start`
 
 ---
 
@@ -76,21 +74,12 @@ At first, we have to run the with data preloaded stateful services (MySQL, Elast
 MinIO) which are used by the backend. I created a docker image of each service preloaded with 
 data, so we just have to execute the `docker-compose.yaml`.
 
-```shell
-cd infrastructure/deployment/development
-docker-compose up -d
-```
-
-Be aware: the images are x86-based. So when you're using ARM-based apple silicone machine, 
-pay attention that emulation is activated. All images will be pulled and mounted to your device. 
-The spring boot backend can connect to the containers now.
-
 For more information on how data were collected, processed and imported look into 
 the [infrastructure](./infrastructure/README.md)-folder.
 
 --- 
 
-### 2. Set Up Spring Boot Backend
+### 1. Set Up Spring Boot Backend and Stateful Services: MySQL, Elasticsearch and MinIO
 
 Now we can start the Spring Boot app:
 
@@ -99,12 +88,16 @@ Now we can start the Spring Boot app:
 ./gradlew bootRun
 ```
 
+This will automatically pull/start the stateful containers. So pulling might take time depending 
+on your bandwidth. For more information on how data were collected, processed and imported look into
+the [infrastructure](./infrastructure/README.md)-folder.
+
 The backend can now be reached at port 8080 on localhost. You can test if the backend works properly by 
 sending some http requests. Use the provided [.http](./src/main/resources/api-calls) files.
 
 ---
 
-### 3. Set Up React Frontend
+### 2. Set Up React Frontend
 
 Now we can run the React frontend. We have to move into the frontend-folder and build & run with yarn or npm. 
 
@@ -129,8 +122,8 @@ we use during development.
 - [x] Set up Elasticsearch, Photos / File Storage
 - [x] Deploy on Home Server with Docker-Compose
 - [x] enable HTTPS with reverse-proxy
-- [ ] simplify local development with spring-boot-starter-docker-compose
-- [ ] setup and add integration tests
+- [x] simplify local development with spring-boot-starter-docker-compose
+- [x] setup and add integration tests
 - [ ] Add Monitoring (Prometheus, Grafana, Exporters) and expose Grafana
 - [ ] Add Logging (ELK Stack) and expose Kibana
 - [ ] Create React Frontend
@@ -147,5 +140,5 @@ we use during development.
 - [ ] Deploy on HA K3s Home Server
 - [ ] Use Flux for GitOps CD
 - [ ] Add Integration Namespace in K3s next to the Prod Env for Testing
-- [ ] Add Unit / Integration Tests in BE and FE
+- [ ] Add Selenium for E2E-tests from FE to BE
 - [ ] Add more Features like Chat Functionality

build.gradle

@@ -5,7 +5,7 @@ plugins {
 	id 'io.spring.dependency-management' version '1.1.4'
 	id 'com.diffplug.spotless' version '6.25.0'
 	id 'com.github.ben-manes.versions' version '0.51.0'
-	id "org.openrewrite.rewrite" version "6.10.1"
+	id "org.openrewrite.rewrite" version "6.11.2"
 }
 
 rewrite {
@@ -77,9 +77,6 @@ dependencies {
 	implementation 'com.mysql:mysql-connector-j:8.3.0'
 	implementation "org.springframework.boot:spring-boot-starter-data-jpa:${springBootVersion}"
 
-	//-- Shared Cache
-	implementation "org.springframework.boot:spring-boot-starter-data-redis:${springBootVersion}"
-
 	//-- SearchEngine
 	implementation 'org.springframework.data:spring-data-elasticsearch:5.2.4'
 	implementation 'org.eclipse.parsson:parsson:1.1.5' // fix transitive vulnerability
@@ -117,7 +114,7 @@ dependencies {
 	testImplementation "org.testcontainers:junit-jupiter:${testContainersVersion}"
 	testImplementation "org.testcontainers:mysql:${testContainersVersion}"
 	testImplementation "org.testcontainers:elasticsearch:${testContainersVersion}"
-	implementation "org.testcontainers:minio:${testContainersVersion}"
+	testImplementation "org.testcontainers:minio:${testContainersVersion}"
 
 	if (System.getProperty('os.name').toLowerCase().contains('mac')) {
 		runtimeOnly 'io.netty:netty-resolver-dns-native-macos:4.1.108.Final:osx-aarch_64'

compose.yaml

@@ -1,11 +1,10 @@
-version: '3.8'
 services:
 
   ### ----------------------- MySQL ----------------------- ###
-  mysql:
+  imdb-clone-mysql:
     container_name: imdb-clone-mysql
-    image: mysql:latest
-    restart: always
+    image: mysql:8.3.0
+    restart: unless-stopped
     environment:
       - MYSQL_DATABASE=movie_db
       - MYSQL_ROOT_PASSWORD=supersecret
@@ -18,6 +17,45 @@ services:
     networks:
       - imdb-clone-network
 
+  imdb-clone-elasticsearch:
+    container_name: imdb-clone-elasticsearch
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0
+    restart: unless-stopped
+    environment:
+      - xpack.security.enabled=false
+      - discovery.type=single-node
+      - ES_JAVA_OPTS=-Xms512m -Xmx2g
+    ports:
+      - "9200:9200"
+    volumes:
+      - imdb-clone-elasticsearch-data:/usr/share/elasticsearch/data
+    networks:
+      - imdb-clone-network
+
+  ### ----------------------- MinIO ----------------------- ###
+  imdb-clone-minio:
+    container_name: imdb-clone-minio
+    image: bitnami/minio:2024.3.26
+    restart: unless-stopped
+    user: root
+    environment:
+      - MINIO_ROOT_USER=ROOTNAME
+      - MINIO_ROOT_PASSWORD=CHANGEME123
+    ports:
+      - "9000:9000"
+      - "9090:9090"
+    volumes:
+      - imdb-clone-minio-data:/data
+    networks:
+      - imdb-clone-network
+    command: /opt/bitnami/minio/bin/minio server --console-address :9090 /data
+
+volumes:
+  imdb-clone-elasticsearch-data:
+    name: imdb-clone-elasticsearch-data
+  imdb-clone-minio-data:
+    name: imdb-clone-minio-data
+
 networks:
   imdb-clone-network:
     name: imdb-clone-network

src/main/resources/sql/index.sql renamed to infrastructure/mysql/docker-image-creation/index.sql

@@ -59,8 +59,7 @@ public void indexMovie(Movie movie) {
     } catch (IOException e) {
       logger.error(
           "Document of type movie with [{}] was not indexed successfully.", kv(MOVIE_ID,movie.getId()));
-//      throw new RuntimeException(e);
-      throw new ElasticsearchOperationException("Document of type movie with [{}] was not indexed successfully.", e);
+      throw new ElasticsearchOperationException("Document of type movie with id [%s] was not indexed successfully.".formatted(movie.getId()), e);
     }
   }
 

src/main/java/com/thecodinglab/imdbclone/service/impl/ElasticSearchServiceImpl.java

@@ -167,8 +167,8 @@ public String storeFile(InputStream file, int fileSize, String fileName, String
                   .stream(file, fileSize, -1)
                   .build());
       return "Image was stored with etag [" + resp.etag() + "]";
-    } catch (Exception e) {
-      throw new MinioOperationException("Error while storing file in MinIO", e);
+    } catch (Exception ex) {
+      throw new MinioOperationException("Error while storing file in MinIO", ex);
     }
   }
 
@@ -194,9 +194,9 @@ public void setUpBucket() {
       createBucketPolicyFrom(bucketPolicy);
       logger.info("bucket [{}] was created and bucketPolicy set successfully", bucketName);
 
-    } catch (Exception e) {
+    } catch (Exception ex) {
       logger.error("Creation of bucket [{}] failed", bucketName);
-      throw new MinioOperationException("Error while creating bucket in MinIO", e);
+      throw new MinioOperationException("Error while creating bucket in MinIO", ex);
     }
   }
 
@@ -205,9 +205,9 @@ private void createBucketPolicyFrom(String bucketPolicy) {
     try {
       minioClient.setBucketPolicy(
           SetBucketPolicyArgs.builder().bucket(bucketName).config(policyConfig).build());
-    } catch (Exception e) {
+    } catch (Exception ex) {
       logger.error("Creation of bucket policy failed");
-      throw new MinioOperationException("Error while creating bucket policy in MinIO", e);
+      throw new MinioOperationException("Error while creating bucket policy in MinIO", ex);
     }
   }
 
@@ -227,9 +227,9 @@ private String readResourceFile(String resourcePath) {
         }
         return stringBuilder.toString();
       }
-    } catch (IOException e) {
+    } catch (IOException ex) {
       logger.error("Reading file with path [{}] failed", resourcePath);
-      throw new MinioOperationException("Error while reading policy config", e);
+      throw new MinioOperationException("Error while reading policy config", ex);
     }
   }
 
@@ -246,9 +246,9 @@ public String generateUrl(String imageName) {
                   .expiry(60 * 60 * 24)
                   .build());
       return presignedUrl;
-    } catch (Exception e) {
+    } catch (Exception ex) {
       logger.error("Generate presigned object URL file with image name [{}] failed", imageName);
-      throw new MinioOperationException("Error while generating presigned URL in MinIO", e);
+      throw new MinioOperationException("Error while generating presigned URL in MinIO", ex);
     }
   }
 }

src/main/java/com/thecodinglab/imdbclone/service/impl/FileStorageServiceImpl.java

@@ -1,3 +1,7 @@
+-- Attempt to create the database if it doesn't already exist
+create database if not exists movie_db;
+use movie_db;
+
 -- Drop tables to clean
 drop table if exists comment;
 drop table if exists watched_movie;
@@ -109,9 +113,6 @@ create table verification_token (
 );
 
 
--- Insert records into the tables
+-- Mandatory Inserts
 insert into role (name) values('ROLE_ADMIN');
 insert into role (name) values('ROLE_USER');
-
-# insert into movie(id, primary_title, original_title, start_year, end_year, runtime_minutes, created_at_in_utc, modified_at_in_utc, movie_genre, movie_type, imdb_rating, imdb_rating_count, adult, rating, rating_count, description, image_url_token)
-# values(2872718,'Nightcrawler','Nightcrawler',2014,null,117,'2023-07-31 06:07:12','2023-07-31 06:07:12',16409,1,7.8,528339,0,null,0,'When Lou Bloom, desperate for work, muscles into the world of L.A. crime journalism, he blurs the line between observer and participant to become the star of his own story. Aiding him in his effort is Nina, a TV-news veteran.','9BGAIYNfdY90aIkV66dIJ6Olee7JGn');

src/main/resources/sql/init.sql renamed to src/main/resources/sql/1_init_schema.sql

@@ -0,0 +1,23 @@
+
+insert into movie(id, primary_title, original_title, start_year, end_year, runtime_minutes, created_at_in_utc, modified_at_in_utc, movie_genre, movie_type, imdb_rating, imdb_rating_count, adult, rating, rating_count, description, image_url_token)
+values(2872718,'Nightcrawler','Nightcrawler',2014,null,117,'2023-07-31 06:07:12','2023-07-31 06:07:12',16409,1,7.8,528339,0,null,0,'When Lou Bloom, desperate for work, muscles into the world of L.A. crime journalism, he blurs the line between observer and participant to become the star of his own story. Aiding him in his effort is Nina, a TV-news veteran.','9BGAIYNfdY90aIkV66dIJ6Olee7JGn');
+
+# add typical scenarios
+
+# # test data
+# insert into movie_db.account(id,username,email,password,first_name,last_name,bio,phone,birthday,locked,enabled) values(1,'les_grossman','tom.cruise@yahoo.de','UnencryptedPa55worD','Tom','Cruise','I will massacre you!','01628264723', '1962-07-03 00:00:00',false,true);
+# insert into movie_db.account(id,username,email,password,first_name,last_name,bio,phone,birthday,locked,enabled) values(2,'jeff_portnoy','jack.black@gmail.com','UnencryptedPa55worD','jack','black','You go, girl!','015122973088', '1969-08-28 00:00:00',false,true);
+#
+# insert into movie_db.rating(rating,movie_id,account_id) values(9.2,942385,1);
+# insert into movie_db.rating(rating,movie_id,account_id) values(7.7,369339,1);
+# insert into movie_db.rating(rating,movie_id,account_id) values(7.8,942385,2);
+# insert into movie_db.rating(rating,movie_id,account_id) values(8.1,332379,2);
+#
+# insert into movie_db.watched_movie(movie_id,account_id) values(942385,1);
+# insert into movie_db.watched_movie(movie_id,account_id) values(369339,1);
+# insert into movie_db.watched_movie(movie_id,account_id) values(942385,1);
+#
+# insert into movie_db.comment(message,movie_id,account_id) values('What an outrageous cast!',942385,1);
+# insert into movie_db.comment(message,movie_id,account_id) values('The shoot was sometimes difficult.',369339,1);
+# insert into movie_db.comment(message,movie_id,account_id) values('All except for one ;-)',942385,2);
+# insert into movie_db.comment(message,movie_id,account_id) values('It was fun to play!',332379,2);

src/main/resources/sql/2_init_data.sql

@@ -20,11 +20,11 @@
 public class BaseContainers {
 
   public static MySQLContainer<?> mysqlContainer =
-      new MySQLContainer<>(DockerImageName.parse("mysql:latest"))
-          .withDatabaseName("testdb")
+      new MySQLContainer<>(DockerImageName.parse("mysql:8.3.0"))
+          .withDatabaseName("movie_db")
           .withUsername("test")
           .withPassword("test")
-          .withInitScript("sql/init.sql");
+          .withInitScript("sql/1_init_schema.sql");
 
   @DynamicPropertySource
   static void mysqlProperties(DynamicPropertyRegistry registry) {
@@ -48,7 +48,7 @@ public static void populateTables(String scriptPath) {
 
   static ElasticsearchContainer elasticContainer =
       new ElasticsearchContainer(
-          DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch:8.7.1"));
+          DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch:8.13.0"));
 
   @DynamicPropertySource
   static void setProperties(DynamicPropertyRegistry registry) {
@@ -57,7 +57,7 @@ static void setProperties(DynamicPropertyRegistry registry) {
 
   // Add this to your existing BaseContainers class
   public static MinIOContainer minioContainer =
-      new MinIOContainer(DockerImageName.parse("minio/minio:latest"))
+      new MinIOContainer(DockerImageName.parse("minio/minio:RELEASE.2024-03-26T22-10-45Z"))
           .withEnv("MINIO_ACCESS_KEY", "minioadmin")
           .withEnv("MINIO_SECRET_KEY", "minioadmin")
           .withCommand("server /data");

src/main/resources/sql/test-data.sql

@@ -2,7 +2,8 @@
 
 import com.thecodinglab.imdbclone.integration.BaseContainers;
 import com.thecodinglab.imdbclone.payload.movie.MovieSearchRequest;
-//import org.junit.jupiter.api.Test;
+import java.util.Collections;
+import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
 import org.springframework.boot.test.context.SpringBootTest;
@@ -15,11 +16,10 @@ class SearchControllerTest extends BaseContainers {
 
   @Autowired private WebTestClient webTestClient;
 
-  // TODO: fix not FOUND Nightcrawler movie (although it seemed to be indexed)
-//  @Test
+  @Test
   void search_success() {
     // Arrange
-    var request = new MovieSearchRequest(null, null, null, null, null, null);
+    var request = new MovieSearchRequest(null, null, null, null, Collections.emptySet(), null);
 
     // Act and Assert
     webTestClient
@@ -28,9 +28,12 @@ void search_success() {
         .bodyValue(request)
         .accept(MediaType.APPLICATION_JSON)
         .exchange()
-        .expectStatus().isOk()
-        .expectHeader().contentType(MediaType.APPLICATION_JSON)
+        .expectStatus()
+        .isOk()
+        .expectHeader()
+        .contentType(MediaType.APPLICATION_JSON)
         .expectBody()
-        .jsonPath("$.pageNumber").isEqualTo(0);
+        .jsonPath("$.content[0].primaryTitle")
+        .isEqualTo("Nightcrawler");
   }
 }