Support AWS Kinesis Data Streams as a Source #823
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will build a Java project with Gradle | |
| # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle | |
| name: Kafka plugin integration tests | |
| on: | |
| push: | |
| paths: | |
| - 'data-prepper-plugins/kafka-plugins/**' | |
| - '*gradle*' | |
| pull_request_target: | |
| types: [ opened, synchronize, reopened ] | |
| paths: | |
| - 'data-prepper-plugins/kafka-plugins/**' | |
| - '*gradle*' | |
| workflow_dispatch: | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| integration-tests: | |
| strategy: | |
| matrix: | |
| java: [11] | |
| kafka: [2.8.1, 3.5.1] | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| - name: Checkout Data Prepper | |
| uses: actions/checkout@v2 | |
| - name: Run Kafka Docker | |
| run: | | |
| echo 'KAFKA_VERSION=${{ matrix.kafka }}' > data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/.env | |
| docker compose --project-directory data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/zookeeper --env-file data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/.env up -d | |
| sleep 2 | |
| - name: Wait for Kafka | |
| run: | | |
| ./gradlew data-prepper-plugins:kafka-plugins:integrationTest -Dtests.kafka.bootstrap_servers=localhost:9092 -Dtests.kafka.authconfig.username=admin -Dtests.kafka.authconfig.password=admin --tests KafkaStartIT | |
| - name: Configure AWS credentials | |
| id: aws-credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: ${{ secrets.TEST_IAM_ROLE_ARN }} | |
| aws-region: ${{ secrets.TEST_REGION }} | |
| output-credentials: true | |
| - name: Configure AWS default credentials | |
| run: | | |
| aws configure set default.region ${{ secrets.TEST_REGION }} | |
| aws configure set default.aws_access_key_id ${{ steps.aws-credentials.outputs.aws-access-key-id }} | |
| aws configure set default.aws_secret_access_key ${{ steps.aws-credentials.outputs.aws-secret-access-key }} | |
| aws configure set default.aws_session_token ${{ steps.aws-credentials.outputs.aws-session-token }} | |
| - name: Run Kafka integration tests | |
| run: | | |
| ./gradlew data-prepper-plugins:kafka-plugins:integrationTest \ | |
| -Dtests.kafka.bootstrap_servers=localhost:9092 \ | |
| -Dtests.kafka.authconfig.username=admin -Dtests.kafka.authconfig.password=admin \ | |
| -Dtests.kafka.kms_key=alias/DataPrepperTesting \ | |
| --tests '*kafka.buffer*' --tests KafkaSourceJsonTypeIT --tests KafkaBufferOTelIT | |
| - name: Upload Unit Test Results | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: data-prepper-kafka-integration-tests-kafka-${{ matrix.kafka }}-java-${{ matrix.java }} | |
| path: '**/test-results/**/*.xml' | |
| publish-test-results: | |
| name: "Publish Unit Tests Results" | |
| needs: integration-tests | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: Download Artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: test-results | |
| - name: Publish Unit Test Results | |
| uses: EnricoMi/publish-unit-test-result-action@v1 | |
| with: | |
| files: "test-results/**/*.xml" |