Adding a basic paging support

.github/workflows/maven-release-github.yml

@@ -12,70 +12,19 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-
-      - name: Setup Java
-        uses: actions/setup-java@v1
-        with:
-          java-version: 1.8
-          server-id: github
-
-      - name: Install GPG Private Key
-        run: echo -e "${{ secrets.GPG_PRIVATE_KEY }}" | gpg --batch --import
-
-      - name: Release
-        run: mvn -pl kafka-connect-http -B deploy -P package,sign
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
-
-      - name: Prepare artifacts
-        run: mkdir staging && cp kafka-connect-http/target/*.jar staging && cp kafka-connect-http/target/*.tar.gz staging && cp kafka-connect-http/target/*.zip staging
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v1
+      - uses: actions/checkout@v4
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
         with:
-          name: Package
-          path: staging
+          java-version: '11'
+          distribution: 'temurin'
+          server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
+          settings-path: ${{ github.workspace }} # location for the settings.xml file
 
-      - name: Create Release
-        id: create_release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ${{ github.ref }}
-          release_name: ${{ github.ref }}
-          draft: false
-          prerelease: false
-
-      - name: Extract tar.gz name
-        id: extract_tar_name
-        run: |
-          ARTIFACT_NAME=$(basename staging/*.tar.gz)
-          echo "::set-output name=artifact_name::$ARTIFACT_NAME"
-      - name: Upload tar.gz
-        id: upload_tar
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: staging/${{ steps.extract_tar_name.outputs.artifact_name }}
-          asset_name: ${{ steps.extract_tar_name.outputs.artifact_name }}
-          asset_content_type: application/tar+gzip
+      - name: Build with Maven
+        run: mvn -B package --file pom.xml
 
-      - name: Extract zip name
-        id: extract_zip_name
-        run: |
-          ARTIFACT_NAME=$(basename staging/*.zip)
-          echo "::set-output name=artifact_name::$ARTIFACT_NAME"
-      - name: Upload zip
-        id: upload_zip
-        uses: actions/upload-release-asset@v1
+      - name: Publish to GitHub Packages Apache Maven
+        run: mvn deploy -s $GITHUB_WORKSPACE/settings.xml
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: staging/${{ steps.extract_zip_name.outputs.artifact_name }}
-          asset_name: ${{ steps.extract_zip_name.outputs.artifact_name }}
-          asset_content_type: application/zip
+          GITHUB_TOKEN: ${{ github.token }}

kafka-connect-http-infra/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>kafka-connect-http-parent</artifactId>
         <groupId>com.github.castorm</groupId>
-        <version>0.8.12-SNAPSHOT</version>
+        <version>0.8.13</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

kafka-connect-http-test/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>kafka-connect-http-parent</artifactId>
         <groupId>com.github.castorm</groupId>
-        <version>0.8.12-SNAPSHOT</version>
+        <version>0.8.13</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

kafka-connect-http/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>kafka-connect-http-parent</artifactId>
         <groupId>com.github.castorm</groupId>
-        <version>0.8.12-SNAPSHOT</version>
+        <version>0.8.13</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

kafka-connect-http/src/main/java/com/github/castorm/kafka/connect/http/HttpSourceConnectorConfig.java

@@ -55,6 +55,7 @@ class HttpSourceConnectorConfig extends AbstractConfig {
     private static final String RECORD_SORTER = "http.record.sorter";
     private static final String RECORD_FILTER_FACTORY = "http.record.filter.factory";
     private static final String OFFSET_INITIAL = "http.offset.initial";
+    private static final String NEXT_PAGE_OFFSET = "http.offset.nextpage";
 
     private final TimerThrottler throttler;
     private final HttpRequestFactory requestFactory;
@@ -63,6 +64,7 @@ class HttpSourceConnectorConfig extends AbstractConfig {
     private final SourceRecordFilterFactory recordFilterFactory;
     private final SourceRecordSorter recordSorter;
     private final Map<String, String> initialOffset;
+    private String nextPageOffset;
 
     HttpSourceConnectorConfig(Map<String, ?> originals) {
         super(config(), originals);
@@ -74,6 +76,7 @@ class HttpSourceConnectorConfig extends AbstractConfig {
         recordSorter = getConfiguredInstance(RECORD_SORTER, SourceRecordSorter.class);
         recordFilterFactory = getConfiguredInstance(RECORD_FILTER_FACTORY, SourceRecordFilterFactory.class);
         initialOffset = breakDownMap(getString(OFFSET_INITIAL));
+        nextPageOffset = getString(NEXT_PAGE_OFFSET);
     }
 
     public static ConfigDef config() {
@@ -84,6 +87,7 @@ public static ConfigDef config() {
                 .define(RESPONSE_PARSER, CLASS, PolicyHttpResponseParser.class, HIGH, "Response Parser Class")
                 .define(RECORD_SORTER, CLASS, OrderDirectionSourceRecordSorter.class, LOW, "Record Sorter Class")
                 .define(RECORD_FILTER_FACTORY, CLASS, OffsetRecordFilterFactory.class, LOW, "Record Filter Factory Class")
-                .define(OFFSET_INITIAL, STRING, "", HIGH, "Starting offset");
+                .define(OFFSET_INITIAL, STRING, "", HIGH, "Starting offset")
+                .define(NEXT_PAGE_OFFSET, STRING, "", HIGH, "Next Page offset");
     }
 }