add ci/cd and clippy fixes

.github/workflows/ci.yml

@@ -0,0 +1,136 @@
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+    branches:
+      - '**'
+
+jobs:
+  test:
+    name: Test on ${{ matrix.config.os }} ${{ matrix.config.arch }}
+    runs-on: ${{ matrix.config.platform }}
+    strategy:
+      matrix:
+        config:
+          - {platform: windows-latest,   os: windows, arch: amd64, path: windows-x86_64}
+          - {platform: macos-13,         os: darwin,  arch: amd64, path: darwin-x86_64}
+          - {platform: macos-latest,     os: darwin,  arch: arm64, path: darwin-arm64}
+          - {platform: ubuntu-latest,    os: linux,   arch: amd64, path: linux-x86_64}
+          - {platform: raspbian-private, os: linux,   arch: arm64, path: linux-arm64}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+
+      - name: Install OpenSSL and pkg-config (Linux ARM64 only)
+        if: matrix.config.path == 'linux-arm64'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libssl-dev pkg-config
+
+      - name: Set PKG_CONFIG_PATH (Linux ARM64 only)
+        if: matrix.config.path == 'linux-arm64'
+        run: echo "PKG_CONFIG_PATH=/usr/lib/aarch64-linux-gnu/pkgconfig" >> $GITHUB_ENV
+
+      - name: Install clippy
+        run: rustup component add clippy --toolchain stable
+
+      - name: Run clippy
+        run: make clippy
+
+      - name: Run tests
+        run: make test
+
+      - name: Build release artifact
+        if: github.ref == 'refs/heads/master'
+        run: make build
+
+      - name: Upload artifacts
+        if: github.ref == 'refs/heads/master'
+        uses: actions/upload-artifact@v4
+        with:
+          name: core-etl-${{ matrix.config.path }}
+          path: ./target/release/core-etl${{ matrix.config.os == 'windows' && '.exe' || '' }}
+
+  release:
+    if: github.ref == 'refs/heads/master'
+    needs: test
+    runs-on: ubuntu-latest
+    outputs:
+      upload_url: ${{ steps.create_release.outputs.upload_url }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+
+      - name: Get version from Cargo.toml
+        id: cargo-version
+        run: echo "::set-output name=VERSION::$(grep '^version =' Cargo.toml | sed -E 's/version = \"(.*)\"/\1/')"
+
+      - name: Create new tag
+        id: tag
+        run: |
+          git fetch --tags
+          TAG_EXISTS=$(git tag -l "v${{ steps.cargo-version.outputs.VERSION }}")
+          if [ -n "$TAG_EXISTS" ]; then
+            echo "Tag v${{ steps.cargo-version.outputs.VERSION }} already exists"
+            exit 1
+          fi
+          git tag v${{ steps.cargo-version.outputs.VERSION }}
+          git push origin v${{ steps.cargo-version.outputs.VERSION }}
+
+      - name: Create GitHub Release
+        id: create_release
+        uses: actions/create-release@v1
+        with:
+          tag_name: v${{ steps.cargo-version.outputs.VERSION }}
+          release_name: Release v${{ steps.cargo-version.outputs.VERSION }}
+          draft: false
+          prerelease: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  artifacts:
+    needs: release
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform: [
+          {path: linux-x86_64, file_ext: ""},
+          {path: windows-x86_64, file_ext: ".exe"},
+          {path: linux-arm64, file_ext: ""},
+          {path: darwin-x86_64, file_ext: ""},
+          {path: darwin-arm64, file_ext: ""},
+        ]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: core-etl-${{ matrix.platform.path }}
+          path: ./core-etl-${{ matrix.platform.path }}
+
+      - name: Upload release assets
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ needs.release.outputs.upload_url }}
+          asset_path: ./core-etl-${{ matrix.platform.path }}/core-etl${{ matrix.platform.file_ext }}
+          asset_name: core-etl-${{ matrix.platform.path }}${{ matrix.platform.file_ext }}
+          asset_content_type: application/octet-stream

.github/workflows/docker.yml

@@ -0,0 +1,41 @@
+D
+on:
+  workflow_run:
+    workflows: ["ci.yml"]
+    types:
+      - completed
+    with:
+      ref: main
+      status: success
+      conclusion: success
+      steps:
+        - name: artifacts
+jobs:
+  docker:
+    name: docker
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get version from Cargo.toml
+        id: cargo-version
+        run: echo "::set-output name=VERSION::$(grep '^version =' Cargo.toml | sed -E 's/version = \"(.*)\"/\1/')"
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver-opts: network=host
+      - name: Install QEMU for Multi-Platform Builds
+        uses: docker/setup-qemu-action@v2
+      - name: Log in to GitHub Container Registry
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+      - name: Build and Push Docker Image for Multiple Architectures
+        run: |
+          docker buildx create --use
+          docker buildx build \
+            --platform linux/amd64,linux/arm64 \
+            --tag ghcr.io/core-coin/core-etl:v${{ steps.cargo-version.outputs.VERSION }} \
+            --tag ghcr.io/core-coin/core-etl:latest \
+            --push .
+      - name: Verify Image Digest (Optional)
+        run: docker buildx imagetools inspect ghcr.io/core-coin/core-etl:latest

.gitignore

@@ -7,4 +7,6 @@ db.db-wal
 .xata
 .xatarc
 libsql_data/
-postgres_data/
+postgres_data/
+./docker-comse/libsql_data
+./docker-compose/postgres_data

Cargo.toml

@@ -29,7 +29,7 @@ keywords = ["core blockchain", "xcb", "etl"]
 license = "MIT OR Apache-2.0"
 homepage = "https://github.com/core-coin/core-etl"
 repository = "https://github.com/core-coin/core-etl"
-version = "0.0.1"
+version = "1.0.0"
 
 [workspace.dependencies]
 # Workspace members

Makefile

@@ -2,17 +2,12 @@
 CORE_ETL_FLAGS ?= -s /libsql_data/data.sqld/dbs/default/data
 CORE_ETL_EXPORT_FLAGS ?= -w="ctn" -l
 GO_CORE_FLAGS ?= --ws --ws.addr 0.0.0.0 --syncmode fast --cache=128 --snapshot=false
+CARGO := cargo
 
 # Targets
-.PHONY: build-libsql up-libsql down-libsql clean-libsql init-libsql sync-local-libsql sync-remote-libsql stop-libsql start-libsql clean-volume-libsql 
+.PHONY: build-libsql up-libsql down-libsql clean-libsql init-libsql sync-local-libsql sync-remote-libsql stop-libsql start-libsql clean-volume-libsql
 .PHONY: build-postgres up-postgres down-postgres clean-volume-postgres init-postgres sync-local-postgres sync-remote-postgres stop-postgres start-postgres
-
-clean:
-	@echo "Cleaning up core-etl.log, sqlite3.db and db.db"
-	@rm -f core-etl.log
-	@rm -f sqlite3.db sqlite3.db-shm sqlite3.db-wal sqlite3_dump.sql
-	@rm -f db.db db.db-shm db.db-wal db_dump.sql
-
+.PHONY: build run run-debug test clean fmt clippy
 
 ######################################### LIBSQL #########################################
 
@@ -90,3 +85,33 @@ clean-volume-postgres:
 
 
 
+build:
+	$(CARGO) build --release
+
+# Build the project in debug mode
+debug:
+	$(CARGO) build
+
+# Run the project in release mode
+run:
+	$(CARGO) run --release
+
+# Run the project in debug mode
+run-debug:
+	$(CARGO) run
+
+# Test the project
+test:
+	$(CARGO) test --all-targets --all-features
+
+# Clean the project
+clean:
+	$(CARGO) clean
+
+# Format the code
+fmt:
+	$(CARGO) fmt
+
+# Check for common mistakes
+clippy:
+	$(CARGO) clippy --all-targets --all-features -- -D warnings

bin/core-etl/src/app_storage.rs

@@ -11,17 +11,17 @@ use tracing::info;
 #[serde(rename_all = "kebab-case")]
 pub enum StorageType {
     #[default]
-    Sqlite3Storage,
-    PostgresStorage,
-    MockStorage,
+    Sqlite3,
+    Postgres,
+    Mock,
 }
 
 impl Args {
     pub async fn choose_storage(&self) -> Arc<dyn Storage + Send + Sync> {
         info!("Storing data about: {:?}", self.modules.clone());
         match self.storage {
-            StorageType::MockStorage => Arc::new(MockStorage::new()),
-            StorageType::Sqlite3Storage => {
+            StorageType::Mock => Arc::new(MockStorage::new()),
+            StorageType::Sqlite3 => {
                 if self.sqlite3_path.is_none() {
                     panic!("sqlite3_path is required for Sqlite3 Storage");
                 }
@@ -39,7 +39,7 @@ impl Args {
                 db.prepare_db().await.unwrap();
                 Arc::new(db)
             }
-            StorageType::PostgresStorage => {
+            StorageType::Postgres => {
                 if self.postgres_db_dsn.is_none() {
                     panic!("postgres_db_dsn is required for Postgres Storage");
                 }

bin/core-etl/src/export.rs

@@ -108,9 +108,7 @@ impl ExportArgs {
             let mut split = token.split(':');
             let token_type = split.next().unwrap().to_string();
             let token_address = split.next().unwrap().to_string();
-            map.entry(token_type)
-                .or_insert_with(HashSet::new)
-                .insert(token_address);
+            map.entry(token_type).or_default().insert(token_address);
         }
         map
     }