From c3a68325593cd848ff5b86a72f3ff938f9b8371a Mon Sep 17 00:00:00 2001 From: Olivier Fuxet Date: Thu, 28 Sep 2023 10:09:42 +0200 Subject: [PATCH 01/11] WIP new indexer domain --- Cargo.lock | 199 +++++++++-- Cargo.toml | 1 + api/Cargo.toml | 5 +- common/infrastructure/Cargo.toml | 5 +- .../domain/indexer-domain/Cargo.toml | 38 ++ .../domain/indexer-domain/src/lib.rs | 2 + .../domain/indexer-domain/src/models/mod.rs | 33 ++ .../domain/indexer-domain/src/ports/mod.rs | 1 + .../src/ports/output/github_api/issue.rs | 16 + .../src/ports/output/github_api/mod.rs | 11 + .../src/ports/output/github_api/port.rs | 7 + .../ports/output/github_api/pull_request.rs | 32 ++ .../src/ports/output/github_api/repo.rs | 9 + .../src/ports/output/github_api/user.rs | 9 + .../indexer-domain/src/ports/output/mod.rs | 1 + rest_api_contract/indexer-contract.yaml | 331 ++++++++++++++++++ 16 files changed, 656 insertions(+), 44 deletions(-) create mode 100644 new-github-indexer/domain/indexer-domain/Cargo.toml create mode 100644 new-github-indexer/domain/indexer-domain/src/lib.rs create mode 100644 new-github-indexer/domain/indexer-domain/src/models/mod.rs create mode 100644 new-github-indexer/domain/indexer-domain/src/ports/mod.rs create mode 100644 new-github-indexer/domain/indexer-domain/src/ports/output/github_api/issue.rs create mode 100644 new-github-indexer/domain/indexer-domain/src/ports/output/github_api/mod.rs create mode 100644 new-github-indexer/domain/indexer-domain/src/ports/output/github_api/port.rs create mode 100644 new-github-indexer/domain/indexer-domain/src/ports/output/github_api/pull_request.rs create mode 100644 new-github-indexer/domain/indexer-domain/src/ports/output/github_api/repo.rs create mode 100644 new-github-indexer/domain/indexer-domain/src/ports/output/github_api/user.rs create mode 100644 new-github-indexer/domain/indexer-domain/src/ports/output/mod.rs create mode 100644 rest_api_contract/indexer-contract.yaml diff --git a/Cargo.lock b/Cargo.lock index b1ebe35aa5..63b97392c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -198,7 +198,7 @@ dependencies = [ "mockall", "mockall_double", "mockito", - "octocrab", + "octocrab 0.19.0", "olog", "presentation", "reqwest", @@ -2240,6 +2240,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" + [[package]] name = "httparse" version = "1.8.0" @@ -2286,32 +2292,44 @@ dependencies = [ ] [[package]] -name = "hyper-tls" -version = "0.5.0" +name = "hyper-rustls" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" dependencies = [ - "bytes", + "futures-util", + "http", "hyper", - "native-tls", + "log", + "rustls", + "rustls-native-certs", "tokio", - "tokio-native-tls", + "tokio-rustls", ] [[package]] -name = "hyperx" -version = "1.4.0" +name = "hyper-timeout" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5617e92fc2f2501c3e2bc6ce547cad841adba2bae5b921c7e52510beca6d084c" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "base64 0.13.1", "bytes", - "http", - "httpdate", - "language-tags", - "mime", - "percent-encoding", - "unicase", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", ] [[package]] @@ -2422,6 +2440,25 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "indexer-domain" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "derive-getters", + "derive-new", + "derive_more", + "futures", + "itertools", + "octocrab 0.30.1", + "serde", + "serde_json", + "thiserror", + "uuid 1.3.3", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -2467,7 +2504,7 @@ dependencies = [ "lapin", "lazy_static", "mockall", - "octocrab", + "octocrab 0.19.0", "olog", "opentelemetry", "opentelemetry-datadog", @@ -2691,12 +2728,6 @@ dependencies = [ "log", ] -[[package]] -name = "language-tags" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" - [[package]] name = "lapin" version = "2.3.1" @@ -3099,8 +3130,9 @@ dependencies = [ [[package]] name = "octocrab" -version = "0.18.1" -source = "git+https://github.com/onlydustxyz/octocrab.git#7998b859de586cd705161a031144b0af644aeafc" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "496442a5ec5ad38376a0c49bc0f31ba55dbda5276cf12757498c378c3bc2ea1c" dependencies = [ "arc-swap", "async-trait", @@ -3111,7 +3143,6 @@ dependencies = [ "either", "futures-core", "futures-util", - "hyperx", "jsonwebtoken", "once_cell", "reqwest", @@ -3124,6 +3155,44 @@ dependencies = [ "url", ] +[[package]] +name = "octocrab" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbed1b1298bc70dd4ae89fd44dd7c13f0a1c80a506d731eb1b939f14f5e367de" +dependencies = [ + "arc-swap", + "async-trait", + "base64 0.21.2", + "bytes", + "cfg-if", + "chrono", + "either", + "futures", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-rustls", + "hyper-timeout", + "jsonwebtoken", + "once_cell", + "percent-encoding", + "pin-project", + "secrecy", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "snafu", + "tokio", + "tower", + "tower-http", + "tracing", + "url", +] + [[package]] name = "olog" version = "0.1.0" @@ -3430,9 +3499,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.1.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pin-project" @@ -5144,6 +5213,16 @@ dependencies = [ "uuid 1.3.3", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.1.0" @@ -5176,6 +5255,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -5259,6 +5348,48 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tokio-util 0.7.8", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" +dependencies = [ + "bitflags 2.3.1", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -5272,6 +5403,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -5421,15 +5553,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "unicase" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" -dependencies = [ - "version_check", -] - [[package]] name = "unicode-bidi" version = "0.3.13" diff --git a/Cargo.toml b/Cargo.toml index 8d2e759055..12db24da32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,4 +11,5 @@ members = [ "common/testing", "common/olog", "github-indexer", + "new-github-indexer/domain/indexer-domain", ] diff --git a/api/Cargo.toml b/api/Cargo.toml index 17a208410f..4fa99a4246 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -38,9 +38,8 @@ uuid08 = { package = "uuid", version = "0.8", default_features = false, features ] } # Github -octocrab = { git = "https://github.com/onlydustxyz/octocrab.git", features = [ - "stream", -] } + +octocrab = { package = "octocrab", version = "0.19.0", features = ["stream"] } # Http client http-api-problem = { version = "0.56.0", features = ["rocket"] } diff --git a/common/infrastructure/Cargo.toml b/common/infrastructure/Cargo.toml index 79b2c311b3..6a451ce0ee 100644 --- a/common/infrastructure/Cargo.toml +++ b/common/infrastructure/Cargo.toml @@ -36,9 +36,8 @@ uuid = { version = "1.3.3", default_features = false, features = [ ] } # Github -octocrab = { git = "https://github.com/onlydustxyz/octocrab.git", features = [ - "stream", -] } +octocrab = { package = "octocrab", version = "0.19.0", features = ["stream"] } + reqwest = "0.11.13" # Graphql client diff --git a/new-github-indexer/domain/indexer-domain/Cargo.toml b/new-github-indexer/domain/indexer-domain/Cargo.toml new file mode 100644 index 0000000000..460d2b60f0 --- /dev/null +++ b/new-github-indexer/domain/indexer-domain/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "indexer-domain" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] + +octocrab_indexer = { package = "octocrab", version = "0.30.1", features = ["stream"] } + +# Async +async-trait = "0.1.56" +futures = { version = "0.3.21", features = ["alloc"] } + +# Core types +uuid = { version = "1.3.3", default_features = false, features = [ + "v4", + "serde", +] } + +# Errors +anyhow = "1.0.72" +thiserror = "1.0.43" + +# Serde +serde = { version = "1.0.137", features = ["derive"] } +serde_json = { version = "1.0.81" } + +# Utils +derive-getters = "0.2.0" +derive_more = "0.99.17" +derive-new = "0.5.9" +chrono = { version = "0.4", features = ["serde"] } +itertools = "0.10.5" + +# Local dependecies + diff --git a/new-github-indexer/domain/indexer-domain/src/lib.rs b/new-github-indexer/domain/indexer-domain/src/lib.rs new file mode 100644 index 0000000000..946719eda2 --- /dev/null +++ b/new-github-indexer/domain/indexer-domain/src/lib.rs @@ -0,0 +1,2 @@ +mod models; +mod ports; diff --git a/new-github-indexer/domain/indexer-domain/src/models/mod.rs b/new-github-indexer/domain/indexer-domain/src/models/mod.rs new file mode 100644 index 0000000000..7228669282 --- /dev/null +++ b/new-github-indexer/domain/indexer-domain/src/models/mod.rs @@ -0,0 +1,33 @@ +use std::collections::HashMap; + +pub use octocrab_indexer::models::{Author as User, *}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Repository { + pub repo: octocrab_indexer::models::Repository, + pub languages: Languages, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Languages(HashMap); + +impl Languages { + pub fn get_all(&self) -> Vec { + self.0.keys().cloned().collect() + } +} + +impl TryFrom for serde_json::Value { + type Error = serde_json::Error; + + fn try_from(value: Languages) -> Result { + serde_json::to_value(value.0) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Ord)] +pub enum CiChecks { + Passed, + Failed, +} diff --git a/new-github-indexer/domain/indexer-domain/src/ports/mod.rs b/new-github-indexer/domain/indexer-domain/src/ports/mod.rs new file mode 100644 index 0000000000..fbb67250bb --- /dev/null +++ b/new-github-indexer/domain/indexer-domain/src/ports/mod.rs @@ -0,0 +1 @@ +mod output; \ No newline at end of file diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/issue.rs b/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/issue.rs new file mode 100644 index 0000000000..18b3e93e6d --- /dev/null +++ b/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/issue.rs @@ -0,0 +1,16 @@ +use async_trait::async_trait; +use chrono::{DateTime, Utc}; + +use super::Result; +use crate::models::issues::Issue; + +#[async_trait] +pub trait Port: Send + Sync { + async fn issue_by_repo_id(&self, repo_id: u64, issue_number: u64) -> Result; + + async fn issues_by_repo_id( + &self, + repo_id: u64, + updated_since: Option>, + ) -> Result>; +} diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/mod.rs b/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/mod.rs new file mode 100644 index 0000000000..0cb7831db7 --- /dev/null +++ b/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/mod.rs @@ -0,0 +1,11 @@ +mod issue; +mod port; +mod pull_request; +mod repo; +pub use port::Port as GithubApiPort; +mod user; + +use thiserror::Error; +#[derive(Debug, Error)] +pub enum Error {} +pub type Result = std::result::Result; diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/port.rs b/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/port.rs new file mode 100644 index 0000000000..c997b846f6 --- /dev/null +++ b/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/port.rs @@ -0,0 +1,7 @@ +use async_trait::async_trait; + +#[async_trait] +pub trait Port: + super::issue::Port + super::pull_request::Port + super::repo::Port + super::user::Port +{ +} diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/pull_request.rs b/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/pull_request.rs new file mode 100644 index 0000000000..2b8b25a336 --- /dev/null +++ b/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/pull_request.rs @@ -0,0 +1,32 @@ +use async_trait::async_trait; +use chrono::{DateTime, Utc}; + +use super::Result; +use crate::models::{ + commits::Commit, + pulls::{PullRequest, Review}, + CiChecks, +}; + +#[async_trait] +pub trait Port: Send + Sync { + async fn pull_request_by_repo_id( + &self, + repo_id: u64, + pull_request_number: u64, + ) -> Result; + + async fn pull_requests_by_repo_id( + &self, + repo_id: u64, + updated_since: Option>, + ) -> Result>; + + async fn pull_request_commits(&self, pull_request: PullRequest) -> Result>; + + async fn pull_request_reviews(&self, pull_request: PullRequest) -> Result>; + + async fn pull_request_closing_issue_ids(&self, pull_request: PullRequest) -> Result>; + + async fn pull_request_ci_checks(&self, pull_request: PullRequest) -> Result>; +} diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/repo.rs b/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/repo.rs new file mode 100644 index 0000000000..ccb307cd03 --- /dev/null +++ b/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/repo.rs @@ -0,0 +1,9 @@ +use async_trait::async_trait; + +use super::Result; +use crate::models::Repository; + +#[async_trait] +pub trait Port: Send + Sync { + async fn repo_by_id(&self, repo_id: u64) -> Result; +} diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/user.rs b/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/user.rs new file mode 100644 index 0000000000..103ccb486f --- /dev/null +++ b/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/user.rs @@ -0,0 +1,9 @@ +use async_trait::async_trait; + +use super::Result; +use crate::models::User; + +#[async_trait] +pub trait Port: Send + Sync { + async fn user_by_id(&self, user_id: u64) -> Result; +} diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/mod.rs b/new-github-indexer/domain/indexer-domain/src/ports/output/mod.rs new file mode 100644 index 0000000000..fcc26fec7f --- /dev/null +++ b/new-github-indexer/domain/indexer-domain/src/ports/output/mod.rs @@ -0,0 +1 @@ +mod github_api; diff --git a/rest_api_contract/indexer-contract.yaml b/rest_api_contract/indexer-contract.yaml new file mode 100644 index 0000000000..7d1ae0894c --- /dev/null +++ b/rest_api_contract/indexer-contract.yaml @@ -0,0 +1,331 @@ +openapi: 3.0.3 +info: + title: OnlyDust Indexer API + description: OnlyDust Indexer API + version: 1.0.0 +servers: + - url: 'http://localhost:8001' + description: Local + - url: 'https://develop-indexer.onlydust.xyz' + description: Develop + - url: 'https://staging-indexer.onlydust.xyz' + description: Staging + - url: 'https://indexer.onlydust.xyz' + description: Production + +paths: + /api/v1/indexes/repos/{repoId}: + put: + security: + - apiKey: [ ] + tags: + - Indexes + summary: Add repo to the list of repos to index + operationId: addRepoToIndex + description: | + Add repo to the list of repos to index. Does nothing if the repo is already in the list of repos to indexed. + parameters: + - in: path + name: repoId + required: true + description: The ID of the repo to add + schema: + $ref: '#/components/schemas/RepoId' + responses: + "204": + description: 'The repo will be indexed soon' + "401": + $ref: '#/components/responses/UnauthorizedError' + "500": + $ref: '#/components/responses/InternalServerError' + + /api/v1/indexes/users/{userId}: + put: + security: + - apiKey: [ ] + tags: + - Indexes + summary: Add user to the list of users to index + operationId: addUserToIndex + description: | + Add user to the list of users to index. Does nothing if the user is already in the list of users to index. + parameters: + - in: path + name: userId + required: true + description: The ID of the user to add + schema: + $ref: '#/components/schemas/UserId' + responses: + "204": + description: 'The user will be indexed soon' + "401": + $ref: '#/components/responses/UnauthorizedError' + "500": + $ref: '#/components/responses/InternalServerError' + + + /api/v1/users/{userId}: + put: + security: + - apiKey: [ ] + tags: + - Users + summary: Immediately indexes the user (if necessary). + operationId: indexUser + description: | + Immediately indexes the user if it is not indexed yet. Does nothing if the user is already indexed. + parameters: + - in: path + name: userId + required: true + description: The ID of the user to add + schema: + $ref: '#/components/schemas/UserId' + responses: + "204": + description: 'User indexed successfully' + "401": + $ref: '#/components/responses/UnauthorizedError' + "500": + $ref: '#/components/responses/InternalServerError' + + /api/v1/issues/{issueId}: + put: + security: + - apiKey: [ ] + tags: + - Issues + summary: Immediately indexes the issue (if necessary). + operationId: indexIssue + description: | + Immediately indexes the issue if it is not indexed yet. Does nothing if the issue is already indexed. + parameters: + - in: path + name: issueId + required: true + description: The ID of the issue to add + schema: + $ref: '#/components/schemas/IssueId' + responses: + "204": + description: 'Issue indexed successfully' + "401": + $ref: '#/components/responses/UnauthorizedError' + "500": + $ref: '#/components/responses/InternalServerError' + + /api/v1/pull-requests/{pullRequestId}: + put: + security: + - apiKey: [ ] + tags: + - Pull request + summary: Immediately indexes the pull request (if necessary). + operationId: indexPullRequest + description: | + Immediately indexes the pull request if it is not indexed yet. Does nothing if the pull request is already indexed. + parameters: + - in: path + name: pullRequestId + required: true + description: The ID of the pull request to add + schema: + $ref: '#/components/schemas/PullRequestId' + responses: + "204": + description: 'Pull request indexed successfully' + "401": + $ref: '#/components/responses/UnauthorizedError' + "500": + $ref: '#/components/responses/InternalServerError' + + /api/v1/repos: + get: + security: + - apiKey: [ ] + tags: + - Repos + summary: Get a list of repos from an installation ID + operationId: getRepos + parameters: + - in: query + name: installationId + required: true + description: The (external) installation ID + schema: + $ref: '#/components/schemas/AppInstallationId' + responses: + "200": + description: 'Repos' + content: + application/json: + schema: + $ref: '#/components/schemas/RepoListResponse' + "401": + $ref: '#/components/responses/UnauthorizedError' + "500": + $ref: '#/components/responses/InternalServerError' + + /api/v1/app-installations: + get: + security: + - apiKey: [ ] + tags: + - Github App installations + summary: Get a list of installations from a repo ID or an organization ID + operationId: getAppInstallations + parameters: + - in: query + name: repoId + description: The repo ID + schema: + $ref: '#/components/schemas/RepoId' + - in: query + name: organizationId + description: The organization ID + schema: + $ref: '#/components/schemas/OrganizationId' + responses: + "200": + description: 'Github App installations' + content: + application/json: + schema: + $ref: '#/components/schemas/AppInstallationListResponse' + "401": + $ref: '#/components/responses/UnauthorizedError' + "500": + $ref: '#/components/responses/InternalServerError' + +components: + securitySchemes: + apiKey: + type: apiKey + in: header + name: Api-Key + + responses: + InternalServerError: + description: Internal Server Error + content: + application/problem+json: + example: + type: GITHUB_NOT_RESPONDING + title: Github API is not responding + status: 500 + schema: + $ref: '#/components/schemas/OnlyDustError' + UnauthorizedError: + description: Unauthorized + content: + application/problem+json: + example: + type: NOT_AUTHENTICATED + title: You must be authenticated to make this call + status: 401 + schema: + $ref: '#/components/schemas/OnlyDustError' + + schemas: + # ERRORS ================================================================== + OnlyDustError: + type: object + description: Error contract strongly inspired by JSON problem (rfc7807) specification + required: + - type + - title + - status + properties: + type: + type: string + description: A code that identifies the problem type. + example: F.OUT_OF_CREDIT + title: + type: string + description: A short, human-readable summary of the problem type. It SHOULD NOT change from occurrence to occurrence of the problem. + example: Not enough credit. + status: + type: integer + description: The HTTP status code generated by the origin server for this occurrence of the problem. + example: 400 + details: + type: string + description: A human-readable explanation specific to this occurrence of the problem. + example: Budget current balance is $300, but a payment of $500 was requested. + instance: + type: object + description: Arbitrary data that is useful to identifies the specific occurrence of the problem. + example: + - traceId: '19049023845982314' + budgetId: '497f6eca-6276-4993-bfeb-53cbbbba6f08' + balance: 300 + + + UserId: + type: integer + description: Github user ID (databaseId) + example: 595505 + + RepoId: + type: integer + description: Github repo ID (databaseId) + example: 650626566 + + PullRequestId: + type: string + description: Github pull request ID (databaseId) + example: 27218542 + + IssueId: + type: string + description: Github issue ID (databaseId) + example: 1678794252 + + AppInstallationId: + type: integer + description: Github app installation ID + example: 9827589 + + OrganizationId: + type: integer + description: Github organization ID + example: 1983274 + + RepoListResponse: + type: object + properties: + repos: + type: array + items: + $ref: '#/components/schemas/RepoResponse' + + RepoResponse: + type: object + properties: + id: + $ref: '#/components/schemas/RepoId' + organizationId: + $ref: '#/components/schemas/OrganizationId' + appInstallationId: + $ref: '#/components/schemas/AppInstallationId' + + AppInstallationListResponse: + type: object + properties: + installations: + type: array + items: + $ref: '#/components/schemas/AppInstallationResponse' + + AppInstallationResponse: + type: object + properties: + id: + $ref: '#/components/schemas/AppInstallationId' + organizationId: + $ref: '#/components/schemas/OrganizationId' + repos: + type: array + items: + $ref: '#/components/schemas/RepoId' \ No newline at end of file From 0e9db183e4b290b81f1fc73b017ab1de5666c6bc Mon Sep 17 00:00:00 2001 From: Olivier Fuxet Date: Thu, 28 Sep 2023 10:44:42 +0200 Subject: [PATCH 02/11] Indexer domain: add indexing facade and first usecase impl --- .../domain/indexer-domain/src/lib.rs | 1 + .../src/ports/input/indexing_facade_port.rs | 19 ++++++++++++ .../indexer-domain/src/ports/input/mod.rs | 1 + .../domain/indexer-domain/src/ports/mod.rs | 3 +- .../src/ports/output/clean_storage.rs | 30 +++++++++++++++++++ .../src/ports/output/github_api/mod.rs | 2 +- .../src/ports/output/github_api/port.rs | 4 +-- .../indexer-domain/src/ports/output/mod.rs | 4 ++- .../indexer-domain/src/usecases/indexing.rs | 26 ++++++++++++++++ .../domain/indexer-domain/src/usecases/mod.rs | 1 + 10 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 new-github-indexer/domain/indexer-domain/src/ports/input/indexing_facade_port.rs create mode 100644 new-github-indexer/domain/indexer-domain/src/ports/input/mod.rs create mode 100644 new-github-indexer/domain/indexer-domain/src/ports/output/clean_storage.rs create mode 100644 new-github-indexer/domain/indexer-domain/src/usecases/indexing.rs create mode 100644 new-github-indexer/domain/indexer-domain/src/usecases/mod.rs diff --git a/new-github-indexer/domain/indexer-domain/src/lib.rs b/new-github-indexer/domain/indexer-domain/src/lib.rs index 946719eda2..9cf6d3cd02 100644 --- a/new-github-indexer/domain/indexer-domain/src/lib.rs +++ b/new-github-indexer/domain/indexer-domain/src/lib.rs @@ -1,2 +1,3 @@ mod models; mod ports; +mod usecases; diff --git a/new-github-indexer/domain/indexer-domain/src/ports/input/indexing_facade_port.rs b/new-github-indexer/domain/indexer-domain/src/ports/input/indexing_facade_port.rs new file mode 100644 index 0000000000..d71c87c746 --- /dev/null +++ b/new-github-indexer/domain/indexer-domain/src/ports/input/indexing_facade_port.rs @@ -0,0 +1,19 @@ +use async_trait::async_trait; +use thiserror::Error; + +use crate::ports::output::{clean_storage, github_api}; + +#[derive(Debug, Error)] +pub enum Error { + #[error(transparent)] + CleanStorage(#[from] clean_storage::Error), + #[error(transparent)] + GithubApi(#[from] github_api::Error), +} + +pub type Result = std::result::Result; + +#[async_trait] +pub trait IndexingFacadePort: Send + Sync { + async fn index_repo(&self, repo_id: u64) -> Result<()>; +} diff --git a/new-github-indexer/domain/indexer-domain/src/ports/input/mod.rs b/new-github-indexer/domain/indexer-domain/src/ports/input/mod.rs new file mode 100644 index 0000000000..a803551b6d --- /dev/null +++ b/new-github-indexer/domain/indexer-domain/src/ports/input/mod.rs @@ -0,0 +1 @@ +pub mod indexing_facade_port; diff --git a/new-github-indexer/domain/indexer-domain/src/ports/mod.rs b/new-github-indexer/domain/indexer-domain/src/ports/mod.rs index fbb67250bb..7d1a54844d 100644 --- a/new-github-indexer/domain/indexer-domain/src/ports/mod.rs +++ b/new-github-indexer/domain/indexer-domain/src/ports/mod.rs @@ -1 +1,2 @@ -mod output; \ No newline at end of file +pub mod input; +pub mod output; diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/clean_storage.rs b/new-github-indexer/domain/indexer-domain/src/ports/output/clean_storage.rs new file mode 100644 index 0000000000..b1e0d48e59 --- /dev/null +++ b/new-github-indexer/domain/indexer-domain/src/ports/output/clean_storage.rs @@ -0,0 +1,30 @@ +use thiserror::Error; + +use crate::models::{ + commits::Commit, + issues::Issue, + pulls::{PullRequest, Review}, + CiChecks, Repository, User, +}; + +#[derive(Debug, Error)] +pub enum Error {} + +pub type Result = std::result::Result; + +pub trait CleanStoragePort: Send + Sync { + fn save_repo(&self, repo: Repository) -> Result<()>; + + fn save_issue(&self, issue: Issue) -> Result<()>; + + fn save_pull_request( + &self, + pull_request: PullRequest, + commits: Vec, + reviews: Vec, + closing_issue_ids: Vec, + ci_checks: Option, + ) -> Result<()>; + + fn save_user(&self, user: User) -> Result<()>; +} diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/mod.rs b/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/mod.rs index 0cb7831db7..f8280b2ae8 100644 --- a/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/mod.rs +++ b/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/mod.rs @@ -1,8 +1,8 @@ mod issue; mod port; +pub use port::GithubApiPort; mod pull_request; mod repo; -pub use port::Port as GithubApiPort; mod user; use thiserror::Error; diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/port.rs b/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/port.rs index c997b846f6..7ca1c0dda2 100644 --- a/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/port.rs +++ b/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/port.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; #[async_trait] -pub trait Port: - super::issue::Port + super::pull_request::Port + super::repo::Port + super::user::Port +pub trait GithubApiPort: + super::issue::Port + super::pull_request::Port + super::repo::Port + super::user::Port + Send + Sync { } diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/mod.rs b/new-github-indexer/domain/indexer-domain/src/ports/output/mod.rs index fcc26fec7f..c770d908dc 100644 --- a/new-github-indexer/domain/indexer-domain/src/ports/output/mod.rs +++ b/new-github-indexer/domain/indexer-domain/src/ports/output/mod.rs @@ -1 +1,3 @@ -mod github_api; +pub mod github_api; + +pub mod clean_storage; diff --git a/new-github-indexer/domain/indexer-domain/src/usecases/indexing.rs b/new-github-indexer/domain/indexer-domain/src/usecases/indexing.rs new file mode 100644 index 0000000000..803b04e23f --- /dev/null +++ b/new-github-indexer/domain/indexer-domain/src/usecases/indexing.rs @@ -0,0 +1,26 @@ +use std::sync::Arc; + +use async_trait::async_trait; + +use crate::ports::{ + input::indexing_facade_port::IndexingFacadePort, + output::{clean_storage::CleanStoragePort, github_api::GithubApiPort}, +}; + +pub struct Usecase { + github_api: Arc, + clean_storage: Arc, +} + +#[async_trait] +impl IndexingFacadePort for Usecase { + async fn index_repo( + &self, + repo_id: u64, + ) -> crate::ports::input::indexing_facade_port::Result<()> { + let repo = self.github_api.repo_by_id(repo_id).await?; + self.clean_storage.save_repo(repo)?; + //TODO: expose data in the exposition storage + Ok(()) + } +} diff --git a/new-github-indexer/domain/indexer-domain/src/usecases/mod.rs b/new-github-indexer/domain/indexer-domain/src/usecases/mod.rs new file mode 100644 index 0000000000..2b9539206e --- /dev/null +++ b/new-github-indexer/domain/indexer-domain/src/usecases/mod.rs @@ -0,0 +1 @@ +mod indexing; From e9989ea536b5374acb4095cf9b9c05e2923fdb81 Mon Sep 17 00:00:00 2001 From: Olivier Fuxet Date: Thu, 28 Sep 2023 10:52:08 +0200 Subject: [PATCH 03/11] Move domain files --- Cargo.lock | 29 +++++++++++++++++++ Cargo.toml | 3 +- .../domain/{indexer-domain => }/Cargo.toml | 0 .../domain/{indexer-domain => }/src/lib.rs | 0 .../{indexer-domain => }/src/models/mod.rs | 0 .../src/ports/input/indexing_facade_port.rs | 0 .../src/ports/input/mod.rs | 0 .../{indexer-domain => }/src/ports/mod.rs | 0 .../src/ports/output/clean_storage.rs | 0 .../src/ports/output/github_api/issue.rs | 0 .../src/ports/output/github_api/mod.rs | 0 .../src/ports/output/github_api/port.rs | 0 .../ports/output/github_api/pull_request.rs | 0 .../src/ports/output/github_api/repo.rs | 0 .../src/ports/output/github_api/user.rs | 0 .../src/ports/output/mod.rs | 0 .../src/usecases/indexing.rs | 0 .../{indexer-domain => }/src/usecases/mod.rs | 0 18 files changed, 31 insertions(+), 1 deletion(-) rename new-github-indexer/domain/{indexer-domain => }/Cargo.toml (100%) rename new-github-indexer/domain/{indexer-domain => }/src/lib.rs (100%) rename new-github-indexer/domain/{indexer-domain => }/src/models/mod.rs (100%) rename new-github-indexer/domain/{indexer-domain => }/src/ports/input/indexing_facade_port.rs (100%) rename new-github-indexer/domain/{indexer-domain => }/src/ports/input/mod.rs (100%) rename new-github-indexer/domain/{indexer-domain => }/src/ports/mod.rs (100%) rename new-github-indexer/domain/{indexer-domain => }/src/ports/output/clean_storage.rs (100%) rename new-github-indexer/domain/{indexer-domain => }/src/ports/output/github_api/issue.rs (100%) rename new-github-indexer/domain/{indexer-domain => }/src/ports/output/github_api/mod.rs (100%) rename new-github-indexer/domain/{indexer-domain => }/src/ports/output/github_api/port.rs (100%) rename new-github-indexer/domain/{indexer-domain => }/src/ports/output/github_api/pull_request.rs (100%) rename new-github-indexer/domain/{indexer-domain => }/src/ports/output/github_api/repo.rs (100%) rename new-github-indexer/domain/{indexer-domain => }/src/ports/output/github_api/user.rs (100%) rename new-github-indexer/domain/{indexer-domain => }/src/ports/output/mod.rs (100%) rename new-github-indexer/domain/{indexer-domain => }/src/usecases/indexing.rs (100%) rename new-github-indexer/domain/{indexer-domain => }/src/usecases/mod.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 63b97392c3..5b17a15728 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2459,6 +2459,35 @@ dependencies = [ "uuid 1.3.3", ] +[[package]] +name = "indexer-infrastructure" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "derive", + "derive-getters", + "derive-new", + "derive_more", + "diesel", + "diesel-derive-newtype", + "diesel_json", + "dotenv", + "futures", + "indexer-domain", + "infrastructure", + "itertools", + "olog", + "reqwest", + "serde", + "serde_json", + "sha2 0.10.7", + "thiserror", + "url", + "uuid 1.3.3", +] + [[package]] name = "indexmap" version = "1.9.3" diff --git a/Cargo.toml b/Cargo.toml index 12db24da32..85a6b244d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,5 +11,6 @@ members = [ "common/testing", "common/olog", "github-indexer", - "new-github-indexer/domain/indexer-domain", + "new-github-indexer/domain", + "new-github-indexer/infrastructure" ] diff --git a/new-github-indexer/domain/indexer-domain/Cargo.toml b/new-github-indexer/domain/Cargo.toml similarity index 100% rename from new-github-indexer/domain/indexer-domain/Cargo.toml rename to new-github-indexer/domain/Cargo.toml diff --git a/new-github-indexer/domain/indexer-domain/src/lib.rs b/new-github-indexer/domain/src/lib.rs similarity index 100% rename from new-github-indexer/domain/indexer-domain/src/lib.rs rename to new-github-indexer/domain/src/lib.rs diff --git a/new-github-indexer/domain/indexer-domain/src/models/mod.rs b/new-github-indexer/domain/src/models/mod.rs similarity index 100% rename from new-github-indexer/domain/indexer-domain/src/models/mod.rs rename to new-github-indexer/domain/src/models/mod.rs diff --git a/new-github-indexer/domain/indexer-domain/src/ports/input/indexing_facade_port.rs b/new-github-indexer/domain/src/ports/input/indexing_facade_port.rs similarity index 100% rename from new-github-indexer/domain/indexer-domain/src/ports/input/indexing_facade_port.rs rename to new-github-indexer/domain/src/ports/input/indexing_facade_port.rs diff --git a/new-github-indexer/domain/indexer-domain/src/ports/input/mod.rs b/new-github-indexer/domain/src/ports/input/mod.rs similarity index 100% rename from new-github-indexer/domain/indexer-domain/src/ports/input/mod.rs rename to new-github-indexer/domain/src/ports/input/mod.rs diff --git a/new-github-indexer/domain/indexer-domain/src/ports/mod.rs b/new-github-indexer/domain/src/ports/mod.rs similarity index 100% rename from new-github-indexer/domain/indexer-domain/src/ports/mod.rs rename to new-github-indexer/domain/src/ports/mod.rs diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/clean_storage.rs b/new-github-indexer/domain/src/ports/output/clean_storage.rs similarity index 100% rename from new-github-indexer/domain/indexer-domain/src/ports/output/clean_storage.rs rename to new-github-indexer/domain/src/ports/output/clean_storage.rs diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/issue.rs b/new-github-indexer/domain/src/ports/output/github_api/issue.rs similarity index 100% rename from new-github-indexer/domain/indexer-domain/src/ports/output/github_api/issue.rs rename to new-github-indexer/domain/src/ports/output/github_api/issue.rs diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/mod.rs b/new-github-indexer/domain/src/ports/output/github_api/mod.rs similarity index 100% rename from new-github-indexer/domain/indexer-domain/src/ports/output/github_api/mod.rs rename to new-github-indexer/domain/src/ports/output/github_api/mod.rs diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/port.rs b/new-github-indexer/domain/src/ports/output/github_api/port.rs similarity index 100% rename from new-github-indexer/domain/indexer-domain/src/ports/output/github_api/port.rs rename to new-github-indexer/domain/src/ports/output/github_api/port.rs diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/pull_request.rs b/new-github-indexer/domain/src/ports/output/github_api/pull_request.rs similarity index 100% rename from new-github-indexer/domain/indexer-domain/src/ports/output/github_api/pull_request.rs rename to new-github-indexer/domain/src/ports/output/github_api/pull_request.rs diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/repo.rs b/new-github-indexer/domain/src/ports/output/github_api/repo.rs similarity index 100% rename from new-github-indexer/domain/indexer-domain/src/ports/output/github_api/repo.rs rename to new-github-indexer/domain/src/ports/output/github_api/repo.rs diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/github_api/user.rs b/new-github-indexer/domain/src/ports/output/github_api/user.rs similarity index 100% rename from new-github-indexer/domain/indexer-domain/src/ports/output/github_api/user.rs rename to new-github-indexer/domain/src/ports/output/github_api/user.rs diff --git a/new-github-indexer/domain/indexer-domain/src/ports/output/mod.rs b/new-github-indexer/domain/src/ports/output/mod.rs similarity index 100% rename from new-github-indexer/domain/indexer-domain/src/ports/output/mod.rs rename to new-github-indexer/domain/src/ports/output/mod.rs diff --git a/new-github-indexer/domain/indexer-domain/src/usecases/indexing.rs b/new-github-indexer/domain/src/usecases/indexing.rs similarity index 100% rename from new-github-indexer/domain/indexer-domain/src/usecases/indexing.rs rename to new-github-indexer/domain/src/usecases/indexing.rs diff --git a/new-github-indexer/domain/indexer-domain/src/usecases/mod.rs b/new-github-indexer/domain/src/usecases/mod.rs similarity index 100% rename from new-github-indexer/domain/indexer-domain/src/usecases/mod.rs rename to new-github-indexer/domain/src/usecases/mod.rs From 57ab2addd2b1c0656450037be310f0337bf040c0 Mon Sep 17 00:00:00 2001 From: Olivier Fuxet Date: Thu, 28 Sep 2023 10:52:31 +0200 Subject: [PATCH 04/11] Introduce infrastructure crate --- new-github-indexer/infrastructure/Cargo.toml | 57 ++++++++++++++++++++ new-github-indexer/infrastructure/src/lib.rs | 1 + 2 files changed, 58 insertions(+) create mode 100644 new-github-indexer/infrastructure/Cargo.toml create mode 100644 new-github-indexer/infrastructure/src/lib.rs diff --git a/new-github-indexer/infrastructure/Cargo.toml b/new-github-indexer/infrastructure/Cargo.toml new file mode 100644 index 0000000000..4f8689be05 --- /dev/null +++ b/new-github-indexer/infrastructure/Cargo.toml @@ -0,0 +1,57 @@ +[package] +name = "indexer-infrastructure" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] + +# Async +async-trait = "0.1.56" +futures = { version = "0.3.21", features = ["alloc"] } + +# Database +diesel = { version = "2.1.0", default-features = false, features = [ + "postgres", + "extras", + "serde_json", +] } +diesel_json = "0.2.0" +diesel-derive-newtype = "2.1.0" + + +# Core types +uuid = { version = "1.3.3", default_features = false, features = [ + "v4", + "serde", +] } + +# Crypto +sha2 = "0.10.7" + +# Http client +reqwest = { version = "0.11", default-features = false } +url = { version = "2.2.2", features = ["serde"] } + +# Errors +anyhow = "1.0.72" +thiserror = "1.0.43" + +# Serde +serde = { version = "1.0.137", features = ["derive"] } +serde_json = { version = "1.0.81" } + +# Utils +derive-getters = "0.2.0" +derive_more = "0.99.17" +derive-new = "0.5.9" +dotenv = "0.15.0" +chrono = { version = "0.4", features = ["serde"] } +itertools = "0.10.5" + +# Local dependecies +domain = { package = "indexer-domain", path = "../domain" } +infrastructure = { path = "../../common/infrastructure" } +derive = { path = "../../common/derive" } +olog = { path = "../../common/olog" } diff --git a/new-github-indexer/infrastructure/src/lib.rs b/new-github-indexer/infrastructure/src/lib.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/new-github-indexer/infrastructure/src/lib.rs @@ -0,0 +1 @@ + From c97f8020935da0dce37e6255391f232154cf82cc Mon Sep 17 00:00:00 2001 From: Olivier Fuxet Date: Thu, 28 Sep 2023 11:42:47 +0200 Subject: [PATCH 05/11] WIP first implementation of the github_api_adapter --- Cargo.lock | 1 + new-github-indexer/domain/src/lib.rs | 4 +- .../domain/src/ports/output/github_api/mod.rs | 18 ++- new-github-indexer/infrastructure/Cargo.toml | 2 + .../src/adapters/github_api/error.rs | 45 ++++++++ .../adapters/github_api/github_api_adapter.rs | 103 ++++++++++++++++++ .../src/adapters/github_api/mod.rs | 3 + .../infrastructure/src/adapters/mod.rs | 2 + new-github-indexer/infrastructure/src/lib.rs | 2 +- 9 files changed, 172 insertions(+), 8 deletions(-) create mode 100644 new-github-indexer/infrastructure/src/adapters/github_api/error.rs create mode 100644 new-github-indexer/infrastructure/src/adapters/github_api/github_api_adapter.rs create mode 100644 new-github-indexer/infrastructure/src/adapters/github_api/mod.rs create mode 100644 new-github-indexer/infrastructure/src/adapters/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 5b17a15728..23e690d2ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2478,6 +2478,7 @@ dependencies = [ "indexer-domain", "infrastructure", "itertools", + "octocrab 0.30.1", "olog", "reqwest", "serde", diff --git a/new-github-indexer/domain/src/lib.rs b/new-github-indexer/domain/src/lib.rs index 9cf6d3cd02..adda7865bc 100644 --- a/new-github-indexer/domain/src/lib.rs +++ b/new-github-indexer/domain/src/lib.rs @@ -1,3 +1,3 @@ -mod models; -mod ports; +pub mod models; +pub mod ports; mod usecases; diff --git a/new-github-indexer/domain/src/ports/output/github_api/mod.rs b/new-github-indexer/domain/src/ports/output/github_api/mod.rs index f8280b2ae8..304971d1fd 100644 --- a/new-github-indexer/domain/src/ports/output/github_api/mod.rs +++ b/new-github-indexer/domain/src/ports/output/github_api/mod.rs @@ -1,11 +1,19 @@ -mod issue; +pub mod issue; mod port; pub use port::GithubApiPort; -mod pull_request; -mod repo; -mod user; +pub mod pull_request; +pub mod repo; +pub mod user; use thiserror::Error; #[derive(Debug, Error)] -pub enum Error {} +pub enum Error { + #[error("Not found")] + NotFound(#[source] anyhow::Error), + #[error(transparent)] + InvalidInput(anyhow::Error), + #[error(transparent)] + Other(anyhow::Error), +} + pub type Result = std::result::Result; diff --git a/new-github-indexer/infrastructure/Cargo.toml b/new-github-indexer/infrastructure/Cargo.toml index 4f8689be05..c47bae41e2 100644 --- a/new-github-indexer/infrastructure/Cargo.toml +++ b/new-github-indexer/infrastructure/Cargo.toml @@ -7,6 +7,8 @@ edition = "2021" [dependencies] +octocrab_indexer = { package = "octocrab", version = "0.30.1", features = ["stream"] } + # Async async-trait = "0.1.56" futures = { version = "0.3.21", features = ["alloc"] } diff --git a/new-github-indexer/infrastructure/src/adapters/github_api/error.rs b/new-github-indexer/infrastructure/src/adapters/github_api/error.rs new file mode 100644 index 0000000000..e9933627c6 --- /dev/null +++ b/new-github-indexer/infrastructure/src/adapters/github_api/error.rs @@ -0,0 +1,45 @@ +use anyhow::anyhow; +use domain::ports::output::github_api; +use thiserror::Error; +use url::ParseError; + +#[derive(Debug, Error)] +pub enum GithubApiAdapterError { + #[error("Not found")] + NotFound(#[source] anyhow::Error), + #[error(transparent)] + InvalidInput(anyhow::Error), + #[error(transparent)] + Other(anyhow::Error), +} + +impl From for github_api::Error { + fn from(error: GithubApiAdapterError) -> Self { + match error { + GithubApiAdapterError::NotFound(e) => github_api::Error::NotFound(e), + GithubApiAdapterError::InvalidInput(e) => github_api::Error::InvalidInput(e), + GithubApiAdapterError::Other(e) => github_api::Error::Other(e), + } + } +} + +impl From for GithubApiAdapterError { + fn from(error: octocrab_indexer::Error) -> Self { + match &error { + octocrab_indexer::Error::GitHub { + source, + backtrace: _, + } => match source.message.as_str() { + "Not Found" => GithubApiAdapterError::NotFound(anyhow!(error)), + _ => GithubApiAdapterError::Other(anyhow!(error)), + }, + _ => GithubApiAdapterError::Other(anyhow!(error)), + } + } +} + +impl From for GithubApiAdapterError { + fn from(error: ParseError) -> Self { + Self::InvalidInput(anyhow!(error)) + } +} diff --git a/new-github-indexer/infrastructure/src/adapters/github_api/github_api_adapter.rs b/new-github-indexer/infrastructure/src/adapters/github_api/github_api_adapter.rs new file mode 100644 index 0000000000..574851ac1a --- /dev/null +++ b/new-github-indexer/infrastructure/src/adapters/github_api/github_api_adapter.rs @@ -0,0 +1,103 @@ +use std::fmt::Debug; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use domain::{ + models::{ + commits::Commit, + issues::Issue, + pulls::{PullRequest, Review}, + CiChecks, Languages, Repository, User, + }, + ports::output::github_api::*, +}; +use octocrab_indexer::{FromResponse, Octocrab}; + +use crate::adapters::github_api::error::GithubApiAdapterError; + +pub struct GithubApiAdapter { + octocrab: Octocrab, +} + +impl GithubApiAdapter { + pub async fn get_as(&self, route: U) -> Result + where + U: AsRef + Debug + Send, + R: FromResponse, + { + let response = self + .octocrab + .get(route, None::<&()>) + .await + .map_err(GithubApiAdapterError::from)?; + Ok(response) + } +} + +#[async_trait] +impl issue::Port for GithubApiAdapter { + async fn issue_by_repo_id(&self, repo_id: u64, issue_number: u64) -> Result { + todo!() + } + + async fn issues_by_repo_id( + &self, + repo_id: u64, + updated_since: Option>, + ) -> Result> { + todo!() + } +} + +#[async_trait] +impl pull_request::Port for GithubApiAdapter { + async fn pull_request_by_repo_id( + &self, + repo_id: u64, + pull_request_number: u64, + ) -> Result { + todo!() + } + + async fn pull_requests_by_repo_id( + &self, + repo_id: u64, + updated_since: Option>, + ) -> Result> { + todo!() + } + + async fn pull_request_commits(&self, pull_request: PullRequest) -> Result> { + todo!() + } + + async fn pull_request_reviews(&self, pull_request: PullRequest) -> Result> { + todo!() + } + + async fn pull_request_closing_issue_ids(&self, pull_request: PullRequest) -> Result> { + todo!() + } + + async fn pull_request_ci_checks(&self, pull_request: PullRequest) -> Result> { + todo!() + } +} + +#[async_trait] +impl repo::Port for GithubApiAdapter { + async fn repo_by_id(&self, repo_id: u64) -> Result { + let repo: octocrab_indexer::models::Repository = + self.get_as(format!("/repositories/{repo_id}")).await?; + let languages: Languages = + self.get_as(format!("/repositories/{repo_id}/languages")).await?; + Ok(Repository { repo, languages }) + } +} + +#[async_trait] +impl user::Port for GithubApiAdapter { + async fn user_by_id(&self, user_id: u64) -> Result { + todo!() + } +} diff --git a/new-github-indexer/infrastructure/src/adapters/github_api/mod.rs b/new-github-indexer/infrastructure/src/adapters/github_api/mod.rs new file mode 100644 index 0000000000..227e617b18 --- /dev/null +++ b/new-github-indexer/infrastructure/src/adapters/github_api/mod.rs @@ -0,0 +1,3 @@ +mod error; +mod github_api_adapter; +pub use github_api_adapter::GithubApiAdapter; diff --git a/new-github-indexer/infrastructure/src/adapters/mod.rs b/new-github-indexer/infrastructure/src/adapters/mod.rs new file mode 100644 index 0000000000..2949241ad2 --- /dev/null +++ b/new-github-indexer/infrastructure/src/adapters/mod.rs @@ -0,0 +1,2 @@ +mod github_api; +pub use github_api::GithubApiAdapter; diff --git a/new-github-indexer/infrastructure/src/lib.rs b/new-github-indexer/infrastructure/src/lib.rs index 8b13789179..4e9af61890 100644 --- a/new-github-indexer/infrastructure/src/lib.rs +++ b/new-github-indexer/infrastructure/src/lib.rs @@ -1 +1 @@ - +pub mod adapters; From b40dba0f2048d5954cb2177195eb1c33b185cffd Mon Sep 17 00:00:00 2001 From: Olivier Fuxet Date: Thu, 28 Sep 2023 11:54:17 +0200 Subject: [PATCH 06/11] bootstrap clean_storage_adapter --- .../infrastructure/src/adapters/mod.rs | 2 + .../postgres_clean_storage/adapter.rs | 39 +++++++++++++++++++ .../adapters/postgres_clean_storage/mod.rs | 1 + 3 files changed, 42 insertions(+) create mode 100644 new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/adapter.rs create mode 100644 new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/mod.rs diff --git a/new-github-indexer/infrastructure/src/adapters/mod.rs b/new-github-indexer/infrastructure/src/adapters/mod.rs index 2949241ad2..5c5d766a79 100644 --- a/new-github-indexer/infrastructure/src/adapters/mod.rs +++ b/new-github-indexer/infrastructure/src/adapters/mod.rs @@ -1,2 +1,4 @@ mod github_api; +mod postgres_clean_storage; + pub use github_api::GithubApiAdapter; diff --git a/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/adapter.rs b/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/adapter.rs new file mode 100644 index 0000000000..6ef278f3ad --- /dev/null +++ b/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/adapter.rs @@ -0,0 +1,39 @@ +use domain::{ + models::{ + commits::Commit, + issues::Issue, + pulls::{PullRequest, Review}, + CiChecks, Repository, User, + }, + ports::output::clean_storage::{CleanStoragePort, Result}, +}; +use infrastructure::database; + +pub struct PostgresCleanStorageAdapter { + pub postgres_client: database::Client, +} + +impl CleanStoragePort for PostgresCleanStorageAdapter { + fn save_repo(&self, repo: Repository) -> Result<()> { + todo!() + } + + fn save_issue(&self, issue: Issue) -> Result<()> { + todo!() + } + + fn save_pull_request( + &self, + pull_request: PullRequest, + commits: Vec, + reviews: Vec, + closing_issue_ids: Vec, + ci_checks: Option, + ) -> Result<()> { + todo!() + } + + fn save_user(&self, user: User) -> Result<()> { + todo!() + } +} diff --git a/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/mod.rs b/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/mod.rs new file mode 100644 index 0000000000..b82da9f25e --- /dev/null +++ b/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/mod.rs @@ -0,0 +1 @@ +mod adapter; From 9ee95ebb6282cfacf96d018c00bce19679329b0a Mon Sep 17 00:00:00 2001 From: Olivier Fuxet Date: Thu, 28 Sep 2023 12:23:56 +0200 Subject: [PATCH 07/11] Fix warnings --- .../adapters/github_api/github_api_adapter.rs | 24 +++++++++---------- .../postgres_clean_storage/adapter.rs | 16 ++++++------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/new-github-indexer/infrastructure/src/adapters/github_api/github_api_adapter.rs b/new-github-indexer/infrastructure/src/adapters/github_api/github_api_adapter.rs index 574851ac1a..68b0fcfd8a 100644 --- a/new-github-indexer/infrastructure/src/adapters/github_api/github_api_adapter.rs +++ b/new-github-indexer/infrastructure/src/adapters/github_api/github_api_adapter.rs @@ -36,14 +36,14 @@ impl GithubApiAdapter { #[async_trait] impl issue::Port for GithubApiAdapter { - async fn issue_by_repo_id(&self, repo_id: u64, issue_number: u64) -> Result { + async fn issue_by_repo_id(&self, _repo_id: u64, _issue_number: u64) -> Result { todo!() } async fn issues_by_repo_id( &self, - repo_id: u64, - updated_since: Option>, + _repo_id: u64, + _updated_since: Option>, ) -> Result> { todo!() } @@ -53,33 +53,33 @@ impl issue::Port for GithubApiAdapter { impl pull_request::Port for GithubApiAdapter { async fn pull_request_by_repo_id( &self, - repo_id: u64, - pull_request_number: u64, + _repo_id: u64, + _pull_request_number: u64, ) -> Result { todo!() } async fn pull_requests_by_repo_id( &self, - repo_id: u64, - updated_since: Option>, + _repo_id: u64, + _updated_since: Option>, ) -> Result> { todo!() } - async fn pull_request_commits(&self, pull_request: PullRequest) -> Result> { + async fn pull_request_commits(&self, _pull_request: PullRequest) -> Result> { todo!() } - async fn pull_request_reviews(&self, pull_request: PullRequest) -> Result> { + async fn pull_request_reviews(&self, _pull_request: PullRequest) -> Result> { todo!() } - async fn pull_request_closing_issue_ids(&self, pull_request: PullRequest) -> Result> { + async fn pull_request_closing_issue_ids(&self, _pull_request: PullRequest) -> Result> { todo!() } - async fn pull_request_ci_checks(&self, pull_request: PullRequest) -> Result> { + async fn pull_request_ci_checks(&self, _pull_request: PullRequest) -> Result> { todo!() } } @@ -97,7 +97,7 @@ impl repo::Port for GithubApiAdapter { #[async_trait] impl user::Port for GithubApiAdapter { - async fn user_by_id(&self, user_id: u64) -> Result { + async fn user_by_id(&self, _user_id: u64) -> Result { todo!() } } diff --git a/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/adapter.rs b/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/adapter.rs index 6ef278f3ad..670885c000 100644 --- a/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/adapter.rs +++ b/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/adapter.rs @@ -14,26 +14,26 @@ pub struct PostgresCleanStorageAdapter { } impl CleanStoragePort for PostgresCleanStorageAdapter { - fn save_repo(&self, repo: Repository) -> Result<()> { + fn save_repo(&self, _repo: Repository) -> Result<()> { todo!() } - fn save_issue(&self, issue: Issue) -> Result<()> { + fn save_issue(&self, _issue: Issue) -> Result<()> { todo!() } fn save_pull_request( &self, - pull_request: PullRequest, - commits: Vec, - reviews: Vec, - closing_issue_ids: Vec, - ci_checks: Option, + _pull_request: PullRequest, + _commits: Vec, + _reviews: Vec, + _closing_issue_ids: Vec, + _ci_checks: Option, ) -> Result<()> { todo!() } - fn save_user(&self, user: User) -> Result<()> { + fn save_user(&self, _user: User) -> Result<()> { todo!() } } From 5c4d22f369a518030ee12e19038070c1fe1fe0f7 Mon Sep 17 00:00:00 2001 From: Olivier Fuxet Date: Thu, 28 Sep 2023 13:01:57 +0200 Subject: [PATCH 08/11] Put DB client and DB schema in different modules --- api/src/application/budget/allocate.rs | 2 +- .../project/accept_leader_invitation.rs | 2 +- api/src/application/project/add_sponsor.rs | 2 +- api/src/application/project/create.rs | 2 +- .../project/ignored_contributions.rs | 2 +- api/src/application/project/invite_leader.rs | 2 +- api/src/application/project/remove_sponsor.rs | 2 +- api/src/application/project/update.rs | 7 +- api/src/application/quotes/sync.rs | 2 +- api/src/application/sponsor/create.rs | 2 +- api/src/application/sponsor/mod.rs | 4 +- api/src/application/sponsor/update.rs | 2 +- api/src/application/user/onboard.rs | 2 +- .../application/user/update_payout_info.rs | 4 +- api/src/config.rs | 4 +- api/src/domain/projectors/projections.rs | 2 +- .../models/contact_informations/repository.rs | 8 +-- api/src/models/events/repository.rs | 4 +- .../github_pull_request_indexes/repository.rs | 7 +- .../models/github_repo_indexes/repository.rs | 9 ++- .../models/github_user_indexes/repository.rs | 9 ++- api/src/models/payout_info/repository.rs | 7 +- .../models/project_github_repos/repository.rs | 9 ++- .../projects_contributors/repository.rs | 8 +-- .../repository.rs | 12 ++-- .../projects_rewarded_users/repository.rs | 9 ++- .../models/user_profile_info/repository.rs | 12 ++-- api/src/models/work_items/repository.rs | 5 +- api/src/presentation/cron/quotes_syncer.rs | 4 +- api/src/presentation/event_listeners/mod.rs | 4 +- .../event_listeners/quote_syncer.rs | 2 +- api/src/presentation/graphql/context.rs | 2 +- api/src/presentation/graphql/error.rs | 2 +- api/src/presentation/http/bootstrap.rs | 4 +- api/src/presentation/http/mod.rs | 2 +- api/src/presentation/http/routes/graphql.rs | 2 +- api/tests/budget_allocation_it.rs | 3 +- api/tests/create_project_it.rs | 3 +- api/tests/crypto_quotes_sync_it.rs | 2 +- .../quote_sync_upon_budget_creation_it.rs | 2 +- common/derive/src/diesel_immutable_model.rs | 22 +++--- common/derive/src/diesel_model.rs | 6 +- common/infrastructure/src/database/mapping.rs | 2 +- common/infrastructure/src/database/mod.rs | 69 ------------------- .../src/database/repositories/command.rs | 6 +- .../src/{database => dbclient}/config.rs | 0 .../src/{database => dbclient}/error.rs | 0 common/infrastructure/src/dbclient/mod.rs | 69 +++++++++++++++++++ .../src/{database => dbclient}/model/mod.rs | 0 .../model/repository.rs | 14 ++-- common/infrastructure/src/event_store.rs | 2 +- common/infrastructure/src/lib.rs | 1 + common/testing/src/context/database.rs | 10 +-- github-indexer/src/config.rs | 4 +- github-indexer/src/domain/indexers/error.rs | 2 +- .../src/domain/indexers/issue/mod.rs | 2 +- .../src/domain/indexers/issue/projector.rs | 2 +- .../src/domain/indexers/issues/mod.rs | 2 +- .../src/domain/indexers/issues/projector.rs | 2 +- .../src/domain/indexers/repo/mod.rs | 2 +- .../src/domain/indexers/repo/projector.rs | 2 +- .../src/domain/indexers/simple_repo/mod.rs | 2 +- .../domain/indexers/simple_repo/projector.rs | 2 +- .../src/domain/indexers/user/mod.rs | 2 +- .../src/domain/indexers/user/projector.rs | 2 +- github-indexer/src/domain/repository.rs | 6 +- .../src/models/contributions/repository.rs | 5 +- .../src/models/github_issues/repository.rs | 7 +- .../github_pull_request_indexes/repository.rs | 7 +- .../models/github_pull_requests/repository.rs | 22 +++--- .../models/github_repo_indexes/repository.rs | 9 ++- .../models/github_user_indexes/repository.rs | 9 ++- github-indexer/src/models/mod.rs | 6 +- .../models/project_github_repos/repository.rs | 9 ++- .../projects_contributors/repository.rs | 7 +- .../repository.rs | 12 ++-- .../src/presentation/cron/scheduler.rs | 4 +- .../src/presentation/http/bootstrap.rs | 4 +- github-indexer/src/presentation/http/mod.rs | 4 +- .../src/presentation/http/routes/issue.rs | 4 +- .../presentation/http/routes/pull_request.rs | 6 +- .../src/presentation/http/routes/repo.rs | 4 +- .../src/presentation/http/routes/user.rs | 4 +- github-indexer/tests/github_indexing_it.rs | 10 +-- .../postgres_clean_storage/adapter.rs | 4 +- 85 files changed, 265 insertions(+), 279 deletions(-) rename common/infrastructure/src/{database => dbclient}/config.rs (100%) rename common/infrastructure/src/{database => dbclient}/error.rs (100%) create mode 100644 common/infrastructure/src/dbclient/mod.rs rename common/infrastructure/src/{database => dbclient}/model/mod.rs (100%) rename common/infrastructure/src/{database => dbclient}/model/repository.rs (87%) diff --git a/api/src/application/budget/allocate.rs b/api/src/application/budget/allocate.rs index 520e798aaf..fcd1fc5111 100644 --- a/api/src/application/budget/allocate.rs +++ b/api/src/application/budget/allocate.rs @@ -6,7 +6,7 @@ use domain::{ sponsor, AggregateRepository, Amount, Budget, BudgetId, DomainError, Event, Project, ProjectId, Publisher, }; -use infrastructure::database::Repository; +use infrastructure::dbclient::Repository; use tracing::instrument; use crate::{domain::Publishable, models::Sponsor}; diff --git a/api/src/application/project/accept_leader_invitation.rs b/api/src/application/project/accept_leader_invitation.rs index 514dd796d8..bff030c367 100644 --- a/api/src/application/project/accept_leader_invitation.rs +++ b/api/src/application/project/accept_leader_invitation.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use anyhow::{anyhow, Result}; use domain::{AggregateRepository, DomainError, Event, GithubUserId, Project, Publisher, UserId}; -use infrastructure::database::ImmutableRepository; +use infrastructure::dbclient::ImmutableRepository; use tracing::instrument; use crate::{domain::Publishable, models::*}; diff --git a/api/src/application/project/add_sponsor.rs b/api/src/application/project/add_sponsor.rs index f512413eaa..51589dde43 100644 --- a/api/src/application/project/add_sponsor.rs +++ b/api/src/application/project/add_sponsor.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use anyhow::Result; use domain::{sponsor, DomainError, ProjectId}; -use infrastructure::database::ImmutableRepository; +use infrastructure::dbclient::ImmutableRepository; use tracing::instrument; use crate::models::*; diff --git a/api/src/application/project/create.rs b/api/src/application/project/create.rs index a3edde6631..24c3f212ee 100644 --- a/api/src/application/project/create.rs +++ b/api/src/application/project/create.rs @@ -5,7 +5,7 @@ use derive_more::Constructor; use domain::{ sponsor, Amount, BudgetId, DomainError, Event, Project, ProjectId, ProjectVisibility, Publisher, }; -use infrastructure::database::Repository; +use infrastructure::dbclient::Repository; use reqwest::Url; use tracing::instrument; diff --git a/api/src/application/project/ignored_contributions.rs b/api/src/application/project/ignored_contributions.rs index 5a15ec15f9..25ee209937 100644 --- a/api/src/application/project/ignored_contributions.rs +++ b/api/src/application/project/ignored_contributions.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anyhow::Result; use derive_more::Constructor; use domain::{DomainError, ProjectId}; -use infrastructure::database::ImmutableRepository; +use infrastructure::dbclient::ImmutableRepository; use tracing::instrument; use crate::models::*; diff --git a/api/src/application/project/invite_leader.rs b/api/src/application/project/invite_leader.rs index 28000633ff..0657d19856 100644 --- a/api/src/application/project/invite_leader.rs +++ b/api/src/application/project/invite_leader.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use anyhow::Result; use domain::{DomainError, GithubUserId, ProjectId}; -use infrastructure::database::ImmutableRepository; +use infrastructure::dbclient::ImmutableRepository; use tracing::instrument; use crate::models::*; diff --git a/api/src/application/project/remove_sponsor.rs b/api/src/application/project/remove_sponsor.rs index 71182adeaf..dd8cd3bfac 100644 --- a/api/src/application/project/remove_sponsor.rs +++ b/api/src/application/project/remove_sponsor.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use anyhow::Result; use domain::{sponsor, DomainError, ProjectId}; -use infrastructure::database::ImmutableRepository; +use infrastructure::dbclient::ImmutableRepository; use tracing::instrument; use crate::models::*; diff --git a/api/src/application/project/update.rs b/api/src/application/project/update.rs index 40d3d75ace..0921bc0165 100644 --- a/api/src/application/project/update.rs +++ b/api/src/application/project/update.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use anyhow::Result; use domain::{DomainError, ProjectId, ProjectVisibility}; -use infrastructure::database::Repository; +use infrastructure::dbclient::Repository; use juniper::Nullable; use tracing::instrument; use url::Url; @@ -82,8 +82,9 @@ impl Usecase { mod tests { use ::url::Url; - use infrastructure::database::{ - enums::ProjectVisibility as ProjectVisibilityEnum, ImmutableRepository, Result, + use infrastructure::{ + database::enums::ProjectVisibility as ProjectVisibilityEnum, + dbclient::{ImmutableRepository, Result}, }; use mockall::{mock, predicate::eq}; use rstest::{fixture, rstest}; diff --git a/api/src/application/quotes/sync.rs b/api/src/application/quotes/sync.rs index e1b426993e..20564aa2f8 100644 --- a/api/src/application/quotes/sync.rs +++ b/api/src/application/quotes/sync.rs @@ -4,7 +4,7 @@ use anyhow::Result; use chrono::Utc; use derive_more::Constructor; use domain::{services::quotes::Service as QuoteService, Currency}; -use infrastructure::database::Repository; +use infrastructure::dbclient::Repository; use crate::models::CryptoUsdQuote; diff --git a/api/src/application/sponsor/create.rs b/api/src/application/sponsor/create.rs index 4e35de4080..9712c37ea9 100644 --- a/api/src/application/sponsor/create.rs +++ b/api/src/application/sponsor/create.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anyhow::Result; use derive_more::Constructor; use domain::sponsor; -use infrastructure::database::Repository; +use infrastructure::dbclient::Repository; use reqwest::Url; use tracing::instrument; diff --git a/api/src/application/sponsor/mod.rs b/api/src/application/sponsor/mod.rs index f1a67b0cae..3902ed50e2 100644 --- a/api/src/application/sponsor/mod.rs +++ b/api/src/application/sponsor/mod.rs @@ -1,4 +1,4 @@ -use infrastructure::database::DatabaseError; +use infrastructure::dbclient::DatabaseError; use thiserror::Error; use crate::domain::ImageStoreServiceError; @@ -17,7 +17,7 @@ pub enum Error { #[cfg(test)] mod test { use domain::sponsor::Id; - use infrastructure::database::{ImmutableRepository, Repository, Result}; + use infrastructure::dbclient::{ImmutableRepository, Repository, Result}; use mockall::mock; use crate::models::Sponsor; diff --git a/api/src/application/sponsor/update.rs b/api/src/application/sponsor/update.rs index 1eb82c3d0c..b8124a12b7 100644 --- a/api/src/application/sponsor/update.rs +++ b/api/src/application/sponsor/update.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anyhow::Result; use derive_more::Constructor; use domain::sponsor; -use infrastructure::database::Repository; +use infrastructure::dbclient::Repository; use reqwest::Url; use tracing::instrument; diff --git a/api/src/application/user/onboard.rs b/api/src/application/user/onboard.rs index 3e4c7c40c5..26ae58ffc0 100644 --- a/api/src/application/user/onboard.rs +++ b/api/src/application/user/onboard.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use chrono::Utc; use domain::UserId; -use infrastructure::database::{DatabaseError, Repository}; +use infrastructure::dbclient::{DatabaseError, Repository}; use thiserror::Error; use crate::models::*; diff --git a/api/src/application/user/update_payout_info.rs b/api/src/application/user/update_payout_info.rs index ac3b8b3e35..765b1ffb2a 100644 --- a/api/src/application/user/update_payout_info.rs +++ b/api/src/application/user/update_payout_info.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anyhow::anyhow; use derive_more::Constructor; use domain::blockchain::{aptos, evm, starknet, Network}; -use infrastructure::database::DatabaseError; +use infrastructure::dbclient::DatabaseError; use thiserror::Error; use crate::{domain::IsEnsValid, models::*}; @@ -72,7 +72,7 @@ impl Usecase { #[cfg(test)] mod tests { use domain::{blockchain::evm, UserId}; - use infrastructure::database::Result; + use infrastructure::dbclient::Result; use mockall::{mock, predicate::eq}; use rstest::{fixture, rstest}; diff --git a/api/src/config.rs b/api/src/config.rs index 73299eaab6..ecddb93e7c 100644 --- a/api/src/config.rs +++ b/api/src/config.rs @@ -1,4 +1,4 @@ -use infrastructure::{amqp, coinmarketcap, database, github, http as http_client, tracing, web3}; +use infrastructure::{amqp, coinmarketcap, dbclient, github, http as http_client, tracing, web3}; use presentation::http as http_server; use serde::Deserialize; @@ -9,7 +9,7 @@ pub struct Config { #[serde(default)] pub http: http_server::Config, #[serde(default)] - pub database: database::Config, + pub database: dbclient::Config, #[serde(default)] pub amqp: amqp::Config, #[serde(default)] diff --git a/api/src/domain/projectors/projections.rs b/api/src/domain/projectors/projections.rs index 6f4156320f..f1803bd872 100644 --- a/api/src/domain/projectors/projections.rs +++ b/api/src/domain/projectors/projections.rs @@ -7,7 +7,7 @@ use domain::{ ApplicationEvent, BudgetEvent, Event, EventListener, PaymentEvent, PaymentWorkItem, ProjectEvent, SubscriberCallbackError, }; -use infrastructure::database::{ImmutableRepository, Repository}; +use infrastructure::dbclient::{ImmutableRepository, Repository}; use rust_decimal::Decimal; use tracing::instrument; diff --git a/api/src/models/contact_informations/repository.rs b/api/src/models/contact_informations/repository.rs index a08f321d9d..b33a757547 100644 --- a/api/src/models/contact_informations/repository.rs +++ b/api/src/models/contact_informations/repository.rs @@ -1,13 +1,13 @@ use diesel::{Connection, ExpressionMethods, QueryDsl, RunQueryDsl}; use domain::UserId; use infrastructure::{ - contextualized_error::IntoContextualizedError, - database::{self, schema::contact_informations::dsl, Result}, + contextualized_error::IntoContextualizedError, database::schema::contact_informations::dsl, + dbclient, dbclient::Result, }; use super::ContactInformation; -pub trait Repository: database::Repository { +pub trait Repository: dbclient::Repository { fn replace_all_for_user( &self, user_id: &UserId, @@ -15,7 +15,7 @@ pub trait Repository: database::Repository { ) -> Result<()>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn replace_all_for_user( &self, user_id: &UserId, diff --git a/api/src/models/events/repository.rs b/api/src/models/events/repository.rs index 9e8a74b51e..3932a9286f 100644 --- a/api/src/models/events/repository.rs +++ b/api/src/models/events/repository.rs @@ -1,5 +1,5 @@ use diesel::RunQueryDsl; -use infrastructure::database::{self, schema::events, DatabaseError}; +use infrastructure::{database::schema::events, dbclient, dbclient::DatabaseError}; use super::Event; @@ -7,7 +7,7 @@ pub trait Repository: Send + Sync { fn append(&self, event: Event) -> Result<(), DatabaseError>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn append(&self, event: Event) -> Result<(), DatabaseError> { let mut connection = self.connection()?; diesel::insert_into(events::table).values(&event).execute(&mut *connection)?; diff --git a/api/src/models/github_pull_request_indexes/repository.rs b/api/src/models/github_pull_request_indexes/repository.rs index e0ffbfad14..7c02287ef8 100644 --- a/api/src/models/github_pull_request_indexes/repository.rs +++ b/api/src/models/github_pull_request_indexes/repository.rs @@ -2,13 +2,12 @@ use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; use domain::GithubPullRequestId; use infrastructure::{ contextualized_error::IntoContextualizedError, - database, - database::{schema::github_pull_request_indexes::dsl, Result}, + database::schema::github_pull_request_indexes::dsl, dbclient, dbclient::Result, }; use super::GithubPullRequestIndex; -pub trait Repository: database::Repository { +pub trait Repository: dbclient::Repository { fn select_pull_request_indexer_state( &self, pull_request_id: &GithubPullRequestId, @@ -20,7 +19,7 @@ pub trait Repository: database::Repository { ) -> Result<()>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn select_pull_request_indexer_state( &self, pull_request_id: &GithubPullRequestId, diff --git a/api/src/models/github_repo_indexes/repository.rs b/api/src/models/github_repo_indexes/repository.rs index e1676845bf..e9cad6ed0e 100644 --- a/api/src/models/github_repo_indexes/repository.rs +++ b/api/src/models/github_repo_indexes/repository.rs @@ -1,15 +1,14 @@ use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; use domain::GithubRepoId; use infrastructure::{ - contextualized_error::IntoContextualizedError, - database, - database::{schema::github_repo_indexes::dsl, Result}, + contextualized_error::IntoContextualizedError, database::schema::github_repo_indexes::dsl, + dbclient, dbclient::Result, }; use super::GithubRepoIndex; use crate::diesel::OptionalExtension; -pub trait Repository: database::Repository { +pub trait Repository: dbclient::Repository { fn select_repo_indexer_state( &self, repo_id: &GithubRepoId, @@ -43,7 +42,7 @@ pub trait Repository: database::Repository { fn start_indexing(&self, repo_id: GithubRepoId) -> Result<()>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn select_repo_indexer_state( &self, repo_id: &GithubRepoId, diff --git a/api/src/models/github_user_indexes/repository.rs b/api/src/models/github_user_indexes/repository.rs index bc9eed7e95..71e9640122 100644 --- a/api/src/models/github_user_indexes/repository.rs +++ b/api/src/models/github_user_indexes/repository.rs @@ -1,14 +1,13 @@ use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl}; use domain::GithubUserId; use infrastructure::{ - contextualized_error::IntoContextualizedError, - database, - database::{schema::github_user_indexes::dsl, Result}, + contextualized_error::IntoContextualizedError, database::schema::github_user_indexes::dsl, + dbclient, dbclient::Result, }; use super::GithubUserIndex; -pub trait Repository: database::ImmutableRepository { +pub trait Repository: dbclient::ImmutableRepository { fn select_user_indexer_state( &self, user_id: &GithubUserId, @@ -20,7 +19,7 @@ pub trait Repository: database::ImmutableRepository { ) -> Result<()>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn select_user_indexer_state( &self, user_id: &GithubUserId, diff --git a/api/src/models/payout_info/repository.rs b/api/src/models/payout_info/repository.rs index a05b2a6716..553b7443e8 100644 --- a/api/src/models/payout_info/repository.rs +++ b/api/src/models/payout_info/repository.rs @@ -1,5 +1,8 @@ use diesel::{Connection, ExpressionMethods}; -use infrastructure::database::{self, schema::wallets, ImmutableModel, Model, Result}; +use infrastructure::{ + database::schema::wallets, + dbclient::{self, ImmutableModel, Model, Result}, +}; use super::{BankAccount, UserPayoutInfo, Wallet}; @@ -12,7 +15,7 @@ pub trait Repository: Send + Sync { ) -> Result<()>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn upsert( &self, user_info: UserPayoutInfo, diff --git a/api/src/models/project_github_repos/repository.rs b/api/src/models/project_github_repos/repository.rs index 201b443f12..88a0c8a2f7 100644 --- a/api/src/models/project_github_repos/repository.rs +++ b/api/src/models/project_github_repos/repository.rs @@ -1,18 +1,17 @@ use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; use domain::{GithubRepoId, ProjectId}; use infrastructure::{ - contextualized_error::IntoContextualizedError, - database, - database::{schema::project_github_repos::dsl, Result}, + contextualized_error::IntoContextualizedError, database::schema::project_github_repos::dsl, + dbclient, dbclient::Result, }; use super::ProjectGithubRepo; -pub trait Repository: database::ImmutableRepository { +pub trait Repository: dbclient::ImmutableRepository { fn find_projects_of_repo(&self, github_repo_id: &GithubRepoId) -> Result>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn find_projects_of_repo(&self, github_repo_id: &GithubRepoId) -> Result> { let mut connection = self.connection()?; let projects = dsl::project_github_repos diff --git a/api/src/models/projects_contributors/repository.rs b/api/src/models/projects_contributors/repository.rs index c8df31b01d..778029c8d9 100644 --- a/api/src/models/projects_contributors/repository.rs +++ b/api/src/models/projects_contributors/repository.rs @@ -3,21 +3,21 @@ use domain::{GithubRepoId, GithubUserId, ProjectId}; use infrastructure::{ contextualized_error::IntoContextualizedError, database::{ - self, enums::ContributionStatus, schema::{contributions, project_github_repos, projects_contributors::dsl}, - Result, }, + dbclient, + dbclient::Result, }; use super::ProjectsContributor; -pub trait Repository: database::ImmutableRepository { +pub trait Repository: dbclient::ImmutableRepository { fn refresh_project_contributor_list(&self, project_id: &ProjectId) -> Result>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn refresh_project_contributor_list( &self, project_id: &ProjectId, diff --git a/api/src/models/projects_pending_contributors/repository.rs b/api/src/models/projects_pending_contributors/repository.rs index d702c1afc6..b7c250d4e2 100644 --- a/api/src/models/projects_pending_contributors/repository.rs +++ b/api/src/models/projects_pending_contributors/repository.rs @@ -2,23 +2,21 @@ use diesel::{Connection, ExpressionMethods, QueryDsl, RunQueryDsl}; use domain::{GithubRepoId, GithubUserId, ProjectId}; use infrastructure::{ contextualized_error::IntoContextualizedError, - database, - database::{ - schema::{contributions, project_github_repos, projects_pending_contributors::dsl}, - Result, - }, + database::schema::{contributions, project_github_repos, projects_pending_contributors::dsl}, + dbclient, + dbclient::Result, }; use super::ProjectsPendingContributor; -pub trait Repository: database::ImmutableRepository { +pub trait Repository: dbclient::ImmutableRepository { fn refresh_project_pending_contributor_list( &self, project_id: &ProjectId, ) -> Result>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn refresh_project_pending_contributor_list( &self, project_id: &ProjectId, diff --git a/api/src/models/projects_rewarded_users/repository.rs b/api/src/models/projects_rewarded_users/repository.rs index 7330dc7f93..810b2cf287 100644 --- a/api/src/models/projects_rewarded_users/repository.rs +++ b/api/src/models/projects_rewarded_users/repository.rs @@ -1,14 +1,13 @@ use diesel::{Connection, ExpressionMethods, RunQueryDsl}; use domain::{GithubUserId, ProjectId}; use infrastructure::{ - contextualized_error::IntoContextualizedError, - database, - database::{schema::projects_rewarded_users::dsl, Result}, + contextualized_error::IntoContextualizedError, database::schema::projects_rewarded_users::dsl, + dbclient, dbclient::Result, }; use super::ProjectsRewardedUser; -pub trait Repository: database::ImmutableRepository { +pub trait Repository: dbclient::ImmutableRepository { fn increase_user_reward_count_for_project( &self, project_id: &ProjectId, @@ -22,7 +21,7 @@ pub trait Repository: database::ImmutableRepository { ) -> Result<()>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn increase_user_reward_count_for_project( &self, project_id: &ProjectId, diff --git a/api/src/models/user_profile_info/repository.rs b/api/src/models/user_profile_info/repository.rs index 7a8d211a93..5df52a8626 100644 --- a/api/src/models/user_profile_info/repository.rs +++ b/api/src/models/user_profile_info/repository.rs @@ -1,18 +1,18 @@ use diesel::{ExpressionMethods, RunQueryDsl}; use domain::UserId; use infrastructure::{ - contextualized_error::IntoContextualizedError, database, - database::schema::user_profile_info::dsl, + contextualized_error::IntoContextualizedError, database::schema::user_profile_info::dsl, + dbclient, }; use super::UserProfileInfo; -pub trait Repository: database::Repository { - fn upsert_user_avatar(&self, id: UserId, avatar_url: String) -> database::Result<()>; +pub trait Repository: dbclient::Repository { + fn upsert_user_avatar(&self, id: UserId, avatar_url: String) -> dbclient::Result<()>; } -impl Repository for database::Client { - fn upsert_user_avatar(&self, id: UserId, avatar_url: String) -> database::Result<()> { +impl Repository for dbclient::Client { + fn upsert_user_avatar(&self, id: UserId, avatar_url: String) -> dbclient::Result<()> { let mut connection = self.connection()?; diesel::insert_into(dsl::user_profile_info) .values((dsl::id.eq(id), dsl::avatar_url.eq(avatar_url.clone()))) diff --git a/api/src/models/work_items/repository.rs b/api/src/models/work_items/repository.rs index 559d2464ae..508f2a70bc 100644 --- a/api/src/models/work_items/repository.rs +++ b/api/src/models/work_items/repository.rs @@ -2,12 +2,13 @@ use diesel::{ExpressionMethods, RunQueryDsl}; use domain::PaymentId; use infrastructure::{ contextualized_error::IntoContextualizedError, - database::{self, schema::work_items::dsl, Client, Result}, + database::schema::work_items::dsl, + dbclient::{self, Client, Result}, }; use super::WorkItem; -pub trait Repository: database::ImmutableRepository { +pub trait Repository: dbclient::ImmutableRepository { fn delete_by_payment_id(&self, payment_id: PaymentId) -> Result<()>; } diff --git a/api/src/presentation/cron/quotes_syncer.rs b/api/src/presentation/cron/quotes_syncer.rs index c61f39f88b..8ce65a0569 100644 --- a/api/src/presentation/cron/quotes_syncer.rs +++ b/api/src/presentation/cron/quotes_syncer.rs @@ -2,7 +2,7 @@ use std::{sync::Arc, time::Duration}; use anyhow::Result; use domain::currencies; -use infrastructure::{coinmarketcap, database}; +use infrastructure::{coinmarketcap, dbclient}; use olog::info; use tokio::time::Instant; use tokio_cron_scheduler::Job; @@ -19,7 +19,7 @@ pub async fn bootstrap(config: Config) -> Result { } async fn _bootstrap(config: Config) -> Result<()> { - let database = Arc::new(database::Client::new(database::init_pool( + let database = Arc::new(dbclient::Client::new(dbclient::init_pool( config.database.clone(), )?)); diff --git a/api/src/presentation/event_listeners/mod.rs b/api/src/presentation/event_listeners/mod.rs index f48bbef7ae..7202b3da0e 100644 --- a/api/src/presentation/event_listeners/mod.rs +++ b/api/src/presentation/event_listeners/mod.rs @@ -7,7 +7,7 @@ use std::{sync::Arc, time::Duration}; use anyhow::Result; use domain::{currencies, EventListener, LogErr, Message, Subscriber, SubscriberCallbackError}; use futures::future::try_join_all; -use infrastructure::{amqp::UniqueMessage, coinmarketcap, database, event_bus}; +use infrastructure::{amqp::UniqueMessage, coinmarketcap, dbclient, event_bus}; use olog::{error, IntoField}; use tokio::task::JoinHandle; use tokio_cron_scheduler::Job; @@ -45,7 +45,7 @@ async fn _bootstrap(config: Config) -> Result<()> { pub async fn spawn_all(config: Config) -> Result>> { let reqwest = reqwest::Client::new(); - let database = Arc::new(database::Client::new(database::init_pool( + let database = Arc::new(dbclient::Client::new(dbclient::init_pool( config.database.clone(), )?)); let coinmarketcap = Arc::new(coinmarketcap::Client::new( diff --git a/api/src/presentation/event_listeners/quote_syncer.rs b/api/src/presentation/event_listeners/quote_syncer.rs index 0388080847..ab1912fac6 100644 --- a/api/src/presentation/event_listeners/quote_syncer.rs +++ b/api/src/presentation/event_listeners/quote_syncer.rs @@ -5,7 +5,7 @@ use async_trait::async_trait; use chrono::Utc; use derive_more::Constructor; use domain::{currencies, services::quotes, BudgetEvent, Event, SubscriberCallbackError}; -use infrastructure::database::Repository; +use infrastructure::dbclient::Repository; use tracing::instrument; use super::EventListener; diff --git a/api/src/presentation/graphql/context.rs b/api/src/presentation/graphql/context.rs index 18d5069199..738fbd213a 100644 --- a/api/src/presentation/graphql/context.rs +++ b/api/src/presentation/graphql/context.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use domain::{AggregateRepository, Event, GithubUserId, Payment, Project, Publisher, UserId}; use infrastructure::{ - database::{ImmutableRepository, Repository}, + dbclient::{ImmutableRepository, Repository}, github, }; use presentation::http::guards::Claims; diff --git a/api/src/presentation/graphql/error.rs b/api/src/presentation/graphql/error.rs index 7d46eae5cf..ec29793aeb 100644 --- a/api/src/presentation/graphql/error.rs +++ b/api/src/presentation/graphql/error.rs @@ -1,5 +1,5 @@ use domain::{DomainError, ParseCurrencyError, UserId}; -use infrastructure::{database::DatabaseError, web3::ens}; +use infrastructure::{dbclient::DatabaseError, web3::ens}; use juniper::{graphql_value, DefaultScalarValue, FieldError, IntoFieldError}; use olog::{error, IntoField}; use thiserror::Error; diff --git a/api/src/presentation/http/bootstrap.rs b/api/src/presentation/http/bootstrap.rs index 9fda02b1a9..57678811f3 100644 --- a/api/src/presentation/http/bootstrap.rs +++ b/api/src/presentation/http/bootstrap.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use anyhow::Result; use domain::{AggregateRepository, CompositePublisher, EventPublisher}; -use infrastructure::{amqp, database, event_bus::EXCHANGE_NAME, github}; +use infrastructure::{amqp, dbclient, event_bus::EXCHANGE_NAME, github}; use rocket::{Build, Rocket}; use crate::{ @@ -14,7 +14,7 @@ use crate::{ pub async fn bootstrap(config: Config) -> Result> { info!("Bootstrapping api http server"); - let database = Arc::new(database::Client::new(database::init_pool( + let database = Arc::new(dbclient::Client::new(dbclient::init_pool( config.database.clone(), )?)); database.run_migrations()?; diff --git a/api/src/presentation/http/mod.rs b/api/src/presentation/http/mod.rs index 9a603d24a5..fb35299208 100644 --- a/api/src/presentation/http/mod.rs +++ b/api/src/presentation/http/mod.rs @@ -4,7 +4,7 @@ use ::domain::{AggregateRepository, Project}; use domain::{Application, Budget, Event, EventStore, GithubFetchService, Payment, Publisher}; pub use http::Config; use infrastructure::{ - database::{ImmutableRepository, Repository}, + dbclient::{ImmutableRepository, Repository}, github, }; use presentation::http; diff --git a/api/src/presentation/http/routes/graphql.rs b/api/src/presentation/http/routes/graphql.rs index c72c1bd3ae..c8cd3c8d47 100644 --- a/api/src/presentation/http/routes/graphql.rs +++ b/api/src/presentation/http/routes/graphql.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use domain::{AggregateRepository, Event, Payment, Project, Publisher}; use infrastructure::{ - database::{ImmutableRepository, Repository}, + dbclient::{ImmutableRepository, Repository}, github, }; use juniper_rocket::{GraphQLRequest, GraphQLResponse}; diff --git a/api/tests/budget_allocation_it.rs b/api/tests/budget_allocation_it.rs index 555805ce52..5e1030662f 100644 --- a/api/tests/budget_allocation_it.rs +++ b/api/tests/budget_allocation_it.rs @@ -5,8 +5,7 @@ use anyhow::Result; use api::{models as api_models, presentation::http::routes::projects::budgets::Response}; use domain::{currencies, sponsor, BudgetEvent, BudgetId, Event, ProjectEvent, ProjectId}; use infrastructure::{ - database::{enums::Currency, ImmutableRepository}, - event_bus::EXCHANGE_NAME, + database::enums::Currency, dbclient::ImmutableRepository, event_bus::EXCHANGE_NAME, }; use olog::info; use rocket::{ diff --git a/api/tests/create_project_it.rs b/api/tests/create_project_it.rs index 75cac1191a..72bd8f635d 100644 --- a/api/tests/create_project_it.rs +++ b/api/tests/create_project_it.rs @@ -6,8 +6,7 @@ use api::{models::Sponsor, presentation::http::routes::projects}; use diesel::RunQueryDsl; use domain::{currencies, sponsor, BudgetEvent, Event, ProjectEvent}; use infrastructure::{ - database::{schema::project_details, ImmutableRepository}, - event_bus::EXCHANGE_NAME, + database::schema::project_details, dbclient::ImmutableRepository, event_bus::EXCHANGE_NAME, }; use olog::info; use rocket::{ diff --git a/api/tests/crypto_quotes_sync_it.rs b/api/tests/crypto_quotes_sync_it.rs index 48f07b50af..26b451bf8e 100644 --- a/api/tests/crypto_quotes_sync_it.rs +++ b/api/tests/crypto_quotes_sync_it.rs @@ -6,7 +6,7 @@ use anyhow::Result; use api::models::CryptoUsdQuote; use assert_matches::assert_matches; use chrono::Utc; -use infrastructure::database::{enums::Currency, ImmutableRepository}; +use infrastructure::{database::enums::Currency, dbclient::ImmutableRepository}; use olog::info; use rstest::rstest; use rust_decimal::Decimal; diff --git a/api/tests/quote_sync_upon_budget_creation_it.rs b/api/tests/quote_sync_upon_budget_creation_it.rs index 046fcb04a1..eb6e1adb56 100644 --- a/api/tests/quote_sync_upon_budget_creation_it.rs +++ b/api/tests/quote_sync_upon_budget_creation_it.rs @@ -3,7 +3,7 @@ use api::models::CryptoUsdQuote; use chrono::Utc; use domain::{currencies, BudgetEvent, BudgetId}; use fixtures::*; -use infrastructure::database::{enums::Currency, ImmutableRepository}; +use infrastructure::{database::enums::Currency, dbclient::ImmutableRepository}; use olog::info; use rstest::rstest; use rust_decimal_macros::dec; diff --git a/common/derive/src/diesel_immutable_model.rs b/common/derive/src/diesel_immutable_model.rs index feb16c9e3a..d23bf335fa 100644 --- a/common/derive/src/diesel_immutable_model.rs +++ b/common/derive/src/diesel_immutable_model.rs @@ -5,12 +5,12 @@ pub fn impl_derive(derive_input: syn::DeriveInput) -> TokenStream { let name = &derive_input.ident; let expanded = quote!( - impl ::infrastructure::database::ImmutableModel<::diesel::pg::PgConnection> for #name + impl ::infrastructure::dbclient::ImmutableModel<::diesel::pg::PgConnection> for #name { fn exists( connection: &mut ::diesel::pg::PgConnection, id: ::Id, - ) -> ::infrastructure::database::Result { + ) -> ::infrastructure::dbclient::Result { use ::diesel::{associations::HasTable, QueryDsl, RunQueryDsl}; use infrastructure::contextualized_error::IntoContextualizedError; ::diesel::select(::diesel::dsl::exists(::table().find(id.clone()))) @@ -22,7 +22,7 @@ pub fn impl_derive(derive_input: syn::DeriveInput) -> TokenStream { fn find_by_id( connection: &mut ::diesel::pg::PgConnection, id: ::Id, - ) -> ::infrastructure::database::Result { + ) -> ::infrastructure::dbclient::Result { use ::diesel::{associations::HasTable, QueryDsl, RunQueryDsl}; use infrastructure::contextualized_error::IntoContextualizedError; ::table().find(id.clone()).first(&mut *connection) @@ -34,7 +34,7 @@ pub fn impl_derive(derive_input: syn::DeriveInput) -> TokenStream { fn try_find_by_id( connection: &mut ::diesel::pg::PgConnection, id: ::Id, - ) -> ::infrastructure::database::Result> { + ) -> ::infrastructure::dbclient::Result> { use ::diesel::{associations::HasTable, QueryDsl, RunQueryDsl, OptionalExtension}; use infrastructure::contextualized_error::IntoContextualizedError; ::table().find(id.clone()).first(&mut *connection) @@ -46,7 +46,7 @@ pub fn impl_derive(derive_input: syn::DeriveInput) -> TokenStream { fn list( connection: &mut ::diesel::pg::PgConnection, - ) -> ::infrastructure::database::Result> { + ) -> ::infrastructure::dbclient::Result> { use ::diesel::{associations::HasTable, RunQueryDsl}; use infrastructure::contextualized_error::IntoContextualizedError; ::table().load(connection) @@ -57,7 +57,7 @@ pub fn impl_derive(derive_input: syn::DeriveInput) -> TokenStream { fn insert( self, connection: &mut ::diesel::pg::PgConnection, - ) -> ::infrastructure::database::Result { + ) -> ::infrastructure::dbclient::Result { use ::diesel::{associations::HasTable, RunQueryDsl}; use infrastructure::contextualized_error::IntoContextualizedError; ::diesel::insert_into(::table()) @@ -70,7 +70,7 @@ pub fn impl_derive(derive_input: syn::DeriveInput) -> TokenStream { fn insert_all( connection: &mut ::diesel::pg::PgConnection, values: Vec, - ) -> ::infrastructure::database::Result { + ) -> ::infrastructure::dbclient::Result { use ::diesel::{associations::HasTable, RunQueryDsl}; use infrastructure::contextualized_error::IntoContextualizedError; ::diesel::insert_into(::table()) @@ -83,7 +83,7 @@ pub fn impl_derive(derive_input: syn::DeriveInput) -> TokenStream { fn try_insert( self, connection: &mut ::diesel::pg::PgConnection, - ) -> ::infrastructure::database::Result> { + ) -> ::infrastructure::dbclient::Result> { use ::diesel::{associations::HasTable, OptionalExtension, RunQueryDsl}; use infrastructure::contextualized_error::IntoContextualizedError; ::diesel::insert_into(::table()) @@ -98,7 +98,7 @@ pub fn impl_derive(derive_input: syn::DeriveInput) -> TokenStream { fn delete( connection: &mut ::diesel::pg::PgConnection, id: ::Id, - ) -> ::infrastructure::database::Result> { + ) -> ::infrastructure::dbclient::Result> { use ::diesel::{associations::HasTable, OptionalExtension, EqAll, RunQueryDsl, Table}; use infrastructure::contextualized_error::IntoContextualizedError; diesel::delete(::table()) @@ -112,7 +112,7 @@ pub fn impl_derive(derive_input: syn::DeriveInput) -> TokenStream { fn delete_all

( connection: &mut ::diesel::pg::PgConnection, predicate: P, - ) -> ::infrastructure::database::Result + ) -> ::infrastructure::dbclient::Result where ::diesel::query_builder::DeleteStatement: ::diesel::query_dsl::methods::FilterDsl

, @@ -130,7 +130,7 @@ pub fn impl_derive(derive_input: syn::DeriveInput) -> TokenStream { fn clear( connection: &mut ::diesel::pg::PgConnection, - ) -> ::infrastructure::database::Result<()> { + ) -> ::infrastructure::dbclient::Result<()> { use ::diesel::{associations::HasTable, RunQueryDsl}; use infrastructure::contextualized_error::IntoContextualizedError; diesel::delete(::table()) diff --git a/common/derive/src/diesel_model.rs b/common/derive/src/diesel_model.rs index df438cc9cd..60bc710e7f 100644 --- a/common/derive/src/diesel_model.rs +++ b/common/derive/src/diesel_model.rs @@ -5,12 +5,12 @@ pub fn impl_derive(derive_input: syn::DeriveInput) -> TokenStream { let name = &derive_input.ident; let expanded = quote!( - impl ::infrastructure::database::Model<::diesel::pg::PgConnection> for #name + impl ::infrastructure::dbclient::Model<::diesel::pg::PgConnection> for #name { fn update( self, connection: &mut ::diesel::pg::PgConnection, - ) -> ::infrastructure::database::Result { + ) -> ::infrastructure::dbclient::Result { use ::diesel::{associations::HasTable, RunQueryDsl, Table}; use infrastructure::contextualized_error::IntoContextualizedError; diesel::update(&self).set(&self).get_result(connection) @@ -21,7 +21,7 @@ pub fn impl_derive(derive_input: syn::DeriveInput) -> TokenStream { fn upsert( self, connection: &mut ::diesel::pg::PgConnection, - ) -> ::infrastructure::database::Result { + ) -> ::infrastructure::dbclient::Result { use ::diesel::{associations::HasTable, RunQueryDsl, Table}; use infrastructure::contextualized_error::IntoContextualizedError; diesel::insert_into(::table()) diff --git a/common/infrastructure/src/database/mapping.rs b/common/infrastructure/src/database/mapping.rs index 42a1dc797d..528dd5121b 100644 --- a/common/infrastructure/src/database/mapping.rs +++ b/common/infrastructure/src/database/mapping.rs @@ -1,6 +1,6 @@ use domain::Entity; -use crate::database::DatabaseError; +use crate::dbclient::DatabaseError; pub trait Repository: Send + Sync { fn upsert(&self, id1: &E1::Id, id2: &E2::Id) -> Result<(), DatabaseError>; diff --git a/common/infrastructure/src/database/mod.rs b/common/infrastructure/src/database/mod.rs index 34fd330ab7..558345248f 100644 --- a/common/infrastructure/src/database/mod.rs +++ b/common/infrastructure/src/database/mod.rs @@ -1,72 +1,3 @@ -mod config; pub mod enums; -mod error; pub mod repositories; pub mod schema; - -mod model; -use anyhow::anyhow; -use diesel::{ - pg::PgConnection, - r2d2::{self, ConnectionManager}, -}; -use diesel_migrations::EmbeddedMigrations; -pub use model::{ImmutableModel, ImmutableRepository, Model, Repository}; -use olog::{error, IntoField}; - -pub use self::{ - config::Config, - error::{Error as DatabaseError, Result}, -}; -use crate::diesel_migrations::MigrationHarness; - -pub type Pool = r2d2::Pool>; -type PooledConnection = r2d2::PooledConnection>; - -pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!(); - -pub fn run_migrations(pool: &Pool) { - let mut connection = pool.get().expect("Unable to get connection from pool"); - connection.run_pending_migrations(MIGRATIONS).expect("diesel migration failure"); -} - -pub struct Client { - pool: Pool, -} - -impl Client { - pub fn new(pool: Pool) -> Self { - Self { pool } - } -} - -impl Client { - pub fn connection(&self) -> Result { - self.pool.get().map_err(|e| { - error!( - error = e.to_field(), - "Failed to connect to get connection out of pool" - ); - DatabaseError::Connection(e.into()) - }) - } - - pub fn run_migrations(&self) -> Result<()> { - let mut connection = self.connection()?; - connection.run_pending_migrations(MIGRATIONS).map_err(|e| { - error!(error = e.to_field(), "Failed to run migrations"); - DatabaseError::Migration(anyhow!(e)) - })?; - Ok(()) - } -} - -pub fn init_pool(config: Config) -> Result { - let manager = ConnectionManager::::new(config.url); - let pool = Pool::builder() - .max_size(config.pool_max_size) - .build(manager) - .map_err(|e| DatabaseError::Pool(anyhow!(e)))?; - - Ok(pool) -} diff --git a/common/infrastructure/src/database/repositories/command.rs b/common/infrastructure/src/database/repositories/command.rs index bf0f7dc280..a8f0450bbb 100644 --- a/common/infrastructure/src/database/repositories/command.rs +++ b/common/infrastructure/src/database/repositories/command.rs @@ -3,9 +3,9 @@ use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl}; use domain::{CommandId, CommandRepository}; use serde::{Deserialize, Serialize}; -use crate::database::{ - schema::{commands, commands::dsl}, - Client, +use crate::{ + database::schema::{commands, commands::dsl}, + dbclient::Client, }; #[derive( diff --git a/common/infrastructure/src/database/config.rs b/common/infrastructure/src/dbclient/config.rs similarity index 100% rename from common/infrastructure/src/database/config.rs rename to common/infrastructure/src/dbclient/config.rs diff --git a/common/infrastructure/src/database/error.rs b/common/infrastructure/src/dbclient/error.rs similarity index 100% rename from common/infrastructure/src/database/error.rs rename to common/infrastructure/src/dbclient/error.rs diff --git a/common/infrastructure/src/dbclient/mod.rs b/common/infrastructure/src/dbclient/mod.rs new file mode 100644 index 0000000000..3201fd020c --- /dev/null +++ b/common/infrastructure/src/dbclient/mod.rs @@ -0,0 +1,69 @@ +pub mod config; +pub mod error; +pub mod model; + +use anyhow::anyhow; +use diesel::{ + pg::PgConnection, + r2d2::{self, ConnectionManager}, +}; +use diesel_migrations::EmbeddedMigrations; +pub use model::{ImmutableModel, ImmutableRepository, Model, Repository}; +use olog::{error, IntoField}; + +pub use self::{ + config::Config, + error::{Error as DatabaseError, Result}, +}; +use crate::diesel_migrations::MigrationHarness; + +pub type Pool = r2d2::Pool>; +type PooledConnection = r2d2::PooledConnection>; + +pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!(); + +pub fn run_migrations(pool: &Pool) { + let mut connection = pool.get().expect("Unable to get connection from pool"); + connection.run_pending_migrations(MIGRATIONS).expect("diesel migration failure"); +} + +pub struct Client { + pool: Pool, +} + +impl Client { + pub fn new(pool: Pool) -> Self { + Self { pool } + } +} + +impl Client { + pub fn connection(&self) -> Result { + self.pool.get().map_err(|e| { + error!( + error = e.to_field(), + "Failed to connect to get connection out of pool" + ); + DatabaseError::Connection(e.into()) + }) + } + + pub fn run_migrations(&self) -> Result<()> { + let mut connection = self.connection()?; + connection.run_pending_migrations(MIGRATIONS).map_err(|e| { + error!(error = e.to_field(), "Failed to run migrations"); + DatabaseError::Migration(anyhow!(e)) + })?; + Ok(()) + } +} + +pub fn init_pool(config: Config) -> Result { + let manager = ConnectionManager::::new(config.url); + let pool = Pool::builder() + .max_size(config.pool_max_size) + .build(manager) + .map_err(|e| DatabaseError::Pool(anyhow!(e)))?; + + Ok(pool) +} diff --git a/common/infrastructure/src/database/model/mod.rs b/common/infrastructure/src/dbclient/model/mod.rs similarity index 100% rename from common/infrastructure/src/database/model/mod.rs rename to common/infrastructure/src/dbclient/model/mod.rs diff --git a/common/infrastructure/src/database/model/repository.rs b/common/infrastructure/src/dbclient/model/repository.rs similarity index 87% rename from common/infrastructure/src/database/model/repository.rs rename to common/infrastructure/src/dbclient/model/repository.rs index 32d074b74c..506a3a20b7 100644 --- a/common/infrastructure/src/database/model/repository.rs +++ b/common/infrastructure/src/dbclient/model/repository.rs @@ -1,7 +1,7 @@ use diesel::{Connection, Identifiable, PgConnection}; use super::{ImmutableModel, Model, Result}; -use crate::{contextualized_error::ContextualizedError, database}; +use crate::{contextualized_error::ContextualizedError, dbclient}; pub trait ImmutableRepository: Send + Sync where @@ -20,7 +20,7 @@ where fn try_insert_all(&self, models: Vec) -> Result<()>; } -impl ImmutableRepository for database::Client +impl ImmutableRepository for dbclient::Client where M: ImmutableModel, { @@ -72,7 +72,7 @@ where fn try_insert_all(&self, models: Vec) -> Result<()> { let mut connection = self.connection()?; - connection.transaction::<(), database::error::Error, _>(|tx| { + connection.transaction::<(), dbclient::error::Error, _>(|tx| { for model in models { model.try_insert(&mut *tx)?; } @@ -91,7 +91,7 @@ where fn upsert(&self, model: M) -> Result; } -impl Repository for database::Client +impl Repository for dbclient::Client where M: Model, { @@ -111,10 +111,10 @@ where } } -// Useful to make transaction::<(), database::error::Error, _> transactions -impl From for database::error::Error { +// Useful to make transaction::<(), dbclient::error::Error, _> transactions +impl From for dbclient::error::Error { fn from(error: diesel::result::Error) -> Self { - database::error::Error::Transaction(ContextualizedError::new( + dbclient::error::Error::Transaction(ContextualizedError::new( "Error in transaction".to_string(), error, )) diff --git a/common/infrastructure/src/event_store.rs b/common/infrastructure/src/event_store.rs index 1e8117ee2b..7db1ee876f 100644 --- a/common/infrastructure/src/event_store.rs +++ b/common/infrastructure/src/event_store.rs @@ -4,7 +4,7 @@ use olog::{error, IntoField}; use serde_json::Value; use tracing::instrument; -use crate::database::{schema::events, Client}; +use crate::{database::schema::events, dbclient::Client}; pub trait Named { fn name() -> String; diff --git a/common/infrastructure/src/lib.rs b/common/infrastructure/src/lib.rs index 79adab96eb..88519186f4 100644 --- a/common/infrastructure/src/lib.rs +++ b/common/infrastructure/src/lib.rs @@ -6,6 +6,7 @@ pub mod coinmarketcap; pub mod config; pub mod contextualized_error; pub mod database; +pub mod dbclient; pub mod event_bus; pub mod event_store; pub mod github; diff --git a/common/testing/src/context/database.rs b/common/testing/src/context/database.rs index 013cc3f03a..387324b3f4 100644 --- a/common/testing/src/context/database.rs +++ b/common/testing/src/context/database.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use anyhow::{anyhow, Result}; -use infrastructure::database; +use infrastructure::dbclient; use testcontainers::{ clients::Cli, core::WaitFor, images::generic::GenericImage, Container, RunnableImage, }; @@ -12,8 +12,8 @@ static DATABASE: &str = "marketplace_db"; pub struct Context<'docker> { _container: Container<'docker, GenericImage>, - pub config: database::Config, - pub client: Arc, + pub config: dbclient::Config, + pub client: Arc, } impl<'docker> Context<'docker> { @@ -24,12 +24,12 @@ impl<'docker> Context<'docker> { .map_to_host_port_ipv4(5432) .ok_or(anyhow!("Invalid postgres port"))?; - let config = database::Config { + let config = dbclient::Config { url: format!("postgres://{USER}:{PASSWORD}@localhost:{port}/{DATABASE}"), pool_max_size: 2, }; - let client = database::Client::new(database::init_pool(config.clone())?); + let client = dbclient::Client::new(dbclient::init_pool(config.clone())?); client.run_migrations()?; diff --git a/github-indexer/src/config.rs b/github-indexer/src/config.rs index 283ae75931..1e19373069 100644 --- a/github-indexer/src/config.rs +++ b/github-indexer/src/config.rs @@ -1,11 +1,11 @@ -use infrastructure::{database, github, tracing}; +use infrastructure::{dbclient, github, tracing}; use presentation::http; use serde::Deserialize; #[derive(Deserialize, Default, Clone)] pub struct Config { #[serde(default)] - pub database: database::Config, + pub database: dbclient::Config, #[serde(default)] pub tracer: tracing::Config, #[serde(default)] diff --git a/github-indexer/src/domain/indexers/error.rs b/github-indexer/src/domain/indexers/error.rs index 42c38aea89..c4d7ee4b03 100644 --- a/github-indexer/src/domain/indexers/error.rs +++ b/github-indexer/src/domain/indexers/error.rs @@ -1,5 +1,5 @@ use domain::GithubServiceError; -use infrastructure::database::DatabaseError; +use infrastructure::dbclient::DatabaseError; use olog::warn; use thiserror::Error; diff --git a/github-indexer/src/domain/indexers/issue/mod.rs b/github-indexer/src/domain/indexers/issue/mod.rs index 0644ba04d8..ad7c191137 100644 --- a/github-indexer/src/domain/indexers/issue/mod.rs +++ b/github-indexer/src/domain/indexers/issue/mod.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use domain::{GithubFetchIssueService, GithubIssue, GithubIssueNumber, GithubRepoId}; -use infrastructure::database::Repository; +use infrastructure::dbclient::Repository; use self::{crawler::IssueCrawler, projector::IssueProjector}; use super::{Crawler, IndexerImpl, Projector}; diff --git a/github-indexer/src/domain/indexers/issue/projector.rs b/github-indexer/src/domain/indexers/issue/projector.rs index 705ced2b48..5da684e199 100644 --- a/github-indexer/src/domain/indexers/issue/projector.rs +++ b/github-indexer/src/domain/indexers/issue/projector.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use derive_new::new; use domain::GithubIssue; -use infrastructure::database::Repository; +use infrastructure::dbclient::Repository; use super::{super::error::Result, Projector}; diff --git a/github-indexer/src/domain/indexers/issues/mod.rs b/github-indexer/src/domain/indexers/issues/mod.rs index 0adcfa86b8..87e7b46874 100644 --- a/github-indexer/src/domain/indexers/issues/mod.rs +++ b/github-indexer/src/domain/indexers/issues/mod.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use domain::{GithubFetchIssueService, GithubIssue, GithubRepoId}; -use infrastructure::database::Repository; +use infrastructure::dbclient::Repository; use self::{crawler::IssuesCrawler, projector::IssuesProjector}; use super::{contributors_projector::ContributorsProjector, Crawler, IndexerImpl, Projector}; diff --git a/github-indexer/src/domain/indexers/issues/projector.rs b/github-indexer/src/domain/indexers/issues/projector.rs index de3e4c48a3..bf59d7231f 100644 --- a/github-indexer/src/domain/indexers/issues/projector.rs +++ b/github-indexer/src/domain/indexers/issues/projector.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use derive_new::new; use domain::GithubIssue; -use infrastructure::database::Repository; +use infrastructure::dbclient::Repository; use super::{super::error::Result, Projector}; use crate::{ diff --git a/github-indexer/src/domain/indexers/repo/mod.rs b/github-indexer/src/domain/indexers/repo/mod.rs index c4df0513ee..b76aeaec93 100644 --- a/github-indexer/src/domain/indexers/repo/mod.rs +++ b/github-indexer/src/domain/indexers/repo/mod.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use domain::{GithubFetchRepoService, GithubRepoId, Languages}; -use infrastructure::database::{ImmutableRepository, Repository}; +use infrastructure::dbclient::{ImmutableRepository, Repository}; use serde::{Deserialize, Serialize}; use self::{crawler::RepoCrawler, projector::RepoProjector}; diff --git a/github-indexer/src/domain/indexers/repo/projector.rs b/github-indexer/src/domain/indexers/repo/projector.rs index d0370a77c2..a345161fac 100644 --- a/github-indexer/src/domain/indexers/repo/projector.rs +++ b/github-indexer/src/domain/indexers/repo/projector.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use chrono::Utc; use derive_new::new; -use infrastructure::database::{ImmutableRepository, Repository}; +use infrastructure::dbclient::{ImmutableRepository, Repository}; use super::{super::error::Result, IndexedRepo}; use crate::{ diff --git a/github-indexer/src/domain/indexers/simple_repo/mod.rs b/github-indexer/src/domain/indexers/simple_repo/mod.rs index 20d27d20c4..95a0dcee61 100644 --- a/github-indexer/src/domain/indexers/simple_repo/mod.rs +++ b/github-indexer/src/domain/indexers/simple_repo/mod.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use domain::{GithubFetchRepoService, GithubRepo, GithubRepoId}; -use infrastructure::database::Repository; +use infrastructure::dbclient::Repository; use self::{crawler::RepoCrawler, projector::RepoProjector}; use super::IndexerImpl; diff --git a/github-indexer/src/domain/indexers/simple_repo/projector.rs b/github-indexer/src/domain/indexers/simple_repo/projector.rs index 817db40eb0..31a901d6ca 100644 --- a/github-indexer/src/domain/indexers/simple_repo/projector.rs +++ b/github-indexer/src/domain/indexers/simple_repo/projector.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use chrono::Utc; use derive_new::new; -use infrastructure::database::Repository; +use infrastructure::dbclient::Repository; use serde_json::json; use super::super::error::Result; diff --git a/github-indexer/src/domain/indexers/user/mod.rs b/github-indexer/src/domain/indexers/user/mod.rs index c922315496..8a24b28384 100644 --- a/github-indexer/src/domain/indexers/user/mod.rs +++ b/github-indexer/src/domain/indexers/user/mod.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use domain::{GithubFetchService, GithubFullUser, GithubUserId}; -use infrastructure::database::Repository; +use infrastructure::dbclient::Repository; use self::{crawler::UserCrawler, projector::UserProjector}; use super::{Crawler, IndexerImpl, Projector}; diff --git a/github-indexer/src/domain/indexers/user/projector.rs b/github-indexer/src/domain/indexers/user/projector.rs index 5022839037..7d7cfcfd41 100644 --- a/github-indexer/src/domain/indexers/user/projector.rs +++ b/github-indexer/src/domain/indexers/user/projector.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use derive_new::new; use domain::GithubFullUser; -use infrastructure::database::Repository; +use infrastructure::dbclient::Repository; use super::{super::error::Result, Projector}; use crate::models::GithubUser; diff --git a/github-indexer/src/domain/repository.rs b/github-indexer/src/domain/repository.rs index b07fbd8914..cd8cefcf6d 100644 --- a/github-indexer/src/domain/repository.rs +++ b/github-indexer/src/domain/repository.rs @@ -3,10 +3,8 @@ use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; use domain::{GithubRepoId, GithubUserId}; use infrastructure::{ contextualized_error::IntoContextualizedError, - database::{ - schema::{github_repo_indexes, github_user_indexes}, - Client, Result, - }, + database::schema::{github_repo_indexes, github_user_indexes}, + dbclient::{Client, Result}, }; use crate::models::GithubUserIndex; diff --git a/github-indexer/src/models/contributions/repository.rs b/github-indexer/src/models/contributions/repository.rs index 363c49a311..221071765b 100644 --- a/github-indexer/src/models/contributions/repository.rs +++ b/github-indexer/src/models/contributions/repository.rs @@ -8,11 +8,10 @@ use domain::GithubCodeReviewId; use infrastructure::{ contextualized_error::IntoContextualizedError, database::{ - self, enums::{ContributionStatus, ContributionType, GithubCodeReviewStatus, GithubIssueStatus}, schema::{contributions, github_pull_request_reviews}, - DatabaseError, Result, }, + dbclient::{self, DatabaseError, Result}, }; use super::{Contribution, DetailsId}; @@ -26,7 +25,7 @@ pub trait Repository: Sync + Send { fn upsert_from_github_pull_request(&self, pull_request: GithubPullRequest) -> Result<()>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn upsert_from_github_issue(&self, issue: GithubIssue) -> Result<()> { let contributions: Vec<_> = issue .assignee_ids diff --git a/github-indexer/src/models/github_issues/repository.rs b/github-indexer/src/models/github_issues/repository.rs index 56afffe1c0..d194315cc1 100644 --- a/github-indexer/src/models/github_issues/repository.rs +++ b/github-indexer/src/models/github_issues/repository.rs @@ -1,17 +1,16 @@ use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl}; use domain::{GithubIssueNumber, GithubRepoId}; use infrastructure::{ - contextualized_error::IntoContextualizedError, - database::{self, schema::github_issues}, + contextualized_error::IntoContextualizedError, database::schema::github_issues, dbclient, }; use crate::models::{GithubIssue, IdentifiableRepository}; -impl IdentifiableRepository for database::Client { +impl IdentifiableRepository for dbclient::Client { fn find( &self, (repo_id, number): (GithubRepoId, GithubIssueNumber), - ) -> database::Result> { + ) -> dbclient::Result> { let mut connection = self.connection()?; github_issues::table .filter(github_issues::repo_id.eq(repo_id)) diff --git a/github-indexer/src/models/github_pull_request_indexes/repository.rs b/github-indexer/src/models/github_pull_request_indexes/repository.rs index e0ffbfad14..7c02287ef8 100644 --- a/github-indexer/src/models/github_pull_request_indexes/repository.rs +++ b/github-indexer/src/models/github_pull_request_indexes/repository.rs @@ -2,13 +2,12 @@ use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; use domain::GithubPullRequestId; use infrastructure::{ contextualized_error::IntoContextualizedError, - database, - database::{schema::github_pull_request_indexes::dsl, Result}, + database::schema::github_pull_request_indexes::dsl, dbclient, dbclient::Result, }; use super::GithubPullRequestIndex; -pub trait Repository: database::Repository { +pub trait Repository: dbclient::Repository { fn select_pull_request_indexer_state( &self, pull_request_id: &GithubPullRequestId, @@ -20,7 +19,7 @@ pub trait Repository: database::Repository { ) -> Result<()>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn select_pull_request_indexer_state( &self, pull_request_id: &GithubPullRequestId, diff --git a/github-indexer/src/models/github_pull_requests/repository.rs b/github-indexer/src/models/github_pull_requests/repository.rs index fd050800cb..c71241cf0c 100644 --- a/github-indexer/src/models/github_pull_requests/repository.rs +++ b/github-indexer/src/models/github_pull_requests/repository.rs @@ -2,14 +2,12 @@ use diesel::{ExpressionMethods, JoinOnDsl, OptionalExtension, QueryDsl, RunQuery use domain::{GithubPullRequestNumber, GithubRepoId}; use infrastructure::{ contextualized_error::IntoContextualizedError, - database, - database::{ - schema::{ - closing_issues, github_pull_request_commits, github_pull_request_reviews, - github_pull_requests, github_repos, - }, - Result, + database::schema::{ + closing_issues, github_pull_request_commits, github_pull_request_reviews, + github_pull_requests, github_repos, }, + dbclient, + dbclient::Result, }; use super::{pull_request::GithubPullRequest, PullRequest as GithubFullPullRequest}; @@ -19,7 +17,7 @@ pub trait Repository: Send + Sync { fn upsert(&self, pull_request: GithubFullPullRequest) -> Result<()>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn upsert(&self, pull_request: GithubFullPullRequest) -> Result<()> { let mut connection = self.connection()?; @@ -80,12 +78,12 @@ impl Repository for database::Client { } impl IdentifiableRepository - for database::Client + for dbclient::Client { fn find( &self, (repo_id, number): (GithubRepoId, GithubPullRequestNumber), - ) -> database::Result> { + ) -> dbclient::Result> { let mut connection = self.connection()?; github_pull_requests::table .filter(github_pull_requests::repo_id.eq(repo_id)) @@ -100,12 +98,12 @@ impl IdentifiableRepository - for database::Client + for dbclient::Client { fn find( &self, (repo_owner, repo_name, number): (String, String, GithubPullRequestNumber), - ) -> database::Result> { + ) -> dbclient::Result> { let mut connection = self.connection()?; github_pull_requests::table .inner_join( diff --git a/github-indexer/src/models/github_repo_indexes/repository.rs b/github-indexer/src/models/github_repo_indexes/repository.rs index e1676845bf..e9cad6ed0e 100644 --- a/github-indexer/src/models/github_repo_indexes/repository.rs +++ b/github-indexer/src/models/github_repo_indexes/repository.rs @@ -1,15 +1,14 @@ use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; use domain::GithubRepoId; use infrastructure::{ - contextualized_error::IntoContextualizedError, - database, - database::{schema::github_repo_indexes::dsl, Result}, + contextualized_error::IntoContextualizedError, database::schema::github_repo_indexes::dsl, + dbclient, dbclient::Result, }; use super::GithubRepoIndex; use crate::diesel::OptionalExtension; -pub trait Repository: database::Repository { +pub trait Repository: dbclient::Repository { fn select_repo_indexer_state( &self, repo_id: &GithubRepoId, @@ -43,7 +42,7 @@ pub trait Repository: database::Repository { fn start_indexing(&self, repo_id: GithubRepoId) -> Result<()>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn select_repo_indexer_state( &self, repo_id: &GithubRepoId, diff --git a/github-indexer/src/models/github_user_indexes/repository.rs b/github-indexer/src/models/github_user_indexes/repository.rs index bc9eed7e95..71e9640122 100644 --- a/github-indexer/src/models/github_user_indexes/repository.rs +++ b/github-indexer/src/models/github_user_indexes/repository.rs @@ -1,14 +1,13 @@ use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl}; use domain::GithubUserId; use infrastructure::{ - contextualized_error::IntoContextualizedError, - database, - database::{schema::github_user_indexes::dsl, Result}, + contextualized_error::IntoContextualizedError, database::schema::github_user_indexes::dsl, + dbclient, dbclient::Result, }; use super::GithubUserIndex; -pub trait Repository: database::ImmutableRepository { +pub trait Repository: dbclient::ImmutableRepository { fn select_user_indexer_state( &self, user_id: &GithubUserId, @@ -20,7 +19,7 @@ pub trait Repository: database::ImmutableRepository { ) -> Result<()>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn select_user_indexer_state( &self, user_id: &GithubUserId, diff --git a/github-indexer/src/models/mod.rs b/github-indexer/src/models/mod.rs index c2529dd3f8..f8d9a12650 100644 --- a/github-indexer/src/models/mod.rs +++ b/github-indexer/src/models/mod.rs @@ -24,7 +24,7 @@ pub use github_repo_indexes::{GithubRepoIndex, Repository as GithubRepoIndexRepo pub use github_repos::GithubRepo; pub use github_user_indexes::{GithubUserIndex, Repository as GithubUserIndexRepository}; pub use github_users::GithubUser; -use infrastructure::database::{self, ImmutableModel, ImmutableRepository}; +use infrastructure::dbclient::{self, ImmutableModel, ImmutableRepository}; pub use project_github_repos::{ProjectGithubRepo, Repository as ProjectGithubRepoRepository}; pub use projects_contributors::{ProjectsContributor, Repository as ProjectsContributorRepository}; pub use projects_pending_contributors::{ @@ -33,7 +33,7 @@ pub use projects_pending_contributors::{ pub use technologies::Technology; pub trait IdentifiableRepository: Send + Sync { - fn find(&self, id: Id) -> database::Result>; + fn find(&self, id: Id) -> dbclient::Result>; } impl IdentifiableRepository for R @@ -41,7 +41,7 @@ where R: ImmutableRepository, M: ImmutableModel, { - fn find(&self, id: M::Id) -> database::Result> { + fn find(&self, id: M::Id) -> dbclient::Result> { >::try_find_by_id(self, id) } } diff --git a/github-indexer/src/models/project_github_repos/repository.rs b/github-indexer/src/models/project_github_repos/repository.rs index 201b443f12..88a0c8a2f7 100644 --- a/github-indexer/src/models/project_github_repos/repository.rs +++ b/github-indexer/src/models/project_github_repos/repository.rs @@ -1,18 +1,17 @@ use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; use domain::{GithubRepoId, ProjectId}; use infrastructure::{ - contextualized_error::IntoContextualizedError, - database, - database::{schema::project_github_repos::dsl, Result}, + contextualized_error::IntoContextualizedError, database::schema::project_github_repos::dsl, + dbclient, dbclient::Result, }; use super::ProjectGithubRepo; -pub trait Repository: database::ImmutableRepository { +pub trait Repository: dbclient::ImmutableRepository { fn find_projects_of_repo(&self, github_repo_id: &GithubRepoId) -> Result>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn find_projects_of_repo(&self, github_repo_id: &GithubRepoId) -> Result> { let mut connection = self.connection()?; let projects = dsl::project_github_repos diff --git a/github-indexer/src/models/projects_contributors/repository.rs b/github-indexer/src/models/projects_contributors/repository.rs index c8df31b01d..a883e51eca 100644 --- a/github-indexer/src/models/projects_contributors/repository.rs +++ b/github-indexer/src/models/projects_contributors/repository.rs @@ -3,21 +3,20 @@ use domain::{GithubRepoId, GithubUserId, ProjectId}; use infrastructure::{ contextualized_error::IntoContextualizedError, database::{ - self, enums::ContributionStatus, schema::{contributions, project_github_repos, projects_contributors::dsl}, - Result, }, + dbclient::{self, Result}, }; use super::ProjectsContributor; -pub trait Repository: database::ImmutableRepository { +pub trait Repository: dbclient::ImmutableRepository { fn refresh_project_contributor_list(&self, project_id: &ProjectId) -> Result>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn refresh_project_contributor_list( &self, project_id: &ProjectId, diff --git a/github-indexer/src/models/projects_pending_contributors/repository.rs b/github-indexer/src/models/projects_pending_contributors/repository.rs index d702c1afc6..b7c250d4e2 100644 --- a/github-indexer/src/models/projects_pending_contributors/repository.rs +++ b/github-indexer/src/models/projects_pending_contributors/repository.rs @@ -2,23 +2,21 @@ use diesel::{Connection, ExpressionMethods, QueryDsl, RunQueryDsl}; use domain::{GithubRepoId, GithubUserId, ProjectId}; use infrastructure::{ contextualized_error::IntoContextualizedError, - database, - database::{ - schema::{contributions, project_github_repos, projects_pending_contributors::dsl}, - Result, - }, + database::schema::{contributions, project_github_repos, projects_pending_contributors::dsl}, + dbclient, + dbclient::Result, }; use super::ProjectsPendingContributor; -pub trait Repository: database::ImmutableRepository { +pub trait Repository: dbclient::ImmutableRepository { fn refresh_project_pending_contributor_list( &self, project_id: &ProjectId, ) -> Result>; } -impl Repository for database::Client { +impl Repository for dbclient::Client { fn refresh_project_pending_contributor_list( &self, project_id: &ProjectId, diff --git a/github-indexer/src/presentation/cron/scheduler.rs b/github-indexer/src/presentation/cron/scheduler.rs index 3580aebf2d..2fc83226aa 100644 --- a/github-indexer/src/presentation/cron/scheduler.rs +++ b/github-indexer/src/presentation/cron/scheduler.rs @@ -2,7 +2,7 @@ use std::{sync::Arc, time::Duration}; use anyhow::Result; use domain::{GithubRepoId, GithubUserId}; -use infrastructure::{database, github}; +use infrastructure::{dbclient, github}; use olog::info; use crate::{ @@ -26,7 +26,7 @@ pub struct Scheduler { impl Scheduler { pub fn new(config: Config) -> Result { let github: Arc = github::RoundRobinClient::new(config.github)?.into(); - let database = Arc::new(database::Client::new(database::init_pool(config.database)?)); + let database = Arc::new(dbclient::Client::new(dbclient::init_pool(config.database)?)); let single_rate_limit_conf = rate_limited::RateLimitConf::new( github.clone(), diff --git a/github-indexer/src/presentation/http/bootstrap.rs b/github-indexer/src/presentation/http/bootstrap.rs index 1aeb2230a1..05c25d9e31 100644 --- a/github-indexer/src/presentation/http/bootstrap.rs +++ b/github-indexer/src/presentation/http/bootstrap.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use anyhow::Result; -use infrastructure::{database, github}; +use infrastructure::{dbclient, github}; use olog::info; use rocket::{Build, Rocket}; @@ -10,7 +10,7 @@ use crate::{presentation::http, Config}; pub async fn bootstrap(config: Config) -> Result> { info!("Bootstrapping backend event-listeners api"); - let database = Arc::new(database::Client::new(database::init_pool( + let database = Arc::new(dbclient::Client::new(dbclient::init_pool( config.database.clone(), )?)); diff --git a/github-indexer/src/presentation/http/mod.rs b/github-indexer/src/presentation/http/mod.rs index 9d3b56a6e1..9a0da00ab7 100644 --- a/github-indexer/src/presentation/http/mod.rs +++ b/github-indexer/src/presentation/http/mod.rs @@ -5,13 +5,13 @@ use std::sync::Arc; pub use bootstrap::bootstrap; pub use http::Config; -use infrastructure::{database, github}; +use infrastructure::{dbclient, github}; use presentation::http; use rocket::{Build, Rocket}; pub fn serve( config: Config, - database: Arc, + database: Arc, github: Arc, ) -> Rocket { rocket::custom(http::config::rocket("github-indexer/Rocket.toml")) diff --git a/github-indexer/src/presentation/http/routes/issue.rs b/github-indexer/src/presentation/http/routes/issue.rs index 054477628c..dd34a59d47 100644 --- a/github-indexer/src/presentation/http/routes/issue.rs +++ b/github-indexer/src/presentation/http/routes/issue.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use domain::GithubIssueId; use http_api_problem::{HttpApiProblem, StatusCode}; -use infrastructure::{database, github}; +use infrastructure::{dbclient, github}; use olog::{error, IntoField}; use presentation::http::guards::ApiKey; use rocket::{serde::json::Json, State}; @@ -27,7 +27,7 @@ pub async fn index( _api_key: ApiKey, repo_id: i64, issue_number: i64, - database: &State>, + database: &State>, github: &State>, ) -> Result, HttpApiProblem> { let database = (*database).clone(); diff --git a/github-indexer/src/presentation/http/routes/pull_request.rs b/github-indexer/src/presentation/http/routes/pull_request.rs index 53266dcbb1..146d07fd86 100644 --- a/github-indexer/src/presentation/http/routes/pull_request.rs +++ b/github-indexer/src/presentation/http/routes/pull_request.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use domain::GithubPullRequestId; use http_api_problem::{HttpApiProblem, StatusCode}; -use infrastructure::{database, github}; +use infrastructure::{dbclient, github}; use olog::{error, IntoField}; use presentation::http::guards::ApiKey; use rocket::{serde::json::Json, State}; @@ -29,7 +29,7 @@ pub async fn index_by_repo_id( _api_key: ApiKey, repo_id: i64, pr_number: i64, - database: &State>, + database: &State>, github: &State>, ) -> Result, HttpApiProblem> { let database = (*database).clone(); @@ -75,7 +75,7 @@ pub async fn index_by_repo_owner_name( repo_owner: String, repo_name: String, pr_number: i64, - database: &State>, + database: &State>, github: &State>, ) -> Result, HttpApiProblem> { let database = (*database).clone(); diff --git a/github-indexer/src/presentation/http/routes/repo.rs b/github-indexer/src/presentation/http/routes/repo.rs index fbcc7da234..7f0069343f 100644 --- a/github-indexer/src/presentation/http/routes/repo.rs +++ b/github-indexer/src/presentation/http/routes/repo.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use domain::GithubRepoId; use http_api_problem::{HttpApiProblem, StatusCode}; -use infrastructure::{database, github}; +use infrastructure::{dbclient, github}; use olog::{error, IntoField}; use presentation::http::guards::ApiKey; use rocket::{serde::json::Json, State}; @@ -26,7 +26,7 @@ pub struct Response { pub async fn index( _api_key: ApiKey, id: i64, - database: &State>, + database: &State>, github: &State>, ) -> Result, HttpApiProblem> { let database = (*database).clone(); diff --git a/github-indexer/src/presentation/http/routes/user.rs b/github-indexer/src/presentation/http/routes/user.rs index 756d22e194..d70ad49b90 100644 --- a/github-indexer/src/presentation/http/routes/user.rs +++ b/github-indexer/src/presentation/http/routes/user.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use domain::GithubUserId; use http_api_problem::{HttpApiProblem, StatusCode}; -use infrastructure::{database, github}; +use infrastructure::{dbclient, github}; use olog::{error, IntoField}; use presentation::http::guards::ApiKey; use rocket::{serde::json::Json, State}; @@ -26,7 +26,7 @@ pub struct Response { pub async fn index( _api_key: ApiKey, user_id: i64, - database: &State>, + database: &State>, github: &State>, ) -> Result, HttpApiProblem> { let database = (*database).clone(); diff --git a/github-indexer/tests/github_indexing_it.rs b/github-indexer/tests/github_indexing_it.rs index 489deb32b4..b0bf6f7900 100644 --- a/github-indexer/tests/github_indexing_it.rs +++ b/github-indexer/tests/github_indexing_it.rs @@ -5,10 +5,12 @@ use fixtures::*; use github_indexer::models::{ self, github_pull_requests::ClosingIssue, GithubRepoIndex, ProjectGithubRepo, }; -use infrastructure::database::{ - enums::{ContributionStatus, ContributionType}, - schema::{contributions, projects_contributors, projects_pending_contributors}, - ImmutableRepository, +use infrastructure::{ + database::{ + enums::{ContributionStatus, ContributionType}, + schema::{contributions, projects_contributors, projects_pending_contributors}, + }, + dbclient::ImmutableRepository, }; use olog::info; use rstest::rstest; diff --git a/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/adapter.rs b/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/adapter.rs index 670885c000..33765268ba 100644 --- a/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/adapter.rs +++ b/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/adapter.rs @@ -7,10 +7,10 @@ use domain::{ }, ports::output::clean_storage::{CleanStoragePort, Result}, }; -use infrastructure::database; +use infrastructure::dbclient; pub struct PostgresCleanStorageAdapter { - pub postgres_client: database::Client, + pub postgres_client: dbclient::Client, } impl CleanStoragePort for PostgresCleanStorageAdapter { From f91e8fbc62dfd1082bcdd37b4adc6e5ecc6c6070 Mon Sep 17 00:00:00 2001 From: Olivier Fuxet Date: Thu, 28 Sep 2023 14:53:49 +0200 Subject: [PATCH 09/11] Fix CI --- .../src/adapters/github_api/github_api_adapter.rs | 9 +++++++++ .../src/adapters/postgres_clean_storage/adapter.rs | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/new-github-indexer/infrastructure/src/adapters/github_api/github_api_adapter.rs b/new-github-indexer/infrastructure/src/adapters/github_api/github_api_adapter.rs index 68b0fcfd8a..8e0492d30d 100644 --- a/new-github-indexer/infrastructure/src/adapters/github_api/github_api_adapter.rs +++ b/new-github-indexer/infrastructure/src/adapters/github_api/github_api_adapter.rs @@ -36,10 +36,12 @@ impl GithubApiAdapter { #[async_trait] impl issue::Port for GithubApiAdapter { + #[allow(clippy::all)] async fn issue_by_repo_id(&self, _repo_id: u64, _issue_number: u64) -> Result { todo!() } + #[allow(clippy::all)] async fn issues_by_repo_id( &self, _repo_id: u64, @@ -51,6 +53,7 @@ impl issue::Port for GithubApiAdapter { #[async_trait] impl pull_request::Port for GithubApiAdapter { + #[allow(clippy::all)] async fn pull_request_by_repo_id( &self, _repo_id: u64, @@ -59,6 +62,7 @@ impl pull_request::Port for GithubApiAdapter { todo!() } + #[allow(clippy::all)] async fn pull_requests_by_repo_id( &self, _repo_id: u64, @@ -67,18 +71,22 @@ impl pull_request::Port for GithubApiAdapter { todo!() } + #[allow(clippy::all)] async fn pull_request_commits(&self, _pull_request: PullRequest) -> Result> { todo!() } + #[allow(clippy::all)] async fn pull_request_reviews(&self, _pull_request: PullRequest) -> Result> { todo!() } + #[allow(clippy::all)] async fn pull_request_closing_issue_ids(&self, _pull_request: PullRequest) -> Result> { todo!() } + #[allow(clippy::all)] async fn pull_request_ci_checks(&self, _pull_request: PullRequest) -> Result> { todo!() } @@ -97,6 +105,7 @@ impl repo::Port for GithubApiAdapter { #[async_trait] impl user::Port for GithubApiAdapter { + #[allow(clippy::all)] async fn user_by_id(&self, _user_id: u64) -> Result { todo!() } diff --git a/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/adapter.rs b/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/adapter.rs index 33765268ba..2444fe7e1e 100644 --- a/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/adapter.rs +++ b/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/adapter.rs @@ -14,14 +14,17 @@ pub struct PostgresCleanStorageAdapter { } impl CleanStoragePort for PostgresCleanStorageAdapter { + #[allow(clippy::all)] fn save_repo(&self, _repo: Repository) -> Result<()> { todo!() } + #[allow(clippy::all)] fn save_issue(&self, _issue: Issue) -> Result<()> { todo!() } + #[allow(clippy::all)] fn save_pull_request( &self, _pull_request: PullRequest, @@ -33,6 +36,7 @@ impl CleanStoragePort for PostgresCleanStorageAdapter { todo!() } + #[allow(clippy::all)] fn save_user(&self, _user: User) -> Result<()> { todo!() } From 8fb48b8ae26b5d51c846c7a63a61640fa9f6928c Mon Sep 17 00:00:00 2001 From: Olivier Fuxet Date: Thu, 28 Sep 2023 15:12:49 +0200 Subject: [PATCH 10/11] Refactor dbclient so that we can run different sets of migrations (and therefore, support multiple schemas) --- Cargo.lock | 2 ++ api/Cargo.toml | 1 + api/src/presentation/http/bootstrap.rs | 5 ++++- common/infrastructure/src/dbclient/mod.rs | 10 ++++------ common/infrastructure/src/lib.rs | 1 - common/testing/Cargo.toml | 1 + common/testing/src/context/database.rs | 5 ++++- 7 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 23e690d2ae..ead17cf6ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -182,6 +182,7 @@ dependencies = [ "diesel", "diesel-derive-newtype", "diesel_json", + "diesel_migrations", "domain", "dotenv", "envtestkit", @@ -5088,6 +5089,7 @@ name = "testing" version = "0.1.0" dependencies = [ "anyhow", + "diesel_migrations", "domain", "infrastructure", "lapin", diff --git a/api/Cargo.toml b/api/Cargo.toml index 4fa99a4246..146618d3a7 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -25,6 +25,7 @@ diesel = { version = "2.1.0", default-features = false, features = [ "uuid", "chrono", ] } +diesel_migrations = "2.1.0" diesel-derive-newtype = "2.1.0" diesel_json = "0.2.0" uuid = { version = "1.3.3", default_features = false, features = [ diff --git a/api/src/presentation/http/bootstrap.rs b/api/src/presentation/http/bootstrap.rs index 57678811f3..35c0c26461 100644 --- a/api/src/presentation/http/bootstrap.rs +++ b/api/src/presentation/http/bootstrap.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use anyhow::Result; +use diesel_migrations::{embed_migrations, EmbeddedMigrations}; use domain::{AggregateRepository, CompositePublisher, EventPublisher}; use infrastructure::{amqp, dbclient, event_bus::EXCHANGE_NAME, github}; use rocket::{Build, Rocket}; @@ -12,12 +13,14 @@ use crate::{ Config, }; +pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("../migrations"); + pub async fn bootstrap(config: Config) -> Result> { info!("Bootstrapping api http server"); let database = Arc::new(dbclient::Client::new(dbclient::init_pool( config.database.clone(), )?)); - database.run_migrations()?; + database.run_migrations(MIGRATIONS)?; let github_api_client: Arc = github::RoundRobinClient::new(config.github_api_client.clone())?.into(); diff --git a/common/infrastructure/src/dbclient/mod.rs b/common/infrastructure/src/dbclient/mod.rs index 3201fd020c..c2a059c832 100644 --- a/common/infrastructure/src/dbclient/mod.rs +++ b/common/infrastructure/src/dbclient/mod.rs @@ -20,11 +20,9 @@ use crate::diesel_migrations::MigrationHarness; pub type Pool = r2d2::Pool>; type PooledConnection = r2d2::PooledConnection>; -pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!(); - -pub fn run_migrations(pool: &Pool) { +pub fn run_migrations(pool: &Pool, migrations: EmbeddedMigrations) { let mut connection = pool.get().expect("Unable to get connection from pool"); - connection.run_pending_migrations(MIGRATIONS).expect("diesel migration failure"); + connection.run_pending_migrations(migrations).expect("diesel migration failure"); } pub struct Client { @@ -48,9 +46,9 @@ impl Client { }) } - pub fn run_migrations(&self) -> Result<()> { + pub fn run_migrations(&self, migrations: EmbeddedMigrations) -> Result<()> { let mut connection = self.connection()?; - connection.run_pending_migrations(MIGRATIONS).map_err(|e| { + connection.run_pending_migrations(migrations).map_err(|e| { error!(error = e.to_field(), "Failed to run migrations"); DatabaseError::Migration(anyhow!(e)) })?; diff --git a/common/infrastructure/src/lib.rs b/common/infrastructure/src/lib.rs index 88519186f4..4aaaa28fe2 100644 --- a/common/infrastructure/src/lib.rs +++ b/common/infrastructure/src/lib.rs @@ -15,7 +15,6 @@ pub mod http; pub mod tracing; pub mod web3; -#[macro_use] extern crate diesel_migrations; #[macro_use] extern crate lazy_static; diff --git a/common/testing/Cargo.toml b/common/testing/Cargo.toml index ceb88edf37..2c1dd440d0 100644 --- a/common/testing/Cargo.toml +++ b/common/testing/Cargo.toml @@ -20,6 +20,7 @@ project-root = "0.2.2" serde_json = "1.0.81" serde = { version = "1.0.137", features = ["derive"] } reqwest = { version = "0.11" } +diesel_migrations = "2.1.0" # Local dependecies domain = { path = "../domain" } diff --git a/common/testing/src/context/database.rs b/common/testing/src/context/database.rs index 387324b3f4..343f61daee 100644 --- a/common/testing/src/context/database.rs +++ b/common/testing/src/context/database.rs @@ -1,11 +1,14 @@ use std::sync::Arc; use anyhow::{anyhow, Result}; +use diesel_migrations::{embed_migrations, EmbeddedMigrations}; use infrastructure::dbclient; use testcontainers::{ clients::Cli, core::WaitFor, images::generic::GenericImage, Container, RunnableImage, }; +pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("../../migrations"); + static USER: &str = "postgres"; static PASSWORD: &str = "Passw0rd"; static DATABASE: &str = "marketplace_db"; @@ -31,7 +34,7 @@ impl<'docker> Context<'docker> { let client = dbclient::Client::new(dbclient::init_pool(config.clone())?); - client.run_migrations()?; + client.run_migrations(MIGRATIONS)?; Ok(Self { _container: container, From 1748a947721f868fa7e7c8932316335afd7775e5 Mon Sep 17 00:00:00 2001 From: Olivier Fuxet Date: Thu, 28 Sep 2023 18:05:55 +0200 Subject: [PATCH 11/11] Introduce new DB schema and new migration set. --- new-github-indexer/infrastructure/diesel.toml | 4 ++++ .../2023-09-28-152513_create_schema/down.sql | 1 + .../2023-09-28-152513_create_schema/up.sql | 1 + .../2023-09-28-152914_create_repos_table/down.sql | 1 + .../2023-09-28-152914_create_repos_table/up.sql | 5 +++++ .../src/adapters/postgres_clean_storage/schema.rs | 10 ++++++++++ rustfmt.toml | 1 + 7 files changed, 23 insertions(+) create mode 100644 new-github-indexer/infrastructure/diesel.toml create mode 100644 new-github-indexer/infrastructure/migrations_indexer_clean/2023-09-28-152513_create_schema/down.sql create mode 100644 new-github-indexer/infrastructure/migrations_indexer_clean/2023-09-28-152513_create_schema/up.sql create mode 100644 new-github-indexer/infrastructure/migrations_indexer_clean/2023-09-28-152914_create_repos_table/down.sql create mode 100644 new-github-indexer/infrastructure/migrations_indexer_clean/2023-09-28-152914_create_repos_table/up.sql create mode 100644 new-github-indexer/infrastructure/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/schema.rs diff --git a/new-github-indexer/infrastructure/diesel.toml b/new-github-indexer/infrastructure/diesel.toml new file mode 100644 index 0000000000..43cddf1084 --- /dev/null +++ b/new-github-indexer/infrastructure/diesel.toml @@ -0,0 +1,4 @@ +[print_schema] +file = "new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/schema.rs" +schema = "indexer_clean" +custom_type_derives = ["diesel::query_builder::QueryId"] diff --git a/new-github-indexer/infrastructure/migrations_indexer_clean/2023-09-28-152513_create_schema/down.sql b/new-github-indexer/infrastructure/migrations_indexer_clean/2023-09-28-152513_create_schema/down.sql new file mode 100644 index 0000000000..8e23e04f3e --- /dev/null +++ b/new-github-indexer/infrastructure/migrations_indexer_clean/2023-09-28-152513_create_schema/down.sql @@ -0,0 +1 @@ +drop schema indexer_clean; \ No newline at end of file diff --git a/new-github-indexer/infrastructure/migrations_indexer_clean/2023-09-28-152513_create_schema/up.sql b/new-github-indexer/infrastructure/migrations_indexer_clean/2023-09-28-152513_create_schema/up.sql new file mode 100644 index 0000000000..5c146704af --- /dev/null +++ b/new-github-indexer/infrastructure/migrations_indexer_clean/2023-09-28-152513_create_schema/up.sql @@ -0,0 +1 @@ +create schema indexer_clean; \ No newline at end of file diff --git a/new-github-indexer/infrastructure/migrations_indexer_clean/2023-09-28-152914_create_repos_table/down.sql b/new-github-indexer/infrastructure/migrations_indexer_clean/2023-09-28-152914_create_repos_table/down.sql new file mode 100644 index 0000000000..8eb8e6a981 --- /dev/null +++ b/new-github-indexer/infrastructure/migrations_indexer_clean/2023-09-28-152914_create_repos_table/down.sql @@ -0,0 +1 @@ +drop table indexer_clean.repos; \ No newline at end of file diff --git a/new-github-indexer/infrastructure/migrations_indexer_clean/2023-09-28-152914_create_repos_table/up.sql b/new-github-indexer/infrastructure/migrations_indexer_clean/2023-09-28-152914_create_repos_table/up.sql new file mode 100644 index 0000000000..0455372151 --- /dev/null +++ b/new-github-indexer/infrastructure/migrations_indexer_clean/2023-09-28-152914_create_repos_table/up.sql @@ -0,0 +1,5 @@ +create table indexer_clean.repos +( + repo_id bigint primary key, + value jsonb not null +); \ No newline at end of file diff --git a/new-github-indexer/infrastructure/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/schema.rs b/new-github-indexer/infrastructure/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/schema.rs new file mode 100644 index 0000000000..3b062caac0 --- /dev/null +++ b/new-github-indexer/infrastructure/new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/schema.rs @@ -0,0 +1,10 @@ +// @generated automatically by Diesel CLI. + +pub mod indexer_clean { + diesel::table! { + indexer_clean.repos (repo_id) { + repo_id -> Int8, + value -> Jsonb, + } + } +} diff --git a/rustfmt.toml b/rustfmt.toml index 169bb27110..6c9b72824d 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -30,4 +30,5 @@ use_field_init_shorthand = true # Ignore generated files ignore = [ "common/infrastructure/src/database/schema.rs", + "new-github-indexer/infrastructure/src/adapters/postgres_clean_storage/schema.rs" ]