From dc93a9574718600896614f531e37340443de17b4 Mon Sep 17 00:00:00 2001 From: eason <30045503+Eason0729@users.noreply.github.com> Date: Fri, 8 Dec 2023 11:38:38 +0800 Subject: [PATCH] test: :sparkles: add intensive logging and opentelemetry --- backend/Cargo.lock | 262 +++++++++++++++++++-- backend/Cargo.toml | 18 +- backend/src/controller/judger/route/mod.rs | 15 +- backend/src/endpoint/submit.rs | 2 - backend/src/endpoint/user.rs | 2 + backend/src/init/logger.rs | 117 ++++++++- backend/src/init/mod.rs | 8 +- backend/src/main.rs | 5 +- backend/src/server.rs | 44 ++-- docker/production/docker-compose.yml | 3 + judger/Cargo.lock | 175 +++++++------- judger/Cargo.toml | 9 +- judger/src/grpc/server.rs | 4 - judger/src/init/logger.rs | 24 +- judger/src/sandbox/container.rs | 1 - judger/src/sandbox/utils/limiter/mod.rs | 1 - judger/src/sandbox/utils/semaphore.rs | 3 - proto/backend.proto | 1 - 18 files changed, 509 insertions(+), 185 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index d5f1ad36..96c794ff 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -347,8 +347,13 @@ dependencies = [ "hickory-resolver", "leaky-bucket", "log", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "opentelemetry-stdout", + "opentelemetry_sdk", "paste", - "prost", + "prost 0.12.3", "prost-types", "quick_cache", "ring 0.17.5", @@ -359,10 +364,11 @@ dependencies = [ "tokio", "tokio-stream", "toml 0.7.4", - "tonic", + "tonic 0.10.2", "tonic-build", "tonic-web", "tracing", + "tracing-opentelemetry", "tracing-subscriber", "uuid", ] @@ -688,6 +694,16 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.8" @@ -1523,9 +1539,9 @@ checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" -version = "0.3.58" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27" +checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" dependencies = [ "wasm-bindgen", ] @@ -1874,6 +1890,99 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" +dependencies = [ + "futures-core", + "futures-sink", + "indexmap 2.0.0", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f24cda83b20ed2433c68241f918d0f6fdec8b1d43b7a9590ab4420c5095ca930" +dependencies = [ + "async-trait", + "futures-core", + "http", + "opentelemetry", + "opentelemetry-proto", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", + "prost 0.11.9", + "thiserror", + "tokio", + "tonic 0.9.2", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2e155ce5cc812ea3d1dffbd1539aed653de4bf4882d60e6e04dcf0901d674e1" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost 0.11.9", + "tonic 0.9.2", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "opentelemetry-stdout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13b2df4cd59c176099ac82806725ba340c8fa7b1a7004c0912daad30470f63e" +dependencies = [ + "async-trait", + "chrono", + "opentelemetry", + "opentelemetry_sdk", + "ordered-float 4.2.0", + "serde", + "serde_json", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "968ba3f2ca03e90e5187f5e4f46c791ef7f2c163ae87789c8ce5f5ca3b7b7de5" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "once_cell", + "opentelemetry", + "ordered-float 4.2.0", + "percent-encoding", + "rand", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "ordered-float" version = "3.7.0" @@ -1883,6 +1992,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" +dependencies = [ + "num-traits", +] + [[package]] name = "ouroboros" version = "0.17.2" @@ -2115,6 +2233,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive 0.11.9", +] + [[package]] name = "prost" version = "0.12.3" @@ -2122,7 +2250,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.12.3", ] [[package]] @@ -2139,7 +2267,7 @@ dependencies = [ "once_cell", "petgraph", "prettyplease", - "prost", + "prost 0.12.3", "prost-types", "regex", "syn 2.0.32", @@ -2147,6 +2275,19 @@ dependencies = [ "which", ] +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "prost-derive" version = "0.12.3" @@ -2166,7 +2307,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" dependencies = [ - "prost", + "prost 0.12.3", ] [[package]] @@ -2602,7 +2743,7 @@ dependencies = [ "chrono", "derivative", "inherent", - "ordered-float", + "ordered-float 3.7.0", "rust_decimal", "sea-query-derive", "serde_json", @@ -3345,6 +3486,34 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost 0.11.9", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tonic" version = "0.10.2" @@ -3363,7 +3532,7 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost", + "prost 0.12.3", "rustls", "rustls-pemfile", "tokio", @@ -3401,7 +3570,7 @@ dependencies = [ "hyper", "pin-project", "tokio-stream", - "tonic", + "tonic 0.10.2", "tower-http", "tower-layer", "tower-service", @@ -3502,6 +3671,35 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" +dependencies = [ + "js-sys", + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", + "tracing", + "tracing-core", + "tracing-log 0.2.0", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-serde" version = "0.1.3" @@ -3529,7 +3727,7 @@ dependencies = [ "thread_local", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.1.3", "tracing-serde", ] @@ -3601,6 +3799,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8parse" version = "0.2.1" @@ -3664,9 +3868,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.81" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994" +checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3674,16 +3878,16 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.81" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" +checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" dependencies = [ "bumpalo", - "lazy_static", "log", + "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.32", "wasm-bindgen-shared", ] @@ -3701,9 +3905,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.81" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa" +checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3711,22 +3915,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.81" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048" +checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.32", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.81" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be" +checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" [[package]] name = "web-sys" @@ -3738,6 +3942,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57099a701fb3a8043f993e8228dc24229c7b942e2b009a1b962e54489ba1d3bf" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.24.0" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 8ae520e8..0747619b 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [workspace] -members = ["entity","migration"] +members = ["entity", "migration"] [dependencies] log = "0.4.18" @@ -29,14 +29,20 @@ leaky-bucket = "1.0.1" hickory-resolver = "0.24.0" crossbeam-queue = "0.3.8" dashmap = "5.5.3" +tracing-opentelemetry = "0.22.0" +opentelemetry_sdk = { version = "0.21.1", features = ["rt-tokio"] } +opentelemetry = "0.21.0" +opentelemetry-stdout = { version = "0.2.0", features = ["metrics"] } +opentelemetry-semantic-conventions = "0.13.0" +opentelemetry-otlp = { version = "0.14.0", features = ["metrics"] } [dependencies.tokio-stream] version = "0.1.14" -features =["sync"] +features = ["sync"] [dependencies.tracing-subscriber] version = "0.3.17" -features =["json"] +features = ["json"] [dependencies.tracing] version = "0.1.40" @@ -44,7 +50,7 @@ features = ["async-await", "log"] [dependencies.tokio] version = "1.34.0" -features =["macros", "rt-multi-thread", "full","time"] +features = ["macros", "rt-multi-thread", "full", "time"] [dependencies.sea-orm] version = "0.12.2" @@ -53,7 +59,7 @@ features = [ "macros", "mock", "sqlx-sqlite", - "with-chrono" + "with-chrono", ] [dependencies.serde] @@ -66,7 +72,7 @@ features = ["transport", "channel", "codegen", "tls", "prost"] [dependencies.spin] version = "0.9.8" -features = ["mutex","spin_mutex","rwlock"] +features = ["mutex", "spin_mutex", "rwlock"] [build-dependencies] tonic-build = "0.10.2" diff --git a/backend/src/controller/judger/route/mod.rs b/backend/src/controller/judger/route/mod.rs index bfc4d24d..46f89c84 100644 --- a/backend/src/controller/judger/route/mod.rs +++ b/backend/src/controller/judger/route/mod.rs @@ -106,6 +106,11 @@ async fn discover( config: JudgerConfig, router: Weak, ) -> Result<(), Error> { + let span = tracing::span!( + tracing::Level::DEBUG, + "service discover", + name = config.name + ); let mut instance = I::new(config)?; loop { match instance.discover().await { @@ -131,10 +136,14 @@ async fn discover( } } } - RouteStatus::Wait(dur) => tokio::time::sleep(dur).await, + RouteStatus::Wait(dur) => { + log::trace!("Service Discovery halt for {} seconds", dur.as_secs()); + tokio::time::sleep(dur).await + } _ => break, } } + drop(span); Ok(()) } @@ -197,6 +206,7 @@ pub struct Upstream { healthy: AtomicIsize, clients: SegQueue, connection: ConnectionDetail, + _live_span: tracing::Span, } impl Upstream { @@ -217,6 +227,8 @@ impl Upstream { result.push((uuid, lang)); } + let live_span = tracing::span!(tracing::Level::INFO, "judger livetime", uri = detail.uri); + let clients = SegQueue::default(); clients.push(client); @@ -225,6 +237,7 @@ impl Upstream { healthy: AtomicIsize::new(HEALTHY_THRESHOLD), clients, connection: detail, + _live_span: live_span, }), result, )) diff --git a/backend/src/endpoint/submit.rs b/backend/src/endpoint/submit.rs index 3406b819..c878064a 100644 --- a/backend/src/endpoint/submit.rs +++ b/backend/src/endpoint/submit.rs @@ -162,8 +162,6 @@ impl SubmitSet for Arc { .ok_or(Error::NotInDB("user"))?; } - // if problem - let submit = SubmitBuilder::default() .code(req.info.code) .lang(lang) diff --git a/backend/src/endpoint/user.rs b/backend/src/endpoint/user.rs index dedda83f..cab7060a 100644 --- a/backend/src/endpoint/user.rs +++ b/backend/src/endpoint/user.rs @@ -174,6 +174,8 @@ impl UserSet for Arc { let mut model: ActiveModel = Default::default(); + log::info!("creating user({})", req.info.username); + fill_active_model!(model, req.info, username); let hash = self.crypto.hash(req.info.password.as_str()).into(); diff --git a/backend/src/init/logger.rs b/backend/src/init/logger.rs index 6b27e8b8..95c8fb2d 100644 --- a/backend/src/init/logger.rs +++ b/backend/src/init/logger.rs @@ -1,8 +1,117 @@ +use opentelemetry::{global, KeyValue}; +use opentelemetry_sdk::{ + metrics::{ + reader::{DefaultAggregationSelector, DefaultTemporalitySelector}, + MeterProvider, PeriodicReader, + }, + runtime, + trace::{BatchConfig, RandomIdGenerator, Sampler, Tracer}, + Resource, +}; +use opentelemetry_semantic_conventions::{ + resource::{DEPLOYMENT_ENVIRONMENT, SERVICE_NAME, SERVICE_VERSION}, + SCHEMA_URL, +}; use tracing::Level; +use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; use super::config::GlobalConfig; -pub fn init(config: &GlobalConfig) { +fn resource() -> Resource { + Resource::from_schema_url( + [ + KeyValue::new(SERVICE_NAME, "mdoj-backend"), + KeyValue::new(SERVICE_VERSION, env!("CARGO_PKG_VERSION")), + #[cfg(debug_assertions)] + KeyValue::new(DEPLOYMENT_ENVIRONMENT, "develop"), + #[cfg(not(debug_assertions))] + KeyValue::new(DEPLOYMENT_ENVIRONMENT, "production"), + ], + SCHEMA_URL, + ) +} + +fn init_tracer() -> Tracer { + opentelemetry_otlp::new_pipeline() + .tracing() + .with_trace_config( + opentelemetry_sdk::trace::Config::default() + // Customize sampling strategy + .with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased( + 1.0, + )))) + // If export trace to AWS X-Ray, you can use XrayIdGenerator + .with_id_generator(RandomIdGenerator::default()) + .with_resource(resource()), + ) + .with_batch_config(BatchConfig::default()) + .with_exporter(opentelemetry_otlp::new_exporter().tonic()) + .install_batch(runtime::Tokio) + .unwrap() +} + +fn init_meter_provider() -> MeterProvider { + #[cfg(debug_assertions)] + let exporter = opentelemetry_otlp::new_exporter() + .tonic() + .build_metrics_exporter( + Box::new(DefaultAggregationSelector::new()), + Box::new(DefaultTemporalitySelector::new()), + ) + .unwrap(); + #[cfg(debug_assertions)] + let otlp_reader = PeriodicReader::builder(exporter, runtime::Tokio) + .with_interval(std::time::Duration::from_secs(30)) + .build(); + + // For debugging in development + let stdout_reader = PeriodicReader::builder( + opentelemetry_stdout::MetricsExporter::default(), + runtime::Tokio, + ) + .build(); + + let meter_provider = MeterProvider::builder() + .with_resource(resource()) + .with_reader(stdout_reader); + + #[cfg(debug_assertions)] + let meter_provider = meter_provider.with_reader(otlp_reader); + + let meter_provider = meter_provider.build(); + + global::set_meter_provider(meter_provider.clone()); + + meter_provider +} + +pub struct OtelGuard { + meter_provider: MeterProvider, +} + +impl Drop for OtelGuard { + fn drop(&mut self) { + if let Err(err) = self.meter_provider.shutdown() { + eprintln!("{err:?}"); + } + opentelemetry::global::shutdown_tracer_provider(); + } +} +fn init_tracing_subscriber(level: Level) -> OtelGuard { + let meter_provider = init_meter_provider(); + + tracing_subscriber::registry() + .with(tracing_subscriber::filter::LevelFilter::from_level(level)) + .with(tracing_subscriber::fmt::layer()) + .with(MetricsLayer::new(meter_provider.clone())) + .with(OpenTelemetryLayer::new(init_tracer())) + .init(); + + OtelGuard { meter_provider } +} + +pub fn init(config: &GlobalConfig) -> OtelGuard { let level = match config.log_level { 0 => Level::TRACE, 1 => Level::DEBUG, @@ -12,9 +121,5 @@ pub fn init(config: &GlobalConfig) { _ => Level::INFO, }; - tracing_subscriber::fmt() - .json() - .with_max_level(level) - .with_current_span(false) - .init(); + init_tracing_subscriber(level) } diff --git a/backend/src/init/mod.rs b/backend/src/init/mod.rs index 56380d30..198accc2 100644 --- a/backend/src/init/mod.rs +++ b/backend/src/init/mod.rs @@ -1,12 +1,12 @@ -use self::config::GlobalConfig; +use self::{config::GlobalConfig, logger::OtelGuard}; pub mod config; pub mod db; pub mod logger; -pub async fn new() -> GlobalConfig { +pub async fn new() -> (GlobalConfig, OtelGuard) { let config = config::init().await; - logger::init(&config); + let olp_guard = logger::init(&config); db::init(&config).await; - config + (config, olp_guard) } diff --git a/backend/src/main.rs b/backend/src/main.rs index 485d8ff1..03d10e93 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -7,7 +7,6 @@ pub mod server; #[tokio::main] async fn main() { - init::new().await; - log::info!("starting server"); - server::Server::start().await; + let server = server::Server::new().await; + server.start().await; } diff --git a/backend/src/server.rs b/backend/src/server.rs index 52d76a84..399403e7 100644 --- a/backend/src/server.rs +++ b/backend/src/server.rs @@ -11,7 +11,11 @@ use crate::{ testcase_set_server::TestcaseSetServer, token_set_server::TokenSetServer, user_set_server::UserSetServer, }, - init::config::{self}, + init::{ + self, + config::{self, GlobalConfig}, + logger::{self, OtelGuard}, + }, }; const MAX_FRAME_SIZE: u32 = 1024 * 1024 * 8; @@ -21,11 +25,18 @@ pub struct Server { pub submit: judger::SubmitController, pub dup: duplicate::DupController, pub crypto: crypto::CryptoController, + config: GlobalConfig, + identity: transport::Identity, + _otp_guard: OtelGuard, } impl Server { - pub async fn start() { + pub async fn new() -> Arc { let config = config::init().await; + + init::db::init(&config).await; + let otp_guard = logger::init(&config); + log::info!("Loading TLS certificate..."); let cert = fs::read_to_string(&config.grpc.public_pem) .await @@ -37,25 +48,30 @@ impl Server { log::info!("Constructing server..."); - let server = Arc::new(Server { + Arc::new(Server { token: token::TokenController::new(), submit: judger::SubmitController::new(&config).await.unwrap(), dup: duplicate::DupController::default(), crypto: crypto::CryptoController::new(&config), - }); - + config, + identity, + _otp_guard: otp_guard, + }) + } + pub async fn start(self: Arc) { transport::Server::builder() - .tls_config(transport::ServerTlsConfig::new().identity(identity)) + // .accept_http1(true) + .tls_config(transport::ServerTlsConfig::new().identity(self.identity.clone())) .unwrap() .max_frame_size(Some(MAX_FRAME_SIZE)) - .add_service(tonic_web::enable(ProblemSetServer::new(server.clone()))) - .add_service(tonic_web::enable(EducationSetServer::new(server.clone()))) - .add_service(tonic_web::enable(UserSetServer::new(server.clone()))) - .add_service(tonic_web::enable(TokenSetServer::new(server.clone()))) - .add_service(tonic_web::enable(ContestSetServer::new(server.clone()))) - .add_service(tonic_web::enable(TestcaseSetServer::new(server.clone()))) - .add_service(tonic_web::enable(SubmitSetServer::new(server))) - .serve(config.bind_address.parse().unwrap()) + .add_service(tonic_web::enable(ProblemSetServer::new(self.clone()))) + .add_service(tonic_web::enable(EducationSetServer::new(self.clone()))) + .add_service(tonic_web::enable(UserSetServer::new(self.clone()))) + .add_service(tonic_web::enable(TokenSetServer::new(self.clone()))) + .add_service(tonic_web::enable(ContestSetServer::new(self.clone()))) + .add_service(tonic_web::enable(TestcaseSetServer::new(self.clone()))) + .add_service(tonic_web::enable(SubmitSetServer::new(self.clone()))) + .serve(self.config.bind_address.clone().parse().unwrap()) .await .unwrap(); } diff --git a/docker/production/docker-compose.yml b/docker/production/docker-compose.yml index 4411affd..5a9f531f 100644 --- a/docker/production/docker-compose.yml +++ b/docker/production/docker-compose.yml @@ -1,5 +1,8 @@ version: '3.9' services: + # jaeger: + # image: jaegertracing/all-in-one + # ports: gateway: container_name: mdoj-gateway build: ../../frontend diff --git a/judger/Cargo.lock b/judger/Cargo.lock index 8c54c04e..aeeb9f09 100644 --- a/judger/Cargo.lock +++ b/judger/Cargo.lock @@ -189,6 +189,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "colored" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6" +dependencies = [ + "is-terminal", + "lazy_static", + "windows-sys", +] + +[[package]] +name = "deranged" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +dependencies = [ + "powerfmt", +] + [[package]] name = "either" version = "1.9.0" @@ -487,6 +507,17 @@ dependencies = [ "hashbrown 0.14.1", ] +[[package]] +name = "is-terminal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +dependencies = [ + "hermit-abi", + "rustix", + "windows-sys", +] + [[package]] name = "itertools" version = "0.10.5" @@ -523,6 +554,7 @@ dependencies = [ "prost", "prost-types", "serde", + "simple_logger", "spin 0.9.8", "tempfile", "thiserror", @@ -533,8 +565,6 @@ dependencies = [ "tonic-build", "tower", "tower-layer", - "tracing", - "tracing-subscriber", "uuid", ] @@ -629,22 +659,21 @@ dependencies = [ ] [[package]] -name = "nu-ansi-term" -version = "0.46.0" +name = "num_cpus" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "overload", - "winapi", + "hermit-abi", + "libc", ] [[package]] -name = "num_cpus" -version = "1.16.0" +name = "num_threads" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" dependencies = [ - "hermit-abi", "libc", ] @@ -663,12 +692,6 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "parking_lot" version = "0.12.1" @@ -740,6 +763,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -967,12 +996,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" -[[package]] -name = "ryu" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" - [[package]] name = "scopeguard" version = "1.2.0" @@ -1009,17 +1032,6 @@ dependencies = [ "syn 2.0.38", ] -[[package]] -name = "serde_json" -version = "1.0.108" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" -dependencies = [ - "itoa", - "ryu", - "serde", -] - [[package]] name = "serde_spanned" version = "0.6.3" @@ -1030,21 +1042,24 @@ dependencies = [ ] [[package]] -name = "sharded-slab" -version = "0.1.7" +name = "signal-hook-registry" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ - "lazy_static", + "libc", ] [[package]] -name = "signal-hook-registry" -version = "1.4.1" +name = "simple_logger" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "da0ca6504625ee1aa5fda33913d2005eab98c7a42dd85f116ecce3ff54c9d3ef" dependencies = [ - "libc", + "colored", + "log", + "time", + "windows-sys", ] [[package]] @@ -1159,13 +1174,34 @@ dependencies = [ ] [[package]] -name = "thread_local" -version = "1.1.7" +name = "time" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" dependencies = [ - "cfg-if", - "once_cell", + "deranged", + "itoa", + "libc", + "num_threads", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +dependencies = [ + "time-core", ] [[package]] @@ -1383,45 +1419,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-serde" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" -dependencies = [ - "serde", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" -dependencies = [ - "nu-ansi-term", - "serde", - "serde_json", - "sharded-slab", - "smallvec", - "thread_local", - "tracing-core", - "tracing-log", - "tracing-serde", ] [[package]] @@ -1463,12 +1460,6 @@ dependencies = [ "serde", ] -[[package]] -name = "valuable" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" - [[package]] name = "want" version = "0.3.1" diff --git a/judger/Cargo.toml b/judger/Cargo.toml index a595d669..1e7a5f79 100644 --- a/judger/Cargo.toml +++ b/judger/Cargo.toml @@ -20,6 +20,7 @@ prost = "0.11.9" prost-types = "0.11.9" thiserror = "1.0.40" toml = "0.7.4" +simple_logger="4.3.0" [dependencies.uuid] version = "1.6.1" @@ -45,14 +46,6 @@ features = ["net"] version = "0.9.8" features = ["mutex","spin_mutex"] -[dependencies.tracing-subscriber] -version = "0.3.17" -features =["json"] - -[dependencies.tracing] -version = "0.1.40" -features =["async-await", "log"] - [dev-dependencies] tempfile = "3.8.0" tower = "0.4.13" diff --git a/judger/src/grpc/server.rs b/judger/src/grpc/server.rs index 487a706d..eb9529a4 100644 --- a/judger/src/grpc/server.rs +++ b/judger/src/grpc/server.rs @@ -5,7 +5,6 @@ use std::{pin::Pin, sync::Arc}; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tonic::{codegen::Bytes, metadata, Code, Response, Status}; -use tracing::instrument; use uuid::Uuid; use crate::{ @@ -115,7 +114,6 @@ impl Server { impl Judger for Server { type JudgeStream = Pin> + Send>>; - #[instrument(skip_all, name = "grpc_judge")] async fn judge<'a>( &'a self, request: tonic::Request, @@ -186,7 +184,6 @@ impl Judger for Server { Ok(Response::new(Box::pin(ReceiverStream::new(rx)))) } - #[instrument(skip_all, name = "grpc_info")] async fn judger_info<'a>( &'a self, request: tonic::Request<()>, @@ -209,7 +206,6 @@ impl Judger for Server { #[doc = " Server streaming response type for the Exec method."] type ExecStream = Pin> + Send>>; - #[instrument(skip_all, name = "grpc_exec")] async fn exec( &self, req: tonic::Request, diff --git a/judger/src/init/logger.rs b/judger/src/init/logger.rs index b22dfe6d..26f8efb7 100644 --- a/judger/src/init/logger.rs +++ b/judger/src/init/logger.rs @@ -1,5 +1,4 @@ -use tracing::Level; - +use log::Level; use super::config::CONFIG; // logger @@ -7,18 +6,13 @@ pub fn init() { let config = CONFIG.get().unwrap(); let level = match config.log_level { - 0 => Level::TRACE, - 1 => Level::DEBUG, - 2 => Level::INFO, - 3 => Level::WARN, - 4 => Level::ERROR, - _ => Level::INFO, + 0 => Level::Trace, + 1 => Level::Debug, + 2 => Level::Info, + 3 => Level::Warn, + 4 => Level::Error, + _ => Level::Info, }; - tracing_subscriber::fmt() - .json() - .with_max_level(level) - .with_current_span(false) - .try_init() - .ok(); -} + simple_logger::init_with_level(level).unwrap(); +} \ No newline at end of file diff --git a/judger/src/sandbox/container.rs b/judger/src/sandbox/container.rs index c4a77801..cc54b450 100644 --- a/judger/src/sandbox/container.rs +++ b/judger/src/sandbox/container.rs @@ -43,7 +43,6 @@ impl<'a> Container<'a> { Ok(Container { id, daemon, root }) } - #[tracing::instrument(skip(self, limit))] pub async fn execute(&self, args: Vec<&str>, limit: Limit) -> Result { let config = CONFIG.get().unwrap(); diff --git a/judger/src/sandbox/utils/limiter/mod.rs b/judger/src/sandbox/utils/limiter/mod.rs index e7ba54c4..5a470e09 100644 --- a/judger/src/sandbox/utils/limiter/mod.rs +++ b/judger/src/sandbox/utils/limiter/mod.rs @@ -146,7 +146,6 @@ impl Limiter { (stat.cpu.clone(), stat.mem.clone()) } - #[tracing::instrument(skip(self),level = tracing::Level::DEBUG)] pub fn wait_exhausted(&mut self) -> Receiver { self.limit_oneshot .take() diff --git a/judger/src/sandbox/utils/semaphore.rs b/judger/src/sandbox/utils/semaphore.rs index a45c6d42..a3de63b8 100644 --- a/judger/src/sandbox/utils/semaphore.rs +++ b/judger/src/sandbox/utils/semaphore.rs @@ -26,7 +26,6 @@ pub struct MemoryStatistic { pub struct MemorySemaphore(Arc>); impl MemorySemaphore { - #[tracing::instrument] pub fn new(memory: u64) -> Self { Self(Arc::new(Mutex::new(MemorySemaphoreInner { memory, @@ -43,7 +42,6 @@ impl MemorySemaphore { tasks: self_.tasks, } } - #[tracing::instrument(skip(self),level = tracing::Level::TRACE)] pub async fn allocate(&self, memory: u64) -> Result { log::trace!("preserve {}B memory", memory); let config = CONFIG.get().unwrap(); @@ -75,7 +73,6 @@ impl MemorySemaphore { Ok(MemoryPermit::new(self, memory)) } - #[tracing::instrument(skip(self),level = tracing::Level::TRACE)] fn deallocate(&self, released_memory: u64) { let self_ = &mut *self.0.lock(); diff --git a/proto/backend.proto b/proto/backend.proto index 04304ba3..07bdece6 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -556,7 +556,6 @@ message CreateUserRequest { required Permission permission = 3; } required Info info = 1; - required UserId id = 2; required string request_id = 3; }