From 119dc57073785767d05313aea27c52a18cad9399 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Thu, 15 Aug 2024 16:09:12 +0400 Subject: [PATCH 1/4] expose basic tokio runtime metrics --- libsql-server/src/http/admin/mod.rs | 25 +++++++++++++ libsql-server/src/metrics.rs | 54 +++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/libsql-server/src/http/admin/mod.rs b/libsql-server/src/http/admin/mod.rs index 683d67995e..24c516ec60 100644 --- a/libsql-server/src/http/admin/mod.rs +++ b/libsql-server/src/http/admin/mod.rs @@ -93,9 +93,34 @@ where tokio::task::spawn(async move { loop { + let runtime = tokio::runtime::Handle::current(); + let metrics = runtime.metrics(); + crate::metrics::TOKIO_RUNTIME_BLOCKING_QUEUE_DEPTH + .set(metrics.blocking_queue_depth() as f64); + crate::metrics::TOKIO_RUNTIME_INJECTION_QUEUE_DEPTH + .set(metrics.injection_queue_depth() as f64); + crate::metrics::TOKIO_RUNTIME_NUM_BLOCKING_THREADS + .set(metrics.num_blocking_threads() as f64); + crate::metrics::TOKIO_RUNTIME_NUM_IDLE_BLOCKING_THREADS + .set(metrics.num_idle_blocking_threads() as f64); + crate::metrics::TOKIO_RUNTIME_NUM_WORKERS.set(metrics.num_workers() as f64); + + crate::metrics::TOKIO_RUNTIME_IO_DRIVER_FD_DEREGISTERED_COUNT + .absolute(metrics.io_driver_fd_deregistered_count() as u64); + crate::metrics::TOKIO_RUNTIME_IO_DRIVER_FD_REGISTERED_COUNT + .absolute(metrics.io_driver_fd_registered_count() as u64); + crate::metrics::TOKIO_RUNTIME_IO_DRIVER_READY_COUNT + .absolute(metrics.io_driver_ready_count() as u64); + crate::metrics::TOKIO_RUNTIME_REMOTE_SCHEDULE_COUNT + .absolute(metrics.remote_schedule_count() as u64); tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + }); + tokio::task::spawn(async move { + loop { crate::metrics::SERVER_COUNT.set(1.0); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; } }); diff --git a/libsql-server/src/metrics.rs b/libsql-server/src/metrics.rs index 1ac97435b3..bb9c049fa7 100644 --- a/libsql-server/src/metrics.rs +++ b/libsql-server/src/metrics.rs @@ -158,3 +158,57 @@ pub static QUERY_CANCELED: Lazy = Lazy::new(|| { describe_counter!(NAME, "Number of canceled queries"); register_counter!(NAME) }); + +pub static TOKIO_RUNTIME_BLOCKING_QUEUE_DEPTH: Lazy = Lazy::new(|| { + const NAME: &str = "tokio_runtime_blocking_queue_depth"; + describe_gauge!(NAME, "tokio runtime blocking_queue_depth"); + register_gauge!(NAME) +}); + +pub static TOKIO_RUNTIME_INJECTION_QUEUE_DEPTH: Lazy = Lazy::new(|| { + const NAME: &str = "tokio_runtime_injection_queue_depth"; + describe_gauge!(NAME, "tokio runtime injection_queue_depth"); + register_gauge!(NAME) +}); + +pub static TOKIO_RUNTIME_NUM_BLOCKING_THREADS: Lazy = Lazy::new(|| { + const NAME: &str = "tokio_runtime_num_blocking_threads"; + describe_gauge!(NAME, "tokio runtime num_blocking_threads"); + register_gauge!(NAME) +}); + +pub static TOKIO_RUNTIME_NUM_IDLE_BLOCKING_THREADS: Lazy = Lazy::new(|| { + const NAME: &str = "tokio_runtime_num_idle_blocking_threads"; + describe_gauge!(NAME, "tokio runtime num_idle_blocking_threads"); + register_gauge!(NAME) +}); + +pub static TOKIO_RUNTIME_NUM_WORKERS: Lazy = Lazy::new(|| { + const NAME: &str = "tokio_runtime_num_workers"; + describe_gauge!(NAME, "tokio runtime num_workers"); + register_gauge!(NAME) +}); + +pub static TOKIO_RUNTIME_IO_DRIVER_FD_DEREGISTERED_COUNT: Lazy = Lazy::new(|| { + const NAME: &str = "tokio_runtime_io_driver_fd_deregistered_count"; + describe_counter!(NAME, "tokio runtime io_driver_fd_deregistered_count"); + register_counter!(NAME) +}); + +pub static TOKIO_RUNTIME_IO_DRIVER_FD_REGISTERED_COUNT: Lazy = Lazy::new(|| { + const NAME: &str = "tokio_runtime_io_driver_fd_registered_count"; + describe_counter!(NAME, "tokio runtime io_driver_fd_registered_count"); + register_counter!(NAME) +}); + +pub static TOKIO_RUNTIME_IO_DRIVER_READY_COUNT: Lazy = Lazy::new(|| { + const NAME: &str = "tokio_runtime_io_driver_ready_count"; + describe_counter!(NAME, "tokio runtime io_driver_ready_count"); + register_counter!(NAME) +}); + +pub static TOKIO_RUNTIME_REMOTE_SCHEDULE_COUNT: Lazy = Lazy::new(|| { + const NAME: &str = "tokio_runtime_remote_schedule_count"; + describe_gauge!(NAME, "tokio runtime remote_schedule_count"); + register_counter!(NAME) +}); From 0dcdb176f467d9ac3936294477927238f099b079 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Thu, 15 Aug 2024 21:41:55 +0400 Subject: [PATCH 2/4] add tokio_unstable cfg in Cargo.toml and GH actions --- .github/workflows/nemesis.yml | 2 +- .github/workflows/rust.yml | 10 +++++----- libsql-server/Cargo.toml | 4 ++++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nemesis.yml b/.github/workflows/nemesis.yml index 9cfdbc39b0..090f2626b8 100644 --- a/.github/workflows/nemesis.yml +++ b/.github/workflows/nemesis.yml @@ -18,7 +18,7 @@ jobs: if: github.repository == 'tursodatabase/libsql' name: Run Nemesis Tests env: - RUSTFLAGS: -D warnings + RUSTFLAGS: -D warnings --cfg tokio_unstable steps: - uses: hecrj/setup-rust-action@v2 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1903c0baff..7036c6cf2f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest name: Run Checks env: - RUSTFLAGS: -D warnings + RUSTFLAGS: -D warnings --cfg tokio_unstable steps: - uses: hecrj/setup-rust-action@v2 @@ -80,15 +80,15 @@ jobs: - uses: taiki-e/install-action@cargo-udeps - uses: Swatinem/rust-cache@v2 - run: cargo +nightly hack udeps -p libsql --each-feature - - run: RUSTFLAGS="-D warnings" cargo check -p libsql --no-default-features --features core - - run: RUSTFLAGS="-D warnings" cargo check -p libsql --no-default-features --features replication - - run: RUSTFLAGS="-D warnings" cargo check -p libsql --no-default-features --features remote + - run: RUSTFLAGS="-D warnings --cfg tokio_unstable" cargo check -p libsql --no-default-features --features core + - run: RUSTFLAGS="-D warnings --cfg tokio_unstable" cargo check -p libsql --no-default-features --features replication + - run: RUSTFLAGS="-D warnings --cfg tokio_unstable" cargo check -p libsql --no-default-features --features remote test: runs-on: ubuntu-latest name: Run Tests env: - RUSTFLAGS: -D warnings + RUSTFLAGS: -D warnings --cfg tokio_unstable steps: - uses: hecrj/setup-rust-action@v2 diff --git a/libsql-server/Cargo.toml b/libsql-server/Cargo.toml index 934a400786..b22b9f1458 100644 --- a/libsql-server/Cargo.toml +++ b/libsql-server/Cargo.toml @@ -8,6 +8,9 @@ default-run = "sqld" name = "sqld" path = "src/main.rs" +[build] +rustflags = ["--cfg", "tokio_unstable"] + [dependencies] anyhow = "1.0.66" async-lock = "2.6.0" @@ -116,6 +119,7 @@ vergen = { version = "8", features = ["build", "git", "gitcl"] } [features] default = [] +tokio-metrics = [] debug-tools = ["console-subscriber", "rusqlite/trace", "tokio/tracing"] wasm-udfs = ["rusqlite/libsql-wasm-experimental"] unix-excl-vfs = ["libsql-sys/unix-excl-vfs"] From 4ebff7fdd5b0dd9e6fa250c5c35cd9b27778536f Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Thu, 15 Aug 2024 21:58:47 +0400 Subject: [PATCH 3/4] remove feature --- libsql-server/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/libsql-server/Cargo.toml b/libsql-server/Cargo.toml index b22b9f1458..2db9330fa6 100644 --- a/libsql-server/Cargo.toml +++ b/libsql-server/Cargo.toml @@ -119,7 +119,6 @@ vergen = { version = "8", features = ["build", "git", "gitcl"] } [features] default = [] -tokio-metrics = [] debug-tools = ["console-subscriber", "rusqlite/trace", "tokio/tracing"] wasm-udfs = ["rusqlite/libsql-wasm-experimental"] unix-excl-vfs = ["libsql-sys/unix-excl-vfs"] From f6fe40df5171614d492369ec69e16a58e88e15f0 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Fri, 16 Aug 2024 00:53:54 +0400 Subject: [PATCH 4/4] use single task to report bunch of metrics --- libsql-server/src/http/admin/mod.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/libsql-server/src/http/admin/mod.rs b/libsql-server/src/http/admin/mod.rs index 24c516ec60..7f34b77359 100644 --- a/libsql-server/src/http/admin/mod.rs +++ b/libsql-server/src/http/admin/mod.rs @@ -113,12 +113,7 @@ where .absolute(metrics.io_driver_ready_count() as u64); crate::metrics::TOKIO_RUNTIME_REMOTE_SCHEDULE_COUNT .absolute(metrics.remote_schedule_count() as u64); - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - } - }); - tokio::task::spawn(async move { - loop { crate::metrics::SERVER_COUNT.set(1.0); tokio::time::sleep(std::time::Duration::from_secs(1)).await; }