Merge pull request #5 from sio2project/3-add-metrics-endpoint

Add metrics endpoint

Author: MasloMaslane

Cargo.lock

@@ -27,4 +27,6 @@ async-trait = "0.1.77"
 flate2 = "1.0.28"
 urlencoding = "2.1.3"
 clap = { version = "4.5.0", features = ["derive"] }
-reqwest = { version = "0.12.0", features = ["json", "gzip"] }
+reqwest = { version = "0.12.0", features = ["json", "gzip"] }
+prometheus = "0.13.4"
+lazy_static = "1.5.0"

Cargo.toml

@@ -2,6 +2,7 @@ version: '3.8'
 
 # MinIO and PostgreSQL setup for s3dedup development
 # Buckets are automatically created by the application on startup
+# PostgreSQL creates both 's3dedup' (main) and 's3dedup_test' (for tests) databases
 services:
   minio:
     image: minio/minio:RELEASE.2024-10-02T17-50-41Z
@@ -34,6 +35,7 @@ services:
       POSTGRES_DB: s3dedup
     volumes:
       - postgres_data:/var/lib/postgresql/data
+      - ./docker/init-test-db.sh:/docker-entrypoint-initdb.d/init-test-db.sh
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 10s

docker-compose.yml

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    CREATE DATABASE s3dedup_test;
+EOSQL

docker/init-test-db.sh

@@ -7,6 +7,7 @@ pub mod filetracker_client;
 pub mod kvstorage;
 pub mod locks;
 pub mod logging;
+pub mod metrics;
 pub mod migration;
 pub mod routes;
 pub mod s3storage;
@@ -18,6 +19,7 @@ pub struct AppState {
     pub locks: Arc<Mutex<Box<locks::LocksStorage>>>,
     pub s3storage: Arc<Mutex<Box<s3storage::S3Storage>>>,
     pub filetracker_client: Option<Arc<filetracker_client::FiletrackerClient>>,
+    pub metrics: Arc<metrics::Metrics>,
 }
 
 impl AppState {
@@ -27,12 +29,14 @@ impl AppState {
         let kvstorage = kvstorage::KVStorage::new(config).await?;
         let locks = locks::LocksStorage::new(&config.locks_type);
         let s3storage = s3storage::S3Storage::new(config).await?;
+        let metrics = Arc::new(metrics::Metrics::new());
         Ok(Self {
             bucket_name: config.name.clone(),
             kvstorage: Arc::new(Mutex::new(kvstorage)),
             locks: Arc::new(Mutex::new(locks)),
             s3storage: Arc::new(Mutex::new(s3storage)),
             filetracker_client: None,
+            metrics,
         })
     }
 
@@ -44,12 +48,18 @@ impl AppState {
         let locks = locks::LocksStorage::new(&config.locks_type);
         let s3storage = s3storage::S3Storage::new(config).await?;
         let filetracker_client = filetracker_client::FiletrackerClient::new(filetracker_url);
+        let metrics = Arc::new(metrics::Metrics::new());
+
+        // Mark migration as active
+        metrics::MIGRATION_ACTIVE.set(1);
+
         Ok(Self {
             bucket_name: config.name.clone(),
             kvstorage: Arc::new(Mutex::new(kvstorage)),
             locks: Arc::new(Mutex::new(locks)),
             s3storage: Arc::new(Mutex::new(s3storage)),
             filetracker_client: Some(Arc::new(filetracker_client)),
+            metrics,
         })
     }
 }

src/lib.rs

@@ -104,6 +104,11 @@ async fn run_s3dedup_server(config_path: Option<&str>, use_env: bool) {
                     .put(ft_put_file)
                     .delete(ft_delete_file),
             )
+            .route("/metrics", get(s3dedup::routes::metrics::metrics_handler))
+            .route(
+                "/metrics/json",
+                get(s3dedup::routes::metrics::metrics_json_handler),
+            )
             .layer(
                 // Logging middleware
                 TraceLayer::new_for_http()
@@ -252,6 +257,11 @@ async fn run_live_migrate(config_path: Option<&str>, use_env: bool, max_concurre
                             .put(s3dedup::routes::ft::put_file::ft_put_file)
                             .delete(s3dedup::routes::ft::delete_file::ft_delete_file),
                     )
+                    .route("/metrics", get(s3dedup::routes::metrics::metrics_handler))
+                    .route(
+                        "/metrics/json",
+                        get(s3dedup::routes::metrics::metrics_json_handler),
+                    )
                     .layer(
                         TraceLayer::new_for_http()
                             .make_span_with(DefaultMakeSpan::new().level(Level::INFO))
@@ -316,6 +326,11 @@ async fn run_live_migrate(config_path: Option<&str>, use_env: bool, max_concurre
                     .put(s3dedup::routes::ft::put_file::ft_put_file)
                     .delete(s3dedup::routes::ft::delete_file::ft_delete_file),
             )
+            .route("/metrics", get(s3dedup::routes::metrics::metrics_handler))
+            .route(
+                "/metrics/json",
+                get(s3dedup::routes::metrics::metrics_json_handler),
+            )
             .layer(
                 TraceLayer::new_for_http()
                     .make_span_with(DefaultMakeSpan::new().level(Level::INFO))