Merge pull request #38 from OpenMined/madhava/otel-instrumentation

chore: standardize lint.sh scripts across repos

Author: madhavajay

.github/workflows/ci.yml

@@ -52,15 +52,12 @@ jobs:
         with:
           workspaces: ". -> target"
 
-      - name: Check formatting
+      - name: Run lint checks and tests
         if: matrix.os != 'windows-latest'
-        run: cargo fmt --all -- --check
+        run: ./lint.sh --check --test
 
-      - name: Run clippy (all targets)
-        if: matrix.os != 'windows-latest'
-        run: cargo clippy --all-targets --all-features --no-deps -- -D warnings
-
-      - name: Run tests
+      - name: Run tests (Windows)
+        if: matrix.os == 'windows-latest'
         run: cargo test --all-features --verbose
 
       - name: Build

Cargo.toml

@@ -13,6 +13,7 @@ process = []
 auth = ["reqwest", "tokio"]
 embedded = ["dep:syftbox-rs"]
 test-plaintext = []
+telemetry = ["opentelemetry", "tracing-opentelemetry"]
 
 [dependencies]
 anyhow = "1.0"
@@ -22,6 +23,8 @@ dirs = "5.0"
 walkdir = "2.4"
 thiserror = "1.0"
 tracing = "0.1"
+opentelemetry = { version = "0.27", optional = true }
+tracing-opentelemetry = { version = "0.28", optional = true }
 tokio = { version = "1.40", features = ["macros", "rt"], optional = true }
 reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"], optional = true }
 base64 = "0.22"

clippy.sh

@@ -1,10 +1,82 @@
 #!/usr/bin/env bash
+# lint.sh - Auto-fix + run quick tests (parallel, quiet on success)
+# Usage: ./lint.sh [--check] [--test]
+#   --check  Read-only mode for CI (no auto-fix)
+#   --test   Also run tests (slower)
 set -euo pipefail
 
-echo "==> cargo fmt (check)"
-cargo fmt --all -- --check
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$ROOT_DIR"
 
-echo "==> cargo clippy (warnings as errors)"
-cargo clippy --all-targets --all-features --no-deps -- -D warnings
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+CYAN='\033[0;36m'
+NC='\033[0m'
 
-echo "✓ Lint checks passed"
+CHECK_MODE=0
+RUN_TESTS=0
+for arg in "$@"; do
+  case "$arg" in
+    --check) CHECK_MODE=1 ;;
+    --test) RUN_TESTS=1 ;;
+  esac
+done
+
+TMPDIR_LINT=$(mktemp -d)
+trap "rm -rf $TMPDIR_LINT" EXIT
+
+FAILED=0
+PIDS=()
+TASKS=()
+
+run_task() {
+  local name="$1"
+  local outfile="$TMPDIR_LINT/$name.out"
+  shift
+  TASKS+=("$name")
+  echo -e "${CYAN}→ $name${NC}"
+  (
+    if "$@" > "$outfile" 2>&1; then
+      echo "0" > "$outfile.status"
+    else
+      echo "1" > "$outfile.status"
+    fi
+  ) &
+  PIDS+=($!)
+}
+
+wait_all() {
+  local i=0
+  for pid in "${PIDS[@]}"; do
+    wait "$pid" || true
+    local name="${TASKS[$i]}"
+    local outfile="$TMPDIR_LINT/$name.out"
+    if [[ -f "$outfile.status" && "$(cat "$outfile.status")" != "0" ]]; then
+      echo -e "${RED}✗ $name${NC}"
+      cat "$outfile"
+      echo ""
+      FAILED=1
+    fi
+    i=$((i + 1))
+  done
+}
+
+if [[ "$CHECK_MODE" -eq 1 ]]; then
+  run_task "rust-fmt" cargo fmt --all -- --check
+  run_task "rust-clippy" cargo clippy --all-targets --all-features --no-deps -- -D warnings
+else
+  run_task "rust-fmt" cargo fmt --all
+  run_task "rust-clippy" cargo clippy --fix --allow-dirty --allow-staged --all-targets --all-features --no-deps -- -D warnings
+fi
+
+if [[ "$RUN_TESTS" -eq 1 ]]; then
+  run_task "rust-test" cargo test
+fi
+
+wait_all
+
+if [[ "$FAILED" -eq 0 ]]; then
+  echo -e "${GREEN}✓ All checks passed${NC}"
+else
+  exit 1
+fi

lint.sh

@@ -25,7 +25,10 @@ pub use syftbox::syc::{
     provision_local_identity_with_options, resolve_identity, restore_identity_from_mnemonic,
     IdentityProvisioningOutcome,
 };
-pub use syftbox::types::{RpcHeaders, RpcRequest, RpcResponse};
+pub use syftbox::types::{RpcHeaders, RpcRequest, RpcResponse, COMPONENT_ATTR, TRACEPARENT_HEADER};
+
+#[cfg(feature = "telemetry")]
+pub use syftbox::types::trace_context;
 
 #[cfg(feature = "auth")]
 pub use syftbox::auth::{request_otp, verify_otp, OtpRequestPayload, OtpTokens, OtpVerifyOutcome};

src/lib.rs

@@ -12,6 +12,7 @@ use std::thread;
 use std::time::{Duration, Instant};
 
 const SYFTBOX_PIDFILE_NAME: &str = "syftbox.pid";
+#[cfg(feature = "embedded")]
 const SYFTBOX_EMBEDDED_PIDFILE_NAME: &str = "syftbox.embedded.pid";
 
 #[cfg(target_os = "windows")]

src/syftbox/control.rs

@@ -3,6 +3,7 @@ use crate::syftbox::storage::WritePolicy;
 use crate::syftbox::types::{RpcRequest, RpcResponse};
 use anyhow::{anyhow, bail, Context, Result};
 use std::path::{Path, PathBuf};
+use tracing::instrument;
 
 /// Request with its file path
 pub type RequestWithPath = (PathBuf, RpcRequest);
@@ -19,6 +20,7 @@ pub struct Endpoint {
 
 impl Endpoint {
     /// Create a new endpoint for a SyftBox app
+    #[instrument(skip(app), fields(component = "rpc", endpoint = %name), err)]
     pub fn new(app: &SyftBoxApp, name: &str) -> Result<Self> {
         let path = app.register_endpoint(name)?;
 
@@ -31,13 +33,15 @@ impl Endpoint {
 
     /// Check for request files in this endpoint (your own folder only)
     /// Requests are written by others TO you in datasites/your-email/app_data/<app>/rpc/<endpoint>/
+    #[instrument(skip(self), fields(component = "rpc", endpoint = %self.name), err)]
     pub fn check_requests(&self) -> Result<Vec<(PathBuf, RpcRequest)>> {
         // Only check your own endpoint folder - this is where others write requests TO you
         self.check_requests_in_folder(&self.path)
     }
 
     /// Check for request files and return both successes and failures
     /// Returns (successful_requests, failed_requests)
+    #[instrument(skip(self), fields(component = "rpc", endpoint = %self.name), err)]
     pub fn check_requests_with_failures(
         &self,
     ) -> Result<(Vec<RequestWithPath>, Vec<FailedRequest>)> {
@@ -109,6 +113,12 @@ impl Endpoint {
     }
 
     /// Send a response for a request
+    #[instrument(skip(self, request, response), fields(
+        component = "rpc",
+        endpoint = %self.name,
+        request_id = %request.id,
+        status = %response.status_code
+    ), err)]
     pub fn send_response(
         &self,
         request_path: &Path,
@@ -160,6 +170,13 @@ impl Endpoint {
     }
 
     /// Create a request file (for sending requests to others)
+    #[instrument(skip(self, request), fields(
+        component = "rpc",
+        endpoint = %self.name,
+        request_id = %request.id,
+        method = %request.method,
+        recipient = %request.url
+    ), err)]
     pub fn create_request(&self, request: &RpcRequest) -> Result<PathBuf> {
         let request_filename = format!("{}.request", request.id);
         let request_path = self.path.join(request_filename);
@@ -183,6 +200,7 @@ impl Endpoint {
     /// Check for response files (when we sent a request and are waiting for response)
     /// Responses are in OTHER identity folders (where you wrote requests), NOT your own folder
     /// You write requests to datasites/their-email/, they write responses back there
+    #[instrument(skip(self), fields(component = "rpc", endpoint = %self.name), err)]
     pub fn check_responses(&self) -> Result<Vec<(PathBuf, RpcResponse)>> {
         let mut responses = Vec::new();
 
@@ -294,6 +312,7 @@ impl Endpoint {
     }
 
     /// Clean up a response file after processing
+    #[instrument(skip(self), fields(component = "rpc", endpoint = %self.name), err)]
     pub fn cleanup_response(&self, response_path: &Path) -> Result<()> {
         self.app.storage.remove_path(response_path)?;
         Ok(())

src/syftbox/endpoint.rs

@@ -3,8 +3,10 @@ use crate::syftbox::endpoint::Endpoint;
 use crate::syftbox::types::{RpcRequest, RpcResponse};
 use anyhow::Result;
 use std::path::{Path, PathBuf};
+use tracing::instrument;
 
 /// Check for requests across all endpoints of an app
+#[instrument(skip(app), fields(component = "rpc", app_email = %app.email), err)]
 pub fn check_all_requests(app: &SyftBoxApp) -> Result<Vec<(String, PathBuf, RpcRequest)>> {
     let mut all_requests = Vec::new();
 
@@ -22,12 +24,19 @@ pub fn check_all_requests(app: &SyftBoxApp) -> Result<Vec<(String, PathBuf, RpcR
 }
 
 /// Check for requests on a specific endpoint
+#[instrument(skip(app), fields(component = "rpc", endpoint = %endpoint_name), err)]
 pub fn check_requests(app: &SyftBoxApp, endpoint_name: &str) -> Result<Vec<(PathBuf, RpcRequest)>> {
     let endpoint = Endpoint::new(app, endpoint_name)?;
     endpoint.check_requests()
 }
 
 /// Send a response to a request
+#[instrument(skip(app, request, response), fields(
+    component = "rpc",
+    endpoint = %endpoint_name,
+    request_id = %request.id,
+    status = %response.status_code
+), err)]
 pub fn send_response(
     app: &SyftBoxApp,
     endpoint_name: &str,
@@ -41,6 +50,13 @@ pub fn send_response(
 }
 
 /// Create and send a request to another datasite
+#[instrument(skip(app, request), fields(
+    component = "rpc",
+    endpoint = %endpoint_name,
+    request_id = %request.id,
+    method = %request.method,
+    recipient = %request.url
+), err)]
 pub fn send_request(
     app: &SyftBoxApp,
     endpoint_name: &str,
@@ -51,6 +67,7 @@ pub fn send_request(
 }
 
 /// Check for responses to requests we've sent
+#[instrument(skip(app), fields(component = "rpc", endpoint = %endpoint_name), err)]
 pub fn check_responses(
     app: &SyftBoxApp,
     endpoint_name: &str,
@@ -60,6 +77,12 @@ pub fn check_responses(
 }
 
 /// Helper to process a request and automatically send a response
+#[instrument(skip(app, request, handler), fields(
+    component = "rpc",
+    endpoint = %endpoint_name,
+    request_id = %request.id,
+    method = %request.method
+), err)]
 pub fn process_request<F>(
     app: &SyftBoxApp,
     endpoint_name: &str,

src/syftbox/rpc.rs

@@ -10,7 +10,7 @@ use syft_crypto_protocol::datasite::{
     bytes::{read_bytes, write_bytes, BytesReadOpts, BytesWriteOpts, BytesWriteOutcome},
     context::{ensure_vault_layout, resolve_vault, AppContext},
 };
-use tracing::warn;
+use tracing::{instrument, warn};
 use walkdir::WalkDir;
 
 /// Result from read_with_shadow when metadata is requested
@@ -143,6 +143,7 @@ impl SyftBoxStorage {
     /// Write encrypted file using shadow folder pattern:
     /// 1. Write plaintext to shadow folder
     /// 2. Encrypt from shadow → datasites
+    #[instrument(skip(self, data), fields(component = "storage", size = data.len(), encrypted = true), err)]
     pub fn write_encrypted_with_shadow(
         &self,
         datasite_path: &Path,
@@ -218,6 +219,7 @@ impl SyftBoxStorage {
     /// Read encrypted file using shadow folder pattern:
     /// 1. Decrypt from datasites → shadow
     /// 2. Read plaintext from shadow
+    #[instrument(skip(self), fields(component = "storage"), err)]
     pub fn read_with_shadow(&self, datasite_path: &Path) -> Result<Vec<u8>> {
         match &self.backend {
             StorageBackend::SyctCrypto(backend) => {
@@ -322,6 +324,7 @@ impl SyftBoxStorage {
 
     /// Read encrypted file using shadow folder pattern, returning metadata about the sender.
     /// This is the same as read_with_shadow but also returns verified sender identity.
+    #[instrument(skip(self), fields(component = "storage"), err)]
     pub fn read_with_shadow_metadata(&self, datasite_path: &Path) -> Result<ReadWithShadowResult> {
         match &self.backend {
             StorageBackend::SyctCrypto(backend) => {
@@ -416,6 +419,7 @@ impl SyftBoxStorage {
 
     /// Write with shadow folder pattern - creates both encrypted file and plaintext shadow
     /// This makes unencrypted/ a true mirror of datasites/
+    #[instrument(skip(self, plaintext), fields(component = "storage", size = plaintext.len()), err)]
     pub fn write_with_shadow(
         &self,
         absolute_path: &Path,