From fb21f1d923f67436e8f8923e4aaf84a8660954e1 Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Wed, 19 Mar 2025 23:16:04 -0300 Subject: [PATCH 01/20] feat: added most of the basic implementation for the docker/clickhouse approach --- .gitignore | 35 + oracle/clickhouse/schema.proto | 32 + oracle/clickhouse/tables.sql | 130 ++++ oracle/docker-compose.yml | 114 ++++ oracle/graphql-api/Cargo.toml | 14 + oracle/graphql-api/Dockerfile | 114 ++++ oracle/graphql-api/src/main.rs | 190 ++++++ oracle/test-producer/Cargo.lock | 1030 ++++++++++++++++++++++++++++++ oracle/test-producer/Cargo.toml | 15 + oracle/test-producer/Dockerfile | 144 +++++ oracle/test-producer/build.rs | 21 + oracle/test-producer/src/main.rs | 168 +++++ subgraph/README.md | 1 - 13 files changed, 2007 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 oracle/clickhouse/schema.proto create mode 100644 oracle/clickhouse/tables.sql create mode 100644 oracle/docker-compose.yml create mode 100644 oracle/graphql-api/Cargo.toml create mode 100644 oracle/graphql-api/Dockerfile create mode 100644 oracle/graphql-api/src/main.rs create mode 100644 oracle/test-producer/Cargo.lock create mode 100644 oracle/test-producer/Cargo.toml create mode 100644 oracle/test-producer/Dockerfile create mode 100644 oracle/test-producer/build.rs create mode 100644 oracle/test-producer/src/main.rs delete mode 100644 subgraph/README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d3aad5a --- /dev/null +++ b/.gitignore @@ -0,0 +1,35 @@ +# Docker volumes data +*data/ +*.volume + +# Rust build artifacts +/target/ +*/target/ +**/target/ +**/*.rs.bk + +# Logs +*.log +/logs/ + +# Environment variables +.env +.env.* +!.env.example + +# IDE - Common +.vscode/ +.idea/ +*.iml +.DS_Store + +# ClickHouse specific +preprocessed_configs/ +clickhouse-server.err.log +clickhouse-server.log + +# Temporary files +*~ +*.swp +*.swo +/tmp/ \ No newline at end of file diff --git a/oracle/clickhouse/schema.proto b/oracle/clickhouse/schema.proto new file mode 100644 index 0000000..1cdea70 --- /dev/null +++ b/oracle/clickhouse/schema.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package qos; + +message ClientQueryProtobuf { + string gateway_id = 1; + bytes receipt_signer = 2; + string query_id = 3; + string api_key = 4; + string user_id = 11; + string subgraph = 12; + string result = 5; + uint32 response_time_ms = 6; + uint32 request_bytes = 7; + uint32 response_bytes = 8; + double total_fees_usd = 9; + repeated IndexerQueryProtobuf indexer_queries = 10; +} + +message IndexerQueryProtobuf { + bytes indexer = 1; + bytes deployment = 2; + bytes allocation = 3; + string indexed_chain = 4; + string url = 5; + double fee_grt = 6; + uint32 response_time_ms = 7; + uint32 seconds_behind = 8; + string result = 9; + string indexer_errors = 10; + uint64 blocks_behind = 11; +} \ No newline at end of file diff --git a/oracle/clickhouse/tables.sql b/oracle/clickhouse/tables.sql new file mode 100644 index 0000000..d44f0af --- /dev/null +++ b/oracle/clickhouse/tables.sql @@ -0,0 +1,130 @@ +-- Create the Kafka engine table to ingest Protobuf messages +CREATE TABLE IF NOT EXISTS kafka_qos_data +( + event_time DateTime DEFAULT now(), + gateway_id String, + receipt_signer String, + query_id String, + api_key String, + user_id String, + subgraph Nullable(String), + result String, + response_time_ms UInt32, + request_bytes UInt32, + response_bytes Nullable(UInt32), + total_fees_usd Float64, + indexer_queries Nested( + indexer String, + deployment String, + allocation String, + indexed_chain String, + url String, + fee_grt Float64, + response_time_ms UInt32, + seconds_behind UInt32, + result String, + indexer_errors String, + blocks_behind UInt64 + ) +) ENGINE = Kafka +SETTINGS + kafka_broker_list = 'redpanda:9092', + kafka_topic_list = 'gateway_qos_topic', + kafka_group_name = 'clickhouse_qos_consumer', + kafka_format = 'Protobuf', + kafka_schema = '/etc/clickhouse-server/schema.proto:qos.ClientQueryProtobuf'; + +-- Create the destination table for the raw data +CREATE TABLE IF NOT EXISTS raw_qos_data +( + event_time DateTime, + gateway_id String, + receipt_signer String, + query_id String, + api_key String, + user_id String, + subgraph Nullable(String), + result String, + response_time_ms UInt32, + request_bytes UInt32, + response_bytes Nullable(UInt32), + total_fees_usd Float64, + indexer_queries Nested( + indexer String, + deployment String, + allocation String, + indexed_chain String, + url String, + fee_grt Float64, + response_time_ms UInt32, + seconds_behind UInt32, + result String, + indexer_errors String, + blocks_behind UInt64 + ) +) ENGINE = MergeTree() +ORDER BY (event_time, gateway_id) +TTL event_time + INTERVAL 7 DAY; + +-- Create a simplified materialized view +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_raw_qos_data TO raw_qos_data AS +SELECT + event_time, + gateway_id, + receipt_signer, + query_id, + api_key, + user_id, + subgraph, + result, + response_time_ms, + request_bytes, + response_bytes, + total_fees_usd, + indexer_queries.indexer, + indexer_queries.deployment, + indexer_queries.allocation, + indexer_queries.indexed_chain, + indexer_queries.url, + indexer_queries.fee_grt, + indexer_queries.response_time_ms, + indexer_queries.seconds_behind, + indexer_queries.result, + indexer_queries.indexer_errors, + indexer_queries.blocks_behind +FROM kafka_qos_data; + +-- Create the MaterializedView to process and store data from Kafka +CREATE TABLE IF NOT EXISTS qos_data +( + event_time DateTime DEFAULT now(), + gateway_id String, + receipt_signer String, + query_id String, + api_key String, + user_id String, + subgraph Nullable(String), + result String, + response_time_ms UInt32, + request_bytes UInt32, + response_bytes Nullable(UInt32), + total_fees_usd Float64, + indexer_queries Nested( + indexer String, + deployment String, + allocation String, + indexed_chain String, + url String, + fee_grt Float64, + response_time_ms UInt32, + seconds_behind UInt32, + result String, + indexer_errors String, + blocks_behind UInt64 + ) +) ENGINE = MergeTree() +ORDER BY (event_time, gateway_id, query_id); + +-- Only create the materialized view if it doesn't exist already +CREATE MATERIALIZED VIEW IF NOT EXISTS qos_data_mv TO qos_data AS +SELECT * FROM kafka_qos_data; diff --git a/oracle/docker-compose.yml b/oracle/docker-compose.yml new file mode 100644 index 0000000..0317483 --- /dev/null +++ b/oracle/docker-compose.yml @@ -0,0 +1,114 @@ +# QoS Oracle - Data pipeline for The Graph Network quality metrics +# Common configuration blocks +x-default-healthcheck: &default-healthcheck + interval: 10s + timeout: 5s + retries: 3 + +x-clickhouse-env: &clickhouse-env + CLICKHOUSE_DB: default + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: "" + +# Services +services: + # Core dependencies + clickhouse-server: + image: clickhouse/clickhouse-server:22.1 + profiles: [all, deps, dev] + expose: + - "8123" # HTTP interface + - "9000" # Native protocol + ports: + - "8123:8123" # HTTP: queries/monitoring + - "9000:9000" # Native: client connections + volumes: + - ./clickhouse:/docker-entrypoint-initdb.d:ro # init scripts: read-only + - ./clickhouse/schema.proto:/etc/clickhouse-server/schema.proto:ro # schema: read-only + - clickhouse_data:/var/lib/clickhouse:delegated # data: container writes + ulimits: + nofile: + soft: 262144 + hard: 262144 + environment: + <<: *clickhouse-env + healthcheck: + <<: *default-healthcheck + test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"] + + redpanda: + image: redpandadata/redpanda:latest + profiles: [all, deps, dev] + expose: + - "9092" # Kafka API + - "8082" # Pandaproxy HTTP + ports: + - "9092:9092" # Kafka: producer/consumer + command: + - redpanda + - start + - --smp=1 + - --memory=1G + - --reserve-memory=0M + - --overprovisioned + - --node-id=0 + - --check=false + - --pandaproxy-addr=0.0.0.0:8082 + - --advertise-pandaproxy-addr=redpanda:8082 + - --kafka-addr=0.0.0.0:9092 + - --advertise-kafka-addr=redpanda:9092 + - --rpc-addr=0.0.0.0:33145 + - --advertise-rpc-addr=redpanda:33145 + volumes: + - redpanda_data:/var/lib/redpanda/data:delegated # data: container writes + healthcheck: + <<: *default-healthcheck + test: ["CMD-SHELL", "rpk cluster health | grep -q 'Healthy:.*true'"] + start_period: 30s # Redpanda needs more time to initialize + + # Application services + graphql-api: + build: + context: ./graphql-api + profiles: [all, dev] + expose: + - "8000" # GraphQL API + ports: + - "8000:8000" # API: external access + depends_on: + clickhouse-server: + condition: service_healthy + environment: + CLICKHOUSE_URL: http://clickhouse-server:8123 + CLICKHOUSE_DB: default + healthcheck: + <<: *default-healthcheck + test: ["CMD-SHELL", "curl -f http://localhost:8000/health || exit 1"] + + # Test/development tools + test-producer: + build: + context: ./test-producer + dockerfile: ./Dockerfile + profiles: [all, dev] + depends_on: + redpanda: + condition: service_healthy + clickhouse-server: + condition: service_healthy + volumes: + - ./clickhouse/schema.proto:/usr/src/app/schema.proto:ro # schema: read-only + environment: + KAFKA_BROKER: redpanda:9092 + MESSAGES_PER_SECOND: 5 + restart: on-failure + healthcheck: + <<: *default-healthcheck + test: ["CMD-SHELL", "ps aux | grep -v grep | grep -q qos_kafka_producer || exit 1"] + +# Persistent volumes +volumes: + clickhouse_data: # clickhouse persistent storage + driver: local + redpanda_data: # redpanda persistent storage + driver: local diff --git a/oracle/graphql-api/Cargo.toml b/oracle/graphql-api/Cargo.toml new file mode 100644 index 0000000..c8d9b54 --- /dev/null +++ b/oracle/graphql-api/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "qos_graphql_api" +version = "0.1.0" +edition = "2021" + +[dependencies] +actix-web = "4" +async-graphql = "4" +async-graphql-actix-web = "4" +clickhouse = { version = "0.11", features = ["uuid"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tokio = { version = "1", features = ["full"] } +futures = "0.3" \ No newline at end of file diff --git a/oracle/graphql-api/Dockerfile b/oracle/graphql-api/Dockerfile new file mode 100644 index 0000000..c6f0037 --- /dev/null +++ b/oracle/graphql-api/Dockerfile @@ -0,0 +1,114 @@ +# ----------------------------------------------------------------------------- +# 1. Global Build Arguments & Metadata +# ----------------------------------------------------------------------------- +ARG RUST_VERSION=1.74 +ARG DEBIAN_VERSION=bookworm +ARG CHEF_IMAGE=lukemathwalker/cargo-chef:latest-rust-${RUST_VERSION}-slim-${DEBIAN_VERSION} +ARG RUNTIME_BASE=gcr.io/distroless/cc-debian12:nonroot + +ARG TINI_VERSION=v0.19.0 +ARG TINI_SHA256=c5b0666b4cb676901f90dfcb37106783c5fe2077b04590973b885950611b30ee + +ARG BUILD_VERSION="0.1.0" +ARG VCS_REF="development" +ARG BUILD_DATE="2023-11-02" + +# ----------------------------------------------------------------------------- +# 2. Cargo Chef Stage - Prepare Dependency Recipe +# ----------------------------------------------------------------------------- +FROM ${CHEF_IMAGE} AS chef +WORKDIR /app +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +# ----------------------------------------------------------------------------- +# 3. Builder Stage - Compile Rust with Cached Dependencies +# ----------------------------------------------------------------------------- +FROM ${CHEF_IMAGE} AS builder + +ARG CARGO_BUILD_JOBS="default" +ARG CARGO_PROFILE=release +ENV RUSTFLAGS="-C opt-level=3 -C codegen-units=1" + +# Install build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config \ + libssl-dev \ + ca-certificates \ + binutils \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Cook dependencies separately +COPY --from=chef /app/recipe.json recipe.json +RUN cargo chef cook --profile ${CARGO_PROFILE} --recipe-path recipe.json + +# Build the application +COPY . . +RUN cargo build --profile ${CARGO_PROFILE} --locked --jobs "${CARGO_BUILD_JOBS}" \ + && strip target/${CARGO_PROFILE}/graphql_api \ + && cp target/${CARGO_PROFILE}/graphql_api /app/graphql_api + +# ----------------------------------------------------------------------------- +# 4. Preparation Stage (Tini + Directory Setup) +# ----------------------------------------------------------------------------- +FROM debian:${DEBIAN_VERSION}-slim AS preparation + +ARG TINI_VERSION +ARG TINI_SHA256 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +# Download Tini and verify its checksum +RUN curl -fsSL -o /tini \ + https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static-amd64 \ + && echo "${TINI_SHA256} /tini" | sha256sum --check \ + && chmod +x /tini + +# Create log directory (owned by non-root UID 65532) +RUN mkdir -p /var/log/graphql_api \ + && chown 65532:65532 /var/log/graphql_api + +# ----------------------------------------------------------------------------- +# 5. Final Stage - Minimal Distroless Image +# ----------------------------------------------------------------------------- +FROM ${RUNTIME_BASE} + +COPY --from=preparation /tini /usr/local/bin/tini +COPY --from=builder /app/graphql_api /usr/local/bin/graphql_api +COPY --from=preparation /var/log/graphql_api /var/log/graphql_api + +ENV TZ=UTC \ + LANG=C.UTF-8 \ + SSL_CERT_DIR=/etc/ssl/certs \ + RUST_BACKTRACE=1 + +WORKDIR / +STOPSIGNAL SIGTERM + +# Healthcheck for API +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD ["/usr/local/bin/graphql_api", "--health-check"] || exit 1 + +# Expose GraphQL port +EXPOSE 8000 + +# Distroless:nonroot automatically runs as UID 65532 +USER nonroot + +ENTRYPOINT ["/usr/local/bin/tini", "--", "/usr/local/bin/graphql_api"] + +# ----------------------------------------------------------------------------- +# 6. OCI Labels for Metadata +# ----------------------------------------------------------------------------- +LABEL org.opencontainers.image.version="${BUILD_VERSION}" \ + org.opencontainers.image.revision="${VCS_REF}" \ + org.opencontainers.image.created="${BUILD_DATE}" \ + org.opencontainers.image.source="https://github.com/graphops/qos-oracle" \ + org.opencontainers.image.description="The Graph QoS Oracle GraphQL API" \ + org.opencontainers.image.vendor="GraphOps" \ + org.opencontainers.image.title="QoS GraphQL API" \ + maintainer="GraphOps " diff --git a/oracle/graphql-api/src/main.rs b/oracle/graphql-api/src/main.rs new file mode 100644 index 0000000..40a204f --- /dev/null +++ b/oracle/graphql-api/src/main.rs @@ -0,0 +1,190 @@ +use actix_web::{web, App, HttpServer, HttpResponse, Result}; +use async_graphql::{Schema, EmptyMutation, EmptySubscription, Context, Object, SimpleObject}; +use async_graphql_actix_web::{GraphQLRequest, GraphQLResponse}; +use clickhouse::{Client, Row}; +use serde::{Deserialize, Serialize}; + +// Define the GraphQL types +#[derive(SimpleObject, Deserialize, Serialize)] +struct IndexerQuery { + indexer: String, + deployment: String, + allocation: String, + indexed_chain: String, + url: String, + fee_grt: f64, + response_time_ms: u32, + seconds_behind: u32, + result: String, + indexer_errors: String, + blocks_behind: u64, +} + +#[derive(SimpleObject)] +struct QosReport { + event_time: String, + gateway_id: String, + receipt_signer: String, + query_id: String, + api_key: String, + user_id: String, + subgraph: Option, + result: String, + response_time_ms: u32, + request_bytes: u32, + response_bytes: Option, + total_fees_usd: f64, + indexer_queries: Vec, +} + +// Structure to receive raw data from ClickHouse - needs both Row and Deserialize +#[derive(Row, Deserialize)] +struct ClickhouseRow { + event_time: String, + gateway_id: String, + receipt_signer: String, + query_id: String, + api_key: String, + user_id: String, + subgraph: Option, + result: String, + response_time_ms: u32, + request_bytes: u32, + response_bytes: Option, + total_fees_usd: f64, + indexer_queries: Vec<( + String, String, String, String, String, + f64, u32, u32, String, String, u64 + )>, +} + +struct QueryRoot; + +#[Object] +impl QueryRoot { + /// Query for QoS Reports within a time range. + async fn qos_reports(&self, _ctx: &Context<'_>, from: String, to: String) -> async_graphql::Result> { + let client = Client::default() + .with_url(&std::env::var("CLICKHOUSE_URL").unwrap_or_else(|_| "http://localhost:8123".into())) + .with_database(&std::env::var("CLICKHOUSE_DB").unwrap_or_else(|_| "default".into())); + + let query = format!( + "SELECT event_time, gateway_id, receipt_signer, query_id, api_key, user_id, subgraph, result, response_time_ms, request_bytes, response_bytes, total_fees_usd, indexer_queries \ + FROM raw_qos_data \ + WHERE event_time BETWEEN toDateTime('{}') AND toDateTime('{}') \ + ORDER BY event_time DESC", + from, to + ); + + // Properly handle the ClickHouse cursor + let cursor_result = client.query(&query).fetch::(); + + match cursor_result { + Ok(mut cursor) => { + let mut reports = Vec::new(); + + // Manually collect rows from the cursor + while let Ok(Some(row)) = cursor.next().await { + // Convert the tuple array to IndexerQuery structs + let indexer_queries = row.indexer_queries.into_iter().map(|tuple| { + IndexerQuery { + indexer: tuple.0, + deployment: tuple.1, + allocation: tuple.2, + indexed_chain: tuple.3, + url: tuple.4, + fee_grt: tuple.5, + response_time_ms: tuple.6, + seconds_behind: tuple.7, + result: tuple.8, + indexer_errors: tuple.9, + blocks_behind: tuple.10, + } + }).collect(); + + reports.push(QosReport { + event_time: row.event_time, + gateway_id: row.gateway_id, + receipt_signer: row.receipt_signer, + query_id: row.query_id, + api_key: row.api_key, + user_id: row.user_id, + subgraph: row.subgraph, + result: row.result, + response_time_ms: row.response_time_ms, + request_bytes: row.request_bytes, + response_bytes: row.response_bytes, + total_fees_usd: row.total_fees_usd, + indexer_queries, + }); + } + + Ok(reports) + }, + Err(err) => Err(async_graphql::Error::new(err.to_string())), + } + } +} + +type GQLSchema = Schema; + +async fn graphql_handler(schema: web::Data, req: GraphQLRequest) -> GraphQLResponse { + schema.execute(req.into_inner()).await.into() +} + +async fn graphql_playground() -> Result { + let html = r#" + + + + GraphQL Playground + + + + + + +
+ + +
Loading GraphQL Playground
+
+ + + + "#; + + Ok(HttpResponse::Ok() + .content_type("text/html; charset=utf-8") + .body(html)) +} + +#[actix_web::main] +async fn main() -> std::io::Result<()> { + let schema = Schema::build(QueryRoot, EmptyMutation, EmptySubscription) + .finish(); + + println!("GraphQL playground: http://localhost:8000"); + + HttpServer::new(move || { + App::new() + .app_data(web::Data::new(schema.clone())) + .route("/graphql", web::post().to(graphql_handler)) + .route("/", web::get().to(graphql_playground)) + }) + .bind("0.0.0.0:8000")? + .run() + .await +} \ No newline at end of file diff --git a/oracle/test-producer/Cargo.lock b/oracle/test-producer/Cargo.lock new file mode 100644 index 0000000..25a9e92 --- /dev/null +++ b/oracle/test-producer/Cargo.lock @@ -0,0 +1,1030 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + +[[package]] +name = "bitflags" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cc" +version = "1.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "indexmap" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.171" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" + +[[package]] +name = "libz-sys" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "miniz_oxide" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +dependencies = [ + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.52.0", +] + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "num_enum" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc" + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +dependencies = [ + "bytes", + "heck", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 1.0.109", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + +[[package]] +name = "qos_kafka_producer" +version = "0.1.0" +dependencies = [ + "prost", + "prost-build", + "rand", + "rdkafka", + "tokio", + "uuid", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.15", +] + +[[package]] +name = "rdkafka" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1de127f294f2dba488ed46760b129d5ecbeabbd337ccbf3739cb29d50db2161c" +dependencies = [ + "futures", + "libc", + "log", + "rdkafka-sys", + "serde", + "serde_derive", + "serde_json", + "slab", + "tokio", +] + +[[package]] +name = "rdkafka-sys" +version = "4.8.0+2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced38182dc436b3d9df0c77976f37a67134df26b050df1f0006688e46fc4c8be" +dependencies = [ + "libc", + "libz-sys", + "num_enum", + "pkg-config", +] + +[[package]] +name = "redox_syscall" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.9.3", + "windows-sys 0.59.0", +] + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" + +[[package]] +name = "socket2" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" +dependencies = [ + "fastrand", + "getrandom 0.3.2", + "once_cell", + "rustix 1.0.3", + "windows-sys 0.59.0", +] + +[[package]] +name = "tokio" +version = "1.44.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" + +[[package]] +name = "toml_edit" +version = "0.22.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "uuid" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +dependencies = [ + "getrandom 0.3.2", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "zerocopy" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd97444d05a4328b90e75e503a34bad781f14e28a823ad3557f0750df1ebcbc6" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6352c01d0edd5db859a63e2605f4ea3183ddbd15e2c4a9e7d32184df75e4f154" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] diff --git a/oracle/test-producer/Cargo.toml b/oracle/test-producer/Cargo.toml new file mode 100644 index 0000000..0e639f0 --- /dev/null +++ b/oracle/test-producer/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "qos_kafka_producer" +version = "0.1.0" +edition = "2021" + +[dependencies] +tokio = { version = "1", features = ["full"] } +rdkafka = { version = "0.28", features = ["tokio"] } +prost = "0.11" +uuid = { version = "1", features = ["v4"] } +rand = "0.8" +chrono = "0.4" + +[build-dependencies] +prost-build = "0.11" \ No newline at end of file diff --git a/oracle/test-producer/Dockerfile b/oracle/test-producer/Dockerfile new file mode 100644 index 0000000..56cc30b --- /dev/null +++ b/oracle/test-producer/Dockerfile @@ -0,0 +1,144 @@ +# ----------------------------------------------------------------------------- +# 1. Global Build Arguments & Metadata +# ----------------------------------------------------------------------------- +ARG RUST_VERSION=1.74 +ARG DEBIAN_VERSION=bookworm +ARG CHEF_IMAGE=lukemathwalker/cargo-chef:latest-rust-${RUST_VERSION}-slim-${DEBIAN_VERSION} +ARG RUNTIME_BASE=gcr.io/distroless/cc-debian12:nonroot + +ARG TINI_VERSION=v0.19.0 +ARG TINI_SHA256=c5b0666b4cb676901f90dfcb37106783c5fe2077b04590973b885950611b30ee + +ARG BUILD_VERSION="0.1.0" +ARG VCS_REF="development" +ARG BUILD_DATE="2023-11-02" + +# ----------------------------------------------------------------------------- +# 2. Cargo Chef Stage - Prepare Dependency Recipe +# ----------------------------------------------------------------------------- +FROM ${CHEF_IMAGE} AS chef +WORKDIR /app +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +# ----------------------------------------------------------------------------- +# 3. Builder Stage - Compile Rust with Cached Dependencies +# ----------------------------------------------------------------------------- +FROM ${CHEF_IMAGE} AS builder + +ARG CARGO_BUILD_JOBS="default" +ARG CARGO_PROFILE=release +ENV RUSTFLAGS="-C opt-level=3 -C codegen-units=1" + +# Install build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config \ + libssl-dev \ + protobuf-compiler \ + g++ \ + make \ + python3 \ + libsasl2-dev \ + zlib1g-dev \ + binutils \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Create schema directories +RUN mkdir -p ../clickhouse + +# Cook dependencies separately +COPY --from=chef /app/recipe.json recipe.json +RUN cargo chef cook --profile ${CARGO_PROFILE} --recipe-path recipe.json + +# Create placeholder schema for build time +RUN echo 'syntax = "proto3"; \ + package qos; \ + message ClientQueryProtobuf { \ + bytes query_id = 1; \ + bytes gateway_id = 2; \ + int64 event_time = 3; \ + repeated IndexerQueryProtobuf indexer_queries = 4; \ + string subgraph = 5; \ + } \ + message IndexerQueryProtobuf { \ + bytes indexer = 1; \ + bytes deployment = 2; \ + bytes allocation = 3; \ + string indexed_chain = 4; \ + string url = 5; \ + double fee_grt = 6; \ + int32 response_time_ms = 7; \ + int32 seconds_behind = 8; \ + string result = 9; \ + string indexer_errors = 10; \ + int32 blocks_behind = 11; \ + }' > ../clickhouse/schema.proto + +# Build the application +COPY . . +RUN cargo build --profile ${CARGO_PROFILE} --locked --jobs "${CARGO_BUILD_JOBS}" \ + && strip target/${CARGO_PROFILE}/qos_kafka_producer \ + && cp target/${CARGO_PROFILE}/qos_kafka_producer /app/qos_kafka_producer + +# ----------------------------------------------------------------------------- +# 4. Preparation Stage (Tini + Directory Setup) +# ----------------------------------------------------------------------------- +FROM debian:${DEBIAN_VERSION}-slim AS preparation + +ARG TINI_VERSION +ARG TINI_SHA256 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +# Download Tini and verify its checksum +RUN curl -fsSL -o /tini \ + https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static-amd64 \ + && echo "${TINI_SHA256} /tini" | sha256sum --check \ + && chmod +x /tini + +# Create log directory (owned by non-root UID 65532) +RUN mkdir -p /var/log/qos_producer \ + && chown 65532:65532 /var/log/qos_producer + +# ----------------------------------------------------------------------------- +# 5. Final Stage - Minimal Distroless Image +# ----------------------------------------------------------------------------- +FROM ${RUNTIME_BASE} + +COPY --from=preparation /tini /usr/local/bin/tini +COPY --from=builder /app/qos_kafka_producer /usr/local/bin/qos_kafka_producer +COPY --from=preparation /var/log/qos_producer /var/log/qos_producer + +ENV TZ=UTC \ + LANG=C.UTF-8 \ + SSL_CERT_DIR=/etc/ssl/certs \ + RUST_BACKTRACE=1 + +WORKDIR / +STOPSIGNAL SIGTERM + +# Optional healthcheck +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD ["/usr/local/bin/qos_kafka_producer", "--health-check"] || exit 1 + +# Distroless:nonroot automatically runs as UID 65532 +USER nonroot + +ENTRYPOINT ["/usr/local/bin/tini", "--", "/usr/local/bin/qos_kafka_producer"] + +# ----------------------------------------------------------------------------- +# 6. OCI Labels for Metadata +# ----------------------------------------------------------------------------- +LABEL org.opencontainers.image.version="${BUILD_VERSION}" \ + org.opencontainers.image.revision="${VCS_REF}" \ + org.opencontainers.image.created="${BUILD_DATE}" \ + org.opencontainers.image.source="https://github.com/graphops/qos-oracle" \ + org.opencontainers.image.description="The Graph QoS Oracle Kafka Producer" \ + org.opencontainers.image.vendor="GraphOps" \ + org.opencontainers.image.title="QoS Kafka Producer" \ + maintainer="GraphOps " \ No newline at end of file diff --git a/oracle/test-producer/build.rs b/oracle/test-producer/build.rs new file mode 100644 index 0000000..54835b3 --- /dev/null +++ b/oracle/test-producer/build.rs @@ -0,0 +1,21 @@ +fn main() -> Result<(), Box> { + // Try different locations for schema.proto + let schema_paths = vec![ + "../clickhouse/schema.proto", + "/usr/src/app/schema.proto", + "schema.proto" + ]; + + // Try each path until one works + for path in schema_paths { + if std::path::Path::new(path).exists() { + prost_build::compile_protos(&[path], &["../clickhouse/"])?; + return Ok(()); + } + } + + // If none of the paths worked, use the default one + prost_build::compile_protos(&["../clickhouse/schema.proto"], &["../clickhouse/"])?; + + Ok(()) +} \ No newline at end of file diff --git a/oracle/test-producer/src/main.rs b/oracle/test-producer/src/main.rs new file mode 100644 index 0000000..72acc8b --- /dev/null +++ b/oracle/test-producer/src/main.rs @@ -0,0 +1,168 @@ +use std::env; +use std::time::Duration; + +use rand::seq::SliceRandom; +use rand::Rng; +use rdkafka::config::ClientConfig; +use rdkafka::producer::{FutureProducer, FutureRecord}; +use rdkafka::util::get_rdkafka_version; +use tokio::time; +use uuid::Uuid; +use prost::Message; +use chrono; + +// Assume the generated Protobuf code is available in the `qos` module. +// The build script should generate the Protobuf code from your schema.proto. +mod qos { + include!(concat!(env!("OUT_DIR"), "/qos.rs")); +} +use qos::{ClientQueryProtobuf, IndexerQueryProtobuf}; + +#[tokio::main] +async fn main() { + // Print librdkafka version + let (version_n, version_s) = get_rdkafka_version(); + println!("rd_kafka_version: 0x{:08x}, {}", version_n, version_s); + + // Read messages per second from the environment variable; default to 10 if not provided. + let mps: u64 = env::var("MESSAGES_PER_SECOND") + .unwrap_or_else(|_| "10".to_string()) + .parse() + .expect("MESSAGES_PER_SECOND must be a valid number"); + let delay = Duration::from_micros(1_000_000 / mps); + + // Allow configuring the broker address from environment + let broker = env::var("KAFKA_BROKER") + .unwrap_or_else(|_| "redpanda:9092".to_string()); + + println!("Connecting to Kafka broker at {}", broker); + + // Wait for Redpanda to be fully ready + println!("Waiting 15 seconds for Redpanda to initialize..."); + time::sleep(Duration::from_secs(15)).await; + + // Create a Kafka producer with extensive logging + let producer: FutureProducer = ClientConfig::new() + .set("bootstrap.servers", &broker) + .set("message.timeout.ms", "30000") + .set("security.protocol", "PLAINTEXT") + .set("debug", "all") + .set("enable.idempotence", "false") // Simplify for testing + .set("retries", "5") // Retry a few times + .set("retry.backoff.ms", "1000") // 1 second between retries + .set("socket.timeout.ms", "10000") // 10 seconds socket timeout + .set("socket.keepalive.enable", "true") + .set("statistics.interval.ms", "30000") // Get stats every 30 seconds + .set("api.version.request", "true") + .set("api.version.request.timeout.ms", "5000") + .create() + .expect("Failed to create Kafka producer"); + + // Verify topic exists before sending + println!("Sending test messages to topic 'gateway_qos_topic'"); + + // Message counter + let mut count = 0; + + loop { + let msg = generate_random_message(); + let mut buf = Vec::new(); + msg.encode(&mut buf).expect("Failed to encode message"); + + let query_id = String::from_utf8_lossy(&msg.query_id).to_string(); + + // Produce message to the target Kafka topic. + let record = FutureRecord::to("gateway_qos_topic") + .payload(&buf) + .key(&query_id); + + match producer.send(record, Duration::from_secs(10)).await { + Ok(delivery) => { + count += 1; + if count % 10 == 0 { // Only log every 10 messages to reduce noise + println!("Successfully delivered message #{}: {:?}", count, delivery); + } + }, + Err((e, _)) => eprintln!("Delivery error: {:?}", e), + } + + // Sleep for the configured delay to maintain the desired messages-per-second rate. + time::sleep(delay).await; + } +} + +fn generate_random_message() -> ClientQueryProtobuf { + let mut rng = rand::thread_rng(); + + // Create random indexer queries + let num_indexers = rng.gen_range(1..5); + let mut indexer_queries = Vec::with_capacity(num_indexers); + + for _ in 0..num_indexers { + let result = if rng.gen_bool(0.95) { "SUCCESS" } else { "ERROR" }; + + let indexer_query = IndexerQueryProtobuf { + indexer: Uuid::new_v4().to_string().into_bytes(), + deployment: Uuid::new_v4().to_string().into_bytes(), + allocation: Uuid::new_v4().to_string().into_bytes(), + indexed_chain: format!("chain-{}", rng.gen_range(1..10)), + url: format!("https://indexer-{}.example.com/graphql", rng.gen_range(1..100)), + fee_grt: rng.gen_range(0.0001..0.01), + response_time_ms: rng.gen_range(50..2000), + seconds_behind: rng.gen_range(0..100), + result: result.to_string(), + indexer_errors: if result == "ERROR" { + "Internal server error".to_string() + } else { + "".to_string() + }, + blocks_behind: rng.gen_range(0..50), + }; + + indexer_queries.push(indexer_query); + } + + // Create a random gateway query with the indexer queries + ClientQueryProtobuf { + query_id: Uuid::new_v4().to_string().into_bytes(), + gateway_id: Uuid::new_v4().to_string().into_bytes(), + event_time: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as i64, + indexer_queries, + subgraph: format!("subgraph-{}", rng.gen_range(1..20)), + } +} + +fn generate_random_indexer_query() -> IndexerQueryProtobuf { + let mut rng = rand::thread_rng(); + + // Sample data + let indexed_chains = ["ethereum", "polygon", "arbitrum", "optimism"]; + let urls = ["https://indexer1.io/graphql", "https://indexer2.io/graphql", "https://indexer3.io/graphql"]; + let results = ["success", "error", "timeout"]; + let error_messages = ["", "timeout error", "server error", "validation error"]; + + IndexerQueryProtobuf { + indexer: Uuid::new_v4().to_string().into_bytes(), + deployment: Uuid::new_v4().to_string().into_bytes(), + allocation: Uuid::new_v4().to_string().into_bytes(), + indexed_chain: indexed_chains.choose(&mut rng) + .expect("Indexed chains array should not be empty") + .to_string(), + url: urls.choose(&mut rng) + .expect("URLs array should not be empty") + .to_string(), + fee_grt: rng.gen_range(0.0001..0.1), + response_time_ms: rng.gen_range(10..1000), + seconds_behind: rng.gen_range(0..300), + result: results.choose(&mut rng) + .expect("Results array should not be empty") + .to_string(), + indexer_errors: error_messages.choose(&mut rng) + .expect("Error messages array should not be empty") + .to_string(), + blocks_behind: rng.gen_range(0..1000), + } +} diff --git a/subgraph/README.md b/subgraph/README.md deleted file mode 100644 index 297d806..0000000 --- a/subgraph/README.md +++ /dev/null @@ -1 +0,0 @@ -# QoS Oracle V2 subgraph \ No newline at end of file From 4dde0e441c8dc51d109f6a11efc245bd42b3d3d9 Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Wed, 19 Mar 2025 23:33:43 -0300 Subject: [PATCH 02/20] fix: schema issues with test producer --- oracle/test-producer/Dockerfile | 25 +------- oracle/test-producer/build.rs | 27 +++----- oracle/test-producer/src/main.rs | 103 ++++++++++--------------------- 3 files changed, 42 insertions(+), 113 deletions(-) diff --git a/oracle/test-producer/Dockerfile b/oracle/test-producer/Dockerfile index 56cc30b..069dac5 100644 --- a/oracle/test-producer/Dockerfile +++ b/oracle/test-producer/Dockerfile @@ -53,29 +53,8 @@ RUN mkdir -p ../clickhouse COPY --from=chef /app/recipe.json recipe.json RUN cargo chef cook --profile ${CARGO_PROFILE} --recipe-path recipe.json -# Create placeholder schema for build time -RUN echo 'syntax = "proto3"; \ - package qos; \ - message ClientQueryProtobuf { \ - bytes query_id = 1; \ - bytes gateway_id = 2; \ - int64 event_time = 3; \ - repeated IndexerQueryProtobuf indexer_queries = 4; \ - string subgraph = 5; \ - } \ - message IndexerQueryProtobuf { \ - bytes indexer = 1; \ - bytes deployment = 2; \ - bytes allocation = 3; \ - string indexed_chain = 4; \ - string url = 5; \ - double fee_grt = 6; \ - int32 response_time_ms = 7; \ - int32 seconds_behind = 8; \ - string result = 9; \ - string indexer_errors = 10; \ - int32 blocks_behind = 11; \ - }' > ../clickhouse/schema.proto +# Copy the ACTUAL schema.proto from the project +COPY ../clickhouse/schema.proto ../clickhouse/schema.proto # Build the application COPY . . diff --git a/oracle/test-producer/build.rs b/oracle/test-producer/build.rs index 54835b3..8626b54 100644 --- a/oracle/test-producer/build.rs +++ b/oracle/test-producer/build.rs @@ -1,21 +1,8 @@ -fn main() -> Result<(), Box> { - // Try different locations for schema.proto - let schema_paths = vec![ - "../clickhouse/schema.proto", - "/usr/src/app/schema.proto", - "schema.proto" - ]; - - // Try each path until one works - for path in schema_paths { - if std::path::Path::new(path).exists() { - prost_build::compile_protos(&[path], &["../clickhouse/"])?; - return Ok(()); - } - } - - // If none of the paths worked, use the default one - prost_build::compile_protos(&["../clickhouse/schema.proto"], &["../clickhouse/"])?; - - Ok(()) +fn main() { + // Specify the path to the schema.proto file + let proto_file = "../clickhouse/schema.proto"; + + println!("cargo:rerun-if-changed={}", proto_file); + + prost_build::compile_protos(&[proto_file], &["../"]).unwrap(); } \ No newline at end of file diff --git a/oracle/test-producer/src/main.rs b/oracle/test-producer/src/main.rs index 72acc8b..09b3ed6 100644 --- a/oracle/test-producer/src/main.rs +++ b/oracle/test-producer/src/main.rs @@ -9,10 +9,8 @@ use rdkafka::util::get_rdkafka_version; use tokio::time; use uuid::Uuid; use prost::Message; -use chrono; -// Assume the generated Protobuf code is available in the `qos` module. -// The build script should generate the Protobuf code from your schema.proto. +// Use the exact schema from clickhouse/schema.proto mod qos { include!(concat!(env!("OUT_DIR"), "/qos.rs")); } @@ -52,37 +50,29 @@ async fn main() { .set("retry.backoff.ms", "1000") // 1 second between retries .set("socket.timeout.ms", "10000") // 10 seconds socket timeout .set("socket.keepalive.enable", "true") - .set("statistics.interval.ms", "30000") // Get stats every 30 seconds - .set("api.version.request", "true") - .set("api.version.request.timeout.ms", "5000") .create() - .expect("Failed to create Kafka producer"); + .expect("Producer creation error"); + + // Make sure this matches the topic name in ClickHouse Kafka engine + let topic = "gateway_qos_topic"; + let mut counter = 0; - // Verify topic exists before sending - println!("Sending test messages to topic 'gateway_qos_topic'"); - - // Message counter - let mut count = 0; - loop { - let msg = generate_random_message(); - let mut buf = Vec::new(); - msg.encode(&mut buf).expect("Failed to encode message"); + let message = generate_message_exact_schema(); + let payload = message.encode_to_vec(); - let query_id = String::from_utf8_lossy(&msg.query_id).to_string(); + counter += 1; - // Produce message to the target Kafka topic. - let record = FutureRecord::to("gateway_qos_topic") - .payload(&buf) - .key(&query_id); - - match producer.send(record, Duration::from_secs(10)).await { - Ok(delivery) => { - count += 1; - if count % 10 == 0 { // Only log every 10 messages to reduce noise - println!("Successfully delivered message #{}: {:?}", count, delivery); - } - }, + match producer + .send( + FutureRecord::to(topic) + .payload(&payload) + .key(&format!("{}", counter)), + Duration::from_secs(0), + ) + .await + { + Ok((partition, offset)) => println!("Delivered: ({}, {})", partition, offset), Err((e, _)) => eprintln!("Delivery error: {:?}", e), } @@ -91,7 +81,8 @@ async fn main() { } } -fn generate_random_message() -> ClientQueryProtobuf { +// Function to generate a message with the exact schema from clickhouse/schema.proto +fn generate_message_exact_schema() -> ClientQueryProtobuf { let mut rng = rand::thread_rng(); // Create random indexer queries @@ -122,47 +113,19 @@ fn generate_random_message() -> ClientQueryProtobuf { indexer_queries.push(indexer_query); } - // Create a random gateway query with the indexer queries + // Create a ClientQueryProtobuf with EXACTLY the fields from schema.proto ClientQueryProtobuf { - query_id: Uuid::new_v4().to_string().into_bytes(), - gateway_id: Uuid::new_v4().to_string().into_bytes(), - event_time: std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as i64, - indexer_queries, + gateway_id: Uuid::new_v4().to_string(), + receipt_signer: Uuid::new_v4().to_string().into_bytes(), + query_id: Uuid::new_v4().to_string(), + api_key: format!("api-key-{}", rng.gen_range(1..1000)), + user_id: format!("user-{}", rng.gen_range(1..500)), subgraph: format!("subgraph-{}", rng.gen_range(1..20)), - } -} - -fn generate_random_indexer_query() -> IndexerQueryProtobuf { - let mut rng = rand::thread_rng(); - - // Sample data - let indexed_chains = ["ethereum", "polygon", "arbitrum", "optimism"]; - let urls = ["https://indexer1.io/graphql", "https://indexer2.io/graphql", "https://indexer3.io/graphql"]; - let results = ["success", "error", "timeout"]; - let error_messages = ["", "timeout error", "server error", "validation error"]; - - IndexerQueryProtobuf { - indexer: Uuid::new_v4().to_string().into_bytes(), - deployment: Uuid::new_v4().to_string().into_bytes(), - allocation: Uuid::new_v4().to_string().into_bytes(), - indexed_chain: indexed_chains.choose(&mut rng) - .expect("Indexed chains array should not be empty") - .to_string(), - url: urls.choose(&mut rng) - .expect("URLs array should not be empty") - .to_string(), - fee_grt: rng.gen_range(0.0001..0.1), - response_time_ms: rng.gen_range(10..1000), - seconds_behind: rng.gen_range(0..300), - result: results.choose(&mut rng) - .expect("Results array should not be empty") - .to_string(), - indexer_errors: error_messages.choose(&mut rng) - .expect("Error messages array should not be empty") - .to_string(), - blocks_behind: rng.gen_range(0..1000), + result: if rng.gen_bool(0.95) { "SUCCESS" } else { "ERROR" }, + response_time_ms: rng.gen_range(50..5000), + request_bytes: rng.gen_range(100..5000), + response_bytes: rng.gen_range(200..10000), + total_fees_usd: rng.gen_range(0.001..0.1), + indexer_queries, } } From 88eddc48463e686bad4b227b98b406adb9028dc9 Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Mon, 24 Mar 2025 00:56:10 -0300 Subject: [PATCH 03/20] feat: added README.md, bunch of improvements to clickhouse setting, general cargo setup --- Cargo.lock | 2891 +++++++++++++++++++++++ Cargo.toml | 6 + README.md | 122 +- oracle/Cargo.toml | 32 + oracle/README.md | 1 - oracle/clickhouse/allow_schema_path.xml | 11 + oracle/clickhouse/settings.xml | 5 + oracle/clickhouse/tables.sql | 36 +- oracle/docker-compose.yml | 35 +- oracle/test-producer/src/main.rs | 418 +++- 10 files changed, 3459 insertions(+), 98 deletions(-) create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 oracle/Cargo.toml delete mode 100644 oracle/README.md create mode 100644 oracle/clickhouse/allow_schema_path.xml create mode 100644 oracle/clickhouse/settings.xml diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..2df0b1f --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2891 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + +[[package]] +name = "actix" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de7fa236829ba0841304542f7614c42b80fca007455315c45c785ccfa873a85b" +dependencies = [ + "actix-macros", + "actix-rt", + "actix_derive", + "bitflags", + "bytes", + "crossbeam-channel", + "futures-core", + "futures-sink", + "futures-task", + "futures-util", + "log", + "once_cell", + "parking_lot", + "pin-project-lite", + "smallvec", + "tokio", + "tokio-util", +] + +[[package]] +name = "actix-codec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-sink", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "actix-http" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa882656b67966045e4152c634051e70346939fced7117d5f0b52146a7c74c9" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", + "base64 0.22.1", + "bitflags", + "brotli", + "bytes", + "bytestring", + "derive_more", + "encoding_rs", + "flate2", + "foldhash", + "futures-core", + "h2", + "http", + "httparse", + "httpdate", + "itoa", + "language-tags", + "local-channel", + "mime", + "percent-encoding", + "pin-project-lite", + "rand 0.9.0", + "sha1", + "smallvec", + "tokio", + "tokio-util", + "tracing", + "zstd", +] + +[[package]] +name = "actix-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" +dependencies = [ + "quote", + "syn 2.0.100", +] + +[[package]] +name = "actix-router" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d324164c51f63867b57e73ba5936ea151b8a41a1d23d1031eeb9f70d0236f8" +dependencies = [ + "bytestring", + "cfg-if", + "http", + "regex", + "regex-lite", + "serde", + "tracing", +] + +[[package]] +name = "actix-rt" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eda4e2a6e042aa4e55ac438a2ae052d3b5da0ecf83d7411e1a368946925208" +dependencies = [ + "futures-core", + "tokio", +] + +[[package]] +name = "actix-server" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6398974fd4284f4768af07965701efbbb5fdc0616bff20cade1bb14b77675e24" +dependencies = [ + "actix-rt", + "actix-service", + "actix-utils", + "futures-core", + "futures-util", + "mio", + "socket2", + "tokio", + "tracing", +] + +[[package]] +name = "actix-service" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e46f36bf0e5af44bdc4bdb36fbbd421aa98c79a9bce724e1edeb3894e10dc7f" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "actix-utils" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a1dcdff1466e3c2488e1cb5c36a71822750ad43839937f85d2f4d9f8b705d8" +dependencies = [ + "local-waker", + "pin-project-lite", +] + +[[package]] +name = "actix-web" +version = "4.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2e3b15b3dc6c6ed996e4032389e9849d4ab002b1e92fbfe85b5f307d1479b4d" +dependencies = [ + "actix-codec", + "actix-http", + "actix-macros", + "actix-router", + "actix-rt", + "actix-server", + "actix-service", + "actix-utils", + "actix-web-codegen", + "bytes", + "bytestring", + "cfg-if", + "cookie", + "derive_more", + "encoding_rs", + "foldhash", + "futures-core", + "futures-util", + "impl-more", + "itoa", + "language-tags", + "log", + "mime", + "once_cell", + "pin-project-lite", + "regex", + "regex-lite", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "socket2", + "time", + "tracing", + "url", +] + +[[package]] +name = "actix-web-actors" +version = "4.3.1+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98c5300b38fd004fe7d2a964f9a90813fdbe8a81fed500587e78b1b71c6f980" +dependencies = [ + "actix", + "actix-codec", + "actix-http", + "actix-web", + "bytes", + "bytestring", + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "actix-web-codegen" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f591380e2e68490b5dfaf1dd1aa0ebe78d84ba7067078512b4ea6e4492d622b8" +dependencies = [ + "actix-router", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "actix_derive" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6ac1e58cded18cb28ddc17143c4dea5345b3ad575e14f32f66e4054a56eb271" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" + +[[package]] +name = "ascii_utils" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71938f30533e4d95a6d17aa530939da3842c2ab6f4f84b9dae68447e4129f74a" + +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-graphql" +version = "4.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9ed522678d412d77effe47b3c82314ac36952a35e6e852093dd48287c421f80" +dependencies = [ + "async-graphql-derive", + "async-graphql-parser", + "async-graphql-value", + "async-stream", + "async-trait", + "base64 0.13.1", + "bytes", + "fast_chemail", + "fnv", + "futures-util", + "http", + "indexmap 1.9.3", + "mime", + "multer", + "num-traits", + "once_cell", + "pin-project-lite", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "static_assertions", + "tempfile", + "thiserror 1.0.69", +] + +[[package]] +name = "async-graphql-actix-web" +version = "4.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c23479439e8a3819a937b88f6e8ec8e2185abebdc46a9b6d726e92518bbf858f" +dependencies = [ + "actix", + "actix-http", + "actix-web", + "actix-web-actors", + "async-channel", + "async-graphql", + "futures-channel", + "futures-util", + "serde_json", + "thiserror 1.0.69", +] + +[[package]] +name = "async-graphql-derive" +version = "4.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c121a894495d7d3fc3d4e15e0a9843e422e4d1d9e3c514d8062a1c94b35b005d" +dependencies = [ + "Inflector", + "async-graphql-parser", + "darling", + "proc-macro-crate 1.3.1", + "proc-macro2", + "quote", + "syn 1.0.109", + "thiserror 1.0.69", +] + +[[package]] +name = "async-graphql-parser" +version = "4.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b6c386f398145c6180206c1869c2279f5a3d45db5be4e0266148c6ac5c6ad68" +dependencies = [ + "async-graphql-value", + "pest", + "serde", + "serde_json", +] + +[[package]] +name = "async-graphql-value" +version = "4.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a941b499fead4a3fb5392cabf42446566d18c86313f69f2deab69560394d65f" +dependencies = [ + "bytes", + "indexmap 1.9.3", + "serde", + "serde_json", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "async-trait" +version = "0.1.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bstr" +version = "1.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" +dependencies = [ + "memchr", +] + +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +dependencies = [ + "serde", +] + +[[package]] +name = "bytestring" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e465647ae23b2823b0753f50decb2d5a86d2bb2cac04788fafd1f80e45378e5f" +dependencies = [ + "bytes", +] + +[[package]] +name = "cc" +version = "1.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "clickhouse" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0875e527e299fc5f4faba42870bf199a39ab0bb2dbba1b8aef0a2151451130f" +dependencies = [ + "bstr", + "bytes", + "clickhouse-derive", + "clickhouse-rs-cityhash-sys", + "futures", + "hyper", + "hyper-tls", + "lz4", + "sealed", + "serde", + "static_assertions", + "thiserror 1.0.69", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "clickhouse-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18af5425854858c507eec70f7deb4d5d8cec4216fcb086283a78872387281ea5" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 1.0.109", +] + +[[package]] +name = "clickhouse-rs-cityhash-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4baf9d4700a28d6cb600e17ed6ae2b43298a5245f1f76b4eab63027ebfd592b9" +dependencies = [ + "cc", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "cookie" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "deranged" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_more" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", + "unicode-xid", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "fast_chemail" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "495a39d30d624c2caabe6312bfead73e7717692b44e0b32df168c275a2e8e9e4" +dependencies = [ + "ascii_utils", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "flate2" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap 2.8.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "impl-more" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" +dependencies = [ + "equivalent", + "hashbrown 0.15.2", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "language-tags" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.171" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" + +[[package]] +name = "libz-sys" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" + +[[package]] +name = "litemap" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" + +[[package]] +name = "local-channel" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8" +dependencies = [ + "futures-core", + "futures-sink", + "local-waker", +] + +[[package]] +name = "local-waker" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" + +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.52.0", +] + +[[package]] +name = "multer" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "log", + "memchr", + "mime", + "spin", + "version_check", +] + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_enum" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" +dependencies = [ + "proc-macro-crate 3.3.0", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc" + +[[package]] +name = "openssl" +version = "0.10.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pest" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" +dependencies = [ + "memchr", + "thiserror 2.0.12", + "ucd-trie", +] + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.8.0", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit 0.19.15", +] + +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit 0.22.24", +] + +[[package]] +name = "proc-macro2" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +dependencies = [ + "bytes", + "heck 0.4.1", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 1.0.109", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + +[[package]] +name = "qos_graphql_api" +version = "0.1.0" +dependencies = [ + "actix-web", + "async-graphql", + "async-graphql-actix-web", + "clickhouse", + "futures", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "qos_kafka_producer" +version = "0.1.0" +dependencies = [ + "chrono", + "prost", + "prost-build", + "rand 0.8.5", + "rdkafka", + "tokio", + "uuid", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", + "zerocopy", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.15", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.2", +] + +[[package]] +name = "rdkafka" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1de127f294f2dba488ed46760b129d5ecbeabbd337ccbf3739cb29d50db2161c" +dependencies = [ + "futures", + "libc", + "log", + "rdkafka-sys", + "serde", + "serde_derive", + "serde_json", + "slab", + "tokio", +] + +[[package]] +name = "rdkafka-sys" +version = "4.8.0+2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced38182dc436b3d9df0c77976f37a67134df26b050df1f0006688e46fc4c8be" +dependencies = [ + "libc", + "libz-sys", + "num_enum", + "pkg-config", +] + +[[package]] +name = "redox_syscall" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.9.3", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustversion" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sealed" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b5e421024b5e5edfbaa8e60ecf90bda9dbffc602dbb230e6028763f85f0c68c" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" + +[[package]] +name = "socket2" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "tempfile" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" +dependencies = [ + "fastrand", + "getrandom 0.3.2", + "once_cell", + "rustix 1.0.3", + "windows-sys 0.59.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "time" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" + +[[package]] +name = "time-macros" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.44.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap 2.8.0", + "toml_datetime", + "winnow 0.5.40", +] + +[[package]] +name = "toml_edit" +version = "0.22.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" +dependencies = [ + "indexmap 2.8.0", + "toml_datetime", + "winnow 0.7.4", +] + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "tracing-core" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +dependencies = [ + "getrandom 0.3.2", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn 2.0.100", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + +[[package]] +name = "winnow" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", + "synstructure", +] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.15+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..3a81aeb --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,6 @@ +[workspace] +members = [ + "oracle/test-producer", + "oracle/graphql-api" +] +resolver = "2" \ No newline at end of file diff --git a/README.md b/README.md index ed33bd9..4fe67f8 100644 --- a/README.md +++ b/README.md @@ -1 +1,121 @@ -# qos-oracle-v2 \ No newline at end of file +# QoS Oracle V2 + +## Overview + +QoS Oracle V2 is a data pipeline and reporting system for The Graph Network quality metrics. It collects, processes, and exposes quality-of-service data from Gateway nodes, providing valuable insights into indexer performance and network health. + +## Architecture + +The system consists of three main components: + +1. **Data Ingestion** - Kafka/Redpanda message broker receives QoS data from Gateways +2. **Data Processing** - ClickHouse database ingests, stores, and aggregates the data +3. **Data Exposure** - GraphQL API serves pre-aggregated metrics for analysis + + +## Features + +- High-throughput data ingestion (2,000+ messages/second) +- Efficient storage with automatic TTL for raw data +- Pre-aggregated metrics in 5-minute, hourly, and daily intervals +- GraphQL API for flexible querying of historical data +- Docker-based deployment for easy setup and operation + +## Getting Started + +### Prerequisites + +- Docker and Docker Compose +- 4+ CPU cores recommended +- 16+ GB RAM recommended +- 250+ GB SSD storage recommended + +### Installation + +1. Clone the repository: + ``` + git clone https://github.com/graphops/qos-oracle.git + cd qos-oracle + ``` + +2. Start the services: + ``` + docker-compose -f oracle/docker-compose.yml up -d + ``` + +3. Verify the installation: + ``` + curl http://localhost:8000/health + ``` + +4. Access the GraphQL playground at http://localhost:8000 + +### Configuration + +The system can be configured through environment variables in the docker-compose.yml file: + +- `CLICKHOUSE_URL` - ClickHouse server URL +- `CLICKHOUSE_DB` - ClickHouse database name +- `KAFKA_BROKER` - Kafka/Redpanda broker address +- `MESSAGES_PER_SECOND` - Test producer message rate (for development) + +## Usage + +### GraphQL API + +The GraphQL API provides access to QoS reports and aggregated metrics. Here's an example query: + +```graphql +query { + qosReports( + from: "2023-01-01T00:00:00Z", + to: "2023-01-02T00:00:00Z" + ) { + event_time + gateway_id + response_time_ms + total_fees_usd + indexer_queries { + indexer + deployment + fee_grt + response_time_ms + } + } +} +``` + +## Development + +### Project Structure + +- `oracle/clickhouse/` - ClickHouse configuration and schema files +- `oracle/graphql-api/` - GraphQL API service +- `oracle/test-producer/` - Test data generator for development +- `oracle/docker-compose.yml` - Docker Compose configuration + +### Running in Development Mode + +``` +docker-compose -f oracle/docker-compose.yml --profile dev up +``` + +This will start all services including the test producer, which generates synthetic QoS data. + +### Monitoring + +- ClickHouse HTTP interface: http://localhost:8123 +- Redpanda Console: http://localhost:8080 +- GraphQL API: http://localhost:8000 + +## License + +[Add license information here] + +## Contributing + +[Add contribution guidelines here] + +## Acknowledgements + +This project is maintained by [GraphOps](https://graphops.xyz) and is part of The Graph Network ecosystem. \ No newline at end of file diff --git a/oracle/Cargo.toml b/oracle/Cargo.toml new file mode 100644 index 0000000..f0c9a9f --- /dev/null +++ b/oracle/Cargo.toml @@ -0,0 +1,32 @@ +# This file is a workspace manifest without a package definition +[workspace] +members = [ + "test-producer", + "graphql-api", + "clickhouse" +] + +[workspace.dependencies] +# Common dependencies across multiple crates +tokio = { version = "1.28", features = ["full"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +anyhow = "1.0" +thiserror = "1.0" +tracing = "0.1" +tracing-subscriber = "0.3" +prost = "0.11" +rdkafka = { version = "0.33", features = ["cmake-build"] } +clickhouse = { version = "0.11.2", features = ["uuid", "time"] } +async-graphql = { version = "5.0", features = ["time", "decimal"] } +actix-web = "4.3" +hex = "0.4" +rand = "0.8" + +# For development and tests +env_logger = "0.10" +dotenv = "0.15" + +# Explicitly telling Cargo this is a virtual manifest (workspace only) +[workspace.package] +edition = "2021" \ No newline at end of file diff --git a/oracle/README.md b/oracle/README.md deleted file mode 100644 index f80149b..0000000 --- a/oracle/README.md +++ /dev/null @@ -1 +0,0 @@ -# Qos Oracle V2 \ No newline at end of file diff --git a/oracle/clickhouse/allow_schema_path.xml b/oracle/clickhouse/allow_schema_path.xml new file mode 100644 index 0000000..daf5aa6 --- /dev/null +++ b/oracle/clickhouse/allow_schema_path.xml @@ -0,0 +1,11 @@ + + /var/lib/clickhouse/ + /var/lib/clickhouse/tmp/ + /var/lib/clickhouse/user_files/ + /etc/clickhouse-server/ + + + 0 + + + \ No newline at end of file diff --git a/oracle/clickhouse/settings.xml b/oracle/clickhouse/settings.xml new file mode 100644 index 0000000..4bb8cf2 --- /dev/null +++ b/oracle/clickhouse/settings.xml @@ -0,0 +1,5 @@ + + + 1 + + diff --git a/oracle/clickhouse/tables.sql b/oracle/clickhouse/tables.sql index d44f0af..e79409c 100644 --- a/oracle/clickhouse/tables.sql +++ b/oracle/clickhouse/tables.sql @@ -7,11 +7,11 @@ CREATE TABLE IF NOT EXISTS kafka_qos_data query_id String, api_key String, user_id String, - subgraph Nullable(String), + subgraph String, result String, response_time_ms UInt32, request_bytes UInt32, - response_bytes Nullable(UInt32), + response_bytes UInt32, total_fees_usd Float64, indexer_queries Nested( indexer String, @@ -32,12 +32,14 @@ SETTINGS kafka_topic_list = 'gateway_qos_topic', kafka_group_name = 'clickhouse_qos_consumer', kafka_format = 'Protobuf', - kafka_schema = '/etc/clickhouse-server/schema.proto:qos.ClientQueryProtobuf'; + kafka_schema = 'schema.proto:qos.ClientQueryProtobuf', + kafka_max_block_size = 1, + kafka_poll_timeout_ms = 500; -- Create the destination table for the raw data CREATE TABLE IF NOT EXISTS raw_qos_data ( - event_time DateTime, + event_time DateTime DEFAULT now(), gateway_id String, receipt_signer String, query_id String, @@ -68,31 +70,7 @@ TTL event_time + INTERVAL 7 DAY; -- Create a simplified materialized view CREATE MATERIALIZED VIEW IF NOT EXISTS mv_raw_qos_data TO raw_qos_data AS -SELECT - event_time, - gateway_id, - receipt_signer, - query_id, - api_key, - user_id, - subgraph, - result, - response_time_ms, - request_bytes, - response_bytes, - total_fees_usd, - indexer_queries.indexer, - indexer_queries.deployment, - indexer_queries.allocation, - indexer_queries.indexed_chain, - indexer_queries.url, - indexer_queries.fee_grt, - indexer_queries.response_time_ms, - indexer_queries.seconds_behind, - indexer_queries.result, - indexer_queries.indexer_errors, - indexer_queries.blocks_behind -FROM kafka_qos_data; +SELECT * FROM kafka_qos_data; -- Create the MaterializedView to process and store data from Kafka CREATE TABLE IF NOT EXISTS qos_data diff --git a/oracle/docker-compose.yml b/oracle/docker-compose.yml index 0317483..007c523 100644 --- a/oracle/docker-compose.yml +++ b/oracle/docker-compose.yml @@ -23,9 +23,11 @@ services: - "8123:8123" # HTTP: queries/monitoring - "9000:9000" # Native: client connections volumes: - - ./clickhouse:/docker-entrypoint-initdb.d:ro # init scripts: read-only - - ./clickhouse/schema.proto:/etc/clickhouse-server/schema.proto:ro # schema: read-only - - clickhouse_data:/var/lib/clickhouse:delegated # data: container writes + - ./clickhouse/schema.proto:/etc/clickhouse-server/schema.proto:ro + - ./clickhouse/tables.sql:/docker-entrypoint-initdb.d/tables.sql:ro + - ./clickhouse/allow_schema_path.xml:/etc/clickhouse-server/config.d/allow_schema_path.xml:ro + - clickhouse_data:/var/lib/clickhouse + - clickhouse_logs:/var/log/clickhouse-server ulimits: nofile: soft: 262144 @@ -65,6 +67,9 @@ services: <<: *default-healthcheck test: ["CMD-SHELL", "rpk cluster health | grep -q 'Healthy:.*true'"] start_period: 30s # Redpanda needs more time to initialize + environment: + REDPANDA_RPC_SERVER_LISTEN_ADDR: 0.0.0.0 + REDPANDA_KAFKA_ADDRESS: 0.0.0.0:9092 # Application services graphql-api: @@ -89,7 +94,7 @@ services: test-producer: build: context: ./test-producer - dockerfile: ./Dockerfile + dockerfile: Dockerfile profiles: [all, dev] depends_on: redpanda: @@ -97,7 +102,7 @@ services: clickhouse-server: condition: service_healthy volumes: - - ./clickhouse/schema.proto:/usr/src/app/schema.proto:ro # schema: read-only + - ./clickhouse/schema.proto:/app/../clickhouse/schema.proto:ro environment: KAFKA_BROKER: redpanda:9092 MESSAGES_PER_SECOND: 5 @@ -106,9 +111,29 @@ services: <<: *default-healthcheck test: ["CMD-SHELL", "ps aux | grep -v grep | grep -q qos_kafka_producer || exit 1"] + redpanda-console: + image: redpandadata/console:latest + container_name: redpanda-console + depends_on: + - redpanda + ports: + - "8080:8080" + environment: + KAFKA_BROKERS: redpanda:9092 + PROTOBUF_ENABLED: "true" + volumes: + - ./clickhouse/schema.proto:/etc/redpanda/protobuf/schema.proto:ro + healthcheck: + test: wget --no-verbose --tries=1 --spider http://localhost:8080 || exit 1 + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + # Persistent volumes volumes: clickhouse_data: # clickhouse persistent storage driver: local + clickhouse_logs: redpanda_data: # redpanda persistent storage driver: local diff --git a/oracle/test-producer/src/main.rs b/oracle/test-producer/src/main.rs index 09b3ed6..8fefd9f 100644 --- a/oracle/test-producer/src/main.rs +++ b/oracle/test-producer/src/main.rs @@ -1,20 +1,355 @@ use std::env; use std::time::Duration; -use rand::seq::SliceRandom; use rand::Rng; use rdkafka::config::ClientConfig; use rdkafka::producer::{FutureProducer, FutureRecord}; use rdkafka::util::get_rdkafka_version; use tokio::time; -use uuid::Uuid; use prost::Message; -// Use the exact schema from clickhouse/schema.proto -mod qos { - include!(concat!(env!("OUT_DIR"), "/qos.rs")); +// Mock types to match the gateway's dependencies +mod mock { + #[derive(Clone, Copy, PartialEq, Eq, Hash)] + pub struct DeploymentId(pub [u8; 32]); + + #[derive(Clone, Copy, PartialEq, Eq, Hash)] + pub struct IndexerId(pub [u8; 20]); + + #[derive(Clone)] + pub struct SubgraphId(pub String); + + #[derive(Clone, Copy, PartialEq, Eq, Hash)] + pub struct Address(pub [u8; 20]); + + impl std::ops::Deref for IndexerId { + type Target = Address; + fn deref(&self) -> &Self::Target { + unsafe { std::mem::transmute(self) } + } + } + + impl DeploymentId { + pub fn to_vec(&self) -> Vec { + self.0.to_vec() + } + } + + impl IndexerId { + pub fn to_vec(&self) -> Vec { + self.0.to_vec() + } + } + + impl SubgraphId { + pub fn to_string(&self) -> String { + self.0.clone() + } + } + + impl Address { + pub fn to_vec(&self) -> Vec { + self.0.to_vec() + } + } + + // Mock error types + pub enum Error { + QueryFailed(String), + } + + impl ToString for Error { + fn to_string(&self) -> String { + match self { + Error::QueryFailed(reason) => format!("Query failed: {}", reason), + } + } + } + + pub enum IndexerError { + NetworkError(String), + Timeout, + } + + impl ToString for IndexerError { + fn to_string(&self) -> String { + match self { + IndexerError::NetworkError(err) => format!("Network error: {}", err), + IndexerError::Timeout => "Timeout".to_string(), + } + } + } + + // Mock Receipt + pub struct Receipt { + allocation: Address, + value: u64, + } + + impl Receipt { + pub fn new(allocation: Address, value: u64) -> Self { + Self { allocation, value } + } + + pub fn allocation(&self) -> Address { + self.allocation + } + + pub fn value(&self) -> u64 { + self.value + } + } + + // Simplified IndexerResponse without attestation + pub struct IndexerResponse { + pub errors: Vec, + } +} + +use mock::*; + +// Protobuf definitions - exact copies from the gateway code +#[derive(prost::Message)] +pub struct ClientQueryProtobuf { + #[prost(string, tag = "1")] + pub gateway_id: String, + // 20 bytes + #[prost(bytes, tag = "2")] + pub receipt_signer: Vec, + #[prost(string, tag = "3")] + pub query_id: String, + #[prost(string, tag = "4")] + pub api_key: String, + #[prost(string, tag = "11")] + pub user_id: String, + #[prost(string, optional, tag = "12")] + pub subgraph: Option, + #[prost(string, tag = "5")] + pub result: String, + #[prost(uint32, tag = "6")] + pub response_time_ms: u32, + #[prost(uint32, tag = "7")] + pub request_bytes: u32, + #[prost(uint32, optional, tag = "8")] + pub response_bytes: Option, + #[prost(double, tag = "9")] + pub total_fees_usd: f64, + #[prost(message, repeated, tag = "10")] + pub indexer_queries: Vec, +} + +#[derive(prost::Message)] +pub struct IndexerQueryProtobuf { + /// 20 bytes + #[prost(bytes, tag = "1")] + pub indexer: Vec, + /// 32 bytes + #[prost(bytes, tag = "2")] + pub deployment: Vec, + /// 20 bytes + #[prost(bytes, tag = "3")] + pub allocation: Vec, + #[prost(string, tag = "4")] + pub indexed_chain: String, + #[prost(string, tag = "5")] + pub url: String, + #[prost(double, tag = "6")] + pub fee_grt: f64, + #[prost(uint32, tag = "7")] + pub response_time_ms: u32, + #[prost(uint32, tag = "8")] + pub seconds_behind: u32, + #[prost(string, tag = "9")] + pub result: String, + #[prost(string, tag = "10")] + pub indexer_errors: String, + #[prost(uint64, tag = "11")] + pub blocks_behind: u64, +} + +// Simplified structures to match the gateway's domain model without attestation +pub struct ClientRequest { + pub id: String, + pub response_time_ms: u16, + pub result: Result<(), Error>, + pub api_key: String, + pub user: String, + pub subgraph: Option, + pub grt_per_usd: f64, // Using f64 instead of NotNan for simplicity + pub indexer_requests: Vec, + pub request_bytes: u32, + pub response_bytes: Option, +} + +pub struct IndexerRequest { + pub indexer: IndexerId, + pub deployment: DeploymentId, + pub url: String, + pub receipt: Receipt, + pub subgraph_chain: String, + pub result: Result, + pub response_time_ms: u16, + pub seconds_behind: u32, + pub blocks_behind: u64, +} + +// Helper functions to generate random data +fn random_bytes(len: usize) -> Vec { + let mut rng = rand::thread_rng(); + let mut bytes = vec![0u8; len]; + rng.fill(&mut bytes[..]); + bytes +} + +fn random_address() -> Address { + let mut bytes = [0u8; 20]; + rand::thread_rng().fill(&mut bytes); + Address(bytes) +} + +fn random_deployment_id() -> DeploymentId { + let mut bytes = [0u8; 32]; + rand::thread_rng().fill(&mut bytes); + DeploymentId(bytes) +} + +fn random_indexer_id() -> IndexerId { + let mut bytes = [0u8; 20]; + rand::thread_rng().fill(&mut bytes); + IndexerId(bytes) +} + +fn generate_random_client_request() -> ClientRequest { + let mut rng = rand::thread_rng(); + + // Generate between 1 and 4 indexer requests + let num_indexers = rng.gen_range(1..5); + let mut indexer_requests = Vec::with_capacity(num_indexers); + + for _ in 0..num_indexers { + let success = rng.gen_bool(0.9); // 90% success rate + + let indexer_id = random_indexer_id(); + let allocation = random_address(); + + let result = if success { + Ok(IndexerResponse { + errors: vec![], + }) + } else { + let error_types = ["Timeout", "Bad Gateway", "Internal Server Error"]; + Err(IndexerError::NetworkError( + error_types[rng.gen_range(0..error_types.len())].to_string() + )) + }; + + indexer_requests.push(IndexerRequest { + indexer: indexer_id, + deployment: random_deployment_id(), + url: format!("https://api.thegraph.com/subgraphs/id/QmIndexer{}", rng.gen_range(1000..9999)), + receipt: Receipt::new(allocation, rng.gen_range(100_000_000_000_000..1_000_000_000_000_000)), + subgraph_chain: "mainnet".to_string(), + result, + response_time_ms: rng.gen_range(50..2000) as u16, + seconds_behind: rng.gen_range(0..100), + blocks_behind: rng.gen_range(0..50), + }); + } + + let success = rng.gen_bool(0.95); // 95% success rate for overall query + + ClientRequest { + id: format!("{:x}", rng.gen::()), + response_time_ms: rng.gen_range(50..5000) as u16, + result: if success { + Ok(()) + } else { + Err(Error::QueryFailed("Query validation error".to_string())) + }, + api_key: format!("api-{:x}", rng.gen::()), + user: format!("user-{:x}", rng.gen::()), + subgraph: if rng.gen_bool(0.9) { + Some(SubgraphId(format!("Qm{}", hex::encode(random_bytes(10))))) + } else { + None + }, + grt_per_usd: rng.gen_range(0.05..0.2), // Realistic GRT/USD price range + indexer_requests, + request_bytes: rng.gen_range(100..5000), + response_bytes: if rng.gen_bool(0.95) { + Some(rng.gen_range(200..10000)) + } else { + None + }, + } +} + +// Function that matches the logic in gateway's report function but without attestation +fn encode_client_request(client_request: ClientRequest, tap_signer: Address, graph_env: String) -> Vec { + let indexer_queries = client_request + .indexer_requests + .iter() + .map(|indexer_request| IndexerQueryProtobuf { + indexer: indexer_request.indexer.to_vec(), + deployment: indexer_request.deployment.to_vec(), + allocation: indexer_request.receipt.allocation().to_vec(), + indexed_chain: indexer_request.subgraph_chain.clone(), + url: indexer_request.url.clone(), + fee_grt: indexer_request.receipt.value() as f64 * 1e-18, + response_time_ms: indexer_request.response_time_ms as u32, + seconds_behind: indexer_request.seconds_behind, + result: indexer_request + .result + .as_ref() + .map(|_| "success".to_string()) + .unwrap_or_else(|err| err.to_string()), + indexer_errors: indexer_request + .result + .as_ref() + .map(|r| { + r.errors + .iter() + .map(|err| err.as_str()) + .collect::>() + .join("; ") + }) + .unwrap_or_default(), + blocks_behind: indexer_request.blocks_behind, + }) + .collect(); + + let total_fees_grt: f64 = client_request + .indexer_requests + .iter() + .map(|i| i.receipt.value() as f64 * 1e-18) + .sum(); + let total_fees_usd: f64 = total_fees_grt / client_request.grt_per_usd; + + let client_query_msg = ClientQueryProtobuf { + gateway_id: graph_env, + receipt_signer: tap_signer.to_vec(), + query_id: client_request.id, + api_key: client_request.api_key, + user_id: client_request.user, + subgraph: client_request.subgraph.map(|s| s.to_string()), + result: client_request + .result + .map(|()| "success".to_string()) + .unwrap_or_else(|err| err.to_string()), + response_time_ms: client_request.response_time_ms as u32, + request_bytes: client_request.request_bytes, + response_bytes: client_request.response_bytes, + total_fees_usd, + indexer_queries, + }; + + // Encode to Protobuf + let mut buf = Vec::new(); + client_query_msg.encode(&mut buf).unwrap(); + + // Return the raw protobuf message without length prefix + buf } -use qos::{ClientQueryProtobuf, IndexerQueryProtobuf}; #[tokio::main] async fn main() { @@ -39,7 +374,7 @@ async fn main() { println!("Waiting 15 seconds for Redpanda to initialize..."); time::sleep(Duration::from_secs(15)).await; - // Create a Kafka producer with extensive logging + // Create a Kafka producer with explicit PLAINTEXT protocol let producer: FutureProducer = ClientConfig::new() .set("bootstrap.servers", &broker) .set("message.timeout.ms", "30000") @@ -56,17 +391,26 @@ async fn main() { // Make sure this matches the topic name in ClickHouse Kafka engine let topic = "gateway_qos_topic"; let mut counter = 0; + + // Create a fixed tap_signer and graph_env to use for all messages + let tap_signer = random_address(); + let graph_env = "testnet-gateway".to_string(); loop { - let message = generate_message_exact_schema(); - let payload = message.encode_to_vec(); - + // Generate a random client request + let client_request = generate_random_client_request(); + + // Encode using the gateway's exact logic + let encoded_message = encode_client_request(client_request, tap_signer, graph_env.clone()); + + println!("Encoded message size: {} bytes", encoded_message.len()); + counter += 1; match producer .send( FutureRecord::to(topic) - .payload(&payload) + .payload(&encoded_message) .key(&format!("{}", counter)), Duration::from_secs(0), ) @@ -75,57 +419,7 @@ async fn main() { Ok((partition, offset)) => println!("Delivered: ({}, {})", partition, offset), Err((e, _)) => eprintln!("Delivery error: {:?}", e), } - - // Sleep for the configured delay to maintain the desired messages-per-second rate. + // Sleep for the configured delay time::sleep(delay).await; } } - -// Function to generate a message with the exact schema from clickhouse/schema.proto -fn generate_message_exact_schema() -> ClientQueryProtobuf { - let mut rng = rand::thread_rng(); - - // Create random indexer queries - let num_indexers = rng.gen_range(1..5); - let mut indexer_queries = Vec::with_capacity(num_indexers); - - for _ in 0..num_indexers { - let result = if rng.gen_bool(0.95) { "SUCCESS" } else { "ERROR" }; - - let indexer_query = IndexerQueryProtobuf { - indexer: Uuid::new_v4().to_string().into_bytes(), - deployment: Uuid::new_v4().to_string().into_bytes(), - allocation: Uuid::new_v4().to_string().into_bytes(), - indexed_chain: format!("chain-{}", rng.gen_range(1..10)), - url: format!("https://indexer-{}.example.com/graphql", rng.gen_range(1..100)), - fee_grt: rng.gen_range(0.0001..0.01), - response_time_ms: rng.gen_range(50..2000), - seconds_behind: rng.gen_range(0..100), - result: result.to_string(), - indexer_errors: if result == "ERROR" { - "Internal server error".to_string() - } else { - "".to_string() - }, - blocks_behind: rng.gen_range(0..50), - }; - - indexer_queries.push(indexer_query); - } - - // Create a ClientQueryProtobuf with EXACTLY the fields from schema.proto - ClientQueryProtobuf { - gateway_id: Uuid::new_v4().to_string(), - receipt_signer: Uuid::new_v4().to_string().into_bytes(), - query_id: Uuid::new_v4().to_string(), - api_key: format!("api-key-{}", rng.gen_range(1..1000)), - user_id: format!("user-{}", rng.gen_range(1..500)), - subgraph: format!("subgraph-{}", rng.gen_range(1..20)), - result: if rng.gen_bool(0.95) { "SUCCESS" } else { "ERROR" }, - response_time_ms: rng.gen_range(50..5000), - request_bytes: rng.gen_range(100..5000), - response_bytes: rng.gen_range(200..10000), - total_fees_usd: rng.gen_range(0.001..0.1), - indexer_queries, - } -} From f8661f4c701ada6160ccee7dcd8d012441c5ad55 Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Tue, 25 Mar 2025 00:47:45 -0300 Subject: [PATCH 04/20] feat: update Cargo files --- Cargo.lock | 93 +- oracle/Cargo.lock | 2857 ++++++++++++++++++++++++++++++ oracle/Cargo.toml | 3 +- oracle/test-producer/Cargo.toml | 5 +- oracle/test-producer/src/main.rs | 1 + 5 files changed, 2891 insertions(+), 68 deletions(-) create mode 100644 oracle/Cargo.lock diff --git a/Cargo.lock b/Cargo.lock index 2df0b1f..baa5c5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -845,9 +845,9 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fixedbitset" -version = "0.4.2" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flate2" @@ -1065,18 +1065,15 @@ dependencies = [ [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "home" -version = "0.5.11" +name = "hex" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" -dependencies = [ - "windows-sys 0.59.0", -] +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" @@ -1407,12 +1404,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.9.3" @@ -1530,9 +1521,9 @@ dependencies = [ [[package]] name = "multimap" -version = "0.8.3" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" [[package]] name = "native-tls" @@ -1688,9 +1679,9 @@ dependencies = [ [[package]] name = "petgraph" -version = "0.6.5" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", "indexmap 2.8.0", @@ -1731,12 +1722,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.1.25" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" dependencies = [ "proc-macro2", - "syn 1.0.109", + "syn 2.0.100", ] [[package]] @@ -1769,9 +1760,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.11.9" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", "prost-derive", @@ -1779,44 +1770,42 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.9" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "bytes", - "heck 0.4.1", + "heck 0.5.0", "itertools", - "lazy_static", "log", "multimap", + "once_cell", "petgraph", "prettyplease", "prost", "prost-types", "regex", - "syn 1.0.109", + "syn 2.0.100", "tempfile", - "which", ] [[package]] name = "prost-derive" -version = "0.11.9" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", "itertools", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.100", ] [[package]] name = "prost-types" -version = "0.11.9" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ "prost", ] @@ -1840,6 +1829,7 @@ name = "qos_kafka_producer" version = "0.1.0" dependencies = [ "chrono", + "hex", "prost", "prost-build", "rand 0.8.5", @@ -2002,19 +1992,6 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - [[package]] name = "rustix" version = "1.0.3" @@ -2024,7 +2001,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys 0.9.3", + "linux-raw-sys", "windows-sys 0.59.0", ] @@ -2262,7 +2239,7 @@ dependencies = [ "fastrand", "getrandom 0.3.2", "once_cell", - "rustix 1.0.3", + "rustix", "windows-sys 0.59.0", ] @@ -2627,18 +2604,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.44", -] - [[package]] name = "windows-core" version = "0.52.0" diff --git a/oracle/Cargo.lock b/oracle/Cargo.lock new file mode 100644 index 0000000..45fe973 --- /dev/null +++ b/oracle/Cargo.lock @@ -0,0 +1,2857 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + +[[package]] +name = "actix" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de7fa236829ba0841304542f7614c42b80fca007455315c45c785ccfa873a85b" +dependencies = [ + "actix-macros", + "actix-rt", + "actix_derive", + "bitflags", + "bytes", + "crossbeam-channel", + "futures-core", + "futures-sink", + "futures-task", + "futures-util", + "log", + "once_cell", + "parking_lot", + "pin-project-lite", + "smallvec", + "tokio", + "tokio-util", +] + +[[package]] +name = "actix-codec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-sink", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "actix-http" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa882656b67966045e4152c634051e70346939fced7117d5f0b52146a7c74c9" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", + "base64 0.22.1", + "bitflags", + "brotli", + "bytes", + "bytestring", + "derive_more", + "encoding_rs", + "flate2", + "foldhash", + "futures-core", + "h2", + "http", + "httparse", + "httpdate", + "itoa", + "language-tags", + "local-channel", + "mime", + "percent-encoding", + "pin-project-lite", + "rand 0.9.0", + "sha1", + "smallvec", + "tokio", + "tokio-util", + "tracing", + "zstd", +] + +[[package]] +name = "actix-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" +dependencies = [ + "quote", + "syn 2.0.100", +] + +[[package]] +name = "actix-router" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d324164c51f63867b57e73ba5936ea151b8a41a1d23d1031eeb9f70d0236f8" +dependencies = [ + "bytestring", + "cfg-if", + "http", + "regex", + "regex-lite", + "serde", + "tracing", +] + +[[package]] +name = "actix-rt" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eda4e2a6e042aa4e55ac438a2ae052d3b5da0ecf83d7411e1a368946925208" +dependencies = [ + "futures-core", + "tokio", +] + +[[package]] +name = "actix-server" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6398974fd4284f4768af07965701efbbb5fdc0616bff20cade1bb14b77675e24" +dependencies = [ + "actix-rt", + "actix-service", + "actix-utils", + "futures-core", + "futures-util", + "mio", + "socket2", + "tokio", + "tracing", +] + +[[package]] +name = "actix-service" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e46f36bf0e5af44bdc4bdb36fbbd421aa98c79a9bce724e1edeb3894e10dc7f" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "actix-utils" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a1dcdff1466e3c2488e1cb5c36a71822750ad43839937f85d2f4d9f8b705d8" +dependencies = [ + "local-waker", + "pin-project-lite", +] + +[[package]] +name = "actix-web" +version = "4.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2e3b15b3dc6c6ed996e4032389e9849d4ab002b1e92fbfe85b5f307d1479b4d" +dependencies = [ + "actix-codec", + "actix-http", + "actix-macros", + "actix-router", + "actix-rt", + "actix-server", + "actix-service", + "actix-utils", + "actix-web-codegen", + "bytes", + "bytestring", + "cfg-if", + "cookie", + "derive_more", + "encoding_rs", + "foldhash", + "futures-core", + "futures-util", + "impl-more", + "itoa", + "language-tags", + "log", + "mime", + "once_cell", + "pin-project-lite", + "regex", + "regex-lite", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "socket2", + "time", + "tracing", + "url", +] + +[[package]] +name = "actix-web-actors" +version = "4.3.1+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98c5300b38fd004fe7d2a964f9a90813fdbe8a81fed500587e78b1b71c6f980" +dependencies = [ + "actix", + "actix-codec", + "actix-http", + "actix-web", + "bytes", + "bytestring", + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "actix-web-codegen" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f591380e2e68490b5dfaf1dd1aa0ebe78d84ba7067078512b4ea6e4492d622b8" +dependencies = [ + "actix-router", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "actix_derive" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6ac1e58cded18cb28ddc17143c4dea5345b3ad575e14f32f66e4054a56eb271" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" + +[[package]] +name = "ascii_utils" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71938f30533e4d95a6d17aa530939da3842c2ab6f4f84b9dae68447e4129f74a" + +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-graphql" +version = "4.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9ed522678d412d77effe47b3c82314ac36952a35e6e852093dd48287c421f80" +dependencies = [ + "async-graphql-derive", + "async-graphql-parser", + "async-graphql-value", + "async-stream", + "async-trait", + "base64 0.13.1", + "bytes", + "fast_chemail", + "fnv", + "futures-util", + "http", + "indexmap 1.9.3", + "mime", + "multer", + "num-traits", + "once_cell", + "pin-project-lite", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "static_assertions", + "tempfile", + "thiserror 1.0.69", +] + +[[package]] +name = "async-graphql-actix-web" +version = "4.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c23479439e8a3819a937b88f6e8ec8e2185abebdc46a9b6d726e92518bbf858f" +dependencies = [ + "actix", + "actix-http", + "actix-web", + "actix-web-actors", + "async-channel", + "async-graphql", + "futures-channel", + "futures-util", + "serde_json", + "thiserror 1.0.69", +] + +[[package]] +name = "async-graphql-derive" +version = "4.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c121a894495d7d3fc3d4e15e0a9843e422e4d1d9e3c514d8062a1c94b35b005d" +dependencies = [ + "Inflector", + "async-graphql-parser", + "darling", + "proc-macro-crate 1.3.1", + "proc-macro2", + "quote", + "syn 1.0.109", + "thiserror 1.0.69", +] + +[[package]] +name = "async-graphql-parser" +version = "4.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b6c386f398145c6180206c1869c2279f5a3d45db5be4e0266148c6ac5c6ad68" +dependencies = [ + "async-graphql-value", + "pest", + "serde", + "serde_json", +] + +[[package]] +name = "async-graphql-value" +version = "4.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a941b499fead4a3fb5392cabf42446566d18c86313f69f2deab69560394d65f" +dependencies = [ + "bytes", + "indexmap 1.9.3", + "serde", + "serde_json", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "async-trait" +version = "0.1.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bstr" +version = "1.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" +dependencies = [ + "memchr", +] + +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +dependencies = [ + "serde", +] + +[[package]] +name = "bytestring" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e465647ae23b2823b0753f50decb2d5a86d2bb2cac04788fafd1f80e45378e5f" +dependencies = [ + "bytes", +] + +[[package]] +name = "cc" +version = "1.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "clickhouse" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0875e527e299fc5f4faba42870bf199a39ab0bb2dbba1b8aef0a2151451130f" +dependencies = [ + "bstr", + "bytes", + "clickhouse-derive", + "clickhouse-rs-cityhash-sys", + "futures", + "hyper", + "hyper-tls", + "lz4", + "sealed", + "serde", + "static_assertions", + "thiserror 1.0.69", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "clickhouse-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18af5425854858c507eec70f7deb4d5d8cec4216fcb086283a78872387281ea5" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 1.0.109", +] + +[[package]] +name = "clickhouse-rs-cityhash-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4baf9d4700a28d6cb600e17ed6ae2b43298a5245f1f76b4eab63027ebfd592b9" +dependencies = [ + "cc", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "cookie" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "deranged" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_more" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", + "unicode-xid", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "fast_chemail" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "495a39d30d624c2caabe6312bfead73e7717692b44e0b32df168c275a2e8e9e4" +dependencies = [ + "ascii_utils", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flate2" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap 2.8.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2fd658b06e56721792c5df4475705b6cda790e9298d19d2f8af083457bcd127" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "impl-more" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" +dependencies = [ + "equivalent", + "hashbrown 0.15.2", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "language-tags" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.171" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" + +[[package]] +name = "libz-sys" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" + +[[package]] +name = "litemap" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" + +[[package]] +name = "local-channel" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8" +dependencies = [ + "futures-core", + "futures-sink", + "local-waker", +] + +[[package]] +name = "local-waker" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.52.0", +] + +[[package]] +name = "multer" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "log", + "memchr", + "mime", + "spin", + "version_check", +] + +[[package]] +name = "multimap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" + +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_enum" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" +dependencies = [ + "proc-macro-crate 3.3.0", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc" + +[[package]] +name = "openssl" +version = "0.10.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pest" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" +dependencies = [ + "memchr", + "thiserror 2.0.12", + "ucd-trie", +] + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap 2.8.0", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" +dependencies = [ + "proc-macro2", + "syn 2.0.100", +] + +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit 0.19.15", +] + +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit 0.22.24", +] + +[[package]] +name = "proc-macro2" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck 0.5.0", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.100", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "qos_graphql_api" +version = "0.1.0" +dependencies = [ + "actix-web", + "async-graphql", + "async-graphql-actix-web", + "clickhouse", + "futures", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "qos_kafka_producer" +version = "0.1.0" +dependencies = [ + "chrono", + "hex", + "prost", + "prost-build", + "rand 0.8.5", + "rdkafka", + "tokio", + "uuid", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", + "zerocopy", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.15", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.2", +] + +[[package]] +name = "rdkafka" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1de127f294f2dba488ed46760b129d5ecbeabbd337ccbf3739cb29d50db2161c" +dependencies = [ + "futures", + "libc", + "log", + "rdkafka-sys", + "serde", + "serde_derive", + "serde_json", + "slab", + "tokio", +] + +[[package]] +name = "rdkafka-sys" +version = "4.8.0+2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced38182dc436b3d9df0c77976f37a67134df26b050df1f0006688e46fc4c8be" +dependencies = [ + "libc", + "libz-sys", + "num_enum", + "pkg-config", +] + +[[package]] +name = "redox_syscall" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustix" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustversion" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sealed" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b5e421024b5e5edfbaa8e60ecf90bda9dbffc602dbb230e6028763f85f0c68c" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" + +[[package]] +name = "socket2" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "tempfile" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" +dependencies = [ + "fastrand", + "getrandom 0.3.2", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "time" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" + +[[package]] +name = "time-macros" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.44.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap 2.8.0", + "toml_datetime", + "winnow 0.5.40", +] + +[[package]] +name = "toml_edit" +version = "0.22.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" +dependencies = [ + "indexmap 2.8.0", + "toml_datetime", + "winnow 0.7.4", +] + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "tracing-core" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +dependencies = [ + "getrandom 0.3.2", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn 2.0.100", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + +[[package]] +name = "winnow" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", + "synstructure", +] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.15+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/oracle/Cargo.toml b/oracle/Cargo.toml index f0c9a9f..5a35a05 100644 --- a/oracle/Cargo.toml +++ b/oracle/Cargo.toml @@ -2,8 +2,7 @@ [workspace] members = [ "test-producer", - "graphql-api", - "clickhouse" + "graphql-api" ] [workspace.dependencies] diff --git a/oracle/test-producer/Cargo.toml b/oracle/test-producer/Cargo.toml index 0e639f0..f9ef30d 100644 --- a/oracle/test-producer/Cargo.toml +++ b/oracle/test-producer/Cargo.toml @@ -6,10 +6,11 @@ edition = "2021" [dependencies] tokio = { version = "1", features = ["full"] } rdkafka = { version = "0.28", features = ["tokio"] } -prost = "0.11" +prost = "0.13.1" uuid = { version = "1", features = ["v4"] } rand = "0.8" chrono = "0.4" +hex = "0.4" [build-dependencies] -prost-build = "0.11" \ No newline at end of file +prost-build = "0.13.1" diff --git a/oracle/test-producer/src/main.rs b/oracle/test-producer/src/main.rs index 8fefd9f..3b7399f 100644 --- a/oracle/test-producer/src/main.rs +++ b/oracle/test-producer/src/main.rs @@ -7,6 +7,7 @@ use rdkafka::producer::{FutureProducer, FutureRecord}; use rdkafka::util::get_rdkafka_version; use tokio::time; use prost::Message; +use hex; // Mock types to match the gateway's dependencies mod mock { From 9caaf3a78c0d2720fdd5dcfa514c051729fc0216 Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Tue, 25 Mar 2025 00:48:12 -0300 Subject: [PATCH 05/20] feat: remove need for length delimiter by using ProtobufSingle --- oracle/clickhouse/tables.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oracle/clickhouse/tables.sql b/oracle/clickhouse/tables.sql index e79409c..1b2b8b2 100644 --- a/oracle/clickhouse/tables.sql +++ b/oracle/clickhouse/tables.sql @@ -31,7 +31,7 @@ SETTINGS kafka_broker_list = 'redpanda:9092', kafka_topic_list = 'gateway_qos_topic', kafka_group_name = 'clickhouse_qos_consumer', - kafka_format = 'Protobuf', + kafka_format = 'ProtobufSingle', kafka_schema = 'schema.proto:qos.ClientQueryProtobuf', kafka_max_block_size = 1, kafka_poll_timeout_ms = 500; From ecd55f7e9dd1e2de534fe1fd1486877ba1239855 Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Tue, 25 Mar 2025 02:04:49 -0300 Subject: [PATCH 06/20] fix: all sort of build errors, added debug statements to test producer --- Cargo.lock | 13 +- Cargo.toml | 5 +- oracle/Cargo.lock | 2857 ------------------------------ oracle/Cargo.toml | 31 - oracle/clickhouse/schema.proto | 8 +- oracle/docker-compose.yml | 6 +- oracle/graphql-api/Dockerfile | 31 +- oracle/test-producer/Cargo.lock | 1030 ----------- oracle/test-producer/Cargo.toml | 3 +- oracle/test-producer/Dockerfile | 72 +- oracle/test-producer/build.rs | 4 +- oracle/test-producer/src/main.rs | 11 +- 12 files changed, 97 insertions(+), 3974 deletions(-) delete mode 100644 oracle/Cargo.lock delete mode 100644 oracle/Cargo.toml delete mode 100644 oracle/test-producer/Cargo.lock diff --git a/Cargo.lock b/Cargo.lock index baa5c5f..45fe973 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1147,14 +1147,15 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.61" +version = "0.1.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +checksum = "b2fd658b06e56721792c5df4475705b6cda790e9298d19d2f8af083457bcd127" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log", "wasm-bindgen", "windows-core", ] @@ -1342,9 +1343,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.10.5" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" dependencies = [ "either", ] @@ -1445,9 +1446,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.26" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "lz4" diff --git a/Cargo.toml b/Cargo.toml index 3a81aeb..08da581 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "oracle/test-producer", - "oracle/graphql-api" + "oracle/graphql-api", ] -resolver = "2" \ No newline at end of file + +resolver = "2" diff --git a/oracle/Cargo.lock b/oracle/Cargo.lock deleted file mode 100644 index 45fe973..0000000 --- a/oracle/Cargo.lock +++ /dev/null @@ -1,2857 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "Inflector" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" -dependencies = [ - "lazy_static", - "regex", -] - -[[package]] -name = "actix" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de7fa236829ba0841304542f7614c42b80fca007455315c45c785ccfa873a85b" -dependencies = [ - "actix-macros", - "actix-rt", - "actix_derive", - "bitflags", - "bytes", - "crossbeam-channel", - "futures-core", - "futures-sink", - "futures-task", - "futures-util", - "log", - "once_cell", - "parking_lot", - "pin-project-lite", - "smallvec", - "tokio", - "tokio-util", -] - -[[package]] -name = "actix-codec" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" -dependencies = [ - "bitflags", - "bytes", - "futures-core", - "futures-sink", - "memchr", - "pin-project-lite", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "actix-http" -version = "3.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa882656b67966045e4152c634051e70346939fced7117d5f0b52146a7c74c9" -dependencies = [ - "actix-codec", - "actix-rt", - "actix-service", - "actix-utils", - "base64 0.22.1", - "bitflags", - "brotli", - "bytes", - "bytestring", - "derive_more", - "encoding_rs", - "flate2", - "foldhash", - "futures-core", - "h2", - "http", - "httparse", - "httpdate", - "itoa", - "language-tags", - "local-channel", - "mime", - "percent-encoding", - "pin-project-lite", - "rand 0.9.0", - "sha1", - "smallvec", - "tokio", - "tokio-util", - "tracing", - "zstd", -] - -[[package]] -name = "actix-macros" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" -dependencies = [ - "quote", - "syn 2.0.100", -] - -[[package]] -name = "actix-router" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13d324164c51f63867b57e73ba5936ea151b8a41a1d23d1031eeb9f70d0236f8" -dependencies = [ - "bytestring", - "cfg-if", - "http", - "regex", - "regex-lite", - "serde", - "tracing", -] - -[[package]] -name = "actix-rt" -version = "2.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eda4e2a6e042aa4e55ac438a2ae052d3b5da0ecf83d7411e1a368946925208" -dependencies = [ - "futures-core", - "tokio", -] - -[[package]] -name = "actix-server" -version = "2.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6398974fd4284f4768af07965701efbbb5fdc0616bff20cade1bb14b77675e24" -dependencies = [ - "actix-rt", - "actix-service", - "actix-utils", - "futures-core", - "futures-util", - "mio", - "socket2", - "tokio", - "tracing", -] - -[[package]] -name = "actix-service" -version = "2.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e46f36bf0e5af44bdc4bdb36fbbd421aa98c79a9bce724e1edeb3894e10dc7f" -dependencies = [ - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "actix-utils" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88a1dcdff1466e3c2488e1cb5c36a71822750ad43839937f85d2f4d9f8b705d8" -dependencies = [ - "local-waker", - "pin-project-lite", -] - -[[package]] -name = "actix-web" -version = "4.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2e3b15b3dc6c6ed996e4032389e9849d4ab002b1e92fbfe85b5f307d1479b4d" -dependencies = [ - "actix-codec", - "actix-http", - "actix-macros", - "actix-router", - "actix-rt", - "actix-server", - "actix-service", - "actix-utils", - "actix-web-codegen", - "bytes", - "bytestring", - "cfg-if", - "cookie", - "derive_more", - "encoding_rs", - "foldhash", - "futures-core", - "futures-util", - "impl-more", - "itoa", - "language-tags", - "log", - "mime", - "once_cell", - "pin-project-lite", - "regex", - "regex-lite", - "serde", - "serde_json", - "serde_urlencoded", - "smallvec", - "socket2", - "time", - "tracing", - "url", -] - -[[package]] -name = "actix-web-actors" -version = "4.3.1+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98c5300b38fd004fe7d2a964f9a90813fdbe8a81fed500587e78b1b71c6f980" -dependencies = [ - "actix", - "actix-codec", - "actix-http", - "actix-web", - "bytes", - "bytestring", - "futures-core", - "pin-project-lite", - "tokio", - "tokio-util", -] - -[[package]] -name = "actix-web-codegen" -version = "4.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f591380e2e68490b5dfaf1dd1aa0ebe78d84ba7067078512b4ea6e4492d622b8" -dependencies = [ - "actix-router", - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "actix_derive" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6ac1e58cded18cb28ddc17143c4dea5345b3ad575e14f32f66e4054a56eb271" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" -dependencies = [ - "alloc-no-stdlib", -] - -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "anyhow" -version = "1.0.97" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" - -[[package]] -name = "ascii_utils" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71938f30533e4d95a6d17aa530939da3842c2ab6f4f84b9dae68447e4129f74a" - -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" -dependencies = [ - "concurrent-queue", - "event-listener", - "futures-core", -] - -[[package]] -name = "async-graphql" -version = "4.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9ed522678d412d77effe47b3c82314ac36952a35e6e852093dd48287c421f80" -dependencies = [ - "async-graphql-derive", - "async-graphql-parser", - "async-graphql-value", - "async-stream", - "async-trait", - "base64 0.13.1", - "bytes", - "fast_chemail", - "fnv", - "futures-util", - "http", - "indexmap 1.9.3", - "mime", - "multer", - "num-traits", - "once_cell", - "pin-project-lite", - "regex", - "serde", - "serde_json", - "serde_urlencoded", - "static_assertions", - "tempfile", - "thiserror 1.0.69", -] - -[[package]] -name = "async-graphql-actix-web" -version = "4.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c23479439e8a3819a937b88f6e8ec8e2185abebdc46a9b6d726e92518bbf858f" -dependencies = [ - "actix", - "actix-http", - "actix-web", - "actix-web-actors", - "async-channel", - "async-graphql", - "futures-channel", - "futures-util", - "serde_json", - "thiserror 1.0.69", -] - -[[package]] -name = "async-graphql-derive" -version = "4.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c121a894495d7d3fc3d4e15e0a9843e422e4d1d9e3c514d8062a1c94b35b005d" -dependencies = [ - "Inflector", - "async-graphql-parser", - "darling", - "proc-macro-crate 1.3.1", - "proc-macro2", - "quote", - "syn 1.0.109", - "thiserror 1.0.69", -] - -[[package]] -name = "async-graphql-parser" -version = "4.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b6c386f398145c6180206c1869c2279f5a3d45db5be4e0266148c6ac5c6ad68" -dependencies = [ - "async-graphql-value", - "pest", - "serde", - "serde_json", -] - -[[package]] -name = "async-graphql-value" -version = "4.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a941b499fead4a3fb5392cabf42446566d18c86313f69f2deab69560394d65f" -dependencies = [ - "bytes", - "indexmap 1.9.3", - "serde", - "serde_json", -] - -[[package]] -name = "async-stream" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "async-trait" -version = "0.1.88" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "autocfg" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" - -[[package]] -name = "backtrace" -version = "0.3.74" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets", -] - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "bitflags" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "brotli" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", -] - -[[package]] -name = "brotli-decompressor" -version = "4.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - -[[package]] -name = "bstr" -version = "1.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" -dependencies = [ - "memchr", -] - -[[package]] -name = "bumpalo" -version = "3.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" - -[[package]] -name = "bytes" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" -dependencies = [ - "serde", -] - -[[package]] -name = "bytestring" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e465647ae23b2823b0753f50decb2d5a86d2bb2cac04788fafd1f80e45378e5f" -dependencies = [ - "bytes", -] - -[[package]] -name = "cc" -version = "1.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" -dependencies = [ - "jobserver", - "libc", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "chrono" -version = "0.4.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "js-sys", - "num-traits", - "wasm-bindgen", - "windows-link", -] - -[[package]] -name = "clickhouse" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0875e527e299fc5f4faba42870bf199a39ab0bb2dbba1b8aef0a2151451130f" -dependencies = [ - "bstr", - "bytes", - "clickhouse-derive", - "clickhouse-rs-cityhash-sys", - "futures", - "hyper", - "hyper-tls", - "lz4", - "sealed", - "serde", - "static_assertions", - "thiserror 1.0.69", - "tokio", - "url", - "uuid", -] - -[[package]] -name = "clickhouse-derive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18af5425854858c507eec70f7deb4d5d8cec4216fcb086283a78872387281ea5" -dependencies = [ - "proc-macro2", - "quote", - "serde_derive_internals", - "syn 1.0.109", -] - -[[package]] -name = "clickhouse-rs-cityhash-sys" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4baf9d4700a28d6cb600e17ed6ae2b43298a5245f1f76b4eab63027ebfd592b9" -dependencies = [ - "cc", -] - -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "cookie" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" -dependencies = [ - "percent-encoding", - "time", - "version_check", -] - -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "crc32fast" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "darling" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 1.0.109", -] - -[[package]] -name = "darling_macro" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" -dependencies = [ - "darling_core", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "deranged" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058" -dependencies = [ - "powerfmt", -] - -[[package]] -name = "derive_more" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" -dependencies = [ - "derive_more-impl", -] - -[[package]] -name = "derive_more-impl" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", - "unicode-xid", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - -[[package]] -name = "displaydoc" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" -dependencies = [ - "libc", - "windows-sys 0.59.0", -] - -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - -[[package]] -name = "fast_chemail" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "495a39d30d624c2caabe6312bfead73e7717692b44e0b32df168c275a2e8e9e4" -dependencies = [ - "ascii_utils", -] - -[[package]] -name = "fastrand" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" - -[[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - -[[package]] -name = "flate2" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - -[[package]] -name = "form_urlencoded" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "futures" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" - -[[package]] -name = "futures-executor" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" - -[[package]] -name = "futures-macro" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "futures-sink" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" - -[[package]] -name = "futures-task" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" - -[[package]] -name = "futures-util" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.11.0+wasi-snapshot-preview1", -] - -[[package]] -name = "getrandom" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasi 0.14.2+wasi-0.2.4", -] - -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - -[[package]] -name = "h2" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap 2.8.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hashbrown" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" - -[[package]] -name = "heck" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "0.14.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.62" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2fd658b06e56721792c5df4475705b6cda790e9298d19d2f8af083457bcd127" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "icu_collections" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locid" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" - -[[package]] -name = "icu_normalizer" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "utf16_iter", - "utf8_iter", - "write16", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" - -[[package]] -name = "icu_properties" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locid_transform", - "icu_properties_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" - -[[package]] -name = "icu_provider" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_provider_macros", - "stable_deref_trait", - "tinystr", - "writeable", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "idna" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "impl-more" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2" - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", - "serde", -] - -[[package]] -name = "indexmap" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" -dependencies = [ - "equivalent", - "hashbrown 0.15.2", -] - -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" - -[[package]] -name = "jobserver" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" -dependencies = [ - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - -[[package]] -name = "language-tags" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "libc" -version = "0.2.171" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" - -[[package]] -name = "libz-sys" -version = "1.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "linux-raw-sys" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" - -[[package]] -name = "litemap" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" - -[[package]] -name = "local-channel" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8" -dependencies = [ - "futures-core", - "futures-sink", - "local-waker", -] - -[[package]] -name = "local-waker" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487" - -[[package]] -name = "lock_api" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" - -[[package]] -name = "lz4" -version = "1.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" -dependencies = [ - "lz4-sys", -] - -[[package]] -name = "lz4-sys" -version = "1.11.1+lz4-1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "memchr" -version = "2.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "miniz_oxide" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" -dependencies = [ - "adler2", -] - -[[package]] -name = "mio" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" -dependencies = [ - "libc", - "log", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.52.0", -] - -[[package]] -name = "multer" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2" -dependencies = [ - "bytes", - "encoding_rs", - "futures-util", - "http", - "httparse", - "log", - "memchr", - "mime", - "spin", - "version_check", -] - -[[package]] -name = "multimap" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" - -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - -[[package]] -name = "num-conv" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "num_enum" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" -dependencies = [ - "num_enum_derive", -] - -[[package]] -name = "num_enum_derive" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" -dependencies = [ - "proc-macro-crate 3.3.0", - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "object" -version = "0.36.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" -dependencies = [ - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc" - -[[package]] -name = "openssl" -version = "0.10.71" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - -[[package]] -name = "openssl-sys" -version = "0.9.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "parking_lot" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets", -] - -[[package]] -name = "percent-encoding" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" - -[[package]] -name = "pest" -version = "2.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" -dependencies = [ - "memchr", - "thiserror 2.0.12", - "ucd-trie", -] - -[[package]] -name = "petgraph" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" -dependencies = [ - "fixedbitset", - "indexmap 2.8.0", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "prettyplease" -version = "0.2.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" -dependencies = [ - "proc-macro2", - "syn 2.0.100", -] - -[[package]] -name = "proc-macro-crate" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" -dependencies = [ - "once_cell", - "toml_edit 0.19.15", -] - -[[package]] -name = "proc-macro-crate" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" -dependencies = [ - "toml_edit 0.22.24", -] - -[[package]] -name = "proc-macro2" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "prost" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" -dependencies = [ - "heck 0.5.0", - "itertools", - "log", - "multimap", - "once_cell", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn 2.0.100", - "tempfile", -] - -[[package]] -name = "prost-derive" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "prost-types" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" -dependencies = [ - "prost", -] - -[[package]] -name = "qos_graphql_api" -version = "0.1.0" -dependencies = [ - "actix-web", - "async-graphql", - "async-graphql-actix-web", - "clickhouse", - "futures", - "serde", - "serde_json", - "tokio", -] - -[[package]] -name = "qos_kafka_producer" -version = "0.1.0" -dependencies = [ - "chrono", - "hex", - "prost", - "prost-build", - "rand 0.8.5", - "rdkafka", - "tokio", - "uuid", -] - -[[package]] -name = "quote" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.3", - "zerocopy", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.15", -] - -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom 0.3.2", -] - -[[package]] -name = "rdkafka" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de127f294f2dba488ed46760b129d5ecbeabbd337ccbf3739cb29d50db2161c" -dependencies = [ - "futures", - "libc", - "log", - "rdkafka-sys", - "serde", - "serde_derive", - "serde_json", - "slab", - "tokio", -] - -[[package]] -name = "rdkafka-sys" -version = "4.8.0+2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced38182dc436b3d9df0c77976f37a67134df26b050df1f0006688e46fc4c8be" -dependencies = [ - "libc", - "libz-sys", - "num_enum", - "pkg-config", -] - -[[package]] -name = "redox_syscall" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" -dependencies = [ - "bitflags", -] - -[[package]] -name = "regex" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-lite" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" - -[[package]] -name = "regex-syntax" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" - -[[package]] -name = "rustc-demangle" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" - -[[package]] -name = "rustix" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.59.0", -] - -[[package]] -name = "rustversion" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" - -[[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - -[[package]] -name = "schannel" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "sealed" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b5e421024b5e5edfbaa8e60ecf90bda9dbffc602dbb230e6028763f85f0c68c" -dependencies = [ - "heck 0.3.3", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "serde" -version = "1.0.219" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.219" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "serde_derive_internals" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "serde_json" -version = "1.0.140" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signal-hook-registry" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" -dependencies = [ - "libc", -] - -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - -[[package]] -name = "smallvec" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" - -[[package]] -name = "socket2" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "synstructure" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "tempfile" -version = "3.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" -dependencies = [ - "fastrand", - "getrandom 0.3.2", - "once_cell", - "rustix", - "windows-sys 0.59.0", -] - -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - -[[package]] -name = "thiserror" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" -dependencies = [ - "thiserror-impl 2.0.12", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "time" -version = "0.3.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" -dependencies = [ - "deranged", - "itoa", - "num-conv", - "powerfmt", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" - -[[package]] -name = "time-macros" -version = "0.2.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" -dependencies = [ - "num-conv", - "time-core", -] - -[[package]] -name = "tinystr" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tokio" -version = "1.44.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" -dependencies = [ - "backtrace", - "bytes", - "libc", - "mio", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys 0.52.0", -] - -[[package]] -name = "tokio-macros" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "toml_datetime" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" - -[[package]] -name = "toml_edit" -version = "0.19.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" -dependencies = [ - "indexmap 2.8.0", - "toml_datetime", - "winnow 0.5.40", -] - -[[package]] -name = "toml_edit" -version = "0.22.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" -dependencies = [ - "indexmap 2.8.0", - "toml_datetime", - "winnow 0.7.4", -] - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" -dependencies = [ - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "tracing-core" -version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" -dependencies = [ - "once_cell", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "typenum" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" - -[[package]] -name = "ucd-trie" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" - -[[package]] -name = "unicode-ident" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" - -[[package]] -name = "unicode-segmentation" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" - -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - -[[package]] -name = "url" -version = "2.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "uuid" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" -dependencies = [ - "getrandom 0.3.2", -] - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" -dependencies = [ - "wit-bindgen-rt", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.100", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "windows-core" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-link" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "winnow" -version = "0.5.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" -dependencies = [ - "memchr", -] - -[[package]] -name = "winnow" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" -dependencies = [ - "memchr", -] - -[[package]] -name = "wit-bindgen-rt" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags", -] - -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - -[[package]] -name = "writeable" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" - -[[package]] -name = "yoke" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", - "synstructure", -] - -[[package]] -name = "zerocopy" -version = "0.8.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "zerofrom" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", - "synstructure", -] - -[[package]] -name = "zerovec" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.15+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/oracle/Cargo.toml b/oracle/Cargo.toml deleted file mode 100644 index 5a35a05..0000000 --- a/oracle/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -# This file is a workspace manifest without a package definition -[workspace] -members = [ - "test-producer", - "graphql-api" -] - -[workspace.dependencies] -# Common dependencies across multiple crates -tokio = { version = "1.28", features = ["full"] } -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -anyhow = "1.0" -thiserror = "1.0" -tracing = "0.1" -tracing-subscriber = "0.3" -prost = "0.11" -rdkafka = { version = "0.33", features = ["cmake-build"] } -clickhouse = { version = "0.11.2", features = ["uuid", "time"] } -async-graphql = { version = "5.0", features = ["time", "decimal"] } -actix-web = "4.3" -hex = "0.4" -rand = "0.8" - -# For development and tests -env_logger = "0.10" -dotenv = "0.15" - -# Explicitly telling Cargo this is a virtual manifest (workspace only) -[workspace.package] -edition = "2021" \ No newline at end of file diff --git a/oracle/clickhouse/schema.proto b/oracle/clickhouse/schema.proto index 1cdea70..d3451cc 100644 --- a/oracle/clickhouse/schema.proto +++ b/oracle/clickhouse/schema.proto @@ -7,14 +7,14 @@ message ClientQueryProtobuf { bytes receipt_signer = 2; string query_id = 3; string api_key = 4; - string user_id = 11; - string subgraph = 12; string result = 5; uint32 response_time_ms = 6; uint32 request_bytes = 7; - uint32 response_bytes = 8; + optional uint32 response_bytes = 8; double total_fees_usd = 9; repeated IndexerQueryProtobuf indexer_queries = 10; + string user_id = 11; + optional string subgraph = 12; } message IndexerQueryProtobuf { @@ -29,4 +29,4 @@ message IndexerQueryProtobuf { string result = 9; string indexer_errors = 10; uint64 blocks_behind = 11; -} \ No newline at end of file +} diff --git a/oracle/docker-compose.yml b/oracle/docker-compose.yml index 007c523..500ccdf 100644 --- a/oracle/docker-compose.yml +++ b/oracle/docker-compose.yml @@ -93,8 +93,8 @@ services: # Test/development tools test-producer: build: - context: ./test-producer - dockerfile: Dockerfile + context: . + dockerfile: ./test-producer/Dockerfile profiles: [all, dev] depends_on: redpanda: @@ -102,7 +102,7 @@ services: clickhouse-server: condition: service_healthy volumes: - - ./clickhouse/schema.proto:/app/../clickhouse/schema.proto:ro + - ./clickhouse/schema.proto:/app/clickhouse/schema.proto:ro environment: KAFKA_BROKER: redpanda:9092 MESSAGES_PER_SECOND: 5 diff --git a/oracle/graphql-api/Dockerfile b/oracle/graphql-api/Dockerfile index c6f0037..c7a7755 100644 --- a/oracle/graphql-api/Dockerfile +++ b/oracle/graphql-api/Dockerfile @@ -1,7 +1,7 @@ # ----------------------------------------------------------------------------- # 1. Global Build Arguments & Metadata # ----------------------------------------------------------------------------- -ARG RUST_VERSION=1.74 +ARG RUST_VERSION=1.84 ARG DEBIAN_VERSION=bookworm ARG CHEF_IMAGE=lukemathwalker/cargo-chef:latest-rust-${RUST_VERSION}-slim-${DEBIAN_VERSION} ARG RUNTIME_BASE=gcr.io/distroless/cc-debian12:nonroot @@ -29,26 +29,41 @@ FROM ${CHEF_IMAGE} AS builder ARG CARGO_BUILD_JOBS="default" ARG CARGO_PROFILE=release ENV RUSTFLAGS="-C opt-level=3 -C codegen-units=1" +# Explicitly set C++ compiler path +ENV CXX=/usr/bin/g++ +ENV CC=/usr/bin/gcc -# Install build dependencies +# Install build dependencies with proper C++ compiler setup RUN apt-get update && apt-get install -y --no-install-recommends \ pkg-config \ libssl-dev \ - ca-certificates \ + protobuf-compiler \ + g++ \ + build-essential \ + cmake \ + make \ + python3 \ + libsasl2-dev \ + zlib1g-dev \ binutils \ - && rm -rf /var/lib/apt/lists/* + ca-certificates \ + && rm -rf /var/lib/apt/lists/* \ + && ln -sf /usr/bin/g++ /usr/bin/c++ \ + && ls -la /usr/bin/g++ /usr/bin/c++ \ + && g++ --version WORKDIR /app # Cook dependencies separately COPY --from=chef /app/recipe.json recipe.json -RUN cargo chef cook --profile ${CARGO_PROFILE} --recipe-path recipe.json +RUN CXX=/usr/bin/g++ CC=/usr/bin/gcc cargo chef cook --profile ${CARGO_PROFILE} --recipe-path recipe.json # Build the application COPY . . -RUN cargo build --profile ${CARGO_PROFILE} --locked --jobs "${CARGO_BUILD_JOBS}" \ - && strip target/${CARGO_PROFILE}/graphql_api \ - && cp target/${CARGO_PROFILE}/graphql_api /app/graphql_api +RUN cargo build --profile ${CARGO_PROFILE} --jobs "${CARGO_BUILD_JOBS}" \ + && ls -la target/${CARGO_PROFILE}/ \ + && strip target/${CARGO_PROFILE}/qos_graphql_api \ + && cp target/${CARGO_PROFILE}/qos_graphql_api /app/graphql_api # ----------------------------------------------------------------------------- # 4. Preparation Stage (Tini + Directory Setup) diff --git a/oracle/test-producer/Cargo.lock b/oracle/test-producer/Cargo.lock deleted file mode 100644 index 25a9e92..0000000 --- a/oracle/test-producer/Cargo.lock +++ /dev/null @@ -1,1030 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "anyhow" -version = "1.0.97" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" - -[[package]] -name = "autocfg" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" - -[[package]] -name = "backtrace" -version = "0.3.74" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets", -] - -[[package]] -name = "bitflags" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" - -[[package]] -name = "bytes" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" - -[[package]] -name = "cc" -version = "1.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" -dependencies = [ - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" -dependencies = [ - "libc", - "windows-sys 0.59.0", -] - -[[package]] -name = "fastrand" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" - -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - -[[package]] -name = "futures" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" - -[[package]] -name = "futures-executor" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" - -[[package]] -name = "futures-macro" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "futures-sink" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" - -[[package]] -name = "futures-task" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" - -[[package]] -name = "futures-util" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "getrandom" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.11.0+wasi-snapshot-preview1", -] - -[[package]] -name = "getrandom" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasi 0.14.2+wasi-0.2.4", -] - -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - -[[package]] -name = "hashbrown" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - -[[package]] -name = "home" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "indexmap" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" -dependencies = [ - "equivalent", - "hashbrown", -] - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "libc" -version = "0.2.171" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" - -[[package]] -name = "libz-sys" -version = "1.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - -[[package]] -name = "linux-raw-sys" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" - -[[package]] -name = "lock_api" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" - -[[package]] -name = "memchr" -version = "2.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" - -[[package]] -name = "miniz_oxide" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" -dependencies = [ - "adler2", -] - -[[package]] -name = "mio" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" -dependencies = [ - "libc", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.52.0", -] - -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - -[[package]] -name = "num_enum" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" -dependencies = [ - "num_enum_derive", -] - -[[package]] -name = "num_enum_derive" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "object" -version = "0.36.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" -dependencies = [ - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc" - -[[package]] -name = "parking_lot" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets", -] - -[[package]] -name = "petgraph" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" -dependencies = [ - "fixedbitset", - "indexmap", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2", - "syn 1.0.109", -] - -[[package]] -name = "proc-macro-crate" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" -dependencies = [ - "toml_edit", -] - -[[package]] -name = "proc-macro2" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "prost" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" -dependencies = [ - "bytes", - "heck", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn 1.0.109", - "tempfile", - "which", -] - -[[package]] -name = "prost-derive" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "prost-types" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" -dependencies = [ - "prost", -] - -[[package]] -name = "qos_kafka_producer" -version = "0.1.0" -dependencies = [ - "prost", - "prost-build", - "rand", - "rdkafka", - "tokio", - "uuid", -] - -[[package]] -name = "quote" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.15", -] - -[[package]] -name = "rdkafka" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de127f294f2dba488ed46760b129d5ecbeabbd337ccbf3739cb29d50db2161c" -dependencies = [ - "futures", - "libc", - "log", - "rdkafka-sys", - "serde", - "serde_derive", - "serde_json", - "slab", - "tokio", -] - -[[package]] -name = "rdkafka-sys" -version = "4.8.0+2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced38182dc436b3d9df0c77976f37a67134df26b050df1f0006688e46fc4c8be" -dependencies = [ - "libc", - "libz-sys", - "num_enum", - "pkg-config", -] - -[[package]] -name = "redox_syscall" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" -dependencies = [ - "bitflags", -] - -[[package]] -name = "regex" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" - -[[package]] -name = "rustc-demangle" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" - -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - -[[package]] -name = "rustix" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.9.3", - "windows-sys 0.59.0", -] - -[[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "serde" -version = "1.0.219" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.219" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "serde_json" -version = "1.0.140" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signal-hook-registry" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" -dependencies = [ - "libc", -] - -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - -[[package]] -name = "smallvec" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" - -[[package]] -name = "socket2" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "tempfile" -version = "3.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" -dependencies = [ - "fastrand", - "getrandom 0.3.2", - "once_cell", - "rustix 1.0.3", - "windows-sys 0.59.0", -] - -[[package]] -name = "tokio" -version = "1.44.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" -dependencies = [ - "backtrace", - "bytes", - "libc", - "mio", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys 0.52.0", -] - -[[package]] -name = "tokio-macros" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "toml_datetime" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" - -[[package]] -name = "toml_edit" -version = "0.22.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" -dependencies = [ - "indexmap", - "toml_datetime", - "winnow", -] - -[[package]] -name = "unicode-ident" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" - -[[package]] -name = "uuid" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" -dependencies = [ - "getrandom 0.3.2", -] - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" -dependencies = [ - "wit-bindgen-rt", -] - -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.44", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "winnow" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" -dependencies = [ - "memchr", -] - -[[package]] -name = "wit-bindgen-rt" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags", -] - -[[package]] -name = "zerocopy" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd97444d05a4328b90e75e503a34bad781f14e28a823ad3557f0750df1ebcbc6" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6352c01d0edd5db859a63e2605f4ea3183ddbd15e2c4a9e7d32184df75e4f154" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] diff --git a/oracle/test-producer/Cargo.toml b/oracle/test-producer/Cargo.toml index f9ef30d..2af38da 100644 --- a/oracle/test-producer/Cargo.toml +++ b/oracle/test-producer/Cargo.toml @@ -2,6 +2,7 @@ name = "qos_kafka_producer" version = "0.1.0" edition = "2021" +resolver = "2" [dependencies] tokio = { version = "1", features = ["full"] } @@ -13,4 +14,4 @@ chrono = "0.4" hex = "0.4" [build-dependencies] -prost-build = "0.13.1" +prost-build = "0.13.1" \ No newline at end of file diff --git a/oracle/test-producer/Dockerfile b/oracle/test-producer/Dockerfile index 069dac5..0997030 100644 --- a/oracle/test-producer/Dockerfile +++ b/oracle/test-producer/Dockerfile @@ -1,10 +1,10 @@ # ----------------------------------------------------------------------------- # 1. Global Build Arguments & Metadata # ----------------------------------------------------------------------------- -ARG RUST_VERSION=1.74 +ARG RUST_VERSION=1.84 ARG DEBIAN_VERSION=bookworm ARG CHEF_IMAGE=lukemathwalker/cargo-chef:latest-rust-${RUST_VERSION}-slim-${DEBIAN_VERSION} -ARG RUNTIME_BASE=gcr.io/distroless/cc-debian12:nonroot +ARG RUNTIME_BASE=debian:${DEBIAN_VERSION}-slim ARG TINI_VERSION=v0.19.0 ARG TINI_SHA256=c5b0666b4cb676901f90dfcb37106783c5fe2077b04590973b885950611b30ee @@ -18,7 +18,10 @@ ARG BUILD_DATE="2023-11-02" # ----------------------------------------------------------------------------- FROM ${CHEF_IMAGE} AS chef WORKDIR /app -COPY . . +# Copy the test-producer code, not the entire context +COPY ./test-producer . +# Generate the Cargo.lock file first +RUN cargo update RUN cargo chef prepare --recipe-path recipe.json # ----------------------------------------------------------------------------- @@ -29,6 +32,8 @@ FROM ${CHEF_IMAGE} AS builder ARG CARGO_BUILD_JOBS="default" ARG CARGO_PROFILE=release ENV RUSTFLAGS="-C opt-level=3 -C codegen-units=1" +ENV CXX=/usr/bin/g++ +ENV CC=/usr/bin/gcc # Install build dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -36,29 +41,43 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libssl-dev \ protobuf-compiler \ g++ \ + build-essential \ + cmake \ make \ python3 \ libsasl2-dev \ zlib1g-dev \ binutils \ ca-certificates \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && ln -sf /usr/bin/g++ /usr/bin/c++ WORKDIR /app -# Create schema directories -RUN mkdir -p ../clickhouse +# Create the proper directory structure +RUN mkdir -p /app/clickhouse + +# Copy the actual schema.proto file from the project root +COPY ./clickhouse/schema.proto /app/clickhouse/schema.proto + +# Modify build.rs to use the local path +COPY ./test-producer/build.rs /app/build.rs.original +RUN sed 's|"../clickhouse/schema.proto"|"./clickhouse/schema.proto"|g' /app/build.rs.original > /app/build.rs.new # Cook dependencies separately COPY --from=chef /app/recipe.json recipe.json -RUN cargo chef cook --profile ${CARGO_PROFILE} --recipe-path recipe.json - -# Copy the ACTUAL schema.proto from the project -COPY ../clickhouse/schema.proto ../clickhouse/schema.proto +COPY --from=chef /app/Cargo.lock Cargo.lock +RUN CXX=/usr/bin/g++ CC=/usr/bin/gcc cargo chef cook --profile ${CARGO_PROFILE} --recipe-path recipe.json # Build the application -COPY . . -RUN cargo build --profile ${CARGO_PROFILE} --locked --jobs "${CARGO_BUILD_JOBS}" \ +COPY ./test-producer . +# Replace the build.rs with our modified version +RUN cp /app/build.rs.new /app/build.rs + +# Verify the proto file exists before building +RUN ls -la /app/clickhouse/ && \ + cat /app/build.rs && \ + cargo build --profile ${CARGO_PROFILE} --jobs "${CARGO_BUILD_JOBS}" \ && strip target/${CARGO_PROFILE}/qos_kafka_producer \ && cp target/${CARGO_PROFILE}/qos_kafka_producer /app/qos_kafka_producer @@ -70,42 +89,45 @@ FROM debian:${DEBIAN_VERSION}-slim AS preparation ARG TINI_VERSION ARG TINI_SHA256 +# Install required packages and clean up RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates curl \ + ca-certificates curl zlib1g \ && rm -rf /var/lib/apt/lists/* -# Download Tini and verify its checksum +# Fetch and verify tini RUN curl -fsSL -o /tini \ https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static-amd64 \ - && echo "${TINI_SHA256} /tini" | sha256sum --check \ + && echo "${TINI_SHA256} /tini" | sha256sum -c - \ && chmod +x /tini -# Create log directory (owned by non-root UID 65532) +# Create logging directory RUN mkdir -p /var/log/qos_producer \ && chown 65532:65532 /var/log/qos_producer # ----------------------------------------------------------------------------- -# 5. Final Stage - Minimal Distroless Image +# 5. Final Stage - Minimal Runtime Image # ----------------------------------------------------------------------------- FROM ${RUNTIME_BASE} COPY --from=preparation /tini /usr/local/bin/tini -COPY --from=builder /app/qos_kafka_producer /usr/local/bin/qos_kafka_producer COPY --from=preparation /var/log/qos_producer /var/log/qos_producer +COPY --from=builder /app/qos_kafka_producer /usr/local/bin/qos_kafka_producer + +# Install required runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + zlib1g ca-certificates \ + && rm -rf /var/lib/apt/lists/* ENV TZ=UTC \ - LANG=C.UTF-8 \ - SSL_CERT_DIR=/etc/ssl/certs \ - RUST_BACKTRACE=1 + LANG=C.UTF-8 WORKDIR / STOPSIGNAL SIGTERM -# Optional healthcheck -HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ - CMD ["/usr/local/bin/qos_kafka_producer", "--health-check"] || exit 1 +# Set non-root user (same as distroless) +RUN groupadd --gid 65532 nonroot && \ + useradd --uid 65532 --gid 65532 -s /sbin/nologin nonroot -# Distroless:nonroot automatically runs as UID 65532 USER nonroot ENTRYPOINT ["/usr/local/bin/tini", "--", "/usr/local/bin/qos_kafka_producer"] diff --git a/oracle/test-producer/build.rs b/oracle/test-producer/build.rs index 8626b54..78882a7 100644 --- a/oracle/test-producer/build.rs +++ b/oracle/test-producer/build.rs @@ -1,8 +1,8 @@ fn main() { // Specify the path to the schema.proto file - let proto_file = "../clickhouse/schema.proto"; + let proto_file = "./clickhouse/schema.proto"; println!("cargo:rerun-if-changed={}", proto_file); - prost_build::compile_protos(&[proto_file], &["../"]).unwrap(); + prost_build::compile_protos(&[proto_file], &["."]).unwrap(); } \ No newline at end of file diff --git a/oracle/test-producer/src/main.rs b/oracle/test-producer/src/main.rs index 3b7399f..d99dac0 100644 --- a/oracle/test-producer/src/main.rs +++ b/oracle/test-producer/src/main.rs @@ -121,10 +121,6 @@ pub struct ClientQueryProtobuf { pub query_id: String, #[prost(string, tag = "4")] pub api_key: String, - #[prost(string, tag = "11")] - pub user_id: String, - #[prost(string, optional, tag = "12")] - pub subgraph: Option, #[prost(string, tag = "5")] pub result: String, #[prost(uint32, tag = "6")] @@ -137,6 +133,10 @@ pub struct ClientQueryProtobuf { pub total_fees_usd: f64, #[prost(message, repeated, tag = "10")] pub indexer_queries: Vec, + #[prost(string, tag = "11")] + pub user_id: String, + #[prost(string, optional, tag = "12")] + pub subgraph: Option, } #[derive(prost::Message)] @@ -347,7 +347,7 @@ fn encode_client_request(client_request: ClientRequest, tap_signer: Address, gra // Encode to Protobuf let mut buf = Vec::new(); client_query_msg.encode(&mut buf).unwrap(); - + // Return the raw protobuf message without length prefix buf } @@ -404,6 +404,7 @@ async fn main() { // Encode using the gateway's exact logic let encoded_message = encode_client_request(client_request, tap_signer, graph_env.clone()); + println!("Encoded message: {} ", hex::encode(&encoded_message)); println!("Encoded message size: {} bytes", encoded_message.len()); counter += 1; From fd57d48623d35874432b3cb22f62d12d22897fef Mon Sep 17 00:00:00 2001 From: cjorge-graphops <128147788+cjorge-graphops@users.noreply.github.com> Date: Mon, 31 Mar 2025 11:15:28 +0100 Subject: [PATCH 07/20] feat: add grafana and prometheus to stack (#14) feat: add dashboard feat: add PARTITION BY, to partition data daily. Partitions will optimize data management when aligned with TTL boundaries, so that entire partitions can be dropped, or queries can be optimized by ignoring irrelevant whole partitions. --------- Co-authored-by: Juan Manuel Rodriguez Defago --- oracle/clickhouse/settings.xml | 14 +- oracle/clickhouse/tables.sql | 4 +- oracle/docker-compose.yml | 70 +- .../dashboards/qos_oracle_dashboard.json | 1901 +++++++++++++++++ .../provisioning/dashboards/dashboard.yaml | 10 + .../provisioning/datasources/datasource.yaml | 7 + oracle/prometheus/prometheus.yml | 11 + 7 files changed, 2008 insertions(+), 9 deletions(-) create mode 100644 oracle/grafana/dashboards/qos_oracle_dashboard.json create mode 100644 oracle/grafana/provisioning/dashboards/dashboard.yaml create mode 100644 oracle/grafana/provisioning/datasources/datasource.yaml create mode 100644 oracle/prometheus/prometheus.yml diff --git a/oracle/clickhouse/settings.xml b/oracle/clickhouse/settings.xml index 4bb8cf2..c755f85 100644 --- a/oracle/clickhouse/settings.xml +++ b/oracle/clickhouse/settings.xml @@ -1,5 +1,13 @@ - - 1 - + 0.0.0.0 + 8123 + 9000 + + /metrics + 9363 + true + true + true + true + diff --git a/oracle/clickhouse/tables.sql b/oracle/clickhouse/tables.sql index 1b2b8b2..b9c94f8 100644 --- a/oracle/clickhouse/tables.sql +++ b/oracle/clickhouse/tables.sql @@ -66,6 +66,7 @@ CREATE TABLE IF NOT EXISTS raw_qos_data ) ) ENGINE = MergeTree() ORDER BY (event_time, gateway_id) +PARTITION BY toYYYYMMDD(event_time) TTL event_time + INTERVAL 7 DAY; -- Create a simplified materialized view @@ -101,7 +102,8 @@ CREATE TABLE IF NOT EXISTS qos_data blocks_behind UInt64 ) ) ENGINE = MergeTree() -ORDER BY (event_time, gateway_id, query_id); +ORDER BY (event_time, gateway_id, query_id) +PARTITION BY toYYYYMMDD(event_time); -- Only create the materialized view if it doesn't exist already CREATE MATERIALIZED VIEW IF NOT EXISTS qos_data_mv TO qos_data AS diff --git a/oracle/docker-compose.yml b/oracle/docker-compose.yml index 500ccdf..d486a58 100644 --- a/oracle/docker-compose.yml +++ b/oracle/docker-compose.yml @@ -19,13 +19,16 @@ services: expose: - "8123" # HTTP interface - "9000" # Native protocol + - "9363" # Prometheus ports: - "8123:8123" # HTTP: queries/monitoring - "9000:9000" # Native: client connections + - "9363:9363" # Prometheus volumes: - ./clickhouse/schema.proto:/etc/clickhouse-server/schema.proto:ro - ./clickhouse/tables.sql:/docker-entrypoint-initdb.d/tables.sql:ro - ./clickhouse/allow_schema_path.xml:/etc/clickhouse-server/config.d/allow_schema_path.xml:ro + - ./clickhouse/settings.xml:/etc/clickhouse-server/config.d/settings.xml:ro - clickhouse_data:/var/lib/clickhouse - clickhouse_logs:/var/log/clickhouse-server ulimits: @@ -42,10 +45,12 @@ services: image: redpandadata/redpanda:latest profiles: [all, deps, dev] expose: - - "9092" # Kafka API - - "8082" # Pandaproxy HTTP + - "9092" # Kafka API + - "8082" # Pandaproxy HTTP + - "9644" # Prometheus metrics ports: - - "9092:9092" # Kafka: producer/consumer + - "9092:9092" # Kafka: producer/consumer + - "9644:9644" command: - redpanda - start @@ -112,6 +117,7 @@ services: test: ["CMD-SHELL", "ps aux | grep -v grep | grep -q qos_kafka_producer || exit 1"] redpanda-console: + profiles: [all, dev] image: redpandadata/console:latest container_name: redpanda-console depends_on: @@ -130,10 +136,64 @@ services: retries: 5 start_period: 10s + # Observability: Prometheus + Grafana + prometheus: + profiles: [all, dev, deps] + image: prom/prometheus:latest + container_name: prometheus + depends_on: + - redpanda + volumes: + - ./prometheus:/etc/prometheus:ro + # This volume will store TSDB data so metrics persist across restarts + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=7d' # Example retention: 7 days + - '--web.enable-lifecycle' # If you want to allow config reload via HTTP + expose: + - "9090" + ports: + - '9090:9090' + healthcheck: + <<: *default-healthcheck + test: ["CMD-SHELL", "wget --spider -q http://localhost:9090/-/healthy || exit 1"] + start_period: 10s + + grafana: + profiles: [all, dev, deps] + image: grafana/grafana:latest + container_name: grafana + depends_on: + - prometheus + environment: + GF_PATHS_PROVISIONING: /etc/grafana/provisioning + GF_AUTH_ANONYMOUS_ENABLED: true + GF_AUTH_ANONYMOUS_ORG_ROLE: Admin + GF_AUTH_BASIC_ENABLED: false + expose: + - "3000" + ports: + - "3000:3000" + healthcheck: + <<: *default-healthcheck + test: ["CMD-SHELL", "curl -f http://localhost:3000/api/health || exit 1"] + start_period: 10s + volumes: + - ./grafana/provisioning/:/etc/grafana/provisioning/ + - ./grafana/dashboards/:/var/lib/grafana/dashboards/ + # Persistent volumes volumes: - clickhouse_data: # clickhouse persistent storage + clickhouse_data: driver: local clickhouse_logs: - redpanda_data: # redpanda persistent storage + redpanda_data: + driver: local + prometheus_data: + driver: local + grafana_data: + driver: local + grafana_provisioning: driver: local diff --git a/oracle/grafana/dashboards/qos_oracle_dashboard.json b/oracle/grafana/dashboards/qos_oracle_dashboard.json new file mode 100644 index 0000000..30d944a --- /dev/null +++ b/oracle/grafana/dashboards/qos_oracle_dashboard.json @@ -0,0 +1,1901 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "Redpanda Overview", + "type": "row" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "vectorized_application_uptime", + "legendFormat": "Instance: {{instance}}", + "refId": "A" + } + ], + "title": "Redpanda Uptime (seconds)", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "vectorized_application_build", + "legendFormat": "Build: {{version}}", + "refId": "A" + } + ], + "title": "Redpanda Build Info", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "vectorized_application_fips_mode", + "legendFormat": "{{instance}} - FIPS", + "refId": "A" + } + ], + "title": "FIPS Mode", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "vectorized_cluster_partition_under_replicated_replicas", + "legendFormat": "URP", + "refId": "A" + } + ], + "title": "Under-Replicated Partitions", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 25, + "panels": [], + "title": "Redpanda Traffic & Throughput", + "type": "row" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "rate(vectorized_cluster_partition_records_produced[5m])", + "legendFormat": "Produced", + "refId": "A" + }, + { + "expr": "rate(vectorized_cluster_partition_records_fetched[5m])", + "legendFormat": "Fetched", + "refId": "B" + } + ], + "title": "Records Produced / Fetched per Sec", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "rate(vectorized_cluster_partition_bytes_produced_total[5m])", + "legendFormat": "Bytes Produced", + "refId": "A" + }, + { + "expr": "rate(vectorized_cluster_partition_bytes_fetched_total[5m])", + "legendFormat": "Bytes Fetched", + "refId": "B" + } + ], + "title": "Bytes Produced / Fetched per Sec", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 26, + "panels": [], + "title": "Redpanda Latencies & Connection Health", + "type": "row" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "rate(vectorized_kafka_handler_latency_microseconds_count[5m])", + "legendFormat": "Handler Latency", + "refId": "A" + } + ], + "title": "Kafka Handler Latency (µs)", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "vectorized_kafka_rpc_active_connections", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Active Kafka Connections", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "vectorized_kafka_rpc_connections_rejected", + "legendFormat": "Rejected", + "refId": "A" + } + ], + "title": "Rejected Kafka Connections", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 27, + "panels": [], + "title": "Redpanda Resource Usage", + "type": "row" + }, + { + "datasource": "Prometheus", + "description": "CPU busy time in ms; using irate for approximate CPU usage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "irate(vectorized_reactor_cpu_busy_ms[5m])", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "CPU Busy ms/s (approx)", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "vectorized_memory_allocated_memory", + "legendFormat": "Allocated", + "refId": "A" + }, + { + "expr": "vectorized_memory_available_memory", + "legendFormat": "Available", + "refId": "B" + } + ], + "title": "Memory Used / Available", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "vectorized_space_management_disk_usage_bytes", + "legendFormat": "Used Bytes", + "refId": "A" + }, + { + "expr": "vectorized_space_management_target_disk_size_bytes", + "legendFormat": "Target Disk Size", + "refId": "B" + } + ], + "title": "Disk Usage", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "irate(vectorized_io_queue_delay[5m])", + "legendFormat": "I/O Delay (ms)", + "refId": "A" + }, + { + "expr": "irate(vectorized_io_queue_consumption[5m])", + "legendFormat": "I/O Consumption (ms)", + "refId": "B" + } + ], + "title": "I/O Queue Delays & Consumption", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 28, + "panels": [], + "title": "ClickHouse Overview", + "type": "row" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 41 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "ClickHouseProfileEvents_Query", + "legendFormat": "All Queries", + "refId": "A" + } + ], + "title": "Total Queries", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 41 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "ClickHouseProfileEvents_SelectQuery", + "legendFormat": "Select", + "refId": "A" + } + ], + "title": "Select Queries (Total)", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 41 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "ClickHouseProfileEvents_InsertQuery", + "legendFormat": "Insert", + "refId": "A" + } + ], + "title": "Insert Queries (Total)", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "rate(ClickHouseProfileEvents_Query[5m])", + "legendFormat": "Queries/s", + "refId": "A" + } + ], + "title": "Queries per Second", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "rate(ClickHouseProfileEvents_QueryTimeMicroseconds[5m])", + "legendFormat": "Query Time µs/s", + "refId": "A" + } + ], + "title": "Query Time (µs) Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 29, + "panels": [], + "title": "ClickHouse Inserts & Merges", + "type": "row" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 54 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "rate(ClickHouseProfileEvents_InsertedRows[5m])", + "legendFormat": "Rows/s", + "refId": "A" + } + ], + "title": "Inserted Rows/s", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 54 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "rate(ClickHouseProfileEvents_MergedRows[5m])", + "legendFormat": "Merged/s", + "refId": "A" + } + ], + "title": "Merged Rows/s", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 54 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "rate(ClickHouseProfileEvents_MergesTimeMilliseconds[5m])", + "legendFormat": "Merges Time ms/s", + "refId": "A" + } + ], + "title": "Merges Time (ms/s)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 30, + "panels": [], + "title": "ClickHouse Resource Usage", + "type": "row" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "ClickHouseMetrics_MemoryTracking", + "legendFormat": "Memory In Use", + "refId": "A" + } + ], + "title": "ClickHouse Memory Tracking (bytes)", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "expr": "ClickHouseAsyncMetrics_TotalBytesOfMergeTreeTables", + "legendFormat": "Total Bytes of MergeTree", + "refId": "A" + }, + { + "expr": "ClickHouseAsyncMetrics_DiskAvailable_default", + "legendFormat": "Disk Available (default)", + "refId": "B" + } + ], + "title": "MergeTree Disk Usage / Available", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [ + "qos_oracle" + ], + "templating": { + "list": [] + }, + "timepicker": {}, + "timezone": "", + "title": "QoS Oracle: ClickHouse & Redpanda Overview", + "uid": "qos-oracle-clickhouse-redpanda", + "version": 8 +} diff --git a/oracle/grafana/provisioning/dashboards/dashboard.yaml b/oracle/grafana/provisioning/dashboards/dashboard.yaml new file mode 100644 index 0000000..8f9f56f --- /dev/null +++ b/oracle/grafana/provisioning/dashboards/dashboard.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 +providers: + - name: 'Local Dashboards' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards diff --git a/oracle/grafana/provisioning/datasources/datasource.yaml b/oracle/grafana/provisioning/datasources/datasource.yaml new file mode 100644 index 0000000..0eddf26 --- /dev/null +++ b/oracle/grafana/provisioning/datasources/datasource.yaml @@ -0,0 +1,7 @@ +apiVersion: 1 +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true diff --git a/oracle/prometheus/prometheus.yml b/oracle/prometheus/prometheus.yml new file mode 100644 index 0000000..c973769 --- /dev/null +++ b/oracle/prometheus/prometheus.yml @@ -0,0 +1,11 @@ +global: + scrape_interval: 60s + +scrape_configs: + - job_name: 'redpanda' + static_configs: + - targets: ['redpanda:9644'] + + - job_name: 'clickhouse' + static_configs: + - targets: ['clickhouse-server:9363'] From 5402cd926e32f91f5e4f581ed7c96196d7c8d2c4 Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Mon, 31 Mar 2025 17:32:19 -0300 Subject: [PATCH 08/20] feat: upgraded clickhouse server to 25.3 --- Cargo.lock | 26 +++++++++++++++++ oracle/clickhouse/tables.sql | 53 +++++++++++++++++++++++++++++------ oracle/clickhouse/users.xml | 43 ++++++++++++++++++++++++++++ oracle/docker-compose.yml | 11 ++++---- oracle/graphql-api/Cargo.toml | 4 ++- 5 files changed, 122 insertions(+), 15 deletions(-) create mode 100644 oracle/clickhouse/users.xml diff --git a/Cargo.lock b/Cargo.lock index 45fe973..88c250a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -518,6 +518,15 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "bstr" version = "1.11.3" @@ -1818,8 +1827,10 @@ dependencies = [ "actix-web", "async-graphql", "async-graphql-actix-web", + "bs58", "clickhouse", "futures", + "hex", "serde", "serde_json", "tokio", @@ -2325,6 +2336,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.44.1" diff --git a/oracle/clickhouse/tables.sql b/oracle/clickhouse/tables.sql index b9c94f8..135f171 100644 --- a/oracle/clickhouse/tables.sql +++ b/oracle/clickhouse/tables.sql @@ -1,7 +1,6 @@ -- Create the Kafka engine table to ingest Protobuf messages CREATE TABLE IF NOT EXISTS kafka_qos_data ( - event_time DateTime DEFAULT now(), gateway_id String, receipt_signer String, query_id String, @@ -39,7 +38,7 @@ SETTINGS -- Create the destination table for the raw data CREATE TABLE IF NOT EXISTS raw_qos_data ( - event_time DateTime DEFAULT now(), + event_time DateTime, gateway_id String, receipt_signer String, query_id String, @@ -69,14 +68,28 @@ ORDER BY (event_time, gateway_id) PARTITION BY toYYYYMMDD(event_time) TTL event_time + INTERVAL 7 DAY; --- Create a simplified materialized view +-- Create a simplified materialized view that also captures the timestamp CREATE MATERIALIZED VIEW IF NOT EXISTS mv_raw_qos_data TO raw_qos_data AS -SELECT * FROM kafka_qos_data; +SELECT + now() as event_time, + gateway_id, + receipt_signer, + query_id, + api_key, + user_id, + subgraph, + result, + response_time_ms, + request_bytes, + response_bytes, + total_fees_usd, + indexer_queries +FROM kafka_qos_data; --- Create the MaterializedView to process and store data from Kafka +-- Create the table for processed QoS data CREATE TABLE IF NOT EXISTS qos_data ( - event_time DateTime DEFAULT now(), + event_time DateTime, gateway_id String, receipt_signer String, query_id String, @@ -105,6 +118,30 @@ CREATE TABLE IF NOT EXISTS qos_data ORDER BY (event_time, gateway_id, query_id) PARTITION BY toYYYYMMDD(event_time); --- Only create the materialized view if it doesn't exist already +-- Create the materialized view that processes and transforms the data CREATE MATERIALIZED VIEW IF NOT EXISTS qos_data_mv TO qos_data AS -SELECT * FROM kafka_qos_data; +SELECT + now() as event_time, + gateway_id, + HEX(receipt_signer) AS receipt_signer, + query_id, + api_key, + user_id, + subgraph, + result, + response_time_ms, + request_bytes, + response_bytes, + total_fees_usd, + arrayMap(x -> HEX(x), indexer_queries.indexer) AS `indexer_queries.indexer`, + arrayMap(x -> HEX(x), indexer_queries.deployment) AS `indexer_queries.deployment`, + arrayMap(x -> HEX(x), indexer_queries.allocation) AS `indexer_queries.allocation`, + indexer_queries.indexed_chain AS `indexer_queries.indexed_chain`, + indexer_queries.url AS `indexer_queries.url`, + indexer_queries.fee_grt AS `indexer_queries.fee_grt`, + indexer_queries.response_time_ms AS `indexer_queries.response_time_ms`, + indexer_queries.seconds_behind AS `indexer_queries.seconds_behind`, + indexer_queries.result AS `indexer_queries.result`, + indexer_queries.indexer_errors AS `indexer_queries.indexer_errors`, + indexer_queries.blocks_behind AS `indexer_queries.blocks_behind` +FROM kafka_qos_data; diff --git a/oracle/clickhouse/users.xml b/oracle/clickhouse/users.xml new file mode 100644 index 0000000..dcbe3c3 --- /dev/null +++ b/oracle/clickhouse/users.xml @@ -0,0 +1,43 @@ + + + + + + ::/0 + + default + default + 1 + + + + graphql_password + + ::/0 + + default + default + + + + + + 1 + 10000000000 + 0 + + + + + + + 3600 + 0 + 0 + 0 + 0 + 0 + + + + \ No newline at end of file diff --git a/oracle/docker-compose.yml b/oracle/docker-compose.yml index d486a58..9061e11 100644 --- a/oracle/docker-compose.yml +++ b/oracle/docker-compose.yml @@ -7,14 +7,14 @@ x-default-healthcheck: &default-healthcheck x-clickhouse-env: &clickhouse-env CLICKHOUSE_DB: default - CLICKHOUSE_USER: default - CLICKHOUSE_PASSWORD: "" + CLICKHOUSE_USER: graphql + CLICKHOUSE_PASSWORD: graphql_password # Services services: # Core dependencies clickhouse-server: - image: clickhouse/clickhouse-server:22.1 + image: clickhouse/clickhouse-server:25.3 profiles: [all, deps, dev] expose: - "8123" # HTTP interface @@ -29,14 +29,13 @@ services: - ./clickhouse/tables.sql:/docker-entrypoint-initdb.d/tables.sql:ro - ./clickhouse/allow_schema_path.xml:/etc/clickhouse-server/config.d/allow_schema_path.xml:ro - ./clickhouse/settings.xml:/etc/clickhouse-server/config.d/settings.xml:ro + - ./clickhouse/users.xml:/etc/clickhouse-server/users.d/users.xml:ro - clickhouse_data:/var/lib/clickhouse - clickhouse_logs:/var/log/clickhouse-server ulimits: nofile: soft: 262144 hard: 262144 - environment: - <<: *clickhouse-env healthcheck: <<: *default-healthcheck test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"] @@ -89,8 +88,8 @@ services: clickhouse-server: condition: service_healthy environment: + <<: *clickhouse-env CLICKHOUSE_URL: http://clickhouse-server:8123 - CLICKHOUSE_DB: default healthcheck: <<: *default-healthcheck test: ["CMD-SHELL", "curl -f http://localhost:8000/health || exit 1"] diff --git a/oracle/graphql-api/Cargo.toml b/oracle/graphql-api/Cargo.toml index c8d9b54..23bb2a2 100644 --- a/oracle/graphql-api/Cargo.toml +++ b/oracle/graphql-api/Cargo.toml @@ -11,4 +11,6 @@ clickhouse = { version = "0.11", features = ["uuid"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tokio = { version = "1", features = ["full"] } -futures = "0.3" \ No newline at end of file +futures = "0.3" +bs58 = "0.5.0" +hex = "0.4.3" \ No newline at end of file From 5fd13dd87a6a27804d480a75701fb76814d58304 Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Tue, 1 Apr 2025 02:34:32 -0300 Subject: [PATCH 09/20] fix: graphql clickhouse request not working --- Cargo.lock | 289 +++++++++---------------------- oracle/docker-compose.yml | 4 +- oracle/graphql-api/Cargo.toml | 2 +- oracle/graphql-api/src/main.rs | 265 ++++++++++++++++++---------- oracle/test-producer/build.rs | 2 +- oracle/test-producer/src/main.rs | 146 ++++++++++------ 6 files changed, 353 insertions(+), 355 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88c250a..344d4fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,7 +75,7 @@ dependencies = [ "foldhash", "futures-core", "h2", - "http", + "http 0.2.12", "httparse", "httpdate", "itoa", @@ -111,7 +111,7 @@ checksum = "13d324164c51f63867b57e73ba5936ea151b8a41a1d23d1031eeb9f70d0236f8" dependencies = [ "bytestring", "cfg-if", - "http", + "http 0.2.12", "regex", "regex-lite", "serde", @@ -342,7 +342,7 @@ dependencies = [ "fast_chemail", "fnv", "futures-util", - "http", + "http 0.2.12", "indexmap 1.9.3", "mime", "multer", @@ -591,20 +591,29 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cityhash-rs" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93a719913643003b84bd13022b4b7e703c09342cd03b679c4641c7d2e50dc34d" + [[package]] name = "clickhouse" -version = "0.11.6" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0875e527e299fc5f4faba42870bf199a39ab0bb2dbba1b8aef0a2151451130f" +checksum = "d9894248c4c5a4402f76a56c273836a0c32547ec8a68166aedee7e01b7b8d102" dependencies = [ "bstr", "bytes", + "cityhash-rs", "clickhouse-derive", - "clickhouse-rs-cityhash-sys", "futures", + "futures-channel", + "http-body-util", "hyper", - "hyper-tls", - "lz4", + "hyper-util", + "lz4_flex", + "replace_with", "sealed", "serde", "static_assertions", @@ -616,23 +625,14 @@ dependencies = [ [[package]] name = "clickhouse-derive" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18af5425854858c507eec70f7deb4d5d8cec4216fcb086283a78872387281ea5" +checksum = "d70f3e2893f7d3e017eeacdc9a708fbc29a10488e3ebca21f9df6a5d2b616dbb" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 1.0.109", -] - -[[package]] -name = "clickhouse-rs-cityhash-sys" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4baf9d4700a28d6cb600e17ed6ae2b43298a5245f1f76b4eab63027ebfd592b9" -dependencies = [ - "cc", + "syn 2.0.100", ] [[package]] @@ -655,16 +655,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -880,21 +870,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1043,7 +1018,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", "indexmap 2.8.0", "slab", "tokio", @@ -1063,15 +1038,6 @@ version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" -[[package]] -name = "heck" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "heck" version = "0.5.0" @@ -1095,14 +1061,37 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" -version = "0.4.6" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.3.1", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.3.1", + "http-body", "pin-project-lite", ] @@ -1120,38 +1109,41 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "0.14.32" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" dependencies = [ "bytes", "futures-channel", - "futures-core", "futures-util", - "http", + "http 1.3.1", "http-body", "httparse", - "httpdate", "itoa", "pin-project-lite", - "socket2", + "smallvec", "tokio", - "tower-service", - "tracing", "want", ] [[package]] -name = "hyper-tls" -version = "0.5.0" +name = "hyper-util" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" dependencies = [ "bytes", + "futures-channel", + "futures-util", + "http 1.3.1", + "http-body", "hyper", - "native-tls", + "libc", + "pin-project-lite", + "socket2", "tokio", - "tokio-native-tls", + "tower-service", + "tracing", ] [[package]] @@ -1460,23 +1452,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] -name = "lz4" -version = "1.28.1" +name = "lz4_flex" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" -dependencies = [ - "lz4-sys", -] - -[[package]] -name = "lz4-sys" -version = "1.11.1+lz4-1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" -dependencies = [ - "cc", - "libc", -] +checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" [[package]] name = "memchr" @@ -1520,7 +1499,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-util", - "http", + "http 0.2.12", "httparse", "log", "memchr", @@ -1535,23 +1514,6 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "num-conv" version = "0.1.0" @@ -1603,50 +1565,6 @@ version = "1.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc" -[[package]] -name = "openssl" -version = "0.10.71" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - -[[package]] -name = "openssl-sys" -version = "0.9.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "parking_lot" version = "0.12.3" @@ -1784,7 +1702,7 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck 0.5.0", + "heck", "itertools", "log", "multimap", @@ -1998,6 +1916,12 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "replace_with" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a8614ee435691de62bcffcf4a66d91b3594bf1428a5722e79103249a095690" + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -2029,15 +1953,6 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" -[[package]] -name = "schannel" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" -dependencies = [ - "windows-sys 0.59.0", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -2046,37 +1961,13 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sealed" -version = "0.4.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b5e421024b5e5edfbaa8e60ecf90bda9dbffc602dbb230e6028763f85f0c68c" +checksum = "22f968c5ea23d555e670b449c1c5e7b2fc399fdaec1d304a17cd48e288abc107" dependencies = [ - "heck 0.3.3", "proc-macro2", "quote", - "syn 1.0.109", -] - -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" -dependencies = [ - "core-foundation-sys", - "libc", + "syn 2.0.100", ] [[package]] @@ -2101,13 +1992,13 @@ dependencies = [ [[package]] name = "serde_derive_internals" -version = "0.26.0" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.100", ] [[package]] @@ -2177,9 +2068,9 @@ checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" [[package]] name = "socket2" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2380,16 +2271,6 @@ dependencies = [ "syn 2.0.100", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.14" @@ -2493,12 +2374,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" -[[package]] -name = "unicode-segmentation" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" - [[package]] name = "unicode-xid" version = "0.2.6" diff --git a/oracle/docker-compose.yml b/oracle/docker-compose.yml index 9061e11..18a9d68 100644 --- a/oracle/docker-compose.yml +++ b/oracle/docker-compose.yml @@ -137,7 +137,7 @@ services: # Observability: Prometheus + Grafana prometheus: - profiles: [all, dev, deps] + profiles: [all, deps] image: prom/prometheus:latest container_name: prometheus depends_on: @@ -161,7 +161,7 @@ services: start_period: 10s grafana: - profiles: [all, dev, deps] + profiles: [all, deps] image: grafana/grafana:latest container_name: grafana depends_on: diff --git a/oracle/graphql-api/Cargo.toml b/oracle/graphql-api/Cargo.toml index 23bb2a2..772d3db 100644 --- a/oracle/graphql-api/Cargo.toml +++ b/oracle/graphql-api/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" actix-web = "4" async-graphql = "4" async-graphql-actix-web = "4" -clickhouse = { version = "0.11", features = ["uuid"] } +clickhouse = { version = "0.13.2", features = ["uuid"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tokio = { version = "1", features = ["full"] } diff --git a/oracle/graphql-api/src/main.rs b/oracle/graphql-api/src/main.rs index 40a204f..cc87444 100644 --- a/oracle/graphql-api/src/main.rs +++ b/oracle/graphql-api/src/main.rs @@ -1,10 +1,29 @@ -use actix_web::{web, App, HttpServer, HttpResponse, Result}; -use async_graphql::{Schema, EmptyMutation, EmptySubscription, Context, Object, SimpleObject}; +use actix_web::{web, App, HttpResponse, HttpServer, Result}; +use async_graphql::{Context, EmptyMutation, EmptySubscription, Object, Schema, SimpleObject}; use async_graphql_actix_web::{GraphQLRequest, GraphQLResponse}; use clickhouse::{Client, Row}; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; +// use bs58; +// use hex; -// Define the GraphQL types +// Complete QoS Report matching the database schema +#[derive(SimpleObject, Deserialize)] +struct QosReport { + event_time: String, + gateway_id: String, + receipt_signer: String, + query_id: String, + api_key: String, + user_id: String, + subgraph: Option, + result: String, + response_time_ms: u32, + request_bytes: u32, + response_bytes: Option, + total_fees_usd: f64, +} + +/* #[derive(SimpleObject, Deserialize, Serialize)] struct IndexerQuery { indexer: String, @@ -19,10 +38,12 @@ struct IndexerQuery { indexer_errors: String, blocks_behind: u64, } +*/ -#[derive(SimpleObject)] -struct QosReport { - event_time: String, +// ClickHouse Row implementation for direct native protocol deserialization +#[derive(Row, Deserialize, Debug, Clone)] +struct QosReportRow { + event_time: u32, gateway_id: String, receipt_signer: String, query_id: String, @@ -34,98 +55,163 @@ struct QosReport { request_bytes: u32, response_bytes: Option, total_fees_usd: f64, - indexer_queries: Vec, } -// Structure to receive raw data from ClickHouse - needs both Row and Deserialize -#[derive(Row, Deserialize)] -struct ClickhouseRow { - event_time: String, - gateway_id: String, - receipt_signer: String, - query_id: String, - api_key: String, - user_id: String, - subgraph: Option, - result: String, - response_time_ms: u32, - request_bytes: u32, - response_bytes: Option, - total_fees_usd: f64, - indexer_queries: Vec<( - String, String, String, String, String, - f64, u32, u32, String, String, u64 - )>, +// Count query result struct +#[derive(Row, Deserialize, Debug)] +struct CountResult { + count: u64, } struct QueryRoot; #[Object] impl QueryRoot { - /// Query for QoS Reports within a time range. - async fn qos_reports(&self, _ctx: &Context<'_>, from: String, to: String) -> async_graphql::Result> { + /// Query for QoS Reports with complete fields + async fn qos_reports( + &self, + _ctx: &Context<'_>, + from: Option, + to: Option, + limit: Option, + ) -> async_graphql::Result> { + // Configure client with proper settings and timeouts let client = Client::default() - .with_url(&std::env::var("CLICKHOUSE_URL").unwrap_or_else(|_| "http://localhost:8123".into())) - .with_database(&std::env::var("CLICKHOUSE_DB").unwrap_or_else(|_| "default".into())); - - let query = format!( - "SELECT event_time, gateway_id, receipt_signer, query_id, api_key, user_id, subgraph, result, response_time_ms, request_bytes, response_bytes, total_fees_usd, indexer_queries \ - FROM raw_qos_data \ - WHERE event_time BETWEEN toDateTime('{}') AND toDateTime('{}') \ - ORDER BY event_time DESC", - from, to - ); - - // Properly handle the ClickHouse cursor - let cursor_result = client.query(&query).fetch::(); - - match cursor_result { - Ok(mut cursor) => { - let mut reports = Vec::new(); - - // Manually collect rows from the cursor - while let Ok(Some(row)) = cursor.next().await { - // Convert the tuple array to IndexerQuery structs - let indexer_queries = row.indexer_queries.into_iter().map(|tuple| { - IndexerQuery { - indexer: tuple.0, - deployment: tuple.1, - allocation: tuple.2, - indexed_chain: tuple.3, - url: tuple.4, - fee_grt: tuple.5, - response_time_ms: tuple.6, - seconds_behind: tuple.7, - result: tuple.8, - indexer_errors: tuple.9, - blocks_behind: tuple.10, - } - }).collect(); - - reports.push(QosReport { - event_time: row.event_time, - gateway_id: row.gateway_id, - receipt_signer: row.receipt_signer, - query_id: row.query_id, - api_key: row.api_key, - user_id: row.user_id, - subgraph: row.subgraph, - result: row.result, - response_time_ms: row.response_time_ms, - request_bytes: row.request_bytes, - response_bytes: row.response_bytes, - total_fees_usd: row.total_fees_usd, - indexer_queries, - }); + .with_url( + &std::env::var("CLICKHOUSE_URL").unwrap_or_else(|_| "http://localhost:8123".into()), + ) + .with_database(&std::env::var("CLICKHOUSE_DB").unwrap_or_else(|_| "default".into())) + .with_user(&std::env::var("CLICKHOUSE_USER").unwrap_or_else(|_| "graphql".into())) + .with_password( + &std::env::var("CLICKHOUSE_PASSWORD").unwrap_or_else(|_| "graphql_password".into()), + ); + + // First, check if there's any data using a simple count query + let count_query = "SELECT count() as count FROM qos_data"; + println!("Checking table count: {}", count_query); + + // Use the proper Row trait implementation for count + let count_result = client.query(count_query).fetch_one::().await; + + match count_result { + Ok(count_row) => { + println!("Total records in qos_data: {}", count_row.count); + if count_row.count == 0 { + return Ok(Vec::new()); + } + } + Err(e) => { + println!("Error getting count: {}", e); + return Err(async_graphql::Error::new(format!("Database error: {}", e))); + } + } + + // Build full query with all fields + let mut query = "SELECT + event_time, + gateway_id, + query_id, + api_key, + user_id, + receipt_signer, + subgraph, + result, + response_time_ms, + request_bytes, + response_bytes, + total_fees_usd + FROM qos_data" + .to_string(); + + // event_time, + // gateway_id, + // receipt_signer, + // query_id, + // api_key, + // user_id, + // subgraph, + // result, + // response_time_ms, + // request_bytes, + // response_bytes, + // total_fees_usd + + // Add date filtering if provided + if from.is_some() || to.is_some() { + query.push_str(" WHERE "); + + if let Some(from_val) = &from { + query.push_str(&format!("event_time >= toDateTime('{}')", from_val)); + if to.is_some() { + query.push_str(" AND "); } + } + + if let Some(to_val) = &to { + query.push_str(&format!("event_time <= toDateTime('{}')", to_val)); + } + } + // Use the provided limit or default to 100 + let limit_value = limit.unwrap_or(100).max(1).min(1000); + query.push_str(&format!(" ORDER BY event_time DESC LIMIT {}", limit_value)); + + println!("Executing query: {}", query); + + // Use fetch_all with the Row trait implementation + let rows_result = client.query(&query).fetch_all::().await; + + match rows_result { + Ok(rows) => { + println!("Query returned {} rows", rows.len()); + + // Convert to GraphQL output type + let reports: Vec = rows + .into_iter() + .map(|row| { + println!( + "Processing row: event_time={}, gateway_id={}, query_id={}", + row.event_time, row.gateway_id, row.query_id + ); + + QosReport { + event_time: row.event_time.to_string(), + gateway_id: row.gateway_id, + receipt_signer: row.receipt_signer, + query_id: row.query_id, + api_key: row.api_key, + user_id: row.user_id, + subgraph: row.subgraph, + result: row.result, + response_time_ms: row.response_time_ms, + request_bytes: row.request_bytes, + response_bytes: row.response_bytes, + total_fees_usd: row.total_fees_usd, + } + }) + .collect(); + + println!("Returning {} reports", reports.len()); Ok(reports) - }, - Err(err) => Err(async_graphql::Error::new(err.to_string())), + } + Err(err) => { + println!("Query failed: {}", err); + Err(async_graphql::Error::new(format!( + "Database error: {}", + err + ))) + } } } } +/* +// Additional fields and nested data queries (commented out for debugging) +async fn fetch_nested_data(&self, client: &Client, query_id: &str) -> Result, String> { + // ... +} +*/ + type GQLSchema = Schema; async fn graphql_handler(schema: web::Data, req: GraphQLRequest) -> GraphQLResponse { @@ -165,16 +251,20 @@ async fn graphql_playground() -> Result { "#; - + Ok(HttpResponse::Ok() .content_type("text/html; charset=utf-8") .body(html)) } +// Health check endpoint for Docker +async fn health_check() -> HttpResponse { + HttpResponse::Ok().body("OK") +} + #[actix_web::main] async fn main() -> std::io::Result<()> { - let schema = Schema::build(QueryRoot, EmptyMutation, EmptySubscription) - .finish(); + let schema = Schema::build(QueryRoot, EmptyMutation, EmptySubscription).finish(); println!("GraphQL playground: http://localhost:8000"); @@ -183,8 +273,9 @@ async fn main() -> std::io::Result<()> { .app_data(web::Data::new(schema.clone())) .route("/graphql", web::post().to(graphql_handler)) .route("/", web::get().to(graphql_playground)) + .route("/health", web::get().to(health_check)) }) .bind("0.0.0.0:8000")? .run() .await -} \ No newline at end of file +} diff --git a/oracle/test-producer/build.rs b/oracle/test-producer/build.rs index 78882a7..2eae0bb 100644 --- a/oracle/test-producer/build.rs +++ b/oracle/test-producer/build.rs @@ -5,4 +5,4 @@ fn main() { println!("cargo:rerun-if-changed={}", proto_file); prost_build::compile_protos(&[proto_file], &["."]).unwrap(); -} \ No newline at end of file +} diff --git a/oracle/test-producer/src/main.rs b/oracle/test-producer/src/main.rs index d99dac0..8fae445 100644 --- a/oracle/test-producer/src/main.rs +++ b/oracle/test-producer/src/main.rs @@ -1,35 +1,35 @@ use std::env; use std::time::Duration; +use hex; +use prost::Message; use rand::Rng; use rdkafka::config::ClientConfig; use rdkafka::producer::{FutureProducer, FutureRecord}; use rdkafka::util::get_rdkafka_version; use tokio::time; -use prost::Message; -use hex; // Mock types to match the gateway's dependencies mod mock { #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct DeploymentId(pub [u8; 32]); - + #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct IndexerId(pub [u8; 20]); - + #[derive(Clone)] pub struct SubgraphId(pub String); - + #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct Address(pub [u8; 20]); - + impl std::ops::Deref for IndexerId { type Target = Address; fn deref(&self) -> &Self::Target { unsafe { std::mem::transmute(self) } } } - + impl DeploymentId { pub fn to_vec(&self) -> Vec { self.0.to_vec() @@ -41,24 +41,24 @@ mod mock { self.0.to_vec() } } - + impl SubgraphId { pub fn to_string(&self) -> String { self.0.clone() } } - + impl Address { pub fn to_vec(&self) -> Vec { self.0.to_vec() } } - + // Mock error types pub enum Error { QueryFailed(String), } - + impl ToString for Error { fn to_string(&self) -> String { match self { @@ -66,12 +66,12 @@ mod mock { } } } - + pub enum IndexerError { NetworkError(String), Timeout, } - + impl ToString for IndexerError { fn to_string(&self) -> String { match self { @@ -80,27 +80,27 @@ mod mock { } } } - + // Mock Receipt pub struct Receipt { allocation: Address, value: u64, } - + impl Receipt { pub fn new(allocation: Address, value: u64) -> Self { Self { allocation, value } } - + pub fn allocation(&self) -> Address { self.allocation } - + pub fn value(&self) -> u64 { self.value } } - + // Simplified IndexerResponse without attestation pub struct IndexerResponse { pub errors: Vec, @@ -176,7 +176,7 @@ pub struct ClientRequest { pub api_key: String, pub user: String, pub subgraph: Option, - pub grt_per_usd: f64, // Using f64 instead of NotNan for simplicity + pub grt_per_usd: f64, // Using f64 instead of NotNan for simplicity pub indexer_requests: Vec, pub request_bytes: u32, pub response_bytes: Option, @@ -202,6 +202,24 @@ fn random_bytes(len: usize) -> Vec { bytes } +// Generate a random string that's guaranteed to be valid UTF-8 +fn random_utf8_string(prefix: &str, len: usize) -> String { + use rand::{thread_rng, Rng}; + + let suffix: String = thread_rng() + .sample_iter(&rand::distributions::Alphanumeric) + .take(len) + .map(char::from) + .collect(); + + format!("{}-{}", prefix, suffix) +} + +// Generate random hex string (guaranteed valid UTF-8) from random bytes +fn random_hex_string(len: usize) -> String { + hex::encode(random_bytes(len)) +} + fn random_address() -> Address { let mut bytes = [0u8; 20]; rand::thread_rng().fill(&mut bytes); @@ -222,33 +240,39 @@ fn random_indexer_id() -> IndexerId { fn generate_random_client_request() -> ClientRequest { let mut rng = rand::thread_rng(); - + // Generate between 1 and 4 indexer requests let num_indexers = rng.gen_range(1..5); let mut indexer_requests = Vec::with_capacity(num_indexers); - + for _ in 0..num_indexers { - let success = rng.gen_bool(0.9); // 90% success rate - + let success = rng.gen_bool(0.9); // 90% success rate + let indexer_id = random_indexer_id(); let allocation = random_address(); - + let result = if success { - Ok(IndexerResponse { - errors: vec![], - }) + Ok(IndexerResponse { errors: vec![] }) } else { - let error_types = ["Timeout", "Bad Gateway", "Internal Server Error"]; + // Use safe error types with valid UTF-8 Err(IndexerError::NetworkError( - error_types[rng.gen_range(0..error_types.len())].to_string() + "Error processing request".to_string(), )) }; - + indexer_requests.push(IndexerRequest { indexer: indexer_id, deployment: random_deployment_id(), - url: format!("https://api.thegraph.com/subgraphs/id/QmIndexer{}", rng.gen_range(1000..9999)), - receipt: Receipt::new(allocation, rng.gen_range(100_000_000_000_000..1_000_000_000_000_000)), + // Ensure URL is valid UTF-8 + url: format!( + "https://api.thegraph.com/subgraphs/id/Qm{}", + random_hex_string(10) + ), + receipt: Receipt::new( + allocation, + rng.gen_range(100_000_000_000_000..1_000_000_000_000_000), + ), + // Ensure chain name is valid UTF-8 subgraph_chain: "mainnet".to_string(), result, response_time_ms: rng.gen_range(50..2000) as u16, @@ -256,25 +280,29 @@ fn generate_random_client_request() -> ClientRequest { blocks_behind: rng.gen_range(0..50), }); } - - let success = rng.gen_bool(0.95); // 95% success rate for overall query - + + let success = rng.gen_bool(0.95); // 95% success rate for overall query + ClientRequest { - id: format!("{:x}", rng.gen::()), + // Use random hex for IDs (guaranteed valid UTF-8) + id: random_hex_string(16), response_time_ms: rng.gen_range(50..5000) as u16, result: if success { Ok(()) } else { + // Use predefined error message to ensure UTF-8 Err(Error::QueryFailed("Query validation error".to_string())) }, - api_key: format!("api-{:x}", rng.gen::()), - user: format!("user-{:x}", rng.gen::()), + // Use random alphanumeric strings with prefixes + api_key: random_utf8_string("api", 16), + user: random_utf8_string("user", 16), subgraph: if rng.gen_bool(0.9) { - Some(SubgraphId(format!("Qm{}", hex::encode(random_bytes(10))))) + // Guarantee valid UTF-8 for subgraph ID + Some(SubgraphId(format!("Qm{}", random_hex_string(10)))) } else { None }, - grt_per_usd: rng.gen_range(0.05..0.2), // Realistic GRT/USD price range + grt_per_usd: rng.gen_range(0.05..0.2), // Realistic GRT/USD price range indexer_requests, request_bytes: rng.gen_range(100..5000), response_bytes: if rng.gen_bool(0.95) { @@ -286,7 +314,11 @@ fn generate_random_client_request() -> ClientRequest { } // Function that matches the logic in gateway's report function but without attestation -fn encode_client_request(client_request: ClientRequest, tap_signer: Address, graph_env: String) -> Vec { +fn encode_client_request( + client_request: ClientRequest, + tap_signer: Address, + graph_env: String, +) -> Vec { let indexer_queries = client_request .indexer_requests .iter() @@ -366,25 +398,24 @@ async fn main() { let delay = Duration::from_micros(1_000_000 / mps); // Allow configuring the broker address from environment - let broker = env::var("KAFKA_BROKER") - .unwrap_or_else(|_| "redpanda:9092".to_string()); - + let broker = env::var("KAFKA_BROKER").unwrap_or_else(|_| "redpanda:9092".to_string()); + println!("Connecting to Kafka broker at {}", broker); - + // Wait for Redpanda to be fully ready - println!("Waiting 15 seconds for Redpanda to initialize..."); - time::sleep(Duration::from_secs(15)).await; - + println!("Waiting 5 seconds for Redpanda to initialize..."); + time::sleep(Duration::from_secs(5)).await; + // Create a Kafka producer with explicit PLAINTEXT protocol let producer: FutureProducer = ClientConfig::new() .set("bootstrap.servers", &broker) .set("message.timeout.ms", "30000") .set("security.protocol", "PLAINTEXT") .set("debug", "all") - .set("enable.idempotence", "false") // Simplify for testing - .set("retries", "5") // Retry a few times - .set("retry.backoff.ms", "1000") // 1 second between retries - .set("socket.timeout.ms", "10000") // 10 seconds socket timeout + .set("enable.idempotence", "false") // Simplify for testing + .set("retries", "5") // Retry a few times + .set("retry.backoff.ms", "1000") // 1 second between retries + .set("socket.timeout.ms", "10000") // 10 seconds socket timeout .set("socket.keepalive.enable", "true") .create() .expect("Producer creation error"); @@ -392,23 +423,24 @@ async fn main() { // Make sure this matches the topic name in ClickHouse Kafka engine let topic = "gateway_qos_topic"; let mut counter = 0; - + // Create a fixed tap_signer and graph_env to use for all messages let tap_signer = random_address(); + // Ensure graph_env is valid UTF-8 let graph_env = "testnet-gateway".to_string(); loop { // Generate a random client request let client_request = generate_random_client_request(); - + // Encode using the gateway's exact logic let encoded_message = encode_client_request(client_request, tap_signer, graph_env.clone()); - + println!("Encoded message: {} ", hex::encode(&encoded_message)); println!("Encoded message size: {} bytes", encoded_message.len()); - + counter += 1; - + match producer .send( FutureRecord::to(topic) From 8d202dc17e753a954ac91df4ef36ad9786a06902 Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Wed, 2 Apr 2025 00:45:50 -0300 Subject: [PATCH 10/20] feat: added basic Allocation, Deployment and Indexer aggregation resolvers --- Cargo.lock | 2 + oracle/clickhouse/tables.sql | 343 +++++++++++++++- oracle/graphql-api/Cargo.toml | 3 +- oracle/graphql-api/src/main.rs | 647 +++++++++++++++++++++++-------- oracle/test-producer/Cargo.toml | 1 + oracle/test-producer/src/main.rs | 180 ++++++--- 6 files changed, 947 insertions(+), 229 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 344d4fd..59fa535 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1746,6 +1746,7 @@ dependencies = [ "async-graphql", "async-graphql-actix-web", "bs58", + "chrono", "clickhouse", "futures", "hex", @@ -1760,6 +1761,7 @@ version = "0.1.0" dependencies = [ "chrono", "hex", + "once_cell", "prost", "prost-build", "rand 0.8.5", diff --git a/oracle/clickhouse/tables.sql b/oracle/clickhouse/tables.sql index 135f171..d0de63b 100644 --- a/oracle/clickhouse/tables.sql +++ b/oracle/clickhouse/tables.sql @@ -35,12 +35,14 @@ SETTINGS kafka_max_block_size = 1, kafka_poll_timeout_ms = 500; --- Create the destination table for the raw data +-- Create the destination table for the raw data (with TTL) +-- Note: This table seems less critical now with the qos_data_mv directly populating qos_data, +-- but we keep it for potential raw data inspection if needed. CREATE TABLE IF NOT EXISTS raw_qos_data ( event_time DateTime, gateway_id String, - receipt_signer String, + receipt_signer String, -- Raw bytes query_id String, api_key String, user_id String, @@ -51,9 +53,9 @@ CREATE TABLE IF NOT EXISTS raw_qos_data response_bytes Nullable(UInt32), total_fees_usd Float64, indexer_queries Nested( - indexer String, - deployment String, - allocation String, + indexer String, -- Raw bytes + deployment String, -- Raw bytes + allocation String, -- Raw bytes indexed_chain String, url String, fee_grt Float64, @@ -66,14 +68,14 @@ CREATE TABLE IF NOT EXISTS raw_qos_data ) ENGINE = MergeTree() ORDER BY (event_time, gateway_id) PARTITION BY toYYYYMMDD(event_time) -TTL event_time + INTERVAL 7 DAY; +TTL event_time + INTERVAL 7 DAY; -- Keep raw data for 7 days --- Create a simplified materialized view that also captures the timestamp +-- Materialized view to populate raw_qos_data (optional, could be removed if qos_data is sufficient) CREATE MATERIALIZED VIEW IF NOT EXISTS mv_raw_qos_data TO raw_qos_data AS SELECT now() as event_time, gateway_id, - receipt_signer, + receipt_signer, -- Store raw bytes here query_id, api_key, user_id, @@ -83,15 +85,15 @@ SELECT request_bytes, response_bytes, total_fees_usd, - indexer_queries + indexer_queries -- Store raw nested bytes here FROM kafka_qos_data; --- Create the table for processed QoS data +-- Create the table for processed QoS data (with HEX encoding) CREATE TABLE IF NOT EXISTS qos_data ( event_time DateTime, gateway_id String, - receipt_signer String, + receipt_signer String, -- HEX encoded query_id String, api_key String, user_id String, @@ -102,9 +104,9 @@ CREATE TABLE IF NOT EXISTS qos_data response_bytes Nullable(UInt32), total_fees_usd Float64, indexer_queries Nested( - indexer String, - deployment String, - allocation String, + indexer String, -- HEX encoded + deployment String, -- HEX encoded + allocation String, -- HEX encoded indexed_chain String, url String, fee_grt Float64, @@ -116,14 +118,15 @@ CREATE TABLE IF NOT EXISTS qos_data ) ) ENGINE = MergeTree() ORDER BY (event_time, gateway_id, query_id) -PARTITION BY toYYYYMMDD(event_time); +PARTITION BY toYYYYMMDD(event_time) +TTL event_time + INTERVAL 30 DAY; -- Keep processed data longer, e.g., 30 days --- Create the materialized view that processes and transforms the data +-- Create the materialized view that processes and transforms the data into qos_data CREATE MATERIALIZED VIEW IF NOT EXISTS qos_data_mv TO qos_data AS SELECT now() as event_time, gateway_id, - HEX(receipt_signer) AS receipt_signer, + HEX(receipt_signer) AS receipt_signer, -- Apply HEX encoding query_id, api_key, user_id, @@ -133,6 +136,7 @@ SELECT request_bytes, response_bytes, total_fees_usd, + -- Apply HEX encoding to nested fields arrayMap(x -> HEX(x), indexer_queries.indexer) AS `indexer_queries.indexer`, arrayMap(x -> HEX(x), indexer_queries.deployment) AS `indexer_queries.deployment`, arrayMap(x -> HEX(x), indexer_queries.allocation) AS `indexer_queries.allocation`, @@ -145,3 +149,308 @@ SELECT indexer_queries.indexer_errors AS `indexer_queries.indexer_errors`, indexer_queries.blocks_behind AS `indexer_queries.blocks_behind` FROM kafka_qos_data; + +-- ============================================================ +-- AGGREGATION TABLES & MATERIALIZED VIEWS +-- ============================================================ + +-- ---------------------------------------- +-- Deployment Level Aggregations (by subgraph) +-- ---------------------------------------- + +-- 5-Minute Deployment Aggregations +CREATE TABLE IF NOT EXISTS agg_deployment_5min +( + time_bucket DateTime, + subgraph String, -- Deployment ID is the subgraph ID + gateway_id String, + query_count UInt64, + success_count UInt64, + failure_count UInt64, + avg_response_time_ms Float64, + total_fees_usd Float64 +) ENGINE = SummingMergeTree() -- Use SummingMergeTree for counts/sums +ORDER BY (time_bucket, subgraph, gateway_id); + +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_deployment_5min TO agg_deployment_5min AS +SELECT + toStartOfFiveMinutes(event_time) AS time_bucket, + assumeNotNull(subgraph) AS subgraph, -- Treat Nullable subgraph as String for grouping + gateway_id, + count() AS query_count, + countIf(result = 'success') AS success_count, + countIf(result != 'success') AS failure_count, + avg(response_time_ms) AS avg_response_time_ms, + sum(total_fees_usd) AS total_fees_usd +FROM qos_data +WHERE subgraph IS NOT NULL -- Only aggregate for requests with a subgraph +GROUP BY time_bucket, subgraph, gateway_id; + +-- Hourly Deployment Aggregations +CREATE TABLE IF NOT EXISTS agg_deployment_hourly +( + time_bucket DateTime, + subgraph String, + gateway_id String, + query_count UInt64, + success_count UInt64, + failure_count UInt64, + avg_response_time_ms Float64, + total_fees_usd Float64 +) ENGINE = SummingMergeTree() +ORDER BY (time_bucket, subgraph, gateway_id); + +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_deployment_hourly TO agg_deployment_hourly AS +SELECT + toStartOfHour(event_time) AS time_bucket, + assumeNotNull(subgraph) AS subgraph, + gateway_id, + count() AS query_count, + countIf(result = 'success') AS success_count, + countIf(result != 'success') AS failure_count, + avg(response_time_ms) AS avg_response_time_ms, + sum(total_fees_usd) AS total_fees_usd +FROM qos_data +WHERE subgraph IS NOT NULL +GROUP BY time_bucket, subgraph, gateway_id; + +-- Daily Deployment Aggregations +CREATE TABLE IF NOT EXISTS agg_deployment_daily +( + time_bucket DateTime, + subgraph String, + gateway_id String, + query_count UInt64, + success_count UInt64, + failure_count UInt64, + avg_response_time_ms Float64, + total_fees_usd Float64 +) ENGINE = SummingMergeTree() +ORDER BY (time_bucket, subgraph, gateway_id); + +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_deployment_daily TO agg_deployment_daily AS +SELECT + toStartOfDay(event_time) AS time_bucket, + assumeNotNull(subgraph) AS subgraph, + gateway_id, + count() AS query_count, + countIf(result = 'success') AS success_count, + countIf(result != 'success') AS failure_count, + avg(response_time_ms) AS avg_response_time_ms, + sum(total_fees_usd) AS total_fees_usd +FROM qos_data +WHERE subgraph IS NOT NULL +GROUP BY time_bucket, subgraph, gateway_id; + + +-- ---------------------------------------- +-- Indexer Level Aggregations (by indexer_queries.indexer) +-- ---------------------------------------- + +-- 5-Minute Indexer Aggregations +CREATE TABLE IF NOT EXISTS agg_indexer_5min +( + time_bucket DateTime, + indexer String, -- HEX encoded indexer ID + gateway_id String, + query_count UInt64, + success_count UInt64, + failure_count UInt64, + avg_indexer_response_time_ms Float64, + total_fee_grt Float64, + avg_seconds_behind Float64, + avg_blocks_behind Float64 +) ENGINE = SummingMergeTree() +ORDER BY (time_bucket, indexer, gateway_id); + +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_indexer_5min TO agg_indexer_5min AS +SELECT + toStartOfFiveMinutes(event_time) AS time_bucket, + iq.indexer AS indexer, + gateway_id, + count() AS query_count, + countIf(iq.result = 'success') AS success_count, + countIf(iq.result != 'success') AS failure_count, + avg(iq.response_time_ms) AS avg_indexer_response_time_ms, + sum(iq.fee_grt) AS total_fee_grt, + avg(iq.seconds_behind) AS avg_seconds_behind, + avg(iq.blocks_behind) AS avg_blocks_behind +FROM qos_data +ARRAY JOIN indexer_queries AS iq -- Flatten the nested structure +GROUP BY time_bucket, indexer, gateway_id; + +-- Hourly Indexer Aggregations +CREATE TABLE IF NOT EXISTS agg_indexer_hourly +( + time_bucket DateTime, + indexer String, + gateway_id String, + query_count UInt64, + success_count UInt64, + failure_count UInt64, + avg_indexer_response_time_ms Float64, + total_fee_grt Float64, + avg_seconds_behind Float64, + avg_blocks_behind Float64 +) ENGINE = SummingMergeTree() +ORDER BY (time_bucket, indexer, gateway_id); + +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_indexer_hourly TO agg_indexer_hourly AS +SELECT + toStartOfHour(event_time) AS time_bucket, + iq.indexer AS indexer, + gateway_id, + count() AS query_count, + countIf(iq.result = 'success') AS success_count, + countIf(iq.result != 'success') AS failure_count, + avg(iq.response_time_ms) AS avg_indexer_response_time_ms, + sum(iq.fee_grt) AS total_fee_grt, + avg(iq.seconds_behind) AS avg_seconds_behind, + avg(iq.blocks_behind) AS avg_blocks_behind +FROM qos_data +ARRAY JOIN indexer_queries AS iq +GROUP BY time_bucket, indexer, gateway_id; + +-- Daily Indexer Aggregations +CREATE TABLE IF NOT EXISTS agg_indexer_daily +( + time_bucket DateTime, + indexer String, + gateway_id String, + query_count UInt64, + success_count UInt64, + failure_count UInt64, + avg_indexer_response_time_ms Float64, + total_fee_grt Float64, + avg_seconds_behind Float64, + avg_blocks_behind Float64 +) ENGINE = SummingMergeTree() +ORDER BY (time_bucket, indexer, gateway_id); + +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_indexer_daily TO agg_indexer_daily AS +SELECT + toStartOfDay(event_time) AS time_bucket, + iq.indexer AS indexer, + gateway_id, + count() AS query_count, + countIf(iq.result = 'success') AS success_count, + countIf(iq.result != 'success') AS failure_count, + avg(iq.response_time_ms) AS avg_indexer_response_time_ms, + sum(iq.fee_grt) AS total_fee_grt, + avg(iq.seconds_behind) AS avg_seconds_behind, + avg(iq.blocks_behind) AS avg_blocks_behind +FROM qos_data +ARRAY JOIN indexer_queries AS iq +GROUP BY time_bucket, indexer, gateway_id; + + +-- ---------------------------------------- +-- Allocation Level Aggregations (by subgraph + indexer_queries.indexer) +-- NOTE: The name "Allocation Aggregation" here refers to the grouping level, +-- not the specific `allocation` ID field, which is removed. +-- ---------------------------------------- + +-- 5-Minute Allocation Aggregations +CREATE TABLE IF NOT EXISTS agg_allocation_5min +( + time_bucket DateTime, + subgraph String, -- Deployment ID + indexer String, -- HEX encoded indexer ID + gateway_id String, + query_count UInt64, + success_count UInt64, + failure_count UInt64, + avg_indexer_response_time_ms Float64, + total_fee_grt Float64, + avg_seconds_behind Float64, + avg_blocks_behind Float64 +) ENGINE = SummingMergeTree() +ORDER BY (time_bucket, subgraph, indexer, gateway_id); + +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_allocation_5min TO agg_allocation_5min AS +SELECT + toStartOfFiveMinutes(event_time) AS time_bucket, + assumeNotNull(subgraph) AS subgraph, + iq.indexer AS indexer, + gateway_id, + count() AS query_count, + countIf(iq.result = 'success') AS success_count, + countIf(iq.result != 'success') AS failure_count, + avg(iq.response_time_ms) AS avg_indexer_response_time_ms, + sum(iq.fee_grt) AS total_fee_grt, + avg(iq.seconds_behind) AS avg_seconds_behind, + avg(iq.blocks_behind) AS avg_blocks_behind +FROM qos_data +ARRAY JOIN indexer_queries AS iq +WHERE subgraph IS NOT NULL -- Only aggregate for requests with a subgraph +GROUP BY time_bucket, subgraph, indexer, gateway_id; + +-- Hourly Allocation Aggregations +CREATE TABLE IF NOT EXISTS agg_allocation_hourly +( + time_bucket DateTime, + subgraph String, + indexer String, + gateway_id String, + query_count UInt64, + success_count UInt64, + failure_count UInt64, + avg_indexer_response_time_ms Float64, + total_fee_grt Float64, + avg_seconds_behind Float64, + avg_blocks_behind Float64 +) ENGINE = SummingMergeTree() +ORDER BY (time_bucket, subgraph, indexer, gateway_id); + +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_allocation_hourly TO agg_allocation_hourly AS +SELECT + toStartOfHour(event_time) AS time_bucket, + assumeNotNull(subgraph) AS subgraph, + iq.indexer AS indexer, + gateway_id, + count() AS query_count, + countIf(iq.result = 'success') AS success_count, + countIf(iq.result != 'success') AS failure_count, + avg(iq.response_time_ms) AS avg_indexer_response_time_ms, + sum(iq.fee_grt) AS total_fee_grt, + avg(iq.seconds_behind) AS avg_seconds_behind, + avg(iq.blocks_behind) AS avg_blocks_behind +FROM qos_data +ARRAY JOIN indexer_queries AS iq +WHERE subgraph IS NOT NULL +GROUP BY time_bucket, subgraph, indexer, gateway_id; + +-- Daily Allocation Aggregations +CREATE TABLE IF NOT EXISTS agg_allocation_daily +( + time_bucket DateTime, + subgraph String, + indexer String, + gateway_id String, + query_count UInt64, + success_count UInt64, + failure_count UInt64, + avg_indexer_response_time_ms Float64, + total_fee_grt Float64, + avg_seconds_behind Float64, + avg_blocks_behind Float64 +) ENGINE = SummingMergeTree() +ORDER BY (time_bucket, subgraph, indexer, gateway_id); + +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_allocation_daily TO agg_allocation_daily AS +SELECT + toStartOfDay(event_time) AS time_bucket, + assumeNotNull(subgraph) AS subgraph, + iq.indexer AS indexer, + gateway_id, + count() AS query_count, + countIf(iq.result = 'success') AS success_count, + countIf(iq.result != 'success') AS failure_count, + avg(iq.response_time_ms) AS avg_indexer_response_time_ms, + sum(iq.fee_grt) AS total_fee_grt, + avg(iq.seconds_behind) AS avg_seconds_behind, + avg(iq.blocks_behind) AS avg_blocks_behind +FROM qos_data +ARRAY JOIN indexer_queries AS iq +WHERE subgraph IS NOT NULL +GROUP BY time_bucket, subgraph, indexer, gateway_id; diff --git a/oracle/graphql-api/Cargo.toml b/oracle/graphql-api/Cargo.toml index 772d3db..5bb5a94 100644 --- a/oracle/graphql-api/Cargo.toml +++ b/oracle/graphql-api/Cargo.toml @@ -13,4 +13,5 @@ serde_json = "1.0" tokio = { version = "1", features = ["full"] } futures = "0.3" bs58 = "0.5.0" -hex = "0.4.3" \ No newline at end of file +hex = "0.4.3" +chrono = "0.4" \ No newline at end of file diff --git a/oracle/graphql-api/src/main.rs b/oracle/graphql-api/src/main.rs index cc87444..907b21a 100644 --- a/oracle/graphql-api/src/main.rs +++ b/oracle/graphql-api/src/main.rs @@ -1,15 +1,20 @@ use actix_web::{web, App, HttpResponse, HttpServer, Result}; -use async_graphql::{Context, EmptyMutation, EmptySubscription, Object, Schema, SimpleObject}; +use async_graphql::{ + Context, EmptyMutation, EmptySubscription, Enum, InputObject, Object, Schema, SimpleObject, +}; use async_graphql_actix_web::{GraphQLRequest, GraphQLResponse}; +use chrono::{DateTime, TimeZone, Utc}; use clickhouse::{Client, Row}; use serde::Deserialize; +use std::env; // use bs58; // use hex; // Complete QoS Report matching the database schema -#[derive(SimpleObject, Deserialize)] +#[derive(SimpleObject, Deserialize, Debug, Clone)] +// #[graphql(complex)] // Mark complex to add computed fields if needed later struct QosReport { - event_time: String, + event_time: String, // Keep as String for GraphQL output gateway_id: String, receipt_signer: String, query_id: String, @@ -21,6 +26,7 @@ struct QosReport { request_bytes: u32, response_bytes: Option, total_fees_usd: f64, + // We might add indexer_queries here later if needed, requires nested struct handling } /* @@ -43,7 +49,7 @@ struct IndexerQuery { // ClickHouse Row implementation for direct native protocol deserialization #[derive(Row, Deserialize, Debug, Clone)] struct QosReportRow { - event_time: u32, + event_time: u32, // Use u32 for Unix timestamp gateway_id: String, receipt_signer: String, query_id: String, @@ -55,6 +61,7 @@ struct QosReportRow { request_bytes: u32, response_bytes: Option, total_fees_usd: f64, + // indexer_queries: Vec, // Needs proper handling for Nested type if selected } // Count query result struct @@ -63,207 +70,517 @@ struct CountResult { count: u64, } +// --- Structs for Aggregated Data --- + +#[derive(Enum, Copy, Clone, Eq, PartialEq, Debug)] +enum AggregationInterval { + FiveMinutes, + Hourly, + Daily, +} + +impl AggregationInterval { + fn table_suffix(&self) -> &'static str { + match self { + AggregationInterval::FiveMinutes => "5min", + AggregationInterval::Hourly => "hourly", + AggregationInterval::Daily => "daily", + } + } +} + +// --- Deployment Aggregation Structs --- + +#[derive(SimpleObject, Deserialize, Debug, Clone)] +#[graphql(name = "DeploymentAggregation")] // Explicit name for GraphQL schema +struct DeploymentAggregationOutput { + time_bucket: String, // Output as String + subgraph: String, + gateway_id: String, + query_count: u64, + success_count: u64, + failure_count: u64, + avg_response_time_ms: f64, + total_fees_usd: f64, +} + +#[derive(Row, Deserialize, Debug, Clone)] +struct DeploymentAggregationRow { + time_bucket: u32, // Use u32 for Unix timestamp + subgraph: String, + gateway_id: String, + query_count: u64, + success_count: u64, + failure_count: u64, + avg_response_time_ms: f64, + total_fees_usd: f64, +} + +// --- Indexer Aggregation Structs --- + +#[derive(SimpleObject, Deserialize, Debug, Clone)] +#[graphql(name = "IndexerAggregation")] +struct IndexerAggregationOutput { + time_bucket: String, + indexer: String, + gateway_id: String, + query_count: u64, + success_count: u64, + failure_count: u64, + avg_indexer_response_time_ms: f64, + total_fee_grt: f64, + avg_seconds_behind: f64, + avg_blocks_behind: f64, +} + +#[derive(Row, Deserialize, Debug, Clone)] +struct IndexerAggregationRow { + time_bucket: u32, // Use u32 for Unix timestamp + indexer: String, + gateway_id: String, + query_count: u64, + success_count: u64, + failure_count: u64, + avg_indexer_response_time_ms: f64, + total_fee_grt: f64, + avg_seconds_behind: f64, + avg_blocks_behind: f64, +} + +// --- Allocation Aggregation Structs --- + +#[derive(Row, Deserialize, Debug, Clone)] +struct AllocationAggregationRow { + time_bucket: u32, + subgraph: String, + indexer: String, + gateway_id: String, + query_count: u64, + success_count: u64, + failure_count: u64, + avg_indexer_response_time_ms: f64, + total_fee_grt: f64, + avg_seconds_behind: f64, + avg_blocks_behind: f64, +} + +#[derive(SimpleObject, Debug, Clone)] +struct AllocationAggregationOutput { + time_bucket: String, // Formatted as RFC3339 + subgraph: String, + indexer: String, + gateway_id: String, + query_count: u64, + success_count: u64, + failure_count: u64, + avg_indexer_response_time_ms: f64, + total_fee_grt: f64, + avg_seconds_behind: f64, + avg_blocks_behind: f64, +} + +// --- Input Objects for Filtering --- + +#[derive(InputObject, Debug)] +struct TimeRangeInput { + /// Start time (inclusive), RFC3339 format (e.g., "2023-01-01T00:00:00Z") + from: String, + /// End time (exclusive), RFC3339 format (e.g., "2023-01-02T00:00:00Z") + to: String, +} + +#[derive(InputObject, Debug)] +struct AggregationFilterInput { + /// Optional: Filter by gateway ID + gateway_id: Option, + /// Optional: Filter by subgraph ID (deployment ID) + subgraph: Option, + /// Optional: Filter by indexer ID (HEX encoded) + indexer: Option, + /// Optional: Filter by allocation ID (HEX encoded) + allocation: Option, +} + +// --- GraphQL Query Root --- + struct QueryRoot; #[Object] impl QueryRoot { - /// Query for QoS Reports with complete fields + /// Query for raw QoS Reports (limited fields for now) async fn qos_reports( &self, - _ctx: &Context<'_>, - from: Option, - to: Option, + ctx: &Context<'_>, + time_range: TimeRangeInput, + #[graphql(desc = "Optional filters for the query")] filter: Option, + #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] limit: Option, - ) -> async_graphql::Result> { - // Configure client with proper settings and timeouts - let client = Client::default() - .with_url( - &std::env::var("CLICKHOUSE_URL").unwrap_or_else(|_| "http://localhost:8123".into()), - ) - .with_database(&std::env::var("CLICKHOUSE_DB").unwrap_or_else(|_| "default".into())) - .with_user(&std::env::var("CLICKHOUSE_USER").unwrap_or_else(|_| "graphql".into())) - .with_password( - &std::env::var("CLICKHOUSE_PASSWORD").unwrap_or_else(|_| "graphql_password".into()), - ); - - // First, check if there's any data using a simple count query - let count_query = "SELECT count() as count FROM qos_data"; - println!("Checking table count: {}", count_query); - - // Use the proper Row trait implementation for count - let count_result = client.query(count_query).fetch_one::().await; - - match count_result { - Ok(count_row) => { - println!("Total records in qos_data: {}", count_row.count); - if count_row.count == 0 { - return Ok(Vec::new()); - } + ) -> Result, String> { + let client = get_clickhouse_client()?; + + // Build WHERE clause + let mut conditions = Vec::new(); + let from_dt = DateTime::parse_from_rfc3339(&time_range.from) + .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? + .with_timezone(&Utc); + let to_dt = DateTime::parse_from_rfc3339(&time_range.to) + .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? + .with_timezone(&Utc); + + conditions.push(format!("event_time >= toDateTime({})", from_dt.timestamp())); + conditions.push(format!("event_time < toDateTime({})", to_dt.timestamp())); + + if let Some(f) = filter { + if let Some(gw) = &f.gateway_id { + conditions.push(format!("gateway_id = '{}'", gw)); } - Err(e) => { - println!("Error getting count: {}", e); - return Err(async_graphql::Error::new(format!("Database error: {}", e))); + if let Some(sg) = &f.subgraph { + // Handle nullable subgraph field correctly + conditions.push(format!("subgraph = '{}'", sg)); } + // Filtering by indexer/allocation on raw reports requires arrayExists/has, + // which can be slow. Skipping for now. + // if let Some(ix) = &f.indexer { + // conditions.push(format!("has(indexer_queries.indexer, '{}')", ix)); + // } + // if let Some(alloc) = &f.allocation { + // conditions.push(format!("has(indexer_queries.allocation, '{}')", alloc)); + // } } - // Build full query with all fields - let mut query = "SELECT - event_time, - gateway_id, - query_id, - api_key, - user_id, - receipt_signer, - subgraph, - result, - response_time_ms, - request_bytes, - response_bytes, - total_fees_usd - FROM qos_data" - .to_string(); - - // event_time, - // gateway_id, - // receipt_signer, - // query_id, - // api_key, - // user_id, - // subgraph, - // result, - // response_time_ms, - // request_bytes, - // response_bytes, - // total_fees_usd - - // Add date filtering if provided - if from.is_some() || to.is_some() { - query.push_str(" WHERE "); - - if let Some(from_val) = &from { - query.push_str(&format!("event_time >= toDateTime('{}')", from_val)); - if to.is_some() { - query.push_str(" AND "); - } + let where_clause = if conditions.is_empty() { + String::new() + } else { + format!("WHERE {}", conditions.join(" AND ")) + }; + let query_limit = limit.unwrap_or(1000).max(0).min(10000); // Apply limits + + let query = format!( + "SELECT event_time, gateway_id, receipt_signer, query_id, api_key, user_id, \ + subgraph, result, response_time_ms, request_bytes, response_bytes, total_fees_usd \ + FROM qos_data {} ORDER BY event_time DESC LIMIT {}", + where_clause, query_limit + ); + + println!("Executing query: {}", query); + + let rows = client + .query(&query) + .fetch_all::() + .await + .map_err(|e| format!("Database query failed: {}", e))?; + + // Convert to GraphQL output type + Ok(rows + .into_iter() + .map(|row| QosReport { + event_time: Utc + .timestamp_opt(row.event_time as i64, 0) + .single() + .map_or_else(|| "Invalid Timestamp".to_string(), |dt| dt.to_rfc3339()), + gateway_id: row.gateway_id, + receipt_signer: row.receipt_signer, + query_id: row.query_id, + api_key: row.api_key, + user_id: row.user_id, + subgraph: row.subgraph, + result: row.result, + response_time_ms: row.response_time_ms, + request_bytes: row.request_bytes, + response_bytes: row.response_bytes, + total_fees_usd: row.total_fees_usd, + }) + .collect()) + } + + /// Query for Deployment level aggregations + async fn deployment_aggregations( + &self, + ctx: &Context<'_>, + interval: AggregationInterval, + time_range: TimeRangeInput, + #[graphql(desc = "Optional filters for the query")] filter: Option, + #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] + limit: Option, + ) -> Result, String> { + let client = get_clickhouse_client()?; + let table_name = format!("agg_deployment_{}", interval.table_suffix()); + + // Build WHERE clause + let mut conditions = Vec::new(); + let from_dt = DateTime::parse_from_rfc3339(&time_range.from) + .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? + .with_timezone(&Utc); + let to_dt = DateTime::parse_from_rfc3339(&time_range.to) + .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? + .with_timezone(&Utc); + conditions.push(format!("time_bucket >= toDateTime({})", from_dt.timestamp())); + conditions.push(format!("time_bucket < toDateTime({})", to_dt.timestamp())); + + if let Some(f) = filter { + if let Some(gw) = &f.gateway_id { + conditions.push(format!("gateway_id = '{}'", gw)); + } + if let Some(sg) = &f.subgraph { + conditions.push(format!("subgraph = '{}'", sg)); } + // No indexer/allocation filters applicable here + } - if let Some(to_val) = &to { - query.push_str(&format!("event_time <= toDateTime('{}')", to_val)); + let where_clause = format!("WHERE {}", conditions.join(" AND ")); + let query_limit = limit.unwrap_or(1000).max(0).min(10000); + + // Query directly selects pre-aggregated columns, no further aggregation or GROUP BY + let query = format!( + "SELECT time_bucket, subgraph, gateway_id, \ + query_count, success_count, failure_count, \ + avg_response_time_ms, total_fees_usd \ + FROM {} {} \ + ORDER BY time_bucket DESC, subgraph, gateway_id \ + LIMIT {}", + table_name, where_clause, query_limit + ); + + println!("Executing query: {}", query); + + let rows = client + .query(&query) + .fetch_all::() // Uses existing struct + .await + .map_err(|e| format!("Database query failed: {}", e))?; + + // Map directly, row-by-row (mapping logic remains the same) + Ok(rows + .into_iter() + .map(|row| DeploymentAggregationOutput { + time_bucket: Utc + .timestamp_opt(row.time_bucket as i64, 0) + .single() + .map_or_else(|| "Invalid Timestamp".to_string(), |dt| dt.to_rfc3339()), + subgraph: row.subgraph, + gateway_id: row.gateway_id, + query_count: row.query_count, + success_count: row.success_count, + failure_count: row.failure_count, + avg_response_time_ms: row.avg_response_time_ms, + total_fees_usd: row.total_fees_usd, + }) + .collect()) + } + + /// Query for Indexer level aggregations + async fn indexer_aggregations( + &self, + ctx: &Context<'_>, + interval: AggregationInterval, + time_range: TimeRangeInput, + #[graphql(desc = "Optional filters for the query")] filter: Option, + #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] + limit: Option, + ) -> Result, String> { + let client = get_clickhouse_client()?; + let table_name = format!("agg_indexer_{}", interval.table_suffix()); + + // Build WHERE clause + let mut conditions = Vec::new(); + let from_dt = DateTime::parse_from_rfc3339(&time_range.from) + .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? + .with_timezone(&Utc); + let to_dt = DateTime::parse_from_rfc3339(&time_range.to) + .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? + .with_timezone(&Utc); + conditions.push(format!("time_bucket >= toDateTime({})", from_dt.timestamp())); + conditions.push(format!("time_bucket < toDateTime({})", to_dt.timestamp())); + + if let Some(f) = filter { + if let Some(gw) = &f.gateway_id { + conditions.push(format!("gateway_id = '{}'", gw)); + } + if let Some(ix) = &f.indexer { + conditions.push(format!("indexer = '{}'", ix)); } + // No subgraph/allocation filters applicable here } - // Use the provided limit or default to 100 - let limit_value = limit.unwrap_or(100).max(1).min(1000); - query.push_str(&format!(" ORDER BY event_time DESC LIMIT {}", limit_value)); + let where_clause = format!("WHERE {}", conditions.join(" AND ")); + let query_limit = limit.unwrap_or(1000).max(0).min(10000); + + // Query directly selects pre-aggregated columns, no further aggregation or GROUP BY + let query = format!( + "SELECT time_bucket, indexer, gateway_id, \ + query_count, success_count, failure_count, \ + avg_indexer_response_time_ms, total_fee_grt, \ + avg_seconds_behind, avg_blocks_behind \ + FROM {} {} \ + ORDER BY time_bucket DESC, indexer, gateway_id \ + LIMIT {}", + table_name, where_clause, query_limit + ); println!("Executing query: {}", query); - // Use fetch_all with the Row trait implementation - let rows_result = client.query(&query).fetch_all::().await; - - match rows_result { - Ok(rows) => { - println!("Query returned {} rows", rows.len()); - - // Convert to GraphQL output type - let reports: Vec = rows - .into_iter() - .map(|row| { - println!( - "Processing row: event_time={}, gateway_id={}, query_id={}", - row.event_time, row.gateway_id, row.query_id - ); - - QosReport { - event_time: row.event_time.to_string(), - gateway_id: row.gateway_id, - receipt_signer: row.receipt_signer, - query_id: row.query_id, - api_key: row.api_key, - user_id: row.user_id, - subgraph: row.subgraph, - result: row.result, - response_time_ms: row.response_time_ms, - request_bytes: row.request_bytes, - response_bytes: row.response_bytes, - total_fees_usd: row.total_fees_usd, - } - }) - .collect(); - - println!("Returning {} reports", reports.len()); - Ok(reports) + let rows = client + .query(&query) + .fetch_all::() // Uses existing struct + .await + .map_err(|e| format!("Database query failed: {}", e))?; + + // Map directly, row-by-row (mapping logic remains the same) + Ok(rows + .into_iter() + .map(|row| IndexerAggregationOutput { + time_bucket: Utc + .timestamp_opt(row.time_bucket as i64, 0) + .single() + .map_or_else(|| "Invalid Timestamp".to_string(), |dt| dt.to_rfc3339()), + indexer: row.indexer, + gateway_id: row.gateway_id, + query_count: row.query_count, + success_count: row.success_count, + failure_count: row.failure_count, + avg_indexer_response_time_ms: row.avg_indexer_response_time_ms, + total_fee_grt: row.total_fee_grt, + avg_seconds_behind: row.avg_seconds_behind, + avg_blocks_behind: row.avg_blocks_behind, + }) + .collect()) + } + + /// Query for Allocation level aggregations + async fn allocation_aggregations( + &self, + ctx: &Context<'_>, + interval: AggregationInterval, + time_range: TimeRangeInput, + #[graphql(desc = "Optional filters for the query")] filter: Option, + #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] + limit: Option, + ) -> Result, String> { + let client = get_clickhouse_client()?; + let table_name = format!("agg_allocation_{}", interval.table_suffix()); + + // Build WHERE clause + let mut conditions = Vec::new(); + let from_dt = DateTime::parse_from_rfc3339(&time_range.from) + .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? + .with_timezone(&Utc); + let to_dt = DateTime::parse_from_rfc3339(&time_range.to) + .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? + .with_timezone(&Utc); + conditions.push(format!("time_bucket >= toDateTime({})", from_dt.timestamp())); + conditions.push(format!("time_bucket < toDateTime({})", to_dt.timestamp())); + + if let Some(f) = filter { + if let Some(gw) = &f.gateway_id { + conditions.push(format!("gateway_id = '{}'", gw)); + } + if let Some(sg) = &f.subgraph { + conditions.push(format!("subgraph = '{}'", sg)); } - Err(err) => { - println!("Query failed: {}", err); - Err(async_graphql::Error::new(format!( - "Database error: {}", - err - ))) + if let Some(ix) = &f.indexer { + conditions.push(format!("indexer = '{}'", ix)); } } + + let where_clause = format!("WHERE {}", conditions.join(" AND ")); + let query_limit = limit.unwrap_or(1000).max(0).min(10000); + + // Query directly selects pre-aggregated columns, no further aggregation or GROUP BY + let query = format!( + "SELECT time_bucket, subgraph, indexer, gateway_id, \ + query_count, success_count, failure_count, \ + avg_indexer_response_time_ms, total_fee_grt, \ + avg_seconds_behind, avg_blocks_behind \ + FROM {} {} \ + ORDER BY time_bucket DESC, subgraph, indexer, gateway_id \ + LIMIT {}", + table_name, where_clause, query_limit + ); + + println!("Executing query: {}", query); + + let rows = client + .query(&query) + .fetch_all::() // Uses existing struct + .await + .map_err(|e| format!("Database query failed: {}", e))?; + + // Map directly, row-by-row (mapping logic remains the same) + Ok(rows + .into_iter() + .map(|row| AllocationAggregationOutput { // Uses existing struct + time_bucket: Utc + .timestamp_opt(row.time_bucket as i64, 0) + .single() + .map_or_else(|| "Invalid Timestamp".to_string(), |dt| dt.to_rfc3339()), + subgraph: row.subgraph, + indexer: row.indexer, + gateway_id: row.gateway_id, + query_count: row.query_count, + success_count: row.success_count, + failure_count: row.failure_count, + avg_indexer_response_time_ms: row.avg_indexer_response_time_ms, + total_fee_grt: row.total_fee_grt, + avg_seconds_behind: row.avg_seconds_behind, + avg_blocks_behind: row.avg_blocks_behind, + }) + .collect()) } } -/* -// Additional fields and nested data queries (commented out for debugging) -async fn fetch_nested_data(&self, client: &Client, query_id: &str) -> Result, String> { - // ... +// Helper function to create ClickHouse client +// Returns the configured client builder. Connection happens on first query. +fn get_clickhouse_client() -> Result { + Ok(Client::default() + .with_url(&env::var("CLICKHOUSE_URL").unwrap_or_else(|_| "http://localhost:8123".into())) + .with_database(&env::var("CLICKHOUSE_DB").unwrap_or_else(|_| "default".into())) + .with_user(&env::var("CLICKHOUSE_USER").unwrap_or_else(|_| "graphql".into())) + .with_password(&env::var("CLICKHOUSE_PASSWORD").unwrap_or_else(|_| "graphql_password".into()))) + // Remove .try_into() and .map_err() } -*/ - -type GQLSchema = Schema; -async fn graphql_handler(schema: web::Data, req: GraphQLRequest) -> GraphQLResponse { +// GraphQL handlers +async fn graphql_handler( + schema: web::Data>, + req: GraphQLRequest, +) -> GraphQLResponse { schema.execute(req.into_inner()).await.into() } async fn graphql_playground() -> Result { - let html = r#" - - - - GraphQL Playground - - - - - - -
- - -
Loading GraphQL Playground
-
- - - - "#; - Ok(HttpResponse::Ok() .content_type("text/html; charset=utf-8") - .body(html)) + .body(async_graphql::http::playground_source( + async_graphql::http::GraphQLPlaygroundConfig::new("/graphql"), + ))) } -// Health check endpoint for Docker +// Health check endpoint async fn health_check() -> HttpResponse { - HttpResponse::Ok().body("OK") + // Basic check: Can we create a client builder? + let client_result = get_clickhouse_client(); + if client_result.is_err() { + // Log the actual error if needed + eprintln!("Health check failed (client config): {:?}", client_result.err()); + return HttpResponse::InternalServerError().body("ClickHouse client configuration error"); + } + let client = client_result.unwrap(); // Safe unwrap after check above + + // More advanced check: Can we execute a simple query? + match client.query("SELECT 1").execute().await { // Use execute() for simple queries + Ok(_) => HttpResponse::Ok().body("OK"), + Err(e) => { + eprintln!("Health check failed (query execution): {}", e); // Log the error + HttpResponse::ServiceUnavailable().body(format!("ClickHouse connection error: {}", e)) + } + } } #[actix_web::main] async fn main() -> std::io::Result<()> { + // Initialize environment variables (e.g., from .env file if needed) + // dotenv::dotenv().ok(); + let schema = Schema::build(QueryRoot, EmptyMutation, EmptySubscription).finish(); println!("GraphQL playground: http://localhost:8000"); diff --git a/oracle/test-producer/Cargo.toml b/oracle/test-producer/Cargo.toml index 2af38da..cf33bea 100644 --- a/oracle/test-producer/Cargo.toml +++ b/oracle/test-producer/Cargo.toml @@ -12,6 +12,7 @@ uuid = { version = "1", features = ["v4"] } rand = "0.8" chrono = "0.4" hex = "0.4" +once_cell = "1.19" [build-dependencies] prost-build = "0.13.1" \ No newline at end of file diff --git a/oracle/test-producer/src/main.rs b/oracle/test-producer/src/main.rs index 8fae445..1d824a8 100644 --- a/oracle/test-producer/src/main.rs +++ b/oracle/test-producer/src/main.rs @@ -8,6 +8,8 @@ use rdkafka::config::ClientConfig; use rdkafka::producer::{FutureProducer, FutureRecord}; use rdkafka::util::get_rdkafka_version; use tokio::time; +use once_cell::sync::Lazy; +use rand::seq::SliceRandom; // Mock types to match the gateway's dependencies mod mock { @@ -220,60 +222,155 @@ fn random_hex_string(len: usize) -> String { hex::encode(random_bytes(len)) } +// --- Define Pool Sizes --- +const POOL_SIZE: usize = 30; // Number of unique IDs for each type + +// --- Define Static ID Pools (Place this before the random_* functions) --- + +static INDEXER_ID_POOL: Lazy> = Lazy::new(|| { + (0..POOL_SIZE) + .map(|_| { + let mut bytes = [0u8; 20]; + rand::thread_rng().fill(&mut bytes); + IndexerId(bytes) + }) + .collect() +}); + +static DEPLOYMENT_ID_POOL: Lazy> = Lazy::new(|| { + (0..POOL_SIZE) + .map(|_| { + let mut bytes = [0u8; 32]; + rand::thread_rng().fill(&mut bytes); + DeploymentId(bytes) + }) + .collect() +}); + +// Pool for Addresses (used by random_address and tap_signer) +static ADDRESS_POOL: Lazy> = Lazy::new(|| { + (0..POOL_SIZE) + .map(|_| { + let mut bytes = [0u8; 20]; + rand::thread_rng().fill(&mut bytes); + Address(bytes) + }) + .collect() +}); + +// Pool for Subgraph IDs (derived from Deployment pool for consistency) +static SUBGRAPH_ID_POOL: Lazy> = Lazy::new(|| { + DEPLOYMENT_ID_POOL + .iter() + .map(|d| SubgraphId(format!("Qm{}", hex::encode(d.0)))) + .collect() +}); + +// Pools for String-based IDs +static GATEWAY_ID_POOL: Lazy> = Lazy::new(|| { + (0..POOL_SIZE) + .map(|i| format!("gateway-pool-{}", i)) + .collect() +}); + +static API_KEY_POOL: Lazy> = Lazy::new(|| { + (0..POOL_SIZE) + .map(|i| random_utf8_string(&format!("api-key-pool-{}", i), 10)) + .collect() +}); + +static USER_POOL: Lazy> = Lazy::new(|| { + (0..POOL_SIZE) + .map(|i| random_utf8_string(&format!("user-pool-{}", i), 10)) + .collect() +}); + +// --- Update the random ID generation functions --- + +// Replace the existing random_address function (lines 223-227) fn random_address() -> Address { - let mut bytes = [0u8; 20]; - rand::thread_rng().fill(&mut bytes); - Address(bytes) + // Choose a random Address from the pre-generated pool + *ADDRESS_POOL.choose(&mut rand::thread_rng()).unwrap() } +// Replace the existing random_deployment_id function (lines 229-233) fn random_deployment_id() -> DeploymentId { - let mut bytes = [0u8; 32]; - rand::thread_rng().fill(&mut bytes); - DeploymentId(bytes) + // Choose a random DeploymentId from the pre-generated pool + *DEPLOYMENT_ID_POOL.choose(&mut rand::thread_rng()).unwrap() } +// Replace the existing random_indexer_id function (lines 235-239) fn random_indexer_id() -> IndexerId { - let mut bytes = [0u8; 20]; - rand::thread_rng().fill(&mut bytes); - IndexerId(bytes) + // Choose a random IndexerId from the pre-generated pool + *INDEXER_ID_POOL.choose(&mut rand::thread_rng()).unwrap() +} + +// --- Add/Update functions for String-based IDs --- + +// Add this function to select from the Subgraph ID pool +fn random_subgraph_id() -> SubgraphId { + SUBGRAPH_ID_POOL.choose(&mut rand::thread_rng()).unwrap().clone() +} + +// Add this function to select from the Gateway ID pool +fn random_gateway_id() -> String { + GATEWAY_ID_POOL.choose(&mut rand::thread_rng()).unwrap().clone() } +// Add this function to select from the API Key pool +fn random_api_key() -> String { + API_KEY_POOL.choose(&mut rand::thread_rng()).unwrap().clone() +} + +// Add this function to select from the User pool +fn random_user() -> String { + USER_POOL.choose(&mut rand::thread_rng()).unwrap().clone() +} + +// --- Update generate_random_client_request (lines 241-314) --- +// Modify calls inside this function to use the new pool-based random functions + fn generate_random_client_request() -> ClientRequest { let mut rng = rand::thread_rng(); - // Generate between 1 and 4 indexer requests let num_indexers = rng.gen_range(1..5); let mut indexer_requests = Vec::with_capacity(num_indexers); + // --- Key Change: Pick IDs *once* per ClientRequest where appropriate --- + let deployment_id = random_deployment_id(); // Pick one deployment from pool + let subgraph_id = random_subgraph_id(); // Pick one subgraph from pool (could also derive from deployment) + let api_key = random_api_key(); // Pick one api key from pool + let user = random_user(); // Pick one user from pool + + let subgraph_chain = if rng.gen_bool(0.8) { "mainnet".to_string() } else { "goerli".to_string() }; + for _ in 0..num_indexers { - let success = rng.gen_bool(0.9); // 90% success rate + let success = rng.gen_bool(0.9); + // --- Use pool-based functions for indexer-specific IDs --- let indexer_id = random_indexer_id(); - let allocation = random_address(); + let allocation_id = random_address(); // Allocation can vary per indexer request let result = if success { Ok(IndexerResponse { errors: vec![] }) } else { - // Use safe error types with valid UTF-8 Err(IndexerError::NetworkError( "Error processing request".to_string(), )) }; indexer_requests.push(IndexerRequest { - indexer: indexer_id, - deployment: random_deployment_id(), - // Ensure URL is valid UTF-8 - url: format!( - "https://api.thegraph.com/subgraphs/id/Qm{}", - random_hex_string(10) + indexer: indexer_id, // From pool + deployment: deployment_id, // Use the *same* deployment for all indexers in this request + url: format!( // Use consistent subgraph ID for URL + "https://api.thegraph.com/subgraphs/id/{}", + subgraph_id.0 // Access the inner String ), receipt: Receipt::new( - allocation, + allocation_id, // From pool rng.gen_range(100_000_000_000_000..1_000_000_000_000_000), ), - // Ensure chain name is valid UTF-8 - subgraph_chain: "mainnet".to_string(), + subgraph_chain: subgraph_chain.clone(), result, response_time_ms: rng.gen_range(50..2000) as u16, seconds_behind: rng.gen_range(0..100), @@ -281,28 +378,20 @@ fn generate_random_client_request() -> ClientRequest { }); } - let success = rng.gen_bool(0.95); // 95% success rate for overall query + let success = rng.gen_bool(0.95); ClientRequest { - // Use random hex for IDs (guaranteed valid UTF-8) - id: random_hex_string(16), + id: random_hex_string(16), // Keep request ID unique response_time_ms: rng.gen_range(50..5000) as u16, result: if success { Ok(()) } else { - // Use predefined error message to ensure UTF-8 Err(Error::QueryFailed("Query validation error".to_string())) }, - // Use random alphanumeric strings with prefixes - api_key: random_utf8_string("api", 16), - user: random_utf8_string("user", 16), - subgraph: if rng.gen_bool(0.9) { - // Guarantee valid UTF-8 for subgraph ID - Some(SubgraphId(format!("Qm{}", random_hex_string(10)))) - } else { - None - }, - grt_per_usd: rng.gen_range(0.05..0.2), // Realistic GRT/USD price range + api_key: api_key, // Use the chosen API key + user: user, // Use the chosen user + subgraph: if rng.gen_bool(0.9) { Some(subgraph_id) } else { None }, // Use the chosen subgraph ID + grt_per_usd: rng.gen_range(0.05..0.2), indexer_requests, request_bytes: rng.gen_range(100..5000), response_bytes: if rng.gen_bool(0.95) { @@ -316,9 +405,13 @@ fn generate_random_client_request() -> ClientRequest { // Function that matches the logic in gateway's report function but without attestation fn encode_client_request( client_request: ClientRequest, - tap_signer: Address, - graph_env: String, + // tap_signer: Address, // Remove this parameter + // graph_env: String, // Remove this parameter ) -> Vec { + // --- Key Change: Select gateway and signer from pools here --- + let gateway_id = random_gateway_id(); // Select from pool + let tap_signer = random_address(); // Select from pool + let indexer_queries = client_request .indexer_requests .iter() @@ -359,7 +452,7 @@ fn encode_client_request( let total_fees_usd: f64 = total_fees_grt / client_request.grt_per_usd; let client_query_msg = ClientQueryProtobuf { - gateway_id: graph_env, + gateway_id, receipt_signer: tap_signer.to_vec(), query_id: client_request.id, api_key: client_request.api_key, @@ -422,19 +515,14 @@ async fn main() { // Make sure this matches the topic name in ClickHouse Kafka engine let topic = "gateway_qos_topic"; - let mut counter = 0; - - // Create a fixed tap_signer and graph_env to use for all messages - let tap_signer = random_address(); - // Ensure graph_env is valid UTF-8 - let graph_env = "testnet-gateway".to_string(); + let mut counter: u64 = 0; loop { // Generate a random client request let client_request = generate_random_client_request(); // Encode using the gateway's exact logic - let encoded_message = encode_client_request(client_request, tap_signer, graph_env.clone()); + let encoded_message = encode_client_request(client_request); println!("Encoded message: {} ", hex::encode(&encoded_message)); println!("Encoded message size: {} bytes", encoded_message.len()); From 4f963a1e43f6e3eb852c61b000fced4040a2346f Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Wed, 2 Apr 2025 03:49:55 -0300 Subject: [PATCH 11/20] feat: added all aggregation fields --- oracle/clickhouse/tables.sql | 1176 ++++++++++++++++++++++++++------ oracle/graphql-api/src/main.rs | 461 ++++++++++--- 2 files changed, 1330 insertions(+), 307 deletions(-) diff --git a/oracle/clickhouse/tables.sql b/oracle/clickhouse/tables.sql index d0de63b..004a341 100644 --- a/oracle/clickhouse/tables.sql +++ b/oracle/clickhouse/tables.sql @@ -98,28 +98,22 @@ CREATE TABLE IF NOT EXISTS qos_data api_key String, user_id String, subgraph Nullable(String), - result String, + result String, -- 'success' or other status response_time_ms UInt32, request_bytes UInt32, response_bytes Nullable(UInt32), total_fees_usd Float64, - indexer_queries Nested( - indexer String, -- HEX encoded - deployment String, -- HEX encoded - allocation String, -- HEX encoded - indexed_chain String, - url String, - fee_grt Float64, + indexer_queries Nested ( + indexer String, + result String, -- 'success' or other status response_time_ms UInt32, + fee_grt Float64, seconds_behind UInt32, - result String, - indexer_errors String, blocks_behind UInt64 ) -) ENGINE = MergeTree() -ORDER BY (event_time, gateway_id, query_id) -PARTITION BY toYYYYMMDD(event_time) -TTL event_time + INTERVAL 30 DAY; -- Keep processed data longer, e.g., 30 days +) ENGINE = MergeTree +PARTITION BY toYYYYMM(event_time) +ORDER BY (gateway_id, event_time); -- Create the materialized view that processes and transforms the data into qos_data CREATE MATERIALIZED VIEW IF NOT EXISTS qos_data_mv TO qos_data AS @@ -158,299 +152,1029 @@ FROM kafka_qos_data; -- Deployment Level Aggregations (by subgraph) -- ---------------------------------------- --- 5-Minute Deployment Aggregations +-- 5-Minute Deployment Aggregations (Table Definition) CREATE TABLE IF NOT EXISTS agg_deployment_5min ( time_bucket DateTime, - subgraph String, -- Deployment ID is the subgraph ID + subgraph String, gateway_id String, - query_count UInt64, - success_count UInt64, - failure_count UInt64, - avg_response_time_ms Float64, - total_fees_usd Float64 -) ENGINE = SummingMergeTree() -- Use SummingMergeTree for counts/sums + query_count AggregateFunction(count), + success_count AggregateFunction(countIf, UInt8), + failure_count AggregateFunction(countIf, UInt8), + total_fees_usd AggregateFunction(sum, Float64), + avg_response_time_ms AggregateFunction(avg, UInt32), + max_response_time_ms AggregateFunction(max, UInt32), + quantiles_response_time_ms AggregateFunction(quantiles(0.90, 0.99), UInt32), + stddev_response_time_ms AggregateFunction(stddevSamp, UInt32), + avg_fee_usd AggregateFunction(avg, Float64), + max_fee_usd AggregateFunction(max, Float64), + quantiles_fee_usd AggregateFunction(quantiles(0.90, 0.99), Float64), + stddev_fee_usd AggregateFunction(stddevSamp, Float64) +) +ENGINE = AggregatingMergeTree ORDER BY (time_bucket, subgraph, gateway_id); +-- Materialized View for 5-Minute Deployment Aggregations (Corrected SELECT with ALL -State functions and alias 'qd') CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_deployment_5min TO agg_deployment_5min AS SELECT - toStartOfFiveMinutes(event_time) AS time_bucket, - assumeNotNull(subgraph) AS subgraph, -- Treat Nullable subgraph as String for grouping - gateway_id, - count() AS query_count, - countIf(result = 'success') AS success_count, - countIf(result != 'success') AS failure_count, - avg(response_time_ms) AS avg_response_time_ms, - sum(total_fees_usd) AS total_fees_usd -FROM qos_data -WHERE subgraph IS NOT NULL -- Only aggregate for requests with a subgraph -GROUP BY time_bucket, subgraph, gateway_id; - --- Hourly Deployment Aggregations + toStartOfFiveMinute(qd.event_time) AS time_bucket, + qd.subgraph, + qd.gateway_id, + countState() AS query_count, + countIfState(qd.result = 'success') AS success_count, + countIfState(qd.result != 'success') AS failure_count, + sumState(qd.total_fees_usd) AS total_fees_usd, + avgState(qd.response_time_ms) AS avg_response_time_ms, + maxState(qd.response_time_ms) AS max_response_time_ms, + quantilesState(0.90, 0.99)(qd.response_time_ms) AS quantiles_response_time_ms, + stddevSampState(qd.response_time_ms) AS stddev_response_time_ms, + avgState(qd.total_fees_usd) AS avg_fee_usd, + maxState(qd.total_fees_usd) AS max_fee_usd, + quantilesState(0.90, 0.99)(qd.total_fees_usd) AS quantiles_fee_usd, + stddevSampState(qd.total_fees_usd) AS stddev_fee_usd +FROM qos_data AS qd -- Added alias +WHERE qd.subgraph IS NOT NULL +GROUP BY time_bucket, qd.subgraph, qd.gateway_id; -- Use alias in GROUP BY + +-- Hourly Deployment Aggregations (Table Definition) CREATE TABLE IF NOT EXISTS agg_deployment_hourly ( time_bucket DateTime, subgraph String, gateway_id String, - query_count UInt64, - success_count UInt64, - failure_count UInt64, - avg_response_time_ms Float64, - total_fees_usd Float64 -) ENGINE = SummingMergeTree() + query_count AggregateFunction(count), + success_count AggregateFunction(countIf, UInt8), + failure_count AggregateFunction(countIf, UInt8), + total_fees_usd AggregateFunction(sum, Float64), + avg_response_time_ms AggregateFunction(avg, UInt32), + max_response_time_ms AggregateFunction(max, UInt32), + quantiles_response_time_ms AggregateFunction(quantiles(0.90, 0.99), UInt32), + stddev_response_time_ms AggregateFunction(stddevSamp, UInt32), + avg_fee_usd AggregateFunction(avg, Float64), + max_fee_usd AggregateFunction(max, Float64), + quantiles_fee_usd AggregateFunction(quantiles(0.90, 0.99), Float64), + stddev_fee_usd AggregateFunction(stddevSamp, Float64) +) +ENGINE = AggregatingMergeTree ORDER BY (time_bucket, subgraph, gateway_id); +-- Materialized View for Hourly Deployment Aggregations (Corrected SELECT with ALL -State functions and alias 'qd') CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_deployment_hourly TO agg_deployment_hourly AS SELECT - toStartOfHour(event_time) AS time_bucket, - assumeNotNull(subgraph) AS subgraph, - gateway_id, - count() AS query_count, - countIf(result = 'success') AS success_count, - countIf(result != 'success') AS failure_count, - avg(response_time_ms) AS avg_response_time_ms, - sum(total_fees_usd) AS total_fees_usd -FROM qos_data -WHERE subgraph IS NOT NULL -GROUP BY time_bucket, subgraph, gateway_id; - --- Daily Deployment Aggregations + toStartOfHour(qd.event_time) AS time_bucket, -- Changed time function + qd.subgraph, + qd.gateway_id, + countState() AS query_count, + countIfState(qd.result = 'success') AS success_count, + countIfState(qd.result != 'success') AS failure_count, + sumState(qd.total_fees_usd) AS total_fees_usd, + avgState(qd.response_time_ms) AS avg_response_time_ms, + maxState(qd.response_time_ms) AS max_response_time_ms, + quantilesState(0.90, 0.99)(qd.response_time_ms) AS quantiles_response_time_ms, + stddevSampState(qd.response_time_ms) AS stddev_response_time_ms, + avgState(qd.total_fees_usd) AS avg_fee_usd, + maxState(qd.total_fees_usd) AS max_fee_usd, + quantilesState(0.90, 0.99)(qd.total_fees_usd) AS quantiles_fee_usd, + stddevSampState(qd.total_fees_usd) AS stddev_fee_usd +FROM qos_data AS qd -- Added alias +WHERE qd.subgraph IS NOT NULL +GROUP BY time_bucket, qd.subgraph, qd.gateway_id; -- Use alias in GROUP BY + +-- Daily Deployment Aggregations (Table Definition) CREATE TABLE IF NOT EXISTS agg_deployment_daily ( time_bucket DateTime, subgraph String, gateway_id String, - query_count UInt64, - success_count UInt64, - failure_count UInt64, - avg_response_time_ms Float64, - total_fees_usd Float64 -) ENGINE = SummingMergeTree() + query_count AggregateFunction(count), + success_count AggregateFunction(countIf, UInt8), + failure_count AggregateFunction(countIf, UInt8), + total_fees_usd AggregateFunction(sum, Float64), + avg_response_time_ms AggregateFunction(avg, UInt32), + max_response_time_ms AggregateFunction(max, UInt32), + quantiles_response_time_ms AggregateFunction(quantiles(0.90, 0.99), UInt32), + stddev_response_time_ms AggregateFunction(stddevSamp, UInt32), + avg_fee_usd AggregateFunction(avg, Float64), + max_fee_usd AggregateFunction(max, Float64), + quantiles_fee_usd AggregateFunction(quantiles(0.90, 0.99), Float64), + stddev_fee_usd AggregateFunction(stddevSamp, Float64) +) +ENGINE = AggregatingMergeTree ORDER BY (time_bucket, subgraph, gateway_id); +-- Materialized View for Daily Deployment Aggregations (Corrected SELECT with ALL -State functions and alias 'qd') CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_deployment_daily TO agg_deployment_daily AS SELECT - toStartOfDay(event_time) AS time_bucket, - assumeNotNull(subgraph) AS subgraph, - gateway_id, - count() AS query_count, - countIf(result = 'success') AS success_count, - countIf(result != 'success') AS failure_count, - avg(response_time_ms) AS avg_response_time_ms, - sum(total_fees_usd) AS total_fees_usd -FROM qos_data -WHERE subgraph IS NOT NULL -GROUP BY time_bucket, subgraph, gateway_id; + toStartOfDay(qd.event_time) AS time_bucket, -- Changed time function + qd.subgraph, + qd.gateway_id, + countState() AS query_count, + countIfState(qd.result = 'success') AS success_count, + countIfState(qd.result != 'success') AS failure_count, + sumState(qd.total_fees_usd) AS total_fees_usd, + avgState(qd.response_time_ms) AS avg_response_time_ms, + maxState(qd.response_time_ms) AS max_response_time_ms, + quantilesState(0.90, 0.99)(qd.response_time_ms) AS quantiles_response_time_ms, + stddevSampState(qd.response_time_ms) AS stddev_response_time_ms, + avgState(qd.total_fees_usd) AS avg_fee_usd, + maxState(qd.total_fees_usd) AS max_fee_usd, + quantilesState(0.90, 0.99)(qd.total_fees_usd) AS quantiles_fee_usd, + stddevSampState(qd.total_fees_usd) AS stddev_fee_usd +FROM qos_data AS qd -- Added alias +WHERE qd.subgraph IS NOT NULL +GROUP BY time_bucket, qd.subgraph, qd.gateway_id; -- Use alias in GROUP BY -- ---------------------------------------- --- Indexer Level Aggregations (by indexer_queries.indexer) +-- Allocation Level Aggregations (by subgraph + indexer_queries.indexer) -- ---------------------------------------- --- 5-Minute Indexer Aggregations -CREATE TABLE IF NOT EXISTS agg_indexer_5min +-- 5-Minute Allocation Aggregations (Table Definition) +CREATE TABLE IF NOT EXISTS agg_allocation_5min ( time_bucket DateTime, - indexer String, -- HEX encoded indexer ID + subgraph String, + indexer String, gateway_id String, - query_count UInt64, - success_count UInt64, - failure_count UInt64, - avg_indexer_response_time_ms Float64, - total_fee_grt Float64, - avg_seconds_behind Float64, - avg_blocks_behind Float64 -) ENGINE = SummingMergeTree() -ORDER BY (time_bucket, indexer, gateway_id); + query_count AggregateFunction(count), + success_count AggregateFunction(countIf, UInt8), + failure_count AggregateFunction(countIf, UInt8), + total_fee_grt AggregateFunction(sum, Float64), + avg_indexer_response_time_ms AggregateFunction(avg, UInt32), + max_indexer_response_time_ms AggregateFunction(max, UInt32), + quantiles_indexer_response_time_ms AggregateFunction(quantiles(0.90, 0.99), UInt32), + stddev_indexer_response_time_ms AggregateFunction(stddevSamp, UInt32), + avg_fee_grt AggregateFunction(avg, Float64), + max_fee_grt AggregateFunction(max, Float64), + quantiles_fee_grt AggregateFunction(quantiles(0.90, 0.99), Float64), + stddev_fee_grt AggregateFunction(stddevSamp, Float64), + avg_seconds_behind AggregateFunction(avg, UInt32), + max_seconds_behind AggregateFunction(max, UInt32), + quantiles_seconds_behind AggregateFunction(quantiles(0.90, 0.99), UInt32), + stddev_seconds_behind AggregateFunction(stddevSamp, UInt32), + avg_blocks_behind AggregateFunction(avg, UInt64), + max_blocks_behind AggregateFunction(max, UInt64), + quantiles_blocks_behind AggregateFunction(quantiles(0.90, 0.99), UInt64), + stddev_blocks_behind AggregateFunction(stddevSamp, UInt64) +) +ENGINE = AggregatingMergeTree +ORDER BY (time_bucket, subgraph, indexer, gateway_id); -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_indexer_5min TO agg_indexer_5min AS +-- Materialized View for 5-Minute Allocation Aggregations (Corrected SELECT with ALL -State functions and aliases 'qd', 'iq') +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_allocation_5min TO agg_allocation_5min AS SELECT - toStartOfFiveMinutes(event_time) AS time_bucket, + toStartOfFiveMinute(qd.event_time) AS time_bucket, + qd.subgraph, iq.indexer AS indexer, - gateway_id, - count() AS query_count, - countIf(iq.result = 'success') AS success_count, - countIf(iq.result != 'success') AS failure_count, - avg(iq.response_time_ms) AS avg_indexer_response_time_ms, - sum(iq.fee_grt) AS total_fee_grt, - avg(iq.seconds_behind) AS avg_seconds_behind, - avg(iq.blocks_behind) AS avg_blocks_behind -FROM qos_data -ARRAY JOIN indexer_queries AS iq -- Flatten the nested structure -GROUP BY time_bucket, indexer, gateway_id; - --- Hourly Indexer Aggregations -CREATE TABLE IF NOT EXISTS agg_indexer_hourly + qd.gateway_id, + countState() AS query_count, + countIfState(iq.result = 'success') AS success_count, + countIfState(iq.result != 'success') AS failure_count, + sumState(iq.fee_grt) AS total_fee_grt, + avgState(iq.response_time_ms) AS avg_indexer_response_time_ms, + maxState(iq.response_time_ms) AS max_indexer_response_time_ms, + quantilesState(0.90, 0.99)(iq.response_time_ms) AS quantiles_indexer_response_time_ms, + stddevSampState(iq.response_time_ms) AS stddev_indexer_response_time_ms, + avgState(iq.fee_grt) AS avg_fee_grt, + maxState(iq.fee_grt) AS max_fee_grt, + quantilesState(0.90, 0.99)(iq.fee_grt) AS quantiles_fee_grt, + stddevSampState(iq.fee_grt) AS stddev_fee_grt, + avgState(iq.seconds_behind) AS avg_seconds_behind, + maxState(iq.seconds_behind) AS max_seconds_behind, + quantilesState(0.90, 0.99)(iq.seconds_behind) AS quantiles_seconds_behind, + stddevSampState(iq.seconds_behind) AS stddev_seconds_behind, + avgState(iq.blocks_behind) AS avg_blocks_behind, + maxState(iq.blocks_behind) AS max_blocks_behind, + quantilesState(0.90, 0.99)(iq.blocks_behind) AS quantiles_blocks_behind, + stddevSampState(iq.blocks_behind) AS stddev_blocks_behind +FROM qos_data AS qd -- Added alias +ARRAY JOIN indexer_queries AS iq +WHERE qd.subgraph IS NOT NULL AND length(iq.indexer) > 0 +GROUP BY time_bucket, qd.subgraph, indexer, qd.gateway_id; -- Use aliases in GROUP BY + +-- Hourly Allocation Aggregations (Table Definition) +CREATE TABLE IF NOT EXISTS agg_allocation_hourly ( time_bucket DateTime, + subgraph String, indexer String, gateway_id String, - query_count UInt64, - success_count UInt64, - failure_count UInt64, - avg_indexer_response_time_ms Float64, - total_fee_grt Float64, - avg_seconds_behind Float64, - avg_blocks_behind Float64 -) ENGINE = SummingMergeTree() -ORDER BY (time_bucket, indexer, gateway_id); + query_count AggregateFunction(count), + success_count AggregateFunction(countIf, UInt8), + failure_count AggregateFunction(countIf, UInt8), + total_fee_grt AggregateFunction(sum, Float64), + avg_indexer_response_time_ms AggregateFunction(avg, UInt32), + max_indexer_response_time_ms AggregateFunction(max, UInt32), + quantiles_indexer_response_time_ms AggregateFunction(quantiles(0.90, 0.99), UInt32), + stddev_indexer_response_time_ms AggregateFunction(stddevSamp, UInt32), + avg_fee_grt AggregateFunction(avg, Float64), + max_fee_grt AggregateFunction(max, Float64), + quantiles_fee_grt AggregateFunction(quantiles(0.90, 0.99), Float64), + stddev_fee_grt AggregateFunction(stddevSamp, Float64), + avg_seconds_behind AggregateFunction(avg, UInt32), + max_seconds_behind AggregateFunction(max, UInt32), + quantiles_seconds_behind AggregateFunction(quantiles(0.90, 0.99), UInt32), + stddev_seconds_behind AggregateFunction(stddevSamp, UInt32), + avg_blocks_behind AggregateFunction(avg, UInt64), + max_blocks_behind AggregateFunction(max, UInt64), + quantiles_blocks_behind AggregateFunction(quantiles(0.90, 0.99), UInt64), + stddev_blocks_behind AggregateFunction(stddevSamp, UInt64) +) +ENGINE = AggregatingMergeTree +ORDER BY (time_bucket, subgraph, indexer, gateway_id); -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_indexer_hourly TO agg_indexer_hourly AS +-- Materialized View for Hourly Allocation Aggregations (Corrected SELECT with ALL -State functions and aliases 'qd', 'iq') +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_allocation_hourly TO agg_allocation_hourly AS SELECT - toStartOfHour(event_time) AS time_bucket, + toStartOfHour(qd.event_time) AS time_bucket, -- Changed time function + qd.subgraph, iq.indexer AS indexer, - gateway_id, - count() AS query_count, - countIf(iq.result = 'success') AS success_count, - countIf(iq.result != 'success') AS failure_count, - avg(iq.response_time_ms) AS avg_indexer_response_time_ms, - sum(iq.fee_grt) AS total_fee_grt, - avg(iq.seconds_behind) AS avg_seconds_behind, - avg(iq.blocks_behind) AS avg_blocks_behind -FROM qos_data + qd.gateway_id, + countState() AS query_count, + countIfState(iq.result = 'success') AS success_count, + countIfState(iq.result != 'success') AS failure_count, + sumState(iq.fee_grt) AS total_fee_grt, + avgState(iq.response_time_ms) AS avg_indexer_response_time_ms, + maxState(iq.response_time_ms) AS max_indexer_response_time_ms, + quantilesState(0.90, 0.99)(iq.response_time_ms) AS quantiles_indexer_response_time_ms, + stddevSampState(iq.response_time_ms) AS stddev_indexer_response_time_ms, + avgState(iq.fee_grt) AS avg_fee_grt, + maxState(iq.fee_grt) AS max_fee_grt, + quantilesState(0.90, 0.99)(iq.fee_grt) AS quantiles_fee_grt, + stddevSampState(iq.fee_grt) AS stddev_fee_grt, + avgState(iq.seconds_behind) AS avg_seconds_behind, + maxState(iq.seconds_behind) AS max_seconds_behind, + quantilesState(0.90, 0.99)(iq.seconds_behind) AS quantiles_seconds_behind, + stddevSampState(iq.seconds_behind) AS stddev_seconds_behind, + avgState(iq.blocks_behind) AS avg_blocks_behind, + maxState(iq.blocks_behind) AS max_blocks_behind, + quantilesState(0.90, 0.99)(iq.blocks_behind) AS quantiles_blocks_behind, + stddevSampState(iq.blocks_behind) AS stddev_blocks_behind +FROM qos_data AS qd -- Added alias ARRAY JOIN indexer_queries AS iq -GROUP BY time_bucket, indexer, gateway_id; +WHERE qd.subgraph IS NOT NULL AND length(iq.indexer) > 0 +GROUP BY time_bucket, qd.subgraph, indexer, qd.gateway_id; -- Use aliases in GROUP BY --- Daily Indexer Aggregations -CREATE TABLE IF NOT EXISTS agg_indexer_daily +-- Daily Allocation Aggregations (Table Definition) +CREATE TABLE IF NOT EXISTS agg_allocation_daily ( time_bucket DateTime, + subgraph String, indexer String, gateway_id String, - query_count UInt64, - success_count UInt64, - failure_count UInt64, - avg_indexer_response_time_ms Float64, - total_fee_grt Float64, - avg_seconds_behind Float64, - avg_blocks_behind Float64 -) ENGINE = SummingMergeTree() -ORDER BY (time_bucket, indexer, gateway_id); + query_count AggregateFunction(count), + success_count AggregateFunction(countIf, UInt8), + failure_count AggregateFunction(countIf, UInt8), + total_fee_grt AggregateFunction(sum, Float64), + avg_indexer_response_time_ms AggregateFunction(avg, UInt32), + max_indexer_response_time_ms AggregateFunction(max, UInt32), + quantiles_indexer_response_time_ms AggregateFunction(quantiles(0.90, 0.99), UInt32), + stddev_indexer_response_time_ms AggregateFunction(stddevSamp, UInt32), + avg_fee_grt AggregateFunction(avg, Float64), + max_fee_grt AggregateFunction(max, Float64), + quantiles_fee_grt AggregateFunction(quantiles(0.90, 0.99), Float64), + stddev_fee_grt AggregateFunction(stddevSamp, Float64), + avg_seconds_behind AggregateFunction(avg, UInt32), + max_seconds_behind AggregateFunction(max, UInt32), + quantiles_seconds_behind AggregateFunction(quantiles(0.90, 0.99), UInt32), + stddev_seconds_behind AggregateFunction(stddevSamp, UInt32), + avg_blocks_behind AggregateFunction(avg, UInt64), + max_blocks_behind AggregateFunction(max, UInt64), + quantiles_blocks_behind AggregateFunction(quantiles(0.90, 0.99), UInt64), + stddev_blocks_behind AggregateFunction(stddevSamp, UInt64) +) +ENGINE = AggregatingMergeTree +ORDER BY (time_bucket, subgraph, indexer, gateway_id); -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_indexer_daily TO agg_indexer_daily AS +-- Materialized View for Daily Allocation Aggregations (Corrected SELECT with ALL -State functions and aliases 'qd', 'iq') +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_allocation_daily TO agg_allocation_daily AS SELECT - toStartOfDay(event_time) AS time_bucket, + toStartOfDay(qd.event_time) AS time_bucket, -- Changed time function + qd.subgraph, iq.indexer AS indexer, - gateway_id, - count() AS query_count, - countIf(iq.result = 'success') AS success_count, - countIf(iq.result != 'success') AS failure_count, - avg(iq.response_time_ms) AS avg_indexer_response_time_ms, - sum(iq.fee_grt) AS total_fee_grt, - avg(iq.seconds_behind) AS avg_seconds_behind, - avg(iq.blocks_behind) AS avg_blocks_behind -FROM qos_data + qd.gateway_id, + countState() AS query_count, + countIfState(iq.result = 'success') AS success_count, + countIfState(iq.result != 'success') AS failure_count, + sumState(iq.fee_grt) AS total_fee_grt, + avgState(iq.response_time_ms) AS avg_indexer_response_time_ms, + maxState(iq.response_time_ms) AS max_indexer_response_time_ms, + quantilesState(0.90, 0.99)(iq.response_time_ms) AS quantiles_indexer_response_time_ms, + stddevSampState(iq.response_time_ms) AS stddev_indexer_response_time_ms, + avgState(iq.fee_grt) AS avg_fee_grt, + maxState(iq.fee_grt) AS max_fee_grt, + quantilesState(0.90, 0.99)(iq.fee_grt) AS quantiles_fee_grt, + stddevSampState(iq.fee_grt) AS stddev_fee_grt, + avgState(iq.seconds_behind) AS avg_seconds_behind, + maxState(iq.seconds_behind) AS max_seconds_behind, + quantilesState(0.90, 0.99)(iq.seconds_behind) AS quantiles_seconds_behind, + stddevSampState(iq.seconds_behind) AS stddev_seconds_behind, + avgState(iq.blocks_behind) AS avg_blocks_behind, + maxState(iq.blocks_behind) AS max_blocks_behind, + quantilesState(0.90, 0.99)(iq.blocks_behind) AS quantiles_blocks_behind, + stddevSampState(iq.blocks_behind) AS stddev_blocks_behind +FROM qos_data AS qd -- Added alias ARRAY JOIN indexer_queries AS iq -GROUP BY time_bucket, indexer, gateway_id; +WHERE qd.subgraph IS NOT NULL AND length(iq.indexer) > 0 +GROUP BY time_bucket, qd.subgraph, indexer, qd.gateway_id; -- Use aliases in GROUP BY -- ---------------------------------------- --- Allocation Level Aggregations (by subgraph + indexer_queries.indexer) --- NOTE: The name "Allocation Aggregation" here refers to the grouping level, --- not the specific `allocation` ID field, which is removed. +-- Indexer Level Aggregations (by indexer_queries.indexer) -- ---------------------------------------- --- 5-Minute Allocation Aggregations -CREATE TABLE IF NOT EXISTS agg_allocation_5min +-- 5-Minute Indexer Aggregations (Table Definition) +CREATE TABLE IF NOT EXISTS agg_indexer_5min ( time_bucket DateTime, - subgraph String, -- Deployment ID - indexer String, -- HEX encoded indexer ID + indexer String, gateway_id String, - query_count UInt64, - success_count UInt64, - failure_count UInt64, - avg_indexer_response_time_ms Float64, - total_fee_grt Float64, - avg_seconds_behind Float64, - avg_blocks_behind Float64 -) ENGINE = SummingMergeTree() -ORDER BY (time_bucket, subgraph, indexer, gateway_id); + query_count AggregateFunction(count), + success_count AggregateFunction(countIf, UInt8), + failure_count AggregateFunction(countIf, UInt8), + total_fee_grt AggregateFunction(sum, Float64), + avg_indexer_response_time_ms AggregateFunction(avg, UInt32), + max_indexer_response_time_ms AggregateFunction(max, UInt32), + quantiles_indexer_response_time_ms AggregateFunction(quantiles(0.90, 0.99), UInt32), + stddev_indexer_response_time_ms AggregateFunction(stddevSamp, UInt32), + avg_fee_grt AggregateFunction(avg, Float64), + max_fee_grt AggregateFunction(max, Float64), + quantiles_fee_grt AggregateFunction(quantiles(0.90, 0.99), Float64), + stddev_fee_grt AggregateFunction(stddevSamp, Float64), + avg_seconds_behind AggregateFunction(avg, UInt32), + max_seconds_behind AggregateFunction(max, UInt32), + quantiles_seconds_behind AggregateFunction(quantiles(0.90, 0.99), UInt32), + stddev_seconds_behind AggregateFunction(stddevSamp, UInt32), + avg_blocks_behind AggregateFunction(avg, UInt64), + max_blocks_behind AggregateFunction(max, UInt64), + quantiles_blocks_behind AggregateFunction(quantiles(0.90, 0.99), UInt64), + stddev_blocks_behind AggregateFunction(stddevSamp, UInt64) +) +ENGINE = AggregatingMergeTree +ORDER BY (time_bucket, indexer, gateway_id); -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_allocation_5min TO agg_allocation_5min AS +-- Materialized View for 5-Minute Indexer Aggregations (Corrected SELECT with ALL -State functions and aliases 'qd', 'iq') +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_indexer_5min TO agg_indexer_5min AS SELECT - toStartOfFiveMinutes(event_time) AS time_bucket, - assumeNotNull(subgraph) AS subgraph, + toStartOfFiveMinute(qd.event_time) AS time_bucket, iq.indexer AS indexer, - gateway_id, - count() AS query_count, - countIf(iq.result = 'success') AS success_count, - countIf(iq.result != 'success') AS failure_count, - avg(iq.response_time_ms) AS avg_indexer_response_time_ms, - sum(iq.fee_grt) AS total_fee_grt, - avg(iq.seconds_behind) AS avg_seconds_behind, - avg(iq.blocks_behind) AS avg_blocks_behind -FROM qos_data + qd.gateway_id, + countState() AS query_count, + countIfState(iq.result = 'success') AS success_count, + countIfState(iq.result != 'success') AS failure_count, + sumState(iq.fee_grt) AS total_fee_grt, + avgState(iq.response_time_ms) AS avg_indexer_response_time_ms, + maxState(iq.response_time_ms) AS max_indexer_response_time_ms, + quantilesState(0.90, 0.99)(iq.response_time_ms) AS quantiles_indexer_response_time_ms, + stddevSampState(iq.response_time_ms) AS stddev_indexer_response_time_ms, + avgState(iq.fee_grt) AS avg_fee_grt, + maxState(iq.fee_grt) AS max_fee_grt, + quantilesState(0.90, 0.99)(iq.fee_grt) AS quantiles_fee_grt, + stddevSampState(iq.fee_grt) AS stddev_fee_grt, + avgState(iq.seconds_behind) AS avg_seconds_behind, + maxState(iq.seconds_behind) AS max_seconds_behind, + quantilesState(0.90, 0.99)(iq.seconds_behind) AS quantiles_seconds_behind, + stddevSampState(iq.seconds_behind) AS stddev_seconds_behind, + avgState(iq.blocks_behind) AS avg_blocks_behind, + maxState(iq.blocks_behind) AS max_blocks_behind, + quantilesState(0.90, 0.99)(iq.blocks_behind) AS quantiles_blocks_behind, + stddevSampState(iq.blocks_behind) AS stddev_blocks_behind +FROM qos_data AS qd -- Added alias ARRAY JOIN indexer_queries AS iq -WHERE subgraph IS NOT NULL -- Only aggregate for requests with a subgraph -GROUP BY time_bucket, subgraph, indexer, gateway_id; +WHERE length(iq.indexer) > 0 +GROUP BY time_bucket, indexer, qd.gateway_id; -- Use aliases in GROUP BY --- Hourly Allocation Aggregations -CREATE TABLE IF NOT EXISTS agg_allocation_hourly +-- Hourly Indexer Aggregations (Table Definition) +CREATE TABLE IF NOT EXISTS agg_indexer_hourly ( time_bucket DateTime, - subgraph String, indexer String, gateway_id String, - query_count UInt64, - success_count UInt64, - failure_count UInt64, - avg_indexer_response_time_ms Float64, - total_fee_grt Float64, - avg_seconds_behind Float64, - avg_blocks_behind Float64 -) ENGINE = SummingMergeTree() -ORDER BY (time_bucket, subgraph, indexer, gateway_id); + query_count AggregateFunction(count), + success_count AggregateFunction(countIf, UInt8), + failure_count AggregateFunction(countIf, UInt8), + total_fee_grt AggregateFunction(sum, Float64), + avg_indexer_response_time_ms AggregateFunction(avg, UInt32), + max_indexer_response_time_ms AggregateFunction(max, UInt32), + quantiles_indexer_response_time_ms AggregateFunction(quantiles(0.90, 0.99), UInt32), + stddev_indexer_response_time_ms AggregateFunction(stddevSamp, UInt32), + avg_fee_grt AggregateFunction(avg, Float64), + max_fee_grt AggregateFunction(max, Float64), + quantiles_fee_grt AggregateFunction(quantiles(0.90, 0.99), Float64), + stddev_fee_grt AggregateFunction(stddevSamp, Float64), + avg_seconds_behind AggregateFunction(avg, UInt32), + max_seconds_behind AggregateFunction(max, UInt32), + quantiles_seconds_behind AggregateFunction(quantiles(0.90, 0.99), UInt32), + stddev_seconds_behind AggregateFunction(stddevSamp, UInt32), + avg_blocks_behind AggregateFunction(avg, UInt64), + max_blocks_behind AggregateFunction(max, UInt64), + quantiles_blocks_behind AggregateFunction(quantiles(0.90, 0.99), UInt64), + stddev_blocks_behind AggregateFunction(stddevSamp, UInt64) +) +ENGINE = AggregatingMergeTree +ORDER BY (time_bucket, indexer, gateway_id); -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_allocation_hourly TO agg_allocation_hourly AS +-- Materialized View for Hourly Indexer Aggregations (Corrected SELECT with ALL -State functions and aliases 'qd', 'iq') +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_indexer_hourly TO agg_indexer_hourly AS SELECT - toStartOfHour(event_time) AS time_bucket, - assumeNotNull(subgraph) AS subgraph, + toStartOfHour(qd.event_time) AS time_bucket, -- Changed time function iq.indexer AS indexer, - gateway_id, - count() AS query_count, - countIf(iq.result = 'success') AS success_count, - countIf(iq.result != 'success') AS failure_count, - avg(iq.response_time_ms) AS avg_indexer_response_time_ms, - sum(iq.fee_grt) AS total_fee_grt, - avg(iq.seconds_behind) AS avg_seconds_behind, - avg(iq.blocks_behind) AS avg_blocks_behind -FROM qos_data + qd.gateway_id, + countState() AS query_count, + countIfState(iq.result = 'success') AS success_count, + countIfState(iq.result != 'success') AS failure_count, + sumState(iq.fee_grt) AS total_fee_grt, + avgState(iq.response_time_ms) AS avg_indexer_response_time_ms, + maxState(iq.response_time_ms) AS max_indexer_response_time_ms, + quantilesState(0.90, 0.99)(iq.response_time_ms) AS quantiles_indexer_response_time_ms, + stddevSampState(iq.response_time_ms) AS stddev_indexer_response_time_ms, + avgState(iq.fee_grt) AS avg_fee_grt, + maxState(iq.fee_grt) AS max_fee_grt, + quantilesState(0.90, 0.99)(iq.fee_grt) AS quantiles_fee_grt, + stddevSampState(iq.fee_grt) AS stddev_fee_grt, + avgState(iq.seconds_behind) AS avg_seconds_behind, + maxState(iq.seconds_behind) AS max_seconds_behind, + quantilesState(0.90, 0.99)(iq.seconds_behind) AS quantiles_seconds_behind, + stddevSampState(iq.seconds_behind) AS stddev_seconds_behind, + avgState(iq.blocks_behind) AS avg_blocks_behind, + maxState(iq.blocks_behind) AS max_blocks_behind, + quantilesState(0.90, 0.99)(iq.blocks_behind) AS quantiles_blocks_behind, + stddevSampState(iq.blocks_behind) AS stddev_blocks_behind +FROM qos_data AS qd -- Added alias ARRAY JOIN indexer_queries AS iq -WHERE subgraph IS NOT NULL -GROUP BY time_bucket, subgraph, indexer, gateway_id; +WHERE length(iq.indexer) > 0 +GROUP BY time_bucket, indexer, qd.gateway_id; -- Use aliases in GROUP BY --- Daily Allocation Aggregations -CREATE TABLE IF NOT EXISTS agg_allocation_daily +-- Daily Indexer Aggregations (Table Definition) +CREATE TABLE IF NOT EXISTS agg_indexer_daily ( time_bucket DateTime, - subgraph String, indexer String, gateway_id String, - query_count UInt64, - success_count UInt64, - failure_count UInt64, - avg_indexer_response_time_ms Float64, - total_fee_grt Float64, - avg_seconds_behind Float64, - avg_blocks_behind Float64 -) ENGINE = SummingMergeTree() -ORDER BY (time_bucket, subgraph, indexer, gateway_id); + query_count AggregateFunction(count), + success_count AggregateFunction(countIf, UInt8), + failure_count AggregateFunction(countIf, UInt8), + total_fee_grt AggregateFunction(sum, Float64), + avg_indexer_response_time_ms AggregateFunction(avg, UInt32), + max_indexer_response_time_ms AggregateFunction(max, UInt32), + quantiles_indexer_response_time_ms AggregateFunction(quantiles(0.90, 0.99), UInt32), + stddev_indexer_response_time_ms AggregateFunction(stddevSamp, UInt32), + avg_fee_grt AggregateFunction(avg, Float64), + max_fee_grt AggregateFunction(max, Float64), + quantiles_fee_grt AggregateFunction(quantiles(0.90, 0.99), Float64), + stddev_fee_grt AggregateFunction(stddevSamp, Float64), + avg_seconds_behind AggregateFunction(avg, UInt32), + max_seconds_behind AggregateFunction(max, UInt32), + quantiles_seconds_behind AggregateFunction(quantiles(0.90, 0.99), UInt32), + stddev_seconds_behind AggregateFunction(stddevSamp, UInt32), + avg_blocks_behind AggregateFunction(avg, UInt64), + max_blocks_behind AggregateFunction(max, UInt64), + quantiles_blocks_behind AggregateFunction(quantiles(0.90, 0.99), UInt64), + stddev_blocks_behind AggregateFunction(stddevSamp, UInt64) +) +ENGINE = AggregatingMergeTree +ORDER BY (time_bucket, indexer, gateway_id); -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_allocation_daily TO agg_allocation_daily AS +-- Materialized View for Daily Indexer Aggregations (Corrected SELECT with ALL -State functions and aliases 'qd', 'iq') +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_indexer_daily TO agg_indexer_daily AS SELECT - toStartOfDay(event_time) AS time_bucket, - assumeNotNull(subgraph) AS subgraph, + toStartOfDay(qd.event_time) AS time_bucket, -- Changed time function iq.indexer AS indexer, - gateway_id, - count() AS query_count, - countIf(iq.result = 'success') AS success_count, - countIf(iq.result != 'success') AS failure_count, - avg(iq.response_time_ms) AS avg_indexer_response_time_ms, - sum(iq.fee_grt) AS total_fee_grt, - avg(iq.seconds_behind) AS avg_seconds_behind, - avg(iq.blocks_behind) AS avg_blocks_behind -FROM qos_data + qd.gateway_id, + countState() AS query_count, + countIfState(iq.result = 'success') AS success_count, + countIfState(iq.result != 'success') AS failure_count, + sumState(iq.fee_grt) AS total_fee_grt, + avgState(iq.response_time_ms) AS avg_indexer_response_time_ms, + maxState(iq.response_time_ms) AS max_indexer_response_time_ms, + quantilesState(0.90, 0.99)(iq.response_time_ms) AS quantiles_indexer_response_time_ms, + stddevSampState(iq.response_time_ms) AS stddev_indexer_response_time_ms, + avgState(iq.fee_grt) AS avg_fee_grt, + maxState(iq.fee_grt) AS max_fee_grt, + quantilesState(0.90, 0.99)(iq.fee_grt) AS quantiles_fee_grt, + stddevSampState(iq.fee_grt) AS stddev_fee_grt, + avgState(iq.seconds_behind) AS avg_seconds_behind, + maxState(iq.seconds_behind) AS max_seconds_behind, + quantilesState(0.90, 0.99)(iq.seconds_behind) AS quantiles_seconds_behind, + stddevSampState(iq.seconds_behind) AS stddev_seconds_behind, + avgState(iq.blocks_behind) AS avg_blocks_behind, + maxState(iq.blocks_behind) AS max_blocks_behind, + quantilesState(0.90, 0.99)(iq.blocks_behind) AS quantiles_blocks_behind, + stddevSampState(iq.blocks_behind) AS stddev_blocks_behind +FROM qos_data AS qd -- Added alias ARRAY JOIN indexer_queries AS iq -WHERE subgraph IS NOT NULL -GROUP BY time_bucket, subgraph, indexer, gateway_id; +WHERE length(iq.indexer) > 0 +GROUP BY time_bucket, indexer, qd.gateway_id; -- Use aliases in GROUP BY + +-- ---------------------------------------- +-- Final Aggregation Views (Using -Merge with CTEs) +-- ---------------------------------------- + +-- View for Final 5-Minute Deployment Aggregations (Using CTE) +CREATE VIEW IF NOT EXISTS view_agg_deployment_5min AS +WITH AggregatedValues AS ( + -- Step 1: Perform all merges and grouping + SELECT + time_bucket, + subgraph, + gateway_id, + countMerge(query_count) AS final_query_count, + countIfMerge(success_count) AS final_success_count, + countIfMerge(failure_count) AS final_failure_count, + avgMerge(avg_response_time_ms) AS final_avg_response_time_ms, + maxMerge(max_response_time_ms) AS final_max_response_time_ms, + quantilesMerge(0.90, 0.99)(quantiles_response_time_ms) AS final_quantiles_response_time_ms, -- Keep array + stddevSampMerge(stddev_response_time_ms) AS final_stddev_response_time_ms, + sumMerge(total_fees_usd) AS final_total_fees_usd, + avgMerge(avg_fee_usd) AS final_avg_fee_usd, + maxMerge(max_fee_usd) AS final_max_fee_usd, + quantilesMerge(0.90, 0.99)(quantiles_fee_usd) AS final_quantiles_fee_usd, -- Keep array + stddevSampMerge(stddev_fee_usd) AS final_stddev_fee_usd + FROM agg_deployment_5min + GROUP BY -- Group by dimensions only + time_bucket, + subgraph, + gateway_id +) +-- Step 2: Select from CTE, extract percentiles, calculate proportion +SELECT + time_bucket, + subgraph, + gateway_id, + final_query_count AS query_count, + final_success_count AS success_count, + final_failure_count AS failure_count, + final_avg_response_time_ms AS avg_response_time_ms, + final_max_response_time_ms AS max_response_time_ms, + final_quantiles_response_time_ms[1] AS p90_response_time_ms, -- Extract p90 + final_quantiles_response_time_ms[2] AS p99_response_time_ms, -- Extract p99 + final_stddev_response_time_ms AS stddev_response_time_ms, + final_total_fees_usd AS total_fees_usd, + final_avg_fee_usd AS avg_fee_usd, + final_max_fee_usd AS max_fee_usd, + final_quantiles_fee_usd[1] AS p90_fee_usd, -- Extract p90 + final_quantiles_fee_usd[2] AS p99_fee_usd, -- Extract p99 + final_stddev_fee_usd AS stddev_fee_usd, + -- Calculate success proportion using aliases from CTE + if(final_query_count = 0, 0.0, final_success_count / final_query_count) AS success_proportion +FROM AggregatedValues; + +-- View for Final Hourly Deployment Aggregations (Using CTE) +CREATE VIEW IF NOT EXISTS view_agg_deployment_hourly AS +WITH AggregatedValues AS ( + SELECT + time_bucket, + subgraph, + gateway_id, + countMerge(query_count) AS final_query_count, + countIfMerge(success_count) AS final_success_count, + countIfMerge(failure_count) AS final_failure_count, + avgMerge(avg_response_time_ms) AS final_avg_response_time_ms, + maxMerge(max_response_time_ms) AS final_max_response_time_ms, + quantilesMerge(0.90, 0.99)(quantiles_response_time_ms) AS final_quantiles_response_time_ms, + stddevSampMerge(stddev_response_time_ms) AS final_stddev_response_time_ms, + sumMerge(total_fees_usd) AS final_total_fees_usd, + avgMerge(avg_fee_usd) AS final_avg_fee_usd, + maxMerge(max_fee_usd) AS final_max_fee_usd, + quantilesMerge(0.90, 0.99)(quantiles_fee_usd) AS final_quantiles_fee_usd, + stddevSampMerge(stddev_fee_usd) AS final_stddev_fee_usd + FROM agg_deployment_hourly + GROUP BY time_bucket, subgraph, gateway_id +) +SELECT + time_bucket, + subgraph, + gateway_id, + final_query_count AS query_count, + final_success_count AS success_count, + final_failure_count AS failure_count, + final_avg_response_time_ms AS avg_response_time_ms, + final_max_response_time_ms AS max_response_time_ms, + final_quantiles_response_time_ms[1] AS p90_response_time_ms, + final_quantiles_response_time_ms[2] AS p99_response_time_ms, + final_stddev_response_time_ms AS stddev_response_time_ms, + final_total_fees_usd AS total_fees_usd, + final_avg_fee_usd AS avg_fee_usd, + final_max_fee_usd AS max_fee_usd, + final_quantiles_fee_usd[1] AS p90_fee_usd, + final_quantiles_fee_usd[2] AS p99_fee_usd, + final_stddev_fee_usd AS stddev_fee_usd, + if(final_query_count = 0, 0.0, final_success_count / final_query_count) AS success_proportion +FROM AggregatedValues; + +-- View for Final Daily Deployment Aggregations (Using CTE) +CREATE VIEW IF NOT EXISTS view_agg_deployment_daily AS +WITH AggregatedValues AS ( + SELECT + time_bucket, + subgraph, + gateway_id, + countMerge(query_count) AS final_query_count, + countIfMerge(success_count) AS final_success_count, + countIfMerge(failure_count) AS final_failure_count, + avgMerge(avg_response_time_ms) AS final_avg_response_time_ms, + maxMerge(max_response_time_ms) AS final_max_response_time_ms, + quantilesMerge(0.90, 0.99)(quantiles_response_time_ms) AS final_quantiles_response_time_ms, + stddevSampMerge(stddev_response_time_ms) AS final_stddev_response_time_ms, + sumMerge(total_fees_usd) AS final_total_fees_usd, + avgMerge(avg_fee_usd) AS final_avg_fee_usd, + maxMerge(max_fee_usd) AS final_max_fee_usd, + quantilesMerge(0.90, 0.99)(quantiles_fee_usd) AS final_quantiles_fee_usd, + stddevSampMerge(stddev_fee_usd) AS final_stddev_fee_usd + FROM agg_deployment_daily + GROUP BY time_bucket, subgraph, gateway_id +) +SELECT + time_bucket, + subgraph, + gateway_id, + final_query_count AS query_count, + final_success_count AS success_count, + final_failure_count AS failure_count, + final_avg_response_time_ms AS avg_response_time_ms, + final_max_response_time_ms AS max_response_time_ms, + final_quantiles_response_time_ms[1] AS p90_response_time_ms, + final_quantiles_response_time_ms[2] AS p99_response_time_ms, + final_stddev_response_time_ms AS stddev_response_time_ms, + final_total_fees_usd AS total_fees_usd, + final_avg_fee_usd AS avg_fee_usd, + final_max_fee_usd AS max_fee_usd, + final_quantiles_fee_usd[1] AS p90_fee_usd, + final_quantiles_fee_usd[2] AS p99_fee_usd, + final_stddev_fee_usd AS stddev_fee_usd, + if(final_query_count = 0, 0.0, final_success_count / final_query_count) AS success_proportion +FROM AggregatedValues; + + +-- View for Final 5-Minute Allocation Aggregations (Using CTE) +CREATE VIEW IF NOT EXISTS view_agg_allocation_5min AS +WITH AggregatedValues AS ( + SELECT + time_bucket, + subgraph, + indexer, + gateway_id, + countMerge(query_count) AS final_query_count, + countIfMerge(success_count) AS final_success_count, + countIfMerge(failure_count) AS final_failure_count, + avgMerge(avg_indexer_response_time_ms) AS final_avg_indexer_response_time_ms, + maxMerge(max_indexer_response_time_ms) AS final_max_indexer_response_time_ms, + quantilesMerge(0.90, 0.99)(quantiles_indexer_response_time_ms) AS final_quantiles_indexer_response_time_ms, + stddevSampMerge(stddev_indexer_response_time_ms) AS final_stddev_indexer_response_time_ms, + sumMerge(total_fee_grt) AS final_total_fee_grt, + avgMerge(avg_fee_grt) AS final_avg_fee_grt, + maxMerge(max_fee_grt) AS final_max_fee_grt, + quantilesMerge(0.90, 0.99)(quantiles_fee_grt) AS final_quantiles_fee_grt, + stddevSampMerge(stddev_fee_grt) AS final_stddev_fee_grt, + avgMerge(avg_seconds_behind) AS final_avg_seconds_behind, + maxMerge(max_seconds_behind) AS final_max_seconds_behind, + quantilesMerge(0.90, 0.99)(quantiles_seconds_behind) AS final_quantiles_seconds_behind, + stddevSampMerge(stddev_seconds_behind) AS final_stddev_seconds_behind, + avgMerge(avg_blocks_behind) AS final_avg_blocks_behind, + maxMerge(max_blocks_behind) AS final_max_blocks_behind, + quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, + stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind + FROM agg_allocation_5min + GROUP BY time_bucket, subgraph, indexer, gateway_id +) +SELECT + time_bucket, + subgraph, + indexer, + gateway_id, + final_query_count AS query_count, + final_success_count AS success_count, + final_failure_count AS failure_count, + final_avg_indexer_response_time_ms AS avg_indexer_response_time_ms, + final_max_indexer_response_time_ms AS max_indexer_response_time_ms, + final_quantiles_indexer_response_time_ms[1] AS p90_indexer_response_time_ms, + final_quantiles_indexer_response_time_ms[2] AS p99_indexer_response_time_ms, + final_stddev_indexer_response_time_ms AS stddev_indexer_response_time_ms, + final_total_fee_grt AS total_fee_grt, + final_avg_fee_grt AS avg_fee_grt, + final_max_fee_grt AS max_fee_grt, + final_quantiles_fee_grt[1] AS p90_fee_grt, + final_quantiles_fee_grt[2] AS p99_fee_grt, + final_stddev_fee_grt AS stddev_fee_grt, + final_avg_seconds_behind AS avg_seconds_behind, + final_max_seconds_behind AS max_seconds_behind, + final_quantiles_seconds_behind[1] AS p90_seconds_behind, + final_quantiles_seconds_behind[2] AS p99_seconds_behind, + final_stddev_seconds_behind AS stddev_seconds_behind, + final_avg_blocks_behind AS avg_blocks_behind, + final_max_blocks_behind AS max_blocks_behind, + final_quantiles_blocks_behind[1] AS p90_blocks_behind, + final_quantiles_blocks_behind[2] AS p99_blocks_behind, + final_stddev_blocks_behind AS stddev_blocks_behind, + if(final_query_count = 0, 0.0, final_success_count / final_query_count) AS success_proportion +FROM AggregatedValues; + +-- View for Final Hourly Allocation Aggregations (Using CTE) +CREATE VIEW IF NOT EXISTS view_agg_allocation_hourly AS +WITH AggregatedValues AS ( + SELECT + time_bucket, + subgraph, + indexer, + gateway_id, + countMerge(query_count) AS final_query_count, + countIfMerge(success_count) AS final_success_count, + countIfMerge(failure_count) AS final_failure_count, + avgMerge(avg_indexer_response_time_ms) AS final_avg_indexer_response_time_ms, + maxMerge(max_indexer_response_time_ms) AS final_max_indexer_response_time_ms, + quantilesMerge(0.90, 0.99)(quantiles_indexer_response_time_ms) AS final_quantiles_indexer_response_time_ms, + stddevSampMerge(stddev_indexer_response_time_ms) AS final_stddev_indexer_response_time_ms, + sumMerge(total_fee_grt) AS final_total_fee_grt, + avgMerge(avg_fee_grt) AS final_avg_fee_grt, + maxMerge(max_fee_grt) AS final_max_fee_grt, + quantilesMerge(0.90, 0.99)(quantiles_fee_grt) AS final_quantiles_fee_grt, + stddevSampMerge(stddev_fee_grt) AS final_stddev_fee_grt, + avgMerge(avg_seconds_behind) AS final_avg_seconds_behind, + maxMerge(max_seconds_behind) AS final_max_seconds_behind, + quantilesMerge(0.90, 0.99)(quantiles_seconds_behind) AS final_quantiles_seconds_behind, + stddevSampMerge(stddev_seconds_behind) AS final_stddev_seconds_behind, + avgMerge(avg_blocks_behind) AS final_avg_blocks_behind, + maxMerge(max_blocks_behind) AS final_max_blocks_behind, + quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, + stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind + FROM agg_allocation_hourly + GROUP BY time_bucket, subgraph, indexer, gateway_id +) +SELECT + time_bucket, + subgraph, + indexer, + gateway_id, + final_query_count AS query_count, + final_success_count AS success_count, + final_failure_count AS failure_count, + final_avg_indexer_response_time_ms AS avg_indexer_response_time_ms, + final_max_indexer_response_time_ms AS max_indexer_response_time_ms, + final_quantiles_indexer_response_time_ms[1] AS p90_indexer_response_time_ms, + final_quantiles_indexer_response_time_ms[2] AS p99_indexer_response_time_ms, + final_stddev_indexer_response_time_ms AS stddev_indexer_response_time_ms, + final_total_fee_grt AS total_fee_grt, + final_avg_fee_grt AS avg_fee_grt, + final_max_fee_grt AS max_fee_grt, + final_quantiles_fee_grt[1] AS p90_fee_grt, + final_quantiles_fee_grt[2] AS p99_fee_grt, + final_stddev_fee_grt AS stddev_fee_grt, + final_avg_seconds_behind AS avg_seconds_behind, + final_max_seconds_behind AS max_seconds_behind, + final_quantiles_seconds_behind[1] AS p90_seconds_behind, + final_quantiles_seconds_behind[2] AS p99_seconds_behind, + final_stddev_seconds_behind AS stddev_seconds_behind, + final_avg_blocks_behind AS avg_blocks_behind, + final_max_blocks_behind AS max_blocks_behind, + final_quantiles_blocks_behind[1] AS p90_blocks_behind, + final_quantiles_blocks_behind[2] AS p99_blocks_behind, + final_stddev_blocks_behind AS stddev_blocks_behind, + if(final_query_count = 0, 0.0, final_success_count / final_query_count) AS success_proportion +FROM AggregatedValues; + +-- View for Final Daily Allocation Aggregations (Using CTE) +CREATE VIEW IF NOT EXISTS view_agg_allocation_daily AS +WITH AggregatedValues AS ( + SELECT + time_bucket, + subgraph, + indexer, + gateway_id, + countMerge(query_count) AS final_query_count, + countIfMerge(success_count) AS final_success_count, + countIfMerge(failure_count) AS final_failure_count, + avgMerge(avg_indexer_response_time_ms) AS final_avg_indexer_response_time_ms, + maxMerge(max_indexer_response_time_ms) AS final_max_indexer_response_time_ms, + quantilesMerge(0.90, 0.99)(quantiles_indexer_response_time_ms) AS final_quantiles_indexer_response_time_ms, + stddevSampMerge(stddev_indexer_response_time_ms) AS final_stddev_indexer_response_time_ms, + sumMerge(total_fee_grt) AS final_total_fee_grt, + avgMerge(avg_fee_grt) AS final_avg_fee_grt, + maxMerge(max_fee_grt) AS final_max_fee_grt, + quantilesMerge(0.90, 0.99)(quantiles_fee_grt) AS final_quantiles_fee_grt, + stddevSampMerge(stddev_fee_grt) AS final_stddev_fee_grt, + avgMerge(avg_seconds_behind) AS final_avg_seconds_behind, + maxMerge(max_seconds_behind) AS final_max_seconds_behind, + quantilesMerge(0.90, 0.99)(quantiles_seconds_behind) AS final_quantiles_seconds_behind, + stddevSampMerge(stddev_seconds_behind) AS final_stddev_seconds_behind, + avgMerge(avg_blocks_behind) AS final_avg_blocks_behind, + maxMerge(max_blocks_behind) AS final_max_blocks_behind, + quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, + stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind + FROM agg_allocation_daily + GROUP BY time_bucket, subgraph, indexer, gateway_id +) +SELECT + time_bucket, + subgraph, + indexer, + gateway_id, + final_query_count AS query_count, + final_success_count AS success_count, + final_failure_count AS failure_count, + final_avg_indexer_response_time_ms AS avg_indexer_response_time_ms, + final_max_indexer_response_time_ms AS max_indexer_response_time_ms, + final_quantiles_indexer_response_time_ms[1] AS p90_indexer_response_time_ms, + final_quantiles_indexer_response_time_ms[2] AS p99_indexer_response_time_ms, + final_stddev_indexer_response_time_ms AS stddev_indexer_response_time_ms, + final_total_fee_grt AS total_fee_grt, + final_avg_fee_grt AS avg_fee_grt, + final_max_fee_grt AS max_fee_grt, + final_quantiles_fee_grt[1] AS p90_fee_grt, + final_quantiles_fee_grt[2] AS p99_fee_grt, + final_stddev_fee_grt AS stddev_fee_grt, + final_avg_seconds_behind AS avg_seconds_behind, + final_max_seconds_behind AS max_seconds_behind, + final_quantiles_seconds_behind[1] AS p90_seconds_behind, + final_quantiles_seconds_behind[2] AS p99_seconds_behind, + final_stddev_seconds_behind AS stddev_seconds_behind, + final_avg_blocks_behind AS avg_blocks_behind, + final_max_blocks_behind AS max_blocks_behind, + final_quantiles_blocks_behind[1] AS p90_blocks_behind, + final_quantiles_blocks_behind[2] AS p99_blocks_behind, + final_stddev_blocks_behind AS stddev_blocks_behind, + if(final_query_count = 0, 0.0, final_success_count / final_query_count) AS success_proportion +FROM AggregatedValues; + + +-- View for Final 5-Minute Indexer Aggregations (Using CTE) +CREATE VIEW IF NOT EXISTS view_agg_indexer_5min AS +WITH AggregatedValues AS ( + SELECT + time_bucket, + indexer, + gateway_id, + countMerge(query_count) AS final_query_count, + countIfMerge(success_count) AS final_success_count, + countIfMerge(failure_count) AS final_failure_count, + avgMerge(avg_indexer_response_time_ms) AS final_avg_indexer_response_time_ms, + maxMerge(max_indexer_response_time_ms) AS final_max_indexer_response_time_ms, + quantilesMerge(0.90, 0.99)(quantiles_indexer_response_time_ms) AS final_quantiles_indexer_response_time_ms, + stddevSampMerge(stddev_indexer_response_time_ms) AS final_stddev_indexer_response_time_ms, + sumMerge(total_fee_grt) AS final_total_fee_grt, + avgMerge(avg_fee_grt) AS final_avg_fee_grt, + maxMerge(max_fee_grt) AS final_max_fee_grt, + quantilesMerge(0.90, 0.99)(quantiles_fee_grt) AS final_quantiles_fee_grt, + stddevSampMerge(stddev_fee_grt) AS final_stddev_fee_grt, + avgMerge(avg_seconds_behind) AS final_avg_seconds_behind, + maxMerge(max_seconds_behind) AS final_max_seconds_behind, + quantilesMerge(0.90, 0.99)(quantiles_seconds_behind) AS final_quantiles_seconds_behind, + stddevSampMerge(stddev_seconds_behind) AS final_stddev_seconds_behind, + avgMerge(avg_blocks_behind) AS final_avg_blocks_behind, + maxMerge(max_blocks_behind) AS final_max_blocks_behind, + quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, + stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind + FROM agg_indexer_5min + GROUP BY time_bucket, indexer, gateway_id +) +SELECT + time_bucket, + indexer, + gateway_id, + final_query_count AS query_count, + final_success_count AS success_count, + final_failure_count AS failure_count, + final_avg_indexer_response_time_ms AS avg_indexer_response_time_ms, + final_max_indexer_response_time_ms AS max_indexer_response_time_ms, + final_quantiles_indexer_response_time_ms[1] AS p90_indexer_response_time_ms, + final_quantiles_indexer_response_time_ms[2] AS p99_indexer_response_time_ms, + final_stddev_indexer_response_time_ms AS stddev_indexer_response_time_ms, + final_total_fee_grt AS total_fee_grt, + final_avg_fee_grt AS avg_fee_grt, + final_max_fee_grt AS max_fee_grt, + final_quantiles_fee_grt[1] AS p90_fee_grt, + final_quantiles_fee_grt[2] AS p99_fee_grt, + final_stddev_fee_grt AS stddev_fee_grt, + final_avg_seconds_behind AS avg_seconds_behind, + final_max_seconds_behind AS max_seconds_behind, + final_quantiles_seconds_behind[1] AS p90_seconds_behind, + final_quantiles_seconds_behind[2] AS p99_seconds_behind, + final_stddev_seconds_behind AS stddev_seconds_behind, + final_avg_blocks_behind AS avg_blocks_behind, + final_max_blocks_behind AS max_blocks_behind, + final_quantiles_blocks_behind[1] AS p90_blocks_behind, + final_quantiles_blocks_behind[2] AS p99_blocks_behind, + final_stddev_blocks_behind AS stddev_blocks_behind, + if(final_query_count = 0, 0.0, final_success_count / final_query_count) AS success_proportion +FROM AggregatedValues; + +-- View for Final Hourly Indexer Aggregations (Using CTE) +CREATE VIEW IF NOT EXISTS view_agg_indexer_hourly AS +WITH AggregatedValues AS ( + SELECT + time_bucket, + indexer, + gateway_id, + countMerge(query_count) AS final_query_count, + countIfMerge(success_count) AS final_success_count, + countIfMerge(failure_count) AS final_failure_count, + avgMerge(avg_indexer_response_time_ms) AS final_avg_indexer_response_time_ms, + maxMerge(max_indexer_response_time_ms) AS final_max_indexer_response_time_ms, + quantilesMerge(0.90, 0.99)(quantiles_indexer_response_time_ms) AS final_quantiles_indexer_response_time_ms, + stddevSampMerge(stddev_indexer_response_time_ms) AS final_stddev_indexer_response_time_ms, + sumMerge(total_fee_grt) AS final_total_fee_grt, + avgMerge(avg_fee_grt) AS final_avg_fee_grt, + maxMerge(max_fee_grt) AS final_max_fee_grt, + quantilesMerge(0.90, 0.99)(quantiles_fee_grt) AS final_quantiles_fee_grt, + stddevSampMerge(stddev_fee_grt) AS final_stddev_fee_grt, + avgMerge(avg_seconds_behind) AS final_avg_seconds_behind, + maxMerge(max_seconds_behind) AS final_max_seconds_behind, + quantilesMerge(0.90, 0.99)(quantiles_seconds_behind) AS final_quantiles_seconds_behind, + stddevSampMerge(stddev_seconds_behind) AS final_stddev_seconds_behind, + avgMerge(avg_blocks_behind) AS final_avg_blocks_behind, + maxMerge(max_blocks_behind) AS final_max_blocks_behind, + quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, + stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind + FROM agg_indexer_hourly + GROUP BY time_bucket, indexer, gateway_id +) +SELECT + time_bucket, + indexer, + gateway_id, + final_query_count AS query_count, + final_success_count AS success_count, + final_failure_count AS failure_count, + final_avg_indexer_response_time_ms AS avg_indexer_response_time_ms, + final_max_indexer_response_time_ms AS max_indexer_response_time_ms, + final_quantiles_indexer_response_time_ms[1] AS p90_indexer_response_time_ms, + final_quantiles_indexer_response_time_ms[2] AS p99_indexer_response_time_ms, + final_stddev_indexer_response_time_ms AS stddev_indexer_response_time_ms, + final_total_fee_grt AS total_fee_grt, + final_avg_fee_grt AS avg_fee_grt, + final_max_fee_grt AS max_fee_grt, + final_quantiles_fee_grt[1] AS p90_fee_grt, + final_quantiles_fee_grt[2] AS p99_fee_grt, + final_stddev_fee_grt AS stddev_fee_grt, + final_avg_seconds_behind AS avg_seconds_behind, + final_max_seconds_behind AS max_seconds_behind, + final_quantiles_seconds_behind[1] AS p90_seconds_behind, + final_quantiles_seconds_behind[2] AS p99_seconds_behind, + final_stddev_seconds_behind AS stddev_seconds_behind, + final_avg_blocks_behind AS avg_blocks_behind, + final_max_blocks_behind AS max_blocks_behind, + final_quantiles_blocks_behind[1] AS p90_blocks_behind, + final_quantiles_blocks_behind[2] AS p99_blocks_behind, + final_stddev_blocks_behind AS stddev_blocks_behind, + if(final_query_count = 0, 0.0, final_success_count / final_query_count) AS success_proportion +FROM AggregatedValues; + +-- View for Final Daily Indexer Aggregations (Using CTE) +CREATE VIEW IF NOT EXISTS view_agg_indexer_daily AS +WITH AggregatedValues AS ( + SELECT + time_bucket, + indexer, + gateway_id, + countMerge(query_count) AS final_query_count, + countIfMerge(success_count) AS final_success_count, + countIfMerge(failure_count) AS final_failure_count, + avgMerge(avg_indexer_response_time_ms) AS final_avg_indexer_response_time_ms, + maxMerge(max_indexer_response_time_ms) AS final_max_indexer_response_time_ms, + quantilesMerge(0.90, 0.99)(quantiles_indexer_response_time_ms) AS final_quantiles_indexer_response_time_ms, + stddevSampMerge(stddev_indexer_response_time_ms) AS final_stddev_indexer_response_time_ms, + sumMerge(total_fee_grt) AS final_total_fee_grt, + avgMerge(avg_fee_grt) AS final_avg_fee_grt, + maxMerge(max_fee_grt) AS final_max_fee_grt, + quantilesMerge(0.90, 0.99)(quantiles_fee_grt) AS final_quantiles_fee_grt, + stddevSampMerge(stddev_fee_grt) AS final_stddev_fee_grt, + avgMerge(avg_seconds_behind) AS final_avg_seconds_behind, + maxMerge(max_seconds_behind) AS final_max_seconds_behind, + quantilesMerge(0.90, 0.99)(quantiles_seconds_behind) AS final_quantiles_seconds_behind, + stddevSampMerge(stddev_seconds_behind) AS final_stddev_seconds_behind, + avgMerge(avg_blocks_behind) AS final_avg_blocks_behind, + maxMerge(max_blocks_behind) AS final_max_blocks_behind, + quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, + stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind + FROM agg_indexer_daily + GROUP BY time_bucket, indexer, gateway_id +) +SELECT + time_bucket, + indexer, + gateway_id, + final_query_count AS query_count, + final_success_count AS success_count, + final_failure_count AS failure_count, + final_avg_indexer_response_time_ms AS avg_indexer_response_time_ms, + final_max_indexer_response_time_ms AS max_indexer_response_time_ms, + final_quantiles_indexer_response_time_ms[1] AS p90_indexer_response_time_ms, + final_quantiles_indexer_response_time_ms[2] AS p99_indexer_response_time_ms, + final_stddev_indexer_response_time_ms AS stddev_indexer_response_time_ms, + final_total_fee_grt AS total_fee_grt, + final_avg_fee_grt AS avg_fee_grt, + final_max_fee_grt AS max_fee_grt, + final_quantiles_fee_grt[1] AS p90_fee_grt, + final_quantiles_fee_grt[2] AS p99_fee_grt, + final_stddev_fee_grt AS stddev_fee_grt, + final_avg_seconds_behind AS avg_seconds_behind, + final_max_seconds_behind AS max_seconds_behind, + final_quantiles_seconds_behind[1] AS p90_seconds_behind, + final_quantiles_seconds_behind[2] AS p99_seconds_behind, + final_stddev_seconds_behind AS stddev_seconds_behind, + final_avg_blocks_behind AS avg_blocks_behind, + final_max_blocks_behind AS max_blocks_behind, + final_quantiles_blocks_behind[1] AS p90_blocks_behind, + final_quantiles_blocks_behind[2] AS p99_blocks_behind, + final_stddev_blocks_behind AS stddev_blocks_behind, + if(final_query_count = 0, 0.0, final_success_count / final_query_count) AS success_proportion +FROM AggregatedValues; diff --git a/oracle/graphql-api/src/main.rs b/oracle/graphql-api/src/main.rs index 907b21a..8d46601 100644 --- a/oracle/graphql-api/src/main.rs +++ b/oracle/graphql-api/src/main.rs @@ -65,8 +65,9 @@ struct QosReportRow { } // Count query result struct -#[derive(Row, Deserialize, Debug)] +#[derive(Row, Deserialize, Debug, Clone)] struct CountResult { + #[allow(dead_code)] // Allow dead code as it's used for deserialization only count: u64, } @@ -91,65 +92,161 @@ impl AggregationInterval { // --- Deployment Aggregation Structs --- -#[derive(SimpleObject, Deserialize, Debug, Clone)] -#[graphql(name = "DeploymentAggregation")] // Explicit name for GraphQL schema -struct DeploymentAggregationOutput { - time_bucket: String, // Output as String +#[derive(Row, Deserialize, Debug, Clone, SimpleObject)] +struct DeploymentAggregationRow { + time_bucket: u32, subgraph: String, gateway_id: String, query_count: u64, success_count: u64, failure_count: u64, avg_response_time_ms: f64, + max_response_time_ms: u32, + p90_response_time_ms: f64, + p99_response_time_ms: f64, + stddev_response_time_ms: f64, total_fees_usd: f64, + avg_fee_usd: f64, + max_fee_usd: f64, + p90_fee_usd: f64, + p99_fee_usd: f64, + stddev_fee_usd: f64, + success_proportion: f64, } -#[derive(Row, Deserialize, Debug, Clone)] -struct DeploymentAggregationRow { - time_bucket: u32, // Use u32 for Unix timestamp +#[derive(SimpleObject, Debug, Clone)] +struct DeploymentAggregationOutput { + time_bucket: String, subgraph: String, gateway_id: String, query_count: u64, success_count: u64, failure_count: u64, - avg_response_time_ms: f64, - total_fees_usd: f64, + // Latency + #[graphql(name = "avgResponseTimeMs")] + avg_response_time_ms: Option, + #[graphql(name = "maxResponseTimeMs")] + max_response_time_ms: Option, + #[graphql(name = "p90ResponseTimeMs")] + p90_response_time_ms: Option, + #[graphql(name = "p99ResponseTimeMs")] + p99_response_time_ms: Option, + #[graphql(name = "stddevResponseTimeMs")] + stddev_response_time_ms: Option, + // Fees + #[graphql(name = "totalFeesUsd")] + total_fees_usd: Option, + #[graphql(name = "avgFeeUsd")] + avg_fee_usd: Option, + #[graphql(name = "maxFeeUsd")] + max_fee_usd: Option, + #[graphql(name = "p90FeeUsd")] + p90_fee_usd: Option, + #[graphql(name = "p99FeeUsd")] + p99_fee_usd: Option, + #[graphql(name = "stddevFeeUsd")] + stddev_fee_usd: Option, + // Success + #[graphql(name = "successProportion")] + success_proportion: Option, } -// --- Indexer Aggregation Structs --- +// --- Indexer Aggregation Structs (Updated) --- -#[derive(SimpleObject, Deserialize, Debug, Clone)] -#[graphql(name = "IndexerAggregation")] -struct IndexerAggregationOutput { - time_bucket: String, +#[derive(Row, Deserialize, Debug, Clone, SimpleObject)] +struct IndexerAggregationRow { + time_bucket: u32, indexer: String, gateway_id: String, query_count: u64, success_count: u64, failure_count: u64, avg_indexer_response_time_ms: f64, + max_indexer_response_time_ms: u32, + p90_indexer_response_time_ms: f64, + p99_indexer_response_time_ms: f64, + stddev_indexer_response_time_ms: f64, total_fee_grt: f64, + avg_fee_grt: f64, + max_fee_grt: f64, + p90_fee_grt: f64, + p99_fee_grt: f64, + stddev_fee_grt: f64, avg_seconds_behind: f64, + max_seconds_behind: u32, + p90_seconds_behind: f64, + p99_seconds_behind: f64, + stddev_seconds_behind: f64, avg_blocks_behind: f64, + max_blocks_behind: u64, + p90_blocks_behind: f64, + p99_blocks_behind: f64, + stddev_blocks_behind: f64, + success_proportion: f64, } -#[derive(Row, Deserialize, Debug, Clone)] -struct IndexerAggregationRow { - time_bucket: u32, // Use u32 for Unix timestamp +#[derive(SimpleObject, Debug, Clone)] +#[graphql(name = "IndexerAggregation")] // Keep existing name if desired +struct IndexerAggregationOutput { + time_bucket: String, indexer: String, gateway_id: String, query_count: u64, success_count: u64, failure_count: u64, - avg_indexer_response_time_ms: f64, - total_fee_grt: f64, - avg_seconds_behind: f64, - avg_blocks_behind: f64, + // Latency + #[graphql(name = "avgIndexerResponseTimeMs")] + avg_indexer_response_time_ms: Option, + #[graphql(name = "maxIndexerResponseTimeMs")] + max_indexer_response_time_ms: Option, + #[graphql(name = "p90IndexerResponseTimeMs")] + p90_indexer_response_time_ms: Option, + #[graphql(name = "p99IndexerResponseTimeMs")] + p99_indexer_response_time_ms: Option, + #[graphql(name = "stddevIndexerResponseTimeMs")] + stddev_indexer_response_time_ms: Option, + // Fees + #[graphql(name = "totalFeeGrt")] + total_fee_grt: Option, + #[graphql(name = "avgFeeGrt")] + avg_fee_grt: Option, + #[graphql(name = "maxFeeGrt")] + max_fee_grt: Option, + #[graphql(name = "p90FeeGrt")] + p90_fee_grt: Option, + #[graphql(name = "p99FeeGrt")] + p99_fee_grt: Option, + #[graphql(name = "stddevFeeGrt")] + stddev_fee_grt: Option, + // Behindness + #[graphql(name = "avgSecondsBehind")] + avg_seconds_behind: Option, + #[graphql(name = "maxSecondsBehind")] + max_seconds_behind: Option, + #[graphql(name = "p90SecondsBehind")] + p90_seconds_behind: Option, + #[graphql(name = "p99SecondsBehind")] + p99_seconds_behind: Option, + #[graphql(name = "stddevSecondsBehind")] + stddev_seconds_behind: Option, + #[graphql(name = "avgBlocksBehind")] + avg_blocks_behind: Option, + #[graphql(name = "maxBlocksBehind")] + max_blocks_behind: Option, + #[graphql(name = "p90BlocksBehind")] + p90_blocks_behind: Option, + #[graphql(name = "p99BlocksBehind")] + p99_blocks_behind: Option, + #[graphql(name = "stddevBlocksBehind")] + stddev_blocks_behind: Option, + // Success + #[graphql(name = "successProportion")] + success_proportion: Option, } // --- Allocation Aggregation Structs --- -#[derive(Row, Deserialize, Debug, Clone)] +#[derive(Row, Deserialize, Debug, Clone, SimpleObject)] struct AllocationAggregationRow { time_bucket: u32, subgraph: String, @@ -159,24 +256,86 @@ struct AllocationAggregationRow { success_count: u64, failure_count: u64, avg_indexer_response_time_ms: f64, + max_indexer_response_time_ms: u32, + p90_indexer_response_time_ms: f64, + p99_indexer_response_time_ms: f64, + stddev_indexer_response_time_ms: f64, total_fee_grt: f64, + avg_fee_grt: f64, + max_fee_grt: f64, + p90_fee_grt: f64, + p99_fee_grt: f64, + stddev_fee_grt: f64, avg_seconds_behind: f64, + max_seconds_behind: u32, + p90_seconds_behind: f64, + p99_seconds_behind: f64, + stddev_seconds_behind: f64, avg_blocks_behind: f64, + max_blocks_behind: u64, + p90_blocks_behind: f64, + p99_blocks_behind: f64, + stddev_blocks_behind: f64, + success_proportion: f64, } #[derive(SimpleObject, Debug, Clone)] struct AllocationAggregationOutput { - time_bucket: String, // Formatted as RFC3339 + time_bucket: String, subgraph: String, indexer: String, gateway_id: String, query_count: u64, success_count: u64, failure_count: u64, - avg_indexer_response_time_ms: f64, - total_fee_grt: f64, - avg_seconds_behind: f64, - avg_blocks_behind: f64, + // Latency + #[graphql(name = "avgIndexerResponseTimeMs")] + avg_indexer_response_time_ms: Option, + #[graphql(name = "maxIndexerResponseTimeMs")] + max_indexer_response_time_ms: Option, + #[graphql(name = "p90IndexerResponseTimeMs")] + p90_indexer_response_time_ms: Option, + #[graphql(name = "p99IndexerResponseTimeMs")] + p99_indexer_response_time_ms: Option, + #[graphql(name = "stddevIndexerResponseTimeMs")] + stddev_indexer_response_time_ms: Option, + // Fees + #[graphql(name = "totalFeeGrt")] + total_fee_grt: Option, + #[graphql(name = "avgFeeGrt")] + avg_fee_grt: Option, + #[graphql(name = "maxFeeGrt")] + max_fee_grt: Option, + #[graphql(name = "p90FeeGrt")] + p90_fee_grt: Option, + #[graphql(name = "p99FeeGrt")] + p99_fee_grt: Option, + #[graphql(name = "stddevFeeGrt")] + stddev_fee_grt: Option, + // Behindness + #[graphql(name = "avgSecondsBehind")] + avg_seconds_behind: Option, + #[graphql(name = "maxSecondsBehind")] + max_seconds_behind: Option, + #[graphql(name = "p90SecondsBehind")] + p90_seconds_behind: Option, + #[graphql(name = "p99SecondsBehind")] + p99_seconds_behind: Option, + #[graphql(name = "stddevSecondsBehind")] + stddev_seconds_behind: Option, + #[graphql(name = "avgBlocksBehind")] + avg_blocks_behind: Option, + #[graphql(name = "maxBlocksBehind")] + max_blocks_behind: Option, + #[graphql(name = "p90BlocksBehind")] + p90_blocks_behind: Option, + #[graphql(name = "p99BlocksBehind")] + p99_blocks_behind: Option, + #[graphql(name = "stddevBlocksBehind")] + stddev_blocks_behind: Option, + // Success + #[graphql(name = "successProportion")] + success_proportion: Option, } // --- Input Objects for Filtering --- @@ -207,14 +366,15 @@ struct QueryRoot; #[Object] impl QueryRoot { - /// Query for raw QoS Reports (limited fields for now) + /// Query for raw QoS reports async fn qos_reports( &self, - ctx: &Context<'_>, + _ctx: &Context<'_>, // Prefix ctx with underscore time_range: TimeRangeInput, #[graphql(desc = "Optional filters for the query")] filter: Option, #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] limit: Option, + #[graphql(desc = "Number of records to skip (for pagination)")] offset: Option, ) -> Result, String> { let client = get_clickhouse_client()?; @@ -253,13 +413,14 @@ impl QueryRoot { } else { format!("WHERE {}", conditions.join(" AND ")) }; - let query_limit = limit.unwrap_or(1000).max(0).min(10000); // Apply limits + let query_offset = offset.unwrap_or(0).max(0); + let query_limit = limit.unwrap_or(1000).clamp(0, 10000); // Use clamp let query = format!( "SELECT event_time, gateway_id, receipt_signer, query_id, api_key, user_id, \ subgraph, result, response_time_ms, request_bytes, response_bytes, total_fees_usd \ - FROM qos_data {} ORDER BY event_time DESC LIMIT {}", - where_clause, query_limit + FROM qos_data {} ORDER BY event_time DESC LIMIT {} OFFSET {}", + where_clause, query_limit, query_offset ); println!("Executing query: {}", query); @@ -293,10 +454,68 @@ impl QueryRoot { .collect()) } + /// Query for the total count of QoS reports matching the criteria + async fn qos_reports_count( + &self, + _ctx: &Context<'_>, // Prefix ctx with underscore + time_range: TimeRangeInput, + #[graphql(desc = "Optional filters for the query")] filter: Option, + ) -> Result { + let client = get_clickhouse_client()?; + + // Build WHERE clause + let mut conditions = Vec::new(); + let from_dt = DateTime::parse_from_rfc3339(&time_range.from) + .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? + .with_timezone(&Utc); + let to_dt = DateTime::parse_from_rfc3339(&time_range.to) + .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? + .with_timezone(&Utc); + + conditions.push(format!("event_time >= toDateTime({})", from_dt.timestamp())); + conditions.push(format!("event_time < toDateTime({})", to_dt.timestamp())); + + if let Some(f) = filter { + if let Some(gw) = &f.gateway_id { + conditions.push(format!("gateway_id = '{}'", gw)); + } + if let Some(sg) = &f.subgraph { + // Handle nullable subgraph field correctly + conditions.push(format!("subgraph = '{}'", sg)); + } + // Filtering by indexer/allocation on raw reports requires arrayExists/has, + // which can be slow. Skipping for now. + // if let Some(ix) = &f.indexer { + // conditions.push(format!("has(indexer_queries.indexer, '{}')", ix)); + // } + // if let Some(alloc) = &f.allocation { + // conditions.push(format!("has(indexer_queries.allocation, '{}')", alloc)); + // } + } + + let where_clause = if conditions.is_empty() { + String::new() + } else { + format!("WHERE {}", conditions.join(" AND ")) + }; + + let query = format!("SELECT count(*) FROM qos_data {}", where_clause); + + println!("Executing query: {}", query); + + let count = client + .query(&query) + .fetch_one::() + .await + .map_err(|e| format!("Database query failed: {}", e))?; + + Ok(count.count) + } + /// Query for Deployment level aggregations async fn deployment_aggregations( &self, - ctx: &Context<'_>, + _ctx: &Context<'_>, // Prefix ctx with underscore interval: AggregationInterval, time_range: TimeRangeInput, #[graphql(desc = "Optional filters for the query")] filter: Option, @@ -304,7 +523,7 @@ impl QueryRoot { limit: Option, ) -> Result, String> { let client = get_clickhouse_client()?; - let table_name = format!("agg_deployment_{}", interval.table_suffix()); + let view_name = format!("view_agg_deployment_{}", interval.table_suffix()); // Build WHERE clause let mut conditions = Vec::new(); @@ -314,7 +533,10 @@ impl QueryRoot { let to_dt = DateTime::parse_from_rfc3339(&time_range.to) .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? .with_timezone(&Utc); - conditions.push(format!("time_bucket >= toDateTime({})", from_dt.timestamp())); + conditions.push(format!( + "time_bucket >= toDateTime({})", + from_dt.timestamp() + )); conditions.push(format!("time_bucket < toDateTime({})", to_dt.timestamp())); if let Some(f) = filter { @@ -328,31 +550,34 @@ impl QueryRoot { } let where_clause = format!("WHERE {}", conditions.join(" AND ")); - let query_limit = limit.unwrap_or(1000).max(0).min(10000); + let query_limit = limit.unwrap_or(1000).clamp(0, 10000); // Use clamp // Query directly selects pre-aggregated columns, no further aggregation or GROUP BY let query = format!( "SELECT time_bucket, subgraph, gateway_id, \ query_count, success_count, failure_count, \ - avg_response_time_ms, total_fees_usd \ + avg_response_time_ms, max_response_time_ms, p90_response_time_ms, p99_response_time_ms, stddev_response_time_ms, \ + total_fees_usd, avg_fee_usd, max_fee_usd, p90_fee_usd, p99_fee_usd, stddev_fee_usd, \ + success_proportion \ FROM {} {} \ ORDER BY time_bucket DESC, subgraph, gateway_id \ LIMIT {}", - table_name, where_clause, query_limit + view_name, where_clause, query_limit ); println!("Executing query: {}", query); let rows = client .query(&query) - .fetch_all::() // Uses existing struct + .fetch_all::() // Uses updated struct .await .map_err(|e| format!("Database query failed: {}", e))?; - // Map directly, row-by-row (mapping logic remains the same) + // Map directly, row-by-row, including new fields Ok(rows .into_iter() .map(|row| DeploymentAggregationOutput { + // Uses updated struct time_bucket: Utc .timestamp_opt(row.time_bucket as i64, 0) .single() @@ -362,8 +587,19 @@ impl QueryRoot { query_count: row.query_count, success_count: row.success_count, failure_count: row.failure_count, - avg_response_time_ms: row.avg_response_time_ms, - total_fees_usd: row.total_fees_usd, + // Map all new fields, handling Option for NaN/Inf safety + avg_response_time_ms: Some(row.avg_response_time_ms), + max_response_time_ms: Some(row.max_response_time_ms), + p90_response_time_ms: Some(row.p90_response_time_ms), + p99_response_time_ms: Some(row.p99_response_time_ms), + stddev_response_time_ms: Some(row.stddev_response_time_ms), + total_fees_usd: Some(row.total_fees_usd), + avg_fee_usd: Some(row.avg_fee_usd), + max_fee_usd: Some(row.max_fee_usd), + p90_fee_usd: Some(row.p90_fee_usd), + p99_fee_usd: Some(row.p99_fee_usd), + stddev_fee_usd: Some(row.stddev_fee_usd), + success_proportion: Some(row.success_proportion), }) .collect()) } @@ -371,17 +607,18 @@ impl QueryRoot { /// Query for Indexer level aggregations async fn indexer_aggregations( &self, - ctx: &Context<'_>, + _ctx: &Context<'_>, // Prefix ctx with underscore interval: AggregationInterval, time_range: TimeRangeInput, #[graphql(desc = "Optional filters for the query")] filter: Option, #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] limit: Option, ) -> Result, String> { + // Return updated Output struct let client = get_clickhouse_client()?; - let table_name = format!("agg_indexer_{}", interval.table_suffix()); + let view_name = format!("view_agg_indexer_{}", interval.table_suffix()); - // Build WHERE clause + // Build WHERE clause (logic remains the same, lines 501-521) let mut conditions = Vec::new(); let from_dt = DateTime::parse_from_rfc3339(&time_range.from) .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? @@ -389,7 +626,10 @@ impl QueryRoot { let to_dt = DateTime::parse_from_rfc3339(&time_range.to) .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? .with_timezone(&Utc); - conditions.push(format!("time_bucket >= toDateTime({})", from_dt.timestamp())); + conditions.push(format!( + "time_bucket >= toDateTime({})", + from_dt.timestamp() + )); conditions.push(format!("time_bucket < toDateTime({})", to_dt.timestamp())); if let Some(f) = filter { @@ -403,32 +643,38 @@ impl QueryRoot { } let where_clause = format!("WHERE {}", conditions.join(" AND ")); - let query_limit = limit.unwrap_or(1000).max(0).min(10000); + let query_limit = limit.unwrap_or(1000).clamp(0, 10000); // Use clamp - // Query directly selects pre-aggregated columns, no further aggregation or GROUP BY + // Update SELECT list to include all new fields let query = format!( "SELECT time_bucket, indexer, gateway_id, \ query_count, success_count, failure_count, \ - avg_indexer_response_time_ms, total_fee_grt, \ - avg_seconds_behind, avg_blocks_behind \ + avg_indexer_response_time_ms, max_indexer_response_time_ms, \ + p90_indexer_response_time_ms, p99_indexer_response_time_ms, \ + stddev_indexer_response_time_ms, \ + total_fee_grt, avg_fee_grt, max_fee_grt, p90_fee_grt, p99_fee_grt, stddev_fee_grt, \ + avg_seconds_behind, max_seconds_behind, p90_seconds_behind, p99_seconds_behind, stddev_seconds_behind, \ + avg_blocks_behind, max_blocks_behind, p90_blocks_behind, p99_blocks_behind, stddev_blocks_behind, \ + success_proportion \ FROM {} {} \ - ORDER BY time_bucket DESC, indexer, gateway_id \ + ORDER BY time_bucket DESC, indexer ASC, gateway_id ASC \ LIMIT {}", - table_name, where_clause, query_limit + view_name, where_clause, query_limit ); println!("Executing query: {}", query); let rows = client .query(&query) - .fetch_all::() // Uses existing struct + .fetch_all::() // Use updated Row struct .await .map_err(|e| format!("Database query failed: {}", e))?; - // Map directly, row-by-row (mapping logic remains the same) + // Map directly, row-by-row, including new fields Ok(rows .into_iter() .map(|row| IndexerAggregationOutput { + // Use updated Output struct time_bucket: Utc .timestamp_opt(row.time_bucket as i64, 0) .single() @@ -438,18 +684,37 @@ impl QueryRoot { query_count: row.query_count, success_count: row.success_count, failure_count: row.failure_count, - avg_indexer_response_time_ms: row.avg_indexer_response_time_ms, - total_fee_grt: row.total_fee_grt, - avg_seconds_behind: row.avg_seconds_behind, - avg_blocks_behind: row.avg_blocks_behind, + // Map all new fields, handling Option for NaN/Inf safety + avg_indexer_response_time_ms: Some(row.avg_indexer_response_time_ms), + max_indexer_response_time_ms: Some(row.max_indexer_response_time_ms), + p90_indexer_response_time_ms: Some(row.p90_indexer_response_time_ms), + p99_indexer_response_time_ms: Some(row.p99_indexer_response_time_ms), + stddev_indexer_response_time_ms: Some(row.stddev_indexer_response_time_ms), + total_fee_grt: Some(row.total_fee_grt), + avg_fee_grt: Some(row.avg_fee_grt), + max_fee_grt: Some(row.max_fee_grt), + p90_fee_grt: Some(row.p90_fee_grt), + p99_fee_grt: Some(row.p99_fee_grt), + stddev_fee_grt: Some(row.stddev_fee_grt), + avg_seconds_behind: Some(row.avg_seconds_behind), + max_seconds_behind: Some(row.max_seconds_behind), + p90_seconds_behind: Some(row.p90_seconds_behind), + p99_seconds_behind: Some(row.p99_seconds_behind), + stddev_seconds_behind: Some(row.stddev_seconds_behind), + avg_blocks_behind: Some(row.avg_blocks_behind), + max_blocks_behind: Some(row.max_blocks_behind), + p90_blocks_behind: Some(row.p90_blocks_behind), + p99_blocks_behind: Some(row.p99_blocks_behind), + stddev_blocks_behind: Some(row.stddev_blocks_behind), + success_proportion: Some(row.success_proportion), }) .collect()) } - /// Query for Allocation level aggregations + /// Query for Allocation level aggregations (grouped by subgraph and indexer) async fn allocation_aggregations( &self, - ctx: &Context<'_>, + _ctx: &Context<'_>, // Prefix ctx with underscore interval: AggregationInterval, time_range: TimeRangeInput, #[graphql(desc = "Optional filters for the query")] filter: Option, @@ -457,7 +722,7 @@ impl QueryRoot { limit: Option, ) -> Result, String> { let client = get_clickhouse_client()?; - let table_name = format!("agg_allocation_{}", interval.table_suffix()); + let view_name = format!("view_agg_allocation_{}", interval.table_suffix()); // Build WHERE clause let mut conditions = Vec::new(); @@ -467,14 +732,17 @@ impl QueryRoot { let to_dt = DateTime::parse_from_rfc3339(&time_range.to) .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? .with_timezone(&Utc); - conditions.push(format!("time_bucket >= toDateTime({})", from_dt.timestamp())); + conditions.push(format!( + "time_bucket >= toDateTime({})", + from_dt.timestamp() + )); conditions.push(format!("time_bucket < toDateTime({})", to_dt.timestamp())); if let Some(f) = filter { if let Some(gw) = &f.gateway_id { conditions.push(format!("gateway_id = '{}'", gw)); } - if let Some(sg) = &f.subgraph { + if let Some(sg) = &f.subgraph { conditions.push(format!("subgraph = '{}'", sg)); } if let Some(ix) = &f.indexer { @@ -483,18 +751,23 @@ impl QueryRoot { } let where_clause = format!("WHERE {}", conditions.join(" AND ")); - let query_limit = limit.unwrap_or(1000).max(0).min(10000); + let query_limit = limit.unwrap_or(1000).clamp(0, 10000); // Use clamp // Query directly selects pre-aggregated columns, no further aggregation or GROUP BY let query = format!( "SELECT time_bucket, subgraph, indexer, gateway_id, \ query_count, success_count, failure_count, \ - avg_indexer_response_time_ms, total_fee_grt, \ - avg_seconds_behind, avg_blocks_behind \ + avg_indexer_response_time_ms, max_indexer_response_time_ms, \ + p90_indexer_response_time_ms, p99_indexer_response_time_ms, \ + stddev_indexer_response_time_ms, \ + total_fee_grt, avg_fee_grt, max_fee_grt, p90_fee_grt, p99_fee_grt, stddev_fee_grt, \ + avg_seconds_behind, max_seconds_behind, p90_seconds_behind, p99_seconds_behind, stddev_seconds_behind, \ + avg_blocks_behind, max_blocks_behind, p90_blocks_behind, p99_blocks_behind, stddev_blocks_behind, \ + success_proportion \ FROM {} {} \ - ORDER BY time_bucket DESC, subgraph, indexer, gateway_id \ + ORDER BY time_bucket DESC, subgraph ASC, indexer ASC, gateway_id ASC \ LIMIT {}", - table_name, where_clause, query_limit + view_name, where_clause, query_limit ); println!("Executing query: {}", query); @@ -505,10 +778,11 @@ impl QueryRoot { .await .map_err(|e| format!("Database query failed: {}", e))?; - // Map directly, row-by-row (mapping logic remains the same) + // Map directly, row-by-row, including new fields Ok(rows .into_iter() - .map(|row| AllocationAggregationOutput { // Uses existing struct + .map(|row| AllocationAggregationOutput { + // Uses existing struct time_bucket: Utc .timestamp_opt(row.time_bucket as i64, 0) .single() @@ -519,10 +793,29 @@ impl QueryRoot { query_count: row.query_count, success_count: row.success_count, failure_count: row.failure_count, - avg_indexer_response_time_ms: row.avg_indexer_response_time_ms, - total_fee_grt: row.total_fee_grt, - avg_seconds_behind: row.avg_seconds_behind, - avg_blocks_behind: row.avg_blocks_behind, + // Map all new fields (same mapping logic as Indexer) + avg_indexer_response_time_ms: Some(row.avg_indexer_response_time_ms), + max_indexer_response_time_ms: Some(row.max_indexer_response_time_ms), + p90_indexer_response_time_ms: Some(row.p90_indexer_response_time_ms), + p99_indexer_response_time_ms: Some(row.p99_indexer_response_time_ms), + stddev_indexer_response_time_ms: Some(row.stddev_indexer_response_time_ms), + total_fee_grt: Some(row.total_fee_grt), + avg_fee_grt: Some(row.avg_fee_grt), + max_fee_grt: Some(row.max_fee_grt), + p90_fee_grt: Some(row.p90_fee_grt), + p99_fee_grt: Some(row.p99_fee_grt), + stddev_fee_grt: Some(row.stddev_fee_grt), + avg_seconds_behind: Some(row.avg_seconds_behind), + max_seconds_behind: Some(row.max_seconds_behind), + p90_seconds_behind: Some(row.p90_seconds_behind), + p99_seconds_behind: Some(row.p99_seconds_behind), + stddev_seconds_behind: Some(row.stddev_seconds_behind), + avg_blocks_behind: Some(row.avg_blocks_behind), + max_blocks_behind: Some(row.max_blocks_behind), + p90_blocks_behind: Some(row.p90_blocks_behind), + p99_blocks_behind: Some(row.p99_blocks_behind), + stddev_blocks_behind: Some(row.stddev_blocks_behind), + success_proportion: Some(row.success_proportion), }) .collect()) } @@ -532,10 +825,12 @@ impl QueryRoot { // Returns the configured client builder. Connection happens on first query. fn get_clickhouse_client() -> Result { Ok(Client::default() - .with_url(&env::var("CLICKHOUSE_URL").unwrap_or_else(|_| "http://localhost:8123".into())) - .with_database(&env::var("CLICKHOUSE_DB").unwrap_or_else(|_| "default".into())) - .with_user(&env::var("CLICKHOUSE_USER").unwrap_or_else(|_| "graphql".into())) - .with_password(&env::var("CLICKHOUSE_PASSWORD").unwrap_or_else(|_| "graphql_password".into()))) + .with_url(env::var("CLICKHOUSE_URL").unwrap_or_else(|_| "http://localhost:8123".into())) + .with_database(env::var("CLICKHOUSE_DB").unwrap_or_else(|_| "default".into())) + .with_user(env::var("CLICKHOUSE_USER").unwrap_or_else(|_| "graphql".into())) + .with_password( + env::var("CLICKHOUSE_PASSWORD").unwrap_or_else(|_| "graphql_password".into()), + )) // Remove .try_into() and .map_err() } @@ -561,13 +856,17 @@ async fn health_check() -> HttpResponse { let client_result = get_clickhouse_client(); if client_result.is_err() { // Log the actual error if needed - eprintln!("Health check failed (client config): {:?}", client_result.err()); + eprintln!( + "Health check failed (client config): {:?}", + client_result.err() + ); return HttpResponse::InternalServerError().body("ClickHouse client configuration error"); } let client = client_result.unwrap(); // Safe unwrap after check above // More advanced check: Can we execute a simple query? - match client.query("SELECT 1").execute().await { // Use execute() for simple queries + match client.query("SELECT 1").execute().await { + // Use execute() for simple queries Ok(_) => HttpResponse::Ok().body("OK"), Err(e) => { eprintln!("Health check failed (query execution): {}", e); // Log the error From 71d62fb48e235a1efcdb251dddaca0f2fce62305 Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Wed, 2 Apr 2025 23:34:00 -0300 Subject: [PATCH 12/20] feat: added graceful shutdown logic --- Cargo.lock | 239 ++++++++++++++++++++++++++++++++- oracle/graphql-api/Cargo.toml | 6 +- oracle/graphql-api/src/main.rs | 61 ++++++++- 3 files changed, 297 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 59fa535..67fcfc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -455,6 +455,60 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "axum" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de45108900e1f9b9242f7f2e254aa3e2c029c921c258fe9e6b4217eeebd54288" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http 1.3.1", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" +dependencies = [ + "bytes", + "futures-core", + "http 1.3.1", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -1119,6 +1173,7 @@ dependencies = [ "http 1.3.1", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1457,6 +1512,21 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.7.4" @@ -1514,6 +1584,16 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -1565,6 +1645,12 @@ version = "1.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "parking_lot" version = "0.12.3" @@ -1743,8 +1829,10 @@ name = "qos_graphql_api" version = "0.1.0" dependencies = [ "actix-web", + "anyhow", "async-graphql", "async-graphql-actix-web", + "axum", "bs58", "chrono", "clickhouse", @@ -1753,6 +1841,8 @@ dependencies = [ "serde", "serde_json", "tokio", + "tracing", + "tracing-subscriber", ] [[package]] @@ -1891,8 +1981,17 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -1903,7 +2002,7 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.5", ] [[package]] @@ -1912,6 +2011,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.5" @@ -2015,6 +2120,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a" +dependencies = [ + "itoa", + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2038,6 +2153,15 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" @@ -2124,6 +2248,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + [[package]] name = "synstructure" version = "0.13.1" @@ -2188,6 +2318,16 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "time" version = "0.3.41" @@ -2314,6 +2454,28 @@ dependencies = [ "winnow 0.7.4", ] +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -2350,6 +2512,49 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", ] [[package]] @@ -2414,6 +2619,12 @@ dependencies = [ "getrandom 0.3.2", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -2508,6 +2719,28 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.52.0" diff --git a/oracle/graphql-api/Cargo.toml b/oracle/graphql-api/Cargo.toml index 5bb5a94..e466e97 100644 --- a/oracle/graphql-api/Cargo.toml +++ b/oracle/graphql-api/Cargo.toml @@ -14,4 +14,8 @@ tokio = { version = "1", features = ["full"] } futures = "0.3" bs58 = "0.5.0" hex = "0.4.3" -chrono = "0.4" \ No newline at end of file +chrono = "0.4" +anyhow = "1.0.79" +tracing = { version = "0.1.40", features = ["attributes"] } +tracing-subscriber = { version = "0.3.18", features = ["env-filter", "json"] } +axum = "0.8.1" \ No newline at end of file diff --git a/oracle/graphql-api/src/main.rs b/oracle/graphql-api/src/main.rs index 8d46601..31d849f 100644 --- a/oracle/graphql-api/src/main.rs +++ b/oracle/graphql-api/src/main.rs @@ -7,6 +7,7 @@ use chrono::{DateTime, TimeZone, Utc}; use clickhouse::{Client, Row}; use serde::Deserialize; use std::env; +use tokio::signal::unix::{signal, SignalKind}; // use bs58; // use hex; @@ -877,21 +878,71 @@ async fn health_check() -> HttpResponse { #[actix_web::main] async fn main() -> std::io::Result<()> { + // Initialize logging (optional but recommended) + // You can use a simple logger like env_logger or tracing + // env_logger::init_from_env(env_logger::Env::new().default_filter_or("info")); + // Or using tracing if you prefer (ensure tracing/tracing-subscriber are deps) + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .init(); + // Initialize environment variables (e.g., from .env file if needed) // dotenv::dotenv().ok(); let schema = Schema::build(QueryRoot, EmptyMutation, EmptySubscription).finish(); - println!("GraphQL playground: http://localhost:8000"); + tracing::info!("Starting GraphQL API server..."); // Use tracing/log - HttpServer::new(move || { + // --- Server Initialization --- + // Create the server instance but don't await .run() immediately + let server = HttpServer::new(move || { App::new() .app_data(web::Data::new(schema.clone())) .route("/graphql", web::post().to(graphql_handler)) .route("/", web::get().to(graphql_playground)) - .route("/health", web::get().to(health_check)) + .route("/health", web::get().to(health_check)) // Keep health check }) .bind("0.0.0.0:8000")? - .run() - .await + .run(); // .run() returns a Server instance + // --- End Server Initialization --- + + // --- Graceful Shutdown Logic --- + // Get a handle to the server instance + let server_handle = server.handle(); + + // Spawn a separate task to listen for termination signals + tokio::spawn(async move { + let mut sigint = signal(SignalKind::interrupt()).expect("Failed to install SIGINT handler"); + let mut sigterm = + signal(SignalKind::terminate()).expect("Failed to install SIGTERM handler"); + + // Wait for either SIGINT or SIGTERM + tokio::select! { + _ = sigint.recv() => { + tracing::info!("SIGINT received, initiating graceful shutdown..."); + }, + _ = sigterm.recv() => { + tracing::info!("SIGTERM received, initiating graceful shutdown..."); + }, + }; + + // Initiate graceful shutdown using the server handle. + // stop(true) sends the stop signal gracefully. + // We await it to ensure the signal is processed. + server_handle.stop(true).await; + tracing::info!("Shutdown signal sent to Actix server."); + }); + // --- End Graceful Shutdown Logic --- + + tracing::info!("GraphQL server running at http://0.0.0.0:8000"); + tracing::info!("GraphQL playground available at http://localhost:8000"); // Updated log + + // Wait for the server to stop. + // This will block until the server is shut down, either normally + // or via the signal handler calling server_handle.stop(). + server.await?; + + tracing::info!("GraphQL server has stopped gracefully."); + + Ok(()) } From d36b6ac705a7f1583888d833b30fc2ff50cd3dc4 Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Wed, 2 Apr 2025 23:58:56 -0300 Subject: [PATCH 13/20] feat: simplified raw data table, removed unnecessary resolvers --- oracle/clickhouse/tables.sql | 73 +++--------- oracle/graphql-api/src/main.rs | 209 --------------------------------- 2 files changed, 14 insertions(+), 268 deletions(-) diff --git a/oracle/clickhouse/tables.sql b/oracle/clickhouse/tables.sql index 004a341..ef9a174 100644 --- a/oracle/clickhouse/tables.sql +++ b/oracle/clickhouse/tables.sql @@ -35,27 +35,26 @@ SETTINGS kafka_max_block_size = 1, kafka_poll_timeout_ms = 500; --- Create the destination table for the raw data (with TTL) --- Note: This table seems less critical now with the qos_data_mv directly populating qos_data, --- but we keep it for potential raw data inspection if needed. -CREATE TABLE IF NOT EXISTS raw_qos_data +-- Create the primary data table (qos_data) +-- This table stores the processed data from Kafka, ready for aggregation +CREATE TABLE IF NOT EXISTS qos_data ( event_time DateTime, gateway_id String, - receipt_signer String, -- Raw bytes + receipt_signer String, -- HEX encoded query_id String, api_key String, user_id String, subgraph Nullable(String), - result String, + result String, -- 'success' or other status response_time_ms UInt32, request_bytes UInt32, response_bytes Nullable(UInt32), total_fees_usd Float64, - indexer_queries Nested( - indexer String, -- Raw bytes - deployment String, -- Raw bytes - allocation String, -- Raw bytes + indexer_queries Nested ( + indexer String, -- HEX encoded + deployment String, -- HEX encoded + allocation String, -- HEX encoded indexed_chain String, url String, fee_grt Float64, @@ -65,57 +64,13 @@ CREATE TABLE IF NOT EXISTS raw_qos_data indexer_errors String, blocks_behind UInt64 ) -) ENGINE = MergeTree() -ORDER BY (event_time, gateway_id) -PARTITION BY toYYYYMMDD(event_time) -TTL event_time + INTERVAL 7 DAY; -- Keep raw data for 7 days - --- Materialized view to populate raw_qos_data (optional, could be removed if qos_data is sufficient) -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_raw_qos_data TO raw_qos_data AS -SELECT - now() as event_time, - gateway_id, - receipt_signer, -- Store raw bytes here - query_id, - api_key, - user_id, - subgraph, - result, - response_time_ms, - request_bytes, - response_bytes, - total_fees_usd, - indexer_queries -- Store raw nested bytes here -FROM kafka_qos_data; - --- Create the table for processed QoS data (with HEX encoding) -CREATE TABLE IF NOT EXISTS qos_data -( - event_time DateTime, - gateway_id String, - receipt_signer String, -- HEX encoded - query_id String, - api_key String, - user_id String, - subgraph Nullable(String), - result String, -- 'success' or other status - response_time_ms UInt32, - request_bytes UInt32, - response_bytes Nullable(UInt32), - total_fees_usd Float64, - indexer_queries Nested ( - indexer String, - result String, -- 'success' or other status - response_time_ms UInt32, - fee_grt Float64, - seconds_behind UInt32, - blocks_behind UInt64 - ) ) ENGINE = MergeTree -PARTITION BY toYYYYMM(event_time) -ORDER BY (gateway_id, event_time); +PARTITION BY toYYYYMM(event_time) -- Keep monthly partitioning +ORDER BY (gateway_id, event_time) +TTL event_time + INTERVAL 7 DAY; -- Add TTL: Delete rows older than 7 days -- Create the materialized view that processes and transforms the data into qos_data +-- Ensure ALL necessary fields are selected and HEX encoding is applied correctly CREATE MATERIALIZED VIEW IF NOT EXISTS qos_data_mv TO qos_data AS SELECT now() as event_time, @@ -130,7 +85,7 @@ SELECT request_bytes, response_bytes, total_fees_usd, - -- Apply HEX encoding to nested fields + -- Select ALL nested fields needed downstream, applying HEX where required arrayMap(x -> HEX(x), indexer_queries.indexer) AS `indexer_queries.indexer`, arrayMap(x -> HEX(x), indexer_queries.deployment) AS `indexer_queries.deployment`, arrayMap(x -> HEX(x), indexer_queries.allocation) AS `indexer_queries.allocation`, diff --git a/oracle/graphql-api/src/main.rs b/oracle/graphql-api/src/main.rs index 31d849f..c08c345 100644 --- a/oracle/graphql-api/src/main.rs +++ b/oracle/graphql-api/src/main.rs @@ -8,69 +8,6 @@ use clickhouse::{Client, Row}; use serde::Deserialize; use std::env; use tokio::signal::unix::{signal, SignalKind}; -// use bs58; -// use hex; - -// Complete QoS Report matching the database schema -#[derive(SimpleObject, Deserialize, Debug, Clone)] -// #[graphql(complex)] // Mark complex to add computed fields if needed later -struct QosReport { - event_time: String, // Keep as String for GraphQL output - gateway_id: String, - receipt_signer: String, - query_id: String, - api_key: String, - user_id: String, - subgraph: Option, - result: String, - response_time_ms: u32, - request_bytes: u32, - response_bytes: Option, - total_fees_usd: f64, - // We might add indexer_queries here later if needed, requires nested struct handling -} - -/* -#[derive(SimpleObject, Deserialize, Serialize)] -struct IndexerQuery { - indexer: String, - deployment: String, - allocation: String, - indexed_chain: String, - url: String, - fee_grt: f64, - response_time_ms: u32, - seconds_behind: u32, - result: String, - indexer_errors: String, - blocks_behind: u64, -} -*/ - -// ClickHouse Row implementation for direct native protocol deserialization -#[derive(Row, Deserialize, Debug, Clone)] -struct QosReportRow { - event_time: u32, // Use u32 for Unix timestamp - gateway_id: String, - receipt_signer: String, - query_id: String, - api_key: String, - user_id: String, - subgraph: Option, - result: String, - response_time_ms: u32, - request_bytes: u32, - response_bytes: Option, - total_fees_usd: f64, - // indexer_queries: Vec, // Needs proper handling for Nested type if selected -} - -// Count query result struct -#[derive(Row, Deserialize, Debug, Clone)] -struct CountResult { - #[allow(dead_code)] // Allow dead code as it's used for deserialization only - count: u64, -} // --- Structs for Aggregated Data --- @@ -367,152 +304,6 @@ struct QueryRoot; #[Object] impl QueryRoot { - /// Query for raw QoS reports - async fn qos_reports( - &self, - _ctx: &Context<'_>, // Prefix ctx with underscore - time_range: TimeRangeInput, - #[graphql(desc = "Optional filters for the query")] filter: Option, - #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] - limit: Option, - #[graphql(desc = "Number of records to skip (for pagination)")] offset: Option, - ) -> Result, String> { - let client = get_clickhouse_client()?; - - // Build WHERE clause - let mut conditions = Vec::new(); - let from_dt = DateTime::parse_from_rfc3339(&time_range.from) - .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? - .with_timezone(&Utc); - let to_dt = DateTime::parse_from_rfc3339(&time_range.to) - .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? - .with_timezone(&Utc); - - conditions.push(format!("event_time >= toDateTime({})", from_dt.timestamp())); - conditions.push(format!("event_time < toDateTime({})", to_dt.timestamp())); - - if let Some(f) = filter { - if let Some(gw) = &f.gateway_id { - conditions.push(format!("gateway_id = '{}'", gw)); - } - if let Some(sg) = &f.subgraph { - // Handle nullable subgraph field correctly - conditions.push(format!("subgraph = '{}'", sg)); - } - // Filtering by indexer/allocation on raw reports requires arrayExists/has, - // which can be slow. Skipping for now. - // if let Some(ix) = &f.indexer { - // conditions.push(format!("has(indexer_queries.indexer, '{}')", ix)); - // } - // if let Some(alloc) = &f.allocation { - // conditions.push(format!("has(indexer_queries.allocation, '{}')", alloc)); - // } - } - - let where_clause = if conditions.is_empty() { - String::new() - } else { - format!("WHERE {}", conditions.join(" AND ")) - }; - let query_offset = offset.unwrap_or(0).max(0); - let query_limit = limit.unwrap_or(1000).clamp(0, 10000); // Use clamp - - let query = format!( - "SELECT event_time, gateway_id, receipt_signer, query_id, api_key, user_id, \ - subgraph, result, response_time_ms, request_bytes, response_bytes, total_fees_usd \ - FROM qos_data {} ORDER BY event_time DESC LIMIT {} OFFSET {}", - where_clause, query_limit, query_offset - ); - - println!("Executing query: {}", query); - - let rows = client - .query(&query) - .fetch_all::() - .await - .map_err(|e| format!("Database query failed: {}", e))?; - - // Convert to GraphQL output type - Ok(rows - .into_iter() - .map(|row| QosReport { - event_time: Utc - .timestamp_opt(row.event_time as i64, 0) - .single() - .map_or_else(|| "Invalid Timestamp".to_string(), |dt| dt.to_rfc3339()), - gateway_id: row.gateway_id, - receipt_signer: row.receipt_signer, - query_id: row.query_id, - api_key: row.api_key, - user_id: row.user_id, - subgraph: row.subgraph, - result: row.result, - response_time_ms: row.response_time_ms, - request_bytes: row.request_bytes, - response_bytes: row.response_bytes, - total_fees_usd: row.total_fees_usd, - }) - .collect()) - } - - /// Query for the total count of QoS reports matching the criteria - async fn qos_reports_count( - &self, - _ctx: &Context<'_>, // Prefix ctx with underscore - time_range: TimeRangeInput, - #[graphql(desc = "Optional filters for the query")] filter: Option, - ) -> Result { - let client = get_clickhouse_client()?; - - // Build WHERE clause - let mut conditions = Vec::new(); - let from_dt = DateTime::parse_from_rfc3339(&time_range.from) - .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? - .with_timezone(&Utc); - let to_dt = DateTime::parse_from_rfc3339(&time_range.to) - .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? - .with_timezone(&Utc); - - conditions.push(format!("event_time >= toDateTime({})", from_dt.timestamp())); - conditions.push(format!("event_time < toDateTime({})", to_dt.timestamp())); - - if let Some(f) = filter { - if let Some(gw) = &f.gateway_id { - conditions.push(format!("gateway_id = '{}'", gw)); - } - if let Some(sg) = &f.subgraph { - // Handle nullable subgraph field correctly - conditions.push(format!("subgraph = '{}'", sg)); - } - // Filtering by indexer/allocation on raw reports requires arrayExists/has, - // which can be slow. Skipping for now. - // if let Some(ix) = &f.indexer { - // conditions.push(format!("has(indexer_queries.indexer, '{}')", ix)); - // } - // if let Some(alloc) = &f.allocation { - // conditions.push(format!("has(indexer_queries.allocation, '{}')", alloc)); - // } - } - - let where_clause = if conditions.is_empty() { - String::new() - } else { - format!("WHERE {}", conditions.join(" AND ")) - }; - - let query = format!("SELECT count(*) FROM qos_data {}", where_clause); - - println!("Executing query: {}", query); - - let count = client - .query(&query) - .fetch_one::() - .await - .map_err(|e| format!("Database query failed: {}", e))?; - - Ok(count.count) - } - /// Query for Deployment level aggregations async fn deployment_aggregations( &self, From 6f78105356498a78d4570a0b72874eda8308cfee Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Thu, 3 Apr 2025 00:19:39 -0300 Subject: [PATCH 14/20] feat: added filtering, sorting, default arguments to graphql api --- oracle/graphql-api/src/main.rs | 535 +++++++++++++++++++++++++-------- 1 file changed, 410 insertions(+), 125 deletions(-) diff --git a/oracle/graphql-api/src/main.rs b/oracle/graphql-api/src/main.rs index c08c345..4cfbd31 100644 --- a/oracle/graphql-api/src/main.rs +++ b/oracle/graphql-api/src/main.rs @@ -298,6 +298,204 @@ struct AggregationFilterInput { allocation: Option, } +// --- Enums and Structs for Sorting --- + +#[derive(Enum, Copy, Clone, Eq, PartialEq, Debug)] +enum SortDirection { + Asc, + Desc, +} + +// Define which fields can be sorted for Deployments +#[derive(Enum, Copy, Clone, Eq, PartialEq, Debug)] +enum DeploymentSortField { + TimeBucket, + Subgraph, + GatewayId, + QueryCount, + SuccessCount, + FailureCount, + AvgResponseTimeMs, + MaxResponseTimeMs, + P90ResponseTimeMs, + P99ResponseTimeMs, + TotalFeesUsd, + AvgFeeUsd, + MaxFeeUsd, + P90FeeUsd, + P99FeeUsd, + SuccessProportion, +} + +impl DeploymentSortField { + // Helper to get the corresponding ClickHouse column name + fn column_name(&self) -> &'static str { + match self { + DeploymentSortField::TimeBucket => "time_bucket", + DeploymentSortField::Subgraph => "subgraph", + DeploymentSortField::GatewayId => "gateway_id", + DeploymentSortField::QueryCount => "query_count", + DeploymentSortField::SuccessCount => "success_count", + DeploymentSortField::FailureCount => "failure_count", + DeploymentSortField::AvgResponseTimeMs => "avg_response_time_ms", + DeploymentSortField::MaxResponseTimeMs => "max_response_time_ms", + DeploymentSortField::P90ResponseTimeMs => "p90_response_time_ms", + DeploymentSortField::P99ResponseTimeMs => "p99_response_time_ms", + DeploymentSortField::TotalFeesUsd => "total_fees_usd", + DeploymentSortField::AvgFeeUsd => "avg_fee_usd", + DeploymentSortField::MaxFeeUsd => "max_fee_usd", + DeploymentSortField::P90FeeUsd => "p90_fee_usd", + DeploymentSortField::P99FeeUsd => "p99_fee_usd", + DeploymentSortField::SuccessProportion => "success_proportion", + } + } +} + +// Define which fields can be sorted for Indexers +#[derive(Enum, Copy, Clone, Eq, PartialEq, Debug)] +enum IndexerSortField { + TimeBucket, + Indexer, + GatewayId, + QueryCount, + SuccessCount, + FailureCount, + AvgIndexerResponseTimeMs, + MaxIndexerResponseTimeMs, + P90IndexerResponseTimeMs, + P99IndexerResponseTimeMs, + TotalFeeGrt, + AvgFeeGrt, + MaxFeeGrt, + P90FeeGrt, + P99FeeGrt, + AvgSecondsBehind, + MaxSecondsBehind, + P90SecondsBehind, + P99SecondsBehind, + AvgBlocksBehind, + MaxBlocksBehind, + P90BlocksBehind, + P99BlocksBehind, + SuccessProportion, +} + +impl IndexerSortField { + // Helper to get the corresponding ClickHouse column name + fn column_name(&self) -> &'static str { + match self { + IndexerSortField::TimeBucket => "time_bucket", + IndexerSortField::Indexer => "indexer", + IndexerSortField::GatewayId => "gateway_id", + IndexerSortField::QueryCount => "query_count", + IndexerSortField::SuccessCount => "success_count", + IndexerSortField::FailureCount => "failure_count", + IndexerSortField::AvgIndexerResponseTimeMs => "avg_indexer_response_time_ms", + IndexerSortField::MaxIndexerResponseTimeMs => "max_indexer_response_time_ms", + IndexerSortField::P90IndexerResponseTimeMs => "p90_indexer_response_time_ms", + IndexerSortField::P99IndexerResponseTimeMs => "p99_indexer_response_time_ms", + IndexerSortField::TotalFeeGrt => "total_fee_grt", + IndexerSortField::AvgFeeGrt => "avg_fee_grt", + IndexerSortField::MaxFeeGrt => "max_fee_grt", + IndexerSortField::P90FeeGrt => "p90_fee_grt", + IndexerSortField::P99FeeGrt => "p99_fee_grt", + IndexerSortField::AvgSecondsBehind => "avg_seconds_behind", + IndexerSortField::MaxSecondsBehind => "max_seconds_behind", + IndexerSortField::P90SecondsBehind => "p90_seconds_behind", + IndexerSortField::P99SecondsBehind => "p99_seconds_behind", + IndexerSortField::AvgBlocksBehind => "avg_blocks_behind", + IndexerSortField::MaxBlocksBehind => "max_blocks_behind", + IndexerSortField::P90BlocksBehind => "p90_blocks_behind", + IndexerSortField::P99BlocksBehind => "p99_blocks_behind", + IndexerSortField::SuccessProportion => "success_proportion", + } + } +} + +// Define which fields can be sorted for Allocations +#[derive(Enum, Copy, Clone, Eq, PartialEq, Debug)] +enum AllocationSortField { + TimeBucket, + Subgraph, + Indexer, + GatewayId, + QueryCount, + SuccessCount, + FailureCount, + AvgIndexerResponseTimeMs, + MaxIndexerResponseTimeMs, + P90IndexerResponseTimeMs, + P99IndexerResponseTimeMs, + TotalFeeGrt, + AvgFeeGrt, + MaxFeeGrt, + P90FeeGrt, + P99FeeGrt, + AvgSecondsBehind, + MaxSecondsBehind, + P90SecondsBehind, + P99SecondsBehind, + AvgBlocksBehind, + MaxBlocksBehind, + P90BlocksBehind, + P99BlocksBehind, + SuccessProportion, +} + +impl AllocationSortField { + // Helper to get the corresponding ClickHouse column name + fn column_name(&self) -> &'static str { + match self { + AllocationSortField::TimeBucket => "time_bucket", + AllocationSortField::Subgraph => "subgraph", + AllocationSortField::Indexer => "indexer", + AllocationSortField::GatewayId => "gateway_id", + AllocationSortField::QueryCount => "query_count", + AllocationSortField::SuccessCount => "success_count", + AllocationSortField::FailureCount => "failure_count", + AllocationSortField::AvgIndexerResponseTimeMs => "avg_indexer_response_time_ms", + AllocationSortField::MaxIndexerResponseTimeMs => "max_indexer_response_time_ms", + AllocationSortField::P90IndexerResponseTimeMs => "p90_indexer_response_time_ms", + AllocationSortField::P99IndexerResponseTimeMs => "p99_indexer_response_time_ms", + AllocationSortField::TotalFeeGrt => "total_fee_grt", + AllocationSortField::AvgFeeGrt => "avg_fee_grt", + AllocationSortField::MaxFeeGrt => "max_fee_grt", + AllocationSortField::P90FeeGrt => "p90_fee_grt", + AllocationSortField::P99FeeGrt => "p99_fee_grt", + AllocationSortField::AvgSecondsBehind => "avg_seconds_behind", + AllocationSortField::MaxSecondsBehind => "max_seconds_behind", + AllocationSortField::P90SecondsBehind => "p90_seconds_behind", + AllocationSortField::P99SecondsBehind => "p99_seconds_behind", + AllocationSortField::AvgBlocksBehind => "avg_blocks_behind", + AllocationSortField::MaxBlocksBehind => "max_blocks_behind", + AllocationSortField::P90BlocksBehind => "p90_blocks_behind", + AllocationSortField::P99BlocksBehind => "p99_blocks_behind", + AllocationSortField::SuccessProportion => "success_proportion", + } + } +} + +// Input object for specifying sorting - Generic enough for all types? +// Let's make specific ones for type safety in the resolver signature. + +#[derive(InputObject, Debug)] +struct DeploymentSortInput { + field: DeploymentSortField, + direction: Option, // Default to Desc +} + +#[derive(InputObject, Debug)] +struct IndexerSortInput { + field: IndexerSortField, + direction: Option, // Default to Desc +} + +#[derive(InputObject, Debug)] +struct AllocationSortInput { + field: AllocationSortField, + direction: Option, // Default to Desc +} + // --- GraphQL Query Root --- struct QueryRoot; @@ -307,24 +505,38 @@ impl QueryRoot { /// Query for Deployment level aggregations async fn deployment_aggregations( &self, - _ctx: &Context<'_>, // Prefix ctx with underscore - interval: AggregationInterval, - time_range: TimeRangeInput, + _ctx: &Context<'_>, + #[graphql(desc = "Aggregation interval (default: Hourly)")] interval: Option, + #[graphql(desc = "Time range (RFC3339 format, default: last 24 hours)")] time_range: Option, #[graphql(desc = "Optional filters for the query")] filter: Option, - #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] - limit: Option, + #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] limit: Option, + #[graphql(desc = "Optional sorting (default: time_bucket DESC)")] sort: Option, ) -> Result, String> { let client = get_clickhouse_client()?; - let view_name = format!("view_agg_deployment_{}", interval.table_suffix()); + + // --- Handle Defaults --- + let actual_interval = interval.unwrap_or(AggregationInterval::Hourly); // Default interval + let (from_dt, to_dt) = match time_range { + Some(tr) => ( + DateTime::parse_from_rfc3339(&tr.from) + .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? + .with_timezone(&Utc), + DateTime::parse_from_rfc3339(&tr.to) + .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? + .with_timezone(&Utc), + ), + None => { + // Default to last 24 hours + let now = Utc::now(); + (now - chrono::Duration::hours(24), now) + } + }; + // --- End Handle Defaults --- + + let view_name = format!("view_agg_deployment_{}", actual_interval.table_suffix()); // Build WHERE clause let mut conditions = Vec::new(); - let from_dt = DateTime::parse_from_rfc3339(&time_range.from) - .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? - .with_timezone(&Utc); - let to_dt = DateTime::parse_from_rfc3339(&time_range.to) - .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? - .with_timezone(&Utc); conditions.push(format!( "time_bucket >= toDateTime({})", from_dt.timestamp() @@ -333,18 +545,36 @@ impl QueryRoot { if let Some(f) = filter { if let Some(gw) = &f.gateway_id { - conditions.push(format!("gateway_id = '{}'", gw)); + // Basic validation/sanitization could be added here if needed + conditions.push(format!("gateway_id = '{}'", gw.replace('\'', "''"))); // Simple quote escape } if let Some(sg) = &f.subgraph { - conditions.push(format!("subgraph = '{}'", sg)); + conditions.push(format!("subgraph = '{}'", sg.replace('\'', "''"))); // Simple quote escape } // No indexer/allocation filters applicable here } let where_clause = format!("WHERE {}", conditions.join(" AND ")); - let query_limit = limit.unwrap_or(1000).clamp(0, 10000); // Use clamp + let query_limit = limit.unwrap_or(1000).clamp(1, 10000); // Ensure limit is at least 1 - // Query directly selects pre-aggregated columns, no further aggregation or GROUP BY + // --- Build ORDER BY Clause --- + let order_by_clause = match sort { + Some(s) => { + let direction = match s.direction.unwrap_or(SortDirection::Desc) { // Default direction + SortDirection::Asc => "ASC", + SortDirection::Desc => "DESC", + }; + // Map the enum field to the actual column name + format!("ORDER BY {} {}", s.field.column_name(), direction) + } + None => { + // Default sort order + "ORDER BY time_bucket DESC, subgraph ASC, gateway_id ASC".to_string() + } + }; + // --- End Build ORDER BY Clause --- + + // Query directly selects pre-aggregated columns let query = format!( "SELECT time_bucket, subgraph, gateway_id, \ query_count, success_count, failure_count, \ @@ -352,24 +582,23 @@ impl QueryRoot { total_fees_usd, avg_fee_usd, max_fee_usd, p90_fee_usd, p99_fee_usd, stddev_fee_usd, \ success_proportion \ FROM {} {} \ - ORDER BY time_bucket DESC, subgraph, gateway_id \ + {} \ LIMIT {}", - view_name, where_clause, query_limit + view_name, where_clause, order_by_clause, query_limit // Use dynamic order_by_clause ); - println!("Executing query: {}", query); + println!("Executing query: {}", query); // Keep for debugging if needed let rows = client .query(&query) - .fetch_all::() // Uses updated struct + .fetch_all::() .await .map_err(|e| format!("Database query failed: {}", e))?; - // Map directly, row-by-row, including new fields + // Map results (logic remains the same) Ok(rows .into_iter() .map(|row| DeploymentAggregationOutput { - // Uses updated struct time_bucket: Utc .timestamp_opt(row.time_bucket as i64, 0) .single() @@ -379,19 +608,20 @@ impl QueryRoot { query_count: row.query_count, success_count: row.success_count, failure_count: row.failure_count, - // Map all new fields, handling Option for NaN/Inf safety - avg_response_time_ms: Some(row.avg_response_time_ms), - max_response_time_ms: Some(row.max_response_time_ms), - p90_response_time_ms: Some(row.p90_response_time_ms), - p99_response_time_ms: Some(row.p99_response_time_ms), - stddev_response_time_ms: Some(row.stddev_response_time_ms), - total_fees_usd: Some(row.total_fees_usd), - avg_fee_usd: Some(row.avg_fee_usd), - max_fee_usd: Some(row.max_fee_usd), - p90_fee_usd: Some(row.p90_fee_usd), - p99_fee_usd: Some(row.p99_fee_usd), - stddev_fee_usd: Some(row.stddev_fee_usd), - success_proportion: Some(row.success_proportion), + // Wrap numeric fields in Option, handle potential NaN/Inf from ClickHouse Float64 if necessary + // Although our views calculate these, being defensive is good. + avg_response_time_ms: if row.avg_response_time_ms.is_finite() { Some(row.avg_response_time_ms) } else { None }, + max_response_time_ms: Some(row.max_response_time_ms), // u32 cannot be NaN/Inf + p90_response_time_ms: if row.p90_response_time_ms.is_finite() { Some(row.p90_response_time_ms) } else { None }, + p99_response_time_ms: if row.p99_response_time_ms.is_finite() { Some(row.p99_response_time_ms) } else { None }, + stddev_response_time_ms: if row.stddev_response_time_ms.is_finite() { Some(row.stddev_response_time_ms) } else { None }, + total_fees_usd: if row.total_fees_usd.is_finite() { Some(row.total_fees_usd) } else { None }, + avg_fee_usd: if row.avg_fee_usd.is_finite() { Some(row.avg_fee_usd) } else { None }, + max_fee_usd: if row.max_fee_usd.is_finite() { Some(row.max_fee_usd) } else { None }, + p90_fee_usd: if row.p90_fee_usd.is_finite() { Some(row.p90_fee_usd) } else { None }, + p99_fee_usd: if row.p99_fee_usd.is_finite() { Some(row.p99_fee_usd) } else { None }, + stddev_fee_usd: if row.stddev_fee_usd.is_finite() { Some(row.stddev_fee_usd) } else { None }, + success_proportion: if row.success_proportion.is_finite() { Some(row.success_proportion) } else { None }, }) .collect()) } @@ -399,25 +629,37 @@ impl QueryRoot { /// Query for Indexer level aggregations async fn indexer_aggregations( &self, - _ctx: &Context<'_>, // Prefix ctx with underscore - interval: AggregationInterval, - time_range: TimeRangeInput, + _ctx: &Context<'_>, + #[graphql(desc = "Aggregation interval (default: Hourly)")] interval: Option, + #[graphql(desc = "Time range (RFC3339 format, default: last 24 hours)")] time_range: Option, #[graphql(desc = "Optional filters for the query")] filter: Option, - #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] - limit: Option, + #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] limit: Option, + #[graphql(desc = "Optional sorting (default: time_bucket DESC)")] sort: Option, // Use IndexerSortInput ) -> Result, String> { - // Return updated Output struct let client = get_clickhouse_client()?; - let view_name = format!("view_agg_indexer_{}", interval.table_suffix()); - // Build WHERE clause (logic remains the same, lines 501-521) + // --- Handle Defaults (Similar to deployment_aggregations) --- + let actual_interval = interval.unwrap_or(AggregationInterval::Hourly); + let (from_dt, to_dt) = match time_range { + Some(tr) => ( + DateTime::parse_from_rfc3339(&tr.from) + .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? + .with_timezone(&Utc), + DateTime::parse_from_rfc3339(&tr.to) + .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? + .with_timezone(&Utc), + ), + None => { + let now = Utc::now(); + (now - chrono::Duration::hours(24), now) + } + }; + // --- End Handle Defaults --- + + let view_name = format!("view_agg_indexer_{}", actual_interval.table_suffix()); + + // Build WHERE clause (Similar logic, different filters) let mut conditions = Vec::new(); - let from_dt = DateTime::parse_from_rfc3339(&time_range.from) - .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? - .with_timezone(&Utc); - let to_dt = DateTime::parse_from_rfc3339(&time_range.to) - .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? - .with_timezone(&Utc); conditions.push(format!( "time_bucket >= toDateTime({})", from_dt.timestamp() @@ -426,16 +668,32 @@ impl QueryRoot { if let Some(f) = filter { if let Some(gw) = &f.gateway_id { - conditions.push(format!("gateway_id = '{}'", gw)); + conditions.push(format!("gateway_id = '{}'", gw.replace('\'', "''"))); } if let Some(ix) = &f.indexer { - conditions.push(format!("indexer = '{}'", ix)); + conditions.push(format!("indexer = '{}'", ix.replace('\'', "''"))); // Filter by indexer } - // No subgraph/allocation filters applicable here + // No subgraph/allocation filters applicable here } let where_clause = format!("WHERE {}", conditions.join(" AND ")); - let query_limit = limit.unwrap_or(1000).clamp(0, 10000); // Use clamp + let query_limit = limit.unwrap_or(1000).clamp(1, 10000); + + // --- Build ORDER BY Clause (Using IndexerSortField) --- + let order_by_clause = match sort { + Some(s) => { + let direction = match s.direction.unwrap_or(SortDirection::Desc) { + SortDirection::Asc => "ASC", + SortDirection::Desc => "DESC", + }; + format!("ORDER BY {} {}", s.field.column_name(), direction) // Use IndexerSortField mapping + } + None => { + // Default sort order + "ORDER BY time_bucket DESC, indexer ASC, gateway_id ASC".to_string() + } + }; + // --- End Build ORDER BY Clause --- // Update SELECT list to include all new fields let query = format!( @@ -449,24 +707,23 @@ impl QueryRoot { avg_blocks_behind, max_blocks_behind, p90_blocks_behind, p99_blocks_behind, stddev_blocks_behind, \ success_proportion \ FROM {} {} \ - ORDER BY time_bucket DESC, indexer ASC, gateway_id ASC \ + {} \ LIMIT {}", - view_name, where_clause, query_limit + view_name, where_clause, order_by_clause, query_limit // Use dynamic order_by_clause ); println!("Executing query: {}", query); let rows = client .query(&query) - .fetch_all::() // Use updated Row struct + .fetch_all::() // Use Indexer Row struct .await .map_err(|e| format!("Database query failed: {}", e))?; - // Map directly, row-by-row, including new fields + // Map results (Similar logic, different fields) Ok(rows .into_iter() .map(|row| IndexerAggregationOutput { - // Use updated Output struct time_bucket: Utc .timestamp_opt(row.time_bucket as i64, 0) .single() @@ -476,29 +733,29 @@ impl QueryRoot { query_count: row.query_count, success_count: row.success_count, failure_count: row.failure_count, - // Map all new fields, handling Option for NaN/Inf safety - avg_indexer_response_time_ms: Some(row.avg_indexer_response_time_ms), + // Map all fields, handling Option for NaN/Inf safety + avg_indexer_response_time_ms: if row.avg_indexer_response_time_ms.is_finite() { Some(row.avg_indexer_response_time_ms) } else { None }, max_indexer_response_time_ms: Some(row.max_indexer_response_time_ms), - p90_indexer_response_time_ms: Some(row.p90_indexer_response_time_ms), - p99_indexer_response_time_ms: Some(row.p99_indexer_response_time_ms), - stddev_indexer_response_time_ms: Some(row.stddev_indexer_response_time_ms), - total_fee_grt: Some(row.total_fee_grt), - avg_fee_grt: Some(row.avg_fee_grt), - max_fee_grt: Some(row.max_fee_grt), - p90_fee_grt: Some(row.p90_fee_grt), - p99_fee_grt: Some(row.p99_fee_grt), - stddev_fee_grt: Some(row.stddev_fee_grt), - avg_seconds_behind: Some(row.avg_seconds_behind), + p90_indexer_response_time_ms: if row.p90_indexer_response_time_ms.is_finite() { Some(row.p90_indexer_response_time_ms) } else { None }, + p99_indexer_response_time_ms: if row.p99_indexer_response_time_ms.is_finite() { Some(row.p99_indexer_response_time_ms) } else { None }, + stddev_indexer_response_time_ms: if row.stddev_indexer_response_time_ms.is_finite() { Some(row.stddev_indexer_response_time_ms) } else { None }, + total_fee_grt: if row.total_fee_grt.is_finite() { Some(row.total_fee_grt) } else { None }, + avg_fee_grt: if row.avg_fee_grt.is_finite() { Some(row.avg_fee_grt) } else { None }, + max_fee_grt: if row.max_fee_grt.is_finite() { Some(row.max_fee_grt) } else { None }, + p90_fee_grt: if row.p90_fee_grt.is_finite() { Some(row.p90_fee_grt) } else { None }, + p99_fee_grt: if row.p99_fee_grt.is_finite() { Some(row.p99_fee_grt) } else { None }, + stddev_fee_grt: if row.stddev_fee_grt.is_finite() { Some(row.stddev_fee_grt) } else { None }, + avg_seconds_behind: if row.avg_seconds_behind.is_finite() { Some(row.avg_seconds_behind) } else { None }, max_seconds_behind: Some(row.max_seconds_behind), - p90_seconds_behind: Some(row.p90_seconds_behind), - p99_seconds_behind: Some(row.p99_seconds_behind), - stddev_seconds_behind: Some(row.stddev_seconds_behind), - avg_blocks_behind: Some(row.avg_blocks_behind), + p90_seconds_behind: if row.p90_seconds_behind.is_finite() { Some(row.p90_seconds_behind) } else { None }, + p99_seconds_behind: if row.p99_seconds_behind.is_finite() { Some(row.p99_seconds_behind) } else { None }, + stddev_seconds_behind: if row.stddev_seconds_behind.is_finite() { Some(row.stddev_seconds_behind) } else { None }, + avg_blocks_behind: if row.avg_blocks_behind.is_finite() { Some(row.avg_blocks_behind) } else { None }, max_blocks_behind: Some(row.max_blocks_behind), - p90_blocks_behind: Some(row.p90_blocks_behind), - p99_blocks_behind: Some(row.p99_blocks_behind), - stddev_blocks_behind: Some(row.stddev_blocks_behind), - success_proportion: Some(row.success_proportion), + p90_blocks_behind: if row.p90_blocks_behind.is_finite() { Some(row.p90_blocks_behind) } else { None }, + p99_blocks_behind: if row.p99_blocks_behind.is_finite() { Some(row.p99_blocks_behind) } else { None }, + stddev_blocks_behind: if row.stddev_blocks_behind.is_finite() { Some(row.stddev_blocks_behind) } else { None }, + success_proportion: if row.success_proportion.is_finite() { Some(row.success_proportion) } else { None }, }) .collect()) } @@ -506,24 +763,37 @@ impl QueryRoot { /// Query for Allocation level aggregations (grouped by subgraph and indexer) async fn allocation_aggregations( &self, - _ctx: &Context<'_>, // Prefix ctx with underscore - interval: AggregationInterval, - time_range: TimeRangeInput, + _ctx: &Context<'_>, + #[graphql(desc = "Aggregation interval (default: Hourly)")] interval: Option, + #[graphql(desc = "Time range (RFC3339 format, default: last 24 hours)")] time_range: Option, #[graphql(desc = "Optional filters for the query")] filter: Option, - #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] - limit: Option, + #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] limit: Option, + #[graphql(desc = "Optional sorting (default: time_bucket DESC)")] sort: Option, // Use AllocationSortInput ) -> Result, String> { - let client = get_clickhouse_client()?; - let view_name = format!("view_agg_allocation_{}", interval.table_suffix()); + let client = get_clickhouse_client()?; - // Build WHERE clause + // --- Handle Defaults (Similar to deployment_aggregations) --- + let actual_interval = interval.unwrap_or(AggregationInterval::Hourly); + let (from_dt, to_dt) = match time_range { + Some(tr) => ( + DateTime::parse_from_rfc3339(&tr.from) + .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? + .with_timezone(&Utc), + DateTime::parse_from_rfc3339(&tr.to) + .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? + .with_timezone(&Utc), + ), + None => { + let now = Utc::now(); + (now - chrono::Duration::hours(24), now) + } + }; + // --- End Handle Defaults --- + + let view_name = format!("view_agg_allocation_{}", actual_interval.table_suffix()); + + // Build WHERE clause (Includes subgraph and indexer filters) let mut conditions = Vec::new(); - let from_dt = DateTime::parse_from_rfc3339(&time_range.from) - .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? - .with_timezone(&Utc); - let to_dt = DateTime::parse_from_rfc3339(&time_range.to) - .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? - .with_timezone(&Utc); conditions.push(format!( "time_bucket >= toDateTime({})", from_dt.timestamp() @@ -532,20 +802,37 @@ impl QueryRoot { if let Some(f) = filter { if let Some(gw) = &f.gateway_id { - conditions.push(format!("gateway_id = '{}'", gw)); + conditions.push(format!("gateway_id = '{}'", gw.replace('\'', "''"))); } - if let Some(sg) = &f.subgraph { - conditions.push(format!("subgraph = '{}'", sg)); + if let Some(sg) = &f.subgraph { + conditions.push(format!("subgraph = '{}'", sg.replace('\'', "''"))); } if let Some(ix) = &f.indexer { - conditions.push(format!("indexer = '{}'", ix)); + conditions.push(format!("indexer = '{}'", ix.replace('\'', "''"))); } + // Note: Allocation ID filter is not used here as the view aggregates by subgraph/indexer } let where_clause = format!("WHERE {}", conditions.join(" AND ")); - let query_limit = limit.unwrap_or(1000).clamp(0, 10000); // Use clamp + let query_limit = limit.unwrap_or(1000).clamp(1, 10000); + + // --- Build ORDER BY Clause (Using AllocationSortField) --- + let order_by_clause = match sort { + Some(s) => { + let direction = match s.direction.unwrap_or(SortDirection::Desc) { + SortDirection::Asc => "ASC", + SortDirection::Desc => "DESC", + }; + format!("ORDER BY {} {}", s.field.column_name(), direction) // Use AllocationSortField mapping + } + None => { + // Default sort order + "ORDER BY time_bucket DESC, subgraph ASC, indexer ASC, gateway_id ASC".to_string() + } + }; + // --- End Build ORDER BY Clause --- - // Query directly selects pre-aggregated columns, no further aggregation or GROUP BY + // Query uses the same fields as indexer aggregation but groups differently in the view let query = format!( "SELECT time_bucket, subgraph, indexer, gateway_id, \ query_count, success_count, failure_count, \ @@ -557,57 +844,55 @@ impl QueryRoot { avg_blocks_behind, max_blocks_behind, p90_blocks_behind, p99_blocks_behind, stddev_blocks_behind, \ success_proportion \ FROM {} {} \ - ORDER BY time_bucket DESC, subgraph ASC, indexer ASC, gateway_id ASC \ + {} \ LIMIT {}", - view_name, where_clause, query_limit + view_name, where_clause, order_by_clause, query_limit // Use dynamic order_by_clause ); println!("Executing query: {}", query); let rows = client .query(&query) - .fetch_all::() // Uses existing struct + .fetch_all::() // Use Allocation Row struct .await .map_err(|e| format!("Database query failed: {}", e))?; - // Map directly, row-by-row, including new fields + // Map results (Identical mapping logic to Indexer, just different input row type) Ok(rows .into_iter() .map(|row| AllocationAggregationOutput { - // Uses existing struct - time_bucket: Utc + time_bucket: Utc .timestamp_opt(row.time_bucket as i64, 0) .single() .map_or_else(|| "Invalid Timestamp".to_string(), |dt| dt.to_rfc3339()), - subgraph: row.subgraph, + subgraph: row.subgraph, // Allocation includes subgraph indexer: row.indexer, gateway_id: row.gateway_id, query_count: row.query_count, success_count: row.success_count, failure_count: row.failure_count, - // Map all new fields (same mapping logic as Indexer) - avg_indexer_response_time_ms: Some(row.avg_indexer_response_time_ms), + avg_indexer_response_time_ms: if row.avg_indexer_response_time_ms.is_finite() { Some(row.avg_indexer_response_time_ms) } else { None }, max_indexer_response_time_ms: Some(row.max_indexer_response_time_ms), - p90_indexer_response_time_ms: Some(row.p90_indexer_response_time_ms), - p99_indexer_response_time_ms: Some(row.p99_indexer_response_time_ms), - stddev_indexer_response_time_ms: Some(row.stddev_indexer_response_time_ms), - total_fee_grt: Some(row.total_fee_grt), - avg_fee_grt: Some(row.avg_fee_grt), - max_fee_grt: Some(row.max_fee_grt), - p90_fee_grt: Some(row.p90_fee_grt), - p99_fee_grt: Some(row.p99_fee_grt), - stddev_fee_grt: Some(row.stddev_fee_grt), - avg_seconds_behind: Some(row.avg_seconds_behind), + p90_indexer_response_time_ms: if row.p90_indexer_response_time_ms.is_finite() { Some(row.p90_indexer_response_time_ms) } else { None }, + p99_indexer_response_time_ms: if row.p99_indexer_response_time_ms.is_finite() { Some(row.p99_indexer_response_time_ms) } else { None }, + stddev_indexer_response_time_ms: if row.stddev_indexer_response_time_ms.is_finite() { Some(row.stddev_indexer_response_time_ms) } else { None }, + total_fee_grt: if row.total_fee_grt.is_finite() { Some(row.total_fee_grt) } else { None }, + avg_fee_grt: if row.avg_fee_grt.is_finite() { Some(row.avg_fee_grt) } else { None }, + max_fee_grt: if row.max_fee_grt.is_finite() { Some(row.max_fee_grt) } else { None }, + p90_fee_grt: if row.p90_fee_grt.is_finite() { Some(row.p90_fee_grt) } else { None }, + p99_fee_grt: if row.p99_fee_grt.is_finite() { Some(row.p99_fee_grt) } else { None }, + stddev_fee_grt: if row.stddev_fee_grt.is_finite() { Some(row.stddev_fee_grt) } else { None }, + avg_seconds_behind: if row.avg_seconds_behind.is_finite() { Some(row.avg_seconds_behind) } else { None }, max_seconds_behind: Some(row.max_seconds_behind), - p90_seconds_behind: Some(row.p90_seconds_behind), - p99_seconds_behind: Some(row.p99_seconds_behind), - stddev_seconds_behind: Some(row.stddev_seconds_behind), - avg_blocks_behind: Some(row.avg_blocks_behind), + p90_seconds_behind: if row.p90_seconds_behind.is_finite() { Some(row.p90_seconds_behind) } else { None }, + p99_seconds_behind: if row.p99_seconds_behind.is_finite() { Some(row.p99_seconds_behind) } else { None }, + stddev_seconds_behind: if row.stddev_seconds_behind.is_finite() { Some(row.stddev_seconds_behind) } else { None }, + avg_blocks_behind: if row.avg_blocks_behind.is_finite() { Some(row.avg_blocks_behind) } else { None }, max_blocks_behind: Some(row.max_blocks_behind), - p90_blocks_behind: Some(row.p90_blocks_behind), - p99_blocks_behind: Some(row.p99_blocks_behind), - stddev_blocks_behind: Some(row.stddev_blocks_behind), - success_proportion: Some(row.success_proportion), + p90_blocks_behind: if row.p90_blocks_behind.is_finite() { Some(row.p90_blocks_behind) } else { None }, + p99_blocks_behind: if row.p99_blocks_behind.is_finite() { Some(row.p99_blocks_behind) } else { None }, + stddev_blocks_behind: if row.stddev_blocks_behind.is_finite() { Some(row.stddev_blocks_behind) } else { None }, + success_proportion: if row.success_proportion.is_finite() { Some(row.success_proportion) } else { None }, }) .collect()) } From bae04dce22465e7ed95db731c08a51837d6f5002 Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Thu, 3 Apr 2025 00:58:12 -0300 Subject: [PATCH 15/20] feat: add better input sanitization and improved timeRange filtering --- Cargo.lock | 2 + oracle/graphql-api/Cargo.toml | 4 +- oracle/graphql-api/src/main.rs | 740 ++++++++++++++++++++++++--------- 3 files changed, 558 insertions(+), 188 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67fcfc5..6c575c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1838,6 +1838,8 @@ dependencies = [ "clickhouse", "futures", "hex", + "once_cell", + "regex", "serde", "serde_json", "tokio", diff --git a/oracle/graphql-api/Cargo.toml b/oracle/graphql-api/Cargo.toml index e466e97..476a7d8 100644 --- a/oracle/graphql-api/Cargo.toml +++ b/oracle/graphql-api/Cargo.toml @@ -18,4 +18,6 @@ chrono = "0.4" anyhow = "1.0.79" tracing = { version = "0.1.40", features = ["attributes"] } tracing-subscriber = { version = "0.3.18", features = ["env-filter", "json"] } -axum = "0.8.1" \ No newline at end of file +axum = "0.8.1" +regex = "1" +once_cell = "1" \ No newline at end of file diff --git a/oracle/graphql-api/src/main.rs b/oracle/graphql-api/src/main.rs index 4cfbd31..54f17bf 100644 --- a/oracle/graphql-api/src/main.rs +++ b/oracle/graphql-api/src/main.rs @@ -5,10 +5,49 @@ use async_graphql::{ use async_graphql_actix_web::{GraphQLRequest, GraphQLResponse}; use chrono::{DateTime, TimeZone, Utc}; use clickhouse::{Client, Row}; +use once_cell::sync::Lazy; +use regex::Regex; use serde::Deserialize; use std::env; use tokio::signal::unix::{signal, SignalKind}; +// --- Input Validation Regexes --- +// Define allowed characters/patterns. Adjust these based on actual expected formats. +static GATEWAY_ID_REGEX: Lazy = Lazy::new(|| Regex::new(r"^[a-zA-Z0-9_-]+$").unwrap()); +// Allow alphanumeric, hyphens, slashes (e.g., for ENS names/subgraph names) +static SUBGRAPH_REGEX: Lazy = Lazy::new(|| Regex::new(r"^[a-zA-Z0-9_/-]+$").unwrap()); +// Allow hex characters, optionally prefixed with 0x +static INDEXER_REGEX: Lazy = Lazy::new(|| Regex::new(r"^(0x)?[a-fA-F0-9]+$").unwrap()); + +// --- ClickHouse String Escaping --- +/// Escapes a string for safe inclusion within single quotes in a ClickHouse SQL query. +/// Replaces backslashes (`\`) with `\\` and single quotes (`'`) with `\'`. +fn escape_clickhouse_string_literal(input: &str) -> String { + input.replace('\\', r"\\").replace('\'', r"\'") +} + +// --- Time Parsing Helper --- +/// Parses a string as either a Unix timestamp (seconds) or an RFC3339 datetime. +fn parse_datetime_input(input: &str) -> Result, String> { + // Try parsing as Unix timestamp (integer seconds) first + if let Ok(ts) = input.parse::() { + match Utc.timestamp_opt(ts, 0).single() { + Some(dt) => Ok(dt), + None => Err(format!("Invalid Unix timestamp value: {}", ts)), + } + } else { + // Fallback to parsing as RFC3339 + DateTime::parse_from_rfc3339(input) + .map(|dt| dt.with_timezone(&Utc)) + .map_err(|e| { + format!( + "Invalid format: Expected Unix timestamp or RFC3339. Parse error: {}", + e + ) + }) + } +} + // --- Structs for Aggregated Data --- #[derive(Enum, Copy, Clone, Eq, PartialEq, Debug)] @@ -278,15 +317,13 @@ struct AllocationAggregationOutput { // --- Input Objects for Filtering --- -#[derive(InputObject, Debug)] +#[derive(InputObject, Debug, Clone)] struct TimeRangeInput { - /// Start time (inclusive), RFC3339 format (e.g., "2023-01-01T00:00:00Z") - from: String, - /// End time (exclusive), RFC3339 format (e.g., "2023-01-02T00:00:00Z") - to: String, + from: Option, + to: Option, } -#[derive(InputObject, Debug)] +#[derive(InputObject, Debug, Clone)] struct AggregationFilterInput { /// Optional: Filter by gateway ID gateway_id: Option, @@ -506,75 +543,118 @@ impl QueryRoot { async fn deployment_aggregations( &self, _ctx: &Context<'_>, - #[graphql(desc = "Aggregation interval (default: Hourly)")] interval: Option, - #[graphql(desc = "Time range (RFC3339 format, default: last 24 hours)")] time_range: Option, + #[graphql(desc = "Aggregation interval (default: Hourly)")] interval: Option< + AggregationInterval, + >, + #[graphql( + desc = "Optional time range filter. If omitted, defaults to the last 24 hours. \ + If provided, requires at least 'from' or 'to'." + )] + time_range: Option, #[graphql(desc = "Optional filters for the query")] filter: Option, - #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] limit: Option, - #[graphql(desc = "Optional sorting (default: time_bucket DESC)")] sort: Option, + #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] + limit: Option, + #[graphql(desc = "Optional sorting (default: time_bucket DESC)")] sort: Option< + DeploymentSortInput, + >, ) -> Result, String> { let client = get_clickhouse_client()?; + let actual_interval = interval.unwrap_or(AggregationInterval::Hourly); + let view_name = format!("view_agg_deployment_{}", actual_interval.table_suffix()); - // --- Handle Defaults --- - let actual_interval = interval.unwrap_or(AggregationInterval::Hourly); // Default interval - let (from_dt, to_dt) = match time_range { - Some(tr) => ( - DateTime::parse_from_rfc3339(&tr.from) - .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? - .with_timezone(&Utc), - DateTime::parse_from_rfc3339(&tr.to) - .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? - .with_timezone(&Utc), - ), + // --- Build WHERE clause --- + let mut conditions = Vec::new(); + + // --- Handle Time Range --- + match time_range { + // Case 1: time_range argument is completely omitted - use default None => { - // Default to last 24 hours let now = Utc::now(); - (now - chrono::Duration::hours(24), now) + let default_from = now - chrono::Duration::hours(24); + conditions.push(format!( + "time_bucket >= toDateTime({})", + default_from.timestamp() + )); + conditions.push(format!("time_bucket < toDateTime({})", now.timestamp())); } - }; - // --- End Handle Defaults --- - - let view_name = format!("view_agg_deployment_{}", actual_interval.table_suffix()); - - // Build WHERE clause - let mut conditions = Vec::new(); - conditions.push(format!( - "time_bucket >= toDateTime({})", - from_dt.timestamp() - )); - conditions.push(format!("time_bucket < toDateTime({})", to_dt.timestamp())); + // Case 2: time_range argument is provided + Some(tr) => { + let parsed_from = tr.from.as_deref().map(parse_datetime_input).transpose()?; + let parsed_to = tr.to.as_deref().map(parse_datetime_input).transpose()?; + + match (parsed_from, parsed_to) { + (None, None) => { + // If time_range was provided but both fields are null/missing + return Err( + "Time range filter requires at least 'from' or 'to' to be specified." + .to_string(), + ); + } + (Some(from_dt), None) => { + conditions.push(format!( + "time_bucket >= toDateTime({})", + from_dt.timestamp() + )); + } + (None, Some(to_dt)) => { + conditions.push(format!("time_bucket < toDateTime({})", to_dt.timestamp())); + } + (Some(from_dt), Some(to_dt)) => { + if from_dt >= to_dt { + return Err("'from' time must be earlier than 'to' time.".to_string()); + } + conditions.push(format!( + "time_bucket >= toDateTime({})", + from_dt.timestamp() + )); + conditions.push(format!("time_bucket < toDateTime({})", to_dt.timestamp())); + } + } + } + } + // --- End Handle Time Range --- + // --- Handle Other Filters (gateway_id, subgraph) --- if let Some(f) = filter { if let Some(gw) = &f.gateway_id { - // Basic validation/sanitization could be added here if needed - conditions.push(format!("gateway_id = '{}'", gw.replace('\'', "''"))); // Simple quote escape + // --- Validate and Escape gateway_id --- + if !GATEWAY_ID_REGEX.is_match(gw) { + return Err("Invalid format for gateway_id filter".to_string()); + } + let escaped_gw = escape_clickhouse_string_literal(gw); + conditions.push(format!("gateway_id = '{}'", escaped_gw)); + // --- End Validation --- } if let Some(sg) = &f.subgraph { - conditions.push(format!("subgraph = '{}'", sg.replace('\'', "''"))); // Simple quote escape + // --- Validate and Escape subgraph --- + if !SUBGRAPH_REGEX.is_match(sg) { + return Err("Invalid format for subgraph filter".to_string()); + } + let escaped_sg = escape_clickhouse_string_literal(sg); + conditions.push(format!("subgraph = '{}'", escaped_sg)); + // --- End Validation --- } - // No indexer/allocation filters applicable here + // No indexer filter applicable here } + // --- End Handle Other Filters --- + // Should always have at least one time condition now let where_clause = format!("WHERE {}", conditions.join(" AND ")); - let query_limit = limit.unwrap_or(1000).clamp(1, 10000); // Ensure limit is at least 1 + let query_limit = limit.unwrap_or(1000).clamp(1, 10000); // --- Build ORDER BY Clause --- let order_by_clause = match sort { Some(s) => { - let direction = match s.direction.unwrap_or(SortDirection::Desc) { // Default direction + let direction = match s.direction.unwrap_or(SortDirection::Desc) { SortDirection::Asc => "ASC", SortDirection::Desc => "DESC", }; - // Map the enum field to the actual column name format!("ORDER BY {} {}", s.field.column_name(), direction) } - None => { - // Default sort order - "ORDER BY time_bucket DESC, subgraph ASC, gateway_id ASC".to_string() - } + None => "ORDER BY time_bucket DESC, subgraph ASC, gateway_id ASC".to_string(), }; - // --- End Build ORDER BY Clause --- - // Query directly selects pre-aggregated columns + // --- Build and Execute Query --- let query = format!( "SELECT time_bucket, subgraph, gateway_id, \ query_count, success_count, failure_count, \ @@ -584,10 +664,10 @@ impl QueryRoot { FROM {} {} \ {} \ LIMIT {}", - view_name, where_clause, order_by_clause, query_limit // Use dynamic order_by_clause + view_name, where_clause, order_by_clause, query_limit ); - println!("Executing query: {}", query); // Keep for debugging if needed + println!("Executing query: {}", query); let rows = client .query(&query) @@ -608,20 +688,62 @@ impl QueryRoot { query_count: row.query_count, success_count: row.success_count, failure_count: row.failure_count, - // Wrap numeric fields in Option, handle potential NaN/Inf from ClickHouse Float64 if necessary - // Although our views calculate these, being defensive is good. - avg_response_time_ms: if row.avg_response_time_ms.is_finite() { Some(row.avg_response_time_ms) } else { None }, - max_response_time_ms: Some(row.max_response_time_ms), // u32 cannot be NaN/Inf - p90_response_time_ms: if row.p90_response_time_ms.is_finite() { Some(row.p90_response_time_ms) } else { None }, - p99_response_time_ms: if row.p99_response_time_ms.is_finite() { Some(row.p99_response_time_ms) } else { None }, - stddev_response_time_ms: if row.stddev_response_time_ms.is_finite() { Some(row.stddev_response_time_ms) } else { None }, - total_fees_usd: if row.total_fees_usd.is_finite() { Some(row.total_fees_usd) } else { None }, - avg_fee_usd: if row.avg_fee_usd.is_finite() { Some(row.avg_fee_usd) } else { None }, - max_fee_usd: if row.max_fee_usd.is_finite() { Some(row.max_fee_usd) } else { None }, - p90_fee_usd: if row.p90_fee_usd.is_finite() { Some(row.p90_fee_usd) } else { None }, - p99_fee_usd: if row.p99_fee_usd.is_finite() { Some(row.p99_fee_usd) } else { None }, - stddev_fee_usd: if row.stddev_fee_usd.is_finite() { Some(row.stddev_fee_usd) } else { None }, - success_proportion: if row.success_proportion.is_finite() { Some(row.success_proportion) } else { None }, + avg_response_time_ms: if row.avg_response_time_ms.is_finite() { + Some(row.avg_response_time_ms) + } else { + None + }, + max_response_time_ms: Some(row.max_response_time_ms), + p90_response_time_ms: if row.p90_response_time_ms.is_finite() { + Some(row.p90_response_time_ms) + } else { + None + }, + p99_response_time_ms: if row.p99_response_time_ms.is_finite() { + Some(row.p99_response_time_ms) + } else { + None + }, + stddev_response_time_ms: if row.stddev_response_time_ms.is_finite() { + Some(row.stddev_response_time_ms) + } else { + None + }, + total_fees_usd: if row.total_fees_usd.is_finite() { + Some(row.total_fees_usd) + } else { + None + }, + avg_fee_usd: if row.avg_fee_usd.is_finite() { + Some(row.avg_fee_usd) + } else { + None + }, + max_fee_usd: if row.max_fee_usd.is_finite() { + Some(row.max_fee_usd) + } else { + None + }, + p90_fee_usd: if row.p90_fee_usd.is_finite() { + Some(row.p90_fee_usd) + } else { + None + }, + p99_fee_usd: if row.p99_fee_usd.is_finite() { + Some(row.p99_fee_usd) + } else { + None + }, + stddev_fee_usd: if row.stddev_fee_usd.is_finite() { + Some(row.stddev_fee_usd) + } else { + None + }, + success_proportion: if row.success_proportion.is_finite() { + Some(row.success_proportion) + } else { + None + }, }) .collect()) } @@ -630,51 +752,97 @@ impl QueryRoot { async fn indexer_aggregations( &self, _ctx: &Context<'_>, - #[graphql(desc = "Aggregation interval (default: Hourly)")] interval: Option, - #[graphql(desc = "Time range (RFC3339 format, default: last 24 hours)")] time_range: Option, + #[graphql(desc = "Aggregation interval (default: Hourly)")] interval: Option< + AggregationInterval, + >, + #[graphql( + desc = "Optional time range filter. If omitted, defaults to the last 24 hours. \ + If provided, requires at least 'from' or 'to'." + )] + time_range: Option, #[graphql(desc = "Optional filters for the query")] filter: Option, - #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] limit: Option, - #[graphql(desc = "Optional sorting (default: time_bucket DESC)")] sort: Option, // Use IndexerSortInput + #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] + limit: Option, + #[graphql(desc = "Optional sorting (default: time_bucket DESC)")] sort: Option< + IndexerSortInput, + >, ) -> Result, String> { let client = get_clickhouse_client()?; - - // --- Handle Defaults (Similar to deployment_aggregations) --- let actual_interval = interval.unwrap_or(AggregationInterval::Hourly); - let (from_dt, to_dt) = match time_range { - Some(tr) => ( - DateTime::parse_from_rfc3339(&tr.from) - .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? - .with_timezone(&Utc), - DateTime::parse_from_rfc3339(&tr.to) - .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? - .with_timezone(&Utc), - ), - None => { - let now = Utc::now(); - (now - chrono::Duration::hours(24), now) - } - }; - // --- End Handle Defaults --- - let view_name = format!("view_agg_indexer_{}", actual_interval.table_suffix()); - // Build WHERE clause (Similar logic, different filters) + // --- Build WHERE clause --- let mut conditions = Vec::new(); - conditions.push(format!( - "time_bucket >= toDateTime({})", - from_dt.timestamp() - )); - conditions.push(format!("time_bucket < toDateTime({})", to_dt.timestamp())); + // --- Handle Time Range (Apply same logic as deployment_aggregations) --- + match time_range { + None => { + let now = Utc::now(); + let default_from = now - chrono::Duration::hours(24); + conditions.push(format!( + "time_bucket >= toDateTime({})", + default_from.timestamp() + )); + conditions.push(format!("time_bucket < toDateTime({})", now.timestamp())); + } + Some(tr) => { + let parsed_from = tr.from.as_deref().map(parse_datetime_input).transpose()?; + let parsed_to = tr.to.as_deref().map(parse_datetime_input).transpose()?; + + match (parsed_from, parsed_to) { + (None, None) => { + return Err( + "Time range filter requires at least 'from' or 'to' to be specified." + .to_string(), + ); + } + (Some(from_dt), None) => { + conditions.push(format!( + "time_bucket >= toDateTime({})", + from_dt.timestamp() + )); + } + (None, Some(to_dt)) => { + conditions.push(format!("time_bucket < toDateTime({})", to_dt.timestamp())); + } + (Some(from_dt), Some(to_dt)) => { + if from_dt >= to_dt { + return Err("'from' time must be earlier than 'to' time.".to_string()); + } + conditions.push(format!( + "time_bucket >= toDateTime({})", + from_dt.timestamp() + )); + conditions.push(format!("time_bucket < toDateTime({})", to_dt.timestamp())); + } + } + } + } + // --- End Handle Time Range --- + + // --- Handle Other Filters (gateway_id, indexer) --- if let Some(f) = filter { if let Some(gw) = &f.gateway_id { - conditions.push(format!("gateway_id = '{}'", gw.replace('\'', "''"))); + // --- Validate and Escape gateway_id --- + if !GATEWAY_ID_REGEX.is_match(gw) { + return Err("Invalid format for gateway_id filter".to_string()); + } + let escaped_gw = escape_clickhouse_string_literal(gw); + conditions.push(format!("gateway_id = '{}'", escaped_gw)); + // --- End Validation --- } if let Some(ix) = &f.indexer { - conditions.push(format!("indexer = '{}'", ix.replace('\'', "''"))); // Filter by indexer + // --- Validate and Escape indexer --- + if !INDEXER_REGEX.is_match(ix) { + return Err("Invalid format for indexer filter".to_string()); + } + let escaped_ix = escape_clickhouse_string_literal(ix); + conditions.push(format!("indexer = '{}'", escaped_ix)); + // --- End Validation --- } - // No subgraph/allocation filters applicable here + // No subgraph filter applicable here } + // --- End Handle Other Filters --- let where_clause = format!("WHERE {}", conditions.join(" AND ")); let query_limit = limit.unwrap_or(1000).clamp(1, 10000); @@ -686,16 +854,12 @@ impl QueryRoot { SortDirection::Asc => "ASC", SortDirection::Desc => "DESC", }; - format!("ORDER BY {} {}", s.field.column_name(), direction) // Use IndexerSortField mapping - } - None => { - // Default sort order - "ORDER BY time_bucket DESC, indexer ASC, gateway_id ASC".to_string() + format!("ORDER BY {} {}", s.field.column_name(), direction) } + None => "ORDER BY time_bucket DESC, indexer ASC, gateway_id ASC".to_string(), }; - // --- End Build ORDER BY Clause --- - // Update SELECT list to include all new fields + // --- Build and Execute Query --- let query = format!( "SELECT time_bucket, indexer, gateway_id, \ query_count, success_count, failure_count, \ @@ -709,18 +873,18 @@ impl QueryRoot { FROM {} {} \ {} \ LIMIT {}", - view_name, where_clause, order_by_clause, query_limit // Use dynamic order_by_clause + view_name, where_clause, order_by_clause, query_limit ); println!("Executing query: {}", query); let rows = client .query(&query) - .fetch_all::() // Use Indexer Row struct + .fetch_all::() .await .map_err(|e| format!("Database query failed: {}", e))?; - // Map results (Similar logic, different fields) + // Map results (logic remains the same) Ok(rows .into_iter() .map(|row| IndexerAggregationOutput { @@ -733,29 +897,105 @@ impl QueryRoot { query_count: row.query_count, success_count: row.success_count, failure_count: row.failure_count, - // Map all fields, handling Option for NaN/Inf safety - avg_indexer_response_time_ms: if row.avg_indexer_response_time_ms.is_finite() { Some(row.avg_indexer_response_time_ms) } else { None }, + avg_indexer_response_time_ms: if row.avg_indexer_response_time_ms.is_finite() { + Some(row.avg_indexer_response_time_ms) + } else { + None + }, max_indexer_response_time_ms: Some(row.max_indexer_response_time_ms), - p90_indexer_response_time_ms: if row.p90_indexer_response_time_ms.is_finite() { Some(row.p90_indexer_response_time_ms) } else { None }, - p99_indexer_response_time_ms: if row.p99_indexer_response_time_ms.is_finite() { Some(row.p99_indexer_response_time_ms) } else { None }, - stddev_indexer_response_time_ms: if row.stddev_indexer_response_time_ms.is_finite() { Some(row.stddev_indexer_response_time_ms) } else { None }, - total_fee_grt: if row.total_fee_grt.is_finite() { Some(row.total_fee_grt) } else { None }, - avg_fee_grt: if row.avg_fee_grt.is_finite() { Some(row.avg_fee_grt) } else { None }, - max_fee_grt: if row.max_fee_grt.is_finite() { Some(row.max_fee_grt) } else { None }, - p90_fee_grt: if row.p90_fee_grt.is_finite() { Some(row.p90_fee_grt) } else { None }, - p99_fee_grt: if row.p99_fee_grt.is_finite() { Some(row.p99_fee_grt) } else { None }, - stddev_fee_grt: if row.stddev_fee_grt.is_finite() { Some(row.stddev_fee_grt) } else { None }, - avg_seconds_behind: if row.avg_seconds_behind.is_finite() { Some(row.avg_seconds_behind) } else { None }, + p90_indexer_response_time_ms: if row.p90_indexer_response_time_ms.is_finite() { + Some(row.p90_indexer_response_time_ms) + } else { + None + }, + p99_indexer_response_time_ms: if row.p99_indexer_response_time_ms.is_finite() { + Some(row.p99_indexer_response_time_ms) + } else { + None + }, + stddev_indexer_response_time_ms: if row.stddev_indexer_response_time_ms.is_finite() + { + Some(row.stddev_indexer_response_time_ms) + } else { + None + }, + total_fee_grt: if row.total_fee_grt.is_finite() { + Some(row.total_fee_grt) + } else { + None + }, + avg_fee_grt: if row.avg_fee_grt.is_finite() { + Some(row.avg_fee_grt) + } else { + None + }, + max_fee_grt: if row.max_fee_grt.is_finite() { + Some(row.max_fee_grt) + } else { + None + }, + p90_fee_grt: if row.p90_fee_grt.is_finite() { + Some(row.p90_fee_grt) + } else { + None + }, + p99_fee_grt: if row.p99_fee_grt.is_finite() { + Some(row.p99_fee_grt) + } else { + None + }, + stddev_fee_grt: if row.stddev_fee_grt.is_finite() { + Some(row.stddev_fee_grt) + } else { + None + }, + avg_seconds_behind: if row.avg_seconds_behind.is_finite() { + Some(row.avg_seconds_behind) + } else { + None + }, max_seconds_behind: Some(row.max_seconds_behind), - p90_seconds_behind: if row.p90_seconds_behind.is_finite() { Some(row.p90_seconds_behind) } else { None }, - p99_seconds_behind: if row.p99_seconds_behind.is_finite() { Some(row.p99_seconds_behind) } else { None }, - stddev_seconds_behind: if row.stddev_seconds_behind.is_finite() { Some(row.stddev_seconds_behind) } else { None }, - avg_blocks_behind: if row.avg_blocks_behind.is_finite() { Some(row.avg_blocks_behind) } else { None }, + p90_seconds_behind: if row.p90_seconds_behind.is_finite() { + Some(row.p90_seconds_behind) + } else { + None + }, + p99_seconds_behind: if row.p99_seconds_behind.is_finite() { + Some(row.p99_seconds_behind) + } else { + None + }, + stddev_seconds_behind: if row.stddev_seconds_behind.is_finite() { + Some(row.stddev_seconds_behind) + } else { + None + }, + avg_blocks_behind: if row.avg_blocks_behind.is_finite() { + Some(row.avg_blocks_behind) + } else { + None + }, max_blocks_behind: Some(row.max_blocks_behind), - p90_blocks_behind: if row.p90_blocks_behind.is_finite() { Some(row.p90_blocks_behind) } else { None }, - p99_blocks_behind: if row.p99_blocks_behind.is_finite() { Some(row.p99_blocks_behind) } else { None }, - stddev_blocks_behind: if row.stddev_blocks_behind.is_finite() { Some(row.stddev_blocks_behind) } else { None }, - success_proportion: if row.success_proportion.is_finite() { Some(row.success_proportion) } else { None }, + p90_blocks_behind: if row.p90_blocks_behind.is_finite() { + Some(row.p90_blocks_behind) + } else { + None + }, + p99_blocks_behind: if row.p99_blocks_behind.is_finite() { + Some(row.p99_blocks_behind) + } else { + None + }, + stddev_blocks_behind: if row.stddev_blocks_behind.is_finite() { + Some(row.stddev_blocks_behind) + } else { + None + }, + success_proportion: if row.success_proportion.is_finite() { + Some(row.success_proportion) + } else { + None + }, }) .collect()) } @@ -764,54 +1004,105 @@ impl QueryRoot { async fn allocation_aggregations( &self, _ctx: &Context<'_>, - #[graphql(desc = "Aggregation interval (default: Hourly)")] interval: Option, - #[graphql(desc = "Time range (RFC3339 format, default: last 24 hours)")] time_range: Option, + #[graphql(desc = "Aggregation interval (default: Hourly)")] interval: Option< + AggregationInterval, + >, + #[graphql( + desc = "Optional time range filter. If omitted, defaults to the last 24 hours. \ + If provided, requires at least 'from' or 'to'." + )] + time_range: Option, #[graphql(desc = "Optional filters for the query")] filter: Option, - #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] limit: Option, - #[graphql(desc = "Optional sorting (default: time_bucket DESC)")] sort: Option, // Use AllocationSortInput + #[graphql(desc = "Maximum number of records to return (default 1000, max 10000)")] + limit: Option, + #[graphql(desc = "Optional sorting (default: time_bucket DESC)")] sort: Option< + AllocationSortInput, + >, ) -> Result, String> { - let client = get_clickhouse_client()?; - - // --- Handle Defaults (Similar to deployment_aggregations) --- + let client = get_clickhouse_client()?; let actual_interval = interval.unwrap_or(AggregationInterval::Hourly); - let (from_dt, to_dt) = match time_range { - Some(tr) => ( - DateTime::parse_from_rfc3339(&tr.from) - .map_err(|e| format!("Invalid 'from' timestamp format: {}", e))? - .with_timezone(&Utc), - DateTime::parse_from_rfc3339(&tr.to) - .map_err(|e| format!("Invalid 'to' timestamp format: {}", e))? - .with_timezone(&Utc), - ), - None => { - let now = Utc::now(); - (now - chrono::Duration::hours(24), now) - } - }; - // --- End Handle Defaults --- - let view_name = format!("view_agg_allocation_{}", actual_interval.table_suffix()); - // Build WHERE clause (Includes subgraph and indexer filters) + // --- Build WHERE clause --- let mut conditions = Vec::new(); - conditions.push(format!( - "time_bucket >= toDateTime({})", - from_dt.timestamp() - )); - conditions.push(format!("time_bucket < toDateTime({})", to_dt.timestamp())); + // --- Handle Time Range (Apply same logic as deployment_aggregations) --- + match time_range { + None => { + let now = Utc::now(); + let default_from = now - chrono::Duration::hours(24); + conditions.push(format!( + "time_bucket >= toDateTime({})", + default_from.timestamp() + )); + conditions.push(format!("time_bucket < toDateTime({})", now.timestamp())); + } + Some(tr) => { + let parsed_from = tr.from.as_deref().map(parse_datetime_input).transpose()?; + let parsed_to = tr.to.as_deref().map(parse_datetime_input).transpose()?; + + match (parsed_from, parsed_to) { + (None, None) => { + return Err( + "Time range filter requires at least 'from' or 'to' to be specified." + .to_string(), + ); + } + (Some(from_dt), None) => { + conditions.push(format!( + "time_bucket >= toDateTime({})", + from_dt.timestamp() + )); + } + (None, Some(to_dt)) => { + conditions.push(format!("time_bucket < toDateTime({})", to_dt.timestamp())); + } + (Some(from_dt), Some(to_dt)) => { + if from_dt >= to_dt { + return Err("'from' time must be earlier than 'to' time.".to_string()); + } + conditions.push(format!( + "time_bucket >= toDateTime({})", + from_dt.timestamp() + )); + conditions.push(format!("time_bucket < toDateTime({})", to_dt.timestamp())); + } + } + } + } + // --- End Handle Time Range --- + + // --- Handle Other Filters (gateway_id, subgraph, indexer) --- if let Some(f) = filter { if let Some(gw) = &f.gateway_id { - conditions.push(format!("gateway_id = '{}'", gw.replace('\'', "''"))); + // --- Validate and Escape gateway_id --- + if !GATEWAY_ID_REGEX.is_match(gw) { + return Err("Invalid format for gateway_id filter".to_string()); + } + let escaped_gw = escape_clickhouse_string_literal(gw); + conditions.push(format!("gateway_id = '{}'", escaped_gw)); + // --- End Validation --- } - if let Some(sg) = &f.subgraph { - conditions.push(format!("subgraph = '{}'", sg.replace('\'', "''"))); + if let Some(sg) = &f.subgraph { + // --- Validate and Escape subgraph --- + if !SUBGRAPH_REGEX.is_match(sg) { + return Err("Invalid format for subgraph filter".to_string()); + } + let escaped_sg = escape_clickhouse_string_literal(sg); + conditions.push(format!("subgraph = '{}'", escaped_sg)); + // --- End Validation --- } if let Some(ix) = &f.indexer { - conditions.push(format!("indexer = '{}'", ix.replace('\'', "''"))); + // --- Validate and Escape indexer --- + if !INDEXER_REGEX.is_match(ix) { + return Err("Invalid format for indexer filter".to_string()); + } + let escaped_ix = escape_clickhouse_string_literal(ix); + conditions.push(format!("indexer = '{}'", escaped_ix)); + // --- End Validation --- } - // Note: Allocation ID filter is not used here as the view aggregates by subgraph/indexer } + // --- End Handle Other Filters --- let where_clause = format!("WHERE {}", conditions.join(" AND ")); let query_limit = limit.unwrap_or(1000).clamp(1, 10000); @@ -823,16 +1114,14 @@ impl QueryRoot { SortDirection::Asc => "ASC", SortDirection::Desc => "DESC", }; - format!("ORDER BY {} {}", s.field.column_name(), direction) // Use AllocationSortField mapping + format!("ORDER BY {} {}", s.field.column_name(), direction) } None => { - // Default sort order "ORDER BY time_bucket DESC, subgraph ASC, indexer ASC, gateway_id ASC".to_string() } }; - // --- End Build ORDER BY Clause --- - // Query uses the same fields as indexer aggregation but groups differently in the view + // --- Build and Execute Query --- let query = format!( "SELECT time_bucket, subgraph, indexer, gateway_id, \ query_count, success_count, failure_count, \ @@ -846,53 +1135,130 @@ impl QueryRoot { FROM {} {} \ {} \ LIMIT {}", - view_name, where_clause, order_by_clause, query_limit // Use dynamic order_by_clause + view_name, where_clause, order_by_clause, query_limit ); println!("Executing query: {}", query); let rows = client .query(&query) - .fetch_all::() // Use Allocation Row struct + .fetch_all::() .await .map_err(|e| format!("Database query failed: {}", e))?; - // Map results (Identical mapping logic to Indexer, just different input row type) + // Map results (logic remains the same) Ok(rows .into_iter() .map(|row| AllocationAggregationOutput { - time_bucket: Utc + time_bucket: Utc .timestamp_opt(row.time_bucket as i64, 0) .single() .map_or_else(|| "Invalid Timestamp".to_string(), |dt| dt.to_rfc3339()), - subgraph: row.subgraph, // Allocation includes subgraph + subgraph: row.subgraph, indexer: row.indexer, gateway_id: row.gateway_id, query_count: row.query_count, success_count: row.success_count, failure_count: row.failure_count, - avg_indexer_response_time_ms: if row.avg_indexer_response_time_ms.is_finite() { Some(row.avg_indexer_response_time_ms) } else { None }, + avg_indexer_response_time_ms: if row.avg_indexer_response_time_ms.is_finite() { + Some(row.avg_indexer_response_time_ms) + } else { + None + }, max_indexer_response_time_ms: Some(row.max_indexer_response_time_ms), - p90_indexer_response_time_ms: if row.p90_indexer_response_time_ms.is_finite() { Some(row.p90_indexer_response_time_ms) } else { None }, - p99_indexer_response_time_ms: if row.p99_indexer_response_time_ms.is_finite() { Some(row.p99_indexer_response_time_ms) } else { None }, - stddev_indexer_response_time_ms: if row.stddev_indexer_response_time_ms.is_finite() { Some(row.stddev_indexer_response_time_ms) } else { None }, - total_fee_grt: if row.total_fee_grt.is_finite() { Some(row.total_fee_grt) } else { None }, - avg_fee_grt: if row.avg_fee_grt.is_finite() { Some(row.avg_fee_grt) } else { None }, - max_fee_grt: if row.max_fee_grt.is_finite() { Some(row.max_fee_grt) } else { None }, - p90_fee_grt: if row.p90_fee_grt.is_finite() { Some(row.p90_fee_grt) } else { None }, - p99_fee_grt: if row.p99_fee_grt.is_finite() { Some(row.p99_fee_grt) } else { None }, - stddev_fee_grt: if row.stddev_fee_grt.is_finite() { Some(row.stddev_fee_grt) } else { None }, - avg_seconds_behind: if row.avg_seconds_behind.is_finite() { Some(row.avg_seconds_behind) } else { None }, + p90_indexer_response_time_ms: if row.p90_indexer_response_time_ms.is_finite() { + Some(row.p90_indexer_response_time_ms) + } else { + None + }, + p99_indexer_response_time_ms: if row.p99_indexer_response_time_ms.is_finite() { + Some(row.p99_indexer_response_time_ms) + } else { + None + }, + stddev_indexer_response_time_ms: if row.stddev_indexer_response_time_ms.is_finite() + { + Some(row.stddev_indexer_response_time_ms) + } else { + None + }, + total_fee_grt: if row.total_fee_grt.is_finite() { + Some(row.total_fee_grt) + } else { + None + }, + avg_fee_grt: if row.avg_fee_grt.is_finite() { + Some(row.avg_fee_grt) + } else { + None + }, + max_fee_grt: if row.max_fee_grt.is_finite() { + Some(row.max_fee_grt) + } else { + None + }, + p90_fee_grt: if row.p90_fee_grt.is_finite() { + Some(row.p90_fee_grt) + } else { + None + }, + p99_fee_grt: if row.p99_fee_grt.is_finite() { + Some(row.p99_fee_grt) + } else { + None + }, + stddev_fee_grt: if row.stddev_fee_grt.is_finite() { + Some(row.stddev_fee_grt) + } else { + None + }, + avg_seconds_behind: if row.avg_seconds_behind.is_finite() { + Some(row.avg_seconds_behind) + } else { + None + }, max_seconds_behind: Some(row.max_seconds_behind), - p90_seconds_behind: if row.p90_seconds_behind.is_finite() { Some(row.p90_seconds_behind) } else { None }, - p99_seconds_behind: if row.p99_seconds_behind.is_finite() { Some(row.p99_seconds_behind) } else { None }, - stddev_seconds_behind: if row.stddev_seconds_behind.is_finite() { Some(row.stddev_seconds_behind) } else { None }, - avg_blocks_behind: if row.avg_blocks_behind.is_finite() { Some(row.avg_blocks_behind) } else { None }, + p90_seconds_behind: if row.p90_seconds_behind.is_finite() { + Some(row.p90_seconds_behind) + } else { + None + }, + p99_seconds_behind: if row.p99_seconds_behind.is_finite() { + Some(row.p99_seconds_behind) + } else { + None + }, + stddev_seconds_behind: if row.stddev_seconds_behind.is_finite() { + Some(row.stddev_seconds_behind) + } else { + None + }, + avg_blocks_behind: if row.avg_blocks_behind.is_finite() { + Some(row.avg_blocks_behind) + } else { + None + }, max_blocks_behind: Some(row.max_blocks_behind), - p90_blocks_behind: if row.p90_blocks_behind.is_finite() { Some(row.p90_blocks_behind) } else { None }, - p99_blocks_behind: if row.p99_blocks_behind.is_finite() { Some(row.p99_blocks_behind) } else { None }, - stddev_blocks_behind: if row.stddev_blocks_behind.is_finite() { Some(row.stddev_blocks_behind) } else { None }, - success_proportion: if row.success_proportion.is_finite() { Some(row.success_proportion) } else { None }, + p90_blocks_behind: if row.p90_blocks_behind.is_finite() { + Some(row.p90_blocks_behind) + } else { + None + }, + p99_blocks_behind: if row.p99_blocks_behind.is_finite() { + Some(row.p99_blocks_behind) + } else { + None + }, + stddev_blocks_behind: if row.stddev_blocks_behind.is_finite() { + Some(row.stddev_blocks_behind) + } else { + None + }, + success_proportion: if row.success_proportion.is_finite() { + Some(row.success_proportion) + } else { + None + }, }) .collect()) } From aef572eaccad71878f73e2337a0bd5ba24d3e3e7 Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Thu, 3 Apr 2025 01:44:31 -0300 Subject: [PATCH 16/20] feat: update README.md --- README.md | 233 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 184 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 4fe67f8..6a11d26 100644 --- a/README.md +++ b/README.md @@ -9,23 +9,23 @@ QoS Oracle V2 is a data pipeline and reporting system for The Graph Network qual The system consists of three main components: 1. **Data Ingestion** - Kafka/Redpanda message broker receives QoS data from Gateways -2. **Data Processing** - ClickHouse database ingests, stores, and aggregates the data -3. **Data Exposure** - GraphQL API serves pre-aggregated metrics for analysis - +2. **Data Processing** - ClickHouse database ingests, stores, and aggregates the data into time-based views (5min, hourly, daily) +3. **Data Exposure** - GraphQL API serves these pre-aggregated metrics from Clickhouse views ## Features -- High-throughput data ingestion (2,000+ messages/second) -- Efficient storage with automatic TTL for raw data +- High-throughput data ingestion (via Kafka/Redpanda) +- Efficient storage with automatic TTL for raw data (managed by ClickHouse) - Pre-aggregated metrics in 5-minute, hourly, and daily intervals -- GraphQL API for flexible querying of historical data -- Docker-based deployment for easy setup and operation +- Flexible GraphQL API for querying aggregated data with filtering, sorting, and time range selection +- Docker-based deployment for easy setup and operation using profiles ## Getting Started ### Prerequisites -- Docker and Docker Compose +- Docker and Docker Compose (v2 recommended) +- Git - 4+ CPU cores recommended - 16+ GB RAM recommended - 250+ GB SSD storage recommended @@ -33,80 +33,215 @@ The system consists of three main components: ### Installation 1. Clone the repository: - ``` - git clone https://github.com/graphops/qos-oracle.git + ```bash + git clone https://github.com/graphops/qos-oracle.git # Or your repo URL cd qos-oracle ``` -2. Start the services: - ``` - docker-compose -f oracle/docker-compose.yml up -d +2. Start the services (Development profile includes API, DB, Kafka, and Test Producer): + ```bash + # Ensure you are in the root qos-oracle directory + docker compose -f oracle/docker-compose.yml --profile dev up --build -d ``` + * Use `--profile deps` to start only ClickHouse and Redpanda 3. Verify the installation: - ``` + ```bash + # Check API health curl http://localhost:8000/health + # Check running containers + docker compose -f oracle/docker-compose.yml ps ``` -4. Access the GraphQL playground at http://localhost:8000 +4. Access the GraphQL endpoint at `http://localhost:8000/graphql`. You can use tools like Postman, Insomnia, `curl`, or potentially a browser-based playground if enabled. ### Configuration -The system can be configured through environment variables in the docker-compose.yml file: +The system can be configured through environment variables set in the `oracle/docker-compose.yml` file for the respective services: -- `CLICKHOUSE_URL` - ClickHouse server URL -- `CLICKHOUSE_DB` - ClickHouse database name -- `KAFKA_BROKER` - Kafka/Redpanda broker address -- `MESSAGES_PER_SECOND` - Test producer message rate (for development) +- **`graphql-api` service:** + * `CLICKHOUSE_DSN`: Full ClickHouse connection string (e.g., `tcp://graphql:graphql_password@clickhouse-server:9000/default?compression=lz4`). This replaces separate URL, DB, User, Pass variables. + * `PORT`: Internal port the API listens on (default: `8000`). + * `RUST_LOG`: Logging level (e.g., `info`, `debug`). +- **`test-producer` service:** + * `KAFKA_BROKER`: Kafka/Redpanda broker address (e.g., `redpanda:9092`). + * `MESSAGES_PER_SECOND`: Test producer message rate. + +*(See `x-clickhouse-env` in `docker-compose.yml` for default user/password used in the default DSN)* ## Usage ### GraphQL API -The GraphQL API provides access to QoS reports and aggregated metrics. Here's an example query: - -```graphql -query { - qosReports( - from: "2023-01-01T00:00:00Z", - to: "2023-01-02T00:00:00Z" - ) { - event_time - gateway_id - response_time_ms - total_fees_usd - indexer_queries { - indexer - deployment - fee_grt - response_time_ms - } - } -} -``` +The GraphQL API provides access to aggregated QoS metrics via three main queries: `deploymentAggregations`, `indexerAggregations`, and `allocationAggregations`. + +**Endpoint:** `http://localhost:8000/graphql` + +**Key Arguments:** + +- `interval: AggregationInterval` (Optional: `FiveMinutes`, `Hourly`, `Daily`. Default: `Hourly`) +- `timeRange: TimeRangeInput` (Optional: Defaults to last 24h. Requires `from` or `to` if provided) + * `from: String` (Optional: RFC3339 or Unix timestamp string, inclusive) + * `to: String` (Optional: RFC3339 or Unix timestamp string, exclusive) +- `filter: AggregationFilterInput` (Optional: Filter by IDs) + * `gatewayId: String` + * `subgraph: String` (Deployment ID) + * `indexer: String` (Hex Address) + * `allocation: String` (Hex Address) +- `limit: Int` (Optional: Default 1000, Max 10000) +- `sort: [QuerySpecific]SortInput` (Optional: Sort by field and direction) + +**Example Queries:** + +1. **Get Hourly Deployment Aggregations (Default - Last 24 hours):** + + * **cURL:** + ```bash + curl -X POST http://localhost:8000/graphql \ + -H "Content-Type: application/json" \ + -d '{ "query": "{ deploymentAggregations { timeBucket subgraph gatewayId queryCount successProportion } }" }' + ``` + * **GraphQL:** + ```graphql + { + deploymentAggregations { + timeBucket + subgraph + gatewayId + queryCount + successProportion + } + } + ``` + +2. **Get Daily Indexer Aggregations for a Specific Indexer in October 2023:** + + * **cURL:** + ```bash + curl -X POST http://localhost:8000/graphql \ + -H "Content-Type: application/json" \ + -d '{ + "query": "query IndexerData($indexerId: String!) { indexerAggregations( interval: Daily, timeRange: { from: \"2023-10-01T00:00:00Z\", to: \"2023-11-01T00:00:00Z\" }, filter: { indexer: $indexerId } ) { timeBucket indexer gatewayId queryCount totalFeeGrt avgIndexerResponseTimeMs } }", + "variables": { "indexerId": "0xYourIndexerAddressHere" } + }' + ``` + * **GraphQL (with variables):** + ```graphql + query IndexerData($indexerId: String!) { + indexerAggregations( + interval: Daily, + timeRange: { + from: "2023-10-01T00:00:00Z", + to: "2023-11-01T00:00:00Z" + }, + filter: { indexer: $indexerId } + ) { + timeBucket + indexer + gatewayId + queryCount + totalFeeGrt + avgIndexerResponseTimeMs + } + } + ``` + *Variables:* + ```json + { + "indexerId": "0xYourIndexerAddressHere" + } + ``` + +3. **Get 5-Minute Allocation Aggregations Since a Specific Timestamp (Unix), Sorted by Query Count:** + + * **cURL:** + ```bash + curl -X POST http://localhost:8000/graphql \ + -H "Content-Type: application/json" \ + -d '{ + "query": "{ allocationAggregations( interval: FiveMinutes, timeRange: { from: \"1698300000\" }, sort: { field: QueryCount, direction: Desc }, limit: 50 ) { timeBucket subgraph indexer gatewayId queryCount avgSecondsBehind } }" + }' + ``` + * **GraphQL:** + ```graphql + { + allocationAggregations( + interval: FiveMinutes, + timeRange: { from: "1698300000" }, + sort: { field: QueryCount, direction: Desc }, + limit: 50 + ) { + timeBucket + subgraph + indexer + gatewayId + queryCount + avgSecondsBehind + } + } + ``` + +4. **Get Hourly Deployment Aggregations Up To a Specific Time for a Gateway:** + + * **cURL:** + ```bash + curl -X POST http://localhost:8000/graphql \ + -H "Content-Type: application/json" \ + -d '{ + "query": "query GatewayUntil($gwId: String!) { deploymentAggregations( timeRange: { to: \"2023-10-27T12:00:00Z\" }, filter: { gatewayId: $gwId } ) { timeBucket subgraph gatewayId queryCount avgResponseTimeMs } }", + "variables": { "gwId": "gateway-mainnet-1" } + }' + ``` + * **GraphQL (with variables):** + ```graphql + query GatewayUntil($gwId: String!) { + deploymentAggregations( + timeRange: { to: "2023-10-27T12:00:00Z" }, + filter: { gatewayId: $gwId } + ) { + timeBucket + subgraph + gatewayId + queryCount + avgResponseTimeMs + } + } + ``` + *Variables:* + ```json + { + "gwId": "gateway-mainnet-1" + } + ``` ## Development ### Project Structure - `oracle/clickhouse/` - ClickHouse configuration and schema files -- `oracle/graphql-api/` - GraphQL API service -- `oracle/test-producer/` - Test data generator for development +- `oracle/graphql-api/` - GraphQL API service (Rust) +- `oracle/test-producer/` - Test data generator for development (Rust) - `oracle/docker-compose.yml` - Docker Compose configuration ### Running in Development Mode +```bash +# From the root qos-oracle directory +docker compose -f oracle/docker-compose.yml --profile dev up ``` -docker-compose -f oracle/docker-compose.yml --profile dev up -``` -This will start all services including the test producer, which generates synthetic QoS data. +*(Add `-d` to run detached)* + +This will start all services including the test producer, which generates synthetic QoS data into the Redpanda topic consumed by ClickHouse. ### Monitoring -- ClickHouse HTTP interface: http://localhost:8123 -- Redpanda Console: http://localhost:8080 -- GraphQL API: http://localhost:8000 +- **GraphQL API Health:** `http://localhost:8000/health` +- **GraphQL API Endpoint:** `http://localhost:8000/graphql` +- **ClickHouse HTTP Interface:** `http://localhost:8123` (Credentials: `graphql`/`graphql_password` by default) +- **Redpanda Kafka API:** `localhost:9092` +- **Redpanda Prometheus Metrics:** `http://localhost:9644/metrics` + *(Note: Redpanda Console is not included in the default compose file)* ## License From 4f78d9e81dee064b02b69353b5576674806950dc Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Mon, 7 Apr 2025 22:49:26 -0300 Subject: [PATCH 17/20] feat: added table optimizations for ClickHouse --- oracle/clickhouse/settings.xml | 14 + oracle/clickhouse/tables.sql | 668 +++++++-------------------------- oracle/clickhouse/users.xml | 3 +- 3 files changed, 161 insertions(+), 524 deletions(-) diff --git a/oracle/clickhouse/settings.xml b/oracle/clickhouse/settings.xml index c755f85..3d968cb 100644 --- a/oracle/clickhouse/settings.xml +++ b/oracle/clickhouse/settings.xml @@ -10,4 +10,18 @@ true true + + warning + true + + + + + + + + + + + diff --git a/oracle/clickhouse/tables.sql b/oracle/clickhouse/tables.sql index ef9a174..7b68fa8 100644 --- a/oracle/clickhouse/tables.sql +++ b/oracle/clickhouse/tables.sql @@ -6,7 +6,7 @@ CREATE TABLE IF NOT EXISTS kafka_qos_data query_id String, api_key String, user_id String, - subgraph String, + subgraph Nullable(String), result String, response_time_ms UInt32, request_bytes UInt32, @@ -35,84 +35,24 @@ SETTINGS kafka_max_block_size = 1, kafka_poll_timeout_ms = 500; --- Create the primary data table (qos_data) --- This table stores the processed data from Kafka, ready for aggregation -CREATE TABLE IF NOT EXISTS qos_data -( - event_time DateTime, - gateway_id String, - receipt_signer String, -- HEX encoded - query_id String, - api_key String, - user_id String, - subgraph Nullable(String), - result String, -- 'success' or other status - response_time_ms UInt32, - request_bytes UInt32, - response_bytes Nullable(UInt32), - total_fees_usd Float64, - indexer_queries Nested ( - indexer String, -- HEX encoded - deployment String, -- HEX encoded - allocation String, -- HEX encoded - indexed_chain String, - url String, - fee_grt Float64, - response_time_ms UInt32, - seconds_behind UInt32, - result String, - indexer_errors String, - blocks_behind UInt64 - ) -) ENGINE = MergeTree -PARTITION BY toYYYYMM(event_time) -- Keep monthly partitioning -ORDER BY (gateway_id, event_time) -TTL event_time + INTERVAL 7 DAY; -- Add TTL: Delete rows older than 7 days - --- Create the materialized view that processes and transforms the data into qos_data --- Ensure ALL necessary fields are selected and HEX encoding is applied correctly -CREATE MATERIALIZED VIEW IF NOT EXISTS qos_data_mv TO qos_data AS -SELECT - now() as event_time, - gateway_id, - HEX(receipt_signer) AS receipt_signer, -- Apply HEX encoding - query_id, - api_key, - user_id, - subgraph, - result, - response_time_ms, - request_bytes, - response_bytes, - total_fees_usd, - -- Select ALL nested fields needed downstream, applying HEX where required - arrayMap(x -> HEX(x), indexer_queries.indexer) AS `indexer_queries.indexer`, - arrayMap(x -> HEX(x), indexer_queries.deployment) AS `indexer_queries.deployment`, - arrayMap(x -> HEX(x), indexer_queries.allocation) AS `indexer_queries.allocation`, - indexer_queries.indexed_chain AS `indexer_queries.indexed_chain`, - indexer_queries.url AS `indexer_queries.url`, - indexer_queries.fee_grt AS `indexer_queries.fee_grt`, - indexer_queries.response_time_ms AS `indexer_queries.response_time_ms`, - indexer_queries.seconds_behind AS `indexer_queries.seconds_behind`, - indexer_queries.result AS `indexer_queries.result`, - indexer_queries.indexer_errors AS `indexer_queries.indexer_errors`, - indexer_queries.blocks_behind AS `indexer_queries.blocks_behind` -FROM kafka_qos_data; - -- ============================================================ --- AGGREGATION TABLES & MATERIALIZED VIEWS +-- MINUTE-LEVEL AGGREGATION TABLES & MATERIALIZED VIEWS +-- ============================================================ +-- We store aggregate state only at the 1-minute level. +-- Coarser granularities (5min, hourly, daily) are calculated +-- on the fly in the final views using -Merge functions. +-- Materialized Views now read directly from kafka_qos_data. -- ============================================================ -- ---------------------------------------- --- Deployment Level Aggregations (by subgraph) +-- Deployment Level Aggregations (1-Minute State) -- ---------------------------------------- - --- 5-Minute Deployment Aggregations (Table Definition) -CREATE TABLE IF NOT EXISTS agg_deployment_5min +CREATE TABLE IF NOT EXISTS agg_deployment_1min ( - time_bucket DateTime, + time_bucket DateTime, -- Minute-level bucket subgraph String, gateway_id String, + -- Aggregate state columns (same as before) query_count AggregateFunction(count), success_count AggregateFunction(countIf, UInt8), failure_count AggregateFunction(countIf, UInt8), @@ -127,130 +67,44 @@ CREATE TABLE IF NOT EXISTS agg_deployment_5min stddev_fee_usd AggregateFunction(stddevSamp, Float64) ) ENGINE = AggregatingMergeTree -ORDER BY (time_bucket, subgraph, gateway_id); +PARTITION BY toYYYYMM(time_bucket) -- Partitioning is still useful +ORDER BY (time_bucket, subgraph, gateway_id) -- Grouping keys +TTL time_bucket + INTERVAL 7 DAY DELETE; -- TTL for 1-minute aggregate state (adjust as needed) --- Materialized View for 5-Minute Deployment Aggregations (Corrected SELECT with ALL -State functions and alias 'qd') -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_deployment_5min TO agg_deployment_5min AS +-- MV to populate 1-Minute Deployment Aggregations (Reads from Kafka) +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_deployment_1min TO agg_deployment_1min AS SELECT - toStartOfFiveMinute(qd.event_time) AS time_bucket, - qd.subgraph, - qd.gateway_id, + toStartOfMinute(now()) AS time_bucket, -- Use 1-minute bucket based on processing time + kq.subgraph, + kq.gateway_id, + -- Calculate aggregate states directly from Kafka data countState() AS query_count, - countIfState(qd.result = 'success') AS success_count, - countIfState(qd.result != 'success') AS failure_count, - sumState(qd.total_fees_usd) AS total_fees_usd, - avgState(qd.response_time_ms) AS avg_response_time_ms, - maxState(qd.response_time_ms) AS max_response_time_ms, - quantilesState(0.90, 0.99)(qd.response_time_ms) AS quantiles_response_time_ms, - stddevSampState(qd.response_time_ms) AS stddev_response_time_ms, - avgState(qd.total_fees_usd) AS avg_fee_usd, - maxState(qd.total_fees_usd) AS max_fee_usd, - quantilesState(0.90, 0.99)(qd.total_fees_usd) AS quantiles_fee_usd, - stddevSampState(qd.total_fees_usd) AS stddev_fee_usd -FROM qos_data AS qd -- Added alias -WHERE qd.subgraph IS NOT NULL -GROUP BY time_bucket, qd.subgraph, qd.gateway_id; -- Use alias in GROUP BY - --- Hourly Deployment Aggregations (Table Definition) -CREATE TABLE IF NOT EXISTS agg_deployment_hourly -( - time_bucket DateTime, - subgraph String, - gateway_id String, - query_count AggregateFunction(count), - success_count AggregateFunction(countIf, UInt8), - failure_count AggregateFunction(countIf, UInt8), - total_fees_usd AggregateFunction(sum, Float64), - avg_response_time_ms AggregateFunction(avg, UInt32), - max_response_time_ms AggregateFunction(max, UInt32), - quantiles_response_time_ms AggregateFunction(quantiles(0.90, 0.99), UInt32), - stddev_response_time_ms AggregateFunction(stddevSamp, UInt32), - avg_fee_usd AggregateFunction(avg, Float64), - max_fee_usd AggregateFunction(max, Float64), - quantiles_fee_usd AggregateFunction(quantiles(0.90, 0.99), Float64), - stddev_fee_usd AggregateFunction(stddevSamp, Float64) -) -ENGINE = AggregatingMergeTree -ORDER BY (time_bucket, subgraph, gateway_id); - --- Materialized View for Hourly Deployment Aggregations (Corrected SELECT with ALL -State functions and alias 'qd') -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_deployment_hourly TO agg_deployment_hourly AS -SELECT - toStartOfHour(qd.event_time) AS time_bucket, -- Changed time function - qd.subgraph, - qd.gateway_id, - countState() AS query_count, - countIfState(qd.result = 'success') AS success_count, - countIfState(qd.result != 'success') AS failure_count, - sumState(qd.total_fees_usd) AS total_fees_usd, - avgState(qd.response_time_ms) AS avg_response_time_ms, - maxState(qd.response_time_ms) AS max_response_time_ms, - quantilesState(0.90, 0.99)(qd.response_time_ms) AS quantiles_response_time_ms, - stddevSampState(qd.response_time_ms) AS stddev_response_time_ms, - avgState(qd.total_fees_usd) AS avg_fee_usd, - maxState(qd.total_fees_usd) AS max_fee_usd, - quantilesState(0.90, 0.99)(qd.total_fees_usd) AS quantiles_fee_usd, - stddevSampState(qd.total_fees_usd) AS stddev_fee_usd -FROM qos_data AS qd -- Added alias -WHERE qd.subgraph IS NOT NULL -GROUP BY time_bucket, qd.subgraph, qd.gateway_id; -- Use alias in GROUP BY - --- Daily Deployment Aggregations (Table Definition) -CREATE TABLE IF NOT EXISTS agg_deployment_daily -( - time_bucket DateTime, - subgraph String, - gateway_id String, - query_count AggregateFunction(count), - success_count AggregateFunction(countIf, UInt8), - failure_count AggregateFunction(countIf, UInt8), - total_fees_usd AggregateFunction(sum, Float64), - avg_response_time_ms AggregateFunction(avg, UInt32), - max_response_time_ms AggregateFunction(max, UInt32), - quantiles_response_time_ms AggregateFunction(quantiles(0.90, 0.99), UInt32), - stddev_response_time_ms AggregateFunction(stddevSamp, UInt32), - avg_fee_usd AggregateFunction(avg, Float64), - max_fee_usd AggregateFunction(max, Float64), - quantiles_fee_usd AggregateFunction(quantiles(0.90, 0.99), Float64), - stddev_fee_usd AggregateFunction(stddevSamp, Float64) -) -ENGINE = AggregatingMergeTree -ORDER BY (time_bucket, subgraph, gateway_id); - --- Materialized View for Daily Deployment Aggregations (Corrected SELECT with ALL -State functions and alias 'qd') -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_deployment_daily TO agg_deployment_daily AS -SELECT - toStartOfDay(qd.event_time) AS time_bucket, -- Changed time function - qd.subgraph, - qd.gateway_id, - countState() AS query_count, - countIfState(qd.result = 'success') AS success_count, - countIfState(qd.result != 'success') AS failure_count, - sumState(qd.total_fees_usd) AS total_fees_usd, - avgState(qd.response_time_ms) AS avg_response_time_ms, - maxState(qd.response_time_ms) AS max_response_time_ms, - quantilesState(0.90, 0.99)(qd.response_time_ms) AS quantiles_response_time_ms, - stddevSampState(qd.response_time_ms) AS stddev_response_time_ms, - avgState(qd.total_fees_usd) AS avg_fee_usd, - maxState(qd.total_fees_usd) AS max_fee_usd, - quantilesState(0.90, 0.99)(qd.total_fees_usd) AS quantiles_fee_usd, - stddevSampState(qd.total_fees_usd) AS stddev_fee_usd -FROM qos_data AS qd -- Added alias -WHERE qd.subgraph IS NOT NULL -GROUP BY time_bucket, qd.subgraph, qd.gateway_id; -- Use alias in GROUP BY + countIfState(kq.result = 'success') AS success_count, + countIfState(kq.result != 'success') AS failure_count, + sumState(kq.total_fees_usd) AS total_fees_usd, + avgState(kq.response_time_ms) AS avg_response_time_ms, + maxState(kq.response_time_ms) AS max_response_time_ms, + quantilesState(0.90, 0.99)(kq.response_time_ms) AS quantiles_response_time_ms, + stddevSampState(kq.response_time_ms) AS stddev_response_time_ms, + avgState(kq.total_fees_usd) AS avg_fee_usd, -- Assuming total_fees_usd is per-query fee + maxState(kq.total_fees_usd) AS max_fee_usd, + quantilesState(0.90, 0.99)(kq.total_fees_usd) AS quantiles_fee_usd, + stddevSampState(kq.total_fees_usd) AS stddev_fee_usd +FROM kafka_qos_data AS kq -- Read directly from Kafka table +WHERE kq.subgraph IS NOT NULL +GROUP BY time_bucket, kq.subgraph, kq.gateway_id; -- ---------------------------------------- --- Allocation Level Aggregations (by subgraph + indexer_queries.indexer) +-- Allocation Level Aggregations (1-Minute State) -- ---------------------------------------- - --- 5-Minute Allocation Aggregations (Table Definition) -CREATE TABLE IF NOT EXISTS agg_allocation_5min +CREATE TABLE IF NOT EXISTS agg_allocation_1min ( - time_bucket DateTime, + time_bucket DateTime, -- Minute-level bucket subgraph String, - indexer String, + indexer String, -- HEX encoded gateway_id String, + -- Aggregate state columns query_count AggregateFunction(count), success_count AggregateFunction(countIf, UInt8), failure_count AggregateFunction(countIf, UInt8), @@ -273,15 +127,18 @@ CREATE TABLE IF NOT EXISTS agg_allocation_5min stddev_blocks_behind AggregateFunction(stddevSamp, UInt64) ) ENGINE = AggregatingMergeTree -ORDER BY (time_bucket, subgraph, indexer, gateway_id); +PARTITION BY toYYYYMM(time_bucket) +ORDER BY (time_bucket, subgraph, indexer, gateway_id) -- Grouping keys +TTL time_bucket + INTERVAL 7 DAY DELETE; -- TTL for 1-minute aggregate state --- Materialized View for 5-Minute Allocation Aggregations (Corrected SELECT with ALL -State functions and aliases 'qd', 'iq') -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_allocation_5min TO agg_allocation_5min AS +-- MV to populate 1-Minute Allocation Aggregations (Reads from Kafka) +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_allocation_1min TO agg_allocation_1min AS SELECT - toStartOfFiveMinute(qd.event_time) AS time_bucket, - qd.subgraph, - iq.indexer AS indexer, - qd.gateway_id, + toStartOfMinute(now()) AS time_bucket, -- Use 1-minute bucket based on processing time + kq.subgraph, + HEX(iq.indexer) AS indexer, -- Apply HEX encoding here + kq.gateway_id, + -- Calculate aggregate states directly from Kafka data countState() AS query_count, countIfState(iq.result = 'success') AS success_count, countIfState(iq.result != 'success') AS failure_count, @@ -302,209 +159,21 @@ SELECT maxState(iq.blocks_behind) AS max_blocks_behind, quantilesState(0.90, 0.99)(iq.blocks_behind) AS quantiles_blocks_behind, stddevSampState(iq.blocks_behind) AS stddev_blocks_behind -FROM qos_data AS qd -- Added alias +FROM kafka_qos_data AS kq -- Read directly from Kafka table ARRAY JOIN indexer_queries AS iq -WHERE qd.subgraph IS NOT NULL AND length(iq.indexer) > 0 -GROUP BY time_bucket, qd.subgraph, indexer, qd.gateway_id; -- Use aliases in GROUP BY - --- Hourly Allocation Aggregations (Table Definition) -CREATE TABLE IF NOT EXISTS agg_allocation_hourly -( - time_bucket DateTime, - subgraph String, - indexer String, - gateway_id String, - query_count AggregateFunction(count), - success_count AggregateFunction(countIf, UInt8), - failure_count AggregateFunction(countIf, UInt8), - total_fee_grt AggregateFunction(sum, Float64), - avg_indexer_response_time_ms AggregateFunction(avg, UInt32), - max_indexer_response_time_ms AggregateFunction(max, UInt32), - quantiles_indexer_response_time_ms AggregateFunction(quantiles(0.90, 0.99), UInt32), - stddev_indexer_response_time_ms AggregateFunction(stddevSamp, UInt32), - avg_fee_grt AggregateFunction(avg, Float64), - max_fee_grt AggregateFunction(max, Float64), - quantiles_fee_grt AggregateFunction(quantiles(0.90, 0.99), Float64), - stddev_fee_grt AggregateFunction(stddevSamp, Float64), - avg_seconds_behind AggregateFunction(avg, UInt32), - max_seconds_behind AggregateFunction(max, UInt32), - quantiles_seconds_behind AggregateFunction(quantiles(0.90, 0.99), UInt32), - stddev_seconds_behind AggregateFunction(stddevSamp, UInt32), - avg_blocks_behind AggregateFunction(avg, UInt64), - max_blocks_behind AggregateFunction(max, UInt64), - quantiles_blocks_behind AggregateFunction(quantiles(0.90, 0.99), UInt64), - stddev_blocks_behind AggregateFunction(stddevSamp, UInt64) -) -ENGINE = AggregatingMergeTree -ORDER BY (time_bucket, subgraph, indexer, gateway_id); - --- Materialized View for Hourly Allocation Aggregations (Corrected SELECT with ALL -State functions and aliases 'qd', 'iq') -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_allocation_hourly TO agg_allocation_hourly AS -SELECT - toStartOfHour(qd.event_time) AS time_bucket, -- Changed time function - qd.subgraph, - iq.indexer AS indexer, - qd.gateway_id, - countState() AS query_count, - countIfState(iq.result = 'success') AS success_count, - countIfState(iq.result != 'success') AS failure_count, - sumState(iq.fee_grt) AS total_fee_grt, - avgState(iq.response_time_ms) AS avg_indexer_response_time_ms, - maxState(iq.response_time_ms) AS max_indexer_response_time_ms, - quantilesState(0.90, 0.99)(iq.response_time_ms) AS quantiles_indexer_response_time_ms, - stddevSampState(iq.response_time_ms) AS stddev_indexer_response_time_ms, - avgState(iq.fee_grt) AS avg_fee_grt, - maxState(iq.fee_grt) AS max_fee_grt, - quantilesState(0.90, 0.99)(iq.fee_grt) AS quantiles_fee_grt, - stddevSampState(iq.fee_grt) AS stddev_fee_grt, - avgState(iq.seconds_behind) AS avg_seconds_behind, - maxState(iq.seconds_behind) AS max_seconds_behind, - quantilesState(0.90, 0.99)(iq.seconds_behind) AS quantiles_seconds_behind, - stddevSampState(iq.seconds_behind) AS stddev_seconds_behind, - avgState(iq.blocks_behind) AS avg_blocks_behind, - maxState(iq.blocks_behind) AS max_blocks_behind, - quantilesState(0.90, 0.99)(iq.blocks_behind) AS quantiles_blocks_behind, - stddevSampState(iq.blocks_behind) AS stddev_blocks_behind -FROM qos_data AS qd -- Added alias -ARRAY JOIN indexer_queries AS iq -WHERE qd.subgraph IS NOT NULL AND length(iq.indexer) > 0 -GROUP BY time_bucket, qd.subgraph, indexer, qd.gateway_id; -- Use aliases in GROUP BY - --- Daily Allocation Aggregations (Table Definition) -CREATE TABLE IF NOT EXISTS agg_allocation_daily -( - time_bucket DateTime, - subgraph String, - indexer String, - gateway_id String, - query_count AggregateFunction(count), - success_count AggregateFunction(countIf, UInt8), - failure_count AggregateFunction(countIf, UInt8), - total_fee_grt AggregateFunction(sum, Float64), - avg_indexer_response_time_ms AggregateFunction(avg, UInt32), - max_indexer_response_time_ms AggregateFunction(max, UInt32), - quantiles_indexer_response_time_ms AggregateFunction(quantiles(0.90, 0.99), UInt32), - stddev_indexer_response_time_ms AggregateFunction(stddevSamp, UInt32), - avg_fee_grt AggregateFunction(avg, Float64), - max_fee_grt AggregateFunction(max, Float64), - quantiles_fee_grt AggregateFunction(quantiles(0.90, 0.99), Float64), - stddev_fee_grt AggregateFunction(stddevSamp, Float64), - avg_seconds_behind AggregateFunction(avg, UInt32), - max_seconds_behind AggregateFunction(max, UInt32), - quantiles_seconds_behind AggregateFunction(quantiles(0.90, 0.99), UInt32), - stddev_seconds_behind AggregateFunction(stddevSamp, UInt32), - avg_blocks_behind AggregateFunction(avg, UInt64), - max_blocks_behind AggregateFunction(max, UInt64), - quantiles_blocks_behind AggregateFunction(quantiles(0.90, 0.99), UInt64), - stddev_blocks_behind AggregateFunction(stddevSamp, UInt64) -) -ENGINE = AggregatingMergeTree -ORDER BY (time_bucket, subgraph, indexer, gateway_id); - --- Materialized View for Daily Allocation Aggregations (Corrected SELECT with ALL -State functions and aliases 'qd', 'iq') -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_allocation_daily TO agg_allocation_daily AS -SELECT - toStartOfDay(qd.event_time) AS time_bucket, -- Changed time function - qd.subgraph, - iq.indexer AS indexer, - qd.gateway_id, - countState() AS query_count, - countIfState(iq.result = 'success') AS success_count, - countIfState(iq.result != 'success') AS failure_count, - sumState(iq.fee_grt) AS total_fee_grt, - avgState(iq.response_time_ms) AS avg_indexer_response_time_ms, - maxState(iq.response_time_ms) AS max_indexer_response_time_ms, - quantilesState(0.90, 0.99)(iq.response_time_ms) AS quantiles_indexer_response_time_ms, - stddevSampState(iq.response_time_ms) AS stddev_indexer_response_time_ms, - avgState(iq.fee_grt) AS avg_fee_grt, - maxState(iq.fee_grt) AS max_fee_grt, - quantilesState(0.90, 0.99)(iq.fee_grt) AS quantiles_fee_grt, - stddevSampState(iq.fee_grt) AS stddev_fee_grt, - avgState(iq.seconds_behind) AS avg_seconds_behind, - maxState(iq.seconds_behind) AS max_seconds_behind, - quantilesState(0.90, 0.99)(iq.seconds_behind) AS quantiles_seconds_behind, - stddevSampState(iq.seconds_behind) AS stddev_seconds_behind, - avgState(iq.blocks_behind) AS avg_blocks_behind, - maxState(iq.blocks_behind) AS max_blocks_behind, - quantilesState(0.90, 0.99)(iq.blocks_behind) AS quantiles_blocks_behind, - stddevSampState(iq.blocks_behind) AS stddev_blocks_behind -FROM qos_data AS qd -- Added alias -ARRAY JOIN indexer_queries AS iq -WHERE qd.subgraph IS NOT NULL AND length(iq.indexer) > 0 -GROUP BY time_bucket, qd.subgraph, indexer, qd.gateway_id; -- Use aliases in GROUP BY +WHERE kq.subgraph IS NOT NULL AND length(iq.indexer) > 0 +GROUP BY time_bucket, kq.subgraph, indexer, kq.gateway_id; -- Group by HEX encoded indexer -- ---------------------------------------- --- Indexer Level Aggregations (by indexer_queries.indexer) +-- Indexer Level Aggregations (1-Minute State) -- ---------------------------------------- - --- 5-Minute Indexer Aggregations (Table Definition) -CREATE TABLE IF NOT EXISTS agg_indexer_5min -( - time_bucket DateTime, - indexer String, - gateway_id String, - query_count AggregateFunction(count), - success_count AggregateFunction(countIf, UInt8), - failure_count AggregateFunction(countIf, UInt8), - total_fee_grt AggregateFunction(sum, Float64), - avg_indexer_response_time_ms AggregateFunction(avg, UInt32), - max_indexer_response_time_ms AggregateFunction(max, UInt32), - quantiles_indexer_response_time_ms AggregateFunction(quantiles(0.90, 0.99), UInt32), - stddev_indexer_response_time_ms AggregateFunction(stddevSamp, UInt32), - avg_fee_grt AggregateFunction(avg, Float64), - max_fee_grt AggregateFunction(max, Float64), - quantiles_fee_grt AggregateFunction(quantiles(0.90, 0.99), Float64), - stddev_fee_grt AggregateFunction(stddevSamp, Float64), - avg_seconds_behind AggregateFunction(avg, UInt32), - max_seconds_behind AggregateFunction(max, UInt32), - quantiles_seconds_behind AggregateFunction(quantiles(0.90, 0.99), UInt32), - stddev_seconds_behind AggregateFunction(stddevSamp, UInt32), - avg_blocks_behind AggregateFunction(avg, UInt64), - max_blocks_behind AggregateFunction(max, UInt64), - quantiles_blocks_behind AggregateFunction(quantiles(0.90, 0.99), UInt64), - stddev_blocks_behind AggregateFunction(stddevSamp, UInt64) -) -ENGINE = AggregatingMergeTree -ORDER BY (time_bucket, indexer, gateway_id); - --- Materialized View for 5-Minute Indexer Aggregations (Corrected SELECT with ALL -State functions and aliases 'qd', 'iq') -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_indexer_5min TO agg_indexer_5min AS -SELECT - toStartOfFiveMinute(qd.event_time) AS time_bucket, - iq.indexer AS indexer, - qd.gateway_id, - countState() AS query_count, - countIfState(iq.result = 'success') AS success_count, - countIfState(iq.result != 'success') AS failure_count, - sumState(iq.fee_grt) AS total_fee_grt, - avgState(iq.response_time_ms) AS avg_indexer_response_time_ms, - maxState(iq.response_time_ms) AS max_indexer_response_time_ms, - quantilesState(0.90, 0.99)(iq.response_time_ms) AS quantiles_indexer_response_time_ms, - stddevSampState(iq.response_time_ms) AS stddev_indexer_response_time_ms, - avgState(iq.fee_grt) AS avg_fee_grt, - maxState(iq.fee_grt) AS max_fee_grt, - quantilesState(0.90, 0.99)(iq.fee_grt) AS quantiles_fee_grt, - stddevSampState(iq.fee_grt) AS stddev_fee_grt, - avgState(iq.seconds_behind) AS avg_seconds_behind, - maxState(iq.seconds_behind) AS max_seconds_behind, - quantilesState(0.90, 0.99)(iq.seconds_behind) AS quantiles_seconds_behind, - stddevSampState(iq.seconds_behind) AS stddev_seconds_behind, - avgState(iq.blocks_behind) AS avg_blocks_behind, - maxState(iq.blocks_behind) AS max_blocks_behind, - quantilesState(0.90, 0.99)(iq.blocks_behind) AS quantiles_blocks_behind, - stddevSampState(iq.blocks_behind) AS stddev_blocks_behind -FROM qos_data AS qd -- Added alias -ARRAY JOIN indexer_queries AS iq -WHERE length(iq.indexer) > 0 -GROUP BY time_bucket, indexer, qd.gateway_id; -- Use aliases in GROUP BY - --- Hourly Indexer Aggregations (Table Definition) -CREATE TABLE IF NOT EXISTS agg_indexer_hourly +CREATE TABLE IF NOT EXISTS agg_indexer_1min ( - time_bucket DateTime, - indexer String, + time_bucket DateTime, -- Minute-level bucket + indexer String, -- HEX encoded gateway_id String, + -- Aggregate state columns (same structure as allocation, but grouped differently) query_count AggregateFunction(count), success_count AggregateFunction(countIf, UInt8), failure_count AggregateFunction(countIf, UInt8), @@ -527,14 +196,17 @@ CREATE TABLE IF NOT EXISTS agg_indexer_hourly stddev_blocks_behind AggregateFunction(stddevSamp, UInt64) ) ENGINE = AggregatingMergeTree -ORDER BY (time_bucket, indexer, gateway_id); +PARTITION BY toYYYYMM(time_bucket) +ORDER BY (time_bucket, indexer, gateway_id) -- Grouping keys +TTL time_bucket + INTERVAL 7 DAY DELETE; -- TTL for 1-minute aggregate state --- Materialized View for Hourly Indexer Aggregations (Corrected SELECT with ALL -State functions and aliases 'qd', 'iq') -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_indexer_hourly TO agg_indexer_hourly AS +-- MV to populate 1-Minute Indexer Aggregations (Reads from Kafka) +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_indexer_1min TO agg_indexer_1min AS SELECT - toStartOfHour(qd.event_time) AS time_bucket, -- Changed time function - iq.indexer AS indexer, - qd.gateway_id, + toStartOfMinute(now()) AS time_bucket, -- Use 1-minute bucket based on processing time + HEX(iq.indexer) AS indexer, -- Apply HEX encoding here + kq.gateway_id, + -- Calculate aggregate states directly from Kafka data countState() AS query_count, countIfState(iq.result = 'success') AS success_count, countIfState(iq.result != 'success') AS failure_count, @@ -555,105 +227,48 @@ SELECT maxState(iq.blocks_behind) AS max_blocks_behind, quantilesState(0.90, 0.99)(iq.blocks_behind) AS quantiles_blocks_behind, stddevSampState(iq.blocks_behind) AS stddev_blocks_behind -FROM qos_data AS qd -- Added alias +FROM kafka_qos_data AS kq -- Read directly from Kafka table ARRAY JOIN indexer_queries AS iq -WHERE length(iq.indexer) > 0 -GROUP BY time_bucket, indexer, qd.gateway_id; -- Use aliases in GROUP BY +WHERE length(iq.indexer) > 0 -- Ensure indexer is present +GROUP BY time_bucket, indexer, kq.gateway_id; -- Group by HEX encoded indexer --- Daily Indexer Aggregations (Table Definition) -CREATE TABLE IF NOT EXISTS agg_indexer_daily -( - time_bucket DateTime, - indexer String, - gateway_id String, - query_count AggregateFunction(count), - success_count AggregateFunction(countIf, UInt8), - failure_count AggregateFunction(countIf, UInt8), - total_fee_grt AggregateFunction(sum, Float64), - avg_indexer_response_time_ms AggregateFunction(avg, UInt32), - max_indexer_response_time_ms AggregateFunction(max, UInt32), - quantiles_indexer_response_time_ms AggregateFunction(quantiles(0.90, 0.99), UInt32), - stddev_indexer_response_time_ms AggregateFunction(stddevSamp, UInt32), - avg_fee_grt AggregateFunction(avg, Float64), - max_fee_grt AggregateFunction(max, Float64), - quantiles_fee_grt AggregateFunction(quantiles(0.90, 0.99), Float64), - stddev_fee_grt AggregateFunction(stddevSamp, Float64), - avg_seconds_behind AggregateFunction(avg, UInt32), - max_seconds_behind AggregateFunction(max, UInt32), - quantiles_seconds_behind AggregateFunction(quantiles(0.90, 0.99), UInt32), - stddev_seconds_behind AggregateFunction(stddevSamp, UInt32), - avg_blocks_behind AggregateFunction(avg, UInt64), - max_blocks_behind AggregateFunction(max, UInt64), - quantiles_blocks_behind AggregateFunction(quantiles(0.90, 0.99), UInt64), - stddev_blocks_behind AggregateFunction(stddevSamp, UInt64) -) -ENGINE = AggregatingMergeTree -ORDER BY (time_bucket, indexer, gateway_id); --- Materialized View for Daily Indexer Aggregations (Corrected SELECT with ALL -State functions and aliases 'qd', 'iq') -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_indexer_daily TO agg_indexer_daily AS -SELECT - toStartOfDay(qd.event_time) AS time_bucket, -- Changed time function - iq.indexer AS indexer, - qd.gateway_id, - countState() AS query_count, - countIfState(iq.result = 'success') AS success_count, - countIfState(iq.result != 'success') AS failure_count, - sumState(iq.fee_grt) AS total_fee_grt, - avgState(iq.response_time_ms) AS avg_indexer_response_time_ms, - maxState(iq.response_time_ms) AS max_indexer_response_time_ms, - quantilesState(0.90, 0.99)(iq.response_time_ms) AS quantiles_indexer_response_time_ms, - stddevSampState(iq.response_time_ms) AS stddev_indexer_response_time_ms, - avgState(iq.fee_grt) AS avg_fee_grt, - maxState(iq.fee_grt) AS max_fee_grt, - quantilesState(0.90, 0.99)(iq.fee_grt) AS quantiles_fee_grt, - stddevSampState(iq.fee_grt) AS stddev_fee_grt, - avgState(iq.seconds_behind) AS avg_seconds_behind, - maxState(iq.seconds_behind) AS max_seconds_behind, - quantilesState(0.90, 0.99)(iq.seconds_behind) AS quantiles_seconds_behind, - stddevSampState(iq.seconds_behind) AS stddev_seconds_behind, - avgState(iq.blocks_behind) AS avg_blocks_behind, - maxState(iq.blocks_behind) AS max_blocks_behind, - quantilesState(0.90, 0.99)(iq.blocks_behind) AS quantiles_blocks_behind, - stddevSampState(iq.blocks_behind) AS stddev_blocks_behind -FROM qos_data AS qd -- Added alias -ARRAY JOIN indexer_queries AS iq -WHERE length(iq.indexer) > 0 -GROUP BY time_bucket, indexer, qd.gateway_id; -- Use aliases in GROUP BY +-- ============================================================ +-- FINAL AGGREGATION VIEWS (for GraphQL API) +-- ============================================================ +-- These views read from the 1-minute aggregate state tables +-- and compute the final results for the desired time granularity +-- (5min, hourly, daily) on the fly using -Merge functions. +-- ============================================================ -- ---------------------------------------- --- Final Aggregation Views (Using -Merge with CTEs) +-- Deployment Views (5min, Hourly, Daily) -- ---------------------------------------- --- View for Final 5-Minute Deployment Aggregations (Using CTE) +-- View for Final 5-Minute Deployment Aggregations CREATE VIEW IF NOT EXISTS view_agg_deployment_5min AS WITH AggregatedValues AS ( - -- Step 1: Perform all merges and grouping SELECT - time_bucket, + toStartOfFiveMinute(time_bucket) AS final_time_bucket, subgraph, gateway_id, countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, + sumMerge(total_fees_usd) AS final_total_fees_usd, avgMerge(avg_response_time_ms) AS final_avg_response_time_ms, maxMerge(max_response_time_ms) AS final_max_response_time_ms, - quantilesMerge(0.90, 0.99)(quantiles_response_time_ms) AS final_quantiles_response_time_ms, -- Keep array + quantilesMerge(0.90, 0.99)(quantiles_response_time_ms) AS final_quantiles_response_time_ms, stddevSampMerge(stddev_response_time_ms) AS final_stddev_response_time_ms, - sumMerge(total_fees_usd) AS final_total_fees_usd, avgMerge(avg_fee_usd) AS final_avg_fee_usd, maxMerge(max_fee_usd) AS final_max_fee_usd, - quantilesMerge(0.90, 0.99)(quantiles_fee_usd) AS final_quantiles_fee_usd, -- Keep array + quantilesMerge(0.90, 0.99)(quantiles_fee_usd) AS final_quantiles_fee_usd, stddevSampMerge(stddev_fee_usd) AS final_stddev_fee_usd - FROM agg_deployment_5min - GROUP BY -- Group by dimensions only - time_bucket, - subgraph, - gateway_id + FROM agg_deployment_1min + GROUP BY final_time_bucket, subgraph, gateway_id ) --- Step 2: Select from CTE, extract percentiles, calculate proportion SELECT - time_bucket, + final_time_bucket AS time_bucket, subgraph, gateway_id, final_query_count AS query_count, @@ -661,43 +276,42 @@ SELECT final_failure_count AS failure_count, final_avg_response_time_ms AS avg_response_time_ms, final_max_response_time_ms AS max_response_time_ms, - final_quantiles_response_time_ms[1] AS p90_response_time_ms, -- Extract p90 - final_quantiles_response_time_ms[2] AS p99_response_time_ms, -- Extract p99 + final_quantiles_response_time_ms[1] AS p90_response_time_ms, + final_quantiles_response_time_ms[2] AS p99_response_time_ms, final_stddev_response_time_ms AS stddev_response_time_ms, final_total_fees_usd AS total_fees_usd, final_avg_fee_usd AS avg_fee_usd, final_max_fee_usd AS max_fee_usd, - final_quantiles_fee_usd[1] AS p90_fee_usd, -- Extract p90 - final_quantiles_fee_usd[2] AS p99_fee_usd, -- Extract p99 + final_quantiles_fee_usd[1] AS p90_fee_usd, + final_quantiles_fee_usd[2] AS p99_fee_usd, final_stddev_fee_usd AS stddev_fee_usd, - -- Calculate success proportion using aliases from CTE if(final_query_count = 0, 0.0, final_success_count / final_query_count) AS success_proportion FROM AggregatedValues; --- View for Final Hourly Deployment Aggregations (Using CTE) +-- View for Final Hourly Deployment Aggregations CREATE VIEW IF NOT EXISTS view_agg_deployment_hourly AS WITH AggregatedValues AS ( SELECT - time_bucket, + toStartOfHour(time_bucket) AS final_time_bucket, subgraph, gateway_id, countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, + sumMerge(total_fees_usd) AS final_total_fees_usd, avgMerge(avg_response_time_ms) AS final_avg_response_time_ms, maxMerge(max_response_time_ms) AS final_max_response_time_ms, quantilesMerge(0.90, 0.99)(quantiles_response_time_ms) AS final_quantiles_response_time_ms, stddevSampMerge(stddev_response_time_ms) AS final_stddev_response_time_ms, - sumMerge(total_fees_usd) AS final_total_fees_usd, avgMerge(avg_fee_usd) AS final_avg_fee_usd, maxMerge(max_fee_usd) AS final_max_fee_usd, quantilesMerge(0.90, 0.99)(quantiles_fee_usd) AS final_quantiles_fee_usd, stddevSampMerge(stddev_fee_usd) AS final_stddev_fee_usd - FROM agg_deployment_hourly - GROUP BY time_bucket, subgraph, gateway_id + FROM agg_deployment_1min + GROUP BY final_time_bucket, subgraph, gateway_id ) SELECT - time_bucket, + final_time_bucket AS time_bucket, subgraph, gateway_id, final_query_count AS query_count, @@ -717,30 +331,30 @@ SELECT if(final_query_count = 0, 0.0, final_success_count / final_query_count) AS success_proportion FROM AggregatedValues; --- View for Final Daily Deployment Aggregations (Using CTE) +-- View for Final Daily Deployment Aggregations CREATE VIEW IF NOT EXISTS view_agg_deployment_daily AS WITH AggregatedValues AS ( SELECT - time_bucket, + toStartOfDay(time_bucket) AS final_time_bucket, subgraph, gateway_id, countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, + sumMerge(total_fees_usd) AS final_total_fees_usd, avgMerge(avg_response_time_ms) AS final_avg_response_time_ms, maxMerge(max_response_time_ms) AS final_max_response_time_ms, quantilesMerge(0.90, 0.99)(quantiles_response_time_ms) AS final_quantiles_response_time_ms, stddevSampMerge(stddev_response_time_ms) AS final_stddev_response_time_ms, - sumMerge(total_fees_usd) AS final_total_fees_usd, avgMerge(avg_fee_usd) AS final_avg_fee_usd, maxMerge(max_fee_usd) AS final_max_fee_usd, quantilesMerge(0.90, 0.99)(quantiles_fee_usd) AS final_quantiles_fee_usd, stddevSampMerge(stddev_fee_usd) AS final_stddev_fee_usd - FROM agg_deployment_daily - GROUP BY time_bucket, subgraph, gateway_id + FROM agg_deployment_1min + GROUP BY final_time_bucket, subgraph, gateway_id ) SELECT - time_bucket, + final_time_bucket AS time_bucket, subgraph, gateway_id, final_query_count AS query_count, @@ -761,22 +375,26 @@ SELECT FROM AggregatedValues; --- View for Final 5-Minute Allocation Aggregations (Using CTE) +-- ---------------------------------------- +-- Allocation Views (5min, Hourly, Daily) +-- ---------------------------------------- + +-- View for Final 5-Minute Allocation Aggregations CREATE VIEW IF NOT EXISTS view_agg_allocation_5min AS WITH AggregatedValues AS ( SELECT - time_bucket, + toStartOfFiveMinute(time_bucket) AS final_time_bucket, subgraph, indexer, gateway_id, countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, + sumMerge(total_fee_grt) AS final_total_fee_grt, avgMerge(avg_indexer_response_time_ms) AS final_avg_indexer_response_time_ms, maxMerge(max_indexer_response_time_ms) AS final_max_indexer_response_time_ms, quantilesMerge(0.90, 0.99)(quantiles_indexer_response_time_ms) AS final_quantiles_indexer_response_time_ms, stddevSampMerge(stddev_indexer_response_time_ms) AS final_stddev_indexer_response_time_ms, - sumMerge(total_fee_grt) AS final_total_fee_grt, avgMerge(avg_fee_grt) AS final_avg_fee_grt, maxMerge(max_fee_grt) AS final_max_fee_grt, quantilesMerge(0.90, 0.99)(quantiles_fee_grt) AS final_quantiles_fee_grt, @@ -789,11 +407,11 @@ WITH AggregatedValues AS ( maxMerge(max_blocks_behind) AS final_max_blocks_behind, quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind - FROM agg_allocation_5min - GROUP BY time_bucket, subgraph, indexer, gateway_id + FROM agg_allocation_1min + GROUP BY final_time_bucket, subgraph, indexer, gateway_id ) SELECT - time_bucket, + final_time_bucket AS time_bucket, subgraph, indexer, gateway_id, @@ -824,22 +442,22 @@ SELECT if(final_query_count = 0, 0.0, final_success_count / final_query_count) AS success_proportion FROM AggregatedValues; --- View for Final Hourly Allocation Aggregations (Using CTE) +-- View for Final Hourly Allocation Aggregations CREATE VIEW IF NOT EXISTS view_agg_allocation_hourly AS WITH AggregatedValues AS ( SELECT - time_bucket, + toStartOfHour(time_bucket) AS final_time_bucket, subgraph, indexer, gateway_id, countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, + sumMerge(total_fee_grt) AS final_total_fee_grt, avgMerge(avg_indexer_response_time_ms) AS final_avg_indexer_response_time_ms, maxMerge(max_indexer_response_time_ms) AS final_max_indexer_response_time_ms, quantilesMerge(0.90, 0.99)(quantiles_indexer_response_time_ms) AS final_quantiles_indexer_response_time_ms, stddevSampMerge(stddev_indexer_response_time_ms) AS final_stddev_indexer_response_time_ms, - sumMerge(total_fee_grt) AS final_total_fee_grt, avgMerge(avg_fee_grt) AS final_avg_fee_grt, maxMerge(max_fee_grt) AS final_max_fee_grt, quantilesMerge(0.90, 0.99)(quantiles_fee_grt) AS final_quantiles_fee_grt, @@ -852,11 +470,11 @@ WITH AggregatedValues AS ( maxMerge(max_blocks_behind) AS final_max_blocks_behind, quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind - FROM agg_allocation_hourly - GROUP BY time_bucket, subgraph, indexer, gateway_id + FROM agg_allocation_1min + GROUP BY final_time_bucket, subgraph, indexer, gateway_id ) SELECT - time_bucket, + final_time_bucket AS time_bucket, subgraph, indexer, gateway_id, @@ -887,22 +505,22 @@ SELECT if(final_query_count = 0, 0.0, final_success_count / final_query_count) AS success_proportion FROM AggregatedValues; --- View for Final Daily Allocation Aggregations (Using CTE) +-- View for Final Daily Allocation Aggregations CREATE VIEW IF NOT EXISTS view_agg_allocation_daily AS WITH AggregatedValues AS ( SELECT - time_bucket, + toStartOfDay(time_bucket) AS final_time_bucket, subgraph, indexer, gateway_id, countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, + sumMerge(total_fee_grt) AS final_total_fee_grt, avgMerge(avg_indexer_response_time_ms) AS final_avg_indexer_response_time_ms, maxMerge(max_indexer_response_time_ms) AS final_max_indexer_response_time_ms, quantilesMerge(0.90, 0.99)(quantiles_indexer_response_time_ms) AS final_quantiles_indexer_response_time_ms, stddevSampMerge(stddev_indexer_response_time_ms) AS final_stddev_indexer_response_time_ms, - sumMerge(total_fee_grt) AS final_total_fee_grt, avgMerge(avg_fee_grt) AS final_avg_fee_grt, maxMerge(max_fee_grt) AS final_max_fee_grt, quantilesMerge(0.90, 0.99)(quantiles_fee_grt) AS final_quantiles_fee_grt, @@ -915,11 +533,11 @@ WITH AggregatedValues AS ( maxMerge(max_blocks_behind) AS final_max_blocks_behind, quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind - FROM agg_allocation_daily - GROUP BY time_bucket, subgraph, indexer, gateway_id + FROM agg_allocation_1min + GROUP BY final_time_bucket, subgraph, indexer, gateway_id ) SELECT - time_bucket, + final_time_bucket AS time_bucket, subgraph, indexer, gateway_id, @@ -951,21 +569,25 @@ SELECT FROM AggregatedValues; --- View for Final 5-Minute Indexer Aggregations (Using CTE) +-- ---------------------------------------- +-- Indexer Views (5min, Hourly, Daily) +-- ---------------------------------------- + +-- View for Final 5-Minute Indexer Aggregations CREATE VIEW IF NOT EXISTS view_agg_indexer_5min AS WITH AggregatedValues AS ( SELECT - time_bucket, + toStartOfFiveMinute(time_bucket) AS final_time_bucket, indexer, gateway_id, countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, + sumMerge(total_fee_grt) AS final_total_fee_grt, avgMerge(avg_indexer_response_time_ms) AS final_avg_indexer_response_time_ms, maxMerge(max_indexer_response_time_ms) AS final_max_indexer_response_time_ms, quantilesMerge(0.90, 0.99)(quantiles_indexer_response_time_ms) AS final_quantiles_indexer_response_time_ms, stddevSampMerge(stddev_indexer_response_time_ms) AS final_stddev_indexer_response_time_ms, - sumMerge(total_fee_grt) AS final_total_fee_grt, avgMerge(avg_fee_grt) AS final_avg_fee_grt, maxMerge(max_fee_grt) AS final_max_fee_grt, quantilesMerge(0.90, 0.99)(quantiles_fee_grt) AS final_quantiles_fee_grt, @@ -978,11 +600,11 @@ WITH AggregatedValues AS ( maxMerge(max_blocks_behind) AS final_max_blocks_behind, quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind - FROM agg_indexer_5min - GROUP BY time_bucket, indexer, gateway_id + FROM agg_indexer_1min + GROUP BY final_time_bucket, indexer, gateway_id ) SELECT - time_bucket, + final_time_bucket AS time_bucket, indexer, gateway_id, final_query_count AS query_count, @@ -1012,21 +634,21 @@ SELECT if(final_query_count = 0, 0.0, final_success_count / final_query_count) AS success_proportion FROM AggregatedValues; --- View for Final Hourly Indexer Aggregations (Using CTE) +-- View for Final Hourly Indexer Aggregations CREATE VIEW IF NOT EXISTS view_agg_indexer_hourly AS WITH AggregatedValues AS ( SELECT - time_bucket, + toStartOfHour(time_bucket) AS final_time_bucket, indexer, gateway_id, countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, + sumMerge(total_fee_grt) AS final_total_fee_grt, avgMerge(avg_indexer_response_time_ms) AS final_avg_indexer_response_time_ms, maxMerge(max_indexer_response_time_ms) AS final_max_indexer_response_time_ms, quantilesMerge(0.90, 0.99)(quantiles_indexer_response_time_ms) AS final_quantiles_indexer_response_time_ms, stddevSampMerge(stddev_indexer_response_time_ms) AS final_stddev_indexer_response_time_ms, - sumMerge(total_fee_grt) AS final_total_fee_grt, avgMerge(avg_fee_grt) AS final_avg_fee_grt, maxMerge(max_fee_grt) AS final_max_fee_grt, quantilesMerge(0.90, 0.99)(quantiles_fee_grt) AS final_quantiles_fee_grt, @@ -1039,11 +661,11 @@ WITH AggregatedValues AS ( maxMerge(max_blocks_behind) AS final_max_blocks_behind, quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind - FROM agg_indexer_hourly - GROUP BY time_bucket, indexer, gateway_id + FROM agg_indexer_1min + GROUP BY final_time_bucket, indexer, gateway_id ) SELECT - time_bucket, + final_time_bucket AS time_bucket, indexer, gateway_id, final_query_count AS query_count, @@ -1073,21 +695,21 @@ SELECT if(final_query_count = 0, 0.0, final_success_count / final_query_count) AS success_proportion FROM AggregatedValues; --- View for Final Daily Indexer Aggregations (Using CTE) +-- View for Final Daily Indexer Aggregations CREATE VIEW IF NOT EXISTS view_agg_indexer_daily AS WITH AggregatedValues AS ( SELECT - time_bucket, + toStartOfDay(time_bucket) AS final_time_bucket, indexer, gateway_id, countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, + sumMerge(total_fee_grt) AS final_total_fee_grt, avgMerge(avg_indexer_response_time_ms) AS final_avg_indexer_response_time_ms, maxMerge(max_indexer_response_time_ms) AS final_max_indexer_response_time_ms, quantilesMerge(0.90, 0.99)(quantiles_indexer_response_time_ms) AS final_quantiles_indexer_response_time_ms, stddevSampMerge(stddev_indexer_response_time_ms) AS final_stddev_indexer_response_time_ms, - sumMerge(total_fee_grt) AS final_total_fee_grt, avgMerge(avg_fee_grt) AS final_avg_fee_grt, maxMerge(max_fee_grt) AS final_max_fee_grt, quantilesMerge(0.90, 0.99)(quantiles_fee_grt) AS final_quantiles_fee_grt, @@ -1100,11 +722,11 @@ WITH AggregatedValues AS ( maxMerge(max_blocks_behind) AS final_max_blocks_behind, quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind - FROM agg_indexer_daily - GROUP BY time_bucket, indexer, gateway_id + FROM agg_indexer_1min + GROUP BY final_time_bucket, indexer, gateway_id ) SELECT - time_bucket, + final_time_bucket AS time_bucket, indexer, gateway_id, final_query_count AS query_count, diff --git a/oracle/clickhouse/users.xml b/oracle/clickhouse/users.xml index dcbe3c3..033e5ba 100644 --- a/oracle/clickhouse/users.xml +++ b/oracle/clickhouse/users.xml @@ -22,9 +22,10 @@ - 1 10000000000 0 + 0 + 0 From d13d30d199fba2d37c40dc707c67891a12345d5d Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Tue, 8 Apr 2025 01:55:11 -0300 Subject: [PATCH 18/20] feat: simplified and optimized tables --- oracle/clickhouse/tables.sql | 159 ++++++++++++++++++------------- oracle/docker-compose.yml | 2 +- oracle/test-producer/src/main.rs | 3 - 3 files changed, 94 insertions(+), 70 deletions(-) diff --git a/oracle/clickhouse/tables.sql b/oracle/clickhouse/tables.sql index 7b68fa8..745e681 100644 --- a/oracle/clickhouse/tables.sql +++ b/oracle/clickhouse/tables.sql @@ -1,4 +1,8 @@ --- Create the Kafka engine table to ingest Protobuf messages +-- ============================================================ +-- RAW DATA INGESTION (Kafka -> qos_data) +-- ============================================================ + +-- Kafka Engine Table (Queue) - Remains the same CREATE TABLE IF NOT EXISTS kafka_qos_data ( gateway_id String, @@ -32,27 +36,27 @@ SETTINGS kafka_group_name = 'clickhouse_qos_consumer', kafka_format = 'ProtobufSingle', kafka_schema = 'schema.proto:qos.ClientQueryProtobuf', - kafka_max_block_size = 1, - kafka_poll_timeout_ms = 500; + kafka_thread_per_consumer = 1; -- ============================================================ --- MINUTE-LEVEL AGGREGATION TABLES & MATERIALIZED VIEWS +-- 5-MINUTE LEVEL AGGREGATION TABLES & MATERIALIZED VIEWS -- ============================================================ --- We store aggregate state only at the 1-minute level. --- Coarser granularities (5min, hourly, daily) are calculated +-- We store aggregate state at the 5-minute level. +-- Coarser granularities (hourly, daily) are calculated -- on the fly in the final views using -Merge functions. --- Materialized Views now read directly from kafka_qos_data. +-- Materialized Views read directly from kafka_qos_data. -- ============================================================ -- ---------------------------------------- --- Deployment Level Aggregations (1-Minute State) +-- Deployment Level Aggregations (5-Minute State) -- ---------------------------------------- -CREATE TABLE IF NOT EXISTS agg_deployment_1min +-- Renamed table from agg_deployment_1min to agg_deployment_5min +CREATE TABLE IF NOT EXISTS agg_deployment_5min ( - time_bucket DateTime, -- Minute-level bucket + time_bucket DateTime, -- 5-Minute level bucket subgraph String, gateway_id String, - -- Aggregate state columns (same as before) + -- Aggregate state columns query_count AggregateFunction(count), success_count AggregateFunction(countIf, UInt8), failure_count AggregateFunction(countIf, UInt8), @@ -67,14 +71,15 @@ CREATE TABLE IF NOT EXISTS agg_deployment_1min stddev_fee_usd AggregateFunction(stddevSamp, Float64) ) ENGINE = AggregatingMergeTree -PARTITION BY toYYYYMM(time_bucket) -- Partitioning is still useful -ORDER BY (time_bucket, subgraph, gateway_id) -- Grouping keys -TTL time_bucket + INTERVAL 7 DAY DELETE; -- TTL for 1-minute aggregate state (adjust as needed) +PARTITION BY toYYYYMM(time_bucket) +ORDER BY (time_bucket, subgraph, gateway_id); --- MV to populate 1-Minute Deployment Aggregations (Reads from Kafka) -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_deployment_1min TO agg_deployment_1min AS +-- Renamed MV from mv_agg_deployment_1min to mv_agg_deployment_5min +-- Changed TO clause to point to agg_deployment_5min +-- Changed time bucketing to toStartOfFiveMinute +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_deployment_5min TO agg_deployment_5min AS SELECT - toStartOfMinute(now()) AS time_bucket, -- Use 1-minute bucket based on processing time + toStartOfFiveMinute(now()) AS time_bucket, -- Use 5-minute bucket kq.subgraph, kq.gateway_id, -- Calculate aggregate states directly from Kafka data @@ -86,7 +91,7 @@ SELECT maxState(kq.response_time_ms) AS max_response_time_ms, quantilesState(0.90, 0.99)(kq.response_time_ms) AS quantiles_response_time_ms, stddevSampState(kq.response_time_ms) AS stddev_response_time_ms, - avgState(kq.total_fees_usd) AS avg_fee_usd, -- Assuming total_fees_usd is per-query fee + avgState(kq.total_fees_usd) AS avg_fee_usd, maxState(kq.total_fees_usd) AS max_fee_usd, quantilesState(0.90, 0.99)(kq.total_fees_usd) AS quantiles_fee_usd, stddevSampState(kq.total_fees_usd) AS stddev_fee_usd @@ -96,11 +101,12 @@ GROUP BY time_bucket, kq.subgraph, kq.gateway_id; -- ---------------------------------------- --- Allocation Level Aggregations (1-Minute State) +-- Allocation Level Aggregations (5-Minute State) -- ---------------------------------------- -CREATE TABLE IF NOT EXISTS agg_allocation_1min +-- Renamed table from agg_allocation_1min to agg_allocation_5min +CREATE TABLE IF NOT EXISTS agg_allocation_5min ( - time_bucket DateTime, -- Minute-level bucket + time_bucket DateTime, -- 5-Minute level bucket subgraph String, indexer String, -- HEX encoded gateway_id String, @@ -128,17 +134,18 @@ CREATE TABLE IF NOT EXISTS agg_allocation_1min ) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMM(time_bucket) -ORDER BY (time_bucket, subgraph, indexer, gateway_id) -- Grouping keys -TTL time_bucket + INTERVAL 7 DAY DELETE; -- TTL for 1-minute aggregate state +ORDER BY (time_bucket, subgraph, indexer, gateway_id); -- Grouping keys --- MV to populate 1-Minute Allocation Aggregations (Reads from Kafka) -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_allocation_1min TO agg_allocation_1min AS +-- Renamed MV from mv_agg_allocation_1min to mv_agg_allocation_5min +-- Changed TO clause to point to agg_allocation_5min +-- Changed time bucketing to toStartOfFiveMinute +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_allocation_5min TO agg_allocation_5min AS SELECT - toStartOfMinute(now()) AS time_bucket, -- Use 1-minute bucket based on processing time + toStartOfFiveMinute(now()) AS time_bucket, -- Use 5-minute bucket kq.subgraph, HEX(iq.indexer) AS indexer, -- Apply HEX encoding here kq.gateway_id, - -- Calculate aggregate states directly from Kafka data + -- Calculate aggregate states directly from Kafka data and nested indexer_queries countState() AS query_count, countIfState(iq.result = 'success') AS success_count, countIfState(iq.result != 'success') AS failure_count, @@ -160,20 +167,21 @@ SELECT quantilesState(0.90, 0.99)(iq.blocks_behind) AS quantiles_blocks_behind, stddevSampState(iq.blocks_behind) AS stddev_blocks_behind FROM kafka_qos_data AS kq -- Read directly from Kafka table -ARRAY JOIN indexer_queries AS iq -WHERE kq.subgraph IS NOT NULL AND length(iq.indexer) > 0 +ARRAY JOIN indexer_queries AS iq -- Join with nested data +WHERE kq.subgraph IS NOT NULL AND length(iq.indexer) > 0 -- Ensure subgraph and indexer are present GROUP BY time_bucket, kq.subgraph, indexer, kq.gateway_id; -- Group by HEX encoded indexer -- ---------------------------------------- --- Indexer Level Aggregations (1-Minute State) +-- Indexer Level Aggregations (5-Minute State) -- ---------------------------------------- -CREATE TABLE IF NOT EXISTS agg_indexer_1min +-- Renamed table from agg_indexer_1min to agg_indexer_5min +CREATE TABLE IF NOT EXISTS agg_indexer_5min ( - time_bucket DateTime, -- Minute-level bucket + time_bucket DateTime, -- 5-Minute level bucket indexer String, -- HEX encoded gateway_id String, - -- Aggregate state columns (same structure as allocation, but grouped differently) + -- Aggregate state columns query_count AggregateFunction(count), success_count AggregateFunction(countIf, UInt8), failure_count AggregateFunction(countIf, UInt8), @@ -197,13 +205,14 @@ CREATE TABLE IF NOT EXISTS agg_indexer_1min ) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMM(time_bucket) -ORDER BY (time_bucket, indexer, gateway_id) -- Grouping keys -TTL time_bucket + INTERVAL 7 DAY DELETE; -- TTL for 1-minute aggregate state +ORDER BY (time_bucket, indexer, gateway_id); -- Grouping keys --- MV to populate 1-Minute Indexer Aggregations (Reads from Kafka) -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_indexer_1min TO agg_indexer_1min AS +-- Renamed MV from mv_agg_indexer_1min to mv_agg_indexer_5min +-- Changed TO clause to point to agg_indexer_5min +-- Changed time bucketing to toStartOfFiveMinute +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_agg_indexer_5min TO agg_indexer_5min AS SELECT - toStartOfMinute(now()) AS time_bucket, -- Use 1-minute bucket based on processing time + toStartOfFiveMinute(now()) AS time_bucket, -- Use 5-minute bucket HEX(iq.indexer) AS indexer, -- Apply HEX encoding here kq.gateway_id, -- Calculate aggregate states directly from Kafka data @@ -236,7 +245,7 @@ GROUP BY time_bucket, indexer, kq.gateway_id; -- Group by HEX encoded indexer -- ============================================================ -- FINAL AGGREGATION VIEWS (for GraphQL API) -- ============================================================ --- These views read from the 1-minute aggregate state tables +-- These views read from the 5-minute aggregate state tables -- and compute the final results for the desired time granularity -- (5min, hourly, daily) on the fly using -Merge functions. -- ============================================================ @@ -246,12 +255,14 @@ GROUP BY time_bucket, indexer, kq.gateway_id; -- Group by HEX encoded indexer -- ---------------------------------------- -- View for Final 5-Minute Deployment Aggregations +-- Updated FROM clause to read from agg_deployment_5min CREATE VIEW IF NOT EXISTS view_agg_deployment_5min AS WITH AggregatedValues AS ( SELECT - toStartOfFiveMinute(time_bucket) AS final_time_bucket, + time_bucket AS final_time_bucket, -- Already at 5-minute level subgraph, gateway_id, + -- Merge the 5-minute states (often just one state per group here) countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, @@ -264,8 +275,8 @@ WITH AggregatedValues AS ( maxMerge(max_fee_usd) AS final_max_fee_usd, quantilesMerge(0.90, 0.99)(quantiles_fee_usd) AS final_quantiles_fee_usd, stddevSampMerge(stddev_fee_usd) AS final_stddev_fee_usd - FROM agg_deployment_1min - GROUP BY final_time_bucket, subgraph, gateway_id + FROM agg_deployment_5min -- Read from the 5-minute state table + GROUP BY final_time_bucket, subgraph, gateway_id -- Group by 5-minute bucket and dimensions ) SELECT final_time_bucket AS time_bucket, @@ -289,12 +300,14 @@ SELECT FROM AggregatedValues; -- View for Final Hourly Deployment Aggregations +-- Updated FROM clause to read from agg_deployment_5min CREATE VIEW IF NOT EXISTS view_agg_deployment_hourly AS WITH AggregatedValues AS ( SELECT - toStartOfHour(time_bucket) AS final_time_bucket, + toStartOfHour(time_bucket) AS final_time_bucket, -- Group 5min buckets into hourly subgraph, gateway_id, + -- Merge the 5-minute states countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, @@ -307,8 +320,8 @@ WITH AggregatedValues AS ( maxMerge(max_fee_usd) AS final_max_fee_usd, quantilesMerge(0.90, 0.99)(quantiles_fee_usd) AS final_quantiles_fee_usd, stddevSampMerge(stddev_fee_usd) AS final_stddev_fee_usd - FROM agg_deployment_1min - GROUP BY final_time_bucket, subgraph, gateway_id + FROM agg_deployment_5min -- Read from the 5-minute state table + GROUP BY final_time_bucket, subgraph, gateway_id -- Group by hourly bucket and dimensions ) SELECT final_time_bucket AS time_bucket, @@ -332,12 +345,14 @@ SELECT FROM AggregatedValues; -- View for Final Daily Deployment Aggregations +-- Updated FROM clause to read from agg_deployment_5min CREATE VIEW IF NOT EXISTS view_agg_deployment_daily AS WITH AggregatedValues AS ( SELECT - toStartOfDay(time_bucket) AS final_time_bucket, + toStartOfDay(time_bucket) AS final_time_bucket, -- Group 5min buckets into daily subgraph, gateway_id, + -- Merge the 5-minute states countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, @@ -350,8 +365,8 @@ WITH AggregatedValues AS ( maxMerge(max_fee_usd) AS final_max_fee_usd, quantilesMerge(0.90, 0.99)(quantiles_fee_usd) AS final_quantiles_fee_usd, stddevSampMerge(stddev_fee_usd) AS final_stddev_fee_usd - FROM agg_deployment_1min - GROUP BY final_time_bucket, subgraph, gateway_id + FROM agg_deployment_5min -- Read from the 5-minute state table + GROUP BY final_time_bucket, subgraph, gateway_id -- Group by daily bucket and dimensions ) SELECT final_time_bucket AS time_bucket, @@ -380,13 +395,15 @@ FROM AggregatedValues; -- ---------------------------------------- -- View for Final 5-Minute Allocation Aggregations +-- Updated FROM clause to read from agg_allocation_5min CREATE VIEW IF NOT EXISTS view_agg_allocation_5min AS WITH AggregatedValues AS ( SELECT - toStartOfFiveMinute(time_bucket) AS final_time_bucket, + time_bucket AS final_time_bucket, -- Already at 5-minute level subgraph, indexer, gateway_id, + -- Merge the 5-minute states countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, @@ -407,8 +424,8 @@ WITH AggregatedValues AS ( maxMerge(max_blocks_behind) AS final_max_blocks_behind, quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind - FROM agg_allocation_1min - GROUP BY final_time_bucket, subgraph, indexer, gateway_id + FROM agg_allocation_5min -- Read from the 5-minute state table + GROUP BY final_time_bucket, subgraph, indexer, gateway_id -- Group by 5-minute bucket and dimensions ) SELECT final_time_bucket AS time_bucket, @@ -443,13 +460,15 @@ SELECT FROM AggregatedValues; -- View for Final Hourly Allocation Aggregations +-- Updated FROM clause to read from agg_allocation_5min CREATE VIEW IF NOT EXISTS view_agg_allocation_hourly AS WITH AggregatedValues AS ( SELECT - toStartOfHour(time_bucket) AS final_time_bucket, + toStartOfHour(time_bucket) AS final_time_bucket, -- Group 5min buckets into hourly subgraph, indexer, gateway_id, + -- Merge the 5-minute states countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, @@ -470,8 +489,8 @@ WITH AggregatedValues AS ( maxMerge(max_blocks_behind) AS final_max_blocks_behind, quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind - FROM agg_allocation_1min - GROUP BY final_time_bucket, subgraph, indexer, gateway_id + FROM agg_allocation_5min -- Read from the 5-minute state table + GROUP BY final_time_bucket, subgraph, indexer, gateway_id -- Group by hourly bucket and dimensions ) SELECT final_time_bucket AS time_bucket, @@ -506,13 +525,15 @@ SELECT FROM AggregatedValues; -- View for Final Daily Allocation Aggregations +-- Updated FROM clause to read from agg_allocation_5min CREATE VIEW IF NOT EXISTS view_agg_allocation_daily AS WITH AggregatedValues AS ( SELECT - toStartOfDay(time_bucket) AS final_time_bucket, + toStartOfDay(time_bucket) AS final_time_bucket, -- Group 5min buckets into daily subgraph, indexer, gateway_id, + -- Merge the 5-minute states countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, @@ -533,8 +554,8 @@ WITH AggregatedValues AS ( maxMerge(max_blocks_behind) AS final_max_blocks_behind, quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind - FROM agg_allocation_1min - GROUP BY final_time_bucket, subgraph, indexer, gateway_id + FROM agg_allocation_5min -- Read from the 5-minute state table + GROUP BY final_time_bucket, subgraph, indexer, gateway_id -- Group by daily bucket and dimensions ) SELECT final_time_bucket AS time_bucket, @@ -574,12 +595,14 @@ FROM AggregatedValues; -- ---------------------------------------- -- View for Final 5-Minute Indexer Aggregations +-- Updated FROM clause to read from agg_indexer_5min CREATE VIEW IF NOT EXISTS view_agg_indexer_5min AS WITH AggregatedValues AS ( SELECT - toStartOfFiveMinute(time_bucket) AS final_time_bucket, + time_bucket AS final_time_bucket, -- Already at 5-minute level indexer, gateway_id, + -- Merge the 5-minute states countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, @@ -600,8 +623,8 @@ WITH AggregatedValues AS ( maxMerge(max_blocks_behind) AS final_max_blocks_behind, quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind - FROM agg_indexer_1min - GROUP BY final_time_bucket, indexer, gateway_id + FROM agg_indexer_5min -- Read from the 5-minute state table + GROUP BY final_time_bucket, indexer, gateway_id -- Group by 5-minute bucket and dimensions ) SELECT final_time_bucket AS time_bucket, @@ -635,12 +658,14 @@ SELECT FROM AggregatedValues; -- View for Final Hourly Indexer Aggregations +-- Updated FROM clause to read from agg_indexer_5min CREATE VIEW IF NOT EXISTS view_agg_indexer_hourly AS WITH AggregatedValues AS ( SELECT - toStartOfHour(time_bucket) AS final_time_bucket, + toStartOfHour(time_bucket) AS final_time_bucket, -- Group 5min buckets into hourly indexer, gateway_id, + -- Merge the 5-minute states countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, @@ -661,8 +686,8 @@ WITH AggregatedValues AS ( maxMerge(max_blocks_behind) AS final_max_blocks_behind, quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind - FROM agg_indexer_1min - GROUP BY final_time_bucket, indexer, gateway_id + FROM agg_indexer_5min -- Read from the 5-minute state table + GROUP BY final_time_bucket, indexer, gateway_id -- Group by hourly bucket and dimensions ) SELECT final_time_bucket AS time_bucket, @@ -696,12 +721,14 @@ SELECT FROM AggregatedValues; -- View for Final Daily Indexer Aggregations +-- Updated FROM clause to read from agg_indexer_5min CREATE VIEW IF NOT EXISTS view_agg_indexer_daily AS WITH AggregatedValues AS ( SELECT - toStartOfDay(time_bucket) AS final_time_bucket, + toStartOfDay(time_bucket) AS final_time_bucket, -- Group 5min buckets into daily indexer, gateway_id, + -- Merge the 5-minute states countMerge(query_count) AS final_query_count, countIfMerge(success_count) AS final_success_count, countIfMerge(failure_count) AS final_failure_count, @@ -722,8 +749,8 @@ WITH AggregatedValues AS ( maxMerge(max_blocks_behind) AS final_max_blocks_behind, quantilesMerge(0.90, 0.99)(quantiles_blocks_behind) AS final_quantiles_blocks_behind, stddevSampMerge(stddev_blocks_behind) AS final_stddev_blocks_behind - FROM agg_indexer_1min - GROUP BY final_time_bucket, indexer, gateway_id + FROM agg_indexer_5min -- Read from the 5-minute state table + GROUP BY final_time_bucket, indexer, gateway_id -- Group by daily bucket and dimensions ) SELECT final_time_bucket AS time_bucket, diff --git a/oracle/docker-compose.yml b/oracle/docker-compose.yml index 18a9d68..0f1d0f8 100644 --- a/oracle/docker-compose.yml +++ b/oracle/docker-compose.yml @@ -109,7 +109,7 @@ services: - ./clickhouse/schema.proto:/app/clickhouse/schema.proto:ro environment: KAFKA_BROKER: redpanda:9092 - MESSAGES_PER_SECOND: 5 + MESSAGES_PER_SECOND: 1000 restart: on-failure healthcheck: <<: *default-healthcheck diff --git a/oracle/test-producer/src/main.rs b/oracle/test-producer/src/main.rs index 1d824a8..a6413b9 100644 --- a/oracle/test-producer/src/main.rs +++ b/oracle/test-producer/src/main.rs @@ -524,9 +524,6 @@ async fn main() { // Encode using the gateway's exact logic let encoded_message = encode_client_request(client_request); - println!("Encoded message: {} ", hex::encode(&encoded_message)); - println!("Encoded message size: {} bytes", encoded_message.len()); - counter += 1; match producer From a9f54b999e99fa9916299d6cb1394e326b1baa1a Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Tue, 8 Apr 2025 02:47:54 -0300 Subject: [PATCH 19/20] feat: improvements for test producer and kafka engine settings --- Cargo.lock | 17 ++++ oracle/clickhouse/tables.sql | 3 +- oracle/docker-compose.yml | 2 +- oracle/test-producer/Cargo.toml | 1 + oracle/test-producer/src/main.rs | 149 +++++++++++++++++++++++-------- 5 files changed, 131 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c575c3..cdee352 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1098,6 +1098,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + [[package]] name = "hex" version = "0.4.3" @@ -1609,6 +1615,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "num_enum" version = "0.7.3" @@ -1853,6 +1869,7 @@ version = "0.1.0" dependencies = [ "chrono", "hex", + "num_cpus", "once_cell", "prost", "prost-build", diff --git a/oracle/clickhouse/tables.sql b/oracle/clickhouse/tables.sql index 745e681..b47f4e3 100644 --- a/oracle/clickhouse/tables.sql +++ b/oracle/clickhouse/tables.sql @@ -36,7 +36,8 @@ SETTINGS kafka_group_name = 'clickhouse_qos_consumer', kafka_format = 'ProtobufSingle', kafka_schema = 'schema.proto:qos.ClientQueryProtobuf', - kafka_thread_per_consumer = 1; + kafka_thread_per_consumer = 1, + kafka_flush_interval_ms = 2500; -- ============================================================ -- 5-MINUTE LEVEL AGGREGATION TABLES & MATERIALIZED VIEWS diff --git a/oracle/docker-compose.yml b/oracle/docker-compose.yml index 0f1d0f8..f83a626 100644 --- a/oracle/docker-compose.yml +++ b/oracle/docker-compose.yml @@ -109,7 +109,7 @@ services: - ./clickhouse/schema.proto:/app/clickhouse/schema.proto:ro environment: KAFKA_BROKER: redpanda:9092 - MESSAGES_PER_SECOND: 1000 + MESSAGES_PER_SECOND: 50000 restart: on-failure healthcheck: <<: *default-healthcheck diff --git a/oracle/test-producer/Cargo.toml b/oracle/test-producer/Cargo.toml index cf33bea..1698bd3 100644 --- a/oracle/test-producer/Cargo.toml +++ b/oracle/test-producer/Cargo.toml @@ -13,6 +13,7 @@ rand = "0.8" chrono = "0.4" hex = "0.4" once_cell = "1.19" +num_cpus = "1" [build-dependencies] prost-build = "0.13.1" \ No newline at end of file diff --git a/oracle/test-producer/src/main.rs b/oracle/test-producer/src/main.rs index a6413b9..502c3a0 100644 --- a/oracle/test-producer/src/main.rs +++ b/oracle/test-producer/src/main.rs @@ -10,6 +10,7 @@ use rdkafka::util::get_rdkafka_version; use tokio::time; use once_cell::sync::Lazy; use rand::seq::SliceRandom; +use num_cpus; // Mock types to match the gateway's dependencies mod mock { @@ -477,68 +478,138 @@ fn encode_client_request( buf } +// --- NEW: Asynchronous function to run in each producer task --- +async fn produce_messages(producer: FutureProducer, topic: String, task_id: u32, delay_micros: u64) { + let mut counter: u64 = 0; + println!("[Task {}] Starting producer loop", task_id); + + loop { + // --- Use existing message generation logic --- + let client_request = generate_random_client_request(); + let payload = encode_client_request(client_request); + // --- End Use existing message generation logic --- + + // Use a simple counter or a field from the request for the key + // to help distribute messages across partitions. + // Using modulo is okay for basic distribution. + let key = format!("key-{}", counter % 100); + + // Send asynchronously - *DO NOT* await the future here in the loop! + let send_future = producer.send_result( + FutureRecord::to(&topic) + .payload(&payload) + .key(&key), + ); + + // Check for immediate queuing errors (e.g., queue full). + if let Err((e, _)) = send_future { + eprintln!("[Task {}] Failed to queue message (queue full?): {:?}. Pausing...", task_id, e); + // Pause briefly if the producer queue is full to avoid overwhelming it. + time::sleep(Duration::from_millis(100)).await; + // Optionally: break or implement more robust backoff + } + + counter += 1; + + // Apply throttling delay if configured + if delay_micros > 0 { + time::sleep(Duration::from_micros(delay_micros)).await; + } else { + // Yield control occasionally if the loop is extremely tight (no throttling) + if counter % 1000 == 0 { + tokio::task::yield_now().await; + } + } + } +} + +// --- NEW: Updated main function using Tokio --- #[tokio::main] async fn main() { // Print librdkafka version let (version_n, version_s) = get_rdkafka_version(); println!("rd_kafka_version: 0x{:08x}, {}", version_n, version_s); - // Read messages per second from the environment variable; default to 10 if not provided. - let mps: u64 = env::var("MESSAGES_PER_SECOND") - .unwrap_or_else(|_| "10".to_string()) + // --- Configuration --- + // Read target total messages per second (informational, not used for delay) + let total_mps: u64 = env::var("MESSAGES_PER_SECOND") + .unwrap_or_else(|_| "10000".to_string()) // Default to 10k .parse() .expect("MESSAGES_PER_SECOND must be a valid number"); - let delay = Duration::from_micros(1_000_000 / mps); - // Allow configuring the broker address from environment + // Determine number of producer tasks (use logical cores) + let num_tasks = num_cpus::get().max(1); + println!("Target MPS: {}, Spawning {} producer tasks", total_mps, num_tasks); + + // Calculate delay per task to achieve target MPS + // Each task needs to wait `num_tasks` times longer than the overall target interval + let delay_per_task_micros = if total_mps > 0 { + (1_000_000 * num_tasks as u64) / total_mps + } else { + 0 // No delay if target MPS is 0 (run as fast as possible) + }; + if delay_per_task_micros > 0 { + println!( + "[Config] Throttling enabled: Each task will pause for {} microseconds.", + delay_per_task_micros + ); + } else { + println!("[Config] Throttling disabled (MESSAGES_PER_SECOND=0 or not set appropriately). Running at max speed."); + } + let broker = env::var("KAFKA_BROKER").unwrap_or_else(|_| "redpanda:9092".to_string()); + let topic = "gateway_qos_topic".to_string(); // Ensure this matches ClickHouse println!("Connecting to Kafka broker at {}", broker); - - // Wait for Redpanda to be fully ready println!("Waiting 5 seconds for Redpanda to initialize..."); time::sleep(Duration::from_secs(5)).await; - // Create a Kafka producer with explicit PLAINTEXT protocol + // --- Producer Setup --- + // Configure the Kafka producer client let producer: FutureProducer = ClientConfig::new() .set("bootstrap.servers", &broker) - .set("message.timeout.ms", "30000") + .set("message.timeout.ms", "30000") // Max time librdkafka tries to send one message .set("security.protocol", "PLAINTEXT") - .set("debug", "all") - .set("enable.idempotence", "false") // Simplify for testing - .set("retries", "5") // Retry a few times - .set("retry.backoff.ms", "1000") // 1 second between retries - .set("socket.timeout.ms", "10000") // 10 seconds socket timeout + // .set("debug", "all") // Disable verbose debugging for performance testing + .set("enable.idempotence", "false") // Disable for max throughput test (less overhead) + .set("retries", "5") // Retries for recoverable send failures + .set("retry.backoff.ms", "100") // Backoff between retries (reduced for faster recovery) + .set("socket.timeout.ms", "10000") .set("socket.keepalive.enable", "true") + // --- Batching Configuration (Crucial for Throughput) --- + .set("linger.ms", "5") // Wait up to 5ms to gather messages into a batch + .set("batch.size", "131072") // Batch size in bytes (e.g., 128 KB). Tune based on avg message size. + // --- Buffering Configuration (Increase if producer blocks/errors often) --- + .set("queue.buffering.max.messages", "500000") // Max messages in internal producer queue + .set("queue.buffering.max.ms", "1000") // Max time msgs wait in queue before send() blocks/errors + // --- Optional: Compression (Reduces network bandwidth, increases CPU) --- + // .set("compression.codec", "snappy") // or lz4, gzip, zstd .create() .expect("Producer creation error"); - // Make sure this matches the topic name in ClickHouse Kafka engine - let topic = "gateway_qos_topic"; - let mut counter: u64 = 0; - - loop { - // Generate a random client request - let client_request = generate_random_client_request(); - - // Encode using the gateway's exact logic - let encoded_message = encode_client_request(client_request); - - counter += 1; + // --- Spawn Producer Tasks --- + let mut tasks = vec![]; + println!("Spawning {} producer tasks...", num_tasks); + for i in 0..num_tasks { + let producer_clone = producer.clone(); // Clone producer for each task + let topic_clone = topic.clone(); + tasks.push(tokio::spawn(produce_messages( + producer_clone, + topic_clone, + i as u32, + delay_per_task_micros, // Pass the calculated delay + ))); + } - match producer - .send( - FutureRecord::to(topic) - .payload(&encoded_message) - .key(&format!("{}", counter)), - Duration::from_secs(0), - ) - .await - { - Ok((partition, offset)) => println!("Delivered: ({}, {})", partition, offset), - Err((e, _)) => eprintln!("Delivery error: {:?}", e), + // --- Keep Main Task Alive --- + println!( + "Producer tasks running. Sending to topic '{}'. Press Ctrl+C to stop.", + topic + ); + // Wait for all tasks to complete (they run infinitely, so this waits forever unless they error out) + for task in tasks { + if let Err(e) = task.await { + eprintln!("Producer task failed: {:?}", e); } - // Sleep for the configured delay - time::sleep(delay).await; } } From 5bd0c760606b4055c869c8f708f5a62cff327809 Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Wed, 9 Apr 2025 02:09:43 -0300 Subject: [PATCH 20/20] feat: update README.md --- README.md | 430 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 261 insertions(+), 169 deletions(-) diff --git a/README.md b/README.md index 6a11d26..baf1bc3 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,41 @@ QoS Oracle V2 is a data pipeline and reporting system for The Graph Network quality metrics. It collects, processes, and exposes quality-of-service data from Gateway nodes, providing valuable insights into indexer performance and network health. -## Architecture - -The system consists of three main components: - -1. **Data Ingestion** - Kafka/Redpanda message broker receives QoS data from Gateways -2. **Data Processing** - ClickHouse database ingests, stores, and aggregates the data into time-based views (5min, hourly, daily) -3. **Data Exposure** - GraphQL API serves these pre-aggregated metrics from Clickhouse views +## Architecture & Data Flow + +The system processes data in the following sequence: + +1. **Gateway:** External Gateway nodes (deployed separately) produce Protobuf-encoded QoS messages containing details about client queries and indexer interactions. +2. **Redpanda/Kafka (`gateway_qos_topic`):** These messages are published to a central Kafka topic, typically named `gateway_qos_topic`. In production, this is usually an existing, managed Kafka cluster (like Redpanda). +3. **ClickHouse (`kafka_qos_data` -> MVs -> Aggregates):** The ClickHouse server consumes messages from the `gateway_qos_topic` via a Kafka Engine table (`kafka_qos_data`). Materialized Views (MVs) trigger automatically to process these raw messages, calculating aggregates and storing them efficiently in AggregatingMergeTree tables (e.g., `agg_deployment_5min`, `agg_indexer_5min`, etc.) at a 5-minute granularity. +4. **GraphQL API:** The GraphQL API service queries the aggregated tables in ClickHouse (using the final Views like `deployment_aggregations_hourly`) to serve requests for QoS metrics over different time intervals (5min, hourly, daily). +5. **Monitoring (Prometheus & Grafana):** + * Prometheus (typically an existing instance) scrapes metrics directly from the ClickHouse server's `/metrics` endpoint (port 9363 by default). + * Grafana (typically an existing instance) queries Prometheus for ClickHouse operational metrics and can also query the ClickHouse aggregated tables directly for QoS metric dashboards. + +```mermaid +graph LR + A[Gateway] -- Protobuf Messages --> B(Redpanda/Kafka Topic: gateway_qos_topic); + B -- Consumed by --> C{ClickHouse Server}; + C -- Kafka Engine Table --> D[MV: mv_agg_..._5min]; + D -- Aggregates & Inserts --> E[AggregatingMergeTree: agg_..._5min]; + F[GraphQL API] -- Queries --> E; + G[Prometheus] -- Scrapes /metrics --> C; + H[Grafana] -- Queries --> G; + H -- Queries --> E; + + subgraph "QoS Oracle V2 Components (To Deploy)" + C + F + end + + subgraph "External/Existing Infrastructure" + A + B + G + H + end +``` ## Features @@ -18,9 +46,12 @@ The system consists of three main components: - Efficient storage with automatic TTL for raw data (managed by ClickHouse) - Pre-aggregated metrics in 5-minute, hourly, and daily intervals - Flexible GraphQL API for querying aggregated data with filtering, sorting, and time range selection -- Docker-based deployment for easy setup and operation using profiles +- Docker-based deployment for easy local development setup +- Configurable via environment variables for production deployments + +## Getting Started (Local Development) -## Getting Started +This section describes how to run the *entire* stack locally using Docker Compose, including dependencies like Redpanda and a test data producer. **This is intended for development and testing purposes only.** For production deployment guidance, see the "Production Deployment (Kubernetes/Helm)" section. ### Prerequisites @@ -32,42 +63,105 @@ The system consists of three main components: ### Installation -1. Clone the repository: - ```bash - git clone https://github.com/graphops/qos-oracle.git # Or your repo URL - cd qos-oracle - ``` +1. Clone the repository: + ```bash + git clone https://github.com/graphops/qos-oracle.git # Or your repo URL + cd qos-oracle + ``` + +2. Start the services (Development profile includes API, DB, Kafka, and Test Producer): + ```bash + # Ensure you are in the root qos-oracle directory + docker compose -f oracle/docker-compose.yml --profile dev up --build -d + ``` + * Use `--profile deps` to start only ClickHouse and Redpanda. + * Use `--profile all` to start everything including monitoring tools. + +3. Verify the installation: + ```bash + # Check API health + curl http://localhost:8000/health + # Check running containers + docker compose -f oracle/docker-compose.yml ps + ``` + +4. Access the GraphQL endpoint at `http://localhost:8000/graphql`. + +## Production Deployment (Kubernetes/Helm) + +This section provides guidance for deploying the QoS Oracle V2 components into a production Kubernetes environment, likely managed via Helm, assuming existing infrastructure for Kafka, Prometheus, and Grafana. + +### Components to Deploy + +From the `oracle/docker-compose.yml` definition, you will need to create Kubernetes resources (Deployments, Services, ConfigMaps, etc.) for: + +1. **`clickhouse-server`:** The core data processing and storage engine. +2. **`graphql-api`:** The service exposing the aggregated data. + +### Components NOT to Deploy (Assuming Existing Infrastructure) -2. Start the services (Development profile includes API, DB, Kafka, and Test Producer): - ```bash - # Ensure you are in the root qos-oracle directory - docker compose -f oracle/docker-compose.yml --profile dev up --build -d - ``` - * Use `--profile deps` to start only ClickHouse and Redpanda +You will likely **not** deploy the following components if they already exist in your cluster: -3. Verify the installation: - ```bash - # Check API health - curl http://localhost:8000/health - # Check running containers - docker compose -f oracle/docker-compose.yml ps - ``` +* `redpanda`: Use your existing Kafka/Redpanda cluster. +* `test-producer`: This is only for development; real data comes from Gateways. +* `prometheus`: Integrate with your existing Prometheus instance. +* `grafana`: Integrate with your existing Grafana instance. +* `redpanda-console`: Use existing tools or deploy if needed separately. -4. Access the GraphQL endpoint at `http://localhost:8000/graphql`. You can use tools like Postman, Insomnia, `curl`, or potentially a browser-based playground if enabled. +### Configuration (Environment Variables) -### Configuration +Configure the deployable components using environment variables, typically managed via Helm `values.yaml` or Kubernetes manifests. -The system can be configured through environment variables set in the `oracle/docker-compose.yml` file for the respective services: +**1. `clickhouse-server` Configuration:** -- **`graphql-api` service:** - * `CLICKHOUSE_DSN`: Full ClickHouse connection string (e.g., `tcp://graphql:graphql_password@clickhouse-server:9000/default?compression=lz4`). This replaces separate URL, DB, User, Pass variables. - * `PORT`: Internal port the API listens on (default: `8000`). - * `RUST_LOG`: Logging level (e.g., `info`, `debug`). -- **`test-producer` service:** - * `KAFKA_BROKER`: Kafka/Redpanda broker address (e.g., `redpanda:9092`). - * `MESSAGES_PER_SECOND`: Test producer message rate. +* **Image:** `clickhouse/clickhouse-server:25.3` (or the version specified in `oracle/docker-compose.yml`) +* **Initialization:** The standard ClickHouse Docker image executes `.sh`, `.sql`, and `.sql.gz` files found in `/docker-entrypoint-initdb.d/` upon initial startup. For production: + * You need to provide the necessary SQL schema definitions (defined in `oracle/clickhouse/tables.sql`) to create the `kafka_qos_data` table, Materialized Views (e.g., `mv_agg_deployment_5min`), and AggregatingMergeTree tables (e.g., `agg_deployment_5min`). + * Mount these SQL definition files into the `/docker-entrypoint-initdb.d/` directory within the container. This is typically done using a Kubernetes ConfigMap mounted as volume(s). + * **Crucially:** The `CREATE TABLE kafka_qos_data` statement within your mounted SQL file(s) must use the correct production values for `kafka_broker_list`, `kafka_topic_list`, `kafka_group_name`, and `kafka_schema`. You will likely need to template these values into the ConfigMap using Helm or a similar tool, rather than hardcoding them. +* **Volume Mounts:** + * Mount your SQL initialization file(s) (e.g., from a ConfigMap) to `/docker-entrypoint-initdb.d/`. (This is the tables.sql file) + * Mount `oracle/clickhouse/schema.proto` to `/var/lib/clickhouse/user_files/schema.proto`. (The `kafka_schema` setting in your SQL should reference the relative path: `'schema.proto:qos.ClientQueryProtobuf'`). + * Mount necessary ClickHouse config XMLs (`settings.xml`, `users.xml`) to `/etc/clickhouse-server/config.d/` and `/etc/clickhouse-server/users.d/` respectively. Ensure `users.xml` defines the user/password needed by the GraphQL API. +* **Ports:** + * `8123` (HTTP): For queries, health checks. + * `9000` (Native): For GraphQL API connection. + * `9363` (Metrics): For Prometheus scraping. +* **Persistence:** Configure persistent storage for `/var/lib/clickhouse`. -*(See `x-clickhouse-env` in `docker-compose.yml` for default user/password used in the default DSN)* +**2. `graphql-api` Configuration:** + +* **Image:** Build the image using the Dockerfile at `oracle/graphql-api/Dockerfile`. +* **Required Environment Variables:** + * `CLICKHOUSE_URL`: The endpoint for connecting to the ClickHouse native protocol (Ensure user/pass match ClickHouse config). +* **Ports:** + * `8000` (API): Or the value set in the `PORT` env var. + +### Monitoring Integration + +**1. Prometheus Scrape Configuration:** + +Add the following job to your Prometheus configuration to scrape metrics from the deployed ClickHouse service: + +``` +scrape_configs: + - job_name: 'clickhouse-qos-oracle' + metrics_path: /metrics + static_configs: + - targets: [':9363'] # Replace with your K8s service name and metrics port + # Optional: Add labels specific to your environment + # relabel_configs: + # - source_labels: [__address__] + # target_label: instance + # replacement: '' # e.g., qos-oracle-clickhouse-prod +``` + +**2. Grafana Dashboard:** + +* The Grafana dashboard definition used in development is located in `oracle/grafana/dashboards/`. +* Export this dashboard (if running locally) or retrieve the JSON file from the repository. +* Import this JSON file into your production Grafana instance via the UI ("+" -> "Import"). +* **Important:** After importing, you may need to edit the dashboard's settings to ensure the "ClickHouse" data source points to your production ClickHouse instance (if the data source name differs from the one used in development) and the "Prometheus" data source points to your production Prometheus. ## Usage @@ -75,7 +169,7 @@ The system can be configured through environment variables set in the `oracle/do The GraphQL API provides access to aggregated QoS metrics via three main queries: `deploymentAggregations`, `indexerAggregations`, and `allocationAggregations`. -**Endpoint:** `http://localhost:8000/graphql` +**Endpoint:** `http://:/graphql` (Use the Kubernetes service endpoint in production) **Key Arguments:** @@ -93,155 +187,153 @@ The GraphQL API provides access to aggregated QoS metrics via three main queries **Example Queries:** -1. **Get Hourly Deployment Aggregations (Default - Last 24 hours):** - - * **cURL:** - ```bash - curl -X POST http://localhost:8000/graphql \ - -H "Content-Type: application/json" \ - -d '{ "query": "{ deploymentAggregations { timeBucket subgraph gatewayId queryCount successProportion } }" }' - ``` - * **GraphQL:** - ```graphql - { - deploymentAggregations { - timeBucket - subgraph - gatewayId - queryCount - successProportion - } - } - ``` - -2. **Get Daily Indexer Aggregations for a Specific Indexer in October 2023:** - - * **cURL:** - ```bash - curl -X POST http://localhost:8000/graphql \ - -H "Content-Type: application/json" \ - -d '{ - "query": "query IndexerData($indexerId: String!) { indexerAggregations( interval: Daily, timeRange: { from: \"2023-10-01T00:00:00Z\", to: \"2023-11-01T00:00:00Z\" }, filter: { indexer: $indexerId } ) { timeBucket indexer gatewayId queryCount totalFeeGrt avgIndexerResponseTimeMs } }", - "variables": { "indexerId": "0xYourIndexerAddressHere" } - }' - ``` - * **GraphQL (with variables):** - ```graphql - query IndexerData($indexerId: String!) { - indexerAggregations( - interval: Daily, - timeRange: { - from: "2023-10-01T00:00:00Z", - to: "2023-11-01T00:00:00Z" - }, - filter: { indexer: $indexerId } - ) { - timeBucket - indexer - gatewayId - queryCount - totalFeeGrt - avgIndexerResponseTimeMs - } - } - ``` - *Variables:* - ```json - { - "indexerId": "0xYourIndexerAddressHere" - } - ``` - -3. **Get 5-Minute Allocation Aggregations Since a Specific Timestamp (Unix), Sorted by Query Count:** - - * **cURL:** - ```bash - curl -X POST http://localhost:8000/graphql \ - -H "Content-Type: application/json" \ - -d '{ - "query": "{ allocationAggregations( interval: FiveMinutes, timeRange: { from: \"1698300000\" }, sort: { field: QueryCount, direction: Desc }, limit: 50 ) { timeBucket subgraph indexer gatewayId queryCount avgSecondsBehind } }" - }' - ``` - * **GraphQL:** - ```graphql - { - allocationAggregations( - interval: FiveMinutes, - timeRange: { from: "1698300000" }, - sort: { field: QueryCount, direction: Desc }, - limit: 50 - ) { - timeBucket - subgraph - indexer - gatewayId - queryCount - avgSecondsBehind - } - } - ``` - -4. **Get Hourly Deployment Aggregations Up To a Specific Time for a Gateway:** - - * **cURL:** - ```bash - curl -X POST http://localhost:8000/graphql \ - -H "Content-Type: application/json" \ - -d '{ - "query": "query GatewayUntil($gwId: String!) { deploymentAggregations( timeRange: { to: \"2023-10-27T12:00:00Z\" }, filter: { gatewayId: $gwId } ) { timeBucket subgraph gatewayId queryCount avgResponseTimeMs } }", - "variables": { "gwId": "gateway-mainnet-1" } - }' - ``` - * **GraphQL (with variables):** - ```graphql - query GatewayUntil($gwId: String!) { - deploymentAggregations( - timeRange: { to: "2023-10-27T12:00:00Z" }, - filter: { gatewayId: $gwId } - ) { - timeBucket - subgraph - gatewayId - queryCount - avgResponseTimeMs - } - } - ``` - *Variables:* - ```json - { - "gwId": "gateway-mainnet-1" - } - ``` +1. **Get Hourly Deployment Aggregations (Default - Last 24 hours):** + + * **cURL:** + ```bash + curl -X POST http://localhost:8000/graphql \ + -H "Content-Type: application/json" \ + -d '{ + "query": "{ deploymentAggregations { timeBucket subgraph gatewayId queryCount avgResponseTimeMs } }" + }' + ``` + * **GraphQL:** + ```graphql + { + deploymentAggregations { + timeBucket + subgraph + gatewayId + queryCount + avgResponseTimeMs + } + } + ``` + +2. **Get Daily Indexer Aggregations for the Last 7 Days:** + + * **cURL:** + ```bash + curl -X POST http://localhost:8000/graphql \ + -H "Content-Type: application/json" \ + -d '{ + "query": "{ indexerAggregations( interval: Daily, timeRange: { from: \"$(date -v-7d +%s)\" } ) { timeBucket indexer gatewayId queryCount avgFeeGrt avgSecondsBehind } }" + }' + ``` + * **GraphQL:** + ```graphql + { + indexerAggregations( + interval: Daily, + timeRange: { from: "PAST_7_DAYS_TIMESTAMP" } # Replace with actual timestamp + ) { + timeBucket + indexer + gatewayId + queryCount + avgFeeGrt + avgSecondsBehind + } + } + ``` + +3. **Get Top 50 Allocations by Query Count (5-Minute Interval, Since Specific Time):** + + * **cURL:** + ```bash + curl -X POST http://localhost:8000/graphql \ + -H "Content-Type: application/json" \ + -d '{ + "query": "{ allocationAggregations( interval: FiveMinutes, timeRange: { from: \"1698300000\" }, sort: { field: QueryCount, direction: Desc }, limit: 50 ) { timeBucket subgraph indexer gatewayId queryCount avgSecondsBehind } }" + }' + ``` + * **GraphQL:** + ```graphql + { + allocationAggregations( + interval: FiveMinutes, + timeRange: { from: "1698300000" }, + sort: { field: QueryCount, direction: Desc }, + limit: 50 + ) { + timeBucket + subgraph + indexer + gatewayId + queryCount + avgSecondsBehind + } + } + ``` + +4. **Get Hourly Deployment Aggregations Up To a Specific Time for a Gateway:** + + * **cURL:** + ```bash + curl -X POST http://localhost:8000/graphql \ + -H "Content-Type: application/json" \ + -d '{ + "query": "query GatewayUntil($gwId: String!) { deploymentAggregations( timeRange: { to: \"2023-10-27T12:00:00Z\" }, filter: { gatewayId: $gwId } ) { timeBucket subgraph gatewayId queryCount avgResponseTimeMs } }", + "variables": { "gwId": "gateway-mainnet-1" } + }' + ``` + * **GraphQL (with variables):** + ```graphql + query GatewayUntil($gwId: String!) { + deploymentAggregations( + timeRange: { to: "2023-10-27T12:00:00Z" }, + filter: { gatewayId: $gwId } + ) { + timeBucket + subgraph + gatewayId + queryCount + avgResponseTimeMs + } + } + ``` + *Variables:* + ```json + { + "gwId": "gateway-mainnet-1" + } + ``` ## Development +This section describes the local development environment setup. + ### Project Structure -- `oracle/clickhouse/` - ClickHouse configuration and schema files +- `oracle/clickhouse/` - ClickHouse configuration, schema (`schema.proto`), init template (`init-db.sql.template`), and entrypoint (`entrypoint.sh`). - `oracle/graphql-api/` - GraphQL API service (Rust) -- `oracle/test-producer/` - Test data generator for development (Rust) -- `oracle/docker-compose.yml` - Docker Compose configuration +- `oracle/test-producer/` - **Development only:** Test data generator (Rust) +- `oracle/docker-compose.yml` - **Development only:** Docker Compose configuration +- `oracle/prometheus/` - **Development only:** Prometheus configuration +- `oracle/grafana/` - **Development only:** Grafana provisioning and dashboard definitions ### Running in Development Mode -```bash +``` # From the root qos-oracle directory docker compose -f oracle/docker-compose.yml --profile dev up ``` *(Add `-d` to run detached)* -This will start all services including the test producer, which generates synthetic QoS data into the Redpanda topic consumed by ClickHouse. +This will start the core services (`clickhouse-server`, `graphql-api`), dependencies (`redpanda`), and the `test-producer` using the configuration in `oracle/docker-compose.yml`. + +### Local Monitoring (Development Only) -### Monitoring +When running with `--profile all` or `--profile deps`: - **GraphQL API Health:** `http://localhost:8000/health` - **GraphQL API Endpoint:** `http://localhost:8000/graphql` -- **ClickHouse HTTP Interface:** `http://localhost:8123` (Credentials: `graphql`/`graphql_password` by default) +- **ClickHouse HTTP Interface:** `http://localhost:8123` (Credentials: `graphql`/`graphql_password` by default, see `oracle/docker-compose.yml` `x-clickhouse-env`) - **Redpanda Kafka API:** `localhost:9092` -- **Redpanda Prometheus Metrics:** `http://localhost:9644/metrics` - *(Note: Redpanda Console is not included in the default compose file)* +- **Prometheus UI:** `http://localhost:9090` +- **Grafana UI:** `http://localhost:3000` +- **Redpanda Console:** `http://localhost:8080` ## License @@ -253,4 +345,4 @@ This will start all services including the test producer, which generates synthe ## Acknowledgements -This project is maintained by [GraphOps](https://graphops.xyz) and is part of The Graph Network ecosystem. \ No newline at end of file +This project is maintained by [GraphOps](https://graphops.xyz) and is part of The Graph Network ecosystem.