From fe5fd52f1d9612c3a04b5d369d36049e374bc529 Mon Sep 17 00:00:00 2001 From: IFFranciscoME Date: Tue, 14 Oct 2025 00:00:04 -0600 Subject: [PATCH 1/2] refactor for research --- CHANGELOG.md | 18 - CONTRIBUTE.md | 21 +- Cargo.toml | 32 +- README.md | 44 +- atelier-data/Cargo.toml | 38 +- atelier-data/README.md | 4 + .../src/clients/grpc/grpc_client.rs | 0 atelier-data/src/clients/grpc/mod.rs | 1 + atelier-data/src/clients/http/base_methods.rs | 17 - atelier-data/src/clients/http/get_methods.rs | 1 - atelier-data/src/clients/http/http_client.rs | 600 ++++++++++++++++-- atelier-data/src/clients/http/mod.rs | 4 - atelier-data/src/clients/http/post_methods.rs | 1 - atelier-data/src/clients/mod.rs | 1 + atelier-data/src/clients/wss/decoder.rs | 2 +- atelier-data/src/clients/wss/mod.rs | 1 - atelier-data/src/clients/wss/wss_client.rs | 4 +- atelier-data/src/config/mod.rs | 2 - atelier-data/src/errors.rs | 36 -- atelier-data/src/exchanges/binance/README.md | 1 + .../src/exchanges/binance/binance_client.rs | 214 +++++++ atelier-data/src/exchanges/binance/mod.rs | 1 + .../src/exchanges/binance/resposes/mod.rs | 4 + .../exchanges/binance/resposes/orderbook.rs | 0 .../src/exchanges/binance/resposes/ticker.rs | 29 + .../src/exchanges/binance/resposes/trades.rs | 0 atelier-data/src/exchanges/bitso/README.md | 1 + .../src/exchanges/bitso/bitso_client.rs | 0 atelier-data/src/exchanges/bitso/mod.rs | 0 .../src/exchanges/bitso/responses/mod.rs | 0 .../exchanges/bitso/responses/orderbook.rs | 0 .../src/exchanges/bitso/responses/trades.rs | 0 atelier-data/src/exchanges/bybit/README.md | 1 + .../src/exchanges/bybit/bybit_client.rs | 336 ++++++++++ .../src/exchanges/bybit/bybit_wss_client.rs | 2 +- .../bybit/configs}/connections.toml | 0 .../bybit/configs}/liquidations.toml | 0 atelier-data/src/exchanges/bybit/mod.rs | 7 +- .../src/exchanges/bybit/responses/mod.rs | 3 + .../exchanges/bybit/responses/orderbook.rs | 35 + .../src/exchanges/bybit/ws_decoder.rs | 2 +- atelier-data/src/exchanges/coinbase/README.md | 3 + .../src/exchanges/coinbase/coinbase_client.rs | 196 ++++++ atelier-data/src/exchanges/coinbase/mod.rs | 2 + .../src/exchanges/coinbase/responses/mod.rs | 4 + .../exchanges/coinbase/responses/orderbook.rs | 24 + .../exchanges/coinbase/responses/product.rs | 38 ++ .../exchanges/coinbase/responses/trades.rs | 23 + .../{config/loader.rs => exchanges/config.rs} | 1 + atelier-data/src/exchanges/errors.rs | 78 +++ atelier-data/src/exchanges/gate/README.md | 1 + .../src/exchanges/gate/gate_client.rs | 0 atelier-data/src/exchanges/gate/mod.rs | 0 .../src/exchanges/gate/responses/mod.rs | 0 .../src/exchanges/gate/responses/orderbook.rs | 0 .../src/exchanges/gate/responses/trades.rs | 0 atelier-data/src/exchanges/gemini/README.md | 1 + .../src/exchanges/gemini/gemini_client.rs | 0 atelier-data/src/exchanges/gemini/mod.rs | 0 .../src/exchanges/gemini/responses/mod.rs | 0 .../exchanges/gemini/responses/orderbook.rs | 0 .../src/exchanges/gemini/responses/trades.rs | 0 atelier-data/src/exchanges/kraken/README.md | 1 + .../src/exchanges/kraken/kraken_client.rs | 354 +++++++++++ atelier-data/src/exchanges/kraken/mod.rs | 0 .../src/exchanges/kraken/responses/mod.rs | 0 .../exchanges/kraken/responses/orderbook.rs | 0 .../src/exchanges/kraken/responses/trades.rs | 0 atelier-data/src/exchanges/mod.rs | 17 + atelier-data/src/exchanges/okx/README.md | 1 + atelier-data/src/exchanges/okx/mod.rs | 0 atelier-data/src/exchanges/okx/okx_client.rs | 0 .../src/exchanges/okx/responses/mod.rs | 0 .../src/exchanges/okx/responses/orderbook.rs | 0 .../src/exchanges/okx/responses/trades.rs | 0 atelier-data/src/lib.rs | 9 +- atelier-data/src/models/exchanges.rs | 11 + atelier-data/src/models/mod.rs | 3 + atelier-data/src/models/orderbook.rs | 286 +++++++++ atelier-data/src/models/pairs.rs | 77 +++ atelier-data/src/protocols/mod.rs | 0 atelier-data/src/protocols/solana/mod.rs | 0 .../test/exchanges/binance/test_orderbook.rs | 0 .../test/exchanges/binance/test_trades.rs | 0 .../test/exchanges/bitso/test_orderbook.rs | 0 .../test/exchanges/bitso/test_trades.rs | 0 .../bybit/test_liquidations.rs | 0 .../test/exchanges/bybit/test_orderbook.rs | 0 .../test/exchanges/bybit/test_trades.rs | 0 .../test/exchanges/coinbase/test_orderbook.rs | 30 + .../test/exchanges/coinbase/test_products.rs | 28 + .../test/exchanges/coinbase/test_trades.rs | 0 .../{ => exchanges}/configs/template.toml | 0 .../{ => exchanges}/configs/test_from_toml.rs | 0 .../test/exchanges/gate/test_orderbook.rs | 0 .../test/exchanges/gate/test_trades.rs | 0 .../test/exchanges/gemini/test_orderbook.rs | 0 .../test/exchanges/gemini/test_trades.rs | 0 .../test/exchanges/kraken/test_orderbook.rs | 0 .../test/exchanges/kraken/test_trades.rs | 0 .../test/exchanges/okx/test_orderbook.rs | 0 .../test/exchanges/okx/test_trades.rs | 0 atelier-rs/README.md | 24 +- atelier-rs/src/lib.rs | 23 +- atelier-synth/README.md | 16 +- atelier-synth/src/errors.rs | 71 +-- atelier-synth/src/synthbooks.rs | 4 +- benches/Cargo.toml | 2 +- datasets/README.md | 38 ++ examples/README.md | 39 +- examples/distributed/case_1/README.md | 0 .../config.toml} | 0 .../files}/data/singular_case_data.csv | 0 .../files}/data/singular_case_ob.json | 0 .../singular_data.rs => case_1/synth_data.rs} | 0 .../singular_training.rs => case_1/train.rs} | 0 examples/distributed/case_3/README.md | 4 +- .../files/{ => data}/case_3_ai_00_data.csv | 0 .../files/{ => data}/case_3_ai_00_ob.json | 0 .../files/{ => data}/case_3_am_00_data.csv | 0 .../files/{ => data}/case_3_am_00_ob.json | 0 .../files/{ => data}/case_3_eu_00_data.csv | 0 .../files/{ => data}/case_3_eu_00_ob.json | 0 examples/distributed/case_3/train.rs | 7 +- examples/distributed/case_9/README.md | 0 .../case_9/{config_c.toml => config.toml} | 0 examples/distributed/case_9/synth_data.rs | 0 .../case_9/{topology_c.toml => topology.toml} | 0 .../templates/test_temp_train_00.toml | 1 - examples/quickstart/basic_ob_progressions.rs | 74 --- examples/{advanced => synthetize}/README.md | 0 examples/synthetize/generators/custom.rs | 0 .../synthetize/ob_progressions.rs} | 0 research/ADA2025/README.md | 20 + research/CFE2025/README.md | 5 + research/README.md | 0 136 files changed, 2570 insertions(+), 409 deletions(-) delete mode 100644 CHANGELOG.md rename examples/advanced/generators/custom.rs => atelier-data/src/clients/grpc/grpc_client.rs (100%) create mode 100644 atelier-data/src/clients/grpc/mod.rs delete mode 100644 atelier-data/src/clients/http/base_methods.rs delete mode 100644 atelier-data/src/clients/http/get_methods.rs delete mode 100644 atelier-data/src/clients/http/post_methods.rs delete mode 100644 atelier-data/src/config/mod.rs delete mode 100644 atelier-data/src/errors.rs create mode 100644 atelier-data/src/exchanges/binance/README.md create mode 100644 atelier-data/src/exchanges/binance/binance_client.rs create mode 100644 atelier-data/src/exchanges/binance/mod.rs create mode 100644 atelier-data/src/exchanges/binance/resposes/mod.rs rename examples/distributed/case_9/example_c.rs => atelier-data/src/exchanges/binance/resposes/orderbook.rs (100%) create mode 100644 atelier-data/src/exchanges/binance/resposes/ticker.rs create mode 100644 atelier-data/src/exchanges/binance/resposes/trades.rs create mode 100644 atelier-data/src/exchanges/bitso/README.md create mode 100644 atelier-data/src/exchanges/bitso/bitso_client.rs create mode 100644 atelier-data/src/exchanges/bitso/mod.rs create mode 100644 atelier-data/src/exchanges/bitso/responses/mod.rs create mode 100644 atelier-data/src/exchanges/bitso/responses/orderbook.rs create mode 100644 atelier-data/src/exchanges/bitso/responses/trades.rs create mode 100644 atelier-data/src/exchanges/bybit/README.md create mode 100644 atelier-data/src/exchanges/bybit/bybit_client.rs rename atelier-data/src/{config/bybit => exchanges/bybit/configs}/connections.toml (100%) rename atelier-data/src/{config/bybit => exchanges/bybit/configs}/liquidations.toml (100%) create mode 100644 atelier-data/src/exchanges/bybit/responses/orderbook.rs create mode 100644 atelier-data/src/exchanges/coinbase/README.md create mode 100644 atelier-data/src/exchanges/coinbase/coinbase_client.rs create mode 100644 atelier-data/src/exchanges/coinbase/mod.rs create mode 100644 atelier-data/src/exchanges/coinbase/responses/mod.rs create mode 100644 atelier-data/src/exchanges/coinbase/responses/orderbook.rs create mode 100644 atelier-data/src/exchanges/coinbase/responses/product.rs create mode 100644 atelier-data/src/exchanges/coinbase/responses/trades.rs rename atelier-data/src/{config/loader.rs => exchanges/config.rs} (99%) create mode 100644 atelier-data/src/exchanges/errors.rs create mode 100644 atelier-data/src/exchanges/gate/README.md create mode 100644 atelier-data/src/exchanges/gate/gate_client.rs create mode 100644 atelier-data/src/exchanges/gate/mod.rs create mode 100644 atelier-data/src/exchanges/gate/responses/mod.rs create mode 100644 atelier-data/src/exchanges/gate/responses/orderbook.rs create mode 100644 atelier-data/src/exchanges/gate/responses/trades.rs create mode 100644 atelier-data/src/exchanges/gemini/README.md create mode 100644 atelier-data/src/exchanges/gemini/gemini_client.rs create mode 100644 atelier-data/src/exchanges/gemini/mod.rs create mode 100644 atelier-data/src/exchanges/gemini/responses/mod.rs create mode 100644 atelier-data/src/exchanges/gemini/responses/orderbook.rs create mode 100644 atelier-data/src/exchanges/gemini/responses/trades.rs create mode 100644 atelier-data/src/exchanges/kraken/README.md create mode 100644 atelier-data/src/exchanges/kraken/kraken_client.rs create mode 100644 atelier-data/src/exchanges/kraken/mod.rs create mode 100644 atelier-data/src/exchanges/kraken/responses/mod.rs create mode 100644 atelier-data/src/exchanges/kraken/responses/orderbook.rs create mode 100644 atelier-data/src/exchanges/kraken/responses/trades.rs create mode 100644 atelier-data/src/exchanges/okx/README.md create mode 100644 atelier-data/src/exchanges/okx/mod.rs create mode 100644 atelier-data/src/exchanges/okx/okx_client.rs create mode 100644 atelier-data/src/exchanges/okx/responses/mod.rs create mode 100644 atelier-data/src/exchanges/okx/responses/orderbook.rs create mode 100644 atelier-data/src/exchanges/okx/responses/trades.rs create mode 100644 atelier-data/src/models/exchanges.rs create mode 100644 atelier-data/src/models/mod.rs create mode 100644 atelier-data/src/models/orderbook.rs create mode 100644 atelier-data/src/models/pairs.rs create mode 100644 atelier-data/src/protocols/mod.rs create mode 100644 atelier-data/src/protocols/solana/mod.rs create mode 100644 atelier-data/test/exchanges/binance/test_orderbook.rs create mode 100644 atelier-data/test/exchanges/binance/test_trades.rs create mode 100644 atelier-data/test/exchanges/bitso/test_orderbook.rs create mode 100644 atelier-data/test/exchanges/bitso/test_trades.rs rename atelier-data/test/{ => exchanges}/bybit/test_liquidations.rs (100%) create mode 100644 atelier-data/test/exchanges/bybit/test_orderbook.rs create mode 100644 atelier-data/test/exchanges/bybit/test_trades.rs create mode 100644 atelier-data/test/exchanges/coinbase/test_orderbook.rs create mode 100644 atelier-data/test/exchanges/coinbase/test_products.rs create mode 100644 atelier-data/test/exchanges/coinbase/test_trades.rs rename atelier-data/test/{ => exchanges}/configs/template.toml (100%) rename atelier-data/test/{ => exchanges}/configs/test_from_toml.rs (100%) create mode 100644 atelier-data/test/exchanges/gate/test_orderbook.rs create mode 100644 atelier-data/test/exchanges/gate/test_trades.rs create mode 100644 atelier-data/test/exchanges/gemini/test_orderbook.rs create mode 100644 atelier-data/test/exchanges/gemini/test_trades.rs create mode 100644 atelier-data/test/exchanges/kraken/test_orderbook.rs create mode 100644 atelier-data/test/exchanges/kraken/test_trades.rs create mode 100644 atelier-data/test/exchanges/okx/test_orderbook.rs create mode 100644 atelier-data/test/exchanges/okx/test_trades.rs create mode 100644 datasets/README.md create mode 100644 examples/distributed/case_1/README.md rename examples/distributed/{singular_case/singular_config.toml => case_1/config.toml} (100%) rename examples/distributed/{singular_case => case_1/files}/data/singular_case_data.csv (100%) rename examples/distributed/{singular_case => case_1/files}/data/singular_case_ob.json (100%) rename examples/distributed/{singular_case/singular_data.rs => case_1/synth_data.rs} (100%) rename examples/distributed/{singular_case/singular_training.rs => case_1/train.rs} (100%) rename examples/distributed/case_3/files/{ => data}/case_3_ai_00_data.csv (100%) rename examples/distributed/case_3/files/{ => data}/case_3_ai_00_ob.json (100%) rename examples/distributed/case_3/files/{ => data}/case_3_am_00_data.csv (100%) rename examples/distributed/case_3/files/{ => data}/case_3_am_00_ob.json (100%) rename examples/distributed/case_3/files/{ => data}/case_3_eu_00_data.csv (100%) rename examples/distributed/case_3/files/{ => data}/case_3_eu_00_ob.json (100%) create mode 100644 examples/distributed/case_9/README.md rename examples/distributed/case_9/{config_c.toml => config.toml} (100%) create mode 100644 examples/distributed/case_9/synth_data.rs rename examples/distributed/case_9/{topology_c.toml => topology.toml} (100%) delete mode 100644 examples/quickstart/basic_ob_progressions.rs rename examples/{advanced => synthetize}/README.md (100%) create mode 100644 examples/synthetize/generators/custom.rs rename examples/{advanced/synthetize/synthetize_ob.rs => synthetize/synthetize/ob_progressions.rs} (100%) create mode 100644 research/ADA2025/README.md create mode 100644 research/CFE2025/README.md create mode 100644 research/README.md diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index ae6aab0..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,18 +0,0 @@ -# v0.1.0 - -## Fixed - -- This was fixed - -## Added - -- This was added - -## Changed - -- This was changed - -## Documented - -- This was documented - diff --git a/CONTRIBUTE.md b/CONTRIBUTE.md index d292827..bfeab01 100644 --- a/CONTRIBUTE.md +++ b/CONTRIBUTE.md @@ -3,9 +3,9 @@ Thanks for the interest in contributing to this awesome project !, in order to get you up to speed in terms of sharing your contributed work, or intentions, consider the following: 1. compile. -2. no clippy errors. +2. no clippy warnings/errors. 3. tests runs. -4. code formats its. +4. code is formatted. ## Lints @@ -14,35 +14,32 @@ This project, for the stable version, has implemented the following lints: ```toml [workspace.lints.rust] unsafe_code = "forbid" -unused_extern_crates = "forbid" -unreachable_code = "forbid" -unreachable_patterns = "forbid" +unused_extern_crates = "allow" +unreachable_code = "deny" +unreachable_patterns = "deny" unused_variables = "warn" trivial_casts = "warn" trivial_numeric_casts = "warn" unexpected_cfgs = { level = "warn", check-cfg = ['cfg(nightly)'] } +dead_code = "allow" +too_many_arguments = "allow" ``` ## rustfmt.toml -Consider all defaults to be present, and, the following changed: +For the `atelier-rs` crate, there is a `.rustfmt.toml` config file, even though most of the values are exactly the same as the default, they were included for future-proof purposes in terms of formatting. Consider all defaults to be present, and, the following changed: ```toml reorder_modules = true max_width = 90 ``` -## Code format with rustfmt - -For the `atelier-rs` crate, there is a `.rustfmt.toml` config file, even though must of the values are exactly the same as the default, they were included for future-proof purposes in terms of formatting. - ## Reporting a Bug In the case that you've found a bug, please make sure you are able to answer the following: ``` -- What version of Rust are you using? -- What version of the crate are you using? +- What version of Rust and the atelier related crate are you using? - What operating system are you using? - What did you do? - What did you expect to see? diff --git a/Cargo.toml b/Cargo.toml index a59e363..649606f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,6 @@ default-members = [ "atelier-base", "atelier-data", "atelier-rs", - "atelier-synth", ] [workspace.package] @@ -27,43 +26,40 @@ rust-version = "1.85.0" edition = "2024" description = "Rust Engine for High Frequency, Synthetic and Historical, Market Microstructure Modeling." -authors = ["IteraLabs.xyz"] +authors = ["iteraLabs.xyz"] documentation = "https://docs.rs/atelier-rs/" repository = "https://github.com/iteralabs/atelier-rs" homepage = "https://iteralabs.xyz/atelier-rs" - keywords = ["machine-learning", "framework", "math", "crypto", "trading"] categories = ["data-structures", "development-tools", "finance", "simulation"] include = ["katex-header.html"] exclude = ["assets/*", ".github", "Makefile.toml", "*.log", "tags"] - license = "Apache-2.0" [workspace.dependencies] -atelier_data = { path = "./atelier-data", version = "0.0.10" } -atelier_base = { path = "./atelier-base", version = "0.0.12" } -atelier_dcml = { path = "./atelier-dcml", version = "0.0.11" } -atelier_quant = { path = "./atelier-quant", version = "0.0.10" } -atelier_rs = { path = "./atelier-rs", version = "0.0.10" } -atelier_synth = { path = "./atelier-synth", version = "0.0.10" } +atelier_data = { version = "0.0.10" } +atelier_base = { version = "0.0.12" } +atelier_dcml = { version = "0.0.11" } +atelier_quant = { version = "0.0.10" } +atelier_rs = { version = "0.0.10" } +atelier_synth = { version = "0.0.10" } -thiserror = { version = "1.0.64" } -rand = { version = "0.9.0" } -rand_distr = { version = "0.5.0" } criterion = { version = "0.5", features = ["html_reports"] } -memuse = { version = "0.2.0" } -human_bytes = { version = "0.4.1" } -toml = { version = "0.8" } csv = { version = "1.3" } clap = { version = "4.5", features = ["derive"] } -tokio = { version = "1", features = ["full"] } futures = { version = "0.3" } +human_bytes = { version = "0.4.1" } +memuse = { version = "0.2.0" } reqwest = { version = "0.12", features = ["json"] } rust_decimal = { version = "1.34", features = ["serde"] } - +rand = { version = "0.9.0" } +rand_distr = { version = "0.5.0" } serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0" } +thiserror = { version = "1.0.64" } +toml = { version = "0.8" } +tokio = { version = "1", features = ["full"] } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/README.md b/README.md index d72fa6a..6a4f241 100644 --- a/README.md +++ b/README.md @@ -31,25 +31,36 @@ # Overview -At a high level it provides the following major components: A full orderbook granularity, stochastic process and functions for synthetic data generation, Distributed convex methods for model training/inference. +Atelier-rs is an engine for high frequency, synthetic and historical, market microstructure modeling. Includes functionality for centralized, decentralized exchanges, and also blockchain protocols. At a high level it provides the following major components: -### Full Limit Order Book +- Full Order book granularity (Sides -> Levels -> Orders). +- Stochastic process functions for synthetic data generation. +- Distributed convex methods for model fitting/training/inference. +- Graph structures, spectral analysis and tools. + +## Full Limit Order Book From the standard representation **\(level_price, level_volume\)** Levels, to a by the level order-queue granularity **\(level_orders \[ \(order_id, order_price, order_amount\), \(order_id, order_price, order_amount\), ... \] \)** to provide a true order-driven market representation structure for enriched models. -### Stochastic Process +## Stochastic Processes and Quantitative Models + +Stochastic process generators for rich/complex simulations, implementations include: Uniform, Normal, Exponential, Poisson. -Stochastic process generators for rich/complex simulations, implementations include: Uniform, Brownian, Hawkes. +Quantitative models from classical and modern literature: Brownian motion, Volume-synchronized Probability of Informed Trading (VPIN), self-exciting point process (Hawkes). -e## Distributed Convex Methods +## Distributed Convex Methods Distributed Convex Methods for Linear Models Training and inference. Implementations include: Undirected Acyclic Compute Graph with Gradient Consensus. +## Graph Theory and Algorithms + +Graph fundamentals, Directed Acyclic Graph (static, and, time-varying), *LaPlacian* spectral analysis. + ### Docs - **[Complete API Documentation](https://docs.rs/atelier-rs)** - All crates in one place -# Usage +# Generic usage ## Local clone @@ -76,7 +87,7 @@ docker build \ --no-cache . ``` -the `builder` stage, to compile the rust binary, and the `runner` stage to have a +The `builder` stage, to compile the rust binary, and the `runner` stage to have a minimalistic container to expose a service provided by the binary execution. Generating results by running the containerized atelier. @@ -93,15 +104,24 @@ docker run \ These are the other published crates members of the workspace: -- [atelier-base](https://crates.io/crates/atelier-base): Core data structures and I/O tools. -- [atelier-dcml](https://crates.io/crates/atelier-dcml): Distributed Convex Machine Learning. +- [atelier-base](https://crates.io/crates/atelier-base): Core data structures for the atelier-rs engine. +- [atelier-data](https://crates.io/crates/atelier-data): Data I/O integrations for OnChain/OffChain Protocols for the atelier-rs engine. +- [atelier-dcml](https://crates.io/crates/atelier-dcml): Distributed Convex Machine Learning methods and tooling for the atelier-rs engine. - [atelier-synth](https://crates.io/crates/atelier-synth): Synthetic Data Generation for the atelier-rs engine. +- [atelier-graph](https://crates.io/crates/atelier-graph): Graph theory algorithms and tooling. +- [atelier-quant](https://crates.io/crates/atelier-quant): Intermediate to advance quantitative finance methods for the atelier-rs engine. Github hosted: -- [benches](https://github.com/IteraLabs/atelier-rs/tree/main/benches) -- [examples](https://github.com/IteraLabs/atelier-rs/tree/main/examples) +- [benches](https://github.com/IteraLabs/atelier-rs/tree/main/benches) : Official and Community benchmarks for the atelier-rs engine and/or individual/grouped sub-components of the engine. +- [examples](https://github.com/IteraLabs/atelier-rs/tree/main/examples): Official and Community gallery of implementations, and examples for the atelier-rs engine and/or individual/grouped sub-components of the engine. +- [research](https://github.com/IteraLabs/atelier-rs/tree/main/research): Official and community research with usage of the atelier-rs and/or individual/grouped sub-components. + +# Contribute + +If you would like to contribute, please consider reading the [CONTRIBUTE.md](https://github.com/IteraLabs/atelier-rs/blob/main/CONTRIBUTE.md) guide. # License -This project is licensed under the Apache V2 license. Any contribution intentionally submitted for inclussion in Atelier by you, shall be licensed as Apache V2, without any additional terms or conditions. +This project is licensed under the Apache V2 license. Any contribution intentionally submitted for inclussion in Atelier by you, shall be licensed as Apache V2, without any additional terms or conditions. + diff --git a/atelier-data/Cargo.toml b/atelier-data/Cargo.toml index 8cd13dd..09a7bcc 100644 --- a/atelier-data/Cargo.toml +++ b/atelier-data/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "atelier_data" -description = "Centralized Exchanges REST/WSS integrations for the atelier-rs engine" +description = "Data I/O integrations for OnChain/OffChain Protocols for the atelier-rs engine" publish = true readme = "README.md" @@ -27,10 +27,6 @@ license = "Apache-2.0" [package.metadata.docs.rs] private-doc = true -# [[bin]] -# name = "liquidations" -# path = "src/bin/bybit_wss.rs" - [lib] name = "atelier_data" path = "src/lib.rs" @@ -39,34 +35,34 @@ path = "src/lib.rs" atelier_base = { path = "../atelier-base", version = "0.0.11" } anyhow = { version = "1.0" } +async-rate-limiter = { version = "1.0", features = ["rt-tokio"] } +async-trait = { version = "0.1" } config = "0.13" +chrono = { version = "0.4", features = ["serde"] } +futures = "0.3" futures-util = "0.3" hex = { version = "0.4" } hmac = { version = "0.12" } -reqwest = { workspace = true } -rust_decimal = { workspace = true } +reqwest = { version = "0.12", features = ["json"] } +rust_decimal = { version = "1.34", features = ["serde"] } sha2 = { version = "0.10" } serde = { version = "1.0", features = ["derive"] } -serde_json = { workspace = true } -thiserror = { workspace = true } -tokio = { workspace = true } +serde_json = { version = "1.0" } +thiserror = { version = "1.0.64" } +tokio = { version = "1.0", features = ["full"] } tokio-tungstenite = { version = "0.21", features = ["native-tls"] } -toml = { workspace = true } -async-rate-limiter = { version = "1.0", features = ["rt-tokio"] } -async-trait = { version = "0.1" } -url = { version = "2.0" } +tonic = { version = "0.10" } +toml = { version = "0.8" } tracing = { version = "0.1" } tracing-subscriber = { version = "0.3" } +url = { version = "2.0" } uuid = { version = "1.0", features = ["v4"] } -chrono = { version = "0.4", features = ["serde"] } - -[[test]] -name = "test_config_from_toml" -path = "test/configs/test_from_toml.rs" +yellowstone-grpc-client = { version = "1.13.0" } +yellowstone-grpc-proto = { version = "1.13.0" } [[test]] -name = "test_liquidations" -path = "test/bybit/test_liquidations.rs" +name = "coinbase_http_orderbook" +path = "test/exchanges/coinbase/test_orderbook.rs" [[example]] name = "bybit_streams" diff --git a/atelier-data/README.md b/atelier-data/README.md index 6c79921..4abe2f6 100644 --- a/atelier-data/README.md +++ b/atelier-data/README.md @@ -4,6 +4,10 @@ Data feeds light weighted connectivity +- Binance +- Coinbase +- Kraken + # Workspace These are the other published crates members of the workspace: diff --git a/examples/advanced/generators/custom.rs b/atelier-data/src/clients/grpc/grpc_client.rs similarity index 100% rename from examples/advanced/generators/custom.rs rename to atelier-data/src/clients/grpc/grpc_client.rs diff --git a/atelier-data/src/clients/grpc/mod.rs b/atelier-data/src/clients/grpc/mod.rs new file mode 100644 index 0000000..348b6e6 --- /dev/null +++ b/atelier-data/src/clients/grpc/mod.rs @@ -0,0 +1 @@ +pub mod grpc_client; diff --git a/atelier-data/src/clients/http/base_methods.rs b/atelier-data/src/clients/http/base_methods.rs deleted file mode 100644 index abdb8f7..0000000 --- a/atelier-data/src/clients/http/base_methods.rs +++ /dev/null @@ -1,17 +0,0 @@ -use crate::clients::http::HttpClient; - -#[derive(Debug)] -pub enum RequestType { - Get, - Post, -} - -impl HttpClient { - // get - // get_with_params - // get_with_headers - - // post - // post_with_params - // post_with_headers -} diff --git a/atelier-data/src/clients/http/get_methods.rs b/atelier-data/src/clients/http/get_methods.rs deleted file mode 100644 index 8b13789..0000000 --- a/atelier-data/src/clients/http/get_methods.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/atelier-data/src/clients/http/http_client.rs b/atelier-data/src/clients/http/http_client.rs index 5d4b562..362d5a2 100644 --- a/atelier-data/src/clients/http/http_client.rs +++ b/atelier-data/src/clients/http/http_client.rs @@ -1,76 +1,576 @@ -use crate::exchanges::Exchange; +use crate::exchanges::errors::{ExchangeError, Result}; use async_rate_limiter::RateLimiter; -use reqwest::Client; -use tokio::time::Duration; -#[derive(Clone)] -pub struct HttpClient { - pub client: Client, - pub exchange: Exchange, - pub rate_limiter: RateLimiter, - pub base_url: String, - pub timeout: Duration, +use reqwest::{ + Client, Response, + header::{HeaderMap, HeaderName, HeaderValue}, +}; +use serde::de::DeserializeOwned; +use std::{ + collections::{BTreeMap, HashMap}, + str::FromStr, + time::Duration, +}; +use tracing::{debug, error, info, warn}; +use url::Url; + +#[derive(Debug)] +pub enum RequestType { + Get, + Post, } +/// HTTP client wrapper with rate limiting and error handling #[derive(Clone)] -pub struct HttpClientBuilder { - client: Option, - exchange: Option, - rate_limiter: Option, - base_url: Option, - timeout: Option, +pub struct HttpClient { + client: Client, + rate_limiter: RateLimiter, + exchange_name: String, + base_url: String, + timeout: Duration, } -impl Default for HttpClientBuilder { - fn default() -> Self { - Self::new() - } -} +impl RetryableHttpClient { + /// Make a POST request with parameters and automatic retries + pub async fn post_with_params_retry( + &self, + endpoint: &str, + params: &[(&str, &str)], + ) -> Result + where + T: DeserializeOwned, + { + let mut last_error = None; + let mut delay = self.retry_config.initial_delay; -impl HttpClientBuilder { - pub fn new() -> Self { - HttpClientBuilder { - client: None, - exchange: None, - rate_limiter: None, - base_url: None, - timeout: None, + for attempt in 0..=self.retry_config.max_retries { + match self.client.post_with_params(endpoint, params).await { + Ok(result) => return Ok(result), + + Err(error) => { + last_error = Some(error); + + if attempt < self.retry_config.max_retries { + info!( + "Request failed (attempt {}/{}), retrying in {:?}", + attempt + 1, + self.retry_config.max_retries + 1, + delay, + ); + + tokio::time::sleep(delay).await; + + // Exponential backoff + delay = std::cmp::min( + Duration::from_millis( + (delay.as_millis() as f64 + * self.retry_config.backoff_factor) + as u64, + ), + self.retry_config.max_delay, + ); + } else { + break; + } + } + } } + + Err(last_error.unwrap_or_else(|| ExchangeError::Unknown)) } - pub fn client(mut self, client: Client) -> Self { - self.client = Some(client); - self + /// Make a POST request with headers and retry + pub async fn post_with_headers_retry( + &self, + endpoint: &str, + params: &[(&str, &str)], + headers: HashMap<&str, &str>, + ) -> Result + where + T: DeserializeOwned, + { + // (This retry logic is copied from get_with_params_retry) + let mut last_error = None; + let mut delay = self.retry_config.initial_delay; + + for attempt in 0..=self.retry_config.max_retries { + // Call the new header-aware function + match self + .client + .post_with_headers(endpoint, params, headers.clone()) + .await + { + Ok(result) => return Ok(result), + Err(error) => { + last_error = Some(error); + + if attempt < self.retry_config.max_retries { + info!( + "Request failed (attempt {}/{}), retrying in {:?}", + attempt + 1, + self.retry_config.max_retries + 1, + delay, + ); + tokio::time::sleep(delay).await; + delay = std::cmp::min( + Duration::from_millis( + (delay.as_millis() as f64 + * self.retry_config.backoff_factor) + as u64, + ), + self.retry_config.max_delay, + ); + } else { + break; + } + } + } + } + Err(last_error.unwrap_or_else(|| ExchangeError::Unknown)) } +} - pub fn exchange(mut self, exchange: Exchange) -> Self { - self.exchange = Some(exchange); - self +// --- POST --- // +impl HttpClient { + /// Make a request with rate limiting + pub async fn post(&self, endpoint: &str) -> Result + where + T: DeserializeOwned, + { + self.post_with_params(endpoint, &[]).await } - pub fn rate_limiter(mut self, rate_limiter: RateLimiter) -> Self { - self.rate_limiter = Some(rate_limiter); - self + /// Make a POST request with query parameters + pub async fn post_with_params( + &self, + endpoint: &str, + params: &[(&str, &str)], + ) -> Result + where + T: DeserializeOwned, + { + // Wait for rate limiter + self.rate_limiter.acquire().await; + + let url = self.build_url(endpoint, params)?; + + debug!("Making POST request to: {}", url); + + let response = self + .client + .post(&url) + .send() + .await + .map_err(ExchangeError::Network)?; + + self.handle_response(response).await } - pub fn timeout(mut self, timeout: Duration) -> Self { - self.timeout = Some(timeout); - self + /// Make a GET request with custom headers + pub async fn post_with_headers( + &self, + endpoint: &str, + params: &[(&str, &str)], + headers: HashMap<&str, &str>, + ) -> Result + where + T: DeserializeOwned, + { + // Wait for rate limiter + self.rate_limiter.acquire().await; + + let url = self.build_url(endpoint, &[])?; + + // Build the header map for reqwest + let mut header_map = HeaderMap::new(); + for (key, value) in headers { + let header_name = + HeaderName::from_str(key).map_err(|_| ExchangeError::Configuration { + message: format!("Invalid header name: {}", key), + })?; + let header_value = HeaderValue::from_str(value).map_err(|_| { + ExchangeError::Configuration { + message: "Invalid header value".to_string(), + } + })?; + header_map.insert(header_name, header_value); + } + + debug!("Making POST request to: {} with custom headers", url); + + let json_body = if params.is_empty() { + serde_json::json!({}) + } else { + let body_map: BTreeMap<_, _> = params.iter().cloned().collect(); + serde_json::to_value(&body_map).unwrap_or_default() + }; + + let response = self + .client + .post(&url) + .headers(header_map) // Attach the headers here + .json(&json_body) + .send() + .await + .map_err(ExchangeError::Network)?; + + self.handle_response(response).await } +} - pub fn build(self) -> Result { - let client = self.client.ok_or("Missing client")?; - let exchange = self.exchange.ok_or("Missing exchange")?; - let rate_limiter = self.rate_limiter.ok_or("Missing rate_limiter")?; - let base_url = self.base_url.ok_or("Missing base_url")?; - let timeout = self.timeout.ok_or("Missing timeout")?; +impl HttpClient { + /// Create a new HTTP client for an exchange + pub fn new( + exchange_name: String, + base_url: String, + requests_per_second: u32, + timeout_seconds: u64, + ) -> Result { + let client = Client::builder() + .timeout(Duration::from_secs(timeout_seconds)) + .user_agent("ix_cex/0.0.1") + .build() + .map_err(ExchangeError::Network)?; - Ok(HttpClient { + let rate_limiter = RateLimiter::new(requests_per_second as usize); + + Ok(Self { client, - exchange, rate_limiter, + exchange_name, base_url, - timeout, + timeout: Duration::from_secs(timeout_seconds), }) } + + pub fn get_timeout(&self) -> Duration { + self.timeout + } + + /// Make a request with rate limiting + pub async fn get(&self, endpoint: &str) -> Result + where + T: DeserializeOwned, + { + self.get_with_params(endpoint, &[]).await + } + + /// Make a GET request with custom headers + pub async fn get_with_headers( + &self, + endpoint: &str, + params: &[(&str, &str)], + headers: HashMap<&str, &str>, + ) -> Result + where + T: DeserializeOwned, + { + // Wait for rate limiter + self.rate_limiter.acquire().await; + + let url = self.build_url(endpoint, params)?; + + // Build the header map for reqwest + let mut header_map = HeaderMap::new(); + for (key, value) in headers { + let header_name = + HeaderName::from_str(key).map_err(|_| ExchangeError::Configuration { + message: format!("Invalid header name: {}", key), + })?; + let header_value = HeaderValue::from_str(value).map_err(|_| { + ExchangeError::Configuration { + message: "Invalid header value".to_string(), + } + })?; + header_map.insert(header_name, header_value); + } + + debug!("Making GET request to: {} with custom headers", url); + + let response = self + .client + .get(&url) + .headers(header_map) // Attach the headers here + .send() + .await + .map_err(ExchangeError::Network)?; + + self.handle_response(response).await + } + + /// Make a GET request with query parameters + pub async fn get_with_params( + &self, + endpoint: &str, + params: &[(&str, &str)], + ) -> Result + where + T: DeserializeOwned, + { + // Wait for rate limiter + self.rate_limiter.acquire().await; + + let url = self.build_url(endpoint, params)?; + + debug!("Making GET request to: {}", url); + + let response = self + .client + .get(&url) + .send() + .await + .map_err(ExchangeError::Network)?; + + self.handle_response(response).await + } + + /// Build full URL with query parameters + fn build_url(&self, endpoint: &str, params: &[(&str, &str)]) -> Result { + let base = if endpoint.starts_with("http") { + endpoint.to_string() + } else { + format!( + "{}/{}", + self.base_url.trim_end_matches('/'), + endpoint.trim_start_matches('/') + ) + }; + + if params.is_empty() { + Ok(base) + } else { + let mut url = + Url::parse(&base).map_err(|e| ExchangeError::Configuration { + message: format!("Invalid URL: {e}"), + })?; + + for (key, value) in params { + url.query_pairs_mut().append_pair(key, value); + } + + Ok(url.to_string()) + } + } + + /// Handle HTTP response and deserialize JSON + async fn handle_response(&self, response: Response) -> Result + where + T: DeserializeOwned, + { + let status = response.status(); + let url = response.url().to_string(); + + debug!("Response status: {} for URL: {}", status, url); + + if status.is_success() { + let text = response.text().await.map_err(ExchangeError::Network)?; + + debug!("Response body length: {} bytes", text.len()); + debug!("\nResponse content: {:?}\n", text); + + serde_json::from_str(&text).map_err(|e| { + error!("JSON parsing error for {}: {}", url, e); + error!("Response body: {}", text); + ExchangeError::JsonParsing(e) + }) + } else { + let error_text = response + .text() + .await + .unwrap_or_else(|_| "Unknown error".to_string()); + + let error = match status.as_u16() { + 429 => { + warn!("Rate limit exceeded for {}", self.exchange_name); + ExchangeError::RateLimit { + exchange: self.exchange_name.clone(), + } + } + 400..=499 => ExchangeError::ApiError { + exchange: self.exchange_name.clone(), + message: format!("Client error ({status}): {error_text}"), + }, + 500..=599 => ExchangeError::ApiError { + exchange: self.exchange_name.clone(), + message: format!("Server error ({status}): {error_text}"), + }, + _ => ExchangeError::ApiError { + exchange: self.exchange_name.clone(), + message: format!("HTTP error ({status}): {error_text}"), + }, + }; + + error!("HTTP error for {}: {:?}", url, error); + Err(error) + } + } +} + +impl HttpClient { + /// Get the exchange name + pub fn exchange_name(&self) -> &str { + &self.exchange_name + } + + /// Get the base URL + pub fn base_url(&self) -> &str { + &self.base_url + } + + /// Manually trigger rate limiting (useful for retries) + pub async fn wait_for_rate_limit(&self) { + self.rate_limiter.acquire().await; + } + + /// Check if rate limiter allows immediate request + pub fn can_make_request(&self) -> bool { + self.rate_limiter.try_acquire().is_ok() + } +} + +/// Retry configuration for HTTP requests +#[derive(Debug, Clone)] +pub struct RetryConfig { + pub max_retries: u32, + pub initial_delay: Duration, + pub max_delay: Duration, + pub backoff_factor: f64, +} + +impl Default for RetryConfig { + fn default() -> Self { + Self { + max_retries: 3, + initial_delay: Duration::from_millis(500), + max_delay: Duration::from_secs(30), + backoff_factor: 2.0, + } + } +} + +/// HTTP client with retry capabilities +#[derive(Clone)] +pub struct RetryableHttpClient { + client: HttpClient, + retry_config: RetryConfig, +} + +impl RetryableHttpClient { + // New + pub fn new(client: HttpClient, retry_config: RetryConfig) -> Self { + Self { + client, + retry_config, + } + } + + pub async fn get_with_headers_retry( + &self, + endpoint: &str, + params: &[(&str, &str)], + headers: HashMap<&str, &str>, + ) -> Result + where + T: DeserializeOwned, + { + // (This retry logic is copied from get_with_params_retry) + let mut last_error = None; + let mut delay = self.retry_config.initial_delay; + + for attempt in 0..=self.retry_config.max_retries { + // Call the new header-aware function + match self + .client + .get_with_headers(endpoint, params, headers.clone()) + .await + { + Ok(result) => return Ok(result), + Err(error) => { + last_error = Some(error); + + if attempt < self.retry_config.max_retries { + info!( + "Request failed (attempt {}/{}), retrying in {:?}", + attempt + 1, + self.retry_config.max_retries + 1, + delay, + ); + tokio::time::sleep(delay).await; + delay = std::cmp::min( + Duration::from_millis( + (delay.as_millis() as f64 + * self.retry_config.backoff_factor) + as u64, + ), + self.retry_config.max_delay, + ); + } else { + break; + } + } + } + } + Err(last_error.unwrap_or_else(|| ExchangeError::Unknown)) + } + + /// Make a GET request with automatic retries + pub async fn get_with_retry(&self, endpoint: &str) -> Result + where + T: DeserializeOwned, + { + self.get_with_params_retry(endpoint, &[]).await + } + + /// Make a GET request with parameters and automatic retries + pub async fn get_with_params_retry( + &self, + endpoint: &str, + params: &[(&str, &str)], + ) -> Result + where + T: DeserializeOwned, + { + let mut last_error = None; + let mut delay = self.retry_config.initial_delay; + + for attempt in 0..=self.retry_config.max_retries { + match self.client.get_with_params(endpoint, params).await { + Ok(result) => return Ok(result), + + Err(error) => { + last_error = Some(error); + + if attempt < self.retry_config.max_retries { + info!( + "Request failed (attempt {}/{}), retrying in {:?}", + attempt + 1, + self.retry_config.max_retries + 1, + delay, + ); + + tokio::time::sleep(delay).await; + + // Exponential backoff + delay = std::cmp::min( + Duration::from_millis( + (delay.as_millis() as f64 + * self.retry_config.backoff_factor) + as u64, + ), + self.retry_config.max_delay, + ); + } else { + break; + } + } + } + } + + Err(last_error.unwrap_or_else(|| ExchangeError::Unknown)) + } + + /// Access the underlying client + pub fn client(&self) -> &HttpClient { + &self.client + } } diff --git a/atelier-data/src/clients/http/mod.rs b/atelier-data/src/clients/http/mod.rs index 56db390..8afac74 100644 --- a/atelier-data/src/clients/http/mod.rs +++ b/atelier-data/src/clients/http/mod.rs @@ -1,6 +1,2 @@ -pub mod base_methods; -pub mod get_methods; pub mod http_client; -pub mod post_methods; - pub use http_client::HttpClient; diff --git a/atelier-data/src/clients/http/post_methods.rs b/atelier-data/src/clients/http/post_methods.rs deleted file mode 100644 index 8b13789..0000000 --- a/atelier-data/src/clients/http/post_methods.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/atelier-data/src/clients/mod.rs b/atelier-data/src/clients/mod.rs index 9d79795..ad963a7 100644 --- a/atelier-data/src/clients/mod.rs +++ b/atelier-data/src/clients/mod.rs @@ -1,2 +1,3 @@ pub mod http; pub mod wss; +pub mod grpc; diff --git a/atelier-data/src/clients/wss/decoder.rs b/atelier-data/src/clients/wss/decoder.rs index 77d1d6e..966b354 100644 --- a/atelier-data/src/clients/wss/decoder.rs +++ b/atelier-data/src/clients/wss/decoder.rs @@ -1,4 +1,4 @@ -use crate::errors::ExchangeError; +use crate::exchanges::errors::ExchangeError; #[async_trait::async_trait] pub trait WssDecoder: Send + Sync + 'static { diff --git a/atelier-data/src/clients/wss/mod.rs b/atelier-data/src/clients/wss/mod.rs index a9da1cd..f481b0e 100644 --- a/atelier-data/src/clients/wss/mod.rs +++ b/atelier-data/src/clients/wss/mod.rs @@ -1,4 +1,3 @@ - pub mod decoder; pub use decoder::WssDecoder; diff --git a/atelier-data/src/clients/wss/wss_client.rs b/atelier-data/src/clients/wss/wss_client.rs index e9911a1..de45c1d 100644 --- a/atelier-data/src/clients/wss/wss_client.rs +++ b/atelier-data/src/clients/wss/wss_client.rs @@ -1,8 +1,8 @@ use crate::clients::wss::decoder::WssDecoder; -use crate::errors::ExchangeError; +use crate::exchanges::errors::ExchangeError; use futures_util::{SinkExt, StreamExt}; use std::sync::Arc; -use tokio::sync::{mpsc, Mutex}; +use tokio::sync::{Mutex, mpsc}; use tokio_tungstenite::{connect_async, tungstenite::protocol::Message}; use tracing::{error, info, warn}; use url::Url; diff --git a/atelier-data/src/config/mod.rs b/atelier-data/src/config/mod.rs deleted file mode 100644 index db18de8..0000000 --- a/atelier-data/src/config/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod loader; -pub use loader::Config; diff --git a/atelier-data/src/errors.rs b/atelier-data/src/errors.rs deleted file mode 100644 index d771894..0000000 --- a/atelier-data/src/errors.rs +++ /dev/null @@ -1,36 +0,0 @@ -use thiserror::Error; - -#[derive(Error, Debug, Clone)] -pub enum WssError { - // Connection Error - #[error("Connection Failed")] - WssConnection, -} - -#[derive(Error, Debug)] -pub enum ExchangeError { - #[error("WebSocket connection error: {0}")] - WebSocketError(Box), - - #[error("URL parsing error: {0}")] - UrlParseError(#[from] url::ParseError), - - #[error("JSON deserialization error: {0}")] - JsonError(#[from] serde_json::Error), - - #[error("Configuration error: {0}")] - ConfigError(#[from] config::ConfigError), - - #[error("Channel send error")] - ChannelSendError, - - #[error("An IO error occurred: {0}")] - IoError(#[from] std::io::Error), -} - -impl From for ExchangeError { - fn from(error: tokio_tungstenite::tungstenite::Error) -> Self { - ExchangeError::WebSocketError(Box::new(error)) - } -} - diff --git a/atelier-data/src/exchanges/binance/README.md b/atelier-data/src/exchanges/binance/README.md new file mode 100644 index 0000000..087ed7d --- /dev/null +++ b/atelier-data/src/exchanges/binance/README.md @@ -0,0 +1 @@ +# References for Binance diff --git a/atelier-data/src/exchanges/binance/binance_client.rs b/atelier-data/src/exchanges/binance/binance_client.rs new file mode 100644 index 0000000..fa27147 --- /dev/null +++ b/atelier-data/src/exchanges/binance/binance_client.rs @@ -0,0 +1,214 @@ +use crate::client::http_client::{HttpClient, RetryConfig, RetryableHttpClient}; +use crate::models::orderbook::{Orderbook, PriceLevel, TradingPair}; +use chrono::Utc; +use ix_results::errors::{ExchangeError, Result}; +use serde::Deserialize; +use std::str::FromStr; +use tracing::{debug, info, warn}; + +/// Binance REST API client +#[derive(Clone)] +pub struct BinanceClient { + client: RetryableHttpClient, +} + +impl BinanceClient { + /// Create a new Binance client + pub fn new() -> Result { + let http_client = HttpClient::new( + "Binance".to_string(), + "https://api.binance.com".to_string(), + 10, // 10 requests per second to stay under limits + 30, // 30 second timeout + )?; + + let retry_client = RetryableHttpClient::new(http_client, RetryConfig::default()); + + Ok(Self { + client: retry_client, + }) + } + + /// Get order book snapshot for a trading pair + pub async fn get_orderbook( + &self, + pair: TradingPair, + depth: Option, + ) -> Result { + let symbol = pair.to_exchange_symbol("binance"); + let depth_str = depth.unwrap_or(1000).to_string(); + + info!( + "Fetching Binance orderbook for {} with depth {}", + symbol, depth_str + ); + + let params = vec![("symbol", symbol.as_str()), ("limit", depth_str.as_str())]; + + let response: BinanceDepthResponse = self + .client + .get_with_params_retry("/api/v3/depth", ¶ms) + .await?; + + debug!( + "Received Binance orderbook with {} bids, {} asks", + response.bids.len(), + response.asks.len() + ); + + self.convert_to_orderbook(response, symbol) + } + + /// Convert Binance response to our OrderBook format + fn convert_to_orderbook( + &self, + response: BinanceDepthResponse, + symbol: String, + ) -> Result { + let mut v_bids = Vec::new(); + let mut v_asks = Vec::new(); + + // Convert bids (should already be sorted from highest to lowest) + for bid_array in response.bids { + if bid_array.len() != 2 { + warn!( + "Invalid bid format from Binance: expected 2 elements, got {}", + bid_array.len() + ); + continue; + } + + let price = + f64::from_str(&bid_array[0]).map_err(|e| ExchangeError::ApiError { + exchange: "Binance".to_string(), + message: format!("Invalid bid price '{}': {}", bid_array[0], e), + })?; + + let quantity = + f64::from_str(&bid_array[1]).map_err(|e| ExchangeError::ApiError { + exchange: "Binance".to_string(), + message: format!("Invalid bid quantity '{}': {}", bid_array[1], e), + })?; + + v_bids.push(PriceLevel { price, quantity }); + } + + // Convert asks (should already be sorted from lowest to highest) + for ask_array in response.asks { + if ask_array.len() != 2 { + warn!( + "Invalid ask format from Binance: expected 2 elements, got {}", + ask_array.len() + ); + continue; + } + + let price = + f64::from_str(&ask_array[0]).map_err(|e| ExchangeError::ApiError { + exchange: "Binance".to_string(), + message: format!("Invalid ask price '{}': {}", ask_array[0], e), + })?; + + let quantity = + f64::from_str(&ask_array[1]).map_err(|e| ExchangeError::ApiError { + exchange: "Binance".to_string(), + message: format!("Invalid ask quantity '{}': {}", ask_array[1], e), + })?; + + v_asks.push(PriceLevel { price, quantity }); + } + + // Final value + let orderbook = Orderbook::new( + symbol, + "Binance".to_string(), + Utc::now(), + v_bids, + v_asks, + Some(response.last_update_id), + None, + ); + + // Validate the orderbook + if !orderbook.is_valid() { + return Err(ExchangeError::ApiError { + exchange: "Binance".to_string(), + message: "Received invalid orderbook data".to_string(), + }); + } + + info!( + "Successfully converted Binance orderbook: {} bids, {} asks, spread: {:?}", + orderbook.bids.len(), + orderbook.asks.len(), + orderbook.spread() + ); + + Ok(orderbook) + } + + /// Get exchange information (available symbols, etc.) + pub async fn get_exchange_info(&self) -> Result { + info!("Fetching Binance exchange information"); + + self.client.get_with_retry("/api/v3/exchangeInfo").await + } + + /// Get server time (useful for synchronization) + pub async fn get_server_time(&self) -> Result { + self.client.get_with_retry("/api/v3/time").await + } + + /// Get 24hr ticker statistics + pub async fn get_24hr_ticker(&self, pair: TradingPair) -> Result { + let symbol = pair.to_exchange_symbol("binance"); + let params = vec![("symbol", symbol.as_str())]; + + self.client + .get_with_params_retry("/api/v3/ticker/24hr", ¶ms) + .await + } +} + +/// Binance depth/orderbook response format +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct BinanceDepthResponse { + last_update_id: u64, + bids: Vec>, // [price, quantity] pairs + asks: Vec>, // [price, quantity] pairs +} + +/// Binance exchange info response +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BinanceExchangeInfo { + pub timezone: String, + pub server_time: u64, + pub symbols: Vec, +} + +/// Binance symbol information +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BinanceSymbol { + pub symbol: String, + pub status: String, + pub base_asset: String, + pub quote_asset: String, + pub base_asset_precision: u32, + pub quote_precision: u32, +} + +/// Binance server time response +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BinanceServerTime { + pub server_time: u64, +} + +impl Default for BinanceClient { + fn default() -> Self { + Self::new().expect("Failed to create default Binance client") + } +} diff --git a/atelier-data/src/exchanges/binance/mod.rs b/atelier-data/src/exchanges/binance/mod.rs new file mode 100644 index 0000000..f38ee2a --- /dev/null +++ b/atelier-data/src/exchanges/binance/mod.rs @@ -0,0 +1 @@ +pub mod binance_client; diff --git a/atelier-data/src/exchanges/binance/resposes/mod.rs b/atelier-data/src/exchanges/binance/resposes/mod.rs new file mode 100644 index 0000000..44b3830 --- /dev/null +++ b/atelier-data/src/exchanges/binance/resposes/mod.rs @@ -0,0 +1,4 @@ +pub mod ticker; +pub mod orderbook; +pub mod trades; + diff --git a/examples/distributed/case_9/example_c.rs b/atelier-data/src/exchanges/binance/resposes/orderbook.rs similarity index 100% rename from examples/distributed/case_9/example_c.rs rename to atelier-data/src/exchanges/binance/resposes/orderbook.rs diff --git a/atelier-data/src/exchanges/binance/resposes/ticker.rs b/atelier-data/src/exchanges/binance/resposes/ticker.rs new file mode 100644 index 0000000..0803534 --- /dev/null +++ b/atelier-data/src/exchanges/binance/resposes/ticker.rs @@ -0,0 +1,29 @@ +use serde; + +/// Binance 24hr ticker response +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Binance24hrTicker { + pub symbol: String, + pub price_change: String, + pub price_change_percent: String, + pub weighted_avg_price: String, + pub prev_close_price: String, + pub last_price: String, + pub last_qty: String, + pub bid_price: String, + pub bid_qty: String, + pub ask_price: String, + pub ask_qty: String, + pub open_price: String, + pub high_price: String, + pub low_price: String, + pub volume: String, + pub quote_volume: String, + pub open_time: u64, + pub close_time: u64, + pub first_id: u64, + pub last_id: u64, + pub count: u64, +} + diff --git a/atelier-data/src/exchanges/binance/resposes/trades.rs b/atelier-data/src/exchanges/binance/resposes/trades.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/bitso/README.md b/atelier-data/src/exchanges/bitso/README.md new file mode 100644 index 0000000..417e39d --- /dev/null +++ b/atelier-data/src/exchanges/bitso/README.md @@ -0,0 +1 @@ +# References for Bitso diff --git a/atelier-data/src/exchanges/bitso/bitso_client.rs b/atelier-data/src/exchanges/bitso/bitso_client.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/bitso/mod.rs b/atelier-data/src/exchanges/bitso/mod.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/bitso/responses/mod.rs b/atelier-data/src/exchanges/bitso/responses/mod.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/bitso/responses/orderbook.rs b/atelier-data/src/exchanges/bitso/responses/orderbook.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/bitso/responses/trades.rs b/atelier-data/src/exchanges/bitso/responses/trades.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/bybit/README.md b/atelier-data/src/exchanges/bybit/README.md new file mode 100644 index 0000000..c121786 --- /dev/null +++ b/atelier-data/src/exchanges/bybit/README.md @@ -0,0 +1 @@ +# References for Bybit diff --git a/atelier-data/src/exchanges/bybit/bybit_client.rs b/atelier-data/src/exchanges/bybit/bybit_client.rs new file mode 100644 index 0000000..962ed33 --- /dev/null +++ b/atelier-data/src/exchanges/bybit/bybit_client.rs @@ -0,0 +1,336 @@ + +use crate::client::http_client::{HttpClient, RetryConfig, RetryableHttpClient}; +use crate::exchanges::bybit::responses; +use crate::models::orderbook::{Orderbook, TradingPair, PriceLevel}; +use chrono::Utc; + +use ix_results::errors::{ExchangeError, Result}; +use serde::Deserialize; +use std::collections::HashMap; +use std::str::FromStr; +use tracing::{debug, info}; + +/// Bybit API Client +#[derive(Clone)] +pub struct BybitClient { + pub client: RetryableHttpClient, +} + +impl BybitClient { + + /// Create a new Bybit client + pub fn new() -> Result { + debug!("Fetching Bybit Get New Client"); + + let exchange_name = "Bybit".to_string(); + let base_url = "https://api.bybit.com".to_string(); + let timeout_secs = 30; + let req_per_sec = 10; + + let http_client = + HttpClient::new(exchange_name, base_url, req_per_sec, timeout_secs)?; + + let retry_client = RetryableHttpClient::new(http_client, RetryConfig::default()); + + Ok(Self { + client: retry_client, + }) + } + + /// Get order book snapshot for a trading pair + pub async fn get_orderbook( + &self, + pair: TradingPair, + depth: Option, + ) -> Result { + let symbol_id = pair.to_exchange_symbol("bybit"); + info!("Fetching Bybit orderbook for {}", symbol_id); + + let mut params: Vec<(&str, String)> = vec![("symbol", symbol_id.clone())]; + + if let Some(depth) = depth { + params.push(("limit", depth.to_string())); + } + + // Convert params to the expected type for get_with_params_retry + let params_ref: Vec<(&str, &str)> = + params.iter().map(|(k, v)| (*k, v.as_str())).collect(); + + println!("params_ref: {:?}", params_ref); + + // let response_debug: responses::BybitOrderbookResponse = self + // .client + // .get_with_params_retry("/v5/market/orderbook?category=spot", ¶ms_ref) + // .await?; + // + // println!("response_debug: {:?}", response_debug); + + let response: responses::BybitOrderbookResponse = self + .client + .get_with_params_retry("/v5/market/orderbook?category=spot", ¶ms_ref) + .await?; + + println!("response: {:?}", response); + + println!( + "Received Bybit orderbook with {} bids, {} asks", + response.result.bids.len(), + response.result.asks.len() + ); + + self.convert_to_orderbook(response, symbol_id) + } + + fn convert_to_orderbook( + &self, + response: responses::BybitOrderbookResponse, + symbol: String, + ) -> Result { + + let mut v_bids = Vec::new(); + let mut v_asks = Vec::new(); + + // --- + for bid in response.result.bids { + + let price = + f64::from_str(&bid.price).map_err(|e| ExchangeError::ApiError { + exchange: "Coinbase".to_string(), + message: format!("Invalid bid price '{}': {}", bid.price, e), + })?; + + let quantity = + f64::from_str(&bid.qty).map_err(|e| ExchangeError::ApiError { + exchange: "Coinbase".to_string(), + message: format!("Invalid bid size '{}': {}", bid.qty, e), + })?; + + v_bids.push(PriceLevel { + price, + quantity, + }); + + } + + // --- + for ask in response.result.asks { + + let price = + f64::from_str(&ask.price).map_err(|e| ExchangeError::ApiError { + exchange: "Coinbase".to_string(), + message: format!("Invalid bid price '{}': {}", ask.price, e), + })?; + + let quantity = + f64::from_str(&ask.qty).map_err(|e| ExchangeError::ApiError { + exchange: "Coinbase".to_string(), + message: format!("Invalid bid size '{}': {}", ask.qty, e), + })?; + + v_asks.push(PriceLevel { + price, + quantity, + }); + + } + + let mut orderbook = Orderbook::new( + symbol, + "Bybit".to_string(), + Utc::now(), + v_bids, + v_asks, + None, + None, + ); + + if !orderbook.is_valid() { + return Err(ExchangeError::ApiError { + exchange: "Bybit".to_string(), + message: "Received invalidad orderbook data".to_string(), + }); + } + + info!( + "Succesfully converted Bybit orderbok: {} bids, {} asks, spread: {:?}", + orderbook.bids.len(), + orderbook.asks.len(), + orderbook.spread() + ); + + orderbook.timestamp = Utc::now(); + + Ok(orderbook) + + } + + /// Get Bybit server time + pub async fn get_server_time(&self) -> Result { + debug!("Fetching Bybit Server Time"); + + let response: BybitServerTimeResponse = + self.client.get_with_retry("/v5/market/time").await?; + + println!("get_server_time.response: {:?}", response); + + if response.ret_code != 0 { + return Err(ExchangeError::ApiError { + exchange: "Bybit".to_string(), + message: format!( + "Bybit API Error\n Code: {:?} Message: {:?} \n + Time {:?} ExtInof {:?}", + response.ret_code, + response.ret_msg, + response.ret_ext_info, + response.time, + ), + }); + } + + Ok(response.result) + + } + + /// Get Account Info + pub async fn get_account_info(&self) -> Result { + + debug!("Fetching Bybit Get Account Info"); + + let response: BybitAccountInfoResponse = + self.client.get_with_retry("/v5/account/info").await?; + + println!("get_account_info.response: {:?}", response); + + if response.ret_code != 0 { + return Err(ExchangeError::ApiError { + exchange: "Bybit".to_string(), + message: format!( + "Bybit API Error\n Code: {:?} Message: {:?}", + response.ret_code, response.ret_msg, + ), + }); + } + Ok(response.result) + } + + /// Get Wallet Balence + pub async fn get_wallet_balance(&self) -> Result { + println!("Fetching Bybit Get Wallet Balance"); + + let p_endpoint = "/v5/wallet-balance".to_string(); + let p_params = [("accountType", "UNIFIED")]; + + let response: BybitWalletBalanceResponse = self + .client + .get_with_params_retry(&p_endpoint, &p_params) + .await?; + + if response.ret_code != 0 { + return Err(ExchangeError::ApiError { + exchange: "Bybit".to_string(), + message: format!( + "Bybit API Error\n Code: {:?} Message: {:?}", + response.ret_code, response.ret_msg, + ), + }); + } + Ok(response.result) + } +} + +/// Bybit server time +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BybitServerTime { + pub time_second: String, + pub time_nano: String, +} + +/// Bybit server time response +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct BybitServerTimeResponse { + ret_code: u64, + ret_msg: String, + ret_ext_info: HashMap, + time: u64, + result: BybitServerTime, +} + +/// Bybit account info +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BybitAccountInfo { + pub margin_mode: String, + pub updated_time: String, + pub unified_margin_status: u64, + pub dcp_status: String, + pub time_window: u64, + pub smp_group: u64, + pub is_master_trader: bool, + pub spot_hedging_status: String, +} + +/// Bybit account info response +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct BybitAccountInfoResponse { + ret_code: u64, + ret_msg: String, + result: BybitAccountInfo, +} + +/// Bybit wallet balance response +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BybitWalletBalance { + pub total_equity: String, // "3.31216591", + pub account_im_rate: String, // "0", + pub account_im_rate_bymp: String, // "0", + pub total_margin_balance: String, // "3.00326056", + pub total_initial_margin: String, // "0", + pub total_initial_margin_bymp: String, // "0", + pub account_type: String, // "UNIFIED", + pub total_available_balance: String, // "3.00326056", + pub account_mm_rate: String, // "0", + pub account_mm_rate_bymp: String, // "0", + pub total_perp_upl: String, // "0", + pub total_wallet_balance: String, // "3.00326056", + pub account_ltv: String, // "0", + pub total_maintenance_margin: String, // "0", + pub total_maintenance_margin_bymp: String, // "0", + pub coin: BybitWalletBalanceCoin, +} + +/// Bybit wallet balance response +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BybitWalletBalanceCoin { + pub available_to_borrow: String, // "3", + pub bonus: String, // "0", + pub accrued_interest: String, // "0", + pub available_to_withdraw: String, // "0", + pub total_order_im: String, // "0", + pub equity: String, // "0", + pub total_position_mm: String, // "0", + pub usd_alue: String, // "0", + pub spot_hedging_qty: String, // "0.01592413", + pub unrealised_pnl: String, // "0", + pub collateral_switch: bool, // true, + pub borrow_amount: String, // "0.0", + pub total_position_im: String, // "0", + pub wallet_balance: String, // "0", + pub cum_realised_pnl: String, // "0", + pub locked: String, // "0", + pub margin_collateral: bool, // true, + pub coin: String, // "BTC" +} + +/// Bybit wallet balance +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct BybitWalletBalanceResponse { + ret_code: u64, + ret_msg: String, + result: BybitWalletBalance, +} diff --git a/atelier-data/src/exchanges/bybit/bybit_wss_client.rs b/atelier-data/src/exchanges/bybit/bybit_wss_client.rs index 2f14997..52f10ea 100644 --- a/atelier-data/src/exchanges/bybit/bybit_wss_client.rs +++ b/atelier-data/src/exchanges/bybit/bybit_wss_client.rs @@ -1,6 +1,6 @@ use crate::{ clients::wss::WssDecoder, - errors::ExchangeError, + exchanges::errors::ExchangeError, exchanges::bybit::ws_decoder::{BybitDecoder, BybitWssEvent}, }; use futures_util::{SinkExt, StreamExt}; diff --git a/atelier-data/src/config/bybit/connections.toml b/atelier-data/src/exchanges/bybit/configs/connections.toml similarity index 100% rename from atelier-data/src/config/bybit/connections.toml rename to atelier-data/src/exchanges/bybit/configs/connections.toml diff --git a/atelier-data/src/config/bybit/liquidations.toml b/atelier-data/src/exchanges/bybit/configs/liquidations.toml similarity index 100% rename from atelier-data/src/config/bybit/liquidations.toml rename to atelier-data/src/exchanges/bybit/configs/liquidations.toml diff --git a/atelier-data/src/exchanges/bybit/mod.rs b/atelier-data/src/exchanges/bybit/mod.rs index 462a0fb..50c18b8 100644 --- a/atelier-data/src/exchanges/bybit/mod.rs +++ b/atelier-data/src/exchanges/bybit/mod.rs @@ -1,9 +1,8 @@ -pub mod ws_decoder; +use crate::exchanges::bybit::ws_decoder::BybitWssEvent; +pub mod ws_decoder; pub mod bybit_wss_client; -use crate::exchanges::bybit::ws_decoder::BybitWssEvent; pub use bybit_wss_client::BybitWssClient; - pub mod responses; pub use responses::{ liquidations::{LiquidationData, LiquidationResponse}, @@ -19,7 +18,7 @@ pub async fn stream_data( symbols: Vec, streams: Vec, source: WssExchange, -) -> Result, crate::errors::ExchangeError> { +) -> Result, crate::exchanges::errors::ExchangeError> { let (tx, rx) = tokio::sync::mpsc::channel(1024); let client = match source { diff --git a/atelier-data/src/exchanges/bybit/responses/mod.rs b/atelier-data/src/exchanges/bybit/responses/mod.rs index 3bc800a..6a4d91f 100644 --- a/atelier-data/src/exchanges/bybit/responses/mod.rs +++ b/atelier-data/src/exchanges/bybit/responses/mod.rs @@ -1,4 +1,7 @@ pub mod funding; pub mod liquidations; pub mod trades; +pub mod orderbook; + +pub use orderbook::BybitOrderbookResponse; diff --git a/atelier-data/src/exchanges/bybit/responses/orderbook.rs b/atelier-data/src/exchanges/bybit/responses/orderbook.rs new file mode 100644 index 0000000..f81de24 --- /dev/null +++ b/atelier-data/src/exchanges/bybit/responses/orderbook.rs @@ -0,0 +1,35 @@ +use serde::Deserialize; + +/// Bybit Orderbook response structure +#[derive(Debug, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct BybitOrderbookResponse { + pub ret_code: i32, + pub ret_msg: String, + pub result: BybitOrderbook, + pub ret_ext_info: serde_json::Value, + pub time: u64, +} + +/// Bybit Orderbook result structure +#[derive(Debug, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct BybitOrderbook { + #[serde(rename = "s")] + pub symbol: String, + #[serde(rename = "a")] + pub asks: Vec, + #[serde(rename = "b")] + pub bids: Vec, + #[serde(rename = "ts")] + pub timestamp: u64, +} + +/// Bybit PriceLevel result structure +#[derive(Debug, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct PriceLevel { + pub price: String, + pub qty: String, +} + diff --git a/atelier-data/src/exchanges/bybit/ws_decoder.rs b/atelier-data/src/exchanges/bybit/ws_decoder.rs index ab316ed..b4d482c 100644 --- a/atelier-data/src/exchanges/bybit/ws_decoder.rs +++ b/atelier-data/src/exchanges/bybit/ws_decoder.rs @@ -1,6 +1,6 @@ use crate::{ + exchanges::errors::ExchangeError, clients::wss::WssDecoder, - errors::ExchangeError, exchanges::bybit::responses::{ liquidations::{LiquidationData, LiquidationResponse}, trades::{TradeData, TradeResponse}, diff --git a/atelier-data/src/exchanges/coinbase/README.md b/atelier-data/src/exchanges/coinbase/README.md new file mode 100644 index 0000000..bfce938 --- /dev/null +++ b/atelier-data/src/exchanges/coinbase/README.md @@ -0,0 +1,3 @@ +# References for Coinbase + + diff --git a/atelier-data/src/exchanges/coinbase/coinbase_client.rs b/atelier-data/src/exchanges/coinbase/coinbase_client.rs new file mode 100644 index 0000000..be011fc --- /dev/null +++ b/atelier-data/src/exchanges/coinbase/coinbase_client.rs @@ -0,0 +1,196 @@ +use crate::{ + clients::http::http_client::{HttpClient, RetryConfig, RetryableHttpClient}, + exchanges::{ + coinbase::responses, + errors::{ExchangeError, Result}, + }, + models::{orderbook, pairs}, +}; + +use chrono::Utc; +use std::str::FromStr; +use tracing::{debug, info}; + +/// Coinbase Advanced Trade API client +#[derive(Clone)] +pub struct CoinbaseClient { + client: RetryableHttpClient, +} + +impl CoinbaseClient { + /// Create a new Coinbase client + pub fn new() -> Result { + let http_client = HttpClient::new( + "Coinbase".to_string(), + "https://api.coinbase.com".to_string(), + 10, // 10 requests per second + 30, // 30 second timeout + )?; + + let retry_client = RetryableHttpClient::new(http_client, RetryConfig::default()); + + Ok(Self { + client: retry_client, + }) + } + + /// Get order book snapshot for a trading pair + pub async fn get_orderbook( + &self, + pair: pairs::TradingPair, + depth: Option, + ) -> Result { + let product_id = pair.to_exchange_symbol("coinbase"); + info!("Fetching Coinbase orderbook for {}", product_id); + + let mut params: Vec<(&str, String)> = vec![("product_id", product_id.clone())]; + + if let Some(depth) = depth { + params.push(("limit", depth.to_string())); + } + + // Convert params to the expected type for get_with_params_retry + let params_ref: Vec<(&str, &str)> = + params.iter().map(|(k, v)| (*k, v.as_str())).collect(); + + let response: responses::orderbook::CoinbaseProductBookResponse = self + .client + .get_with_params_retry("/api/v3/brokerage/market/product_book", ¶ms_ref) + .await?; + + debug!( + "Received Coinbase orderbook with {} bids, {} asks", + response.pricebook.bids.len(), + response.pricebook.asks.len() + ); + + self.convert_to_orderbook(response, product_id) + } + + /// Convert Coinbase response to our OrderBook format + fn convert_to_orderbook( + &self, + response: responses::orderbook::CoinbaseProductBookResponse, + symbol: String, + ) -> Result { + let mut v_bids = Vec::new(); + let mut v_asks = Vec::new(); + + // Convert bids + for bid in response.pricebook.bids { + let price = + f64::from_str(&bid.price).map_err(|e| ExchangeError::ApiError { + exchange: "Coinbase".to_string(), + message: format!("Invalid bid price '{}': {}", bid.price, e), + })?; + + let quantity = + f64::from_str(&bid.size).map_err(|e| ExchangeError::ApiError { + exchange: "Coinbase".to_string(), + message: format!("Invalid bid size '{}': {}", bid.size, e), + })?; + + // orderbook.bids.push(PriceLevel { price, quantity }); + v_bids.push(orderbook::PriceLevel { price, quantity }); + } + + // Convert asks + for ask in response.pricebook.asks { + let price = + f64::from_str(&ask.price).map_err(|e| ExchangeError::ApiError { + exchange: "Coinbase".to_string(), + message: format!("Invalid ask price '{}': {}", ask.price, e), + })?; + + let quantity = + f64::from_str(&ask.size).map_err(|e| ExchangeError::ApiError { + exchange: "Coinbase".to_string(), + message: format!("Invalid ask size '{}': {}", ask.size, e), + })?; + + // orderbook.asks.push(PriceLevel { price, quantity }); + v_asks.push(orderbook::PriceLevel { price, quantity }); + } + + // Final value + let orderbook = orderbook::Orderbook::new( + symbol, + "Coinbase".to_string(), + Utc::now(), + v_bids, + v_asks, + None, + None, + ); + + // Validate the orderbook + if !orderbook.is_valid() { + return Err(ExchangeError::ApiError { + exchange: "Coinbase".to_string(), + message: "Received invalid orderbook data".to_string(), + }); + } + + info!( + "Successfully converted Coinbase orderbook: {} bids, {} asks, spread: {:?}", + orderbook.bids.len(), + orderbook.asks.len(), + orderbook.spread() + ); + + Ok(orderbook) + } + + /// Get all products (trading pairs) + pub async fn get_products(&self) -> Result> { + info!("Fetching Coinbase products"); + + let response: responses::product::CoinbaseProductsResponse = self + .client + .get_with_retry("/api/v3/brokerage/products") + .await?; + + Ok(response.products) + } + + /// Get specific product information + pub async fn get_product( + &self, + product_id: &str, + ) -> Result { + info!("Fetching Coinbase product info for {}", product_id); + + let endpoint = format!("/api/v3/brokerage/products/{product_id}"); + self.client.get_with_retry(&endpoint).await + } + + /// Get market trades + pub async fn get_market_trades( + &self, + product_id: &str, + depth: Option, + ) -> Result { + let endpoint = format!("/api/v3/brokerage/products/{product_id}/ticker"); + let mut params: Vec<(&str, String)> = + vec![("product_id", product_id.to_string())]; + + if let Some(depth) = depth { + params.push(("depth", depth.to_string())); + } + + // Convert params to the expected type for get_with_params_retry + let params_ref: Vec<(&str, &str)> = + params.iter().map(|(k, v)| (*k, v.as_str())).collect(); + + self.client + .get_with_params_retry(&endpoint, ¶ms_ref) + .await + } +} + +/// Coinbase client default +impl Default for CoinbaseClient { + fn default() -> Self { + Self::new().expect("Failed to create default Coinbase client") + } +} diff --git a/atelier-data/src/exchanges/coinbase/mod.rs b/atelier-data/src/exchanges/coinbase/mod.rs new file mode 100644 index 0000000..27000db --- /dev/null +++ b/atelier-data/src/exchanges/coinbase/mod.rs @@ -0,0 +1,2 @@ +pub mod coinbase_client; +pub mod responses; diff --git a/atelier-data/src/exchanges/coinbase/responses/mod.rs b/atelier-data/src/exchanges/coinbase/responses/mod.rs new file mode 100644 index 0000000..659b0ae --- /dev/null +++ b/atelier-data/src/exchanges/coinbase/responses/mod.rs @@ -0,0 +1,4 @@ +pub mod orderbook; +pub mod trades; +pub mod product; + diff --git a/atelier-data/src/exchanges/coinbase/responses/orderbook.rs b/atelier-data/src/exchanges/coinbase/responses/orderbook.rs new file mode 100644 index 0000000..cbde7fe --- /dev/null +++ b/atelier-data/src/exchanges/coinbase/responses/orderbook.rs @@ -0,0 +1,24 @@ +use serde::Deserialize; + +/// Coinbase product book response +#[derive(Debug, Deserialize)] +pub struct CoinbaseProductBookResponse { + pub pricebook: CoinbasePricebook, +} + +/// Coinbase pricebook +#[derive(Debug, Deserialize)] +pub struct CoinbasePricebook { + pub product_id: String, + pub bids: Vec, + pub asks: Vec, + pub time: String, +} + +/// Coinbase price level +#[derive(Debug, Deserialize)] +pub struct CoinbasePriceLevel { + pub price: String, + pub size: String, +} + diff --git a/atelier-data/src/exchanges/coinbase/responses/product.rs b/atelier-data/src/exchanges/coinbase/responses/product.rs new file mode 100644 index 0000000..0f95c88 --- /dev/null +++ b/atelier-data/src/exchanges/coinbase/responses/product.rs @@ -0,0 +1,38 @@ +use serde::Deserialize; + +/// Coinbase products response +#[derive(Debug, Deserialize)] +pub struct CoinbaseProductsResponse { + pub products: Vec, +} + +/// Coinbase product information +#[derive(Debug, Deserialize)] +pub struct CoinbaseProduct { + pub product_id: String, + pub price: Option, + pub price_percentage_change_24h: Option, + pub volume_24h: Option, + pub volume_percentage_change_24h: Option, + pub base_increment: String, + pub quote_increment: String, + pub quote_min_size: String, + pub quote_max_size: String, + pub base_min_size: String, + pub base_max_size: String, + pub base_name: String, + pub quote_name: String, + pub watched: bool, + pub is_disabled: bool, + pub new: bool, + pub status: String, + pub cancel_only: bool, + pub depth_only: bool, + pub post_only: bool, + pub trading_disabled: bool, + pub auction_mode: bool, + pub product_type: String, + pub quote_currency_id: String, + pub base_currency_id: String, +} + diff --git a/atelier-data/src/exchanges/coinbase/responses/trades.rs b/atelier-data/src/exchanges/coinbase/responses/trades.rs new file mode 100644 index 0000000..372ca4a --- /dev/null +++ b/atelier-data/src/exchanges/coinbase/responses/trades.rs @@ -0,0 +1,23 @@ +use serde::Deserialize; + +/// Coinbase trades response +#[derive(Debug, Deserialize)] +pub struct CoinbaseTradesResponse { + pub trades: Vec, + pub best_bid: String, + pub best_ask: String, +} + +/// Coinbase trade +#[derive(Debug, Deserialize)] +pub struct CoinbaseTrade { + pub trade_id: String, + pub product_id: String, + pub price: String, + pub size: String, + pub time: String, + pub side: String, + pub bid: String, + pub ask: String, +} + diff --git a/atelier-data/src/config/loader.rs b/atelier-data/src/exchanges/config.rs similarity index 99% rename from atelier-data/src/config/loader.rs rename to atelier-data/src/exchanges/config.rs index ef25f9f..10aa05c 100644 --- a/atelier-data/src/config/loader.rs +++ b/atelier-data/src/exchanges/config.rs @@ -1,3 +1,4 @@ + use serde::Deserialize; use std::{error::Error, fs}; use toml; diff --git a/atelier-data/src/exchanges/errors.rs b/atelier-data/src/exchanges/errors.rs new file mode 100644 index 0000000..6c9f956 --- /dev/null +++ b/atelier-data/src/exchanges/errors.rs @@ -0,0 +1,78 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum ExchangeError { + #[error("WebSocket connection error: {0}")] + WebSocketError(Box), + + #[error("URL parsing error: {0}")] + UrlParseError(#[from] url::ParseError), + + #[error("JSON parsing error: {0}")] + JsonParsing(#[from] serde_json::Error), + + #[error("Configuration error: {0}")] + ConfigError(#[from] config::ConfigError), + + #[error("Configuration error: {message}")] + Configuration { message: String }, + + #[error("Channel send error")] + ChannelSendError, + + #[error("An IO error occurred: {0}")] + IoError(#[from] std::io::Error), + + #[error("Unknown error")] + Unknown, + + #[error("Network error: {0}")] + Network(#[from] reqwest::Error), + + #[error("Rate limit exceeded for exchange: {exchange}")] + RateLimit { exchange: String }, + + #[error("Timeout error: {0}")] + Timeout(String), + + #[error("API error from {exchange}: {message}")] + ApiError { exchange: String, message: String }, +} + +impl ExchangeError { + /// Check if the error is retryable + pub fn is_retryable(&self) -> bool { + matches!( + self, + ExchangeError::Network(_) + | ExchangeError::RateLimit { .. } + | ExchangeError::Timeout(_) + ) + } + + /// Get the exchange name if applicable + pub fn exchange(&self) -> Option<&str> { + match self { + ExchangeError::RateLimit { exchange, .. } + | ExchangeError::ApiError { exchange, .. } => Some(exchange), + _ => None, + } + } +} + +impl From for ExchangeError { + fn from(error: tokio_tungstenite::tungstenite::Error) -> Self { + ExchangeError::WebSocketError(Box::new(error)) + } +} + +#[derive(Error, Debug, Clone)] +pub enum WssError { + // Connection Error + #[error("Connection Failed")] + WssConnection, +} + +/// Result type alias using our custom error +pub type Result = std::result::Result; + diff --git a/atelier-data/src/exchanges/gate/README.md b/atelier-data/src/exchanges/gate/README.md new file mode 100644 index 0000000..eb4b69d --- /dev/null +++ b/atelier-data/src/exchanges/gate/README.md @@ -0,0 +1 @@ +# References for Gate diff --git a/atelier-data/src/exchanges/gate/gate_client.rs b/atelier-data/src/exchanges/gate/gate_client.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/gate/mod.rs b/atelier-data/src/exchanges/gate/mod.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/gate/responses/mod.rs b/atelier-data/src/exchanges/gate/responses/mod.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/gate/responses/orderbook.rs b/atelier-data/src/exchanges/gate/responses/orderbook.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/gate/responses/trades.rs b/atelier-data/src/exchanges/gate/responses/trades.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/gemini/README.md b/atelier-data/src/exchanges/gemini/README.md new file mode 100644 index 0000000..6a6ed04 --- /dev/null +++ b/atelier-data/src/exchanges/gemini/README.md @@ -0,0 +1 @@ +# References for Gemini diff --git a/atelier-data/src/exchanges/gemini/gemini_client.rs b/atelier-data/src/exchanges/gemini/gemini_client.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/gemini/mod.rs b/atelier-data/src/exchanges/gemini/mod.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/gemini/responses/mod.rs b/atelier-data/src/exchanges/gemini/responses/mod.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/gemini/responses/orderbook.rs b/atelier-data/src/exchanges/gemini/responses/orderbook.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/gemini/responses/trades.rs b/atelier-data/src/exchanges/gemini/responses/trades.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/kraken/README.md b/atelier-data/src/exchanges/kraken/README.md new file mode 100644 index 0000000..82e7e68 --- /dev/null +++ b/atelier-data/src/exchanges/kraken/README.md @@ -0,0 +1 @@ +# References for Kraken diff --git a/atelier-data/src/exchanges/kraken/kraken_client.rs b/atelier-data/src/exchanges/kraken/kraken_client.rs new file mode 100644 index 0000000..fdc71a4 --- /dev/null +++ b/atelier-data/src/exchanges/kraken/kraken_client.rs @@ -0,0 +1,354 @@ +use crate::client::http_client::{HttpClient, RetryConfig, RetryableHttpClient}; +use crate::models::orderbook::{Orderbook, PriceLevel, TradingPair}; + +use chrono::Utc; +use ix_results::errors::{ExchangeError, Result}; +use serde::Deserialize; +use std::collections::HashMap; +use std::str::FromStr; +use tracing::{debug, info}; + +/// Kraken REST API client +#[derive(Clone)] +pub struct KrakenClient { + client: RetryableHttpClient, +} + +impl KrakenClient { + /// Create a new Kraken client + pub fn new() -> Result { + let http_client = HttpClient::new( + "Kraken".to_string(), + "https://api.kraken.com".to_string(), + 1, // Conservative rate limit for Kraken (1 request per second) + 30, // 30 second timeout + )?; + + let retry_client = RetryableHttpClient::new(http_client, RetryConfig::default()); + + Ok(Self { + client: retry_client, + }) + } + + /// Get order book snapshot for a trading pair + pub async fn get_orderbook( + &self, + pair: TradingPair, + count: Option, + ) -> Result { + let product_id = pair.to_exchange_symbol("kraken"); + info!("Fetching Kraken orderbook for {}", product_id); + + let mut params: Vec<(&str, String)> = vec![("pair", product_id.clone())]; + + if let Some(count) = count { + params.push(("count", count.to_string())); + } + + // Convert params to the expected type for get_with_params_retry + let params_ref: Vec<(&str, &str)> = + params.iter().map(|(k, v)| (*k, v.as_str())).collect(); + + // debug!("Params: {:?}", params_ref); + + let response: KrakenDepthResponse = self + .client + .get_with_params_retry("/0/public/Depth", ¶ms_ref) + .await?; + + // Check for API errors + if !response.error.is_empty() { + return Err(ExchangeError::ApiError { + exchange: "Kraken".to_string(), + message: format!("Kraken API errors: {:?}", response.error), + }); + } + + debug!( + "Received Kraken orderbook response with {} pairs", + response.result.len() + ); + + // Find the orderbook data for our pair + let orderbook_data = + response + .result + .values() + .next() + .ok_or_else(|| ExchangeError::ApiError { + exchange: "Kraken".to_string(), + message: "No orderbook data found in response".to_string(), + })?; + + self.convert_to_orderbook(orderbook_data.clone(), product_id) + } + + /// Convert Kraken response to our OrderBook format + fn convert_to_orderbook( + &self, + data: KrakenOrderbookData, + symbol: String, + ) -> Result { + let mut ob_ts: u64 = 0; + let mut v_bids = Vec::new(); + let mut v_asks = Vec::new(); + + for bid in data.bids { + // Update orderbook timestamp to the newst timestamp found in the levels + // exclusive from Kraken response. + if bid.2 > ob_ts { + let delta_ts = bid.2 - ob_ts; + ob_ts += delta_ts; + } + + let price = f64::from_str(&bid.0).map_err(|e| ExchangeError::ApiError { + exchange: "Kraken".to_string(), + message: format!("Invalid bid price '{}': {}", bid.0, e), + })?; + let quantity = + f64::from_str(&bid.1).map_err(|e| ExchangeError::ApiError { + exchange: "Kraken".to_string(), + message: format!("Invalid bid volume '{}': {}", bid.1, e), + })?; + v_bids.push(PriceLevel { price, quantity }); + } + + for ask in data.asks { + if ask.2 > ob_ts { + let delta_ts = ask.2 - ob_ts; + ob_ts += delta_ts; + } + + let price = f64::from_str(&ask.0).map_err(|e| ExchangeError::ApiError { + exchange: "Kraken".to_string(), + message: format!("Invalid ask price '{}': {}", ask.0, e), + })?; + let quantity = + f64::from_str(&ask.1).map_err(|e| ExchangeError::ApiError { + exchange: "Kraken".to_string(), + message: format!("Invalid ask volume '{}': {}", ask.1, e), + })?; + v_asks.push(PriceLevel { price, quantity }); + } + + // Final value + let mut orderbook = Orderbook::new( + symbol, + "Kraken".to_string(), + Utc::now(), + v_bids, + v_asks, + None, + None, + ); + + if !orderbook.is_valid() { + return Err(ExchangeError::ApiError { + exchange: "Kraken".to_string(), + message: "Received invalid orderbook data".to_string(), + }); + } + + info!( + "Successfully converted Kraken orderbook: {} bids, {} asks, spread: {:?}", + orderbook.bids.len(), + orderbook.asks.len(), + orderbook.spread() + ); + + // exchange_ts + // orderbook.timestamp = DateTime::::from_timestamp(ob_ts as i64, 3).unwrap(); + + // response_ts + orderbook.timestamp = Utc::now(); + + Ok(orderbook) + } + + /// Get server time + pub async fn get_server_time(&self) -> Result { + debug!("Fetching Kraken server time"); + + let response: KrakenServerTimeResponse = + self.client.get_with_retry("/0/public/Time").await?; + + if !response.error.is_empty() { + return Err(ExchangeError::ApiError { + exchange: "Kraken".to_string(), + message: format!("Kraken API errors: {:?}", response.error), + }); + } + + Ok(response.result) + } + + /// Get system status + pub async fn get_system_status(&self) -> Result { + info!("Fetching Kraken system status"); + + let response: KrakenSystemStatusResponse = + self.client.get_with_retry("/0/public/SystemStatus").await?; + + if !response.error.is_empty() { + return Err(ExchangeError::ApiError { + exchange: "Kraken".to_string(), + message: format!("Kraken API errors: {:?}", response.error), + }); + } + + Ok(response.result) + } + + /// Get asset pairs information + pub async fn get_asset_pairs(&self) -> Result> { + println!("Fetching Kraken asset pairs"); + + let response: KrakenAssetPairsResponse = + self.client.get_with_retry("/0/public/AssetPairs").await?; + + if !response.error.is_empty() { + return Err(ExchangeError::ApiError { + exchange: "Kraken".to_string(), + message: format!("Kraken API errors: {:?}", response.error), + }); + } + + Ok(response.result) + } + + /// Get ticker information + pub async fn get_ticker( + &self, + pair: TradingPair, + ) -> Result> { + let pair_name = pair.to_exchange_symbol("kraken"); + info!("Fetching Kraken ticker for {}", pair_name); + + let params = vec![("pair", pair_name.as_str())]; + let response: KrakenTickerResponse = self + .client + .get_with_params_retry("/0/public/Ticker", ¶ms) + .await?; + + if !response.error.is_empty() { + return Err(ExchangeError::ApiError { + exchange: "Kraken".to_string(), + message: format!("Kraken API errors: {:?}", response.error), + }); + } + + Ok(response.result) + } +} + +/// Kraken depth response +#[derive(Debug, Deserialize)] +struct KrakenDepthResponse { + error: Vec, + result: HashMap, +} + +#[derive(Debug, Clone, Deserialize)] +struct KrakenPriceLevel( + String, // price + String, // volume + u64, // timestamp +); + +#[derive(Debug, Clone, Deserialize)] +struct KrakenOrderbookData { + bids: Vec, + asks: Vec, +} + +/// Kraken server time response +#[derive(Debug, Deserialize)] +struct KrakenServerTimeResponse { + error: Vec, + result: KrakenServerTime, +} + +/// Kraken server time +#[derive(Debug, Deserialize)] +pub struct KrakenServerTime { + pub unixtime: u64, + pub rfc1123: String, +} + +/// Kraken system status response +#[derive(Debug, Deserialize)] +struct KrakenSystemStatusResponse { + error: Vec, + result: KrakenSystemStatus, +} + +/// Kraken system status +#[derive(Debug, Deserialize)] +pub struct KrakenSystemStatus { + pub status: String, + pub timestamp: String, +} + +/// Kraken asset pairs response +#[derive(Debug, Deserialize)] +struct KrakenAssetPairsResponse { + error: Vec, + result: HashMap, +} + +/// Kraken asset pair information +#[derive(Debug, Deserialize)] +pub struct KrakenAssetPair { + pub altname: String, + pub wsname: Option, + pub aclass_base: String, + pub base: String, + pub aclass_quote: String, + pub quote: String, + pub pair_decimals: u32, + pub cost_decimals: u32, + pub lot_decimals: u32, + pub lot_multiplier: u32, + pub leverage_buy: Vec, + pub leverage_sell: Vec, + pub fees: Vec>, + pub fees_maker: Vec>, + pub fee_volume_currency: String, + pub margin_call: u32, + pub margin_stop: u32, + pub ordermin: String, + pub costmin: Option, + pub tick_size: Option, + pub status: String, + pub long_position_limit: Option, + pub short_position_limit: Option, +} + +/// Kraken ticker response +#[derive(Debug, Deserialize)] +struct KrakenTickerResponse { + error: Vec, + result: HashMap, +} + +/// Kraken ticker data +#[derive(Debug, Deserialize)] +pub struct KrakenTicker { + pub a: Vec, // ask [price, whole_lot_volume, lot_volume] + pub b: Vec, // bid [price, whole_lot_volume, lot_volume] + pub c: Vec, // last trade closed [price, lot_volume] + pub v: Vec, // volume [today, last_24_hours] + pub p: Vec, // volume weighted average price [today, last_24_hours] + pub t: Vec, // number of trades [today, last_24_hours] + pub l: Vec, // low [today, last_24_hours] + pub h: Vec, // high [today, last_24_hours] + pub o: String, // today's opening price +} + +impl Default for KrakenClient { + fn default() -> Self { + Self::new().expect("Failed to create default Kraken client") + } +} + diff --git a/atelier-data/src/exchanges/kraken/mod.rs b/atelier-data/src/exchanges/kraken/mod.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/kraken/responses/mod.rs b/atelier-data/src/exchanges/kraken/responses/mod.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/kraken/responses/orderbook.rs b/atelier-data/src/exchanges/kraken/responses/orderbook.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/kraken/responses/trades.rs b/atelier-data/src/exchanges/kraken/responses/trades.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/mod.rs b/atelier-data/src/exchanges/mod.rs index 3fbb0fd..1695949 100644 --- a/atelier-data/src/exchanges/mod.rs +++ b/atelier-data/src/exchanges/mod.rs @@ -1,6 +1,23 @@ +pub mod binance; +pub mod bitso; pub mod bybit; +pub mod coinbase; +pub mod gate; +pub mod gemini; +pub mod kraken; +pub mod okx; + +pub mod config; +pub mod errors; #[derive(Debug, Clone)] pub enum Exchange { Bybit, + Binance, + Coinbase, + Kraken, + Gate, + Okx, + Bitso, + Gemini, } diff --git a/atelier-data/src/exchanges/okx/README.md b/atelier-data/src/exchanges/okx/README.md new file mode 100644 index 0000000..0b36223 --- /dev/null +++ b/atelier-data/src/exchanges/okx/README.md @@ -0,0 +1 @@ +# References for Okx diff --git a/atelier-data/src/exchanges/okx/mod.rs b/atelier-data/src/exchanges/okx/mod.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/okx/okx_client.rs b/atelier-data/src/exchanges/okx/okx_client.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/okx/responses/mod.rs b/atelier-data/src/exchanges/okx/responses/mod.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/okx/responses/orderbook.rs b/atelier-data/src/exchanges/okx/responses/orderbook.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/exchanges/okx/responses/trades.rs b/atelier-data/src/exchanges/okx/responses/trades.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/lib.rs b/atelier-data/src/lib.rs index d7a4901..6918b98 100644 --- a/atelier-data/src/lib.rs +++ b/atelier-data/src/lib.rs @@ -1,17 +1,14 @@ -/// Configuration tempaltes -pub mod config; - /// Communication clients pub mod clients; /// Exchanges and properties pub mod exchanges; -/// Error messages -pub mod errors; +/// Data models +pub mod models; // Re-export common types for convenience pub use clients::wss::{WssClient, WssClientBuilder, WssDecoder}; -pub use errors::ExchangeError; pub use exchanges::bybit::{stream_data, BybitWssClient, LiquidationData}; +pub use exchanges::errors::ExchangeError; diff --git a/atelier-data/src/models/exchanges.rs b/atelier-data/src/models/exchanges.rs new file mode 100644 index 0000000..c997deb --- /dev/null +++ b/atelier-data/src/models/exchanges.rs @@ -0,0 +1,11 @@ +#[derive(Clone, Debug)] +pub enum Exchange { + Binance, + Bitso, + Bybit, + Coinbase, + Gate, + Gemini, + Kraken, + Okx, +} diff --git a/atelier-data/src/models/mod.rs b/atelier-data/src/models/mod.rs new file mode 100644 index 0000000..8dbede5 --- /dev/null +++ b/atelier-data/src/models/mod.rs @@ -0,0 +1,3 @@ +pub mod exchanges; +pub mod orderbook; +pub mod pairs; diff --git a/atelier-data/src/models/orderbook.rs b/atelier-data/src/models/orderbook.rs new file mode 100644 index 0000000..f603afc --- /dev/null +++ b/atelier-data/src/models/orderbook.rs @@ -0,0 +1,286 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; + +/// Represents a single price level in the order book +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct PriceLevel { + pub price: f64, + pub quantity: f64, +} + +impl PriceLevel { + pub fn new(price: f64, quantity: f64) -> Self { + Self { price, quantity } + } +} + +impl Default for PriceLevel { + fn default() -> Self { + Self { + price: 0.0, + quantity: 0.0, + } + } +} + +/// Input structure for JSON parsing +#[derive(Debug, Serialize, Deserialize)] +pub struct OrderbookInput { + pub symbol: String, + pub exchange: String, + pub timestamp: String, + pub bids: Vec, + pub asks: Vec, + pub last_update_id: u64, + pub sequence: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct PriceLevelInput { + pub price: String, + pub quantity: String, +} + +/// Complete order book snapshot +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Orderbook { + pub symbol: String, + pub exchange: String, + pub timestamp: DateTime, + pub bids: Vec, + pub asks: Vec, + pub last_update_id: Option, + pub sequence: Option, +} + +impl TryFrom for Orderbook { + type Error = chrono::ParseError; + + fn try_from(input: OrderbookInput) -> Result { + let timestamp = + DateTime::parse_from_rfc3339(&input.timestamp)?.with_timezone(&Utc); + + let bids = input + .bids + .into_iter() + .map(|level| { + PriceLevel::new( + f64::from_str(&level.price).unwrap(), + f64::from_str(&level.quantity).unwrap(), + ) + }) + .collect(); + + let asks = input + .asks + .into_iter() + .map(|level| { + PriceLevel::new( + f64::from_str(&level.price).unwrap(), + f64::from_str(&level.quantity).unwrap(), + ) + }) + .collect(); + + Ok(Orderbook::new( + input.symbol, + input.exchange, + timestamp, + bids, + asks, + Some(input.last_update_id), + input.sequence, + )) + } +} + +impl Default for Orderbook { + fn default() -> Self { + Self { + symbol: "default_symbol".to_string(), + exchange: "default_exchange".to_string(), + timestamp: DateTime::default(), + bids: vec![PriceLevel::default()], + asks: vec![PriceLevel::default()], + last_update_id: Some(1234), + sequence: Some(4321), + } + } +} + +impl Orderbook { + /// Create a new empty order book + pub fn new( + symbol: String, + exchange: String, + timestamp: DateTime, + bids: Vec, + asks: Vec, + last_update_id: Option, + sequence: Option, + ) -> Self { + Self { + symbol, + exchange, + timestamp, + bids, + asks, + last_update_id, + sequence, + } + } + + /// Get the best bid (highest buy price) + pub fn best_bid(&self) -> Option<&PriceLevel> { + self.bids.first() + } + + /// Get the best ask (lowest sell price) + pub fn best_ask(&self) -> Option<&PriceLevel> { + self.asks.first() + } + + /// Calculate the bid-ask spread + pub fn spread(&self) -> Option { + match (self.best_bid(), self.best_ask()) { + (Some(bid), Some(ask)) => Some(ask.price - bid.price), + _ => None, + } + } + + /// Calculate the mid price + pub fn mid_price(&self) -> Option { + match (self.best_bid(), self.best_ask()) { + (Some(bid), Some(ask)) => Some((bid.price + ask.price) / 2.0), + _ => None, + } + } + + /// Get total liquidity within a certain percentage of the mid price + pub fn liquidity_within_percentage(&self, percentage: f64) -> (f64, f64) { + let mid = match self.mid_price() { + Some(mid) => mid, + None => return (0.0, 0.0), + }; + + let threshold = mid * percentage / 100.0; + let bid_threshold = mid - threshold; + let ask_threshold = mid + threshold; + + let bid_liquidity = self + .bids + .iter() + .filter(|level| level.price >= bid_threshold) + .map(|level| level.quantity) + .sum(); + + let ask_liquidity = self + .asks + .iter() + .filter(|level| level.price <= ask_threshold) + .map(|level| level.quantity) + .sum(); + + (bid_liquidity, ask_liquidity) + } + + /// Validate that the order book is properly sorted and has no crossed spread + pub fn is_valid(&self) -> bool { + // Check if bids are sorted in descending order + for i in 1..self.bids.len() { + if self.bids[i].price > self.bids[i - 1].price { + return false; + } + } + + // Check if asks are sorted in ascending order + for i in 1..self.asks.len() { + if self.asks[i].price < self.asks[i - 1].price { + return false; + } + } + + // Check that best bid < best ask (no crossed spread) + if let (Some(best_bid), Some(best_ask)) = (self.best_bid(), self.best_ask()) { + if best_bid.price >= best_ask.price { + return false; + } + } + + true + } + + /// Get total volume on bid side + pub fn bid_volume(&self) -> f64 { + self.bids.iter().map(|level| level.quantity).sum() + } + + /// Get total volume on ask side + pub fn ask_volume(&self) -> f64 { + self.asks.iter().map(|level| level.quantity).sum() + } + + /// Create partitioned file path for parquet storage + pub fn parquet_path(&self) -> String { + format!( + "{}-{}-{}-{}-{}.parquet", + self.exchange, + self.timestamp.format("%Y%m%d"), + self.timestamp.format("%H"), + self.timestamp.format("%M"), + self.symbol + ) + } + + /// Validate Orderbook data + pub fn validate(&self) -> Result<(), String> { + if self.symbol.is_empty() { + return Err("Symbol cannot be empty".to_string()); + } + + if self.exchange.is_empty() { + return Err("Exchange cannot be empty".to_string()); + } + + if self.bids.is_empty() && self.asks.is_empty() { + return Err("Orderbook must have at least one bid or ask".to_string()); + } + + Ok(()) + } +} + +/// Summary statistics for an order book +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OrderbookSummary { + pub symbol: String, + pub exchange: String, + pub timestamp: DateTime, + pub best_bid: Option, + pub best_ask: Option, + pub spread: Option, + pub mid_price: Option, + pub bid_count: usize, + pub ask_count: usize, + pub total_bid_volume: f64, + pub total_ask_volume: f64, +} + +impl From<&Orderbook> for OrderbookSummary { + fn from(orderbook: &Orderbook) -> Self { + Self { + symbol: orderbook.symbol.clone(), + exchange: orderbook.exchange.clone(), + timestamp: orderbook.timestamp, + best_bid: orderbook.best_bid().map(|b| b.price), + best_ask: orderbook.best_ask().map(|a| a.price), + spread: orderbook.spread(), + mid_price: orderbook.mid_price(), + bid_count: orderbook.bids.len(), + ask_count: orderbook.asks.len(), + total_bid_volume: orderbook.bids.iter().map(|b| b.quantity).sum(), + total_ask_volume: orderbook.asks.iter().map(|a| a.quantity).sum(), + } + } +} diff --git a/atelier-data/src/models/pairs.rs b/atelier-data/src/models/pairs.rs new file mode 100644 index 0000000..dc265bd --- /dev/null +++ b/atelier-data/src/models/pairs.rs @@ -0,0 +1,77 @@ +use serde::{Deserialize, Serialize}; + +/// Supported trading pairs +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub enum TradingPair { + BtcUsdt, + BtcUsdc, + EthUsdt, + EthUsdc, + SolUsdt, + SolUsdc, +} + +impl TradingPair { + /// Convert to exchange-specific symbol format + pub fn to_exchange_symbol(&self, exchange: &str) -> String { + match (self, exchange.to_lowercase().as_str()) { + (TradingPair::BtcUsdt, "binance") => "BTCUSDT".to_string(), + (TradingPair::SolUsdt, "binance") => "SOLUSDT".to_string(), + (TradingPair::EthUsdt, "binance") => "ETHUSDT".to_string(), + (TradingPair::BtcUsdc, "binance") => "BTCUSDC".to_string(), + (TradingPair::SolUsdc, "binance") => "SOLUSDC".to_string(), + (TradingPair::EthUsdc, "binance") => "ETHUSDC".to_string(), + + (TradingPair::BtcUsdt, "bybit") => "BTCUSDT".to_string(), + (TradingPair::SolUsdt, "bybit") => "SOLUSDT".to_string(), + (TradingPair::EthUsdt, "bybit") => "ETHUSDT".to_string(), + (TradingPair::BtcUsdc, "bybit") => "BTCUSDC".to_string(), + (TradingPair::SolUsdc, "bybit") => "SOLUSDC".to_string(), + (TradingPair::EthUsdc, "bybit") => "ETHUSDC".to_string(), + + (TradingPair::BtcUsdt, "coinbase") => "BTC-USDT".to_string(), + (TradingPair::SolUsdt, "coinbase") => "SOL-USDT".to_string(), + (TradingPair::EthUsdt, "coinbase") => "ETH-USDT".to_string(), + (TradingPair::BtcUsdc, "coinbase") => "BTC-USDC".to_string(), + (TradingPair::SolUsdc, "coinbase") => "SOL-USDC".to_string(), + (TradingPair::EthUsdc, "coinbase") => "ETH-USDC".to_string(), + + (TradingPair::BtcUsdt, "kraken") => "BTCUSDT".to_string(), + (TradingPair::SolUsdt, "kraken") => "SOLUSDT".to_string(), + (TradingPair::EthUsdt, "kraken") => "ETHUSDT".to_string(), + (TradingPair::BtcUsdc, "kraken") => "BTCUSDC".to_string(), + (TradingPair::SolUsdc, "kraken") => "SOLUSDC".to_string(), + (TradingPair::EthUsdc, "kraken") => "ETHUSDC".to_string(), + + _ => format!("{self:?}"), // Fallback + } + } + + /// Parse from string + pub fn parse_from_str(s: &str) -> Option { + match s.to_uppercase().as_str() { + "BTCUSDT" | "BTC-USDT" | "BTC/USDT" => Some(TradingPair::BtcUsdt), + "SOLUSDT" | "SOL-USDT" | "SOL/USDT" => Some(TradingPair::SolUsdt), + "ETHUSDT" | "ETH-USDT" | "ETH/USDT" => Some(TradingPair::EthUsdt), + "BTCUSDC" | "BTC-USDC" | "BTC/USDC" => Some(TradingPair::BtcUsdc), + "SOLUSDC" | "SOL-USDC" | "SOL/USDC" => Some(TradingPair::SolUsdc), + "ETHUSDC" | "ETH-USDC" | "ETH/USDC" => Some(TradingPair::EthUsdc), + _ => None, + } + } +} + +impl std::fmt::Display for TradingPair { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + TradingPair::BtcUsdt => "BTC/USDT", + TradingPair::SolUsdt => "SOL/USDT", + TradingPair::EthUsdt => "ETH/USDT", + TradingPair::BtcUsdc => "BTC/USDC", + TradingPair::SolUsdc => "SOL/USDC", + TradingPair::EthUsdc => "ETH/USDC", + }; + write!(f, "{s}") + } +} + diff --git a/atelier-data/src/protocols/mod.rs b/atelier-data/src/protocols/mod.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/src/protocols/solana/mod.rs b/atelier-data/src/protocols/solana/mod.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/test/exchanges/binance/test_orderbook.rs b/atelier-data/test/exchanges/binance/test_orderbook.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/test/exchanges/binance/test_trades.rs b/atelier-data/test/exchanges/binance/test_trades.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/test/exchanges/bitso/test_orderbook.rs b/atelier-data/test/exchanges/bitso/test_orderbook.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/test/exchanges/bitso/test_trades.rs b/atelier-data/test/exchanges/bitso/test_trades.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/test/bybit/test_liquidations.rs b/atelier-data/test/exchanges/bybit/test_liquidations.rs similarity index 100% rename from atelier-data/test/bybit/test_liquidations.rs rename to atelier-data/test/exchanges/bybit/test_liquidations.rs diff --git a/atelier-data/test/exchanges/bybit/test_orderbook.rs b/atelier-data/test/exchanges/bybit/test_orderbook.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/test/exchanges/bybit/test_trades.rs b/atelier-data/test/exchanges/bybit/test_trades.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/test/exchanges/coinbase/test_orderbook.rs b/atelier-data/test/exchanges/coinbase/test_orderbook.rs new file mode 100644 index 0000000..dcaaabc --- /dev/null +++ b/atelier-data/test/exchanges/coinbase/test_orderbook.rs @@ -0,0 +1,30 @@ +#[cfg(test)] + +mod tests { + + use atelier_data::{ + exchanges::coinbase::coinbase_client::CoinbaseClient, + models::pairs::TradingPair, + }; + + #[tokio::test] + async fn test_coinbase_client() { + let exchange_client = CoinbaseClient::new(); + } + + #[tokio::test] + async fn test_coinbase_orderbook() { + + let exchange_client = CoinbaseClient::new(); + let depth = 10; + let r_orderbook = exchange_client.unwrap() + .get_orderbook(TradingPair::SolUsdc.clone(), Some(depth)) + .await + .unwrap(); + + println!("Orderbook from Coinbase: {:?}", r_orderbook); + + } + +} + diff --git a/atelier-data/test/exchanges/coinbase/test_products.rs b/atelier-data/test/exchanges/coinbase/test_products.rs new file mode 100644 index 0000000..32186a0 --- /dev/null +++ b/atelier-data/test/exchanges/coinbase/test_products.rs @@ -0,0 +1,28 @@ +#[cfg(test)] +mod tests { + + use atelier_data::exchanges::coinbase::coinbase_client::CoinbaseClient; + + #[tokio::test] + async fn test_coinbase_client_creation() { + let client = CoinbaseClient::new(); + assert!(client.is_ok()); + } + + #[tokio::test] + async fn test_get_products() { + let client = CoinbaseClient::new().unwrap(); + let result = client.get_products().await; + + // This test might fail if no internet connection + match result { + Ok(products) => { + assert!(!products.is_empty()); + println!("Found {} Coinbase products", products.len()); + } + Err(e) => { + println!("Expected network error in test environment: {e:?}"); + } + } + } +} diff --git a/atelier-data/test/exchanges/coinbase/test_trades.rs b/atelier-data/test/exchanges/coinbase/test_trades.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/test/configs/template.toml b/atelier-data/test/exchanges/configs/template.toml similarity index 100% rename from atelier-data/test/configs/template.toml rename to atelier-data/test/exchanges/configs/template.toml diff --git a/atelier-data/test/configs/test_from_toml.rs b/atelier-data/test/exchanges/configs/test_from_toml.rs similarity index 100% rename from atelier-data/test/configs/test_from_toml.rs rename to atelier-data/test/exchanges/configs/test_from_toml.rs diff --git a/atelier-data/test/exchanges/gate/test_orderbook.rs b/atelier-data/test/exchanges/gate/test_orderbook.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/test/exchanges/gate/test_trades.rs b/atelier-data/test/exchanges/gate/test_trades.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/test/exchanges/gemini/test_orderbook.rs b/atelier-data/test/exchanges/gemini/test_orderbook.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/test/exchanges/gemini/test_trades.rs b/atelier-data/test/exchanges/gemini/test_trades.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/test/exchanges/kraken/test_orderbook.rs b/atelier-data/test/exchanges/kraken/test_orderbook.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/test/exchanges/kraken/test_trades.rs b/atelier-data/test/exchanges/kraken/test_trades.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/test/exchanges/okx/test_orderbook.rs b/atelier-data/test/exchanges/okx/test_orderbook.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-data/test/exchanges/okx/test_trades.rs b/atelier-data/test/exchanges/okx/test_trades.rs new file mode 100644 index 0000000..e69de29 diff --git a/atelier-rs/README.md b/atelier-rs/README.md index 3f7eecd..4941ce7 100644 --- a/atelier-rs/README.md +++ b/atelier-rs/README.md @@ -1,5 +1,7 @@ # atelier-rs +(This create is the re-export from the other crates, and this content is exactly the same as the Repository's main [README.md](https://github.com/IteraLabs/atelier-rs/blob/main/README.md)) +
[![Crates.io][badge-crates]][url-crates] @@ -31,27 +33,31 @@ # Overview -Engine for High Frequency, Synthetic and Historical, Market Microstructure Modeling. At a high level it provides the following major components: +Engine for High Frequency, Synthetic and Historical, Market Microstructure Modeling for Centralized and Decentralized Exchanges and Protocols. At a high level it provides the following major components: -- Full orderbook granularity. +- Full Order book granularity (Sides -> Levels -> Orders). - Stochastic process functions for synthetic data generation. -- Distributed convex methods for model training/inference. +- Distributed convex methods for model fitting/training/inference. +- Graph structures, spectral analysis and tools. # Workspace These are the other published crates members of the workspace: -- [atelier-base](https://crates.io/crates/atelier-base): Core data structures and I/O tools. -- [atelier-dcml](https://crates.io/crates/atelier-dcml): Distributed Convex Machine Learning. +- [atelier-base](https://crates.io/crates/atelier-base): Core data structures for the atelier-rs engine. +- [atelier-data](https://crates.io/crates/atelier-data): Data I/O integrations for OnChain/OffChain Protocols for the atelier-rs engine. +- [atelier-dcml](https://crates.io/crates/atelier-dcml): Distributed Convex Machine Learning methods and tooling for the atelier-rs engine. - [atelier-synth](https://crates.io/crates/atelier-synth): Synthetic Data Generation for the atelier-rs engine. +- [atelier-graph](https://crates.io/crates/atelier-graph): Graph theory algorithms and tooling. +- [atelier-quant](https://crates.io/crates/atelier-quant): Intermediate to advance quantitative finance methods for the atelier-rs engine. Github hosted: -- [benches](https://github.com/IteraLabs/atelier-rs/tree/main/benches) -- [examples](https://github.com/IteraLabs/atelier-rs/tree/main/examples) -- [tests](https://github.com/IteraLabs/atelier-rs/tree/main/tests) +- [benches](https://github.com/IteraLabs/atelier-rs/tree/main/benches) : Official and Community benchmarks for the atelier-rs engine and/or individual/grouped sub-components of the engine. +- [examples](https://github.com/IteraLabs/atelier-rs/tree/main/examples): Official and Community gallery of implementations, and examples for the atelier-rs engine and/or individual/grouped sub-components of the engine. +- [research](https://github.com/IteraLabs/atelier-rs/tree/main/research): Official and community research with usage of the atelier-rs and/or individual/grouped sub-components.
--- -atelier-rs is the "entrypoint" for the [atelier-rs](https://github.com/IteraLabs/atelier-rs) workspace +atelier-rs is the "entrypoint" for the [atelier-rs](https://github.com/IteraLabs/atelier-rs) workspace. diff --git a/atelier-rs/src/lib.rs b/atelier-rs/src/lib.rs index 493798d..9b6ac7d 100644 --- a/atelier-rs/src/lib.rs +++ b/atelier-rs/src/lib.rs @@ -6,33 +6,30 @@ //! //! # Modules //! -//! This SDK is organized into the following modules: +//! This SDK is organized into the following modules: //! //! - `atelier-base`: Base data structures (Orderbook, Trades, Liquidations, etc). //! - `atelier-data`: Data I/O, External sources connectivity. //! - `atelier-dcml`: Distributed Convex Machine Learning. //! - `atelier-quant`: Quantitative Models. //! - `atelier-synth`: Synthetic Data Generation Proceadures. +//! - `atelier-graph`: Graph theory, algorithms and tooling. -pub use atelier_base::{ - orderbooks::Orderbook, Order, OrderType, OrderSide, Level -}; +pub use atelier_base::{Level, Order, OrderSide, OrderType, orderbooks::Orderbook}; pub use atelier_data::{ clients::wss::{WssClient, WssClientBuilder, WssDecoder}, errors::ExchangeError, - exchanges::bybit::{stream_data, BybitWssClient, LiquidationData} + exchanges::bybit::{BybitWssClient, LiquidationData, stream_data}, }; pub use atelier_synth::{ - probabilistic, synthbooks, agent, errors -}; - -pub use atelier_dcml::{ - features, math + agent::Agent, + errors::SynthetizerErrors, + probabilistic, + synthbooks, }; -pub use atelier_quant::{ - vpin, hawkes, brownian -}; +pub use atelier_dcml::{features, math}; +pub use atelier_quant::{brownian, hawkes, vpin}; diff --git a/atelier-synth/README.md b/atelier-synth/README.md index 60cfa5d..43b982a 100644 --- a/atelier-synth/README.md +++ b/atelier-synth/README.md @@ -32,17 +32,19 @@ prices to start the generation. These are the other published crates members of the workspace: -- [atelier-base](https://crates.io/crates/atelier-base): Core data structures and I/O tools. -- [atelier-dcml](https://crates.io/crates/atelier-dcml): Distributed Convex Machine Learning. -- [atelier-synth](https://crates.io/crates/atelier-synth): Synthetic Data Generation for the atelier-rs engine. +- [atelier-base](https://crates.io/crates/atelier-base): Core data structures for the atelier-rs engine. +- [atelier-data](https://crates.io/crates/atelier-data): Data I/O integrations for OnChain/OffChain Protocols for the atelier-rs engine. +- [atelier-dcml](https://crates.io/crates/atelier-dcml): Distributed Convex Machine Learning methods and tooling for the atelier-rs engine. +- [atelier-graph](https://crates.io/crates/atelier-graph): Graph theory algorithms and tooling. +- [atelier-quant](https://crates.io/crates/atelier-quant): Intermediate to advance quantitative finance methods for the atelier-rs engine. Github hosted: -- [benches](https://github.com/IteraLabs/atelier-rs/tree/main/benches) -- [examples](https://github.com/IteraLabs/atelier-rs/tree/main/examples) -- [tests](https://github.com/IteraLabs/atelier-rs/tree/main/tests) +- [benches](https://github.com/IteraLabs/atelier-rs/tree/main/benches) : Official and Community benchmarks for the atelier-rs engine and/or individual/grouped sub-components of the engine. +- [examples](https://github.com/IteraLabs/atelier-rs/tree/main/examples): Official and Community gallery of implementations, and examples for the atelier-rs engine and/or individual/grouped sub-components of the engine. +- [research](https://github.com/IteraLabs/atelier-rs/tree/main/research): Official and community research with usage of the atelier-rs and/or individual/grouped sub-components.
--- -atelier-synth is a member of the [atelier-rs](https://github.com/iteralabs/atelier-rs) workspace +atelier-synth is a member of the [atelier-rs](https://github.com/iteralabs/atelier-rs) workspace. diff --git a/atelier-synth/src/errors.rs b/atelier-synth/src/errors.rs index 5d76e7b..8bdeb67 100644 --- a/atelier-synth/src/errors.rs +++ b/atelier-synth/src/errors.rs @@ -1,76 +1,7 @@ use thiserror::Error; -#[derive(Error, Debug, Clone)] -pub enum LevelError { - // Level not found - #[error("Level not found")] - LevelNotFound, - - // Level info not available - #[error("Level info not available")] - LevelInfoNotAvailable, - - // Level deletion not successful - #[error("Level deletion not successful")] - LevelDeletionFailed, - - // Level modification not succesful - #[error("Level modification not successful")] - LevelModificationFailed, - - // Level insertion not successful - #[error("Level insertion not successful")] - LevelInsertionFailed, -} - -#[derive(Error, Debug)] -pub enum OrderError { - // Order not found - #[error("Order not found")] - OrderNotFound, - - // Order info not available - #[error("Order info not available")] - OrderInfoNotAvailable, - - // Order deletion not successful - #[error("Order deletion not successful")] - OrderDeletionFailed, - - // Order modification not successful - #[error("Order modification not successful")] - OrderModificationFailed, - - // Order insertion not succesful - #[error("Order insertion not successful")] - OrderInsertionFailed, -} - -#[derive(Error, Debug)] -pub enum GeneratorError { - // Undefined Generator Error - #[error("The Generator presented an Undefined Error")] - GeneratorUndefinedError, - - // Not a valid number on the input - #[error("The Generator did not recived a valid number")] - GeneratorInputTypeFailure, - - // Not a valid number on the output - #[error("The Generator did not produced a valid number")] - GeneratorOutputTypeFailure, -} - -#[derive(Error, Debug)] -pub enum EventError { - // Event generator failed - #[error("The Event Generator function failed")] - EventFailure, -} - #[derive(Error, Debug)] -pub enum SynthetizerError { - // Progression Generation Error +pub enum SynthetizerErrors { #[error("The progression generation was unsuccessful {0}")] GenerationError(String), } diff --git a/atelier-synth/src/synthbooks.rs b/atelier-synth/src/synthbooks.rs index 10696f5..12aea71 100644 --- a/atelier-synth/src/synthbooks.rs +++ b/atelier-synth/src/synthbooks.rs @@ -230,7 +230,7 @@ pub async fn async_progressions( orderbook_templates: Vec, model_templates: Vec, n_progres: usize, -) -> Result>, errors::SynthetizerError> { +) -> Result>, errors::SynthetizerErrors> { let tasks: Vec<_> = orderbook_templates .into_iter() .zip(model_templates.into_iter()) @@ -239,7 +239,7 @@ pub async fn async_progressions( progressions(&ob_config, &template_model, n_progres).unwrap() }) .await - .map_err(|e| errors::SynthetizerError::GenerationError(e.to_string())) + .map_err(|e| errors::SynthetizerErrors::GenerationError(e.to_string())) }) .collect(); diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 58d4260..33d6635 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -29,7 +29,7 @@ private-doc = true [dependencies] -atelier_base = { path = "../atelier-base", version = "0.0.11" } +atelier_base = { version = "0.0.11" } rand = { version = "0.9.0" } criterion = { version = "0.5", features = ["html_reports"] } diff --git a/datasets/README.md b/datasets/README.md new file mode 100644 index 0000000..3adb1ae --- /dev/null +++ b/datasets/README.md @@ -0,0 +1,38 @@ +# Datasets + +A series of small/brief datasets used as reference in the examples/, or to be used for your own dev-test cycle. + +There are two types of supported logics for the synthetizer: + +- **Model-driven progressions**: From a state definition, data is generated by models. +- **Agent-driven progressions**: From an agent definition, data is generated by inter-agents interactions. + +## Independent Synthetic Order books + +- **Model-Driven**: From the best bid and ask prices, at the top-of-book, the deeper levels are generated outwards by the model(s) of choice. + +Upcoming ... + +- **Agent-Driven**: From a set of agent definitions, for agents >> 2, the order book is generated as the consequence of agents behavior and a matching engine implementation. + +- Deterministic Trend + Random White Noise + +Both best-bid and best-ask prices increase 0.01% every innovation of the order book, everything else, as mentioned below, is a Uniformly distributed random choice. + +- Generator logic: Order book snapshot (generates TOB first, then expand towards full order book) +- Range logic: [start, end, step] +- Seed: 1 +- No. of levels [5, 10, 1]. +- No. of orders per level [20, 40, 1]. +- Amount for each order [0.01, 1.0, 0.01]. + +## Independent Synthetic Public Trades + +- **Model-Driven**: From a defined market price, trades are generated in time by the model(s) of choice. + +- **Agent-Driven**: From a set of agent definitions, for agents >> 2, the order book is generated as the consequence of agents behavior and a matching engine implementation. + +## Full Synthetic Market + +- Order books generated by agents and public trades as results of interactions of the orders, as processed by the matching engine. + diff --git a/examples/README.md b/examples/README.md index 9eabc23..61c88b9 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,34 +1,35 @@ -# examples - -# Overview +# Examples Research and implementation examples for the atelier-rs engine -# General Layout +# Quick start -## distributed/singular_case +## Order books -You can take this as a benchmark for more advanced implementations. +- basic order: [examples/quickstart/basic_orders.rs](https://github.com/iteralabs/atelier-rs/examples/quickstart/basic_order.rs) +- basic order book: [examples/quickstart/basic_ob.rs]("https://github.com/iteralabs/atelier-rs/examples/quickstart/basic_ob.rs") -- Single-threaded synthetic Orderbook data generation. -- Single-threaded orderbook features generation. -- Single-threaded batch training using gradient descent. +# Integrations -# Workspace +- bybit wss: +- helius gRPC: +- coinbase wss: +- kraken wss: +- binance wss: -These are the other published crates members of the workspace: +# Distributed -- [atelier-base](https://crates.io/crates/atelier-base): Core data structures and I/O tools. -- [atelier-dcml](https://crates.io/crates/atelier-dcml): Distributed Convex Machine Learning. -- [atelier-synth](https://crates.io/crates/atelier-synth): Synthetic Data Generation for the atelier-rs engine. +- Case 1 (Benchmark) : 1 model learning from 1 CEX. +- Case 3 (Independent) : 3 models learning from 3 CEXes across 3 regions in equal importance topology. +- Case 9 (Leader-Follower) : 9 models learning from 9 CEXs across 3 regions in leader-follower topology. -Github hosted: +# Synthetize -- [benches](https://github.com/IteraLabs/atelier-rs/tree/main/benches) -- [examples](https://github.com/IteraLabs/atelier-rs/tree/main/examples) -- [tests](https://github.com/IteraLabs/atelier-rs/tree/main/tests) +- generators/custom.rs: Create a custom probabilistic random number generator. +- synthetize/ob_progressions.rs: Create a series of synthetic orderbooks from a template.
+If you would like to contribute, please consider reading the [CONTRIBUTE.md](https://github.com/IteraLabs/atelier-rs/blob/main/CONTRIBUTE.md) guide. --- -examples is a member of the [atelier-rs](https://github.com/iteralabs/atelier-rs) workspace +This content is part of *examples*, which is a crate member of the [atelier-rs](https://github.com/iteralabs/atelier-rs) workspace. diff --git a/examples/distributed/case_1/README.md b/examples/distributed/case_1/README.md new file mode 100644 index 0000000..e69de29 diff --git a/examples/distributed/singular_case/singular_config.toml b/examples/distributed/case_1/config.toml similarity index 100% rename from examples/distributed/singular_case/singular_config.toml rename to examples/distributed/case_1/config.toml diff --git a/examples/distributed/singular_case/data/singular_case_data.csv b/examples/distributed/case_1/files/data/singular_case_data.csv similarity index 100% rename from examples/distributed/singular_case/data/singular_case_data.csv rename to examples/distributed/case_1/files/data/singular_case_data.csv diff --git a/examples/distributed/singular_case/data/singular_case_ob.json b/examples/distributed/case_1/files/data/singular_case_ob.json similarity index 100% rename from examples/distributed/singular_case/data/singular_case_ob.json rename to examples/distributed/case_1/files/data/singular_case_ob.json diff --git a/examples/distributed/singular_case/singular_data.rs b/examples/distributed/case_1/synth_data.rs similarity index 100% rename from examples/distributed/singular_case/singular_data.rs rename to examples/distributed/case_1/synth_data.rs diff --git a/examples/distributed/singular_case/singular_training.rs b/examples/distributed/case_1/train.rs similarity index 100% rename from examples/distributed/singular_case/singular_training.rs rename to examples/distributed/case_1/train.rs diff --git a/examples/distributed/case_3/README.md b/examples/distributed/case_3/README.md index b33dac0..03f4896 100644 --- a/examples/distributed/case_3/README.md +++ b/examples/distributed/case_3/README.md @@ -1,6 +1,6 @@ -# case_3 +# Distributed ML (3 Models) -3 different exchanges across 3 different regions, each considered a high-volume, high-activity Centralized Exchange, but generating data of the same market Btc/Usdt. +Uses 3 different exchanges across 3 different regions, each considered a high-volume, high-activity Centralized Exchange, but generating data of the same market *SOL/USDc*. # Modeling Process diff --git a/examples/distributed/case_3/files/case_3_ai_00_data.csv b/examples/distributed/case_3/files/data/case_3_ai_00_data.csv similarity index 100% rename from examples/distributed/case_3/files/case_3_ai_00_data.csv rename to examples/distributed/case_3/files/data/case_3_ai_00_data.csv diff --git a/examples/distributed/case_3/files/case_3_ai_00_ob.json b/examples/distributed/case_3/files/data/case_3_ai_00_ob.json similarity index 100% rename from examples/distributed/case_3/files/case_3_ai_00_ob.json rename to examples/distributed/case_3/files/data/case_3_ai_00_ob.json diff --git a/examples/distributed/case_3/files/case_3_am_00_data.csv b/examples/distributed/case_3/files/data/case_3_am_00_data.csv similarity index 100% rename from examples/distributed/case_3/files/case_3_am_00_data.csv rename to examples/distributed/case_3/files/data/case_3_am_00_data.csv diff --git a/examples/distributed/case_3/files/case_3_am_00_ob.json b/examples/distributed/case_3/files/data/case_3_am_00_ob.json similarity index 100% rename from examples/distributed/case_3/files/case_3_am_00_ob.json rename to examples/distributed/case_3/files/data/case_3_am_00_ob.json diff --git a/examples/distributed/case_3/files/case_3_eu_00_data.csv b/examples/distributed/case_3/files/data/case_3_eu_00_data.csv similarity index 100% rename from examples/distributed/case_3/files/case_3_eu_00_data.csv rename to examples/distributed/case_3/files/data/case_3_eu_00_data.csv diff --git a/examples/distributed/case_3/files/case_3_eu_00_ob.json b/examples/distributed/case_3/files/data/case_3_eu_00_ob.json similarity index 100% rename from examples/distributed/case_3/files/case_3_eu_00_ob.json rename to examples/distributed/case_3/files/data/case_3_eu_00_ob.json diff --git a/examples/distributed/case_3/train.rs b/examples/distributed/case_3/train.rs index e135f49..2ee8d14 100644 --- a/examples/distributed/case_3/train.rs +++ b/examples/distributed/case_3/train.rs @@ -43,10 +43,13 @@ pub fn main() { .join("distributed") .join("case_3") .join("files") + .join("data") .to_str() .unwrap() .to_owned() - + "/case_3_" + v_datafiles[i_case] + "_data.csv"; + + "/case_3_" + + v_datafiles[i_case] + + "_data.csv"; let header = true; let column_types = None; @@ -155,6 +158,4 @@ pub fn main() { // Train the distributed system let _ = distributed_trainer.train(10); - } - diff --git a/examples/distributed/case_9/README.md b/examples/distributed/case_9/README.md new file mode 100644 index 0000000..e69de29 diff --git a/examples/distributed/case_9/config_c.toml b/examples/distributed/case_9/config.toml similarity index 100% rename from examples/distributed/case_9/config_c.toml rename to examples/distributed/case_9/config.toml diff --git a/examples/distributed/case_9/synth_data.rs b/examples/distributed/case_9/synth_data.rs new file mode 100644 index 0000000..e69de29 diff --git a/examples/distributed/case_9/topology_c.toml b/examples/distributed/case_9/topology.toml similarity index 100% rename from examples/distributed/case_9/topology_c.toml rename to examples/distributed/case_9/topology.toml diff --git a/examples/distributed/templates/test_temp_train_00.toml b/examples/distributed/templates/test_temp_train_00.toml index f5ebd26..2f04bae 100644 --- a/examples/distributed/templates/test_temp_train_00.toml +++ b/examples/distributed/templates/test_temp_train_00.toml @@ -9,4 +9,3 @@ label = "GD" description = "Gradient Descent" params_labels = ["learning_rate", "epsilon"] params_values = [0.1, 0.001] - diff --git a/examples/quickstart/basic_ob_progressions.rs b/examples/quickstart/basic_ob_progressions.rs deleted file mode 100644 index bf13193..0000000 --- a/examples/quickstart/basic_ob_progressions.rs +++ /dev/null @@ -1,74 +0,0 @@ -use atelier_base::orderbooks::Orderbook; -use rand::Rng; -use rand_distr::{Bernoulli, Distribution, Uniform}; - -fn main() { - // -- Orderbook parameters -- // - let ini_bid_price = 100_000.00; - let ini_bid_levels = Some((1, 2)); - let ini_bid_orders = Some((1, 10)); - - let ini_ask_price = 100_001.00; - let ini_ask_levels = Some((1, 2)); - let ini_ask_orders = Some((1, 10)); - - let ini_ticksize = Some((0.1, 1.1)); - - let mut v_orderbook = vec![]; - let n_progressions = 10; - - // -- Probabilistic parameters -- // - - let uni_params = [0.001, 0.005]; - let ber_params = [0.5]; - - let mut rng = rand::rng(); - - // ------------------------------------------------------------------------------- // - for _ in 0..n_progressions { - let uni_rand = Uniform::new(uni_params[0], uni_params[1]) - .expect("Failed to create Uniform distribution sampler"); - let r_amount_ret = rng.sample(uni_rand); - - let bernoulli = Bernoulli::new(ber_params[0]).unwrap(); - let r_sign_ret = if bernoulli.sample(&mut rng) { - 1.0 - } else { - -1.0 - }; - - let v_bid_price = ini_bid_price + ini_bid_price * r_amount_ret * r_sign_ret; - let v_ask_price = ini_ask_price + ini_ask_price * r_amount_ret * r_sign_ret; - - let r_ob = Orderbook::random( - None, - v_bid_price, - ini_bid_levels, - ini_bid_orders, - ini_ticksize, - v_ask_price, - ini_ask_levels, - ini_ask_orders, - ); - - v_orderbook.push(r_ob); - } - - println!("\nNumber of progressions: {n_progressions:?}\n"); - - println!( - "\nfirst 4 bid prices: {:?}, {:?}, {:?}, {:?}", - v_orderbook[0].bids[0].price, - v_orderbook[1].bids[0].price, - v_orderbook[2].bids[0].price, - v_orderbook[3].bids[0].price - ); - - println!( - "\nfirst 4 ask prices: {:?}, {:?}, {:?}, {:?}", - v_orderbook[0].asks[0].price, - v_orderbook[1].asks[0].price, - v_orderbook[2].asks[0].price, - v_orderbook[3].asks[0].price - ); -} diff --git a/examples/advanced/README.md b/examples/synthetize/README.md similarity index 100% rename from examples/advanced/README.md rename to examples/synthetize/README.md diff --git a/examples/synthetize/generators/custom.rs b/examples/synthetize/generators/custom.rs new file mode 100644 index 0000000..e69de29 diff --git a/examples/advanced/synthetize/synthetize_ob.rs b/examples/synthetize/synthetize/ob_progressions.rs similarity index 100% rename from examples/advanced/synthetize/synthetize_ob.rs rename to examples/synthetize/synthetize/ob_progressions.rs diff --git a/research/ADA2025/README.md b/research/ADA2025/README.md new file mode 100644 index 0000000..625bca2 --- /dev/null +++ b/research/ADA2025/README.md @@ -0,0 +1,20 @@ +# Algorithms Design and Analysis + +Title: Observable Patterns on Block Transactions and Validator Nodes on the Solana Blockchain. + +## Introduction + +The Solana blockchain is one of the most high-performant distributed processing engines amongst all permissionless Layer 1 blockchains in existence today. + +## Problem Statement + +## Data & Methods + +## Results + +## Analysis and Conclusions + +## Future Work + +## References + diff --git a/research/CFE2025/README.md b/research/CFE2025/README.md new file mode 100644 index 0000000..23e1ff1 --- /dev/null +++ b/research/CFE2025/README.md @@ -0,0 +1,5 @@ +# CFE-2025 + +Computational and Financial Econometrics Conference 2025, London, UK. + + diff --git a/research/README.md b/research/README.md new file mode 100644 index 0000000..e69de29 From 74dce391f881a6b47a29b4ea8d6d4fbc1f6b542d Mon Sep 17 00:00:00 2001 From: IFFranciscoME Date: Tue, 14 Oct 2025 23:14:29 -0600 Subject: [PATCH 2/2] Major refactor, part.1 --- Cargo.toml | 5 +- atelier-data/Cargo.toml | 52 +++++- .../src/exchanges/binance/binance_client.rs | 76 ++------ atelier-data/src/exchanges/binance/mod.rs | 3 + .../src/exchanges/binance/responses/misc.rs | 19 ++ .../binance/{resposes => responses}/mod.rs | 4 +- .../exchanges/binance/responses/orderbook.rs | 11 ++ .../ticker.rs => responses/symbols.rs} | 14 +- .../binance/{resposes => responses}/trades.rs | 0 .../src/exchanges/coinbase/coinbase_client.rs | 10 +- atelier-data/src/exchanges/coinbase/mod.rs | 2 + .../responses/misc.rs} | 0 .../src/exchanges/coinbase/responses/mod.rs | 4 +- .../responses/{product.rs => symbols.rs} | 0 .../src/exchanges/kraken/kraken_client.rs | 166 +++-------------- atelier-data/src/exchanges/kraken/mod.rs | 3 + .../src/exchanges/kraken/responses/misc.rs | 52 ++++++ .../src/exchanges/kraken/responses/mod.rs | 4 + .../exchanges/kraken/responses/orderbook.rs | 23 +++ .../src/exchanges/kraken/responses/symbols.rs | 38 ++++ atelier-data/src/exchanges/mod.rs | 1 + .../test/exchanges/binance/test_connection.rs | 29 +++ .../test/exchanges/binance/test_orderbook.rs | 17 ++ .../test/exchanges/binance/test_symbols.rs | 12 ++ .../test/exchanges/binance/test_trades.rs | 12 ++ .../exchanges/coinbase/test_connection.rs | 12 ++ .../test/exchanges/coinbase/test_orderbook.rs | 33 +--- .../{test_products.rs => test_symbols.rs} | 6 +- .../test/exchanges/coinbase/test_trades.rs | 12 ++ .../test/exchanges/kraken/test_connection.rs | 12 ++ .../test/exchanges/kraken/test_orderbook.rs | 17 ++ .../test/exchanges/kraken/test_symbols.rs | 11 ++ .../test/exchanges/kraken/test_trades.rs | 11 ++ atelier-rs/Cargo.toml | 1 + atelier-rs/database/clickhouse/config.xml | 0 .../database/clickhouse/init-ob-schema.sql | 20 ++ atelier-rs/database/clickhouse/users.xml | 59 ++++++ atelier-rs/database/database.Dockerfile | 42 +++++ atelier-rs/database/database_entrypoint.sh | 63 +++++++ .../datacollector/datacollector.Dockerfile | 41 ++++ .../datacollector/datacollector_config.toml | 49 +++++ atelier-rs/src/bin/datacollector.rs | 175 ++++++++++++++++++ atelier-rs/src/lib.rs | 1 - atelier-rs/src/queries/create_table.sql | 18 ++ atelier-rs/src/queries/mod.rs | 3 + atelier-rs/src/queries/read_table.sql | 0 atelier-rs/src/queries/write_table.sql | 0 atelier-synth/Cargo.toml | 1 + 48 files changed, 896 insertions(+), 248 deletions(-) create mode 100644 atelier-data/src/exchanges/binance/responses/misc.rs rename atelier-data/src/exchanges/binance/{resposes => responses}/mod.rs (53%) create mode 100644 atelier-data/src/exchanges/binance/responses/orderbook.rs rename atelier-data/src/exchanges/binance/{resposes/ticker.rs => responses/symbols.rs} (68%) rename atelier-data/src/exchanges/binance/{resposes => responses}/trades.rs (100%) rename atelier-data/src/exchanges/{binance/resposes/orderbook.rs => coinbase/responses/misc.rs} (100%) rename atelier-data/src/exchanges/coinbase/responses/{product.rs => symbols.rs} (100%) create mode 100644 atelier-data/src/exchanges/kraken/responses/misc.rs create mode 100644 atelier-data/src/exchanges/kraken/responses/symbols.rs create mode 100644 atelier-data/test/exchanges/binance/test_connection.rs create mode 100644 atelier-data/test/exchanges/binance/test_symbols.rs create mode 100644 atelier-data/test/exchanges/coinbase/test_connection.rs rename atelier-data/test/exchanges/coinbase/{test_products.rs => test_symbols.rs} (79%) create mode 100644 atelier-data/test/exchanges/kraken/test_connection.rs create mode 100644 atelier-data/test/exchanges/kraken/test_symbols.rs create mode 100644 atelier-rs/database/clickhouse/config.xml create mode 100644 atelier-rs/database/clickhouse/init-ob-schema.sql create mode 100644 atelier-rs/database/clickhouse/users.xml create mode 100644 atelier-rs/database/database.Dockerfile create mode 100644 atelier-rs/database/database_entrypoint.sh create mode 100644 atelier-rs/datacollector/datacollector.Dockerfile create mode 100644 atelier-rs/datacollector/datacollector_config.toml create mode 100644 atelier-rs/src/bin/datacollector.rs create mode 100644 atelier-rs/src/queries/create_table.sql create mode 100644 atelier-rs/src/queries/mod.rs create mode 100644 atelier-rs/src/queries/read_table.sql create mode 100644 atelier-rs/src/queries/write_table.sql diff --git a/Cargo.toml b/Cargo.toml index 649606f..aabfd99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,13 +38,14 @@ license = "Apache-2.0" [workspace.dependencies] -atelier_data = { version = "0.0.10" } -atelier_base = { version = "0.0.12" } +atelier_data = { version = "0.0.12" } +atelier_base = { version = "0.0.11" } atelier_dcml = { version = "0.0.11" } atelier_quant = { version = "0.0.10" } atelier_rs = { version = "0.0.10" } atelier_synth = { version = "0.0.10" } +anyhow = { version = "1.0" } criterion = { version = "0.5", features = ["html_reports"] } csv = { version = "1.3" } clap = { version = "4.5", features = ["derive"] } diff --git a/atelier-data/Cargo.toml b/atelier-data/Cargo.toml index 09a7bcc..251fa16 100644 --- a/atelier-data/Cargo.toml +++ b/atelier-data/Cargo.toml @@ -60,13 +60,59 @@ uuid = { version = "1.0", features = ["v4"] } yellowstone-grpc-client = { version = "1.13.0" } yellowstone-grpc-proto = { version = "1.13.0" } +# --- Coinbase --- # + +[[test]] +name = "coinbase_http_connection" +path = "test/exchanges/coinbase/test_connection.rs" + [[test]] name = "coinbase_http_orderbook" path = "test/exchanges/coinbase/test_orderbook.rs" -[[example]] -name = "bybit_streams" -path = "examples/bybit_streams.rs" +[[test]] +name = "coinbase_http_trades" +path = "test/exchanges/coinbase/test_trades.rs" + +[[test]] +name = "coinbase_http_symbols" +path = "test/exchanges/coinbase/test_symbols.rs" + +# --- Binance --- # + +[[test]] +name = "binance_http_connection" +path = "test/exchanges/binance/test_connection.rs" + +[[test]] +name = "binance_http_orderbook" +path = "test/exchanges/binance/test_orderbook.rs" + +[[test]] +name = "binance_http_trades" +path = "test/exchanges/binance/test_trades.rs" + +[[test]] +name = "binance_http_symbols" +path = "test/exchanges/binance/test_symbols.rs" + +# --- Kraken --- # + +[[test]] +name = "kraken_http_connection" +path = "test/exchanges/kraken/test_connection.rs" + +[[test]] +name = "kraken_http_orderbook" +path = "test/exchanges/kraken/test_orderbook.rs" + +[[test]] +name = "kraken_http_trades" +path = "test/exchanges/kraken/test_trades.rs" + +[[test]] +name = "kraken_http_symbols" +path = "test/exchanges/kraken/test_symbols.rs" [lints.rust] unsafe_code = "forbid" diff --git a/atelier-data/src/exchanges/binance/binance_client.rs b/atelier-data/src/exchanges/binance/binance_client.rs index fa27147..e988977 100644 --- a/atelier-data/src/exchanges/binance/binance_client.rs +++ b/atelier-data/src/exchanges/binance/binance_client.rs @@ -1,8 +1,16 @@ -use crate::client::http_client::{HttpClient, RetryConfig, RetryableHttpClient}; -use crate::models::orderbook::{Orderbook, PriceLevel, TradingPair}; +use crate::{ + clients::http::http_client::{HttpClient, RetryConfig, RetryableHttpClient}, + exchanges::{ + binance::responses, + errors::{ExchangeError, Result}, + }, + models::{ + orderbook::{Orderbook, PriceLevel}, + pairs::TradingPair, + }, +}; + use chrono::Utc; -use ix_results::errors::{ExchangeError, Result}; -use serde::Deserialize; use std::str::FromStr; use tracing::{debug, info, warn}; @@ -45,7 +53,7 @@ impl BinanceClient { let params = vec![("symbol", symbol.as_str()), ("limit", depth_str.as_str())]; - let response: BinanceDepthResponse = self + let response: responses::orderbook::BinanceDepthResponse = self .client .get_with_params_retry("/api/v3/depth", ¶ms) .await?; @@ -62,7 +70,7 @@ impl BinanceClient { /// Convert Binance response to our OrderBook format fn convert_to_orderbook( &self, - response: BinanceDepthResponse, + response: responses::orderbook::BinanceDepthResponse, symbol: String, ) -> Result { let mut v_bids = Vec::new(); @@ -147,64 +155,10 @@ impl BinanceClient { Ok(orderbook) } - /// Get exchange information (available symbols, etc.) - pub async fn get_exchange_info(&self) -> Result { - info!("Fetching Binance exchange information"); - - self.client.get_with_retry("/api/v3/exchangeInfo").await - } - /// Get server time (useful for synchronization) - pub async fn get_server_time(&self) -> Result { + pub async fn get_server_time(&self) -> Result { self.client.get_with_retry("/api/v3/time").await } - - /// Get 24hr ticker statistics - pub async fn get_24hr_ticker(&self, pair: TradingPair) -> Result { - let symbol = pair.to_exchange_symbol("binance"); - let params = vec![("symbol", symbol.as_str())]; - - self.client - .get_with_params_retry("/api/v3/ticker/24hr", ¶ms) - .await - } -} - -/// Binance depth/orderbook response format -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct BinanceDepthResponse { - last_update_id: u64, - bids: Vec>, // [price, quantity] pairs - asks: Vec>, // [price, quantity] pairs -} - -/// Binance exchange info response -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct BinanceExchangeInfo { - pub timezone: String, - pub server_time: u64, - pub symbols: Vec, -} - -/// Binance symbol information -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct BinanceSymbol { - pub symbol: String, - pub status: String, - pub base_asset: String, - pub quote_asset: String, - pub base_asset_precision: u32, - pub quote_precision: u32, -} - -/// Binance server time response -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct BinanceServerTime { - pub server_time: u64, } impl Default for BinanceClient { diff --git a/atelier-data/src/exchanges/binance/mod.rs b/atelier-data/src/exchanges/binance/mod.rs index f38ee2a..bab9a79 100644 --- a/atelier-data/src/exchanges/binance/mod.rs +++ b/atelier-data/src/exchanges/binance/mod.rs @@ -1 +1,4 @@ pub mod binance_client; +pub mod responses; + +pub use binance_client::BinanceClient; diff --git a/atelier-data/src/exchanges/binance/responses/misc.rs b/atelier-data/src/exchanges/binance/responses/misc.rs new file mode 100644 index 0000000..571c04f --- /dev/null +++ b/atelier-data/src/exchanges/binance/responses/misc.rs @@ -0,0 +1,19 @@ +use serde::Deserialize; +use crate::exchanges::binance::responses::symbols; + +/// Binance exchange info response +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BinanceExchangeInfo { + pub timezone: String, + pub server_time: u64, + pub symbols: Vec, +} + +/// Binance server time response +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BinanceServerTime { + pub server_time: u64, +} + diff --git a/atelier-data/src/exchanges/binance/resposes/mod.rs b/atelier-data/src/exchanges/binance/responses/mod.rs similarity index 53% rename from atelier-data/src/exchanges/binance/resposes/mod.rs rename to atelier-data/src/exchanges/binance/responses/mod.rs index 44b3830..f8d4b19 100644 --- a/atelier-data/src/exchanges/binance/resposes/mod.rs +++ b/atelier-data/src/exchanges/binance/responses/mod.rs @@ -1,4 +1,4 @@ -pub mod ticker; +pub mod misc; pub mod orderbook; +pub mod symbols; pub mod trades; - diff --git a/atelier-data/src/exchanges/binance/responses/orderbook.rs b/atelier-data/src/exchanges/binance/responses/orderbook.rs new file mode 100644 index 0000000..9a37712 --- /dev/null +++ b/atelier-data/src/exchanges/binance/responses/orderbook.rs @@ -0,0 +1,11 @@ +use serde::Deserialize; + +/// Binance depth/orderbook response format +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BinanceDepthResponse { + pub last_update_id: u64, + pub bids: Vec>, // [price, quantity] pairs + pub asks: Vec>, // [price, quantity] pairs +} + diff --git a/atelier-data/src/exchanges/binance/resposes/ticker.rs b/atelier-data/src/exchanges/binance/responses/symbols.rs similarity index 68% rename from atelier-data/src/exchanges/binance/resposes/ticker.rs rename to atelier-data/src/exchanges/binance/responses/symbols.rs index 0803534..19cf768 100644 --- a/atelier-data/src/exchanges/binance/resposes/ticker.rs +++ b/atelier-data/src/exchanges/binance/responses/symbols.rs @@ -1,4 +1,4 @@ -use serde; +use serde::Deserialize; /// Binance 24hr ticker response #[derive(Debug, Deserialize)] @@ -27,3 +27,15 @@ pub struct Binance24hrTicker { pub count: u64, } +/// Binance symbol information +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BinanceSymbol { + pub symbol: String, + pub status: String, + pub base_asset: String, + pub quote_asset: String, + pub base_asset_precision: u32, + pub quote_precision: u32, +} + diff --git a/atelier-data/src/exchanges/binance/resposes/trades.rs b/atelier-data/src/exchanges/binance/responses/trades.rs similarity index 100% rename from atelier-data/src/exchanges/binance/resposes/trades.rs rename to atelier-data/src/exchanges/binance/responses/trades.rs diff --git a/atelier-data/src/exchanges/coinbase/coinbase_client.rs b/atelier-data/src/exchanges/coinbase/coinbase_client.rs index be011fc..f878463 100644 --- a/atelier-data/src/exchanges/coinbase/coinbase_client.rs +++ b/atelier-data/src/exchanges/coinbase/coinbase_client.rs @@ -142,10 +142,10 @@ impl CoinbaseClient { } /// Get all products (trading pairs) - pub async fn get_products(&self) -> Result> { + pub async fn get_symbols(&self) -> Result> { info!("Fetching Coinbase products"); - let response: responses::product::CoinbaseProductsResponse = self + let response: responses::symbols::CoinbaseProductsResponse = self .client .get_with_retry("/api/v3/brokerage/products") .await?; @@ -154,10 +154,10 @@ impl CoinbaseClient { } /// Get specific product information - pub async fn get_product( + pub async fn get_symbol( &self, product_id: &str, - ) -> Result { + ) -> Result { info!("Fetching Coinbase product info for {}", product_id); let endpoint = format!("/api/v3/brokerage/products/{product_id}"); @@ -165,7 +165,7 @@ impl CoinbaseClient { } /// Get market trades - pub async fn get_market_trades( + pub async fn get_public_trades( &self, product_id: &str, depth: Option, diff --git a/atelier-data/src/exchanges/coinbase/mod.rs b/atelier-data/src/exchanges/coinbase/mod.rs index 27000db..ccb7441 100644 --- a/atelier-data/src/exchanges/coinbase/mod.rs +++ b/atelier-data/src/exchanges/coinbase/mod.rs @@ -1,2 +1,4 @@ pub mod coinbase_client; pub mod responses; + +pub use coinbase_client::CoinbaseClient; diff --git a/atelier-data/src/exchanges/binance/resposes/orderbook.rs b/atelier-data/src/exchanges/coinbase/responses/misc.rs similarity index 100% rename from atelier-data/src/exchanges/binance/resposes/orderbook.rs rename to atelier-data/src/exchanges/coinbase/responses/misc.rs diff --git a/atelier-data/src/exchanges/coinbase/responses/mod.rs b/atelier-data/src/exchanges/coinbase/responses/mod.rs index 659b0ae..f8d4b19 100644 --- a/atelier-data/src/exchanges/coinbase/responses/mod.rs +++ b/atelier-data/src/exchanges/coinbase/responses/mod.rs @@ -1,4 +1,4 @@ +pub mod misc; pub mod orderbook; +pub mod symbols; pub mod trades; -pub mod product; - diff --git a/atelier-data/src/exchanges/coinbase/responses/product.rs b/atelier-data/src/exchanges/coinbase/responses/symbols.rs similarity index 100% rename from atelier-data/src/exchanges/coinbase/responses/product.rs rename to atelier-data/src/exchanges/coinbase/responses/symbols.rs diff --git a/atelier-data/src/exchanges/kraken/kraken_client.rs b/atelier-data/src/exchanges/kraken/kraken_client.rs index fdc71a4..e14630e 100644 --- a/atelier-data/src/exchanges/kraken/kraken_client.rs +++ b/atelier-data/src/exchanges/kraken/kraken_client.rs @@ -1,9 +1,18 @@ -use crate::client::http_client::{HttpClient, RetryConfig, RetryableHttpClient}; -use crate::models::orderbook::{Orderbook, PriceLevel, TradingPair}; +use crate::{ + clients::http::http_client::{HttpClient, RetryConfig, RetryableHttpClient}, + exchanges::{ + errors::{ExchangeError, Result}, + kraken::responses, + }, +}; + +// use crate::client::http_client::{HttpClient, RetryConfig, RetryableHttpClient}; +use crate::models::{ + orderbook::{Orderbook, PriceLevel}, + pairs::TradingPair +}; use chrono::Utc; -use ix_results::errors::{ExchangeError, Result}; -use serde::Deserialize; use std::collections::HashMap; use std::str::FromStr; use tracing::{debug, info}; @@ -52,7 +61,7 @@ impl KrakenClient { // debug!("Params: {:?}", params_ref); - let response: KrakenDepthResponse = self + let response: responses::orderbook::KrakenDepthResponse = self .client .get_with_params_retry("/0/public/Depth", ¶ms_ref) .await?; @@ -87,7 +96,7 @@ impl KrakenClient { /// Convert Kraken response to our OrderBook format fn convert_to_orderbook( &self, - data: KrakenOrderbookData, + data: responses::orderbook::KrakenOrderbookData, symbol: String, ) -> Result { let mut ob_ts: u64 = 0; @@ -167,10 +176,10 @@ impl KrakenClient { } /// Get server time - pub async fn get_server_time(&self) -> Result { + pub async fn get_server_time(&self) -> Result { debug!("Fetching Kraken server time"); - let response: KrakenServerTimeResponse = + let response: responses::misc::KrakenServerTimeResponse = self.client.get_with_retry("/0/public/Time").await?; if !response.error.is_empty() { @@ -184,10 +193,10 @@ impl KrakenClient { } /// Get system status - pub async fn get_system_status(&self) -> Result { + pub async fn get_system_status(&self) -> Result { info!("Fetching Kraken system status"); - let response: KrakenSystemStatusResponse = + let response: responses::misc::KrakenSystemStatusResponse = self.client.get_with_retry("/0/public/SystemStatus").await?; if !response.error.is_empty() { @@ -201,10 +210,10 @@ impl KrakenClient { } /// Get asset pairs information - pub async fn get_asset_pairs(&self) -> Result> { + pub async fn get_symbols(&self) -> Result> { println!("Fetching Kraken asset pairs"); - let response: KrakenAssetPairsResponse = + let response: responses::symbols::KrakenAssetPairsResponse = self.client.get_with_retry("/0/public/AssetPairs").await?; if !response.error.is_empty() { @@ -217,138 +226,5 @@ impl KrakenClient { Ok(response.result) } - /// Get ticker information - pub async fn get_ticker( - &self, - pair: TradingPair, - ) -> Result> { - let pair_name = pair.to_exchange_symbol("kraken"); - info!("Fetching Kraken ticker for {}", pair_name); - - let params = vec![("pair", pair_name.as_str())]; - let response: KrakenTickerResponse = self - .client - .get_with_params_retry("/0/public/Ticker", ¶ms) - .await?; - - if !response.error.is_empty() { - return Err(ExchangeError::ApiError { - exchange: "Kraken".to_string(), - message: format!("Kraken API errors: {:?}", response.error), - }); - } - - Ok(response.result) - } -} - -/// Kraken depth response -#[derive(Debug, Deserialize)] -struct KrakenDepthResponse { - error: Vec, - result: HashMap, -} - -#[derive(Debug, Clone, Deserialize)] -struct KrakenPriceLevel( - String, // price - String, // volume - u64, // timestamp -); - -#[derive(Debug, Clone, Deserialize)] -struct KrakenOrderbookData { - bids: Vec, - asks: Vec, -} - -/// Kraken server time response -#[derive(Debug, Deserialize)] -struct KrakenServerTimeResponse { - error: Vec, - result: KrakenServerTime, -} - -/// Kraken server time -#[derive(Debug, Deserialize)] -pub struct KrakenServerTime { - pub unixtime: u64, - pub rfc1123: String, -} - -/// Kraken system status response -#[derive(Debug, Deserialize)] -struct KrakenSystemStatusResponse { - error: Vec, - result: KrakenSystemStatus, -} - -/// Kraken system status -#[derive(Debug, Deserialize)] -pub struct KrakenSystemStatus { - pub status: String, - pub timestamp: String, -} - -/// Kraken asset pairs response -#[derive(Debug, Deserialize)] -struct KrakenAssetPairsResponse { - error: Vec, - result: HashMap, -} - -/// Kraken asset pair information -#[derive(Debug, Deserialize)] -pub struct KrakenAssetPair { - pub altname: String, - pub wsname: Option, - pub aclass_base: String, - pub base: String, - pub aclass_quote: String, - pub quote: String, - pub pair_decimals: u32, - pub cost_decimals: u32, - pub lot_decimals: u32, - pub lot_multiplier: u32, - pub leverage_buy: Vec, - pub leverage_sell: Vec, - pub fees: Vec>, - pub fees_maker: Vec>, - pub fee_volume_currency: String, - pub margin_call: u32, - pub margin_stop: u32, - pub ordermin: String, - pub costmin: Option, - pub tick_size: Option, - pub status: String, - pub long_position_limit: Option, - pub short_position_limit: Option, -} - -/// Kraken ticker response -#[derive(Debug, Deserialize)] -struct KrakenTickerResponse { - error: Vec, - result: HashMap, -} - -/// Kraken ticker data -#[derive(Debug, Deserialize)] -pub struct KrakenTicker { - pub a: Vec, // ask [price, whole_lot_volume, lot_volume] - pub b: Vec, // bid [price, whole_lot_volume, lot_volume] - pub c: Vec, // last trade closed [price, lot_volume] - pub v: Vec, // volume [today, last_24_hours] - pub p: Vec, // volume weighted average price [today, last_24_hours] - pub t: Vec, // number of trades [today, last_24_hours] - pub l: Vec, // low [today, last_24_hours] - pub h: Vec, // high [today, last_24_hours] - pub o: String, // today's opening price -} - -impl Default for KrakenClient { - fn default() -> Self { - Self::new().expect("Failed to create default Kraken client") - } } diff --git a/atelier-data/src/exchanges/kraken/mod.rs b/atelier-data/src/exchanges/kraken/mod.rs index e69de29..a846843 100644 --- a/atelier-data/src/exchanges/kraken/mod.rs +++ b/atelier-data/src/exchanges/kraken/mod.rs @@ -0,0 +1,3 @@ +pub mod kraken_client; +pub use kraken_client::KrakenClient; +pub mod responses; diff --git a/atelier-data/src/exchanges/kraken/responses/misc.rs b/atelier-data/src/exchanges/kraken/responses/misc.rs new file mode 100644 index 0000000..59242c7 --- /dev/null +++ b/atelier-data/src/exchanges/kraken/responses/misc.rs @@ -0,0 +1,52 @@ +use serde::Deserialize; +use std::collections::HashMap; + +/// Kraken server time response +#[derive(Debug, Deserialize)] +pub struct KrakenServerTimeResponse { + pub error: Vec, + pub result: KrakenServerTime, +} + +/// Kraken server time +#[derive(Debug, Deserialize)] +pub struct KrakenServerTime { + pub unixtime: u64, + pub rfc1123: String, +} + +/// Kraken system status response +#[derive(Debug, Deserialize)] +pub struct KrakenSystemStatusResponse { + pub error: Vec, + pub result: KrakenSystemStatus, +} + +/// Kraken system status +#[derive(Debug, Deserialize)] +pub struct KrakenSystemStatus { + pub status: String, + pub timestamp: String, +} + +/// Kraken ticker response +#[derive(Debug, Deserialize)] +pub struct KrakenTickerResponse { + pub error: Vec, + pub result: HashMap, +} + +/// Kraken ticker data +#[derive(Debug, Deserialize)] +pub struct KrakenTicker { + pub a: Vec, // ask [price, whole_lot_volume, lot_volume] + pub b: Vec, // bid [price, whole_lot_volume, lot_volume] + pub c: Vec, // last trade closed [price, lot_volume] + pub v: Vec, // volume [today, last_24_hours] + pub p: Vec, // volume weighted average price [today, last_24_hours] + pub t: Vec, // number of trades [today, last_24_hours] + pub l: Vec, // low [today, last_24_hours] + pub h: Vec, // high [today, last_24_hours] + pub o: String, // today's opening price +} + diff --git a/atelier-data/src/exchanges/kraken/responses/mod.rs b/atelier-data/src/exchanges/kraken/responses/mod.rs index e69de29..f8d4b19 100644 --- a/atelier-data/src/exchanges/kraken/responses/mod.rs +++ b/atelier-data/src/exchanges/kraken/responses/mod.rs @@ -0,0 +1,4 @@ +pub mod misc; +pub mod orderbook; +pub mod symbols; +pub mod trades; diff --git a/atelier-data/src/exchanges/kraken/responses/orderbook.rs b/atelier-data/src/exchanges/kraken/responses/orderbook.rs index e69de29..f91dd31 100644 --- a/atelier-data/src/exchanges/kraken/responses/orderbook.rs +++ b/atelier-data/src/exchanges/kraken/responses/orderbook.rs @@ -0,0 +1,23 @@ +use serde::Deserialize; +use std::collections::HashMap; + +/// Kraken depth response +#[derive(Debug, Deserialize)] +pub struct KrakenDepthResponse { + pub error: Vec, + pub result: HashMap, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct KrakenPriceLevel( + pub String, // price + pub String, // volume + pub u64, // timestamp +); + +#[derive(Debug, Clone, Deserialize)] +pub struct KrakenOrderbookData { + pub bids: Vec, + pub asks: Vec, +} + diff --git a/atelier-data/src/exchanges/kraken/responses/symbols.rs b/atelier-data/src/exchanges/kraken/responses/symbols.rs new file mode 100644 index 0000000..87ec63b --- /dev/null +++ b/atelier-data/src/exchanges/kraken/responses/symbols.rs @@ -0,0 +1,38 @@ +use serde::Deserialize; +use std::collections::HashMap; + +/// Kraken asset pairs response +#[derive(Debug, Deserialize)] +pub struct KrakenAssetPairsResponse { + pub error: Vec, + pub result: HashMap, +} + +/// Kraken asset pair information +#[derive(Debug, Deserialize)] +pub struct KrakenAssetPair { + pub altname: String, + pub wsname: Option, + pub aclass_base: String, + pub base: String, + pub aclass_quote: String, + pub quote: String, + pub pair_decimals: u32, + pub cost_decimals: u32, + pub lot_decimals: u32, + pub lot_multiplier: u32, + pub leverage_buy: Vec, + pub leverage_sell: Vec, + pub fees: Vec>, + pub fees_maker: Vec>, + pub fee_volume_currency: String, + pub margin_call: u32, + pub margin_stop: u32, + pub ordermin: String, + pub costmin: Option, + pub tick_size: Option, + pub status: String, + pub long_position_limit: Option, + pub short_position_limit: Option, +} + diff --git a/atelier-data/src/exchanges/mod.rs b/atelier-data/src/exchanges/mod.rs index 1695949..0cec134 100644 --- a/atelier-data/src/exchanges/mod.rs +++ b/atelier-data/src/exchanges/mod.rs @@ -21,3 +21,4 @@ pub enum Exchange { Bitso, Gemini, } + diff --git a/atelier-data/test/exchanges/binance/test_connection.rs b/atelier-data/test/exchanges/binance/test_connection.rs new file mode 100644 index 0000000..6c2bfca --- /dev/null +++ b/atelier-data/test/exchanges/binance/test_connection.rs @@ -0,0 +1,29 @@ +#[cfg(test)] +mod tests { + + use atelier_data::exchanges::binance::BinanceClient; + + #[tokio::test] + async fn test_binance_client_creation() { + let client = BinanceClient::new(); + assert!(client.is_ok()); + } + + #[tokio::test] + async fn test_get_server_time() { + let client = BinanceClient::new().unwrap(); + let result = client.get_server_time().await; + + // This test might fail if no internet connection + match result { + Ok(server_time) => { + assert!(server_time.server_time > 0); + println!("Binance server time: {}", server_time.server_time); + } + Err(e) => { + println!("Expected network error in test environment: {e:?}"); + } + } + } +} + diff --git a/atelier-data/test/exchanges/binance/test_orderbook.rs b/atelier-data/test/exchanges/binance/test_orderbook.rs index e69de29..a8f9404 100644 --- a/atelier-data/test/exchanges/binance/test_orderbook.rs +++ b/atelier-data/test/exchanges/binance/test_orderbook.rs @@ -0,0 +1,17 @@ +#[cfg(test)] +mod tests { + + use atelier_data::{exchanges::binance::BinanceClient, models::pairs::TradingPair}; + + #[tokio::test] + async fn test_binance_orderbook() { + let binance_client = BinanceClient::new(); + let r_ob = binance_client + .unwrap() + .get_orderbook(TradingPair::SolUsdt, Some(25)) + .await; + + assert!(r_ob.is_ok()); + println!("Orderbook: {:?}", r_ob); + } +} diff --git a/atelier-data/test/exchanges/binance/test_symbols.rs b/atelier-data/test/exchanges/binance/test_symbols.rs new file mode 100644 index 0000000..7fd3039 --- /dev/null +++ b/atelier-data/test/exchanges/binance/test_symbols.rs @@ -0,0 +1,12 @@ +#[cfg(test)] +mod tests { + + use atelier_data::exchanges::binance::BinanceClient; + + #[tokio::test] + async fn test_binance_client_creation() { + let client = BinanceClient::new(); + assert!(client.is_ok()); + } +} + diff --git a/atelier-data/test/exchanges/binance/test_trades.rs b/atelier-data/test/exchanges/binance/test_trades.rs index e69de29..7fd3039 100644 --- a/atelier-data/test/exchanges/binance/test_trades.rs +++ b/atelier-data/test/exchanges/binance/test_trades.rs @@ -0,0 +1,12 @@ +#[cfg(test)] +mod tests { + + use atelier_data::exchanges::binance::BinanceClient; + + #[tokio::test] + async fn test_binance_client_creation() { + let client = BinanceClient::new(); + assert!(client.is_ok()); + } +} + diff --git a/atelier-data/test/exchanges/coinbase/test_connection.rs b/atelier-data/test/exchanges/coinbase/test_connection.rs new file mode 100644 index 0000000..ad0c5ce --- /dev/null +++ b/atelier-data/test/exchanges/coinbase/test_connection.rs @@ -0,0 +1,12 @@ +#[cfg(test)] +mod tests { + + use atelier_data::exchanges::coinbase::CoinbaseClient; + + #[tokio::test] + async fn test_coinbase_client_creation() { + let client = CoinbaseClient::new(); + assert!(client.is_ok()); + } + +} diff --git a/atelier-data/test/exchanges/coinbase/test_orderbook.rs b/atelier-data/test/exchanges/coinbase/test_orderbook.rs index dcaaabc..97e2a26 100644 --- a/atelier-data/test/exchanges/coinbase/test_orderbook.rs +++ b/atelier-data/test/exchanges/coinbase/test_orderbook.rs @@ -1,30 +1,17 @@ #[cfg(test)] - mod tests { - use atelier_data::{ - exchanges::coinbase::coinbase_client::CoinbaseClient, - models::pairs::TradingPair, - }; + use atelier_data::{exchanges::coinbase::CoinbaseClient, models::pairs::TradingPair}; #[tokio::test] - async fn test_coinbase_client() { - let exchange_client = CoinbaseClient::new(); + async fn test_coinbase_client_creation() { + let coinbase_client = CoinbaseClient::new(); + let r_ob = coinbase_client + .unwrap() + .get_orderbook(TradingPair::SolUsdt, Some(25)) + .await; + + assert!(r_ob.is_ok()); + println!("Orderbook: {:?}", r_ob); } - - #[tokio::test] - async fn test_coinbase_orderbook() { - - let exchange_client = CoinbaseClient::new(); - let depth = 10; - let r_orderbook = exchange_client.unwrap() - .get_orderbook(TradingPair::SolUsdc.clone(), Some(depth)) - .await - .unwrap(); - - println!("Orderbook from Coinbase: {:?}", r_orderbook); - - } - } - diff --git a/atelier-data/test/exchanges/coinbase/test_products.rs b/atelier-data/test/exchanges/coinbase/test_symbols.rs similarity index 79% rename from atelier-data/test/exchanges/coinbase/test_products.rs rename to atelier-data/test/exchanges/coinbase/test_symbols.rs index 32186a0..2681b8d 100644 --- a/atelier-data/test/exchanges/coinbase/test_products.rs +++ b/atelier-data/test/exchanges/coinbase/test_symbols.rs @@ -1,7 +1,7 @@ #[cfg(test)] mod tests { - use atelier_data::exchanges::coinbase::coinbase_client::CoinbaseClient; + use atelier_data::exchanges::coinbase::CoinbaseClient; #[tokio::test] async fn test_coinbase_client_creation() { @@ -10,9 +10,9 @@ mod tests { } #[tokio::test] - async fn test_get_products() { + async fn test_get_symbols() { let client = CoinbaseClient::new().unwrap(); - let result = client.get_products().await; + let result = client.get_symbols().await; // This test might fail if no internet connection match result { diff --git a/atelier-data/test/exchanges/coinbase/test_trades.rs b/atelier-data/test/exchanges/coinbase/test_trades.rs index e69de29..ad0c5ce 100644 --- a/atelier-data/test/exchanges/coinbase/test_trades.rs +++ b/atelier-data/test/exchanges/coinbase/test_trades.rs @@ -0,0 +1,12 @@ +#[cfg(test)] +mod tests { + + use atelier_data::exchanges::coinbase::CoinbaseClient; + + #[tokio::test] + async fn test_coinbase_client_creation() { + let client = CoinbaseClient::new(); + assert!(client.is_ok()); + } + +} diff --git a/atelier-data/test/exchanges/kraken/test_connection.rs b/atelier-data/test/exchanges/kraken/test_connection.rs new file mode 100644 index 0000000..d299be7 --- /dev/null +++ b/atelier-data/test/exchanges/kraken/test_connection.rs @@ -0,0 +1,12 @@ +#[cfg(test)] +mod tests { + + use atelier_data::exchanges::kraken::KrakenClient; + + #[tokio::test] + async fn test_kraken_client_creation() { + let client = KrakenClient::new(); + assert!(client.is_ok()); + } +} + diff --git a/atelier-data/test/exchanges/kraken/test_orderbook.rs b/atelier-data/test/exchanges/kraken/test_orderbook.rs index e69de29..81e470e 100644 --- a/atelier-data/test/exchanges/kraken/test_orderbook.rs +++ b/atelier-data/test/exchanges/kraken/test_orderbook.rs @@ -0,0 +1,17 @@ +#[cfg(test)] +mod tests { + + use atelier_data::{exchanges::kraken::KrakenClient, models::pairs::TradingPair}; + + #[tokio::test] + async fn test_kraken_client_creation() { + let kraken_client = KrakenClient::new(); + let r_ob = kraken_client + .unwrap() + .get_orderbook(TradingPair::SolUsdt, Some(25)) + .await; + + assert!(r_ob.is_ok()); + println!("Orderbook: {:?}", r_ob); + } +} diff --git a/atelier-data/test/exchanges/kraken/test_symbols.rs b/atelier-data/test/exchanges/kraken/test_symbols.rs new file mode 100644 index 0000000..e24286d --- /dev/null +++ b/atelier-data/test/exchanges/kraken/test_symbols.rs @@ -0,0 +1,11 @@ +#[cfg(test)] +mod tests { + + use atelier_data::exchanges::kraken::KrakenClient; + + #[tokio::test] + async fn test_kraken_client_creation() { + let client = KrakenClient::new(); + assert!(client.is_ok()); + } +} diff --git a/atelier-data/test/exchanges/kraken/test_trades.rs b/atelier-data/test/exchanges/kraken/test_trades.rs index e69de29..e24286d 100644 --- a/atelier-data/test/exchanges/kraken/test_trades.rs +++ b/atelier-data/test/exchanges/kraken/test_trades.rs @@ -0,0 +1,11 @@ +#[cfg(test)] +mod tests { + + use atelier_data::exchanges::kraken::KrakenClient; + + #[tokio::test] + async fn test_kraken_client_creation() { + let client = KrakenClient::new(); + assert!(client.is_ok()); + } +} diff --git a/atelier-rs/Cargo.toml b/atelier-rs/Cargo.toml index 64920dd..cb01a8b 100644 --- a/atelier-rs/Cargo.toml +++ b/atelier-rs/Cargo.toml @@ -36,6 +36,7 @@ atelier_dcml = { path = "../atelier-dcml", version = "0.0.11" } atelier_quant = { path = "../atelier-quant", version = "0.0.10" } atelier_synth = { path = "../atelier-synth", version = "0.0.10" } +anyhow = { version = "1.0" } thiserror = { version = "1.0.64" } rand = { version = "0.9.0" } rand_distr = { version = "0.5.0" } diff --git a/atelier-rs/database/clickhouse/config.xml b/atelier-rs/database/clickhouse/config.xml new file mode 100644 index 0000000..e69de29 diff --git a/atelier-rs/database/clickhouse/init-ob-schema.sql b/atelier-rs/database/clickhouse/init-ob-schema.sql new file mode 100644 index 0000000..6dd46fd --- /dev/null +++ b/atelier-rs/database/clickhouse/init-ob-schema.sql @@ -0,0 +1,20 @@ + + +-- Create the database if does not exist +CREATE DATABASE IF NOT EXISTS operations; + +-- Use the trading database +USE operations; + +-- Orderbooks table +CREATE TABLE IF NOT EXISTS orderbooks ( + timestamp DateTime64(6, 'UTC'), + symbol String, + exchange String, + bids Array(Tuple(String, String)), + asks Array(Tuple(String, String)) +) ENGINE = MergeTree() +PARTITION BY toYYYYMM(timestamp) +ORDER BY (symbol, exchange, timestamp) +SETTINGS index_granularity = 8192; + diff --git a/atelier-rs/database/clickhouse/users.xml b/atelier-rs/database/clickhouse/users.xml new file mode 100644 index 0000000..05eda68 --- /dev/null +++ b/atelier-rs/database/clickhouse/users.xml @@ -0,0 +1,59 @@ + + + + + + + ::/0 + + default + default + 1 + + default + + + + + + + ::/0 + + readonly + default + + + + + + 10000000000 + 0 + random + 16 + 16 + 8 + 8 + 16 + + + + 1 + 10000000000 + 0 + random + + + + + + + 3600 + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/atelier-rs/database/database.Dockerfile b/atelier-rs/database/database.Dockerfile new file mode 100644 index 0000000..1b42eb2 --- /dev/null +++ b/atelier-rs/database/database.Dockerfile @@ -0,0 +1,42 @@ +FROM clickhouse/clickhouse-server:latest + +# Remove default password file that may override configuration +RUN rm -f /etc/clickhouse-server/users.d/default-password.xml + +# Create required directories with proper permissions +RUN echo "create /var/lib/clickhouse folders" +RUN mkdir -p /var/lib/clickhouse/format_schemas && \ + mkdir -p /var/lib/clickhouse/access && \ + mkdir -p /var/lib/clickhouse/user_files && \ + mkdir -p /var/lib/clickhouse/tmp && \ + mkdir -p /var/log/clickhouse-server && \ + chown -R clickhouse:clickhouse /var/lib/clickhouse && \ + chown -R clickhouse:clickhouse /var/log/clickhouse-server + +# Copy initialization SQL scripts to the init directory +RUN echo "Copy initialization queries.." +RUN mkdir -p /docker-entrypoint-initdb.d +COPY clickhouse/init-ft-schema.sql /docker-entrypoint-initdb.d/init-ft-schema.sql +COPY clickhouse/init-lq-schema.sql /docker-entrypoint-initdb.d/init-lq-schema.sql +COPY clickhouse/init-ob-schema.sql /docker-entrypoint-initdb.d/init-ob-schema.sql +COPY clickhouse/init-pt-schema.sql /docker-entrypoint-initdb.d/init-pt-schema.sql +COPY clickhouse/init-sn-schema.sql /docker-entrypoint-initdb.d/init-sn-schema.sql + +# Set proper permissions for init directory +RUN chown -R clickhouse:clickhouse /docker-entrypoint-initdb.d/ && \ + chmod 644 /docker-entrypoint-initdb.d/*.sql + +# Copy and set up custom entrypoint script +COPY database_entrypoint.sh /custom-entrypoint.sh +RUN chmod +x /custom-entrypoint.sh && \ + chown clickhouse:clickhouse /custom-entrypoint.sh + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ + CMD wget --spider -q localhost:8123/ping || exit 1 + +# Expose standard ClickHouse ports +EXPOSE 8123 9000 9009 + +# Use custom entrypoint +ENTRYPOINT ["/custom-entrypoint.sh"] diff --git a/atelier-rs/database/database_entrypoint.sh b/atelier-rs/database/database_entrypoint.sh new file mode 100644 index 0000000..e473674 --- /dev/null +++ b/atelier-rs/database/database_entrypoint.sh @@ -0,0 +1,63 @@ + +#!/bin/bash +set -e + +echo "=== ClickHouse Database Initialization Entrypoint ===" + +# Remove any default password files that might override our configuration +rm -f /etc/clickhouse-server/users.d/default-password.xml +rm -f /etc/clickhouse-server/users.d/default-user.xml + +# Ensure proper ownership +chown -R clickhouse:clickhouse /var/lib/clickhouse +chown -R clickhouse:clickhouse /var/log/clickhouse-server +chown -R clickhouse:clickhouse /etc/clickhouse-server + +# Function to wait for ClickHouse to be ready +wait_for_clickhouse() { + echo "Waiting for ClickHouse to start..." + for i in {1..15}; do + if clickhouse-client --query "SELECT 1" >/dev/null 2>&1; then + echo "ClickHouse is ready!" + return 0 + fi + echo "Waiting for ClickHouse... attempt $i/30" + sleep 5 + done + echo "ERROR: ClickHouse failed to start within 60 seconds" + return 1 +} + +# Function to execute SQL files +execute_init_scripts() { + echo "Executing initialization scripts..." + + if [ -d "/docker-entrypoint-initdb.d" ]; then + for f in /docker-entrypoint-initdb.d/*.sql; do + if [ -f "$f" ]; then + echo "Executing $f..." + clickhouse-client --multiquery < "$f" + echo "Completed $f" + fi + done + fi + + echo "All initialization scripts completed!" +} + +# Start ClickHouse server in background +echo "Starting ClickHouse server..." +exec /entrypoint.sh "$@" + +# Wait for ClickHouse to be ready +if wait_for_clickhouse; then + # Execute initialization scripts + execute_init_scripts + + echo "=== ClickHouse initialization completed successfully ===" + echo "ClickHouse is ready to accept connections" +else + echo "=== ClickHouse initialization FAILED ===" + exit 1 +fi + diff --git a/atelier-rs/datacollector/datacollector.Dockerfile b/atelier-rs/datacollector/datacollector.Dockerfile new file mode 100644 index 0000000..2d26be6 --- /dev/null +++ b/atelier-rs/datacollector/datacollector.Dockerfile @@ -0,0 +1,41 @@ +FROM --platform=linux/amd64 debian:bookworm-slim + +# Install runtime dependencies +RUN apt-get update && apt-get install -y \ + ca-certificates \ + curl \ + wget \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + +# Create app user +RUN useradd -r -u 1001 -s /bin/bash appuser + +# Create directories +RUN mkdir -p /app /app/logs && \ + chown -R appuser:appuser /app + +WORKDIR /app + +# Copy built binary from builder stage +COPY datacollector /usr/local/bin/datacollector +COPY datacollector_config.toml /app + +# Set permissions +RUN chmod +x /usr/local/bin/datacollector && \ + chown root:root /usr/local/bin/datacollector && \ + chown appuser:appuser /app/datacollector_config.toml + +# Switch to app user +USER appuser + +# Environment variables +ENV RUST_LOG=info +ENV CLICKHOUSE_URL=http://database:8123 + +# Expose port +EXPOSE 9009 + +# Run the application +CMD ["/usr/local/bin/datacollector"] + diff --git a/atelier-rs/datacollector/datacollector_config.toml b/atelier-rs/datacollector/datacollector_config.toml new file mode 100644 index 0000000..3165b70 --- /dev/null +++ b/atelier-rs/datacollector/datacollector_config.toml @@ -0,0 +1,49 @@ + +# File: docker/datacollector-config.toml +# Configuration for the CEX data collector + +[datacollector] +name = "datacollector" +log_level = "info" + +[database] +clickhouse_url = "http://localhost:8123" +database = "operations" +username = "default" +password = "" + +[connection] +timeout = "30s" +retry_attempts = 3 +batch_size = 1000 +flush_interval_seconds = 30 + +[exchanges] + +[exchanges.binance] +enabled = true +websocket_url = "wss://stream.binance.com:9443/ws/" +rest_api_url = "https://api.binance.com/api/v3/" +symbols = ["BTCUSDT", "ETHUSDT", "ADAUSDT", "DOTUSDT"] +data_types = ["orderbook", "trades", "klines"] + +[exchanges.coinbase] +enabled = true +websocket_url = "wss://ws-feed.pro.coinbase.com" +rest_api_url = "https://api.pro.coinbase.com/" +symbols = ["BTC-USD", "ETH-USD", "ADA-USD", "DOT-USD"] +data_types = ["orderbook", "trades"] + +[exchanges.kraken] +enabled = false +websocket_url = "wss://ws.kraken.com" +rest_api_url = "https://api.kraken.com/0/public/" + +[features] +calculate_technical_indicators = true +store_raw_data = true +enable_signal_generation = false + +[monitoring] +metrics_port = 8090 +health_check_port = 8091 diff --git a/atelier-rs/src/bin/datacollector.rs b/atelier-rs/src/bin/datacollector.rs new file mode 100644 index 0000000..2894b0c --- /dev/null +++ b/atelier-rs/src/bin/datacollector.rs @@ -0,0 +1,175 @@ +// src/bin/datacollector.rs + +use atelier_data::exchanges::Exchange; +use atelier_data::models::pairs::TradingPair; +use atelier_data::exchanges::binance::BinanceClient; +use atelier_data::exchanges::coinbase::CoinbaseClient; +use atelier_data::exchanges::kraken::KrakenClient; + +use atelier_rs::{ + queries, queries::trades::ClickhouseTradeData, ClickHouseClient, +}; + +use atelier_data::{ + exchanges::bybit::{ws_decoder::BybitWssEvent, WssExchange}, + stream_data, +}; +use std::{ + env, + thread::sleep, + time::{Duration, Instant}, +}; + + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + + // -- take from env + // Environment variable from docker-compose.yaml + let ch_url = env::var("CLICKHOUSE_URL").unwrap_or_else(|_| "http://database:8123".to_string()); + let ch_db = env::var("CLICKHOUSE_DB").unwrap_or_else(|_| "operations".to_string()); + + // --- LIQUIDATIONS Datacollector --- // + let ch_lq_client = ClickHouseClient::builder() + .url(ch_url.clone()) + .database(ch_db.clone()) + .build() + .await + .unwrap(); + + let streams_task = tokio::spawn(async move { + println!("started streams_task"); + + let symbols = vec![ + "SOLUSDT".to_string(), + "LINKUSDT".to_string(), + ]; + + let streams = vec!["allLiquidation".to_string(), "publicTrade".to_string()]; + let source = WssExchange::Bybit; + let mut rx = stream_data(symbols, streams, source) + .await + .expect("failed to open Bybit WS"); + + while let Some(recv_event) = rx.recv().await { + println!("\nrecv_event"); + match recv_event { + BybitWssEvent::LiquidationData(event_data) => { + let i_liq = LiquidationNew { + ts: event_data.liquidation_ts, + symbol: event_data.symbol.clone(), + side: event_data.side, + amount: event_data.amount, + price: event_data.price, + exchange: "Bybit".to_string(), + }; + + println!( + "\n ---- allLiquidation event received... {:?} ----\n ", + event_data.symbol + ); + + let liquidation_query = + queries::liquidations::write_tables::q_insert_liquidations( + &i_liq, + ) + .unwrap(); + println!("\n ---- liquidation_query {:?} ---- \n", liquidation_query); + + let ch_lq_result = ch_lq_client.write_table(&liquidation_query).await; + println!("\n ---- ch_lq_result {:?} ---- \n", ch_lq_result); + } + + BybitWssEvent::TradeData(event_data) => { + let i_trade = ClickhouseTradeData { + timestamp: event_data.trade_ts, + symbol: event_data.symbol.clone(), + side: event_data.side, + amount: event_data.amount.parse().expect("failed to parse amount to f64"), + price: event_data.price.parse().expect("failed to parse price to f64"), + exchange: "Bybit".to_string(), + }; + + println!("\n ---- publicTrade event received... {:?} ---- ", event_data.symbol); + + let trade_query = + queries::trades::write_tables::q_insert_trades(&i_trade).unwrap(); + println!("\n ---- trade_query {:?} --- \n", trade_query); + + let ch_pt_result = ch_lq_client.write_table(&trade_query).await; + println!("\n ---- ch_pt_result {:?} ---- \n", ch_pt_result); + + } + } + } + }); + + // --- ORDERBOOKS Dataollector --- // + let ch_ob_client = ClickHouseClient::builder() + .url(ch_url.clone()) + .database(ch_db.clone()) + .build() + .await + .unwrap(); + + let orderbook_task = tokio::spawn(async move { + let interval = Duration::from_millis(800); + let mut next_time = Instant::now() + interval; + + loop { + + let v_exchanges = vec![ + Exchange::Kraken, + Exchange::Binance, + Exchange::Bybit, + Exchange::Coinbase + ]; + + let v_pairs = vec![ + TradingPair::SolUsdt, + TradingPair::LinkUsdt, + ]; + + let depth = 25; + + for i_exchange in v_exchanges { + + println!("exchange {:?}", i_exchange); + + let exchange_client: Box = + match i_exchange { + Exchange::Binance => Box::new(BinanceClient::new().unwrap()), + Exchange::Coinbase => Box::new(CoinbaseClient::new().unwrap()), + Exchange::Kraken => Box::new(KrakenClient::new().unwrap()), + Exchange::Bybit => Box::new(BybitClient::new().unwrap()), + }; + + for i_pair in v_pairs.clone() { + println!("\nOrderbook query for pair: {:?}", i_pair); + + let r_orderbook = exchange_client + .get_orderbook(i_pair.clone(), Some(depth)) + .await + .unwrap(); + + let ob_query: String = + queries::orderbooks::write_tables::q_insert_orderbook(&r_orderbook) + .unwrap(); + println!("\n ---- ob_query {:?} ---- \n", ob_query); + + let ch_ob_result = ch_ob_client.write_table(&ob_query).await; + println!("\n ---- ch_ob_result {:?} ---- \n", ch_ob_result); + + sleep(next_time - Instant::now()); + next_time += interval; + + } + } + } + }); + + // Wait for both tasks + tokio::try_join!(streams_task, orderbook_task)?; + Ok(()) + +} diff --git a/atelier-rs/src/lib.rs b/atelier-rs/src/lib.rs index 9b6ac7d..fef4c7c 100644 --- a/atelier-rs/src/lib.rs +++ b/atelier-rs/src/lib.rs @@ -19,7 +19,6 @@ pub use atelier_base::{Level, Order, OrderSide, OrderType, orderbooks::Orderbook pub use atelier_data::{ clients::wss::{WssClient, WssClientBuilder, WssDecoder}, - errors::ExchangeError, exchanges::bybit::{BybitWssClient, LiquidationData, stream_data}, }; diff --git a/atelier-rs/src/queries/create_table.sql b/atelier-rs/src/queries/create_table.sql new file mode 100644 index 0000000..9ff30a4 --- /dev/null +++ b/atelier-rs/src/queries/create_table.sql @@ -0,0 +1,18 @@ + +// Create the orderbooks table DDL +pub fn create_orderbooks_table_ddl() -> String { + r#" +CREATE TABLE IF NOT EXISTS orderbooks ( + timestamp DateTime64(6, 'UTC'), + symbol String, + exchange String, + bids Array(Tuple(String, String)), + asks Array(Tuple(String, String)), +) ENGINE = MergeTree() +PARTITION BY toYYYYMM(timestamp) +ORDER BY (symbol, exchange, timestamp) +SETTINGS index_granularity = 8192 +"# + .trim() + .to_string() +} diff --git a/atelier-rs/src/queries/mod.rs b/atelier-rs/src/queries/mod.rs new file mode 100644 index 0000000..6707361 --- /dev/null +++ b/atelier-rs/src/queries/mod.rs @@ -0,0 +1,3 @@ +pub mod create_tables; +pub mod read_tables; +pub mod write_tables; diff --git a/atelier-rs/src/queries/read_table.sql b/atelier-rs/src/queries/read_table.sql new file mode 100644 index 0000000..e69de29 diff --git a/atelier-rs/src/queries/write_table.sql b/atelier-rs/src/queries/write_table.sql new file mode 100644 index 0000000..e69de29 diff --git a/atelier-synth/Cargo.toml b/atelier-synth/Cargo.toml index 39e54b6..0a5cf31 100644 --- a/atelier-synth/Cargo.toml +++ b/atelier-synth/Cargo.toml @@ -37,6 +37,7 @@ atelier_base = { path = "../atelier-base", version = "0.0.11" } atelier_dcml = { path = "../atelier-dcml", version = "0.0.11" } atelier_quant = { path = "../atelier-quant", version = "0.0.10" } +anyhow = { version = "1.0" } rand = { version = "0.9.0" } rand_distr = { version = "0.5.0" } tokio = { version = "1", features = ["full"] }