Refactor docs and core data functionality

CONTRIBUTE.md

@@ -3,9 +3,9 @@
 Thanks for the interest in contributing to this awesome project !, in order to get you up to speed in terms of sharing your contributed work, or intentions, consider the following: 
 
 1. compile.
-2. no clippy errors.
+2. no clippy warnings/errors.
 3. tests runs.
-4. code formats its.
+4. code is formatted.
 
 ## Lints
 
@@ -14,35 +14,32 @@ This project, for the stable version, has implemented the following lints:
 ```toml
 [workspace.lints.rust]
 unsafe_code = "forbid"
-unused_extern_crates = "forbid"
-unreachable_code = "forbid"
-unreachable_patterns = "forbid"
+unused_extern_crates = "allow"
+unreachable_code = "deny"
+unreachable_patterns = "deny"
 unused_variables = "warn"
 trivial_casts = "warn"
 trivial_numeric_casts = "warn"
 unexpected_cfgs = { level = "warn", check-cfg = ['cfg(nightly)'] }
+dead_code = "allow"
+too_many_arguments = "allow"
 ```
 
 ## rustfmt.toml
 
-Consider all defaults to be present, and, the following changed:
+For the `atelier-rs` crate, there is a `.rustfmt.toml` config file, even though most of the values are exactly the same as the default, they were included for future-proof purposes in terms of formatting. Consider all defaults to be present, and, the following changed:
 
 ```toml
 reorder_modules = true
 max_width = 90
 ```
 
-## Code format with rustfmt
-
-For the `atelier-rs` crate, there is a `.rustfmt.toml` config file, even though must of the values are exactly the same as the default, they were included for future-proof purposes in terms of formatting. 
-
 ## Reporting a Bug
 
 In the case that you've found a bug, please make sure you are able to answer the following:
 
 ```
-- What version of Rust are you using?
-- What version of the crate are you using?
+- What version of Rust and the atelier related crate are you using?
 - What operating system are you using?
 - What did you do?
 - What did you expect to see?

Cargo.toml

@@ -19,51 +19,48 @@ default-members = [
     "atelier-base",
     "atelier-data",
     "atelier-rs",
-    "atelier-synth",
 ]
 
 [workspace.package]
 rust-version = "1.85.0"
 edition = "2024"
 
 description = "Rust Engine for High Frequency, Synthetic and Historical, Market Microstructure Modeling."
-authors = ["IteraLabs.xyz"]
+authors = ["iteraLabs.xyz"]
 documentation = "https://docs.rs/atelier-rs/"
 repository = "https://github.com/iteralabs/atelier-rs"
 homepage = "https://iteralabs.xyz/atelier-rs"
-
 keywords = ["machine-learning", "framework", "math", "crypto", "trading"]
 categories = ["data-structures", "development-tools", "finance", "simulation"]
 include = ["katex-header.html"]
 exclude = ["assets/*", ".github", "Makefile.toml", "*.log", "tags"]
-
 license = "Apache-2.0"
 
 [workspace.dependencies]
 
-atelier_data = { path = "./atelier-data", version = "0.0.10" }
-atelier_base = { path = "./atelier-base", version = "0.0.12" }
-atelier_dcml = { path = "./atelier-dcml", version = "0.0.11" }
-atelier_quant = { path = "./atelier-quant", version = "0.0.10" }
-atelier_rs = { path = "./atelier-rs", version = "0.0.10" }
-atelier_synth = { path = "./atelier-synth", version = "0.0.10" }
+atelier_data = { version = "0.0.12" }
+atelier_base = { version = "0.0.11" }
+atelier_dcml = { version = "0.0.11" }
+atelier_quant = { version = "0.0.10" }
+atelier_rs = { version = "0.0.10" }
+atelier_synth = { version = "0.0.10" }
 
-thiserror = { version = "1.0.64" }
-rand = { version = "0.9.0" }
-rand_distr = { version = "0.5.0" }
+anyhow = { version = "1.0" }
 criterion = { version = "0.5", features = ["html_reports"] }
-memuse = { version = "0.2.0" }
-human_bytes = { version = "0.4.1" }
-toml = { version = "0.8" }
 csv = { version = "1.3" }
 clap = { version = "4.5", features = ["derive"] }
-tokio = { version = "1", features = ["full"] }
 futures = { version = "0.3" }
+human_bytes = { version = "0.4.1" }
+memuse = { version = "0.2.0" }
 reqwest = { version = "0.12", features = ["json"] }
 rust_decimal = { version = "1.34", features = ["serde"] }
-
+rand = { version = "0.9.0" }
+rand_distr = { version = "0.5.0" }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0" }
+thiserror = { version = "1.0.64" }
+toml = { version = "0.8" }
+tokio = { version = "1", features = ["full"] }
 
 [workspace.lints.rust]
 unsafe_code = "forbid"

README.md

@@ -31,25 +31,36 @@
 
 # Overview
 
-At a high level it provides the following major components: A full orderbook granularity, stochastic process and functions for synthetic data generation, Distributed convex methods for model training/inference.
+Atelier-rs is an engine for high frequency, synthetic and historical, market microstructure modeling. Includes functionality for centralized, decentralized exchanges, and also blockchain protocols. At a high level it provides the following major components:
 
-### Full Limit Order Book
+- Full Order book granularity (Sides -> Levels -> Orders).
+- Stochastic process functions for synthetic data generation.
+- Distributed convex methods for model fitting/training/inference.
+- Graph structures, spectral analysis and tools.
+
+## Full Limit Order Book
 
 From the standard representation **\(level_price, level_volume\)** Levels, to a by the level order-queue granularity **\(level_orders \[ \(order_id, order_price, order_amount\), \(order_id, order_price, order_amount\), ... \] \)** to provide a true order-driven market representation structure for enriched models.
 
-### Stochastic Process
+## Stochastic Processes and Quantitative Models
+
+Stochastic process generators for rich/complex simulations, implementations include: Uniform, Normal, Exponential, Poisson.
 
-Stochastic process generators for rich/complex simulations, implementations include: Uniform, Brownian, Hawkes.
+Quantitative models from classical and modern literature: Brownian motion, Volume-synchronized Probability of Informed Trading (VPIN), self-exciting point process (Hawkes). 
 
-e## Distributed Convex Methods
+## Distributed Convex Methods
 
 Distributed Convex Methods for Linear Models Training and inference. Implementations include: Undirected Acyclic Compute Graph with Gradient Consensus. 
 
+## Graph Theory and Algorithms
+
+Graph fundamentals, Directed Acyclic Graph (static, and, time-varying), *LaPlacian* spectral analysis.
+
 ### Docs
 
 - **[Complete API Documentation](https://docs.rs/atelier-rs)** - All crates in one place
 
-# Usage
+# Generic usage
 
 ## Local clone
 
@@ -76,7 +87,7 @@ docker build \
     --no-cache . 
 ```
 
-the `builder` stage, to compile the rust binary, and the `runner` stage to have a 
+The `builder` stage, to compile the rust binary, and the `runner` stage to have a 
 minimalistic container to expose a service provided by the binary execution.
 
 Generating results by running the containerized atelier.
@@ -93,15 +104,24 @@ docker run \
 
 These are the other published crates members of the workspace: 
 
-- [atelier-base](https://crates.io/crates/atelier-base): Core data structures and I/O tools.
-- [atelier-dcml](https://crates.io/crates/atelier-dcml): Distributed Convex Machine Learning. 
+- [atelier-base](https://crates.io/crates/atelier-base): Core data structures for the atelier-rs engine.
+- [atelier-data](https://crates.io/crates/atelier-data): Data I/O integrations for OnChain/OffChain Protocols for the atelier-rs engine.
+- [atelier-dcml](https://crates.io/crates/atelier-dcml): Distributed Convex Machine Learning methods and tooling for the atelier-rs engine.
 - [atelier-synth](https://crates.io/crates/atelier-synth): Synthetic Data Generation for the atelier-rs engine.
+- [atelier-graph](https://crates.io/crates/atelier-graph): Graph theory algorithms and tooling.
+- [atelier-quant](https://crates.io/crates/atelier-quant): Intermediate to advance quantitative finance methods for the atelier-rs engine.
 
 Github hosted:
 
-- [benches](https://github.com/IteraLabs/atelier-rs/tree/main/benches)
-- [examples](https://github.com/IteraLabs/atelier-rs/tree/main/examples)
+- [benches](https://github.com/IteraLabs/atelier-rs/tree/main/benches) : Official and Community benchmarks for the atelier-rs engine and/or individual/grouped sub-components of the engine.
+- [examples](https://github.com/IteraLabs/atelier-rs/tree/main/examples): Official and Community gallery of implementations, and examples for the atelier-rs engine and/or individual/grouped sub-components of the engine.
+- [research](https://github.com/IteraLabs/atelier-rs/tree/main/research): Official and community research with usage of the atelier-rs and/or individual/grouped sub-components.
+
+# Contribute
+
+If you would like to contribute, please consider reading the [CONTRIBUTE.md](https://github.com/IteraLabs/atelier-rs/blob/main/CONTRIBUTE.md) guide.
 
 # License
 
-This project is licensed under the Apache V2 license. Any contribution intentionally submitted for inclussion in Atelier by you, shall be licensed as Apache V2, without any additional terms or conditions. 
+This project is licensed under the Apache V2 license. Any contribution intentionally submitted for inclussion in Atelier by you, shall be licensed as Apache V2, without any additional terms or conditions.
+

atelier-data/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "atelier_data"
-description = "Centralized Exchanges REST/WSS integrations for the atelier-rs engine"
+description = "Data I/O integrations for OnChain/OffChain Protocols for the atelier-rs engine"
 publish = true
 readme = "README.md"
 
@@ -27,10 +27,6 @@ license = "Apache-2.0"
 [package.metadata.docs.rs]
 private-doc = true
 
-# [[bin]]
-# name = "liquidations"
-# path = "src/bin/bybit_wss.rs"
-
 [lib]
 name = "atelier_data"
 path = "src/lib.rs"
@@ -39,38 +35,84 @@ path = "src/lib.rs"
 atelier_base = { path = "../atelier-base", version = "0.0.11" }
 
 anyhow = { version = "1.0" }
+async-rate-limiter = { version = "1.0", features = ["rt-tokio"] }
+async-trait = { version = "0.1" }
 config = "0.13"
+chrono = { version = "0.4", features = ["serde"] }
+futures = "0.3"
 futures-util = "0.3"
 hex = { version = "0.4" }
 hmac = { version = "0.12" }
-reqwest = { workspace = true }
-rust_decimal = { workspace = true }
+reqwest = { version = "0.12", features = ["json"] }
+rust_decimal = { version = "1.34", features = ["serde"] }
 sha2 = { version = "0.10" }
 serde = { version = "1.0", features = ["derive"] }
-serde_json = { workspace = true }
-thiserror = { workspace = true }
-tokio = { workspace = true }
+serde_json = { version = "1.0" }
+thiserror = { version = "1.0.64" }
+tokio = { version = "1.0", features = ["full"] }
 tokio-tungstenite = { version = "0.21", features = ["native-tls"] }
-toml = { workspace = true }
-async-rate-limiter = { version = "1.0", features = ["rt-tokio"] }
-async-trait = { version = "0.1" }
-url = { version = "2.0" }
+tonic = { version = "0.10" }
+toml = { version = "0.8" }
 tracing = { version = "0.1" }
 tracing-subscriber = { version = "0.3" }
+url = { version = "2.0" }
 uuid = { version = "1.0", features = ["v4"] }
-chrono = { version = "0.4", features = ["serde"] }
+yellowstone-grpc-client = { version = "1.13.0" }
+yellowstone-grpc-proto = { version = "1.13.0" }
+
+# --- Coinbase --- #
+
+[[test]]
+name = "coinbase_http_connection"
+path = "test/exchanges/coinbase/test_connection.rs"
 
 [[test]]
-name = "test_config_from_toml"
-path = "test/configs/test_from_toml.rs"
+name = "coinbase_http_orderbook"
+path = "test/exchanges/coinbase/test_orderbook.rs"
 
 [[test]]
-name = "test_liquidations"
-path = "test/bybit/test_liquidations.rs"
+name = "coinbase_http_trades"
+path = "test/exchanges/coinbase/test_trades.rs"
 
-[[example]]
-name = "bybit_streams"
-path = "examples/bybit_streams.rs"
+[[test]]
+name = "coinbase_http_symbols"
+path = "test/exchanges/coinbase/test_symbols.rs"
+
+# --- Binance --- #
+
+[[test]]
+name = "binance_http_connection"
+path = "test/exchanges/binance/test_connection.rs"
+
+[[test]]
+name = "binance_http_orderbook"
+path = "test/exchanges/binance/test_orderbook.rs"
+
+[[test]]
+name = "binance_http_trades"
+path = "test/exchanges/binance/test_trades.rs"
+
+[[test]]
+name = "binance_http_symbols"
+path = "test/exchanges/binance/test_symbols.rs"
+
+# --- Kraken --- #
+
+[[test]]
+name = "kraken_http_connection"
+path = "test/exchanges/kraken/test_connection.rs"
+
+[[test]]
+name = "kraken_http_orderbook"
+path = "test/exchanges/kraken/test_orderbook.rs"
+
+[[test]]
+name = "kraken_http_trades"
+path = "test/exchanges/kraken/test_trades.rs"
+
+[[test]]
+name = "kraken_http_symbols"
+path = "test/exchanges/kraken/test_symbols.rs"
 
 [lints.rust]
 unsafe_code = "forbid"

atelier-data/README.md

@@ -4,6 +4,10 @@
 
 Data feeds light weighted connectivity
 
+- Binance
+- Coinbase
+- Kraken
+
 # Workspace
 
 These are the other published crates members of the workspace: 

atelier-data/src/clients/grpc/mod.rs

@@ -0,0 +1 @@
+pub mod grpc_client;