From c85929dbbc009994827df301d8ac8836edb4bc33 Mon Sep 17 00:00:00 2001 From: jojii Date: Tue, 1 Apr 2025 18:59:28 +0200 Subject: [PATCH] Warn on large upsert requests --- Cargo.toml | 7 ++++++- src/qdrant_client/points.rs | 12 +++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4dddcaf..0bb031a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,13 +19,18 @@ prost-types = "0.13.3" anyhow = "1.0.89" serde = { version = "1.0.210", features = ["derive"], optional = true } serde_json = { version = "1.0.128", optional = true } -reqwest = { version = "0.12.8", optional = true, default-features = false, features = ["stream", "rustls-tls", "http2"] } +reqwest = { version = "0.12.8", optional = true, default-features = false, features = [ + "stream", + "rustls-tls", + "http2", +] } futures = { version = "0.3.31" } tokio = { version = "1.40.0", features = ["rt-multi-thread"] } futures-util = { version = "0.3.31", optional = true } derive_builder = { version = "0.20.2" } thiserror = "1.0.64" semver = "1.0.24" +log = { version = "0.4.27", default-features = false } [dev-dependencies] tonic-build = { version = "0.12.3", features = ["prost"] } diff --git a/src/qdrant_client/points.rs b/src/qdrant_client/points.rs index 023f84e..3d12d2b 100644 --- a/src/qdrant_client/points.rs +++ b/src/qdrant_client/points.rs @@ -14,6 +14,9 @@ use crate::qdrant::{ }; use crate::qdrant_client::{Qdrant, QdrantResult}; +/// Max amount of points that should be upserted in a single request/chunk. +const TOO_MANY_POINTS: usize = 50_000; + /// # Point operations /// /// Manage points and vectors. @@ -103,7 +106,14 @@ impl Qdrant { &self, request: impl Into, ) -> QdrantResult { - let request = &request.into(); + let request: &UpsertPoints = &request.into(); + + if request.points.len() > TOO_MANY_POINTS { + log::warn!( + "Trying to upload more than 50k points. Consider using upsert_points_batch instead." + ); + } + self.with_points_client(|mut points_api| async move { Ok(points_api.upsert(request.clone()).await?.into_inner()) })