Advanced reconnection strategy

- added reconnection config with 3 possible strategies: constant, linear or exponential
- BREAKING CHANGE: removed client::Config::max_command_attempts
fix #66

Author: mcatanzariti

src/client/client.rs

@@ -33,7 +33,7 @@ use crate::{
 };
 use futures_channel::{mpsc, oneshot};
 use futures_util::Stream;
-use log::trace;
+use log::{info, trace};
 use serde::de::DeserializeOwned;
 use std::{
     future::IntoFuture,
@@ -269,8 +269,10 @@ impl Client {
     fn send_message(&self, message: Message) -> Result<()> {
         if let Some(msg_sender) = &self.msg_sender as &Option<MsgSender> {
             trace!("Will enqueue message: {message:?}");
-            msg_sender.unbounded_send(message)?;
-            Ok(())
+            Ok(msg_sender.unbounded_send(message).map_err(|e| {
+                info!("{}", e.to_string());
+                Error::Client("Disconnected from server".to_string())
+            })?)
         } else {
             Err(Error::Client(
                 "Invalid channel to send messages to the network handler".to_owned(),

src/client/config.rs

@@ -13,7 +13,6 @@ const DEFAULT_AUTO_RESUBSCRTBE: bool = true;
 const DEFAULT_AUTO_REMONITOR: bool = true;
 const DEFAULT_KEEP_ALIVE: Option<Duration> = None;
 const DEFAULT_NO_DELAY: bool = true;
-const DEFAULT_MAX_COMMAND_ATTEMPTS: usize = 3;
 const DEFAULT_RETRY_ON_ERROR: bool = false;
 
 type Uri<'a> = (
@@ -84,8 +83,6 @@ pub struct Config {
     ///
     /// See [`TcpStream::set_nodelay`](https://docs.rs/tokio/latest/tokio/net/struct.TcpStream.html#method.set_nodelay)    
     pub no_delay: bool,
-    /// Maximum number of retry attempts to send a command to the Redis server (default `3`).
-    pub max_command_attempts: usize,
     /// Defines the default strategy for retries on network error (default `false`):
     /// * `true` - retry sending the command/batch of commands on network error
     /// * `false` - do not retry sending the command/batch of commands on network error
@@ -99,6 +96,8 @@ pub struct Config {
     /// * [`Client::send_and_forget`](crate::client::Client::send_and_forget)
     /// * [`Client::send_batch`](crate::client::Client::send_batch)
     pub retry_on_error: bool,
+    /// Reconnection configuration (Constant, Linear or Exponential)
+    pub reconnection: ReconnectionConfig,
 }
 
 impl Default for Config {
@@ -117,8 +116,8 @@ impl Default for Config {
             connection_name: String::from(""),
             keep_alive: DEFAULT_KEEP_ALIVE,
             no_delay: DEFAULT_NO_DELAY,
-            max_command_attempts: DEFAULT_MAX_COMMAND_ATTEMPTS,
             retry_on_error: DEFAULT_RETRY_ON_ERROR,
+            reconnection: Default::default(),
         }
     }
 }
@@ -315,12 +314,6 @@ impl Config {
                 }
             }
 
-            if let Some(max_command_attempts) = query.remove("max_command_attempts") {
-                if let Ok(max_command_attempts) = max_command_attempts.parse::<usize>() {
-                    config.max_command_attempts = max_command_attempts;
-                }
-            }
-
             if let Some(retry_on_error) = query.remove("retry_on_error") {
                 if let Ok(retry_on_error) = retry_on_error.parse::<bool>() {
                     config.retry_on_error = retry_on_error;
@@ -583,19 +576,6 @@ impl ToString for Config {
             s.push_str(&format!("no_delay={}", self.no_delay));
         }
 
-        if self.max_command_attempts != DEFAULT_MAX_COMMAND_ATTEMPTS {
-            if !query_separator {
-                query_separator = true;
-                s.push('?');
-            } else {
-                s.push('&');
-            }
-            s.push_str(&format!(
-                "max_command_attempts={}",
-                self.max_command_attempts
-            ));
-        }
-
         if self.retry_on_error != DEFAULT_RETRY_ON_ERROR {
             if !query_separator {
                 query_separator = true;
@@ -874,3 +854,111 @@ impl IntoConfig for Url {
         Config::from_uri(self)
     }
 }
+
+/// The type of reconnection policy to use. This will apply to every connection used by the client.
+/// This code has been mostly inpisred by [fred ReconnectPolicy](https://docs.rs/fred/latest/fred/types/enum.ReconnectPolicy.html)
+#[derive(Debug, Clone)]
+pub enum ReconnectionConfig {
+    /// Wait a constant amount of time between reconnection attempts, in ms.
+    Constant {
+        /// Maximum number of attemps, set `0` to retry forever.
+        max_attempts: u32,
+        /// Delay in ms to wait between reconnection attempts
+        delay: u32,
+        /// Add jitter in ms to each delay
+        jitter: u32,
+    },
+    /// Backoff reconnection attempts linearly, adding `delay` each time.
+    Linear {
+        /// Maximum number of attemps, set `0` to retry forever.
+        max_attempts: u32,
+        /// Maximum delay in ms
+        max_delay: u32,
+        /// Delay in ms to add to the total waiting time at each attemp
+        delay: u32,
+        /// Add jitter in ms to each delay
+        jitter: u32,
+    },
+    /// Backoff reconnection attempts exponentially, multiplying the last delay by `multiplicative_factor` each time.
+    /// see https://en.wikipedia.org/wiki/Exponential_backoff
+    Exponential {
+        /// Maximum number of attemps, set `0` to retry forever.
+        max_attempts: u32,
+        /// Minimum delay in ms
+        min_delay: u32,
+        /// Maximum delay in ms
+        max_delay: u32,
+        // multiplicative factor
+        multiplicative_factor: u32,
+        /// Add jitter in ms to each delay
+        jitter: u32,
+    },
+}
+
+/// The default amount of jitter when waiting to reconnect.
+const DEFAULT_JITTER_MS: u32 = 100;
+const DEFAULT_DELAY_MS: u32 = 1000;
+
+impl Default for ReconnectionConfig {
+    fn default() -> Self {
+        Self::Constant {
+            max_attempts: 0,
+            delay: DEFAULT_DELAY_MS,
+            jitter: DEFAULT_JITTER_MS,
+        }
+    }
+}
+
+impl ReconnectionConfig {
+    /// Create a new reconnect policy with a constant backoff.
+    pub fn new_constant(max_attempts: u32, delay: u32) -> Self {
+        Self::Constant {
+            max_attempts,
+            delay,
+            jitter: DEFAULT_JITTER_MS,
+        }
+    }
+
+    /// Create a new reconnect policy with a linear backoff.
+    pub fn new_linear(max_attempts: u32, max_delay: u32, delay: u32) -> Self {
+        Self::Linear {
+            max_attempts,
+            max_delay,
+            delay,
+            jitter: DEFAULT_JITTER_MS,
+        }
+    }
+
+    /// Create a new reconnect policy with an exponential backoff.
+    pub fn new_exponential(
+        max_attempts: u32,
+        min_delay: u32,
+        max_delay: u32,
+        multiplicative_factor: u32,
+    ) -> Self {
+        Self::Exponential {
+            max_delay,
+            max_attempts,
+            min_delay,
+            multiplicative_factor,
+            jitter: DEFAULT_JITTER_MS,
+        }
+    }
+
+    /// Set the amount of jitter to add to each reconnection delay.
+    ///
+    /// Default: 100 ms
+    pub fn set_jitter(&mut self, jitter_ms: u32) {
+        match self {
+            Self::Constant { ref mut jitter, .. } => {
+                *jitter = jitter_ms;
+            }
+            Self::Linear { ref mut jitter, .. } => {
+                *jitter = jitter_ms;
+            }
+            Self::Exponential { ref mut jitter, .. } => {
+                *jitter = jitter_ms;
+            }
+        }
+    }
+}

src/client/message.rs

@@ -1,14 +1,19 @@
+use log::warn;
 use smallvec::SmallVec;
 
-use crate::{resp::Command, PushSender, PubSubSender, RetryReason, network::{ResultSender, ResultsSender}};
+use crate::{
+    network::{ResultSender, ResultsSender},
+    resp::Command,
+    Error, PubSubSender, PushSender, RetryReason,
+};
 
 #[cfg(debug_assertions)]
 use std::sync::atomic::{AtomicUsize, Ordering};
 
 #[cfg(debug_assertions)]
 static MESSAGE_SEQUENCE_COUNTER: AtomicUsize = AtomicUsize::new(0);
 
-#[allow(clippy::large_enum_variant)] 
+#[allow(clippy::large_enum_variant)]
 #[derive(Debug)]
 pub(crate) enum Commands {
     None,
@@ -24,6 +29,26 @@ impl Commands {
             Commands::Batch(commands, _) => commands.len(),
         }
     }
+
+    pub fn send_error(self, tag: &str, error: Error) {
+        match self {
+            Commands::Single(_, Some(result_sender)) => {
+                if let Err(e) = result_sender.send(Err(error)) {
+                    warn!(
+                    "[{tag}] Cannot send value to caller because receiver is not there anymore: {e:?}",
+                );
+                }
+            }
+            Commands::Batch(_, results_sender) => {
+                if let Err(e) = results_sender.send(Err(error)) {
+                    warn!(
+                    "[{tag}] Cannot send value to caller because receiver is not there anymore: {e:?}",
+                );
+                }
+            }
+            _ => (),
+        }
+    }
 }
 
 impl IntoIterator for Commands {
@@ -65,7 +90,7 @@ impl<'a> IntoIterator for &'a mut Commands {
     }
 }
 
-#[allow(clippy::large_enum_variant)] 
+#[allow(clippy::large_enum_variant)]
 pub enum CommandsIterator {
     Single(Option<Command>),
     Batch(std::vec::IntoIter<Command>),
@@ -123,7 +148,7 @@ pub(crate) struct Message {
     pub retry_on_error: bool,
     #[cfg(debug_assertions)]
     #[allow(unused)]
-    pub (crate) message_seq: usize,
+    pub(crate) message_seq: usize,
 }
 
 impl Message {
@@ -154,7 +179,11 @@ impl Message {
     }
 
     #[inline(always)]
-    pub fn batch(commands: Vec<Command>, results_sender: ResultsSender, retry_on_error: bool) -> Self {
+    pub fn batch(
+        commands: Vec<Command>,
+        results_sender: ResultsSender,
+        retry_on_error: bool,
+    ) -> Self {
         Message {
             commands: Commands::Batch(commands, results_sender),
             pub_sub_senders: None,
@@ -184,11 +213,7 @@ impl Message {
     }
 
     #[inline(always)]
-    pub fn monitor(
-        command: Command,
-        result_sender: ResultSender,
-        push_sender: PushSender,
-    ) -> Self {
+    pub fn monitor(command: Command, result_sender: ResultSender, push_sender: PushSender) -> Self {
         Message {
             commands: Commands::Single(command, Some(result_sender)),
             pub_sub_senders: None,

src/error.rs

@@ -1,6 +1,6 @@
-use crate::{client::Message, Result};
+use crate::Result;
 use futures_channel::{
-    mpsc::{self, TrySendError},
+    mpsc::{self},
     oneshot,
 };
 use smallvec::SmallVec;
@@ -102,12 +102,6 @@ impl From<std::io::Error> for Error {
     }
 }
 
-impl From<TrySendError<Message>> for Error {
-    fn from(e: TrySendError<Message>) -> Self {
-        Error::Client(e.to_string())
-    }
-}
-
 impl From<oneshot::Canceled> for Error {
     fn from(e: oneshot::Canceled) -> Self {
         Error::Client(e.to_string())

src/network/mod.rs

@@ -3,6 +3,7 @@ mod cluster_connection;
 mod command_info_manager;
 mod connection;
 mod network_handler;
+mod reconnection_state;
 mod sentinel_connection;
 mod standalone_connection;
 mod util;
@@ -13,6 +14,7 @@ pub(crate) use cluster_connection::*;
 pub(crate) use command_info_manager::*;
 pub(crate) use connection::*;
 pub(crate) use network_handler::*;
+pub(crate) use reconnection_state::*;
 pub(crate) use sentinel_connection::*;
 pub(crate) use standalone_connection::*;
 pub(crate) use version::*;