refactor: tune circuit breaker config (#6202)

At the moment it takes way to long to half-open and close circuits ones
they were opened.

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>

querier/src/ingester/circuit_breaker.rs

@@ -242,13 +242,14 @@ impl CircuitBreakerFlightClient {
         time_provider: Arc<dyn TimeProvider>,
         metric_registry: Arc<Registry>,
         open_circuit_after_n_errors: u64,
+        backoff_config: BackoffConfig,
     ) -> Self {
         Self {
             inner,
             open_circuit_after_n_errors,
             time_provider,
             metric_registry,
-            backoff_config: BackoffConfig::default(),
+            backoff_config,
             circuits: Default::default(),
             rng_overwrite: None,
         }
@@ -1110,6 +1111,7 @@ mod tests {
                 Arc::clone(&time_provider) as _,
                 Arc::clone(&metric_registry),
                 2,
+                BackoffConfig::default(),
             );
 
             // set up "RNG" that always generates the maximum, so we can test things easier

querier/src/ingester/mod.rs

@@ -145,15 +145,27 @@ pub fn create_ingester_connections_by_shard(
     catalog_cache: Arc<CatalogCache>,
     open_circuit_after_n_errors: u64,
 ) -> Arc<dyn IngesterConnection> {
+    // This backoff config is used to retry requests for a specific table-scoped query.
+    let retry_backoff_config = BackoffConfig {
+        init_backoff: Duration::from_millis(100),
+        max_backoff: Duration::from_secs(1),
+        base: 3.0,
+        deadline: Some(Duration::from_secs(10)),
+    };
+
+    // This backoff config is used to half-open the circuit after it was opened. Circuits are ingester-scoped.
+    let circuit_breaker_backoff_config = BackoffConfig {
+        init_backoff: Duration::from_secs(1),
+        max_backoff: Duration::from_secs(60),
+        base: 3.0,
+        deadline: None,
+    };
+
     Arc::new(IngesterConnectionImpl::by_shard(
         shard_to_ingesters,
         catalog_cache,
-        BackoffConfig {
-            init_backoff: Duration::from_millis(100),
-            max_backoff: Duration::from_secs(1),
-            base: 3.0,
-            deadline: Some(Duration::from_secs(10)),
-        },
+        retry_backoff_config,
+        circuit_breaker_backoff_config,
         open_circuit_after_n_errors,
     ))
 }
@@ -341,7 +353,8 @@ impl IngesterConnectionImpl {
     pub fn by_shard(
         shard_to_ingesters: HashMap<ShardIndex, IngesterMapping>,
         catalog_cache: Arc<CatalogCache>,
-        backoff_config: BackoffConfig,
+        retry_backoff_config: BackoffConfig,
+        circuit_breaker_backoff_config: BackoffConfig,
         open_circuit_after_n_errors: u64,
     ) -> Self {
         let flight_client = Arc::new(FlightClientImpl::new());
@@ -350,12 +363,13 @@ impl IngesterConnectionImpl {
             catalog_cache.time_provider(),
             catalog_cache.metric_registry(),
             open_circuit_after_n_errors,
+            circuit_breaker_backoff_config,
         ));
         Self::by_shard_with_flight_client(
             shard_to_ingesters,
             flight_client,
             catalog_cache,
-            backoff_config,
+            retry_backoff_config,
         )
     }