feat(qconnect): recover from terminal states via idle-retry + startup backoff

Add `reconnect_idle_retry_ms` (default 0 = legacy terminate). When > 0, the
transport loop no longer breaks permanently on Exhausted: the new
`idle_retry_after_exhausted` helper idles (shutdown-cancellable), resets the
attempt counter / backoff to base, and retries (gap #7). The real config
sets 60s. The reset happens ONLY after Exhausted plus a deliberate idle
wait, never on connect, so the issue #358 latch is untouched.

Critical cross-file coupling: `service.rs`'s `MaxReconnectAttemptsExceeded`
handler previously dropped the runtime and broke the event loop. With
idle-retry active the transport stays alive and re-emits events, so the
handler now surfaces lifecycle Exhausted to the UI but keeps the runtime and
continues the loop (only the legacy path still drops + breaks). When the
re-armed transport later reaches SessionEstablished, the existing handler
moves the lifecycle back to Connected.

Startup auto-connect now retries on a bounded, monotonic schedule
(`startup_retry_schedule` = 2s/5s/15s/30s) instead of giving up after one
failed attempt (gap #8), re-resolving the transport config each iteration.

cc

crates/qconnect-transport-ws/src/config.rs

@@ -18,6 +18,12 @@ pub struct WsTransportConfig {
     ///
     /// `None` means unlimited (legacy behavior, retained for tests).
     pub reconnect_max_attempts: Option<u32>,
+    /// When `> 0`, reaching `Exhausted` no longer terminates the transport
+    /// loop: it idles this long (shutdown-cancellable), resets the attempt
+    /// counter / backoff to base, and retries instead of giving up (gap #7).
+    /// Default `0` preserves the legacy terminate-on-exhausted behavior used
+    /// by tests; the real config sets 60s.
+    pub reconnect_idle_retry_ms: u64,
     pub connect_timeout_ms: u64,
     pub keepalive_interval_ms: u64,
     pub auto_subscribe: bool,
@@ -34,6 +40,7 @@ impl Default for WsTransportConfig {
             reconnect_backoff_ms: 2_000,
             reconnect_backoff_max_ms: 30_000,
             reconnect_max_attempts: Some(10),
+            reconnect_idle_retry_ms: 0,
             connect_timeout_ms: 10_000,
             keepalive_interval_ms: 30_000,
             auto_subscribe: true,

crates/qconnect-transport-ws/src/transport.rs

@@ -356,7 +356,21 @@ async fn run_native_transport_loop(
                 .await
                 {
                     ReconnectOutcome::Continue => continue,
-                    ReconnectOutcome::Shutdown | ReconnectOutcome::Exhausted => break,
+                    ReconnectOutcome::Shutdown => break,
+                    ReconnectOutcome::Exhausted => {
+                        if idle_retry_after_exhausted(
+                            &mut shutdown_rx,
+                            &mut reconnect_attempt,
+                            &mut backoff,
+                            base_backoff,
+                            config.reconnect_idle_retry_ms,
+                        )
+                        .await
+                        {
+                            continue;
+                        }
+                        break;
+                    }
                 }
             }
             Err(_) => {
@@ -372,7 +386,21 @@ async fn run_native_transport_loop(
                 .await
                 {
                     ReconnectOutcome::Continue => continue,
-                    ReconnectOutcome::Shutdown | ReconnectOutcome::Exhausted => break,
+                    ReconnectOutcome::Shutdown => break,
+                    ReconnectOutcome::Exhausted => {
+                        if idle_retry_after_exhausted(
+                            &mut shutdown_rx,
+                            &mut reconnect_attempt,
+                            &mut backoff,
+                            base_backoff,
+                            config.reconnect_idle_retry_ms,
+                        )
+                        .await
+                        {
+                            continue;
+                        }
+                        break;
+                    }
                 }
             }
         };
@@ -417,7 +445,21 @@ async fn run_native_transport_loop(
                 .await
                 {
                     ReconnectOutcome::Continue => continue,
-                    ReconnectOutcome::Shutdown | ReconnectOutcome::Exhausted => break,
+                    ReconnectOutcome::Shutdown => break,
+                    ReconnectOutcome::Exhausted => {
+                        if idle_retry_after_exhausted(
+                            &mut shutdown_rx,
+                            &mut reconnect_attempt,
+                            &mut backoff,
+                            base_backoff,
+                            config.reconnect_idle_retry_ms,
+                        )
+                        .await
+                        {
+                            continue;
+                        }
+                        break;
+                    }
                 }
             }
             emit(&events_tx, TransportEvent::Authenticated);
@@ -451,7 +493,21 @@ async fn run_native_transport_loop(
             .await
             {
                 ReconnectOutcome::Continue => continue,
-                ReconnectOutcome::Shutdown | ReconnectOutcome::Exhausted => break,
+                ReconnectOutcome::Shutdown => break,
+                ReconnectOutcome::Exhausted => {
+                    if idle_retry_after_exhausted(
+                        &mut shutdown_rx,
+                        &mut reconnect_attempt,
+                        &mut backoff,
+                        base_backoff,
+                        config.reconnect_idle_retry_ms,
+                    )
+                    .await
+                    {
+                        continue;
+                    }
+                    break;
+                }
             }
         }
 
@@ -489,7 +545,21 @@ async fn run_native_transport_loop(
                 .await
                 {
                     ReconnectOutcome::Continue => continue,
-                    ReconnectOutcome::Shutdown | ReconnectOutcome::Exhausted => break,
+                    ReconnectOutcome::Shutdown => break,
+                    ReconnectOutcome::Exhausted => {
+                        if idle_retry_after_exhausted(
+                            &mut shutdown_rx,
+                            &mut reconnect_attempt,
+                            &mut backoff,
+                            base_backoff,
+                            config.reconnect_idle_retry_ms,
+                        )
+                        .await
+                        {
+                            continue;
+                        }
+                        break;
+                    }
                 }
             }
             emit(&events_tx, TransportEvent::Subscribed);
@@ -635,7 +705,21 @@ async fn run_native_transport_loop(
         .await
         {
             ReconnectOutcome::Continue => continue,
-            ReconnectOutcome::Shutdown | ReconnectOutcome::Exhausted => break,
+            ReconnectOutcome::Shutdown => break,
+            ReconnectOutcome::Exhausted => {
+                if idle_retry_after_exhausted(
+                    &mut shutdown_rx,
+                    &mut reconnect_attempt,
+                    &mut backoff,
+                    base_backoff,
+                    config.reconnect_idle_retry_ms,
+                )
+                .await
+                {
+                    continue;
+                }
+                break;
+            }
         }
     }
 
@@ -701,6 +785,39 @@ async fn handle_reconnect_delay(
     ReconnectOutcome::Continue
 }
 
+/// Long-backoff idle retry after the reconnect loop has Exhausted its bounded
+/// attempts (gap #7). When `idle_retry_ms == 0` this is a no-op returning
+/// `false` (legacy terminate behavior). Otherwise it sleeps `idle_retry_ms`
+/// (cancellable by shutdown); on a real shutdown it returns `false` without
+/// re-arming. On a clean wake it resets the attempt counter / backoff to base
+/// and returns `true` so the caller can `continue` the loop.
+///
+/// NOTE: this resets the counters ONLY after Exhausted + a deliberate idle
+/// wait — never on connect. The issue #358 latch (reset only on the first
+/// SESSION_STATE) is therefore untouched.
+async fn idle_retry_after_exhausted(
+    shutdown_rx: &mut watch::Receiver<bool>,
+    reconnect_attempt: &mut u32,
+    backoff: &mut u64,
+    base_backoff: u64,
+    idle_retry_ms: u64,
+) -> bool {
+    if idle_retry_ms == 0 {
+        return false;
+    }
+    tokio::select! {
+        _ = tokio::time::sleep(Duration::from_millis(idle_retry_ms)) => {}
+        _ = shutdown_rx.changed() => {
+            if *shutdown_rx.borrow() {
+                return false;
+            }
+        }
+    }
+    *reconnect_attempt = 0;
+    *backoff = base_backoff;
+    true
+}
+
 /// Side-channel result from `handle_incoming_binary`: we need to know whether
 /// any decoded frame contained a `MESSAGE_TYPE_SRVR_CTRL_SESSION_STATE` event,
 /// because that is the signal the reconnect loop uses to reset its backoff
@@ -1061,6 +1178,28 @@ mod tests {
         assert!(keepalive_is_stale(now, now - deadline - 1, 2, deadline));
     }
 
+    /// gap #7: with `idle_retry_ms == 0` the helper is a no-op — it returns
+    /// `false` (terminate) and leaves the counters untouched.
+    #[tokio::test]
+    async fn idle_retry_disabled_when_interval_zero() {
+        let (_tx, mut rx) = watch::channel(false);
+        let mut attempt = 5u32;
+        let mut backoff = 8u64;
+        assert!(!idle_retry_after_exhausted(&mut rx, &mut attempt, &mut backoff, 2, 0).await);
+        assert_eq!(attempt, 5);
+    }
+
+    /// gap #7: a shutdown signalled during the idle wait must cancel the retry
+    /// and NOT re-arm (returns `false`, counters untouched).
+    #[tokio::test]
+    async fn idle_retry_cancelled_by_shutdown_does_not_rearm() {
+        let (tx, mut rx) = watch::channel(false);
+        let _ = tx.send(true);
+        let mut attempt = 5u32;
+        let mut backoff = 8u64;
+        assert!(!idle_retry_after_exhausted(&mut rx, &mut attempt, &mut backoff, 2, 60_000).await);
+    }
+
     #[test]
     fn qcloud_frame_roundtrip() {
         let payload = vec![0x01, 0x02, 0x03, 0x04];