Stop usage chips flashing and back off retries for failing providers (#8772)

* Stop usage chips flashing and back off retries for failing providers

Two behaviors made the status bar unusable when any provider was
persistently failing (bad auth, unsupported plan):

1. Every refetch repainted all providers as 'fetching', so a settled
   error chip flashed to a loading "…" chip and back on every cycle.
   withFetchingStatus now keeps settled states (ok/error/unavailable)
   visible until the new result lands; only providers with no settled
   state (first load, explicit account-switch clear) show loading.

2. Error providers on the fast activation-retry lane (claude/codex/
   grok) were retried every 30s on any focus/show/restore event —
   forever. Repeated hits drove Claude's tight-budget usage endpoint
   into 429s, flipping the chip between 'Limited' and its actual error.
   Retries now back off exponentially per consecutive applied failure
   (30s, 60s, 120s, … capped at the 15-minute poll cadence) and reset
   on success or account/target switch.

* Count full fetches as failure-lane retries and keep Grok pane refresh feedback

- Stamp failing providers' activation-retry clocks when a stale-driven full
  fetch runs, so the individual failure lane does not fire a redundant retry
  right after the full fetch already retried them.
- GrokUsagePane: manual refresh spinner/disable is now renderer-local, since
  settled snapshots no longer repaint as 'fetching' during refetches.
- Strengthen the backoff-reset test so it distinguishes a reset streak from a
  stale retry timestamp (CodeRabbit), and add a regression test for the
  full-fetch retry stamping.
This commit is contained in:
Brennan Benson 2026-07-14 16:35:31 -07:00 committed by GitHub
parent 0a0b400ada
commit dfabd85513
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 282 additions and 4 deletions

View File

@ -120,6 +120,19 @@ function unavailableProvider(
}
}
// Why: beforeEach caches snapshot objects whose updatedAt is pinned at suite
// start, so after 5 fake minutes every healthy provider looks stale and every
// activation degrades to a full fetch. Backoff tests that reason about the
// individual retry lane need healthy providers minted fresh at fetch time.
function mockFreshBackgroundProviderFetches(): void {
vi.mocked(fetchCodexRateLimits).mockImplementation(async () => okProvider('codex', 24))
vi.mocked(fetchGeminiRateLimits).mockImplementation(async () => okProvider('gemini', 0))
vi.mocked(fetchOpenCodeGoRateLimits).mockImplementation(async () => okProvider('opencode-go', 0))
vi.mocked(fetchKimiRateLimits).mockImplementation(async () => okProvider('kimi', 0))
vi.mocked(fetchMiniMaxRateLimits).mockImplementation(async () => okProvider('minimax', 0))
vi.mocked(fetchGrokRateLimits).mockImplementation(async () => unavailableProvider('grok'))
}
function serviceInternals(service: RateLimitService): { fetchAll: () => Promise<void> } {
return service as unknown as { fetchAll: () => Promise<void> }
}
@ -393,7 +406,7 @@ describe('RateLimitService', () => {
}
})
it('throttles repeated active-window retries while Claude is still failing', async () => {
it('backs off repeated active-window retries while Claude is still failing', async () => {
vi.useFakeTimers()
try {
vi.mocked(fetchClaudeRateLimits).mockResolvedValue(errorProvider('claude', 'still failing'))
@ -407,6 +420,7 @@ describe('RateLimitService', () => {
await vi.advanceTimersByTimeAsync(1000)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(1)
// First activation recovers immediately (retry timestamps start at 0).
window.emit('focus')
await vi.advanceTimersByTimeAsync(0)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2)
@ -415,11 +429,175 @@ describe('RateLimitService', () => {
await vi.advanceTimersByTimeAsync(0)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2)
// Two consecutive failures: the retry window doubled to 60s, so an
// activation at +30s must not hammer the endpoint again.
await vi.advanceTimersByTimeAsync(30 * 1000)
window.emit('restore')
await vi.advanceTimersByTimeAsync(0)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(30 * 1000)
window.emit('focus')
await vi.advanceTimersByTimeAsync(0)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3)
// Three consecutive failures: 120s window.
await vi.advanceTimersByTimeAsync(60 * 1000)
window.emit('show')
await vi.advanceTimersByTimeAsync(0)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3)
await vi.advanceTimersByTimeAsync(60 * 1000)
window.emit('restore')
await vi.advanceTimersByTimeAsync(0)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(4)
service.stop()
} finally {
vi.useRealTimers()
}
})
it('resets the retry backoff once Claude recovers', async () => {
vi.useFakeTimers()
try {
vi.mocked(fetchClaudeRateLimits)
.mockResolvedValueOnce(errorProvider('claude', 'still failing'))
.mockResolvedValueOnce(errorProvider('claude', 'still failing'))
.mockResolvedValueOnce(okProvider('claude', 12))
.mockResolvedValue(errorProvider('claude', 'failing again'))
mockFreshBackgroundProviderFetches()
const service = new RateLimitService()
const window = new FakeRateLimitWindow()
service.attach(asRateLimitWindow(window))
service.start({ fetchImmediately: false })
await vi.advanceTimersByTimeAsync(1000)
window.emit('focus')
await vi.advanceTimersByTimeAsync(0)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2)
// Recovery fetch succeeds and must clear the failure streak.
await vi.advanceTimersByTimeAsync(60 * 1000)
window.emit('focus')
await vi.advanceTimersByTimeAsync(0)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3)
expect(service.getState().claude?.status).toBe('ok')
// The stale-ok snapshot forces a full refresh, which fails again.
await vi.advanceTimersByTimeAsync(5 * 60 * 1000)
window.emit('focus')
await vi.advanceTimersByTimeAsync(0)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(4)
expect(service.getState().claude?.status).toBe('error')
// Consume the stale active-retry timestamp so the next windows measure
// the post-recovery streak rather than time elapsed before recovery.
window.emit('focus')
await vi.advanceTimersByTimeAsync(0)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(5)
// Post-recovery streak is 2 (not the pre-recovery 4): the window must be
// 60s, so +30s stays throttled and +60s retries.
await vi.advanceTimersByTimeAsync(30 * 1000)
window.emit('focus')
await vi.advanceTimersByTimeAsync(0)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(5)
await vi.advanceTimersByTimeAsync(30 * 1000)
window.emit('focus')
await vi.advanceTimersByTimeAsync(0)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(6)
service.stop()
} finally {
vi.useRealTimers()
}
})
it('counts a stale-driven full fetch as the failing provider retry attempt', async () => {
vi.useFakeTimers()
try {
vi.mocked(fetchClaudeRateLimits).mockImplementation(async () =>
errorProvider('claude', 'still failing')
)
mockFreshBackgroundProviderFetches()
const service = new RateLimitService()
const window = new FakeRateLimitWindow()
service.attach(asRateLimitWindow(window))
service.start({ fetchImmediately: false })
await vi.advanceTimersByTimeAsync(1000)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(1)
window.emit('focus')
await vi.advanceTimersByTimeAsync(0)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2)
// Healthy providers go stale after 5 minutes, so this activation runs a
// full fetch that also retries failing Claude (streak now 3 → 120s).
await vi.advanceTimersByTimeAsync(5 * 60 * 1000)
window.emit('focus')
await vi.advanceTimersByTimeAsync(0)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3)
// Why: the full fetch was itself a retry. An activation moments later
// must not fire the individual failure lane ahead of the backoff window.
window.emit('focus')
await vi.advanceTimersByTimeAsync(0)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3)
// Once the 120s window elapses, the individual retry lane fires again.
await vi.advanceTimersByTimeAsync(120 * 1000)
window.emit('focus')
await vi.advanceTimersByTimeAsync(0)
expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(4)
service.stop()
} finally {
vi.useRealTimers()
}
})
it('keeps a settled error chip settled during background refetches instead of flashing fetching', async () => {
vi.useFakeTimers()
try {
const secondClaude = deferred<ProviderRateLimits>()
vi.mocked(fetchClaudeRateLimits)
.mockResolvedValueOnce(errorProvider('claude', 'still failing'))
.mockImplementationOnce(() => secondClaude.promise)
vi.mocked(fetchCodexRateLimits).mockResolvedValue(okProvider('codex', 24))
const service = new RateLimitService()
const window = new FakeRateLimitWindow()
const claudeStatuses: string[] = []
service.onStateChange((state) => {
if (state.claude) {
claudeStatuses.push(state.claude.status)
}
})
service.attach(asRateLimitWindow(window))
service.start({ fetchImmediately: false })
await vi.advanceTimersByTimeAsync(1000)
expect(service.getState().claude?.status).toBe('error')
// Why: the retry must not repaint the settled error chip as a loading
// "…" chip while the refetch is in flight — that is the flash users see
// every cycle when a provider is stuck failing.
window.emit('focus')
await vi.advanceTimersByTimeAsync(0)
expect(service.getState().claude?.status).toBe('error')
secondClaude.resolve(okProvider('claude', 12))
await vi.advanceTimersByTimeAsync(0)
expect(service.getState().claude?.status).toBe('ok')
const statusesAfterFirstSettle = claudeStatuses.slice(claudeStatuses.indexOf('error'))
expect(statusesAfterFirstSettle).not.toContain('fetching')
service.stop()
} finally {
vi.useRealTimers()

View File

@ -77,6 +77,12 @@ const MIN_POLL_MS = 30 * 1000 // 30 seconds — renderer input should never crea
const MAX_POLL_MS = 2_147_483_647 // Max safe setInterval delay before Node clamps back to 1ms.
const MIN_REFETCH_MS = 5 * 60 * 1000 // 5 minutes — debounce resume/manual refresh bursts
const ACTIVE_FAILURE_REFETCH_MS = MIN_POLL_MS
// Why: a persistent failure (bad auth, unsupported plan) retried at the 30s
// floor hammers provider endpoints — Claude's tight-budget usage endpoint
// starts returning 429s — without ever recovering. Back off per consecutive
// failure, capped at the background poll cadence.
const MAX_ACTIVE_FAILURE_REFETCH_MS = DEFAULT_POLL_MS
const MAX_ACTIVE_FAILURE_STREAK = 8
// Why: these providers have a dedicated fetch cycle, so an activation retry can
// refresh just the failing one. Providers without one force a full fetchAll, so
// their error retries stay on the 5-minute cadence to protect Claude's budget.
@ -152,6 +158,18 @@ export class RateLimitService {
grok: 0,
antigravity: 0
}
// Why: consecutive applied failures per provider drive exponential backoff of
// the fast activation-retry lane; reset on any successful/unavailable result.
private activeFailureStreakByProvider: Record<ActiveRateLimitProvider, number> = {
claude: 0,
codex: 0,
gemini: 0,
'opencode-go': 0,
kimi: 0,
minimax: 0,
grok: 0,
antigravity: 0
}
private mainWindow: BrowserWindow | null = null
private detachWindowListeners: (() => void) | null = null
private isFetching = false
@ -358,6 +376,8 @@ export class RateLimitService {
}
this.codexFetchTarget = nextTarget
this.codexFetchGeneration += 1
// Why: a new account/target starts with a clean retry schedule.
this.activeFailureStreakByProvider.codex = 0
this.inactiveCodexAccountsGeneration += 1
this.pruneInactiveCodexState()
this.lastInactiveCodexFetchAt = 0
@ -377,6 +397,7 @@ export class RateLimitService {
const targetChanged = !this.isSameCodexTarget(this.codexFetchTarget, nextTarget)
this.codexFetchTarget = nextTarget
this.codexFetchGeneration += 1
this.activeFailureStreakByProvider.codex = 0
this.updateState({
...this.state,
codex: this.withFetchingStatus(targetChanged ? null : this.state.codex, 'codex')
@ -426,6 +447,8 @@ export class RateLimitService {
this.inactiveClaudeAccountsGeneration += 1
this.pruneInactiveClaudeState()
this.claudeFetchGeneration += 1
// Why: a new account/target starts with a clean retry schedule.
this.activeFailureStreakByProvider.claude = 0
this.lastInactiveClaudeFetchAt = 0
this.updateState({
...this.state,
@ -440,6 +463,7 @@ export class RateLimitService {
const targetChanged = !this.isSameClaudeTarget(this.claudeFetchTarget, nextTarget)
this.claudeFetchTarget = nextTarget
this.claudeFetchGeneration += 1
this.activeFailureStreakByProvider.claude = 0
this.updateState({
...this.state,
claude: this.withFetchingStatus(targetChanged ? null : this.state.claude, 'claude')
@ -764,7 +788,11 @@ export class RateLimitService {
if (limits.status === 'error') {
const lastRetryAt = this.lastActiveFailureRetryAtByProvider[provider]
const throttleMs = INDIVIDUALLY_REFRESHABLE_PROVIDERS.has(provider)
? ACTIVE_FAILURE_REFETCH_MS
? Math.min(
ACTIVE_FAILURE_REFETCH_MS *
2 ** Math.max(0, this.activeFailureStreakByProvider[provider] - 1),
MAX_ACTIVE_FAILURE_REFETCH_MS
)
: MIN_REFETCH_MS
if (now - lastRetryAt >= throttleMs) {
retryableFailures.push(provider)
@ -783,6 +811,18 @@ export class RateLimitService {
return
}
if (plan.kind === 'full') {
// Why: a full fetch retries failing providers too. Restart their retry
// clocks so the individual failure lane doesn't fire again right after,
// ahead of its backoff window. Skip when a fetch is already in flight —
// fetchAll would no-op and the throttle must not be consumed for free.
if (!this.isFetching) {
const now = Date.now()
for (const { provider, limits } of this.getActiveProviderState()) {
if (limits?.status === 'error') {
this.lastActiveFailureRetryAtByProvider[provider] = now
}
}
}
await this.fetchAll()
return
}
@ -1242,6 +1282,22 @@ export class RateLimitService {
}
}
private trackActiveFailureStreak(
provider: ActiveRateLimitProvider,
fresh: ProviderRateLimits
): void {
if (fresh.status === 'error') {
this.activeFailureStreakByProvider[provider] = Math.min(
this.activeFailureStreakByProvider[provider] + 1,
MAX_ACTIVE_FAILURE_STREAK
)
return
}
if (fresh.status === 'ok' || fresh.status === 'unavailable') {
this.activeFailureStreakByProvider[provider] = 0
}
}
private withFetchingStatus(
current: ProviderRateLimits | null,
provider:
@ -1264,6 +1320,13 @@ export class RateLimitService {
status: 'fetching'
}
}
// Why: repainting a settled chip as "fetching" on every background refetch
// makes the status bar flash "…" → error each retry cycle when a provider
// is persistently failing. Keep the settled state visible until the new
// result lands; only providers with no settled state show a loading chip.
if (current.status === 'ok' || current.status === 'error' || current.status === 'unavailable') {
return current
}
return { ...current, status: 'fetching' }
}
@ -1481,6 +1544,22 @@ export class RateLimitService {
const shouldApplyOpencode = opencodeGeneration === this.opencodeFetchGeneration
const shouldApplyMiniMax = miniMaxGeneration === this.minimaxFetchGeneration
if (shouldApplyClaude) {
this.trackActiveFailureStreak('claude', claude)
}
if (shouldApplyCodex) {
this.trackActiveFailureStreak('codex', codex)
}
this.trackActiveFailureStreak('gemini', gemini)
this.trackActiveFailureStreak('antigravity', antigravity)
if (shouldApplyOpencode) {
this.trackActiveFailureStreak('opencode-go', opencodeGo)
}
this.trackActiveFailureStreak('kimi', kimi)
if (shouldApplyMiniMax) {
this.trackActiveFailureStreak('minimax', miniMax)
}
// Why: account switches can race in-flight Codex fetches. Only apply a
// Codex result if both the selected-account provenance and the request
// generation still match, otherwise an old account could overwrite the
@ -1523,6 +1602,7 @@ export class RateLimitService {
error: grokResult.reason instanceof Error ? grokResult.reason.message : 'Unknown error',
status: 'error'
} satisfies ProviderRateLimits)
this.trackActiveFailureStreak('grok', grok)
this.updateState({
...this.state,
grok: this.applyStalePolicy(grok, previousState.grok)
@ -1575,6 +1655,9 @@ export class RateLimitService {
const shouldApplyCodex =
codexGeneration === this.codexFetchGeneration && codexProvenance === latestCodexProvenance
if (shouldApplyCodex) {
this.trackActiveFailureStreak('codex', codex)
}
this.updateState({
...this.state,
codex: shouldApplyCodex ? this.applyStalePolicy(codex, previousState.codex) : this.state.codex
@ -1630,6 +1713,9 @@ export class RateLimitService {
claudeProvenance === latestClaudeProvenance &&
this.isSameClaudeTarget(claudeTarget, this.claudeFetchTarget)
if (shouldApplyClaude) {
this.trackActiveFailureStreak('claude', claude)
}
this.updateState({
...this.state,
claude: shouldApplyClaude
@ -1669,6 +1755,7 @@ export class RateLimitService {
return
}
this.trackActiveFailureStreak('grok', grok)
this.updateState({
...this.state,
grok: this.applyStalePolicy(grok, previousState.grok)

View File

@ -1,3 +1,4 @@
import { useState } from 'react'
import { CalendarClock, ExternalLink, RefreshCw, Sparkles } from 'lucide-react'
import { translate } from '@/i18n/i18n'
import { useAppStore } from '../../store'
@ -13,6 +14,18 @@ export function GrokUsagePane(): React.JSX.Element {
const openSettingsPage = useAppStore((s) => s.openSettingsPage)
const openSettingsTarget = useAppStore((s) => s.openSettingsTarget)
const recordFeatureInteraction = useAppStore((s) => s.recordFeatureInteraction)
// Why: settled snapshots keep their status during refetches (no 'fetching'
// repaint), so manual-refresh feedback must be renderer-local, matching the
// StatusBar refresh button.
const [isRefreshing, setIsRefreshing] = useState(false)
const handleRefresh = (): void => {
if (isRefreshing) {
return
}
setIsRefreshing(true)
void refreshGrokRateLimits().finally(() => setIsRefreshing(false))
}
const openGrokAccounts = (): void => {
openSettingsTarget({ pane: 'accounts', repoId: null, sectionId: 'accounts-grok' })
@ -57,7 +70,7 @@ export function GrokUsagePane(): React.JSX.Element {
grok?.weekly && typeof grok.weekly.usedPercent === 'number'
? Math.round(grok.weekly.usedPercent)
: null
const isFetching = grok?.status === 'fetching'
const isFetching = isRefreshing || grok?.status === 'fetching'
return (
<div
@ -83,7 +96,7 @@ export function GrokUsagePane(): React.JSX.Element {
<Button
variant="ghost"
size="icon-xs"
onClick={() => void refreshGrokRateLimits()}
onClick={handleRefresh}
disabled={isFetching}
aria-label={translate(
'auto.components.stats.GrokUsagePane.i0j1k2l3m4',