diff --git a/src/main/rate-limits/service.test.ts b/src/main/rate-limits/service.test.ts index 261a94f36..9bd66cb80 100644 --- a/src/main/rate-limits/service.test.ts +++ b/src/main/rate-limits/service.test.ts @@ -120,6 +120,19 @@ function unavailableProvider( } } +// Why: beforeEach caches snapshot objects whose updatedAt is pinned at suite +// start, so after 5 fake minutes every healthy provider looks stale and every +// activation degrades to a full fetch. Backoff tests that reason about the +// individual retry lane need healthy providers minted fresh at fetch time. +function mockFreshBackgroundProviderFetches(): void { + vi.mocked(fetchCodexRateLimits).mockImplementation(async () => okProvider('codex', 24)) + vi.mocked(fetchGeminiRateLimits).mockImplementation(async () => okProvider('gemini', 0)) + vi.mocked(fetchOpenCodeGoRateLimits).mockImplementation(async () => okProvider('opencode-go', 0)) + vi.mocked(fetchKimiRateLimits).mockImplementation(async () => okProvider('kimi', 0)) + vi.mocked(fetchMiniMaxRateLimits).mockImplementation(async () => okProvider('minimax', 0)) + vi.mocked(fetchGrokRateLimits).mockImplementation(async () => unavailableProvider('grok')) +} + function serviceInternals(service: RateLimitService): { fetchAll: () => Promise } { return service as unknown as { fetchAll: () => Promise } } @@ -393,7 +406,7 @@ describe('RateLimitService', () => { } }) - it('throttles repeated active-window retries while Claude is still failing', async () => { + it('backs off repeated active-window retries while Claude is still failing', async () => { vi.useFakeTimers() try { vi.mocked(fetchClaudeRateLimits).mockResolvedValue(errorProvider('claude', 'still failing')) @@ -407,6 +420,7 @@ describe('RateLimitService', () => { await vi.advanceTimersByTimeAsync(1000) expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(1) + // First activation recovers immediately (retry timestamps start at 0). window.emit('focus') await vi.advanceTimersByTimeAsync(0) expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2) @@ -415,11 +429,175 @@ describe('RateLimitService', () => { await vi.advanceTimersByTimeAsync(0) expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2) + // Two consecutive failures: the retry window doubled to 60s, so an + // activation at +30s must not hammer the endpoint again. await vi.advanceTimersByTimeAsync(30 * 1000) window.emit('restore') await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2) + + await vi.advanceTimersByTimeAsync(30 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3) + // Three consecutive failures: 120s window. + await vi.advanceTimersByTimeAsync(60 * 1000) + window.emit('show') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3) + + await vi.advanceTimersByTimeAsync(60 * 1000) + window.emit('restore') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(4) + + service.stop() + } finally { + vi.useRealTimers() + } + }) + + it('resets the retry backoff once Claude recovers', async () => { + vi.useFakeTimers() + try { + vi.mocked(fetchClaudeRateLimits) + .mockResolvedValueOnce(errorProvider('claude', 'still failing')) + .mockResolvedValueOnce(errorProvider('claude', 'still failing')) + .mockResolvedValueOnce(okProvider('claude', 12)) + .mockResolvedValue(errorProvider('claude', 'failing again')) + mockFreshBackgroundProviderFetches() + + const service = new RateLimitService() + const window = new FakeRateLimitWindow() + service.attach(asRateLimitWindow(window)) + service.start({ fetchImmediately: false }) + + await vi.advanceTimersByTimeAsync(1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2) + + // Recovery fetch succeeds and must clear the failure streak. + await vi.advanceTimersByTimeAsync(60 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3) + expect(service.getState().claude?.status).toBe('ok') + + // The stale-ok snapshot forces a full refresh, which fails again. + await vi.advanceTimersByTimeAsync(5 * 60 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(4) + expect(service.getState().claude?.status).toBe('error') + + // Consume the stale active-retry timestamp so the next windows measure + // the post-recovery streak rather than time elapsed before recovery. + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(5) + + // Post-recovery streak is 2 (not the pre-recovery 4): the window must be + // 60s, so +30s stays throttled and +60s retries. + await vi.advanceTimersByTimeAsync(30 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(5) + + await vi.advanceTimersByTimeAsync(30 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(6) + + service.stop() + } finally { + vi.useRealTimers() + } + }) + + it('counts a stale-driven full fetch as the failing provider retry attempt', async () => { + vi.useFakeTimers() + try { + vi.mocked(fetchClaudeRateLimits).mockImplementation(async () => + errorProvider('claude', 'still failing') + ) + mockFreshBackgroundProviderFetches() + + const service = new RateLimitService() + const window = new FakeRateLimitWindow() + service.attach(asRateLimitWindow(window)) + service.start({ fetchImmediately: false }) + + await vi.advanceTimersByTimeAsync(1000) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(1) + + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2) + + // Healthy providers go stale after 5 minutes, so this activation runs a + // full fetch that also retries failing Claude (streak now 3 → 120s). + await vi.advanceTimersByTimeAsync(5 * 60 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3) + + // Why: the full fetch was itself a retry. An activation moments later + // must not fire the individual failure lane ahead of the backoff window. + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3) + + // Once the 120s window elapses, the individual retry lane fires again. + await vi.advanceTimersByTimeAsync(120 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(4) + + service.stop() + } finally { + vi.useRealTimers() + } + }) + + it('keeps a settled error chip settled during background refetches instead of flashing fetching', async () => { + vi.useFakeTimers() + try { + const secondClaude = deferred() + vi.mocked(fetchClaudeRateLimits) + .mockResolvedValueOnce(errorProvider('claude', 'still failing')) + .mockImplementationOnce(() => secondClaude.promise) + vi.mocked(fetchCodexRateLimits).mockResolvedValue(okProvider('codex', 24)) + + const service = new RateLimitService() + const window = new FakeRateLimitWindow() + const claudeStatuses: string[] = [] + service.onStateChange((state) => { + if (state.claude) { + claudeStatuses.push(state.claude.status) + } + }) + service.attach(asRateLimitWindow(window)) + service.start({ fetchImmediately: false }) + + await vi.advanceTimersByTimeAsync(1000) + expect(service.getState().claude?.status).toBe('error') + + // Why: the retry must not repaint the settled error chip as a loading + // "…" chip while the refetch is in flight — that is the flash users see + // every cycle when a provider is stuck failing. + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(service.getState().claude?.status).toBe('error') + + secondClaude.resolve(okProvider('claude', 12)) + await vi.advanceTimersByTimeAsync(0) + expect(service.getState().claude?.status).toBe('ok') + + const statusesAfterFirstSettle = claudeStatuses.slice(claudeStatuses.indexOf('error')) + expect(statusesAfterFirstSettle).not.toContain('fetching') + service.stop() } finally { vi.useRealTimers() diff --git a/src/main/rate-limits/service.ts b/src/main/rate-limits/service.ts index cd301cfe5..84fb016fe 100644 --- a/src/main/rate-limits/service.ts +++ b/src/main/rate-limits/service.ts @@ -77,6 +77,12 @@ const MIN_POLL_MS = 30 * 1000 // 30 seconds — renderer input should never crea const MAX_POLL_MS = 2_147_483_647 // Max safe setInterval delay before Node clamps back to 1ms. const MIN_REFETCH_MS = 5 * 60 * 1000 // 5 minutes — debounce resume/manual refresh bursts const ACTIVE_FAILURE_REFETCH_MS = MIN_POLL_MS +// Why: a persistent failure (bad auth, unsupported plan) retried at the 30s +// floor hammers provider endpoints — Claude's tight-budget usage endpoint +// starts returning 429s — without ever recovering. Back off per consecutive +// failure, capped at the background poll cadence. +const MAX_ACTIVE_FAILURE_REFETCH_MS = DEFAULT_POLL_MS +const MAX_ACTIVE_FAILURE_STREAK = 8 // Why: these providers have a dedicated fetch cycle, so an activation retry can // refresh just the failing one. Providers without one force a full fetchAll, so // their error retries stay on the 5-minute cadence to protect Claude's budget. @@ -152,6 +158,18 @@ export class RateLimitService { grok: 0, antigravity: 0 } + // Why: consecutive applied failures per provider drive exponential backoff of + // the fast activation-retry lane; reset on any successful/unavailable result. + private activeFailureStreakByProvider: Record = { + claude: 0, + codex: 0, + gemini: 0, + 'opencode-go': 0, + kimi: 0, + minimax: 0, + grok: 0, + antigravity: 0 + } private mainWindow: BrowserWindow | null = null private detachWindowListeners: (() => void) | null = null private isFetching = false @@ -358,6 +376,8 @@ export class RateLimitService { } this.codexFetchTarget = nextTarget this.codexFetchGeneration += 1 + // Why: a new account/target starts with a clean retry schedule. + this.activeFailureStreakByProvider.codex = 0 this.inactiveCodexAccountsGeneration += 1 this.pruneInactiveCodexState() this.lastInactiveCodexFetchAt = 0 @@ -377,6 +397,7 @@ export class RateLimitService { const targetChanged = !this.isSameCodexTarget(this.codexFetchTarget, nextTarget) this.codexFetchTarget = nextTarget this.codexFetchGeneration += 1 + this.activeFailureStreakByProvider.codex = 0 this.updateState({ ...this.state, codex: this.withFetchingStatus(targetChanged ? null : this.state.codex, 'codex') @@ -426,6 +447,8 @@ export class RateLimitService { this.inactiveClaudeAccountsGeneration += 1 this.pruneInactiveClaudeState() this.claudeFetchGeneration += 1 + // Why: a new account/target starts with a clean retry schedule. + this.activeFailureStreakByProvider.claude = 0 this.lastInactiveClaudeFetchAt = 0 this.updateState({ ...this.state, @@ -440,6 +463,7 @@ export class RateLimitService { const targetChanged = !this.isSameClaudeTarget(this.claudeFetchTarget, nextTarget) this.claudeFetchTarget = nextTarget this.claudeFetchGeneration += 1 + this.activeFailureStreakByProvider.claude = 0 this.updateState({ ...this.state, claude: this.withFetchingStatus(targetChanged ? null : this.state.claude, 'claude') @@ -764,7 +788,11 @@ export class RateLimitService { if (limits.status === 'error') { const lastRetryAt = this.lastActiveFailureRetryAtByProvider[provider] const throttleMs = INDIVIDUALLY_REFRESHABLE_PROVIDERS.has(provider) - ? ACTIVE_FAILURE_REFETCH_MS + ? Math.min( + ACTIVE_FAILURE_REFETCH_MS * + 2 ** Math.max(0, this.activeFailureStreakByProvider[provider] - 1), + MAX_ACTIVE_FAILURE_REFETCH_MS + ) : MIN_REFETCH_MS if (now - lastRetryAt >= throttleMs) { retryableFailures.push(provider) @@ -783,6 +811,18 @@ export class RateLimitService { return } if (plan.kind === 'full') { + // Why: a full fetch retries failing providers too. Restart their retry + // clocks so the individual failure lane doesn't fire again right after, + // ahead of its backoff window. Skip when a fetch is already in flight — + // fetchAll would no-op and the throttle must not be consumed for free. + if (!this.isFetching) { + const now = Date.now() + for (const { provider, limits } of this.getActiveProviderState()) { + if (limits?.status === 'error') { + this.lastActiveFailureRetryAtByProvider[provider] = now + } + } + } await this.fetchAll() return } @@ -1242,6 +1282,22 @@ export class RateLimitService { } } + private trackActiveFailureStreak( + provider: ActiveRateLimitProvider, + fresh: ProviderRateLimits + ): void { + if (fresh.status === 'error') { + this.activeFailureStreakByProvider[provider] = Math.min( + this.activeFailureStreakByProvider[provider] + 1, + MAX_ACTIVE_FAILURE_STREAK + ) + return + } + if (fresh.status === 'ok' || fresh.status === 'unavailable') { + this.activeFailureStreakByProvider[provider] = 0 + } + } + private withFetchingStatus( current: ProviderRateLimits | null, provider: @@ -1264,6 +1320,13 @@ export class RateLimitService { status: 'fetching' } } + // Why: repainting a settled chip as "fetching" on every background refetch + // makes the status bar flash "…" → error each retry cycle when a provider + // is persistently failing. Keep the settled state visible until the new + // result lands; only providers with no settled state show a loading chip. + if (current.status === 'ok' || current.status === 'error' || current.status === 'unavailable') { + return current + } return { ...current, status: 'fetching' } } @@ -1481,6 +1544,22 @@ export class RateLimitService { const shouldApplyOpencode = opencodeGeneration === this.opencodeFetchGeneration const shouldApplyMiniMax = miniMaxGeneration === this.minimaxFetchGeneration + if (shouldApplyClaude) { + this.trackActiveFailureStreak('claude', claude) + } + if (shouldApplyCodex) { + this.trackActiveFailureStreak('codex', codex) + } + this.trackActiveFailureStreak('gemini', gemini) + this.trackActiveFailureStreak('antigravity', antigravity) + if (shouldApplyOpencode) { + this.trackActiveFailureStreak('opencode-go', opencodeGo) + } + this.trackActiveFailureStreak('kimi', kimi) + if (shouldApplyMiniMax) { + this.trackActiveFailureStreak('minimax', miniMax) + } + // Why: account switches can race in-flight Codex fetches. Only apply a // Codex result if both the selected-account provenance and the request // generation still match, otherwise an old account could overwrite the @@ -1523,6 +1602,7 @@ export class RateLimitService { error: grokResult.reason instanceof Error ? grokResult.reason.message : 'Unknown error', status: 'error' } satisfies ProviderRateLimits) + this.trackActiveFailureStreak('grok', grok) this.updateState({ ...this.state, grok: this.applyStalePolicy(grok, previousState.grok) @@ -1575,6 +1655,9 @@ export class RateLimitService { const shouldApplyCodex = codexGeneration === this.codexFetchGeneration && codexProvenance === latestCodexProvenance + if (shouldApplyCodex) { + this.trackActiveFailureStreak('codex', codex) + } this.updateState({ ...this.state, codex: shouldApplyCodex ? this.applyStalePolicy(codex, previousState.codex) : this.state.codex @@ -1630,6 +1713,9 @@ export class RateLimitService { claudeProvenance === latestClaudeProvenance && this.isSameClaudeTarget(claudeTarget, this.claudeFetchTarget) + if (shouldApplyClaude) { + this.trackActiveFailureStreak('claude', claude) + } this.updateState({ ...this.state, claude: shouldApplyClaude @@ -1669,6 +1755,7 @@ export class RateLimitService { return } + this.trackActiveFailureStreak('grok', grok) this.updateState({ ...this.state, grok: this.applyStalePolicy(grok, previousState.grok) diff --git a/src/renderer/src/components/stats/GrokUsagePane.tsx b/src/renderer/src/components/stats/GrokUsagePane.tsx index 8e51049e3..086006215 100644 --- a/src/renderer/src/components/stats/GrokUsagePane.tsx +++ b/src/renderer/src/components/stats/GrokUsagePane.tsx @@ -1,3 +1,4 @@ +import { useState } from 'react' import { CalendarClock, ExternalLink, RefreshCw, Sparkles } from 'lucide-react' import { translate } from '@/i18n/i18n' import { useAppStore } from '../../store' @@ -13,6 +14,18 @@ export function GrokUsagePane(): React.JSX.Element { const openSettingsPage = useAppStore((s) => s.openSettingsPage) const openSettingsTarget = useAppStore((s) => s.openSettingsTarget) const recordFeatureInteraction = useAppStore((s) => s.recordFeatureInteraction) + // Why: settled snapshots keep their status during refetches (no 'fetching' + // repaint), so manual-refresh feedback must be renderer-local, matching the + // StatusBar refresh button. + const [isRefreshing, setIsRefreshing] = useState(false) + + const handleRefresh = (): void => { + if (isRefreshing) { + return + } + setIsRefreshing(true) + void refreshGrokRateLimits().finally(() => setIsRefreshing(false)) + } const openGrokAccounts = (): void => { openSettingsTarget({ pane: 'accounts', repoId: null, sectionId: 'accounts-grok' }) @@ -57,7 +70,7 @@ export function GrokUsagePane(): React.JSX.Element { grok?.weekly && typeof grok.weekly.usedPercent === 'number' ? Math.round(grok.weekly.usedPercent) : null - const isFetching = grok?.status === 'fetching' + const isFetching = isRefreshing || grok?.status === 'fetching' return (
void refreshGrokRateLimits()} + onClick={handleRefresh} disabled={isFetching} aria-label={translate( 'auto.components.stats.GrokUsagePane.i0j1k2l3m4',