> resolution per site, connection and model: assistants sharing a setup share one lookup */ private array $resolved = []; /** @var array> recorded failures per site and connection */ private array $facts = []; /** @var array last success stamp per site and connection */ private array $successes = []; /** @var list>|null */ private ?array $bots = null; /** @var list|null */ private ?array $tripped = null; /** @var list>|null */ private ?array $itemsCache = null; public function __construct(private Plugin $plugin) { } /** @return list> */ private function bots(): array { return $this->bots ??= $this->plugin->bots->all(get_current_blog_id()); } /** @param array $bot @return array */ private function resolve(array $bot): array { $key = (int) ($bot['site_id'] ?? 0) . '|' . (string) ($bot['connection_id'] ?? '-') . '|' . (string) ($bot['model'] ?? ''); return $this->resolved[$key] ??= $this->plugin->ai()->resolve($bot); } /** One report instance lives for one page render, so these memos never outlive the request or cross a site (the key carries the site). */ private function factsFor(int $site, int $connectionId): array { return $this->facts[$site . '|' . $connectionId] ??= $this->plugin->connectionHealth()->recent($connectionId); } private function successFor(ProviderBreaker $breaker, int $site, int $connectionId): int { return $this->successes[$site . '|' . $connectionId] ??= $breaker->lastSuccessAt($site, $connectionId); } private function tripped(ProviderBreaker $breaker, int $site): array { return $this->tripped ??= $breaker->tripped($site); } /** * @return list, failures:list, connection_id:?int}> */ public function assistants(): array { $p = $this->plugin; $site = get_current_blog_id(); $now = $p->clock->now(); $ai = $p->ai(); $breaker = new ProviderBreaker($p->db, $p->clock, $p->options); $open = []; foreach ($this->tripped($breaker, $site) as $t) { if ($t['open']) { $open[(int) $t['connection_id']] = true; } } $balance = $p->ledger->balance($site, 'site'); $upgrade = $p->upgradeNeedsAttention(); $out = []; foreach ($this->bots() as $bot) { if (($bot['status'] ?? '') !== 'active' || empty($bot['is_public'])) { continue; // a draft or private assistant is not answering the public, so it has no public availability to report } $r = $this->resolve($bot); $connectionId = isset($r['connection']['id']) ? (int) $r['connection']['id'] : null; $verdict = AssistantAvailability::evaluate([ 'chat_enabled' => $p->modules->isEnabled('chat'), 'ai_problem' => $r['problem'], 'simulated' => ($r['mode'] ?? '') === 'demo' && $r['ready'], 'paused' => $p->ledger->isHalted(), 'budget' => $balance, 'grounded' => $p->sources()->botHasSources($site, (int) $bot['id']), 'knowledge_enabled' => $p->modules->isEnabled('knowledge'), 'upgrade_pending' => $upgrade, 'facts' => $connectionId !== null ? $this->factsFor($site, $connectionId) : [], 'last_success' => $connectionId !== null ? $this->successFor($breaker, $site, $connectionId) : 0, 'breaker_open' => $connectionId !== null && isset($open[$connectionId]), 'now' => $now, ]); $out[] = ['id' => (int) $bot['id'], 'name' => (string) $bot['name'], 'status' => $verdict['status'], 'reasons' => $verdict['reasons'], 'failures' => $verdict['failures'], 'connection_id' => $connectionId]; } return $out; } /** * @return list}}> */ public function items(): array { if ($this->itemsCache !== null) { return $this->itemsCache; } $p = $this->plugin; $site = get_current_blog_id(); $now = $p->clock->now(); $items = []; // 1. Failures observed on live requests, per saved connection. $breaker = new ProviderBreaker($p->db, $p->clock, $p->options); $repo = $p->connections(); foreach ($repo?->list($site) ?? [] as $c) { $id = (int) $c['id']; $success = $this->successFor($breaker, $site, $id); foreach ($this->factsFor($site, $id) as $f) { $current = $f['last'] > $success && $now - $f['last'] <= AssistantAvailability::WINDOW && $f['category'] !== FailureCatalog::NO_ANSWER; $items[] = ['id' => 'conn:' . $id . ':' . $f['category'], 'severity' => $current ? 'danger' : 'info', 'category' => $f['category'], 'billing' => $f['billing'], 'component' => __('AI connection', 'aiwend'), 'subject' => (string) $c['label'], 'since' => $f['first'], 'last' => $f['last'], 'link' => [__('AI connections', 'aiwend'), AdminView::url('aiwend-connections', ['view' => 'connection', 'id' => $id])], 'action' => null]; } } foreach ($this->tripped($breaker, $site) as $t) { $connection = $repo?->find($site, (int) $t['connection_id']); $items[] = ['id' => 'breaker:' . (int) $t['connection_id'], 'severity' => 'warn', 'category' => FailureCatalog::UNCERTAIN_USAGE, 'billing' => FailureCatalog::BILLING_UNCERTAIN, 'component' => __('Circuit breaker', 'aiwend'), 'subject' => $connection !== null ? (string) $connection['label'] : '#' . (int) $t['connection_id'], 'since' => null, 'last' => null, 'link' => [__('Usage', 'aiwend'), AdminView::url('aiwend-usage')], 'action' => null]; } // 2. Uncertain usage. // Bounded: at most 100 held requests are read (state index), never the whole table. $held = $p->ledger->uncertain($site, 100); if ($held !== []) { $units = array_sum(array_map(static fn(\Aiwend\Usage\Reservation $r): int => $r->reservedUnits, $held)); $count = count($held); try { $oldestAge = $p->ledger->holdSummary($site)['uncertain']['oldest_age']; // bounded, state index; read-only } catch (\Throwable) { $oldestAge = null; // fail safe: the dashboard must not break } $items[] = ['id' => 'uncertain', 'severity' => 'warn', 'category' => FailureCatalog::UNCERTAIN_USAGE, 'billing' => FailureCatalog::BILLING_UNCERTAIN, 'component' => __('Usage', 'aiwend'), 'subject' => sprintf(/* translators: 1: number of requests, 2: held usage units */ _n('%1$d request, %2$s units held', '%1$d requests, %2$s units held', $count, 'aiwend'), $count, number_format_i18n($units)) . ($count >= 100 ? '+' : '') . ($oldestAge !== null ? ', ' . sprintf(/* translators: %s: time span such as "3 days" */ __('oldest %s', 'aiwend'), human_time_diff($now - $oldestAge, $now)) : ''), 'since' => null, 'last' => null, 'link' => [__('Resolve on the Usage page', 'aiwend'), AdminView::url('aiwend-usage')], 'action' => null]; } // 2b. Reservations still `reserved` or `dispatched` long after they normally finish (Phase 1 visibility; read-only, bounded to // HOLD_SCAN_CAP rows by one query on the state index). They hold units against the limits but are not on the resolve list above. // Nothing is released, charged or expired by showing them; no ids, prompts or provider text are read. try { $pending = $p->ledger->holdSummary($site)['pending']; } catch (\Throwable) { $pending = ['count' => 0, 'units' => 0, 'oldest_age' => null, 'capped' => false]; // fail safe: the dashboard must not break } if ($pending['count'] > 0) { $site_balance = $p->ledger->balance($site, 'site'); $more = $pending['capped'] ? '+' : ''; $text = sprintf(/* translators: 1: number of reservations, 2: reserved units (local usage units, not provider billing) */ _n('%1$s%3$s reservation, %2$s%3$s units reserved', '%1$s%3$s reservations, %2$s%3$s units reserved', $pending['count'], 'aiwend'), number_format_i18n($pending['count']), number_format_i18n($pending['units']), $more); if ($pending['oldest_age'] !== null) { $text .= ', ' . sprintf($pending['capped'] ? /* translators: %s: time span such as "2 hours" */ __('oldest seen %s', 'aiwend') : /* translators: %s: time span such as "2 hours" */ __('oldest %s', 'aiwend'), human_time_diff($now - $pending['oldest_age'], $now)); } if ($pending['capped']) { $text .= ' ' . sprintf(/* translators: %d: number of rows read */ __('(at least: only the first %d were counted)', 'aiwend'), \Aiwend\Usage\Ledger::HOLD_SCAN_CAP); } if ($site_balance !== null) { $text .= '; ' . sprintf(/* translators: 1: site limit, 2: units still available */ __('site limit %1$s units, %2$s available', 'aiwend'), number_format_i18n($site_balance['limit']), number_format_i18n(max(0, $site_balance['available']))); } $items[] = ['id' => 'pending_holds', 'severity' => 'warn', 'category' => FailureCatalog::PENDING_HOLD, 'billing' => FailureCatalog::BILLING_NOT_APPLICABLE, 'component' => __('Usage', 'aiwend'), 'subject' => $text, 'since' => null, 'last' => null, 'link' => [__('Usage', 'aiwend'), AdminView::url('aiwend-usage')], 'action' => null]; } // 3. Background jobs: dead jobs and cron. $dead = count($p->queue->deadJobs($site, 100)); // bounded; the (state, next run) index finds dead jobs without scanning finished ones if ($dead > 0) { $items[] = ['id' => 'jobs', 'severity' => 'danger', 'category' => FailureCatalog::JOB_FAILED, 'billing' => FailureCatalog::BILLING_NOT_APPLICABLE, 'component' => __('Background jobs', 'aiwend'), 'subject' => sprintf(/* translators: %d: number of failed jobs */ _n('%d failed job', '%d failed jobs', $dead, 'aiwend'), $dead), 'since' => null, 'last' => null, 'link' => null, 'action' => null]; } $last = (int) get_option(CronDiagnostics::LAST_RUN_OPTION, 0); if (CronDiagnostics::count() !== 1 || ($last > 0 && $now - $last >= 900) || $last === 0) { $items[] = ['id' => 'cron', 'severity' => 'warn', 'category' => FailureCatalog::CRON_DELAYED, 'billing' => FailureCatalog::BILLING_NOT_APPLICABLE, 'component' => __('Background jobs', 'aiwend'), 'subject' => $last > 0 ? sprintf(/* translators: %s: time span */ __('last run %s ago', 'aiwend'), human_time_diff($last, $now)) : __('no run recorded yet', 'aiwend'), 'since' => null, 'last' => $last ?: null, 'link' => null, 'action' => null]; } // A scheduled run that started but ran nothing while indexing jobs were ready: the scheduler works, the worker did not pick the work up. // Reported with "Process now", which uses the same worker for a few jobs, so the administrator can proceed and the evidence is visible. $result = get_option(CronDiagnostics::LAST_RESULT_OPTION); if (is_array($result) && (int) ($result['waiting'] ?? 0) > 0 && (int) ($result['ran'] ?? 0) === 0 && (int) ($result['failed'] ?? 0) === 0 && $now - (int) ($result['at'] ?? 0) < 900) { $idle = (int) $result['waiting']; $items[] = ['id' => 'cron_idle', 'severity' => 'warn', 'category' => FailureCatalog::CRON_DELAYED, 'billing' => FailureCatalog::BILLING_NOT_APPLICABLE, 'component' => __('Background jobs', 'aiwend'), 'subject' => sprintf(/* translators: %d: number of sources */ _n('the last run started but %d source was still waiting to be indexed', 'the last run started but %d sources were still waiting to be indexed', $idle, 'aiwend'), $idle), 'since' => null, 'last' => (int) $result['at'], 'link' => null, 'action' => current_user_can(Capabilities::MANAGE_KNOWLEDGE) ? ['do' => 'process_now', 'label' => __('Process now', 'aiwend'), 'fields' => []] : null]; } // 4. Knowledge. $failed = $p->sources()->countFailed($site); if ($failed > 0) { $items[] = ['id' => 'knowledge', 'severity' => 'danger', 'category' => FailureCatalog::KNOWLEDGE_FAILED, 'billing' => FailureCatalog::BILLING_NOT_APPLICABLE, 'component' => __('Knowledge', 'aiwend'), 'subject' => sprintf(/* translators: %d: number of sources */ _n('%d source', '%d sources', $failed, 'aiwend'), $failed), 'since' => null, 'last' => null, 'link' => [__('Open Knowledge', 'aiwend'), AdminView::url('aiwend-knowledge')], 'action' => current_user_can(Capabilities::MANAGE_KNOWLEDGE) ? ['do' => 'run_repair', 'label' => __('Check for sources that were never queued', 'aiwend'), 'fields' => []] : null]; } // 5. Database update. $guard = new UpgradeGuard($p->db, $p->options, $p->clock); $failure = $guard->failure(); if ($failure !== null && $p->upgradeNeedsAttention()) { $items[] = ['id' => 'upgrade', 'severity' => 'danger', 'category' => FailureCatalog::MIGRATION_FAILED, 'billing' => FailureCatalog::BILLING_NOT_APPLICABLE, 'component' => __('Database', 'aiwend'), 'subject' => null, 'since' => $failure['at'], 'last' => $failure['at'], 'link' => null, 'action' => current_user_can(Capabilities::MANAGE) ? ['do' => 'retry_upgrade', 'label' => __('Try again now', 'aiwend'), 'fields' => []] : null]; } // 6. The encryption key. foreach ($this->bots() as $bot) { if (($this->resolve($bot)['problem'] ?? null) === 'encryption_missing') { $items[] = ['id' => 'encryption', 'severity' => 'danger', 'category' => FailureCatalog::ENCRYPTION_KEY, 'billing' => FailureCatalog::BILLING_NOT_SENT, 'component' => __('Setup', 'aiwend'), 'subject' => null, 'since' => null, 'last' => null, 'link' => [__('Overview', 'aiwend'), AdminView::url('aiwend')], 'action' => null]; break; } } $order = ['danger' => 0, 'warn' => 1, 'info' => 2]; usort($items, static fn(array $a, array $b): int => [$order[$a['severity']], -($a['last'] ?? 0)] <=> [$order[$b['severity']], -($b['last'] ?? 0)]); return $this->itemsCache = $items; } /** @return list */ public function deadJobs(): array { // The queue stored a redacted summary; it is redacted again for display, so a value that was stored some other way never reaches a screen. return array_map(fn(array $j): array => ['error' => $this->plugin->redactor->summary($j['error'])] + $j + ['label' => RetryPolicy::label($j['type']), 'retryable' => RetryPolicy::allows($j['type'])], $this->plugin->queue->deadJobs(get_current_blog_id(), 20)); } /** @return array{at:int, count:int}|null the last run of the daily knowledge repair */ public function lastRepair(): ?array { $v = get_option(KnowledgeModule::LAST_REPAIR_OPTION, null); return is_array($v) && isset($v['at'], $v['count']) ? ['at' => (int) $v['at'], 'count' => (int) $v['count']] : null; } }