This is an automated email from the ASF dual-hosted git repository.
hsluoyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/casbin-gateway.git
The following commit(s) were added to refs/heads/master by this push:
new 478207a feat(provider): audit channels from kept records and by
probing upstreams
478207a is described below
commit 478207a5edf52afba826259cf3a0d611ffe68ec9
Author: Yang Luo <[email protected]>
AuthorDate: Thu Sep 3 00:51:54 2026 +0800
feat(provider): audit channels from kept records and by probing upstreams
---
conf/app.conf | 10 +
controllers/llm_record.go | 17 +
controllers/provider_probe.go | 101 +++++
main.go | 3 +
object/llm_audit.go | 405 ++++++++++++++++++++
object/ormer.go | 7 +
object/provider.go | 17 +
object/provider_probe.go | 370 +++++++++++++++++++
object/provider_probe_checks.go | 591 ++++++++++++++++++++++++++++++
object/provider_probe_run.go | 343 +++++++++++++++++
object/setting.go | 18 +
routers/router.go | 4 +
web/src/backend/LlmRecordBackend.ts | 15 +-
web/src/backend/ProviderBackend.ts | 32 +-
web/src/components/ProviderAuditCard.tsx | 487 ++++++++++++++++++++++++
web/src/components/ProviderAuditPanel.tsx | 284 ++++++++++++++
web/src/locales/en/data.json | 82 +++++
web/src/locales/zh/data.json | 82 +++++
web/src/pages/ProviderListPage.tsx | 24 +-
web/src/pages/SettingPage.tsx | 18 +
web/src/types.ts | 99 +++++
21 files changed, 3004 insertions(+), 5 deletions(-)
diff --git a/conf/app.conf b/conf/app.conf
index 9397a23..d7e189c 100644
--- a/conf/app.conf
+++ b/conf/app.conf
@@ -66,6 +66,16 @@ agentRecordCapacity = 1000
; How often to rescan the append-only logs of file-tailed agents, in seconds.
agentMonitorPollSeconds = 5
+; Channel probes. A probe sends four short requests to a provider's upstream to
+; check that what answers is what it claims to be: the model name it reports,
+; the vendor headers it carries, the shape of its event stream, whether prompt
+; caching is really accounted for, and whether two identical requests are
billed
+; the same. It spends a few cents of that provider's own credit.
+; "auto" probes a provider when it is added, when its endpoint, type or key
+; changes, and once for any provider that has never been probed. "manual" only
+; probes when the button on the Providers page is pressed. "off" never probes.
+providerProbeMode = "auto"
+
; LLM request records. "off" keeps nothing, "metadata" records who called which
; model with which outcome, and "full" also stores the sanitized request body.
; The default is "full": the records are what the dashboards are made of, and
diff --git a/controllers/llm_record.go b/controllers/llm_record.go
index c566e1c..77b6c6a 100644
--- a/controllers/llm_record.go
+++ b/controllers/llm_record.go
@@ -106,6 +106,23 @@ func (c *ApiController) GetLlmRecordStats() {
c.ResponseOk(stats)
}
+// GetLlmProviderAudit reports what the kept records say about each provider
+// that served them: the same window the stats cover, read for how an upstream
+// behaved rather than for what it cost.
+func (c *ApiController) GetLlmProviderAudit() {
+ if c.RequireAdmin() {
+ return
+ }
+
+ report, err := object.GetLlmProviderAudit(c.readLlmRecordFilter())
+ if err != nil {
+ c.ResponseError(err.Error())
+ return
+ }
+
+ c.ResponseOk(report)
+}
+
// GetLlmUsageTrend groups the same window into time buckets, which is what the
// usage dashboard draws. "bucket" is "hour" or "day"; anything else is a day.
func (c *ApiController) GetLlmUsageTrend() {
diff --git a/controllers/provider_probe.go b/controllers/provider_probe.go
new file mode 100644
index 0000000..b2449fb
--- /dev/null
+++ b/controllers/provider_probe.go
@@ -0,0 +1,101 @@
+// Copyright 2026 The casbin Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controllers
+
+import (
+ "github.com/apache/casbin-gateway/object"
+)
+
+// GetProviderProbes is the newest probe of every provider that has one. The
+// second payload is how probes are started, so the page can say whether an
+// unprobed provider is waiting for a sweep or for a button.
+func (c *ApiController) GetProviderProbes() {
+ if c.RequireAdmin() {
+ return
+ }
+
+ probes, err := object.GetProviderProbes()
+ if err != nil {
+ c.ResponseError(err.Error())
+ return
+ }
+
+ c.ResponseOk(probes, object.GetProviderProbeMode())
+}
+
+// GetProviderProbeHistory is every kept run for one provider, newest first:
two
+// runs of the same suite that disagree are what says a backend was swapped.
+func (c *ApiController) GetProviderProbeHistory() {
+ if c.RequireAdmin() {
+ return
+ }
+
+ id := c.Input().Get("id")
+ owner, _, ok := getProviderOwnerAndName(id)
+ if !ok {
+ c.ResponseError("invalid provider ID: " + id)
+ return
+ }
+ if !c.providerAccess(owner) {
+ c.ResponseError("unauthorized")
+ return
+ }
+
+ probes, err := object.GetProviderProbeHistory(id)
+ if err != nil {
+ c.ResponseError(err.Error())
+ return
+ }
+
+ c.ResponseOk(probes)
+}
+
+// ProbeProvider runs the suite against one stored provider and answers with
the
+// report. It spends the account's own credit, which is why it is a request of
+// its own rather than something a page load does.
+func (c *ApiController) ProbeProvider() {
+ if c.RequireAdmin() {
+ return
+ }
+
+ id := c.Input().Get("id")
+ owner, _, ok := getProviderOwnerAndName(id)
+ if !ok {
+ c.ResponseError("invalid provider ID: " + id)
+ return
+ }
+ if !c.providerAccess(owner) {
+ c.ResponseError("unauthorized")
+ return
+ }
+
+ provider, err := object.GetProvider(id)
+ if err != nil {
+ c.ResponseError(err.Error())
+ return
+ }
+ if provider == nil {
+ c.ResponseError("the provider does not exist")
+ return
+ }
+
+ probe, err := object.ProbeProviderNow(provider,
object.ProbeTriggerManual)
+ if err != nil {
+ c.ResponseError(err.Error())
+ return
+ }
+
+ c.ResponseOk(probe)
+}
diff --git a/main.go b/main.go
index 080a8ad..9edc80a 100644
--- a/main.go
+++ b/main.go
@@ -75,6 +75,9 @@ func main() {
casdoor.InitCasdoorConfig()
object.InitUsers()
proxy.InitHttpClient()
+ // Probes go out over the same transport, so they can only start once
it is
+ // configured. The sweep waits a while longer before spending anything.
+ object.StartProviderProbeSweep()
agentmonitor.Configure(
conf.GetAgentPatchStateDir(),
diff --git a/object/llm_audit.go b/object/llm_audit.go
new file mode 100644
index 0000000..a99be13
--- /dev/null
+++ b/object/llm_audit.go
@@ -0,0 +1,405 @@
+// Copyright 2026 The casbin Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package object
+
+import (
+ "sort"
+
+ "github.com/apache/casbin-gateway/protocol"
+)
+
+// The audit reads records that were kept anyway and adds nothing to the
request
+// path. It measures what an upstream reported about itself, which is evidence
+// about a provider rather than a verdict on one: every number here is meant to
+// be quoted back together with the sample it was measured over.
+
+// llmAuditScanLimit bounds how many records one audit reads, so the page stays
+// answerable on a table configured to keep far more than the default.
+const llmAuditScanLimit = 50000
+
+// llmAuditMinSample is how many requests a check needs before its level means
+// anything. Below it the check reports itself as unknown rather than guessing.
+const llmAuditMinSample = 20
+
+// llmAuditCacheMinTokens is the prompt size below which a missing cache says
+// nothing: Anthropic will not cache a prefix shorter than its own minimum, so
+// short requests are left out of the cache sample entirely.
+const llmAuditCacheMinTokens = 2048
+
+// llmAuditMaxModels bounds the model names carried on one report.
+const llmAuditMaxModels = 24
+
+// The levels a check ends in. "unknown" is not a failure: it is the honest
+// answer when the window did not hold enough requests to measure.
+const (
+ LlmAuditOk = "ok"
+ LlmAuditWarn = "warn"
+ LlmAuditAlert = "alert"
+ LlmAuditUnknown = "unknown"
+)
+
+// The checks, named so that the page can word each one itself.
+const (
+ LlmAuditCache = "cache"
+ LlmAuditErrors = "errors"
+ LlmAuditLatency = "latency"
+ LlmAuditPricing = "pricing"
+)
+
+// LlmAuditCheck is one measurement, kept as a number and a level rather than a
+// sentence: what it means is worded by whatever renders it.
+type LlmAuditCheck struct {
+ Key string `json:"key"`
+ Level string `json:"level"`
+ // Value is a share between 0 and 1 for every check but latency, where
it is
+ // the ratio of the slow tail to the median.
+ Value float64 `json:"value"`
+ // Sample is how many requests the value was measured over, which is the
+ // half of a rate that says whether to believe it.
+ Sample int64 `json:"sample"`
+}
+
+// LlmProviderAudit is what the kept records say about one provider.
+type LlmProviderAudit struct {
+ Provider string `json:"provider"`
+
+ // Requests counts the records this provider answered, failures
included.
+ Requests int64 `json:"requests"`
+ Failed int64 `json:"failed"`
+ // FailedOver counts the attempts this provider lost the request on,
which
+ // the chain then had another provider answer. They are not in
Requests: a
+ // record belongs to whoever answered it.
+ FailedOver int64 `json:"failedOver"`
+ Retried int64 `json:"retried"`
+
+ PromptTokens int64 `json:"promptTokens"`
+ CompletionTokens int64 `json:"completionTokens"`
+ CacheReadTokens int64 `json:"cacheReadTokens"`
+ CacheWriteTokens int64 `json:"cacheWriteTokens"`
+ TotalTokens int64 `json:"totalTokens"`
+ Cost float64 `json:"cost"`
+
+ // Cacheable counts the answered requests a cache could have shown up
on at
+ // all: the Anthropic protocol accounts a cache separately, and the
prompt
+ // was long enough to be worth caching. CacheHits is how many of those
came
+ // back with any cache accounting on them.
+ Cacheable int64 `json:"cacheable"`
+ CacheHits int64 `json:"cacheHits"`
+
+ LatencyP50Ms int64 `json:"latencyP50Ms"`
+ LatencyP95Ms int64 `json:"latencyP95Ms"`
+
+ // Unpriced counts answered requests whose model has no rate anywhere,
and
+ // UnpricedModels names them: a model nobody publishes a rate for is a
model
+ // name this upstream invented.
+ Unpriced int64 `json:"unpriced"`
+ UnpricedModels []string `json:"unpricedModels"`
+ Models []string `json:"models"`
+
+ FirstTime string `json:"firstTime"`
+ LastTime string `json:"lastTime"`
+
+ Checks []LlmAuditCheck `json:"checks"`
+
+ // The working sets the exported fields above are cut from.
+ latencies []int64
+ models map[string]bool
+ unpricedModels map[string]bool
+}
+
+// LlmAuditReport is one window's worth of providers, with what it was measured
+// over: a report that read fewer records than the window holds has to say so.
+type LlmAuditReport struct {
+ Scanned int64 `json:"scanned"`
+ Truncated bool `json:"truncated"`
+ Providers []*LlmProviderAudit `json:"providers"`
+}
+
+// llmAuditCols are the columns an audit reads. The bodies are left behind: the
+// report is about how an upstream answered, not about what it was asked.
+var llmAuditCols = []string{
+ "created_time", "protocol", "model", "provider", "status",
"duration_ms",
+ "attempts", "failures", "prompt_tokens", "completion_tokens",
+ "cache_read_tokens", "cache_write_tokens", "total_tokens", "cost",
"priced",
+}
+
+// GetLlmProviderAudit reads the kept records of one window and reports what
+// each provider in them looks like from the outside.
+func GetLlmProviderAudit(filter LlmRecordFilter) (*LlmAuditReport, error) {
+ session := llmRecordSession(filter)
+ defer session.Close()
+
+ records := []*LlmRecord{}
+ err :=
session.Cols(llmAuditCols...).Desc("id").Limit(llmAuditScanLimit).Find(&records)
+ if err != nil {
+ return nil, err
+ }
+
+ report := &LlmAuditReport{
+ Scanned: int64(len(records)),
+ Truncated: len(records) >= llmAuditScanLimit,
+ Providers: []*LlmProviderAudit{},
+ }
+
+ audits := map[string]*LlmProviderAudit{}
+ for _, record := range records {
+ if record.Provider != "" {
+ addLlmAuditRecord(audits, record)
+ }
+ for _, failure := range llmAuditFailoversOf(record) {
+ if failure.Provider != "" {
+ auditOf(audits, failure.Provider).FailedOver++
+ }
+ }
+ }
+
+ for _, audit := range audits {
+ finishLlmAudit(audit)
+ report.Providers = append(report.Providers, audit)
+ }
+ sort.Slice(report.Providers, func(i, j int) bool {
+ left, right := report.Providers[i], report.Providers[j]
+ if left.Requests != right.Requests {
+ return left.Requests > right.Requests
+ }
+ return left.Provider < right.Provider
+ })
+ return report, nil
+}
+
+func auditOf(audits map[string]*LlmProviderAudit, provider string)
*LlmProviderAudit {
+ audit, ok := audits[provider]
+ if !ok {
+ audit = &LlmProviderAudit{
+ Provider: provider,
+ UnpricedModels: []string{},
+ Models: []string{},
+ Checks: []LlmAuditCheck{},
+ models: map[string]bool{},
+ unpricedModels: map[string]bool{},
+ }
+ audits[provider] = audit
+ }
+ return audit
+}
+
+// llmAuditFailoversOf is the attempts a record failed over from, without the
+// one the record itself already counts. A chain whose last provider failed
+// before writing anything appends that attempt to both, and counting it twice
+// would make that provider look worse than it was.
+func llmAuditFailoversOf(record *LlmRecord) []LlmFailure {
+ failures := record.Failures
+ if len(failures) == 0 {
+ return nil
+ }
+ last := failures[len(failures)-1]
+ if last.Provider == record.Provider && !llmAuditAnswered(record) {
+ failures = failures[:len(failures)-1]
+ }
+ return failures
+}
+
+func llmAuditAnswered(record *LlmRecord) bool {
+ return record.Status >= 200 && record.Status < 300
+}
+
+func addLlmAuditRecord(audits map[string]*LlmProviderAudit, record *LlmRecord)
{
+ audit := auditOf(audits, record.Provider)
+ audit.Requests++
+ if !llmAuditAnswered(record) {
+ audit.Failed++
+ }
+ if record.Attempts > 1 {
+ audit.Retried++
+ }
+
+ audit.PromptTokens += int64(record.PromptTokens)
+ audit.CompletionTokens += int64(record.CompletionTokens)
+ audit.CacheReadTokens += int64(record.CacheReadTokens)
+ audit.CacheWriteTokens += int64(record.CacheWriteTokens)
+ audit.TotalTokens += int64(record.TotalTokens)
+ audit.Cost += record.Cost
+
+ // The records arrive newest first, so the first one seen is the last
one
+ // there was, and every later one moves the beginning further back.
+ if audit.LastTime == "" {
+ audit.LastTime = record.CreatedTime
+ }
+ audit.FirstTime = record.CreatedTime
+
+ if record.Model != "" && len(audit.models) < llmAuditMaxModels {
+ audit.models[record.Model] = true
+ }
+ if !llmAuditAnswered(record) {
+ return
+ }
+
+ if record.TotalTokens > 0 && !record.Priced {
+ audit.Unpriced++
+ if record.Model != "" && len(audit.unpricedModels) <
llmAuditMaxModels {
+ audit.unpricedModels[record.Model] = true
+ }
+ }
+ if record.DurationMs > 0 {
+ audit.latencies = append(audit.latencies, record.DurationMs)
+ }
+
+ if llmAuditCacheable(record) {
+ audit.Cacheable++
+ if record.CacheReadTokens > 0 || record.CacheWriteTokens > 0 {
+ audit.CacheHits++
+ }
+ }
+}
+
+// llmAuditCacheable reports whether a cache could have been accounted for on
+// this request at all. Only the Anthropic protocol reports a cache write, and
+// only a prompt past the minimum is eligible for one, so counting anything
else
+// would make a zero look like a finding when it is the shape of the request.
+func llmAuditCacheable(record *LlmRecord) bool {
+ if record.Protocol != protocol.Anthropic {
+ return false
+ }
+ return record.PromptTokens+record.CacheReadTokens >=
llmAuditCacheMinTokens
+}
+
+func finishLlmAudit(audit *LlmProviderAudit) {
+ audit.Models = sortedKeys(audit.models)
+ audit.UnpricedModels = sortedKeys(audit.unpricedModels)
+
+ sort.Slice(audit.latencies, func(i, j int) bool { return
audit.latencies[i] < audit.latencies[j] })
+ audit.LatencyP50Ms = llmAuditPercentile(audit.latencies, 0.5)
+ audit.LatencyP95Ms = llmAuditPercentile(audit.latencies, 0.95)
+
+ audit.Checks = []LlmAuditCheck{
+ llmAuditCacheCheck(audit),
+ llmAuditErrorCheck(audit),
+ llmAuditLatencyCheck(audit),
+ llmAuditPricingCheck(audit),
+ }
+ audit.models, audit.unpricedModels, audit.latencies = nil, nil, nil
+}
+
+func sortedKeys(set map[string]bool) []string {
+ keys := make([]string, 0, len(set))
+ for key := range set {
+ keys = append(keys, key)
+ }
+ sort.Strings(keys)
+ return keys
+}
+
+func llmAuditPercentile(sorted []int64, share float64) int64 {
+ if len(sorted) == 0 {
+ return 0
+ }
+ return sorted[int(share*float64(len(sorted)-1))]
+}
+
+// llmAuditCacheCheck is the share of cacheable requests the upstream reported
+// any cache accounting on. One that never reports any is either not passing
+// prompt caching through or not implementing it, and is billing the whole
+// prompt as fresh input either way.
+func llmAuditCacheCheck(audit *LlmProviderAudit) LlmAuditCheck {
+ check := LlmAuditCheck{Key: LlmAuditCache, Level: LlmAuditUnknown,
Sample: audit.Cacheable}
+ if audit.Cacheable == 0 {
+ return check
+ }
+
+ check.Value = float64(audit.CacheHits) / float64(audit.Cacheable)
+ if audit.Cacheable < llmAuditMinSample {
+ return check
+ }
+ switch {
+ case audit.CacheHits == 0:
+ check.Level = LlmAuditAlert
+ case check.Value < 0.2:
+ check.Level = LlmAuditWarn
+ default:
+ check.Level = LlmAuditOk
+ }
+ return check
+}
+
+func llmAuditErrorCheck(audit *LlmProviderAudit) LlmAuditCheck {
+ attempts := audit.Requests + audit.FailedOver
+ check := LlmAuditCheck{Key: LlmAuditErrors, Level: LlmAuditUnknown,
Sample: attempts}
+ if attempts == 0 {
+ return check
+ }
+
+ check.Value = float64(audit.Failed+audit.FailedOver) / float64(attempts)
+ if attempts < llmAuditMinSample {
+ return check
+ }
+ switch {
+ case check.Value >= 0.2:
+ check.Level = LlmAuditAlert
+ case check.Value >= 0.05:
+ check.Level = LlmAuditWarn
+ default:
+ check.Level = LlmAuditOk
+ }
+ return check
+}
+
+// llmAuditLatencyCheck measures the spread rather than the duration: how long
a
+// request takes depends on how much was asked for, but how far the slow tail
+// sits from the middle is about the route it took, and a long chain of
resellers
+// is what makes that tail long.
+func llmAuditLatencyCheck(audit *LlmProviderAudit) LlmAuditCheck {
+ answered := audit.Requests - audit.Failed
+ check := LlmAuditCheck{Key: LlmAuditLatency, Level: LlmAuditUnknown,
Sample: answered}
+ if audit.LatencyP50Ms == 0 {
+ return check
+ }
+
+ check.Value = float64(audit.LatencyP95Ms) / float64(audit.LatencyP50Ms)
+ if answered < llmAuditMinSample {
+ return check
+ }
+ switch {
+ case check.Value >= 10:
+ check.Level = LlmAuditAlert
+ case check.Value >= 5:
+ check.Level = LlmAuditWarn
+ default:
+ check.Level = LlmAuditOk
+ }
+ return check
+}
+
+// llmAuditPricingCheck is the share of answered requests whose model has no
+// rate anywhere. It says nothing on its own about an honest provider serving
+// something new, and quite a lot about one serving a model name that exists
+// nowhere else.
+func llmAuditPricingCheck(audit *LlmProviderAudit) LlmAuditCheck {
+ answered := audit.Requests - audit.Failed
+ check := LlmAuditCheck{Key: LlmAuditPricing, Level: LlmAuditUnknown,
Sample: answered}
+ if answered == 0 {
+ return check
+ }
+
+ check.Value = float64(audit.Unpriced) / float64(answered)
+ switch {
+ case check.Value >= 0.5:
+ check.Level = LlmAuditAlert
+ case audit.Unpriced > 0:
+ check.Level = LlmAuditWarn
+ default:
+ check.Level = LlmAuditOk
+ }
+ return check
+}
diff --git a/object/ormer.go b/object/ormer.go
index c44a314..473d99e 100644
--- a/object/ormer.go
+++ b/object/ormer.go
@@ -274,6 +274,13 @@ func (a *Ormer) createTable() {
panic(err)
}
+ // What an active probe found the last few times it asked a provider's
+ // upstream the questions the passive records cannot answer.
+ err = a.Engine.Sync2(new(ProviderProbe))
+ if err != nil {
+ panic(err)
+ }
+
// Token prices edited on the Pricing page or written by a models.dev
sync,
// which override the built-in list prices.
err = a.Engine.Sync2(new(LlmPriceEntry))
diff --git a/object/provider.go b/object/provider.go
index cf39084..41eea2b 100644
--- a/object/provider.go
+++ b/object/provider.go
@@ -526,11 +526,16 @@ func AddProvider(provider *Provider) (bool, error) {
if err := encryptQuotaToken(provider); err != nil {
return false, err
}
+ // The probe needs the key as it was typed, and encryption is in place.
+ probeTarget := *provider
if err := encryptApiKey(provider); err != nil {
return false, err
}
affected, err := ormer.Engine.Insert(provider)
+ if err == nil && affected != 0 {
+ ProbeNewProvider(&probeTarget)
+ }
return affected != 0, err
}
@@ -574,6 +579,14 @@ func UpdateProvider(id string, provider *Provider) (bool,
error) {
provider.Name = name
provider.UpdatedTime = util.GetCurrentTime()
+ // Snapshotted before the key is encrypted, and before the mask that
means
+ // "the field was not touched" is resolved back to the stored key.
+ keyChanged := provider.ApiKey != ApiKeyMask
+ probeTarget := *provider
+ if !keyChanged {
+ probeTarget.ApiKey = stored.ApiKey
+ }
+
// The quota token is masked on read like the API key, but it shares its
// column with the rest of the quota configuration, so the stored one
is put
// back rather than the column left out of the write.
@@ -602,6 +615,7 @@ func UpdateProvider(id string, provider *Provider) (bool,
error) {
// provider starts from a clean slate.
ClearProviderHealth(provider.GetId())
ClearProviderQuota(provider.GetId())
+ ProbeEditedProvider(stored, &probeTarget, keyChanged)
}
return affected != 0, err
}
@@ -611,6 +625,9 @@ func DeleteProvider(provider *Provider) (bool, error) {
if err == nil {
ClearProviderHealth(provider.GetId())
ClearProviderQuota(provider.GetId())
+ if err := DeleteProviderProbes(provider.GetId()); err != nil {
+ fmt.Printf("DeleteProvider(): probes of [%s]: %v\n",
provider.GetId(), err)
+ }
}
return affected != 0, err
}
diff --git a/object/provider_probe.go b/object/provider_probe.go
new file mode 100644
index 0000000..c90fed1
--- /dev/null
+++ b/object/provider_probe.go
@@ -0,0 +1,370 @@
+// Copyright 2026 The casbin Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package object
+
+import (
+ "fmt"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/apache/casbin-gateway/conf"
+ "github.com/apache/casbin-gateway/util"
+ "github.com/beego/beego"
+)
+
+// A probe is the other half of the channel audit. The passive side reads what
+// the agents already asked for, which cannot say anything a real request never
+// happened to show; a probe asks the upstream a question chosen to have
exactly
+// one right answer, and costs a few cents of the user's own credit to do it.
+//
+// Everything here is written so a finding is reproducible: the request bodies
+// are fixed, the checks compare against what the vendor's own API documents,
+// and the report keeps what came back rather than a conclusion drawn from it.
+
+// The triggers a probe can run under, kept on the row so a report says why it
+// exists — and so an automatic one is told from a button press.
+const (
+ ProbeTriggerAdded = "added"
+ ProbeTriggerEdited = "edited"
+ ProbeTriggerUnused = "unused"
+ ProbeTriggerManual = "manual"
+)
+
+// The checks a probe runs.
+const (
+ ProbeIdentity = "identity"
+ ProbeVendor = "vendor"
+ ProbeStream = "stream"
+ ProbeCache = "cache"
+ ProbeTools = "tools"
+ ProbeBilling = "billing"
+)
+
+// probeKeepPerProvider is how many past probes are kept, which is what makes a
+// provider that quietly changed backends visible later.
+const probeKeepPerProvider = 20
+
+// probeMaxParallel bounds a sweep, so adopting a machine with twenty providers
+// does not open twenty upstream connections at once.
+const probeMaxParallel = 3
+
+// ProbeCheck is one measurement, in the same shape the passive audit uses: a
+// level and the fact it was drawn from, with the wording left to the page.
+type ProbeCheck struct {
+ Key string `json:"key"`
+ Level string `json:"level"`
+ // Facts are what actually came back, as data rather than as a
sentence: the
+ // model name that answered, the headers that were present, the events
that
+ // were missing, the two counts that did not agree. The page words
them, so
+ // a finding reads in the reader's own language and still quotes the
value
+ // it was drawn from.
+ Facts []string `json:"facts"`
+ // Value is the number the level was decided from, where there is one.
+ Value float64 `json:"value"`
+}
+
+// ProviderProbe is one run of the suite against one provider.
+type ProviderProbe struct {
+ Id int64 `xorm:"int notnull pk autoincr" json:"id"`
+ Provider string `xorm:"varchar(201) index" json:"provider"`
+ CreatedTime string `xorm:"varchar(100) index" json:"createdTime"`
+
+ Trigger string `xorm:"varchar(20)" json:"trigger"`
+ Protocol string `xorm:"varchar(20)" json:"protocol"`
+ // Model is what the probe asked for, UpstreamModel what answered.
+ Model string `xorm:"varchar(255)" json:"model"`
+ UpstreamModel string `xorm:"varchar(255)" json:"upstreamModel"`
+
+ // Ok is whether the suite got far enough to judge anything. Error
holds the
+ // reason it did not, which is a finding of its own.
+ Ok bool `xorm:"bool" json:"ok"`
+ Error string `xorm:"varchar(500)" json:"error"`
+
+ // TtftMs is the wait before the first streamed byte, which is what a
chain
+ // of resellers adds to and a total duration hides.
+ TtftMs int64 `xorm:"bigint" json:"ttftMs"`
+ DurationMs int64 `xorm:"bigint" json:"durationMs"`
+
+ // What the probe itself spent, so the price of knowing is on the report
+ // rather than only on the bill.
+ Requests int `xorm:"int" json:"requests"`
+ PromptTokens int `xorm:"int" json:"promptTokens"`
+ CompletionTokens int `xorm:"int" json:"completionTokens"`
+ CacheReadTokens int `xorm:"int" json:"cacheReadTokens"`
+ CacheWriteTokens int `xorm:"int" json:"cacheWriteTokens"`
+ Cost float64 `xorm:"double" json:"cost"`
+ Priced bool `xorm:"bool" json:"priced"`
+
+ // VendorHeaders are the response headers of the vendor's own API that
were
+ // present, by name. A relay in front of a real API usually keeps some.
+ VendorHeaders []string `xorm:"varchar(1000)" json:"vendorHeaders"`
+ Checks []ProbeCheck `xorm:"mediumtext" json:"checks"`
+}
+
+// GetProviderProbeMode is how probes are started: "auto" runs them for a
+// provider that was just added, one whose endpoint or key changed, and one
that
+// has never been probed; "manual" only ever runs one that was asked for; "off"
+// disables them entirely.
+func GetProviderProbeMode() string {
+ switch
strings.ToLower(conf.GetConfigStringUnquoted("providerProbeMode")) {
+ case "off":
+ return "off"
+ case "manual":
+ return "manual"
+ default:
+ return "auto"
+ }
+}
+
+func isProbeAutomatic() bool {
+ return GetProviderProbeMode() == "auto"
+}
+
+var probesRunning sync.Map
+
+// ProbeProviderNow runs the suite and stores the result. It is the one entry
+// point: everything else decides whether to call it.
+func ProbeProviderNow(provider *Provider, trigger string) (*ProviderProbe,
error) {
+ if provider == nil {
+ return nil, fmt.Errorf("no provider to probe")
+ }
+ if GetProviderProbeMode() == "off" {
+ return nil, fmt.Errorf("probing is turned off")
+ }
+
+ id := provider.GetId()
+ if _, busy := probesRunning.LoadOrStore(id, true); busy {
+ return nil, fmt.Errorf("a probe of %s is already running", id)
+ }
+ defer probesRunning.Delete(id)
+
+ probe := runProviderProbe(provider, trigger)
+ if err := saveProviderProbe(probe); err != nil {
+ return probe, err
+ }
+ return probe, nil
+}
+
+// probeProviderInBackground is the automatic path. It never blocks the caller
+// and never reports to it: a probe that could not run leaves the provider
+// unprobed, which the page shows as such.
+func probeProviderInBackground(provider *Provider, trigger string) {
+ if provider == nil || !isProbeAutomatic() {
+ return
+ }
+ copied := *provider
+ go func() {
+ if _, err := ProbeProviderNow(&copied, trigger); err != nil {
+ beego.Error("provider probe failed for",
copied.GetId()+":", err)
+ }
+ }()
+}
+
+// ProbeNewProvider runs after a provider is stored. Adding a provider already
+// sends one request to the upstream to check the key, so the credential is
+// spent either way; this asks the same upstream the questions that say whether
+// what is behind the key is what it claims to be.
+func ProbeNewProvider(provider *Provider) {
+ probeProviderInBackground(provider, ProbeTriggerAdded)
+}
+
+// ProbeEditedProvider runs when the part of a provider a probe measured has
+// changed. Renaming one or editing its notes does not reprobe it; repointing
it
+// at another endpoint, or giving it another key, does.
+func ProbeEditedProvider(stored *Provider, updated *Provider, keyChanged bool)
{
+ if stored == nil || updated == nil {
+ return
+ }
+ if !keyChanged && stored.BaseUrl == updated.BaseUrl &&
+ stored.Type == updated.Type && stored.AuthMode ==
updated.AuthMode {
+ return
+ }
+ probeProviderInBackground(updated, ProbeTriggerEdited)
+}
+
+// probeSweepDelay lets the process finish starting before it spends anything,
+// and probeSweepInterval catches a provider added while probing was off.
+const (
+ probeSweepDelay = 45 * time.Second
+ probeSweepInterval = 6 * time.Hour
+)
+
+// StartProviderProbeSweep probes the providers that were configured before
+// probing existed, or before it was turned on. A provider is swept once: the
+// sweep looks for the ones with no probe at all, so it goes quiet by itself.
+func StartProviderProbeSweep() {
+ go func() {
+ time.Sleep(probeSweepDelay)
+ for {
+ sweepUnprobedProviders()
+ time.Sleep(probeSweepInterval)
+ }
+ }()
+}
+
+func sweepUnprobedProviders() {
+ if !isProbeAutomatic() || ormer == nil || ormer.Engine == nil {
+ return
+ }
+
+ // An empty owner lists every account's providers, which is what a
sweep of
+ // this machine has to cover.
+ providers, err := GetProviders("")
+ if err != nil {
+ beego.Error("provider probe sweep could not list providers:",
err)
+ return
+ }
+ probed, err := probedProviderIds()
+ if err != nil {
+ beego.Error("provider probe sweep could not read past probes:",
err)
+ return
+ }
+
+ pending := []*Provider{}
+ for _, provider := range providers {
+ if provider.Status == "disabled" || probed[provider.GetId()] ||
!isProbable(provider) {
+ continue
+ }
+ pending = append(pending, provider)
+ }
+ if len(pending) == 0 {
+ return
+ }
+
+ gate := make(chan struct{}, probeMaxParallel)
+ group := sync.WaitGroup{}
+ for _, provider := range pending {
+ group.Add(1)
+ gate <- struct{}{}
+ go func(target *Provider) {
+ defer group.Done()
+ defer func() { <-gate }()
+ if _, err := ProbeProviderNow(target,
ProbeTriggerUnused); err != nil {
+ beego.Error("provider probe failed for",
target.GetId()+":", err)
+ }
+ }(provider)
+ }
+ group.Wait()
+}
+
+// isProbable reports whether there is anything to probe with. A provider that
+// forwards the caller's own login has no credential of its own here, so the
+// suite would only ever measure a 401.
+func isProbable(provider *Provider) bool {
+ if !IsProviderTypeSupported(provider) || provider.BaseUrl == "" {
+ return false
+ }
+ return !UsesClientAuth(provider) && provider.ApiKey != ""
+}
+
+func probedProviderIds() (map[string]bool, error) {
+ rows := []ProviderProbe{}
+ err :=
ormer.Engine.Table("provider_probe").Distinct("provider").Find(&rows)
+ if err != nil {
+ return nil, err
+ }
+ probed := map[string]bool{}
+ for _, row := range rows {
+ probed[row.Provider] = true
+ }
+ return probed, nil
+}
+
+func saveProviderProbe(probe *ProviderProbe) error {
+ if ormer == nil || ormer.Engine == nil {
+ return fmt.Errorf("no database")
+ }
+ if _, err := ormer.Engine.Insert(probe); err != nil {
+ return err
+ }
+ prunePastProbes(probe.Provider)
+ return nil
+}
+
+// prunePastProbes keeps the recent history of one provider and drops the rest.
+func prunePastProbes(providerId string) {
+ ids := []int64{}
+ err := ormer.Engine.Table("provider_probe").Cols("id").
+ Where("provider = ?", providerId).Desc("id").Limit(1,
probeKeepPerProvider).
+ Find(&ids)
+ if err != nil || len(ids) == 0 {
+ return
+ }
+ if _, err := ormer.Engine.In("id", ids).Delete(&ProviderProbe{}); err
!= nil {
+ beego.Error("provider probe cleanup failed:", err)
+ }
+}
+
+// GetProviderProbes is the newest probe of every provider that has one, which
+// is what the audit page shows beside the traffic.
+func GetProviderProbes() ([]*ProviderProbe, error) {
+ if ormer == nil || ormer.Engine == nil {
+ return []*ProviderProbe{}, nil
+ }
+
+ probes := []*ProviderProbe{}
+ // Every kept probe is read and the newest per provider taken here: the
+ // history is bounded per provider, and the three drivers Gateway
supports
+ // spell a per-group maximum differently.
+ err := ormer.Engine.Desc("id").Find(&probes)
+ if err != nil {
+ return nil, err
+ }
+
+ latest := []*ProviderProbe{}
+ seen := map[string]bool{}
+ for _, probe := range probes {
+ if seen[probe.Provider] {
+ continue
+ }
+ seen[probe.Provider] = true
+ latest = append(latest, probe)
+ }
+ return latest, nil
+}
+
+// GetProviderProbeHistory is every kept run for one provider, newest first.
Two
+// runs of the same suite months apart are what says a backend was swapped.
+func GetProviderProbeHistory(providerId string) ([]*ProviderProbe, error) {
+ probes := []*ProviderProbe{}
+ if ormer == nil || ormer.Engine == nil {
+ return probes, nil
+ }
+ err := ormer.Engine.Where("provider = ?",
providerId).Desc("id").Find(&probes)
+ return probes, err
+}
+
+// DeleteProviderProbes drops the history of a provider that no longer exists.
+func DeleteProviderProbes(providerId string) error {
+ if ormer == nil || ormer.Engine == nil {
+ return nil
+ }
+ _, err := ormer.Engine.Where("provider = ?",
providerId).Delete(&ProviderProbe{})
+ return err
+}
+
+// newProviderProbe is the row a run starts from, so a failure before the first
+// request still produces a report saying so.
+func newProviderProbe(provider *Provider, trigger string) *ProviderProbe {
+ return &ProviderProbe{
+ Provider: provider.GetId(),
+ CreatedTime: util.GetCurrentTime(),
+ Trigger: trigger,
+ Protocol: ProviderProtocol(provider),
+ VendorHeaders: []string{},
+ Checks: []ProbeCheck{},
+ }
+}
diff --git a/object/provider_probe_checks.go b/object/provider_probe_checks.go
new file mode 100644
index 0000000..4052d94
--- /dev/null
+++ b/object/provider_probe_checks.go
@@ -0,0 +1,591 @@
+// Copyright 2026 The casbin Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The checks themselves. Each one asks a question the vendor's own API
+// documents an answer to, so a finding can be argued with rather than
believed.
+
+package object
+
+import (
+ "encoding/json"
+ "fmt"
+ "math"
+ "net/http"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/apache/casbin-gateway/auditutil"
+)
+
+// ---------------------------------------------------------------------------
+// One: a forced tool call, which also carries the identity and header checks
+// ---------------------------------------------------------------------------
+
+const probeToolName = "record_station_reading"
+
+const probeToolSystem = "You are answering a capability probe. Call the tool
exactly once with plausible values."
+
+const probeToolPrompt = "Station BRAVO-7 is tagged coastal and tidal. It
reported 12.5 degrees Celsius " +
+ "and the sensor was working. Record that reading."
+
+// probeToolSchema is nested and has required fields at two levels. A model
that
+// is not the one being sold usually flattens it or drops the array.
+func probeToolSchema() map[string]any {
+ return map[string]any{
+ "type": "object",
+ "properties": map[string]any{
+ "station": map[string]any{
+ "type": "object",
+ "properties": map[string]any{
+ "id": map[string]any{"type":
"string"},
+ "tags": map[string]any{"type": "array",
"items": map[string]any{"type": "string"}},
+ },
+ "required": []string{"id", "tags"},
+ },
+ "samples": map[string]any{
+ "type": "array",
+ "items": map[string]any{
+ "type": "object",
+ "properties": map[string]any{
+ "celsius":
map[string]any{"type": "number"},
+ "ok":
map[string]any{"type": "boolean"},
+ },
+ "required": []string{"celsius", "ok"},
+ },
+ },
+ },
+ "required": []string{"station", "samples"},
+ }
+}
+
+func probeToolBody(upstream string, model string) map[string]any {
+ schema := probeToolSchema()
+ if upstream == ProtocolAnthropic {
+ return map[string]any{
+ "model": model,
+ "max_tokens": probeToolMaxTokens,
+ "system": probeToolSystem,
+ "messages": []any{map[string]any{"role": "user",
"content": probeToolPrompt}},
+ "tools": []any{map[string]any{
+ "name": probeToolName,
+ "description": "Record one temperature reading
for a station.",
+ "input_schema": schema,
+ }},
+ "tool_choice": map[string]any{"type": "tool", "name":
probeToolName},
+ }
+ }
+
+ return map[string]any{
+ "model": model,
+ "max_tokens": probeToolMaxTokens,
+ "messages": []any{
+ map[string]any{"role": "system", "content":
probeToolSystem},
+ map[string]any{"role": "user", "content":
probeToolPrompt},
+ },
+ "tools": []any{map[string]any{
+ "type": "function",
+ "function": map[string]any{
+ "name": probeToolName,
+ "description": "Record one temperature reading
for a station.",
+ "parameters": schema,
+ },
+ }},
+ "tool_choice": map[string]any{
+ "type": "function",
+ "function": map[string]any{"name": probeToolName},
+ },
+ }
+}
+
+// probeToolCall is the first request. It decides whether the rest of the suite
+// runs at all, and answers three questions at once: what the upstream says it
+// is, what its headers look like, and whether it can hold a nested schema.
+func probeToolCall(provider *Provider, model string, probe *ProviderProbe)
*probeBody {
+ upstream := ProviderProtocol(provider)
+ answer, err := sendProbe(provider, probeToolBody(upstream, model),
false)
+ if err != nil {
+ probe.Error = auditutil.BoundString(err.Error(),
probeErrorChars)
+ return nil
+ }
+ if !answer.ok() {
+ probe.Error = answer.failure()
+ return nil
+ }
+
+ parsed := probeBody{}
+ if err := json.Unmarshal(answer.body, &parsed); err != nil {
+ probe.Error = "the upstream did not answer with a completion
this API defines"
+ return nil
+ }
+
+ probe.addUsage(parsed.Usage)
+ probe.UpstreamModel = parsed.Model
+ probe.VendorHeaders = vendorHeadersOf(upstream, answer.header)
+
+ probe.addCheck(probeIdentityCheck(model, parsed.Model))
+ probe.addCheck(probeVendorCheck(probe.VendorHeaders))
+ probe.addCheck(probeToolsCheck(upstream, &parsed))
+ return &parsed
+}
+
+// probeIdentityCheck compares the model name that came back with the one that
+// was asked for. Matching proves little on its own — echoing a name back is
the
+// easiest thing in the world to do — but not matching is decisive.
+func probeIdentityCheck(asked string, answered string) ProbeCheck {
+ check := ProbeCheck{Key: ProbeIdentity, Level: LlmAuditUnknown, Facts:
[]string{}}
+ if strings.TrimSpace(answered) == "" {
+ return check
+ }
+ check.Facts = []string{answered}
+ if sameModelName(asked, answered) {
+ check.Level = LlmAuditOk
+ return check
+ }
+ check.Level = LlmAuditAlert
+ return check
+}
+
+// sameModelName treats a name that only adds a version or date suffix as the
+// same model, which is what a vendor answering "claude-opus-4-5-20251101" to a
+// request for "claude-opus-4-5" is doing.
+func sameModelName(asked string, answered string) bool {
+ left := strings.ToLower(strings.TrimSpace(asked))
+ right := strings.ToLower(strings.TrimSpace(answered))
+ if left == right {
+ return true
+ }
+ // An aggregator prefixes the vendor, e.g. "anthropic/claude-opus-4-5".
+ if index := strings.LastIndex(left, "/"); index >= 0 {
+ left = left[index+1:]
+ }
+ if index := strings.LastIndex(right, "/"); index >= 0 {
+ right = right[index+1:]
+ }
+ return strings.HasPrefix(left, right) || strings.HasPrefix(right, left)
+}
+
+// The response headers each vendor's own API sets. A reseller in front of the
+// real thing usually passes some of them through; a backend that never talked
+// to the vendor has none to pass.
+var probeVendorHeaders = map[string][]string{
+ ProtocolAnthropic: {
+ "Request-Id",
+ "Anthropic-Organization-Id",
+ "Anthropic-Ratelimit-Requests-Limit",
+ "Anthropic-Ratelimit-Input-Tokens-Limit",
+ "X-Should-Retry",
+ },
+ ProtocolOpenAi: {
+ "X-Request-Id",
+ "Openai-Organization",
+ "Openai-Processing-Ms",
+ "Openai-Version",
+ "X-Ratelimit-Limit-Requests",
+ },
+}
+
+func vendorHeadersOf(upstream string, header http.Header) []string {
+ found := []string{}
+ for _, name := range probeVendorHeaders[upstream] {
+ if header.Get(name) != "" {
+ found = append(found, name)
+ }
+ }
+ return found
+}
+
+// probeVendorCheck never reaches "alert": a relay that strips headers is being
+// tidy, not necessarily dishonest. It is a corroborating signal, and the page
+// words it as one.
+func probeVendorCheck(found []string) ProbeCheck {
+ check := ProbeCheck{
+ Key: ProbeVendor,
+ Level: LlmAuditWarn,
+ Facts: found,
+ Value: float64(len(found)),
+ }
+ if len(found) >= 2 {
+ check.Level = LlmAuditOk
+ }
+ return check
+}
+
+// probeToolsCheck reads the forced call back. A model that cannot fill a
+// two-level schema is not the frontier model it was sold as, whatever the
+// response says its name is.
+func probeToolsCheck(upstream string, parsed *probeBody) ProbeCheck {
+ check := ProbeCheck{Key: ProbeTools, Level: LlmAuditAlert, Facts:
[]string{}}
+
+ arguments := ""
+ if upstream == ProtocolAnthropic {
+ for _, block := range parsed.Content {
+ if block.Type == "tool_use" && block.Name ==
probeToolName {
+ arguments = string(block.Input)
+ }
+ }
+ } else if len(parsed.Choices) > 0 {
+ for _, call := range parsed.Choices[0].Message.ToolCalls {
+ if call.Function.Name == probeToolName {
+ arguments = call.Function.Arguments
+ }
+ }
+ }
+
+ if arguments == "" {
+ return check
+ }
+
+ var filled struct {
+ Station struct {
+ Id string `json:"id"`
+ Tags []string `json:"tags"`
+ } `json:"station"`
+ Samples []struct {
+ Celsius *float64 `json:"celsius"`
+ Ok *bool `json:"ok"`
+ } `json:"samples"`
+ }
+ // Arguments that are not valid JSON and a half-filled object are the
same
+ // finding for whoever is paying: the model was made to call the tool
and
+ // could not hold the shape it was given.
+ if err := json.Unmarshal([]byte(arguments), &filled); err != nil {
+ check.Level = LlmAuditWarn
+ return check
+ }
+ if filled.Station.Id == "" || len(filled.Samples) == 0 ||
filled.Samples[0].Celsius == nil {
+ check.Level = LlmAuditWarn
+ return check
+ }
+
+ check.Level = LlmAuditOk
+ return check
+}
+
+// ---------------------------------------------------------------------------
+// Two: the shape of the event stream
+// ---------------------------------------------------------------------------
+
+// probeAnthropicEvents is the sequence the Anthropic streaming API documents.
A
+// backend translating some other API into this one is where these go missing.
+var probeAnthropicEvents = []string{
+ "message_start", "content_block_start", "content_block_delta",
+ "content_block_stop", "message_delta", "message_stop",
+}
+
+func probeStreamBody(upstream string, model string) map[string]any {
+ body := map[string]any{
+ "model": model,
+ "max_tokens": probeStreamMaxTokens,
+ "messages": []any{map[string]any{"role": "user", "content":
"Reply with the single word: ready."}},
+ "stream": true,
+ }
+ if upstream != ProtocolAnthropic {
+ body["stream_options"] = map[string]any{"include_usage": true}
+ }
+ return body
+}
+
+func probeStreamShape(provider *Provider, model string, probe *ProviderProbe) {
+ upstream := ProviderProtocol(provider)
+ answer, err := sendProbe(provider, probeStreamBody(upstream, model),
true)
+ if err != nil {
+ probe.addCheck(ProbeCheck{Key: ProbeStream, Level:
LlmAuditUnknown, Facts: []string{}})
+ return
+ }
+ if !answer.ok() {
+ probe.addCheck(ProbeCheck{Key: ProbeStream, Level:
LlmAuditUnknown, Facts: []string{}})
+ return
+ }
+
+ probe.TtftMs = answer.ttftMs
+ probe.Requests++
+ if upstream == ProtocolAnthropic {
+ probe.addCheck(probeAnthropicStreamCheck(answer))
+ return
+ }
+ probe.addCheck(probeOpenAiStreamCheck(answer))
+}
+
+func probeAnthropicStreamCheck(answer *probeAnswer) ProbeCheck {
+ seen := map[string]bool{}
+ for _, name := range answer.events {
+ seen[name] = true
+ }
+
+ missing := []string{}
+ for _, name := range probeAnthropicEvents {
+ if !seen[name] {
+ missing = append(missing, name)
+ }
+ }
+
+ check := ProbeCheck{Key: ProbeStream, Level: LlmAuditOk, Facts:
missing, Value: float64(len(missing))}
+ switch {
+ case !seen["message_start"] || !seen["message_delta"]:
+ check.Level = LlmAuditAlert
+ return check
+ case len(missing) > 0:
+ check.Level = LlmAuditWarn
+ return check
+ }
+
+ // message_start carries the input count before a token has been
generated.
+ // A backend translating some other API has nothing to put there yet and
+ // sends a zero, which is as good as a missing event.
+ if probeStreamOpeningInput(answer) == 0 {
+ check.Level, check.Facts = LlmAuditWarn,
[]string{"message_start.usage"}
+ }
+ return check
+}
+
+func probeStreamOpeningInput(answer *probeAnswer) int {
+ for index, name := range answer.events {
+ if name != "message_start" {
+ continue
+ }
+ var parsed struct {
+ Message struct {
+ Usage probeUsage `json:"usage"`
+ } `json:"message"`
+ }
+ if json.Unmarshal(answer.payloads[index], &parsed) == nil {
+ return parsed.Message.Usage.billedInput()
+ }
+ }
+ return 0
+}
+
+func probeOpenAiStreamCheck(answer *probeAnswer) ProbeCheck {
+ chunks, finished, done, usage := 0, false, false, false
+ for _, payload := range answer.payloads {
+ if strings.TrimSpace(string(payload)) == "[DONE]" {
+ done = true
+ continue
+ }
+ var parsed struct {
+ Choices []struct {
+ FinishReason *string `json:"finish_reason"`
+ } `json:"choices"`
+ Usage *probeUsage `json:"usage"`
+ }
+ if json.Unmarshal(payload, &parsed) != nil {
+ continue
+ }
+ chunks++
+ if len(parsed.Choices) > 0 && parsed.Choices[0].FinishReason !=
nil {
+ finished = true
+ }
+ if parsed.Usage != nil && parsed.Usage.billedInput() > 0 {
+ usage = true
+ }
+ }
+
+ missing := []string{}
+ if chunks == 0 {
+ missing = append(missing, "chat.completion.chunk")
+ }
+ if !finished {
+ missing = append(missing, "finish_reason")
+ }
+ if !done {
+ missing = append(missing, "[DONE]")
+ }
+
+ check := ProbeCheck{Key: ProbeStream, Level: LlmAuditOk, Facts:
missing, Value: float64(len(missing))}
+ if len(missing) > 0 {
+ check.Level = LlmAuditAlert
+ return check
+ }
+ if !usage {
+ // Asking for it is standard; not answering it is common among
the
+ // OpenAI-compatible vendors, so this is a note rather than a
finding.
+ check.Level, check.Facts = LlmAuditWarn, []string{"usage"}
+ }
+ return check
+}
+
+// ---------------------------------------------------------------------------
+// Three: the cache, sent twice, which also settles the billing question
+// ---------------------------------------------------------------------------
+
+// probeFillerChars is long enough to clear the largest minimum any of these
+// vendors caches at, and no longer: every character of it is paid for twice.
+const probeFillerChars = 9000
+
+const probeFillerParagraph = "This paragraph is a fixed cache probe sent by
Casbin Gateway on behalf of " +
+ "the account that owns this key. It carries no user data and asks for
no work. It exists only so that the " +
+ "same request can be sent twice and the two answers compared against
each other, which is the one way to " +
+ "tell whether an upstream is really accounting for a prompt cache."
+
+var (
+ probeFillerOnce sync.Once
+ probeFillerText string
+)
+
+// probeFiller is byte-identical on every call, which is what a cache needs:
two
+// requests that differ anywhere in the cached prefix are two different
prompts.
+func probeFiller() string {
+ probeFillerOnce.Do(func() {
+ builder := strings.Builder{}
+ for index := 0; builder.Len() < probeFillerChars; index++ {
+ fmt.Fprintf(&builder, "Section %d. %s\n", index,
probeFillerParagraph)
+ }
+ probeFillerText = builder.String()
+ })
+ return probeFillerText
+}
+
+const probeCachePrompt = "Answer with the single word: ok."
+
+func probeCacheBody(upstream string, model string) map[string]any {
+ if upstream == ProtocolAnthropic {
+ return map[string]any{
+ "model": model,
+ "max_tokens": probeCacheMaxTokens,
+ "system": []any{map[string]any{
+ "type": "text",
+ "text": probeFiller(),
+ "cache_control": map[string]any{"type":
"ephemeral"},
+ }},
+ "messages": []any{map[string]any{"role": "user",
"content": probeCachePrompt}},
+ }
+ }
+
+ return map[string]any{
+ "model": model,
+ "max_tokens": probeCacheMaxTokens,
+ "messages": []any{
+ map[string]any{"role": "system", "content":
probeFiller()},
+ map[string]any{"role": "user", "content":
probeCachePrompt},
+ },
+ }
+}
+
+// probeCacheGap lets a written cache become readable before it is asked for.
+const probeCacheGap = 1500 * time.Millisecond
+
+// probeCachePair sends the same long request twice. The pair answers two
+// questions no single request can: whether the cache is real, and whether the
+// input counter is stable — two identical requests that are billed different
+// amounts of input were not counted, they were invented.
+func probeCachePair(provider *Provider, model string, probe *ProviderProbe) {
+ upstream := ProviderProtocol(provider)
+ body := probeCacheBody(upstream, model)
+
+ first, err := sendProbeUsage(provider, body, probe)
+ if err != nil {
+ probe.addCacheUnknown()
+ return
+ }
+ time.Sleep(probeCacheGap)
+ second, err := sendProbeUsage(provider, body, probe)
+ if err != nil {
+ probe.addCacheUnknown()
+ return
+ }
+
+ probe.addCheck(probeCacheCheck(upstream, first, second))
+ probe.addCheck(probeBillingCheck(first, second))
+}
+
+// sendProbeUsage sends one of the pair and returns what it reported.
+func sendProbeUsage(provider *Provider, body map[string]any, probe
*ProviderProbe) (probeUsage, error) {
+ answer, err := sendProbe(provider, body, false)
+ if err != nil {
+ return probeUsage{}, err
+ }
+ if !answer.ok() {
+ return probeUsage{}, fmt.Errorf("%s", answer.failure())
+ }
+
+ parsed := probeBody{}
+ if err := json.Unmarshal(answer.body, &parsed); err != nil {
+ return probeUsage{}, fmt.Errorf("the answer could not be read")
+ }
+ probe.addUsage(parsed.Usage)
+ return parsed.Usage, nil
+}
+
+// addCacheUnknown records that the pair could not be sent, so the two checks
it
+// answers are shown as unmeasured rather than left off the report.
+func (probe *ProviderProbe) addCacheUnknown() {
+ probe.addCheck(ProbeCheck{Key: ProbeCache, Level: LlmAuditUnknown,
Facts: []string{}})
+ probe.addCheck(ProbeCheck{Key: ProbeBilling, Level: LlmAuditUnknown,
Facts: []string{}})
+}
+
+func probeCacheCheck(upstream string, first probeUsage, second probeUsage)
ProbeCheck {
+ check := ProbeCheck{
+ Key: ProbeCache,
+ Value: float64(second.cached()),
+ Facts: []string{strconv.Itoa(first.written()),
strconv.Itoa(second.cached())},
+ }
+
+ switch {
+ case second.cached() > 0:
+ check.Level = LlmAuditOk
+ case first.written() > 0:
+ // Written but never read back: the write may simply not have
landed in
+ // time, which is why this is not the harder finding.
+ check.Level = LlmAuditWarn
+ case upstream == ProtocolAnthropic:
+ // This request carried cache_control, and this API documents a
counter
+ // for it. Nothing came back in either field.
+ check.Level = LlmAuditAlert
+ default:
+ // The OpenAI-compatible vendors are not all expected to cache
at all.
+ check.Level = LlmAuditWarn
+ }
+ return check
+}
+
+// probeBillingEstimate is what the cache request should count as input, from
+// the bytes that were actually sent. Four characters to the token is rough, so
+// only a large gap is treated as a finding.
+func probeBillingEstimate() int {
+ return (len(probeFiller()) + len(probeCachePrompt)) / 4
+}
+
+func probeBillingCheck(first probeUsage, second probeUsage) ProbeCheck {
+ billed, again := first.billedInput(), second.billedInput()
+ check := ProbeCheck{Key: ProbeBilling, Level: LlmAuditUnknown, Facts:
[]string{}}
+ if billed == 0 {
+ return check
+ }
+
+ // The two requests were byte-identical, so the counts have to be too.
This
+ // part needs no tokenizer and no assumption.
+ drift := math.Abs(float64(billed-again)) / float64(billed)
+ if drift > 0.02 {
+ check.Level = LlmAuditAlert
+ check.Value = drift
+ // The leading token says which of the two comparisons the
numbers are.
+ check.Facts = []string{"drift", strconv.Itoa(billed),
strconv.Itoa(again)}
+ return check
+ }
+
+ estimate := probeBillingEstimate()
+ check.Value = float64(billed) / float64(estimate)
+ check.Facts = []string{"estimate", strconv.Itoa(billed),
strconv.Itoa(estimate)}
+ switch {
+ case check.Value >= 2.5 || check.Value <= 0.4:
+ check.Level = LlmAuditAlert
+ case check.Value >= 1.5 || check.Value <= 0.7:
+ check.Level = LlmAuditWarn
+ default:
+ check.Level = LlmAuditOk
+ }
+ return check
+}
diff --git a/object/provider_probe_run.go b/object/provider_probe_run.go
new file mode 100644
index 0000000..98b2dfc
--- /dev/null
+++ b/object/provider_probe_run.go
@@ -0,0 +1,343 @@
+// Copyright 2026 The casbin Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file sends the probe requests and reads the answers. The bodies are
+// fixed on purpose: a check is only worth showing if the same request can be
+// sent again and produce the same finding.
+
+package object
+
+import (
+ "bufio"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+
+ "github.com/apache/casbin-gateway/auditutil"
+ "github.com/apache/casbin-gateway/proxy"
+)
+
+const (
+ probeRequestTimeout = 90 * time.Second
+ // probeMaxBody bounds one answer. A probe asks for a few tokens, so
+ // anything past this is an error page.
+ probeMaxBody = 1 << 20
+ // probeErrorChars is how much of an upstream failure is kept on the
report.
+ probeErrorChars = 400
+)
+
+// The answer lengths asked for. They are small on purpose: what is being
+// measured is the envelope, not the writing.
+const (
+ probeToolMaxTokens = 256
+ probeStreamMaxTokens = 32
+ probeCacheMaxTokens = 16
+)
+
+// runProviderProbe is the suite, in the order that stops early where a later
+// question would be meaningless. The first request decides whether the
upstream
+// is usable at all.
+func runProviderProbe(provider *Provider, trigger string) *ProviderProbe {
+ probe := newProviderProbe(provider, trigger)
+ start := time.Now()
+ defer func() { probe.DurationMs = time.Since(start).Milliseconds() }()
+
+ if !isProbable(provider) {
+ probe.Error = "this provider has no credential of its own to
probe with"
+ return probe
+ }
+
+ model, err := probeModelOf(provider)
+ if err != nil {
+ probe.Error = auditutil.BoundString(err.Error(),
probeErrorChars)
+ return probe
+ }
+ probe.Model = model
+
+ answer := probeToolCall(provider, model, probe)
+ if answer == nil {
+ return probe
+ }
+ probe.Ok = true
+
+ probeStreamShape(provider, model, probe)
+ probeCachePair(provider, model, probe)
+ priceProviderProbe(probe)
+ return probe
+}
+
+// probeModelOf is the model a probe asks for. The first model configured on
the
+// provider is used rather than a cheap one: a report about a model nobody
routes
+// to would answer a question nobody asked. A provider naming none is asked
what
+// it serves.
+func probeModelOf(provider *Provider) (string, error) {
+ for _, model := range provider.Models {
+ if strings.TrimSpace(model) != "" {
+ return strings.TrimSpace(model), nil
+ }
+ }
+
+ models, err := FetchProviderModels(provider)
+ if err != nil {
+ return "", err
+ }
+ return models[0], nil
+}
+
+// ---------------------------------------------------------------------------
+// The requests
+// ---------------------------------------------------------------------------
+
+// probeAnswer is one upstream answer, kept whole so every check reads the same
+// bytes rather than a summary of them.
+type probeAnswer struct {
+ status int
+ header http.Header
+ body []byte
+ ttftMs int64
+ // events are the SSE event names in the order they arrived, empty for a
+ // request that was not streamed.
+ events []string
+ // payloads are the SSE data payloads, in the same order.
+ payloads [][]byte
+}
+
+func (answer *probeAnswer) ok() bool {
+ return answer.status >= 200 && answer.status < 300
+}
+
+// probeFailure is the one-line reason a request did not answer, with the
+// upstream's own words: the status code alone almost never says which of key,
+// URL, model name or plan was the problem.
+func (answer *probeAnswer) failure() string {
+ text := strings.Join(strings.Fields(string(answer.body)), " ")
+ if _, message := readProbeError(answer.body); message != "" {
+ text = message
+ }
+ return auditutil.BoundString(fmt.Sprintf("the upstream answered %d:
%s", answer.status, text), probeErrorChars)
+}
+
+func readProbeError(body []byte) (string, string) {
+ var parsed struct {
+ Error struct {
+ Message string `json:"message"`
+ Type string `json:"type"`
+ } `json:"error"`
+ }
+ if err := json.Unmarshal(body, &parsed); err != nil {
+ return "", ""
+ }
+ return parsed.Error.Type, parsed.Error.Message
+}
+
+// sendProbe posts one probe body to the provider's completion endpoint. An
+// upstream that rejects "max_tokens" is asked again the way the newer OpenAI
+// models spell it, so a provider serving those is measured rather than
skipped.
+func sendProbe(provider *Provider, body map[string]any, stream bool)
(*probeAnswer, error) {
+ answer, err := postProbe(provider, body, stream)
+ if err != nil || answer.ok() {
+ return answer, err
+ }
+
+ if _, message := readProbeError(answer.body);
strings.Contains(strings.ToLower(message), "max_tokens") {
+ if limit, ok := body["max_tokens"]; ok {
+ retried := map[string]any{}
+ for key, value := range body {
+ retried[key] = value
+ }
+ delete(retried, "max_tokens")
+ retried["max_completion_tokens"] = limit
+ return postProbe(provider, retried, stream)
+ }
+ }
+ return answer, nil
+}
+
+func postProbe(provider *Provider, body map[string]any, stream bool)
(*probeAnswer, error) {
+ upstream := ProviderProtocol(provider)
+ endpoint := "/chat/completions"
+ if upstream == ProtocolAnthropic {
+ endpoint = "/v1/messages"
+ }
+
+ target, err := BuildProviderUrl(provider.BaseUrl, upstream, endpoint)
+ if err != nil {
+ return nil, err
+ }
+ raw, err := json.Marshal(body)
+ if err != nil {
+ return nil, err
+ }
+
+ request, err := http.NewRequest(http.MethodPost, target,
strings.NewReader(string(raw)))
+ if err != nil {
+ return nil, err
+ }
+ request.Header.Set("Content-Type", "application/json")
+ if upstream == ProtocolAnthropic {
+ request.Header.Set("Anthropic-Version", AnthropicVersion)
+ }
+ SetProviderAuth(request.Header, provider)
+
+ client := &http.Client{Timeout: probeRequestTimeout, Transport:
proxy.Transport()}
+ start := time.Now()
+ response, err := client.Do(request)
+ if err != nil {
+ return nil, err
+ }
+ defer response.Body.Close()
+
+ answer := &probeAnswer{status: response.StatusCode, header:
response.Header}
+ if !stream || !answer.ok() {
+ answer.body, err = io.ReadAll(io.LimitReader(response.Body,
probeMaxBody))
+ answer.ttftMs = time.Since(start).Milliseconds()
+ return answer, err
+ }
+
+ readProbeStream(answer, response.Body, start)
+ return answer, nil
+}
+
+// readProbeStream keeps the shape of an event stream: the event names in
order,
+// their payloads, and how long the first one took. The names are the point —
+// a backend pretending to be another API usually gets the envelope wrong long
+// before it gets the text wrong.
+func readProbeStream(answer *probeAnswer, body io.Reader, start time.Time) {
+ reader := bufio.NewReaderSize(body, 64*1024)
+ name, data := "", []byte{}
+ first := true
+
+ for {
+ line, err := reader.ReadString('\n')
+ trimmed := strings.TrimRight(line, "\r\n")
+ switch {
+ case strings.HasPrefix(trimmed, "event:"):
+ name = strings.TrimSpace(strings.TrimPrefix(trimmed,
"event:"))
+ case strings.HasPrefix(trimmed, "data:"):
+ data = append(data,
strings.TrimPrefix(strings.TrimPrefix(trimmed, "data:"), " ")...)
+ case trimmed == "" && len(data) > 0:
+ if first {
+ answer.ttftMs = time.Since(start).Milliseconds()
+ first = false
+ }
+ answer.events = append(answer.events, name)
+ answer.payloads = append(answer.payloads,
append([]byte{}, data...))
+ name, data = "", data[:0]
+ }
+ if err != nil {
+ if len(data) > 0 {
+ answer.events = append(answer.events, name)
+ answer.payloads = append(answer.payloads,
append([]byte{}, data...))
+ }
+ return
+ }
+ }
+}
+
+// ---------------------------------------------------------------------------
+// The usage counters, in both spellings
+// ---------------------------------------------------------------------------
+
+type probeUsage struct {
+ InputTokens int `json:"input_tokens"`
+ OutputTokens int `json:"output_tokens"`
+ CacheCreationInputTokens int `json:"cache_creation_input_tokens"`
+ CacheReadInputTokens int `json:"cache_read_input_tokens"`
+
+ PromptTokens int `json:"prompt_tokens"`
+ CompletionTokens int `json:"completion_tokens"`
+ PromptTokensDetails struct {
+ CachedTokens int `json:"cached_tokens"`
+ } `json:"prompt_tokens_details"`
+ PromptCacheHitTokens int `json:"prompt_cache_hit_tokens"`
+}
+
+// fresh, cached, written and answered are the four counters the rest of the
+// gateway prices, read out of whichever spelling arrived.
+func (usage probeUsage) fresh() int {
+ if usage.PromptTokens > 0 {
+ return maxInt(usage.PromptTokens-usage.cached(), 0)
+ }
+ return usage.InputTokens
+}
+
+func (usage probeUsage) cached() int {
+ return maxInt(maxInt(usage.CacheReadInputTokens,
usage.PromptTokensDetails.CachedTokens), usage.PromptCacheHitTokens)
+}
+
+func (usage probeUsage) written() int {
+ return usage.CacheCreationInputTokens
+}
+
+func (usage probeUsage) answered() int {
+ return maxInt(usage.OutputTokens, usage.CompletionTokens)
+}
+
+// billedInput is everything the upstream counted as input, however it split
it.
+// Two identical requests have to be billed the same amount of it.
+func (usage probeUsage) billedInput() int {
+ return usage.fresh() + usage.cached() + usage.written()
+}
+
+func maxInt(left int, right int) int {
+ if left > right {
+ return left
+ }
+ return right
+}
+
+// probeBody is the outer shape both APIs answer a completion in, as far as a
+// probe needs to read it.
+type probeBody struct {
+ Model string `json:"model"`
+ Usage probeUsage `json:"usage"`
+ Content []struct {
+ Type string `json:"type"`
+ Name string `json:"name"`
+ Input json.RawMessage `json:"input"`
+ } `json:"content"`
+ Choices []struct {
+ Message struct {
+ ToolCalls []struct {
+ Function struct {
+ Name string `json:"name"`
+ Arguments string `json:"arguments"`
+ } `json:"function"`
+ } `json:"tool_calls"`
+ } `json:"message"`
+ } `json:"choices"`
+}
+
+func (probe *ProviderProbe) addUsage(usage probeUsage) {
+ probe.Requests++
+ probe.PromptTokens += usage.fresh()
+ probe.CompletionTokens += usage.answered()
+ probe.CacheReadTokens += usage.cached()
+ probe.CacheWriteTokens += usage.written()
+}
+
+func (probe *ProviderProbe) addCheck(check ProbeCheck) {
+ probe.Checks = append(probe.Checks, check)
+}
+
+// priceProviderProbe costs what the probe itself spent, so the report can say
+// what it took to produce rather than leaving it to the next invoice.
+func priceProviderProbe(probe *ProviderProbe) {
+ cost, priced := GetLlmCost(probe.Model, probe.PromptTokens,
probe.CompletionTokens,
+ probe.CacheWriteTokens, probe.CacheReadTokens)
+ probe.Cost, probe.Priced = cost, priced
+}
diff --git a/object/setting.go b/object/setting.go
index 3df85a4..4d5253d 100644
--- a/object/setting.go
+++ b/object/setting.go
@@ -40,6 +40,10 @@ type Setting struct {
LlmRecordMaxPayloadBytes int `xorm:"int"
json:"llmRecordMaxPayloadBytes"`
LlmPricingFile string `xorm:"varchar(500)"
json:"llmPricingFile"`
+ // ProviderProbeMode is "auto", "manual" or "off". An automatic probe
spends
+ // a few cents of the account's own credit, so it is one setting away.
+ ProviderProbeMode string `xorm:"varchar(20)" json:"providerProbeMode"`
+
AgentPatchStateDir string `xorm:"varchar(500)"
json:"agentPatchStateDir"`
AgentRecordCapacity int `xorm:"int" json:"agentRecordCapacity"`
AgentMonitorPollSeconds int `xorm:"int"
json:"agentMonitorPollSeconds"`
@@ -73,6 +77,8 @@ func SyncSettingToConf(setting *Setting) {
"llmRecordMaxPayloadBytes":
strconv.Itoa(setting.LlmRecordMaxPayloadBytes),
"llmPricingFile": setting.LlmPricingFile,
+ "providerProbeMode": setting.ProviderProbeMode,
+
"agentPatchStateDir": setting.AgentPatchStateDir,
"agentRecordCapacity":
strconv.Itoa(setting.AgentRecordCapacity),
"agentMonitorPollSeconds":
strconv.Itoa(setting.AgentMonitorPollSeconds),
@@ -169,6 +175,16 @@ func InitBuiltInSetting() {
}
}
+ // Likewise for a column added after the row was written: an empty value
+ // would show as an empty choice on the Settings page rather than as the
+ // default the code already applies.
+ if setting.ProviderProbeMode == "" {
+ setting.ProviderProbeMode = GetProviderProbeMode()
+ if _, err = ormer.Engine.ID(core.PK{setting.Owner,
setting.Name}).Cols("provider_probe_mode").Update(setting); err != nil {
+ panic(err)
+ }
+ }
+
SyncSettingToConf(setting)
}
@@ -189,6 +205,8 @@ func newSettingFromConf() *Setting {
LlmRecordMaxPayloadBytes: conf.GetLlmRecordMaxPayloadBytes(),
LlmPricingFile: conf.GetLlmPricingFile(),
+ ProviderProbeMode: GetProviderProbeMode(),
+
AgentPatchStateDir: conf.GetAgentPatchStateDir(),
AgentRecordCapacity: conf.GetAgentRecordCapacity(),
AgentMonitorPollSeconds: conf.GetAgentMonitorPollSeconds(),
diff --git a/routers/router.go b/routers/router.go
index 63e994f..0e210d5 100644
--- a/routers/router.go
+++ b/routers/router.go
@@ -93,6 +93,9 @@ func initAPI() {
beego.Router("/api/get-provider-health", &controllers.ApiController{},
"GET:GetProviderHealth")
beego.Router("/api/get-provider-quotas", &controllers.ApiController{},
"GET:GetProviderQuotas")
beego.Router("/api/refresh-provider-quotas",
&controllers.ApiController{}, "POST:RefreshProviderQuotas")
+ beego.Router("/api/get-provider-probes", &controllers.ApiController{},
"GET:GetProviderProbes")
+ beego.Router("/api/get-provider-probe-history",
&controllers.ApiController{}, "GET:GetProviderProbeHistory")
+ beego.Router("/api/probe-provider", &controllers.ApiController{},
"POST:ProbeProvider")
beego.Router("/api/get-llm-records", &controllers.ApiController{},
"GET:GetLlmRecords")
beego.Router("/api/get-llm-record", &controllers.ApiController{},
"GET:GetLlmRecord")
@@ -100,6 +103,7 @@ func initAPI() {
beego.Router("/api/get-llm-record-stats", &controllers.ApiController{},
"GET:GetLlmRecordStats")
beego.Router("/api/get-llm-agent-stats", &controllers.ApiController{},
"GET:GetLlmAgentStats")
beego.Router("/api/get-llm-usage-trend", &controllers.ApiController{},
"GET:GetLlmUsageTrend")
+ beego.Router("/api/get-llm-provider-audit",
&controllers.ApiController{}, "GET:GetLlmProviderAudit")
beego.Router("/api/stream-llm-records", &controllers.ApiController{},
"GET:StreamLlmRecords")
beego.Router("/api/delete-llm-record", &controllers.ApiController{},
"POST:DeleteLlmRecord")
beego.Router("/api/clear-llm-records", &controllers.ApiController{},
"POST:ClearLlmRecords")
diff --git a/web/src/backend/LlmRecordBackend.ts
b/web/src/backend/LlmRecordBackend.ts
index ce4cca9..4953fc5 100644
--- a/web/src/backend/LlmRecordBackend.ts
+++ b/web/src/backend/LlmRecordBackend.ts
@@ -14,7 +14,15 @@
import {query, request} from "@/backend/request";
import {ServerUrl} from "@/Setting";
-import type {LlmAgentStat, LlmPrice, LlmRecord, LlmRecordStats,
LlmRecordStatus, LlmTrendPoint} from "@/types";
+import type {
+ LlmAgentStat,
+ LlmAuditReport,
+ LlmPrice,
+ LlmRecord,
+ LlmRecordStats,
+ LlmRecordStatus,
+ LlmTrendPoint,
+} from "@/types";
export interface LlmRecordFilter {
model?: string;
@@ -51,6 +59,11 @@ export function getLlmUsageTrend(filter: LlmRecordFilter =
{}, bucket: "hour" |
return request<LlmTrendPoint[]>(`/api/get-llm-usage-trend${query({...filter,
bucket: bucket})}`);
}
+/** The same window read for how each provider behaved rather than what it
cost. */
+export function getLlmProviderAudit(filter: LlmRecordFilter = {}) {
+ return
request<LlmAuditReport>(`/api/get-llm-provider-audit${query({...filter})}`);
+}
+
/** The same totals per agent, in one request rather than one request each. */
export function getLlmAgentStats(filter: LlmRecordFilter = {}) {
return
request<LlmAgentStat[]>(`/api/get-llm-agent-stats${query({...filter})}`);
diff --git a/web/src/backend/ProviderBackend.ts
b/web/src/backend/ProviderBackend.ts
index a447452..c2fbeb5 100644
--- a/web/src/backend/ProviderBackend.ts
+++ b/web/src/backend/ProviderBackend.ts
@@ -13,7 +13,14 @@
// limitations under the License.
import {itemId, query, request} from "@/backend/request";
-import type {Provider, ProviderHealth, ProviderQuota, ProviderTestResult} from
"@/types";
+import type {
+ Provider,
+ ProviderHealth,
+ ProviderProbe,
+ ProviderProbeMode,
+ ProviderQuota,
+ ProviderTestResult,
+} from "@/types";
export function getProviders(
owner: string,
@@ -82,3 +89,26 @@ export function getProviderQuotas() {
export function refreshProviderQuotas(id = "", force = false) {
return request<ProviderQuota[]>("/api/refresh-provider-quotas", "POST", {id:
id, force: force});
}
+
+/**
+ * The newest probe of every provider that has one. The second payload is how
+ * probes are started, so an unprobed provider can say whether it is waiting
for
+ * a sweep or for a button.
+ */
+export function getProviderProbes() {
+ return request<ProviderProbe[],
ProviderProbeMode>("/api/get-provider-probes");
+}
+
+/** Every kept run for one provider, newest first. */
+export function getProviderProbeHistory(id: string) {
+ return request<ProviderProbe[]>(`/api/get-provider-probe-history${query({id:
id})}`);
+}
+
+/**
+ * Runs the probe suite now. It sends four short requests to the upstream and
so
+ * spends a little of that provider's credit, which is why nothing calls this
on
+ * a page load.
+ */
+export function probeProvider(id: string) {
+ return request<ProviderProbe>(`/api/probe-provider${query({id: id})}`,
"POST");
+}
diff --git a/web/src/components/ProviderAuditCard.tsx
b/web/src/components/ProviderAuditCard.tsx
new file mode 100644
index 0000000..5260968
--- /dev/null
+++ b/web/src/components/ProviderAuditCard.tsx
@@ -0,0 +1,487 @@
+// Copyright 2026 The casbin Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import * as React from "react";
+import {CircleAlert, CircleCheck, CircleHelp, Stethoscope, TriangleAlert} from
"lucide-react";
+import i18next from "i18next";
+
+import * as Setting from "@/Setting";
+import {ProviderIcon} from "@/components/ProviderIcon";
+import {Badge} from "@/components/ui/badge";
+import {Button} from "@/components/ui/button";
+import {Card, CardContent} from "@/components/ui/card";
+import {SimpleTooltip} from "@/components/ui/tooltip";
+import {cn} from "@/lib/utils";
+import {formatCost, formatTokens} from "@/lib/usage";
+import type {
+ LlmAuditCheck,
+ LlmAuditLevel,
+ LlmProviderAudit,
+ ProbeCheck,
+ Provider,
+ ProviderProbe,
+ ProviderProbeMode,
+} from "@/types";
+
+/** How many model badges fit before the rest become a count. */
+const visibleModels = 4;
+
+/** Mirrors llmAuditMinSample on the server, for the sentence that explains
it. */
+const minSample = 20;
+
+const levelStyles: Record<LlmAuditLevel, {text: string; border: string; icon:
React.ElementType}> = {
+ ok: {text: "text-success", border: "border-success/25 bg-success/5", icon:
CircleCheck},
+ warn: {text: "text-warning", border: "border-warning/30 bg-warning/5", icon:
TriangleAlert},
+ alert: {text: "text-destructive", border: "border-destructive/30
bg-destructive/5", icon: CircleAlert},
+ unknown: {text: "text-muted-foreground", border: "border-border", icon:
CircleHelp},
+};
+
+function percent(share: number) {
+ const value = share * 100;
+ if (value > 0 && value < 1) {
+ return "<1%";
+ }
+ return `${Math.round(value)}%`;
+}
+
+function formatMs(ms: number) {
+ if (ms < 1000) {
+ return `${ms}ms`;
+ }
+ return `${(ms / 1000).toFixed(ms < 10000 ? 1 : 0)}s`;
+}
+
+/** Fills {name} placeholders, which is all the interpolation these strings
need. */
+function fill(key: string, values: Record<string, string | number>) {
+ return Object.entries(values).reduce(
+ (text, [name, value]) => text.split(`{${name}}`).join(String(value)),
+ i18next.t(key),
+ );
+}
+
+/** One measurement, whichever side it was measured from. */
+function CheckTile({
+ title,
+ value,
+ detail,
+ level,
+}: {
+ title: React.ReactNode;
+ value: React.ReactNode;
+ detail: React.ReactNode;
+ level: LlmAuditLevel;
+}) {
+ const style = levelStyles[level] ?? levelStyles.unknown;
+ const Icon = style.icon;
+
+ return (
+ <div className={cn("flex flex-col gap-1 rounded-lg border p-3",
style.border)}>
+ <div className="text-muted-foreground flex items-center gap-1.5 text-xs
font-medium">
+ <Icon className={cn("size-3.5 shrink-0", style.text)} />
+ <span className="truncate">{title}</span>
+ </div>
+ <span className={cn("text-lg font-semibold tabular-nums",
style.text)}>{value}</span>
+ <span className="text-muted-foreground text-xs
leading-snug">{detail}</span>
+ </div>
+ );
+}
+
+// ---------------------------------------------------------------------------
+// The probe: what the upstream answered when it was asked directly
+// ---------------------------------------------------------------------------
+
+/** Strongest evidence first, so the tile that matters is the one read first.
*/
+const probeOrder: ProbeCheck["key"][] = ["identity", "cache", "billing",
"stream", "tools", "vendor"];
+
+function probeTitle(key: ProbeCheck["key"]) {
+ switch (key) {
+ case "identity":
+ return i18next.t("audit:Model identity");
+ case "cache":
+ return i18next.t("audit:Prompt cache");
+ case "billing":
+ return i18next.t("audit:Token billing");
+ case "stream":
+ return i18next.t("audit:Stream shape");
+ case "tools":
+ return i18next.t("audit:Tool schema");
+ default:
+ return i18next.t("audit:Vendor headers");
+ }
+}
+
+function probeValue(check: ProbeCheck, probe: ProviderProbe) {
+ if (check.level === "unknown") {
+ return "—";
+ }
+ switch (check.key) {
+ case "identity":
+ return check.level === "ok" ? i18next.t("audit:Matches") :
i18next.t("audit:Different");
+ case "cache":
+ return check.level === "ok" ? formatTokens(Number(check.facts[1] ?? 0)) :
i18next.t("audit:None");
+ case "billing":
+ return check.facts[0] === "drift" ? i18next.t("audit:Unstable") :
`${check.value.toFixed(2)}x`;
+ case "stream":
+ return check.level === "ok" ? i18next.t("audit:Complete") :
String(check.facts.length);
+ case "tools":
+ return check.level === "ok" ? i18next.t("audit:Held") :
i18next.t("audit:Lost");
+ default:
+ return String(probe.vendorHeaders.length);
+ }
+}
+
+/**
+ * What the probe found, worded from the data the check carries. Nothing here
+ * calls a provider dishonest: it says what was asked and what came back, which
+ * is the part that can be sent to the provider and argued about.
+ */
+function probeDetail(check: ProbeCheck, probe: ProviderProbe) {
+ if (check.level === "unknown" && check.key !== "identity") {
+ return i18next.t("audit:This question could not be asked");
+ }
+
+ switch (check.key) {
+ case "identity":
+ if (check.facts.length === 0) {
+ return i18next.t("audit:No model name came back");
+ }
+ return check.level === "ok"
+ ? fill("audit:Identity ok", {answered: check.facts[0]})
+ : fill("audit:Identity different", {asked: probe.model, answered:
check.facts[0]});
+ case "cache": {
+ const written = Number(check.facts[0] ?? 0);
+ const read = Number(check.facts[1] ?? 0);
+ if (read > 0) {
+ return fill("audit:Cache probe ok", {written: written, read: read});
+ }
+ if (written > 0) {
+ return fill("audit:Cache probe written only", {written: written});
+ }
+ return i18next.t("audit:Cache probe none");
+ }
+ case "billing":
+ return check.facts[0] === "drift"
+ ? fill("audit:Billing unstable", {first: check.facts[1], second:
check.facts[2]})
+ : fill("audit:Billing detail", {billed: check.facts[1], estimate:
check.facts[2]});
+ case "stream":
+ return check.facts.length === 0
+ ? i18next.t("audit:Stream complete")
+ : fill("audit:Stream missing", {items: check.facts.join(", ")});
+ case "tools":
+ if (check.level === "ok") {
+ return i18next.t("audit:Tools ok");
+ }
+ return check.level === "warn" ? i18next.t("audit:Tools partial") :
i18next.t("audit:Tools none");
+ default:
+ return probe.vendorHeaders.length === 0
+ ? i18next.t("audit:No vendor header")
+ : fill("audit:Vendor detail", {names: probe.vendorHeaders.join(", ")});
+ }
+}
+
+function ProbeSection({
+ probe,
+ probeMode,
+ probing,
+ onProbe,
+}: {
+ probe?: ProviderProbe;
+ probeMode: ProviderProbeMode;
+ probing: boolean;
+ onProbe?: () => void;
+}) {
+ const label = (
+ <span className="text-xs font-semibold tracking-wide
uppercase">{i18next.t("audit:Probe")}</span>
+ );
+ const runButton = onProbe ? (
+ <Button size="xs" variant="outline" onClick={onProbe} loading={probing}>
+ <Stethoscope />
+ {probe ? i18next.t("audit:Probe again") : i18next.t("audit:Probe now")}
+ </Button>
+ ) : null;
+
+ if (!probe) {
+ return (
+ <div className="flex flex-col gap-2">
+ <div className="flex flex-wrap items-center gap-2">
+ {label}
+ <div className="ml-auto">{runButton}</div>
+ </div>
+ <p className="text-muted-foreground text-xs">
+ {probeMode === "auto"
+ ? i18next.t("audit:Not probed yet auto")
+ : probeMode === "manual"
+ ? i18next.t("audit:Not probed yet manual")
+ : i18next.t("audit:Probing is off")}
+ </p>
+ </div>
+ );
+ }
+
+ const checks = [...probe.checks].sort(
+ (left, right) => probeOrder.indexOf(left.key) -
probeOrder.indexOf(right.key),
+ );
+
+ return (
+ <div className="flex flex-col gap-2">
+ <div className="flex flex-wrap items-center gap-2">
+ {label}
+ <Badge variant="muted">{i18next.t(`audit:Trigger
${probe.trigger}`)}</Badge>
+ <span className="text-muted-foreground
text-xs">{Setting.getFormattedDate(probe.createdTime)}</span>
+ {probe.requests > 0 ? (
+ <SimpleTooltip title={i18next.t("audit:Probe cost hint")}>
+ <span className="text-muted-foreground text-xs">
+ {fill("audit:Probe cost", {
+ requests: probe.requests,
+ cost: probe.priced ? formatCost(probe.cost) : "?",
+ })}
+ </span>
+ </SimpleTooltip>
+ ) : null}
+ <div className="ml-auto">{runButton}</div>
+ </div>
+
+ {probe.ok ? (
+ <>
+ <div className="grid grid-cols-2 gap-2 md:grid-cols-3
2xl:grid-cols-6">
+ {checks.map(check => (
+ <CheckTile
+ key={check.key}
+ title={probeTitle(check.key)}
+ value={probeValue(check, probe)}
+ detail={probeDetail(check, probe)}
+ level={check.level}
+ />
+ ))}
+ </div>
+ {probe.ttftMs > 0 ? (
+ <p className="text-muted-foreground text-xs">
+ {fill("audit:First byte", {ttft: formatMs(probe.ttftMs)})}
+ </p>
+ ) : null}
+ </>
+ ) : (
+ <p className="text-destructive text-xs break-words">
+ {fill("audit:Probe failed", {reason: probe.error})}
+ </p>
+ )}
+ </div>
+ );
+}
+
+// ---------------------------------------------------------------------------
+// The traffic: what the kept records already said
+// ---------------------------------------------------------------------------
+
+function trafficTitle(key: LlmAuditCheck["key"]) {
+ switch (key) {
+ case "cache":
+ return i18next.t("audit:Cache accounting");
+ case "errors":
+ return i18next.t("audit:Failed attempts");
+ case "latency":
+ return i18next.t("audit:Response time");
+ default:
+ return i18next.t("audit:Unpriced models");
+ }
+}
+
+function trafficValue(check: LlmAuditCheck, audit: LlmProviderAudit) {
+ if (check.key === "latency") {
+ return audit.latencyP50Ms === 0 ? "—" : formatMs(audit.latencyP50Ms);
+ }
+ if (check.level === "unknown" && check.sample === 0) {
+ return "—";
+ }
+ return percent(check.value);
+}
+
+function trafficDetail(check: LlmAuditCheck, audit: LlmProviderAudit) {
+ if (check.level === "unknown" && check.sample > 0 && check.sample <
minSample) {
+ return fill("audit:Too few to measure", {min: minSample});
+ }
+ if (check.sample === 0) {
+ return check.key === "cache"
+ ? i18next.t("audit:Nothing could report a cache")
+ : i18next.t("audit:Nothing was relayed in this window");
+ }
+
+ switch (check.key) {
+ case "cache":
+ return check.value === 0
+ ? fill("audit:Cache never reported", {sample: check.sample})
+ : fill("audit:Cache detail", {hits: audit.cacheHits, sample:
check.sample});
+ case "errors":
+ return fill("audit:Failures detail", {
+ failed: audit.failed + audit.failedOver,
+ sample: check.sample,
+ });
+ case "latency":
+ return fill("audit:Latency detail", {
+ p95: formatMs(audit.latencyP95Ms),
+ ratio: check.value.toFixed(1),
+ });
+ default:
+ return audit.unpriced === 0
+ ? i18next.t("audit:Every model has a rate")
+ : fill("audit:Pricing detail", {unpriced: audit.unpriced, sample:
check.sample});
+ }
+}
+
+function Fact({label, value, tone}: {label: React.ReactNode; value:
React.ReactNode; tone?: string}) {
+ return (
+ <div className="flex flex-col gap-0.5">
+ <span className="text-muted-foreground text-xs">{label}</span>
+ <span className={cn("text-sm font-medium tabular-nums",
tone)}>{value}</span>
+ </div>
+ );
+}
+
+function TrafficSection({audit}: {audit?: LlmProviderAudit}) {
+ const label = (
+ <span className="text-xs font-semibold tracking-wide
uppercase">{i18next.t("audit:Traffic")}</span>
+ );
+
+ if (!audit) {
+ return (
+ <div className="flex flex-col gap-2">
+ {label}
+ <p className="text-muted-foreground text-xs">{i18next.t("audit:No
traffic in this window")}</p>
+ </div>
+ );
+ }
+
+ const shownModels = audit.models.slice(0, visibleModels);
+
+ return (
+ <div className="flex flex-col gap-2">
+ {label}
+ <div className="grid grid-cols-2 gap-2 xl:grid-cols-4">
+ {audit.checks.map(check => (
+ <CheckTile
+ key={check.key}
+ title={trafficTitle(check.key)}
+ value={trafficValue(check, audit)}
+ detail={trafficDetail(check, audit)}
+ level={check.level}
+ />
+ ))}
+ </div>
+
+ <div className="grid grid-cols-3 gap-3 pt-1 sm:grid-cols-6">
+ <Fact label={i18next.t("llm:Requests")}
value={audit.requests.toLocaleString()} />
+ <Fact
+ label={i18next.t("llm:Failed")}
+ value={audit.failed.toLocaleString()}
+ tone={audit.failed > 0 ? "text-warning" : undefined}
+ />
+ <SimpleTooltip title={i18next.t("audit:Failed over hint")}>
+ <div>
+ <Fact
+ label={i18next.t("audit:Failed over")}
+ value={audit.failedOver.toLocaleString()}
+ tone={audit.failedOver > 0 ? "text-warning" : undefined}
+ />
+ </div>
+ </SimpleTooltip>
+ <Fact label={i18next.t("llm:Tokens")}
value={formatTokens(audit.totalTokens)} />
+ <Fact label={i18next.t("llm:Cost")} value={formatCost(audit.cost)} />
+ <Fact label={i18next.t("llm:Cache read")}
value={formatTokens(audit.cacheReadTokens)} />
+ </div>
+
+ {audit.models.length > 0 ? (
+ <div className="flex flex-wrap items-center gap-1">
+ {shownModels.map(model => (
+ <Badge
+ key={model}
+ variant={audit.unpricedModels.includes(model) ? "warning" :
"muted"}
+ className="font-mono"
+ >
+ {model}
+ </Badge>
+ ))}
+ {audit.models.length > shownModels.length ? (
+ <SimpleTooltip title={audit.models.slice(visibleModels).join(",
")}>
+ <Badge variant="outline">{`+${audit.models.length -
shownModels.length}`}</Badge>
+ </SimpleTooltip>
+ ) : null}
+ </div>
+ ) : null}
+ </div>
+ );
+}
+
+// ---------------------------------------------------------------------------
+
+/**
+ * One provider from both sides: what it answered when it was asked directly,
+ * and what the kept records say it has been doing. The card is a report and
not
+ * a score — every tile shows what came back and what it was compared against,
+ * and the conclusion is left to whoever is paying the bill.
+ */
+export function ProviderAuditCard({
+ audit,
+ probe,
+ provider,
+ providersKnown,
+ probeMode,
+ probing,
+ onProbe,
+}: {
+ audit?: LlmProviderAudit;
+ probe?: ProviderProbe;
+ /** Missing when the records outlive the provider they name. */
+ provider?: Provider;
+ /** False while the provider listing has not landed, when a missing provider
+ * means "not known yet" rather than "no longer configured". */
+ providersKnown: boolean;
+ probeMode: ProviderProbeMode;
+ probing: boolean;
+ /** Missing for a provider there is nothing left to probe. */
+ onProbe?: () => void;
+}) {
+ const id = provider ? `${provider.owner}/${provider.name}` :
(audit?.provider ?? probe?.provider ?? "");
+ const name = provider?.displayName || id.split("/").slice(-1)[0];
+ const lastSeen = audit?.lastTime || probe?.createdTime || "";
+
+ return (
+ <Card>
+ <CardContent className="flex flex-col gap-4 p-4">
+ <div className="flex min-w-0 items-center gap-2">
+ <ProviderIcon icon={provider?.icon} baseUrl={provider?.baseUrl}
alt={name} size={20} />
+ <div className="flex min-w-0 flex-col">
+ <span className="truncate text-sm font-medium">{name}</span>
+ <span className="text-muted-foreground truncate font-mono
text-xs">{id}</span>
+ </div>
+ <div className="ml-auto flex shrink-0 items-center gap-2">
+ {/* The records outlive the provider they name, and a report about
+ one that is no longer configured has to say so before it is
read
+ as advice about the current setup. */}
+ {providersKnown && !provider ? (
+ <Badge variant="muted">{i18next.t("audit:Removed")}</Badge>
+ ) : null}
+ {lastSeen ? (
+ <SimpleTooltip title={i18next.t("audit:Last seen")}>
+ <span className="text-muted-foreground
text-xs">{Setting.getFormattedDate(lastSeen)}</span>
+ </SimpleTooltip>
+ ) : null}
+ </div>
+ </div>
+
+ <ProbeSection probe={probe} probeMode={probeMode} probing={probing}
onProbe={onProbe} />
+ <TrafficSection audit={audit} />
+ </CardContent>
+ </Card>
+ );
+}
diff --git a/web/src/components/ProviderAuditPanel.tsx
b/web/src/components/ProviderAuditPanel.tsx
new file mode 100644
index 0000000..452e190
--- /dev/null
+++ b/web/src/components/ProviderAuditPanel.tsx
@@ -0,0 +1,284 @@
+// Copyright 2026 The casbin Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import * as React from "react";
+import {Link} from "react-router-dom";
+import {Logs, RefreshCw, Stethoscope} from "lucide-react";
+import i18next from "i18next";
+
+import * as LlmRecordBackend from "@/backend/LlmRecordBackend";
+import * as ProviderBackend from "@/backend/ProviderBackend";
+import * as Setting from "@/Setting";
+import {ProviderAuditCard} from "@/components/ProviderAuditCard";
+import {EmptyState, ErrorState} from "@/components/shared/empty-state";
+import {Loading} from "@/components/shared/loading";
+import {SimpleSelect} from "@/components/shared/simple-select";
+import {MessageAlert} from "@/components/ui/alert";
+import {Button} from "@/components/ui/button";
+import {Card} from "@/components/ui/card";
+import {providerIdOf} from "@/lib/providers";
+import type {
+ LlmAuditReport,
+ LlmProviderAudit,
+ Provider,
+ ProviderProbe,
+ ProviderProbeMode,
+} from "@/types";
+
+/** The windows the traffic side offers, in hours. 0 is every record still
kept. */
+const windows = [24, 24 * 7, 24 * 30, 0];
+
+function windowLabel(hours: number) {
+ if (hours === 0) {
+ return i18next.t("llm:All time");
+ }
+ if (hours === 24) {
+ return i18next.t("llm:Last 24 hours");
+ }
+ return i18next.t("usage:Last {count} days").replace("{count}", String(hours
/ 24));
+}
+
+/** One provider, from whichever of the three sources happen to know it. */
+interface AuditRow {
+ id: string;
+ provider?: Provider;
+ audit?: LlmProviderAudit;
+ probe?: ProviderProbe;
+}
+
+/**
+ * Rows for every provider any source knows about: the ones configured now,
plus
+ * the ones only the records or an old probe still name. A provider that was
+ * added a minute ago has no traffic and belongs here just as much as one that
+ * has served for a month.
+ */
+function buildRows(
+ providers: Provider[],
+ audits: LlmProviderAudit[],
+ probes: ProviderProbe[],
+): AuditRow[] {
+ const rows = new Map<string, AuditRow>();
+ const rowOf = (id: string) => {
+ const existing = rows.get(id);
+ if (existing) {
+ return existing;
+ }
+ const created: AuditRow = {id: id};
+ rows.set(id, created);
+ return created;
+ };
+
+ providers.forEach(provider => {
+ rowOf(providerIdOf(provider)).provider = provider;
+ });
+ audits.forEach(audit => {
+ rowOf(audit.provider).audit = audit;
+ });
+ probes.forEach(probe => {
+ rowOf(probe.provider).probe = probe;
+ });
+
+ // Configured providers first, then by how much traffic they carried: a
+ // provider that is gone is history, not something to act on.
+ return [...rows.values()].sort((left, right) => {
+ if (Boolean(left.provider) !== Boolean(right.provider)) {
+ return left.provider ? -1 : 1;
+ }
+ const traffic = (right.audit?.requests ?? 0) - (left.audit?.requests ?? 0);
+ return traffic !== 0 ? traffic : left.id.localeCompare(right.id);
+ });
+}
+
+/**
+ * The channel audit: what each upstream looks like from here, from both sides.
+ * The traffic side reads records the proxy already kept and costs nothing. The
+ * probe side asks the upstream directly — four short requests whose answers
are
+ * documented, which is the only way to learn something a real request never
+ * happened to show — and spends a few cents of that provider's own credit.
+ */
+export function ProviderAuditPanel({owner}: {owner: string}) {
+ const [hours, setHours] = React.useState(24 * 7);
+ const [report, setReport] = React.useState<LlmAuditReport | null>(null);
+ // Null until a listing lands. An empty list and "not asked yet" must not
+ // render the same: every card would claim its provider is no longer
+ // configured, which is the one thing this page must never get wrong.
+ const [providers, setProviders] = React.useState<Provider[] | null>(null);
+ const [probes, setProbes] = React.useState<ProviderProbe[]>([]);
+ const [probeMode, setProbeMode] = React.useState<ProviderProbeMode>("auto");
+ const [probing, setProbing] = React.useState<string[]>([]);
+ const [loading, setLoading] = React.useState(true);
+ const [error, setError] = React.useState("");
+ const [recording, setRecording] = React.useState(true);
+
+ React.useEffect(() => {
+ LlmRecordBackend.getLlmRecordStatus()
+ .then(res => setRecording(res.status !== "ok" || (res.data?.mode ??
"off") !== "off"))
+ .catch(() => undefined);
+ }, []);
+
+ const loadProbes = React.useCallback(() => {
+ ProviderBackend.getProviderProbes()
+ .then(res => {
+ if (res.status !== "ok") {
+ return;
+ }
+ setProbes(res.data ?? []);
+ setProbeMode(res.data2 ?? "auto");
+ })
+ .catch(() => undefined);
+ }, []);
+
+ const load = React.useCallback(() => {
+ setLoading(true);
+ loadProbes();
+ // Reloaded with the rest, so a listing that failed once is not left to
+ // mislabel every card until the page is opened again.
+ ProviderBackend.getProviders(owner, 1, 100)
+ .then(res => setProviders(res.status === "ok" ? (res.data ?? []) : null))
+ .catch(() => setProviders(null));
+ LlmRecordBackend.getLlmProviderAudit({windowHours: hours})
+ .then(res => {
+ if (res.status === "ok" && res.data) {
+ setReport(res.data);
+ setError("");
+ } else {
+ setError(res.msg || i18next.t("general:Failed to get data"));
+ }
+ })
+ .catch(failure => setError(failure.message || String(failure)))
+ .then(() => setLoading(false));
+ }, [hours, owner, loadProbes]);
+
+ React.useEffect(() => {
+ load();
+ }, [load]);
+
+ // A probe started automatically finishes after the page has loaded, so the
+ // list is asked again for a while rather than only on a refresh.
+ React.useEffect(() => {
+ const interval = setInterval(loadProbes, 20000);
+ return () => clearInterval(interval);
+ }, [loadProbes]);
+
+ const runProbe = (id: string) => {
+ setProbing(current => [...current, id]);
+ ProviderBackend.probeProvider(id)
+ .then(res => {
+ if (res.status !== "ok" || !res.data) {
+ Setting.showMessage("error", res.msg || i18next.t("audit:The probe
could not run"));
+ return;
+ }
+ const probe = res.data;
+ setProbes(current => [probe, ...current.filter(item => item.provider
!== id)]);
+ })
+ .catch(failure => Setting.showMessage("error", `${failure}`))
+ .then(() => setProbing(current => current.filter(item => item !== id)));
+ };
+
+ const rows = buildRows(providers ?? [], report?.providers ?? [], probes);
+
+ return (
+ <div className="space-y-3">
+ <div className="flex flex-wrap items-start justify-between gap-2">
+ <p className="text-muted-foreground max-w-3xl
text-sm">{i18next.t("audit:Page description")}</p>
+ <div className="flex shrink-0 gap-2">
+ <SimpleSelect
+ className="w-[150px]"
+ value={String(hours)}
+ onChange={value => setHours(Number(value))}
+ options={windows.map(option => ({label: windowLabel(option),
value: String(option)}))}
+ />
+ <Button variant="outline" onClick={load} loading={loading}>
+ <RefreshCw />
+ {i18next.t("general:Refresh")}
+ </Button>
+ </div>
+ </div>
+
+ <MessageAlert
+ variant="info"
+ showIcon={false}
+ title={i18next.t(`audit:Probe mode ${probeMode}`)}
+ description={i18next.t("audit:Probe mode detail")}
+ />
+
+ {!recording ? (
+ <MessageAlert
+ variant="warning"
+ title={i18next.t("llm:Recording is off")}
+ description={i18next.t("audit:Recording is off detail")}
+ action={
+ <Button asChild size="sm" variant="outline">
+ <Link to="/llm-records">
+ <Logs />
+ {i18next.t("llm:LLM Records")}
+ </Link>
+ </Button>
+ }
+ />
+ ) : null}
+
+ {report?.truncated ? (
+ <MessageAlert
+ variant="info"
+ title={i18next
+ .t("audit:Only the newest records were read")
+ .replace("{count}", report.scanned.toLocaleString())}
+ />
+ ) : null}
+
+ {error !== "" ? (
+ <Card>
+ <ErrorState error={error} onRetry={load} />
+ </Card>
+ ) : loading && report === null ? (
+ <Loading />
+ ) : rows.length === 0 ? (
+ <Card>
+ <EmptyState
+ icon={Stethoscope}
+ title={i18next.t("audit:Nothing to audit yet")}
+ description={i18next.t("audit:Nothing to audit yet detail")}
+ />
+ </Card>
+ ) : (
+ <>
+ <div className="space-y-3">
+ {rows.map(row => (
+ <ProviderAuditCard
+ key={row.id}
+ audit={row.audit}
+ probe={row.probe}
+ provider={row.provider}
+ providersKnown={providers !== null}
+ probeMode={probeMode}
+ probing={probing.includes(row.id)}
+ // A provider the records still name but nothing is configured
+ // for has no endpoint left to ask.
+ onProbe={row.provider && probeMode !== "off" ? () =>
runProbe(row.id) : undefined}
+ />
+ ))}
+ </div>
+ <p className="text-muted-foreground text-xs">
+ {i18next
+ .t("audit:Read from {count} records")
+ .replace("{count}", (report?.scanned ?? 0).toLocaleString())}
+ {" · "}
+ {i18next.t("audit:What this cannot tell you")}
+ </p>
+ </>
+ )}
+ </div>
+ );
+}
diff --git a/web/src/locales/en/data.json b/web/src/locales/en/data.json
index 79900c1..05c7471 100644
--- a/web/src/locales/en/data.json
+++ b/web/src/locales/en/data.json
@@ -357,6 +357,84 @@
"{count} skills, fetched {when}": "{count} skills, fetched {when}",
"{files} files": "{files} files"
},
+ "audit": {
+ "Billing detail": "{billed} input tokens billed for about {estimate} sent.
Four characters to the token is rough, so only a wide gap means anything.",
+ "Billing unstable": "Two byte-identical requests were billed {first} and
{second} input tokens. The same prompt has to cost the same twice.",
+ "Cache accounting": "Cache accounting",
+ "Cache detail": "{hits} of {sample} long prompts came back with cache
tokens on them.",
+ "Cache never reported": "None of {sample} long prompts came back with any
cache tokens, so each was billed as fresh input.",
+ "Cache probe none": "The probe asked for a cache and the answer accounted
for none of it, so the whole prompt was billed as fresh input.",
+ "Cache probe ok": "{written} tokens written, {read} read back on the
identical second request.",
+ "Cache probe written only": "{written} tokens were written but none were
read back a second later.",
+ "Channel audit": "Channel audit",
+ "Complete": "Complete",
+ "Different": "Different",
+ "Every model has a rate": "Every model this provider served has a
published rate.",
+ "Failed attempts": "Failed attempts",
+ "Failed over": "Failed over",
+ "Failed over hint": "Attempts this provider lost, which another one in the
chain then answered. They are not counted as its requests.",
+ "Failures detail": "{failed} of {sample} attempts did not answer.",
+ "First byte": "First byte after {ttft}.",
+ "Held": "Held",
+ "Identity different": "Asked for {asked} and the answer named {answered}.",
+ "Identity ok": "Answered as {answered}. Echoing a name back is easy, so
this is a floor rather than a proof.",
+ "Last seen": "Last seen",
+ "Latency detail": "Slowest 5% from {p95}, {ratio}x the median.",
+ "Lost": "Lost",
+ "Matches": "Matches",
+ "Model identity": "Model identity",
+ "No model name came back": "The answer carried no model name.",
+ "No published rate for": "No published rate for",
+ "No traffic in this window": "No relayed request in this window.",
+ "No vendor header": "No response header of this vendor's own API came
back. A relay may simply strip them.",
+ "None": "None",
+ "Not probed yet auto": "Not probed yet. A provider is probed when it is
added, when its endpoint or key changes, and once if it has never been.",
+ "Not probed yet manual": "Not probed yet. Probing only runs when it is
asked for.",
+ "Nothing could report a cache": "Nothing here could have reported a cache:
the OpenAI protocol does not account for one, and short prompts are not
eligible.",
+ "Nothing to audit yet": "No relayed request to read yet",
+ "Nothing to audit yet detail": "The audit reads records the proxy already
kept. Point an agent at a provider and its first requests will show up here.",
+ "Nothing was relayed in this window": "Nothing was relayed in this
window.",
+ "Only the newest records were read": "This window holds more records than
one audit reads, so only the newest {count} were counted.",
+ "Page description": "What each upstream looks like from here, from both
sides. Traffic reads records Gateway already kept and costs nothing. A probe
asks the upstream directly, with four short requests whose right answers the
vendor's own API documents, and spends a few cents of that provider's credit.
Every figure names what it was measured over, because a rate over four requests
is not a finding.",
+ "Pricing detail": "{unpriced} of {sample} answered requests used a model
no price list names.",
+ "Probe": "Probe",
+ "Probe again": "Probe again",
+ "Probe cost": "{requests} requests, {cost}",
+ "Probe cost hint": "What the probe itself spent of this provider's
credit.",
+ "Probe failed": "The probe could not be completed: {reason}",
+ "Probe mode auto": "Probe automatically",
+ "Probe mode detail": "A probe sends four short requests and spends a few
cents of the probed provider's own credit. Change this under Settings.",
+ "Probe mode manual": "Probe only when asked",
+ "Probe mode off": "Never probe",
+ "Probe now": "Probe now",
+ "Probing is off": "Probing is turned off, so nothing here was asked
directly.",
+ "Prompt cache": "Prompt cache",
+ "Read from {count} records": "Read from {count} records",
+ "Recording is off detail": "The audit has nothing to read while
llmRecordMode is \"off\". Metadata alone is enough for every check here;
prompts do not have to be stored.",
+ "Removed": "No longer configured",
+ "Response time": "Response time",
+ "Stream complete": "Every event this API documents arrived, in order.",
+ "Stream missing": "Missing or empty: {items}.",
+ "Stream shape": "Stream shape",
+ "The probe could not run": "The probe could not run",
+ "This question could not be asked": "This request did not go through, so
the question stays unanswered.",
+ "Token billing": "Token billing",
+ "Too few to measure": "Fewer than {min} requests, too few to read anything
into.",
+ "Tool schema": "Tool schema",
+ "Tools none": "The tool call was forced and the answer contained none.",
+ "Tools ok": "The forced call came back with both levels of the schema
filled in.",
+ "Tools partial": "The tool was called but the nested fields it requires
were not filled in.",
+ "Traffic": "Traffic",
+ "Trigger added": "on adding",
+ "Trigger edited": "after an edit",
+ "Trigger manual": "asked for",
+ "Trigger unused": "first sweep",
+ "Unpriced models": "Unpriced models",
+ "Unstable": "Unstable",
+ "Vendor detail": "Carries {names}.",
+ "Vendor headers": "Vendor headers",
+ "What this cannot tell you": "These are the upstream's own numbers. They
show what it reported, not what it ran."
+ },
"general": {
"Action": "Action",
"Advanced": "Advanced",
@@ -665,6 +743,8 @@
"Casdoor application": "Casdoor application",
"Casdoor endpoint": "Casdoor endpoint",
"Casdoor organization": "Casdoor organization",
+ "Channel probes": "Channel probes",
+ "Channel probes description": "A probe sends four short requests to a
provider's upstream and checks the answers against what the vendor's API
documents: the model name reported, the vendor headers carried, the shape of
the event stream, whether prompt caching is really accounted for, and whether
two identical requests are billed the same. It spends a few cents of that
provider's own credit.",
"Client ID": "Client ID",
"Client secret": "Client secret",
"LLM records": "LLM records",
@@ -677,6 +757,8 @@
"Page description": "Everything Casbin Gateway reads out of its
configuration. Changes are stored in the database and applied at once.",
"Pricing file": "Pricing file",
"Pricing file hint": "Optional JSON file of token prices in US dollars per
million, overriding the built-in list prices.",
+ "Probe mode": "When to probe",
+ "Probe mode hint": "Automatic probing runs when a provider is added, when
its endpoint, type or key changes, and once for any provider that has never
been probed.",
"Queue capacity": "Write queue capacity",
"Queue capacity hint": "A full queue drops records rather than slowing the
proxy down.",
"Record mode": "Record mode",
diff --git a/web/src/locales/zh/data.json b/web/src/locales/zh/data.json
index 493a85f..3915412 100644
--- a/web/src/locales/zh/data.json
+++ b/web/src/locales/zh/data.json
@@ -357,6 +357,84 @@
"{count} skills, fetched {when}": "{count} 个技能，拉取于 {when}",
"{files} files": "{files} 个文件"
},
+ "audit": {
+ "Billing detail": "实际发出约 {estimate} token，被计了 {billed}。四字符折一 token
只是粗估，差得多才说明问题。",
+ "Billing unstable": "两次字节完全相同的请求，被计成了 {first} 和 {second} 个输入
token。同样的提示词两次不该是两个价。",
+ "Cache accounting": "缓存计费",
+ "Cache detail": "{sample} 次长提示词中，有 {hits} 次返回了缓存 token。",
+ "Cache never reported": "{sample} 次长提示词全部没有返回任何缓存 token，也就是每一次都按全新输入计费。",
+ "Cache probe none": "探针明确要求了缓存，返回里一个缓存 token 都没有——整段提示词按全新输入计费。",
+ "Cache probe ok": "写入 {written} token，一秒后的相同请求读回 {read}。",
+ "Cache probe written only": "写入了 {written} token，但一秒后并没有读回。",
+ "Channel audit": "渠道体检",
+ "Complete": "齐全",
+ "Different": "不一致",
+ "Every model has a rate": "该渠道服务过的模型都能查到公开价格。",
+ "Failed attempts": "失败请求",
+ "Failed over": "被切走",
+ "Failed over hint": "该渠道没答上、由链路中其它渠道接管的尝试。它们不计入该渠道的请求数。",
+ "Failures detail": "{sample} 次尝试中有 {failed} 次没有返回结果。",
+ "First byte": "首字节耗时 {ttft}。",
+ "Held": "接住了",
+ "Identity different": "请求的是 {asked}，返回里写的是 {answered}。",
+ "Identity ok": "返回 {answered}。把名字原样回显是最容易做到的事，所以这只是底线，不是证明。",
+ "Last seen": "最后一次",
+ "Latency detail": "最慢的 5% 从 {p95} 起，是中位数的 {ratio} 倍。",
+ "Lost": "没接住",
+ "Matches": "一致",
+ "Model identity": "模型身份",
+ "No model name came back": "返回里没有模型名。",
+ "No published rate for": "查不到公开价格的模型：",
+ "No traffic in this window": "这段时间内没有转发过请求。",
+ "No vendor header": "没有返回该厂商官方 API 的任何响应头。中转站也可能只是把它们去掉了。",
+ "None": "无",
+ "Not probed yet auto": "还没探测过。新增渠道、改了地址或密钥、以及从没测过的渠道都会自动探测一次。",
+ "Not probed yet manual": "还没探测过。当前只在手动触发时探测。",
+ "Nothing could report a cache": "这里没有能体现缓存的请求：OpenAI 协议不单独计缓存，短提示词也不到门槛。",
+ "Nothing to audit yet": "还没有可读的转发记录",
+ "Nothing to audit yet detail": "体检读的是代理本来就已经留下的记录。把某个 Agent
指向某个渠道，它的第一批请求就会出现在这里。",
+ "Nothing was relayed in this window": "这段时间内没有转发过请求。",
+ "Only the newest records were read": "这段时间的记录条数超过了单次体检的上限，只统计了最新的 {count}
条。",
+ "Page description":
"从这台机器看，每个上游渠道两面各是什么样。「真实流量」读的是网关本来就保存下来的记录，不花一分钱；「主动探测」直接去问上游——4
个短请求，正确答案写在厂商自己的 API 文档里——会花掉该渠道几分钱额度。每个数字都标着它是在多少次请求上算出来的：四次请求算出来的比例算不上结论。",
+ "Pricing detail": "{sample} 次成功请求中，有 {unpriced} 次用的模型在任何价格表里都查不到。",
+ "Probe": "主动探测",
+ "Probe again": "重新探测",
+ "Probe cost": "{requests} 次请求，{cost}",
+ "Probe cost hint": "探测本身花掉的该渠道额度。",
+ "Probe failed": "探测没能完成：{reason}",
+ "Probe mode auto": "自动探测",
+ "Probe mode detail": "一次探测会发 4 个短请求，花掉被测渠道几分钱额度。可在「设置」里改。",
+ "Probe mode manual": "只在手动触发时探测",
+ "Probe mode off": "从不探测",
+ "Probe now": "立即探测",
+ "Probing is off": "探测已关闭，这里没有直接问过上游。",
+ "Prompt cache": "提示词缓存",
+ "Read from {count} records": "读自 {count} 条记录",
+ "Recording is off detail": "llmRecordMode 为 \"off\"
时体检没有数据可读。这里的每项检查只需要元数据，不必保存提示词内容。",
+ "Removed": "已删除",
+ "Response time": "响应耗时",
+ "Stream complete": "该 API 文档里的事件全部按顺序到齐。",
+ "Stream missing": "缺失或为空：{items}。",
+ "Stream shape": "流式结构",
+ "The probe could not run": "探测无法执行",
+ "This question could not be asked": "这个请求没发出去，所以这项没有结论。",
+ "Token billing": "Token 计费",
+ "Too few to measure": "不足 {min} 次请求，还读不出什么。",
+ "Tool schema": "工具 Schema",
+ "Tools none": "强制要求调用工具，返回里却没有调用。",
+ "Tools ok": "强制调用返回了完整填好的两层嵌套结构。",
+ "Tools partial": "调用了工具，但 schema 要求的嵌套字段没填上。",
+ "Traffic": "真实流量",
+ "Trigger added": "新增时",
+ "Trigger edited": "修改后",
+ "Trigger manual": "手动",
+ "Trigger unused": "首次巡检",
+ "Unpriced models": "无价模型",
+ "Unstable": "不稳定",
+ "Vendor detail": "带有 {names}。",
+ "Vendor headers": "厂商响应头",
+ "What this cannot tell you": "这些是上游自己报的数。它们说明的是上游报了什么，而不是上游实际跑了什么。"
+ },
"general": {
"Action": "操作",
"Advanced": "高级",
@@ -665,6 +743,8 @@
"Casdoor application": "Casdoor 应用",
"Casdoor endpoint": "Casdoor 地址",
"Casdoor organization": "Casdoor 组织",
+ "Channel probes": "渠道探测",
+ "Channel probes description": "一次探测会向渠道上游发 4 个短请求，把返回和厂商 API
文档对照：返回的模型名、携带的厂商响应头、流式事件结构、是否真的在计缓存、以及两次相同请求是否同价。它会花掉该渠道几分钱额度。",
"Client ID": "Client ID",
"Client secret": "Client Secret",
"LLM records": "LLM 记录",
@@ -677,6 +757,8 @@
"Page description": "Casbin Gateway 的全部配置项。修改会保存到数据库并立即生效。",
"Pricing file": "价格文件",
"Pricing file hint": "可选的 JSON 文件，按每百万 Token 的美元价格覆盖内置价目表。",
+ "Probe mode": "何时探测",
+ "Probe mode hint": "自动探测会在新增渠道时、改了地址/类型/密钥后、以及对从未探测过的渠道各跑一次。",
"Queue capacity": "写入队列容量",
"Queue capacity hint": "队列写满时会丢弃记录，而不是拖慢代理。",
"Record mode": "记录模式",
diff --git a/web/src/pages/ProviderListPage.tsx
b/web/src/pages/ProviderListPage.tsx
index 43698af..68551c9 100644
--- a/web/src/pages/ProviderListPage.tsx
+++ b/web/src/pages/ProviderListPage.tsx
@@ -27,6 +27,7 @@ import i18next from "i18next";
import * as AgentBackend from "@/backend/AgentBackend";
import * as ProviderBackend from "@/backend/ProviderBackend";
import * as Setting from "@/Setting";
+import {ProviderAuditPanel} from "@/components/ProviderAuditPanel";
import {ProviderIconField} from "@/components/ProviderIcon";
import {ProviderGridCard} from "@/components/ProviderGridCard";
import {ProviderModelsField} from "@/components/ProviderModelsField";
@@ -43,6 +44,7 @@ import {Badge} from "@/components/ui/badge";
import {Button} from "@/components/ui/button";
import {Card, CardContent, CardDescription, CardHeader, CardTitle} from
"@/components/ui/card";
import {Input} from "@/components/ui/input";
+import {Tabs, TabsList, TabsTrigger} from "@/components/ui/tabs";
import {Textarea} from "@/components/ui/textarea";
import {
authProvider,
@@ -106,8 +108,13 @@ function Advanced({defaultOpen, children}: {defaultOpen:
boolean; children: Reac
);
}
+/** The grid of providers, or what the relayed records say about them. */
+type ProviderTab = "providers" | "audit";
+
export default function ProviderListPage({account}: {account: Account}) {
const navigate = useNavigate();
+ const isAdmin = Setting.isAdminUser(account);
+ const [tab, setTab] = React.useState<ProviderTab>("providers");
const [data, setData] = React.useState<Provider[]>([]);
const [total, setTotal] = React.useState(0);
const [loading, setLoading] = React.useState(false);
@@ -199,13 +206,13 @@ export default function ProviderListPage({account}:
{account: Account}) {
// so the agents are loaded next to them. A non-admin cannot list them, and
the
// card simply leaves that line out.
const loadAgents = React.useCallback(() => {
- if (!Setting.isAdminUser(account)) {
+ if (!isAdmin) {
return;
}
AgentBackend.getAgents()
.then(res => setAgents(res.status === "ok" ? (res.data ?? []) : []))
.catch(() => setAgents([]));
- }, [account]);
+ }, [isAdmin]);
React.useEffect(() => {
loadAgents();
@@ -336,7 +343,18 @@ export default function ProviderListPage({account}:
{account: Account}) {
}
/>
- {error !== "" && data.length === 0 ? (
+ {isAdmin ? (
+ <Tabs value={tab} onValueChange={value => setTab(value as
ProviderTab)}>
+ <TabsList>
+ <TabsTrigger
value="providers">{i18next.t("provider:Providers")}</TabsTrigger>
+ <TabsTrigger value="audit">{i18next.t("audit:Channel
audit")}</TabsTrigger>
+ </TabsList>
+ </Tabs>
+ ) : null}
+
+ {tab === "audit" ? (
+ <ProviderAuditPanel owner={account.name} />
+ ) : error !== "" && data.length === 0 ? (
<Card>
<ErrorState error={error} onRetry={() => fetchProviders()} />
</Card>
diff --git a/web/src/pages/SettingPage.tsx b/web/src/pages/SettingPage.tsx
index 9137277..6808fed 100644
--- a/web/src/pages/SettingPage.tsx
+++ b/web/src/pages/SettingPage.tsx
@@ -146,6 +146,24 @@ export default function SettingPage({account}: {account:
Account}) {
{textField("llmPricingFile", i18next.t("setting:Pricing file"),
i18next.t("setting:Pricing file hint"))}
</Section>
+ <Section
+ columns={2}
+ title={i18next.t("setting:Channel probes")}
+ description={i18next.t("setting:Channel probes description")}
+ >
+ <Field label={i18next.t("setting:Probe mode")}
hint={i18next.t("setting:Probe mode hint")}>
+ <SimpleSelect
+ value={setting.providerProbeMode}
+ onChange={value => updateField("providerProbeMode", value as
SettingType["providerProbeMode"])}
+ options={[
+ {label: i18next.t("audit:Probe mode auto"), value: "auto"},
+ {label: i18next.t("audit:Probe mode manual"), value: "manual"},
+ {label: i18next.t("audit:Probe mode off"), value: "off"},
+ ]}
+ />
+ </Field>
+ </Section>
+
<Section columns={2} title={i18next.t("setting:Agents")}>
{textField("agentPatchStateDir", i18next.t("setting:Agent state dir"),
i18next.t("setting:Agent state dir hint"))}
{numberField("agentRecordCapacity", i18next.t("setting:Agent record
capacity"), i18next.t("setting:Agent record capacity hint"), 1)}
diff --git a/web/src/types.ts b/web/src/types.ts
index 9e530a1..d3a1f9d 100644
--- a/web/src/types.ts
+++ b/web/src/types.ts
@@ -446,6 +446,103 @@ export interface LlmTrendPoint {
cost: number;
}
+/** The checks a channel audit runs, and how sure each one is of its answer. */
+export type LlmAuditKey = "cache" | "errors" | "latency" | "pricing";
+export type LlmAuditLevel = "ok" | "warn" | "alert" | "unknown";
+
+/**
+ * One measurement of a provider. The server sends the number and the level and
+ * leaves the wording here, so a check reads as a fact with the sample it was
+ * measured over rather than as a verdict.
+ */
+export interface LlmAuditCheck {
+ key: LlmAuditKey;
+ level: LlmAuditLevel;
+ /** A share of `sample` for every check but latency, which is P95 over P50.
*/
+ value: number;
+ sample: number;
+}
+
+/** What the kept records say about one provider that served them. */
+export interface LlmProviderAudit {
+ provider: string;
+ requests: number;
+ failed: number;
+ /** Attempts this provider lost, which another one then answered. */
+ failedOver: number;
+ retried: number;
+ promptTokens: number;
+ completionTokens: number;
+ cacheReadTokens: number;
+ cacheWriteTokens: number;
+ totalTokens: number;
+ cost: number;
+ /** Answered requests a cache could have been reported on at all. */
+ cacheable: number;
+ cacheHits: number;
+ latencyP50Ms: number;
+ latencyP95Ms: number;
+ unpriced: number;
+ unpricedModels: string[];
+ models: string[];
+ firstTime: string;
+ lastTime: string;
+ checks: LlmAuditCheck[];
+}
+
+export interface LlmAuditReport {
+ scanned: number;
+ /** The window held more records than one audit reads. */
+ truncated: boolean;
+ providers: LlmProviderAudit[];
+}
+
+/** How probes are started. An automatic probe spends a little of the credit.
*/
+export type ProviderProbeMode = "auto" | "manual" | "off";
+
+/** The questions an active probe asks, none of which the records can answer.
*/
+export type ProbeKey = "identity" | "vendor" | "stream" | "cache" | "tools" |
"billing";
+
+/**
+ * One probe measurement. `facts` is what actually came back, as data rather
+ * than as a sentence — the model name that answered, the headers present, the
+ * events missing, the counts that disagreed — so the page can word it in the
+ * reader's language and still quote the value the level was drawn from.
+ */
+export interface ProbeCheck {
+ key: ProbeKey;
+ level: LlmAuditLevel;
+ facts: string[];
+ value: number;
+}
+
+/** One run of the probe suite against one provider. */
+export interface ProviderProbe {
+ id: number;
+ provider: string;
+ createdTime: string;
+ /** Why it ran: "added", "edited", "unused" or "manual". */
+ trigger: string;
+ protocol: string;
+ model: string;
+ /** The model name the upstream answered with, which may not be the one
asked. */
+ upstreamModel: string;
+ ok: boolean;
+ error: string;
+ ttftMs: number;
+ durationMs: number;
+ /** What the probe itself spent. */
+ requests: number;
+ promptTokens: number;
+ completionTokens: number;
+ cacheReadTokens: number;
+ cacheWriteTokens: number;
+ cost: number;
+ priced: boolean;
+ vendorHeaders: string[];
+ checks: ProbeCheck[];
+}
+
/** One agent's share of the relayed requests, with what it last asked for. */
export interface LlmAgentStat {
agent: string;
@@ -516,6 +613,8 @@ export interface Setting {
llmRecordMaxPayloadBytes: number;
llmPricingFile: string;
+ providerProbeMode: ProviderProbeMode;
+
agentPatchStateDir: string;
agentRecordCapacity: number;
agentMonitorPollSeconds: number;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
