This is an automated email from the ASF dual-hosted git repository. tomsun28 pushed a commit to branch 2.0.0 in repository https://gitbox.apache.org/repos/asf/hertzbeat.git
commit 6a5e6099056bd88b7490bc45ae8eb711a5ac5b23 Author: tomsun28 <[email protected]> AuthorDate: Fri Oct 9 17:03:11 2026 +0800 fix(ai): let grounded answers use empty reads and discoverable namespaces Two agent UX defects found by real DeepSeek testing: - The grounding guard rejected every final answer when allowlisted read tools returned well-formed but empty results, so "are there any alerts" could never be answered with "none". A validated read that observed zero rows is still a real observation, so list-style classifiers now emit zero-count observations while malformed or untrusted output stays fail-closed. Collector detect/collect_once and protocol row reads keep requiring rows. - tool.search results now carry availableNamespaces so the model learns the full on-demand namespace catalog even when a query misses, instead of having to guess names. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../runtime/AgentReadGroundingEvaluator.java | 4 +-- .../runtime/AgentReadObservationClassifier.java | 20 +++++++---- .../ai/gateway/tool/core/AgentToolRegistry.java | 11 ++++++ .../tool/discovery/AgentToolDiscoveryService.java | 5 ++- .../runtime/AgentReadGroundingEvaluatorTest.java | 42 ++++++++++++++++++++++ .../gateway/tool/core/AgentToolRegistryTest.java | 11 ++++++ 6 files changed, 84 insertions(+), 9 deletions(-) diff --git a/hertzbeat-ai-gateway/src/main/java/org/apache/hertzbeat/ai/gateway/runtime/AgentReadGroundingEvaluator.java b/hertzbeat-ai-gateway/src/main/java/org/apache/hertzbeat/ai/gateway/runtime/AgentReadGroundingEvaluator.java index eabb041b39..a36f45d410 100644 --- a/hertzbeat-ai-gateway/src/main/java/org/apache/hertzbeat/ai/gateway/runtime/AgentReadGroundingEvaluator.java +++ b/hertzbeat-ai-gateway/src/main/java/org/apache/hertzbeat/ai/gateway/runtime/AgentReadGroundingEvaluator.java @@ -40,7 +40,7 @@ final class AgentReadGroundingEvaluator { return Optional.empty(); } AgentReadObservationClassifier.Observation observation = classifier.classify(call, result.getOutput()); - if (observation == null || observation.count() <= 0) { + if (observation == null || observation.count() == null || observation.count() < 0) { return Optional.empty(); } return Optional.of(AgentGroundingProof.builder() @@ -74,7 +74,7 @@ final class AgentReadGroundingEvaluator { && Objects.equals(outputHash(message.text()), proof.getOutputHash()) && StringUtils.hasText(proof.getObservationKind()) && proof.getObservationCount() != null - && proof.getObservationCount() > 0 + && proof.getObservationCount() >= 0 && proof.getTargetFingerprint() == null && proof.getTargetVersion() == null && proof.getEntityId() == null diff --git a/hertzbeat-ai-gateway/src/main/java/org/apache/hertzbeat/ai/gateway/runtime/AgentReadObservationClassifier.java b/hertzbeat-ai-gateway/src/main/java/org/apache/hertzbeat/ai/gateway/runtime/AgentReadObservationClassifier.java index 7f824735e0..53b54ea389 100644 --- a/hertzbeat-ai-gateway/src/main/java/org/apache/hertzbeat/ai/gateway/runtime/AgentReadObservationClassifier.java +++ b/hertzbeat-ai-gateway/src/main/java/org/apache/hertzbeat/ai/gateway/runtime/AgentReadObservationClassifier.java @@ -94,7 +94,7 @@ final class AgentReadObservationClassifier { } } } - Integer returned = positiveInteger(output.get("returnedPoints")); + Integer returned = nonNegativeInteger(output.get("returnedPoints")); Long total = nonNegativeLong(output.get("totalPoints")); return returned != null && returned == visible && total != null && total >= visible ? new Observation("metric-points", visible) : null; @@ -103,7 +103,7 @@ final class AgentReadObservationClassifier { private Observation page(Map<String, Object> output, String kind, RowShape rowShape) { List<?> content = list(output.get("content")); Long total = nonNegativeLong(output.get("totalElements")); - return content != null && !content.isEmpty() && content.stream().allMatch(rowShape::valid) + return content != null && content.stream().allMatch(rowShape::valid) && total != null && total >= content.size() ? new Observation(kind, content.size()) : null; } @@ -115,8 +115,11 @@ final class AgentReadObservationClassifier { } Observation single = page(cast(map(output.get("single"))), "alert-records", RowShape.ALERT); Observation group = page(cast(map(output.get("group"))), "alert-records", RowShape.ALERT); + if (single == null && group == null) { + return null; + } int count = (single == null ? 0 : single.count()) + (group == null ? 0 : group.count()); - return count > 0 ? new Observation("alert-records", count) : null; + return new Observation("alert-records", count); } private Observation alertGet(AgentRuntimeToolCall call, Map<String, Object> output) { @@ -133,7 +136,7 @@ final class AgentReadObservationClassifier { private Observation topology(Map<String, Object> output) { List<?> nodes = list(output.get("nodes")); - return Boolean.TRUE.equals(output.get("apiBacked")) && nodes != null && !nodes.isEmpty() + return Boolean.TRUE.equals(output.get("apiBacked")) && nodes != null && nodes.stream().allMatch(RowShape.TOPOLOGY_NODE::valid) ? new Observation("topology-nodes", nodes.size()) : null; } @@ -204,7 +207,7 @@ final class AgentReadObservationClassifier { private Observation exactListCount(Map<String, Object> output, String listKey, String countKey, String kind, RowShape rowShape) { List<?> values = list(output.get(listKey)); - Integer count = positiveInteger(output.get(countKey)); + Integer count = nonNegativeInteger(output.get(countKey)); return values != null && count != null && count == values.size() && values.stream().allMatch(rowShape::valid) ? new Observation(kind, count) : null; } @@ -266,7 +269,7 @@ final class AgentReadObservationClassifier { } private Observation positiveNumber(Map<String, Object> output, String key, String kind) { - Long count = positiveLong(output.get(key)); + Long count = nonNegativeLong(output.get(key)); return count != null && count <= Integer.MAX_VALUE ? new Observation(kind, count.intValue()) : null; } @@ -302,6 +305,11 @@ final class AgentReadObservationClassifier { return converted != null && converted <= Integer.MAX_VALUE ? converted.intValue() : null; } + private Integer nonNegativeInteger(Object value) { + Long converted = nonNegativeLong(value); + return converted != null && converted <= Integer.MAX_VALUE ? converted.intValue() : null; + } + private Long positiveLong(Object value) { Long converted = nonNegativeLong(value); return converted != null && converted > 0 ? converted : null; diff --git a/hertzbeat-ai-gateway/src/main/java/org/apache/hertzbeat/ai/gateway/tool/core/AgentToolRegistry.java b/hertzbeat-ai-gateway/src/main/java/org/apache/hertzbeat/ai/gateway/tool/core/AgentToolRegistry.java index 74c26f3f6a..f7fbe855fb 100644 --- a/hertzbeat-ai-gateway/src/main/java/org/apache/hertzbeat/ai/gateway/tool/core/AgentToolRegistry.java +++ b/hertzbeat-ai-gateway/src/main/java/org/apache/hertzbeat/ai/gateway/tool/core/AgentToolRegistry.java @@ -53,6 +53,17 @@ public class AgentToolRegistry { .toList(); } + public List<String> discoverableNamespaces() { + return handlers.values().stream() + .map(RegisteredTool::descriptor) + .filter(descriptor -> descriptor.getExposure() == AgentToolExposure.MODEL_ON_DEMAND) + .map(AgentToolDescriptor::getNamespace) + .filter(Objects::nonNull) + .distinct() + .sorted() + .toList(); + } + public List<AgentToolDescriptor> discoverableDescriptors(String namespace, String query) { // An unscoped discovery call would inject the entire on-demand catalog into the model context. if ((namespace == null || namespace.isBlank()) && (query == null || query.isBlank())) { diff --git a/hertzbeat-ai-gateway/src/main/java/org/apache/hertzbeat/ai/gateway/tool/discovery/AgentToolDiscoveryService.java b/hertzbeat-ai-gateway/src/main/java/org/apache/hertzbeat/ai/gateway/tool/discovery/AgentToolDiscoveryService.java index 92775a79b4..8d18f8ec64 100644 --- a/hertzbeat-ai-gateway/src/main/java/org/apache/hertzbeat/ai/gateway/tool/discovery/AgentToolDiscoveryService.java +++ b/hertzbeat-ai-gateway/src/main/java/org/apache/hertzbeat/ai/gateway/tool/discovery/AgentToolDiscoveryService.java @@ -56,7 +56,8 @@ public class AgentToolDiscoveryService { + "Loaded tools receive structured schemas on the next model request.") Boolean includeSchema) { boolean schemas = Boolean.TRUE.equals(includeSchema); - List<Map<String, Object>> tools = registryProvider.getObject() + AgentToolRegistry registry = registryProvider.getObject(); + List<Map<String, Object>> tools = registry .discoverableDescriptors(namespace, query).stream() .map(descriptor -> toolRow(descriptor, schemas)) .toList(); @@ -65,6 +66,8 @@ public class AgentToolDiscoveryService { result.put("query", query); result.put("tools", tools); result.put("count", tools.size()); + // Surface the full namespace catalog so a missed query still teaches the model where to look next. + result.put("availableNamespaces", registry.discoverableNamespaces()); return result; } diff --git a/hertzbeat-ai-gateway/src/test/java/org/apache/hertzbeat/ai/gateway/runtime/AgentReadGroundingEvaluatorTest.java b/hertzbeat-ai-gateway/src/test/java/org/apache/hertzbeat/ai/gateway/runtime/AgentReadGroundingEvaluatorTest.java index c98bbc48b2..d9f676f0bb 100644 --- a/hertzbeat-ai-gateway/src/test/java/org/apache/hertzbeat/ai/gateway/runtime/AgentReadGroundingEvaluatorTest.java +++ b/hertzbeat-ai-gateway/src/test/java/org/apache/hertzbeat/ai/gateway/runtime/AgentReadGroundingEvaluatorTest.java @@ -46,6 +46,43 @@ class AgentReadGroundingEvaluatorTest { }); } + @Test + void wellFormedEmptyReadsShouldStillCountAsObservations() { + Map<String, Object> emptyHistoryArguments = Map.of( + "monitorId", 7L, "metricKey", "basic.qps", "start", 1_000L, "end", 2_000L); + Map<String, Object> emptyHistory = Map.of( + "monitorId", 7L, "metricKey", "basic.qps", "start", 1_000L, "end", 2_000L, + "values", Map.of(), "returnedPoints", 0, "totalPoints", 0); + List<Observation> emptyReads = List.of( + observation("monitor.query", Map.of(), page(List.of()), "monitor-list", 0), + observation("logs.query", Map.of(), page(List.of()), "log-records", 0), + observation("traces.query", Map.of(), page(List.of()), "trace-records", 0), + observation("entity.query", Map.of(), page(List.of()), "entity-list", 0), + observation("collector.list", Map.of(), page(List.of()), "collector-list", 0), + observation("topology.query", Map.of(), + Map.of("apiBacked", true, "nodes", List.of()), "topology-nodes", 0), + observation("alert.query", Map.of(), + Map.of("result", page(List.of())), "alert-records", 0), + observation("alert.query", Map.of(), + Map.of("single", page(List.of()), "group", page(List.of())), "alert-records", 0), + observation("alert.similar", Map.of(), + Map.of("content", List.of(), "returnedCount", 0), "similar-alerts", 0), + observation("alert.summary", Map.of(), Map.of("total", 0), "alert-summary", 0), + observation("metrics.history", emptyHistoryArguments, emptyHistory, "metric-points", 0), + observation("metrics.realtime", Map.of("monitorId", 7L, "metrics", "basic"), + Map.of("monitorId", 7L, "metrics", "basic", "valueRows", List.of(), + "rowCount", 0), "metric-rows", 0), + observation("database.mysql_slow_queries", Map.of(), + Map.of("rows", List.of(), "rowCount", 0), "database-rows", 0)); + + emptyReads.forEach(observation -> { + AgentGroundingProof proof = evaluator.evaluate("run-1", call(observation), result(observation)) + .orElseThrow(() -> new AssertionError(observation.toolName())); + assertEquals(observation.kind(), proof.getObservationKind(), observation.toolName()); + assertEquals(0, proof.getObservationCount(), observation.toolName()); + }); + } + @Test void identifiersRowsAndUnknownDatabaseNamespaceMustFailClosed() { List<Observation> denied = List.of( @@ -142,6 +179,11 @@ class AgentReadGroundingEvaluatorTest { return new Observation(toolName, arguments, output, kind, 1); } + private Observation observation(String toolName, Map<String, Object> arguments, Map<String, Object> output, + String kind, int count) { + return new Observation(toolName, arguments, output, kind, count); + } + private AgentRuntimeToolCall call(Observation observation) { return AgentRuntimeToolCall.builder().toolCallId("call-1").toolName(observation.toolName()) .arguments(observation.arguments()).build(); diff --git a/hertzbeat-ai-gateway/src/test/java/org/apache/hertzbeat/ai/gateway/tool/core/AgentToolRegistryTest.java b/hertzbeat-ai-gateway/src/test/java/org/apache/hertzbeat/ai/gateway/tool/core/AgentToolRegistryTest.java index 7c9537fb6a..7a3ab5cdc0 100644 --- a/hertzbeat-ai-gateway/src/test/java/org/apache/hertzbeat/ai/gateway/tool/core/AgentToolRegistryTest.java +++ b/hertzbeat-ai-gateway/src/test/java/org/apache/hertzbeat/ai/gateway/tool/core/AgentToolRegistryTest.java @@ -89,6 +89,17 @@ class AgentToolRegistryTest { assertTrue(registry.discoverableDescriptors(null, "no-matching-token").isEmpty()); } + @Test + void shouldListOnlyOnDemandNamespaces() { + AgentToolRegistry registry = new AgentToolRegistry(); + registry.register(handler("jdbc.query", AgentToolExposure.MODEL_ON_DEMAND)); + registry.register(handler("alert_analysis_policy.create", AgentToolExposure.MODEL_ON_DEMAND)); + registry.register(handler("monitor.get", AgentToolExposure.MODEL_VISIBLE)); + + assertEquals(java.util.List.of("alert_analysis_policy", "jdbc"), + registry.discoverableNamespaces()); + } + @Test void shouldRejectUnscopedDiscovery() { AgentToolRegistry registry = new AgentToolRegistry(); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
