This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 3cb9efc2a1 [Fix][seatunnel-cli] Keep the original request when
answering a clarification (#13000)
3cb9efc2a1 is described below
commit 3cb9efc2a15223372bc1f87a92d027845a8ca800
Author: SEZ <[email protected]>
AuthorDate: Thu Nov 5 07:05:21 2026 +0000
[Fix][seatunnel-cli] Keep the original request when answering a
clarification (#13000)
Co-authored-by: SEZ9 <[email protected]>
---
seatunnel-cli/seatunnel_cli/agents.py | 68 +++++-
seatunnel-cli/seatunnel_cli/cli.py | 13 +-
.../tests/test_orchestrator_clarification.py | 268 +++++++++++++++++++++
3 files changed, 335 insertions(+), 14 deletions(-)
diff --git a/seatunnel-cli/seatunnel_cli/agents.py
b/seatunnel-cli/seatunnel_cli/agents.py
index 81c42ed7a4..76ae359b89 100644
--- a/seatunnel-cli/seatunnel_cli/agents.py
+++ b/seatunnel-cli/seatunnel_cli/agents.py
@@ -1270,6 +1270,12 @@ class Orchestrator:
self.on_stream = on_stream or (lambda *a: None)
self.memory_store = memory_store
self.pending_question: str | None = None
+ # The request that started the current exchange. A clarifying question
+ # splits one requirement across two turns, and only the planner reads
+ # the conversation history -- skill matching, slot filling and the
+ # enriched prompt are all built from the latest message alone. Without
+ # this the reply replaces the requirement instead of adding to it.
+ self._original_request: str | None = None
self._connector_metadata_block: str = ""
# ─── Context window management ───
@@ -1283,8 +1289,15 @@ class Orchestrator:
SUMMARY_THRESHOLD = 24 # only summarize when total exceeds this
def load_history(self, history: list[dict]) -> None:
- """Replace conversation history (e.g., when resuming a session)."""
+ """Replace conversation history (e.g., when resuming a session).
+
+ The clarification state belongs to the exchange being replaced, so it
+ is cleared too. Leaving it would merge the first message after a resume
+ into whatever requirement happened to be open before.
+ """
self.conversation_history = list(history)
+ self.pending_question = None
+ self._original_request = None
def _trimmed_history(self) -> list[dict]:
"""Return a context-window-friendly view of conversation_history.
@@ -1366,6 +1379,34 @@ class Orchestrator:
system = system + "\n\n" + block
return system
+ def _effective_request(self, user_input: str) -> str:
+ """The requirement as it now stands, across a clarifying question.
+
+ With no question outstanding this is just the message, which also
+ becomes the requirement the next reply will be added to. While a
+ question is outstanding the reply is appended rather than substituted,
+ and stays appended across further rounds.
+
+ A message sent while a question is outstanding is treated as an answer
+ to it. There is no reliable way to tell an answer from an unrelated new
+ request, and merging is the safer default: the planner reads the whole
+ history regardless, so the alternative loses the requirement outright.
+ """
+ if not self.pending_question:
+ self._original_request = user_input
+ return user_input
+
+ if not self._original_request:
+ self._original_request = user_input
+ return user_input
+
+ self._original_request = (
+ f"{self._original_request}\n\n"
+ f"Additional details, in reply to a clarifying question:\n"
+ f"{user_input}"
+ )
+ return self._original_request
+
def process_user_input(self, user_input: str) -> dict:
"""Process a user message and return result.
@@ -1382,14 +1423,24 @@ class Orchestrator:
"content": [{"text": user_input}],
})
+ # When this message answers a question we asked, the requirement lives
+ # in the earlier turn, so everything downstream has to see both. A
+ # reply like "use sensible defaults" carries no source, no sink and no
+ # table name, and on its own it produces an empty plan and an empty
+ # connector-metadata block.
+ effective_request = self._effective_request(user_input)
+
# Phase 1: Planning
self.on_status("thinking", "Analyzing your request...")
plan_result = self._run_planner()
if plan_result["type"] == "question":
+ self.pending_question = plan_result.get("content")
return plan_result
if plan_result["type"] == "chat":
+ self.pending_question = None
+ self._original_request = None
self.conversation_history.append({
"role": "assistant",
"content": [{"text": plan_result["content"]}],
@@ -1402,16 +1453,18 @@ class Orchestrator:
structured_plan = parse_structured_plan(plan_result["content"])
# Trigger-based skill matching: uses plan + user input keywords
- matched_skills = SkillRouter.match(structured_plan, user_input)
+ matched_skills = SkillRouter.match(structured_plan, effective_request)
skill = SkillExecutor(structured_plan, matched_skills)
# Fill slots (with pipeline expansion) and check for missing info
- missing = skill.fill_and_check(user_input, self.memory_store,
self.client)
+ missing = skill.fill_and_check(
+ effective_request, self.memory_store, self.client)
if missing:
question = (
"I need a few more details to generate the config:\n"
+ "\n".join(f"- {m}" for m in missing)
)
+ self.pending_question = question
return {
"type": "question",
"content": question,
@@ -1422,7 +1475,7 @@ class Orchestrator:
# Fetch metadata and build enriched prompt
self._connector_metadata_block =
skill.fetch_all_metadata(self.on_status)
enriched_prompt = skill.build_enriched_prompt(
- user_input, self.on_status, self.memory_store,
+ effective_request, self.on_status, self.memory_store,
)
# Phase 2: Config Generation (with enriched prompt from Skill)
@@ -1450,6 +1503,11 @@ class Orchestrator:
elif dryrun["valid"]:
dryrun_note = "\n\n**Dry-run:** PASSED"
+ # The requirement has been served, so the next message starts a
+ # new one rather than adding to this.
+ self.pending_question = None
+ self._original_request = None
+
# Add assistant message to history
self.conversation_history.append({
"role": "assistant",
@@ -1474,6 +1532,8 @@ class Orchestrator:
break
# Return best effort after max rounds
+ self.pending_question = None
+ self._original_request = None
self.conversation_history.append({
"role": "assistant",
"content": [{"text": f"Here is the generated
config:\n```hocon\n{config}\n```\n\n{explanation}"}],
diff --git a/seatunnel-cli/seatunnel_cli/cli.py
b/seatunnel-cli/seatunnel_cli/cli.py
index c81db9bf44..40fe715224 100644
--- a/seatunnel-cli/seatunnel_cli/cli.py
+++ b/seatunnel-cli/seatunnel_cli/cli.py
@@ -870,16 +870,9 @@ class SeaTunnelCLI:
self._handle_command(user_input)
continue
- # If previous turn was a clarification question, merge the
- # original request with the user's answer so the planner and
- # slot-checker see the full context.
- if self._pending_request:
- user_input = (
- f"{self._pending_request}\n\n"
- f"Additional details from user: {user_input}"
- )
- self._pending_request = None
-
+ # The Orchestrator merges a clarification reply with the request it
+ # answers; it owns `pending_question`, so doing it here as well
+ # would prepend the requirement twice.
# Process natural language request
self._process_request(user_input)
diff --git a/seatunnel-cli/tests/test_orchestrator_clarification.py
b/seatunnel-cli/tests/test_orchestrator_clarification.py
new file mode 100644
index 0000000000..918ca747fa
--- /dev/null
+++ b/seatunnel-cli/tests/test_orchestrator_clarification.py
@@ -0,0 +1,268 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""A clarifying question must not drop the requirement that preceded it.
+
+Only the planner reads the conversation history. Skill matching, slot filling
+and the enriched prompt are all built from the latest message, so before this
+was fixed a reply such as "use sensible defaults" became the whole request and
+the generator never saw what the user had actually asked for.
+"""
+
+import unittest
+from unittest import mock
+
+from seatunnel_cli.agents import Orchestrator
+
+
+REQUEST = (
+ "Sync the users table from MySQL at localhost:3306, database shop, "
+ "user root into the existing users_replica table in the same database."
+)
+REPLY = (
+ "Use sensible defaults for anything unspecified. Do not ask further "
+ "questions - generate the config now."
+)
+
+
+class EffectiveRequestTest(unittest.TestCase):
+ def setUp(self):
+ self.orchestrator = Orchestrator(client=mock.MagicMock())
+
+ def test_plain_message_is_the_request(self):
+ self.assertEqual(self.orchestrator._effective_request(REQUEST),
REQUEST)
+
+ def test_reply_to_a_question_is_added_not_substituted(self):
+ self.orchestrator._effective_request(REQUEST)
+ self.orchestrator.pending_question = "Which table should I write to?"
+
+ merged = self.orchestrator._effective_request(REPLY)
+
+ # The requirement has to survive; the reply is extra detail, and on its
+ # own it names no source, no sink and no table.
+ self.assertIn("users_replica", merged)
+ self.assertIn("localhost:3306", merged)
+ self.assertIn(REPLY, merged)
+
+ def test_several_rounds_keep_accumulating(self):
+ self.orchestrator._effective_request(REQUEST)
+ self.orchestrator.pending_question = "q1"
+ self.orchestrator._effective_request("The target table already
exists.")
+ self.orchestrator.pending_question = "q2"
+
+ merged = self.orchestrator._effective_request("Batch mode is fine.")
+
+ self.assertIn("users_replica", merged)
+ self.assertIn("The target table already exists.", merged)
+ self.assertIn("Batch mode is fine.", merged)
+
+ def test_the_requirement_is_not_prepended_twice(self):
+ """Only one component may do the merge.
+
+ The interactive loop in `cli.py` used to merge the reply with the
+ request before calling in, which together with this would have
+ prepended the requirement twice. That merge was removed in favour of
+ this one, because the Orchestrator owns `pending_question` and the
+ programmatic callers -- the benchmark among them -- never went through
+ `cli.py` at all.
+ """
+ self.orchestrator._effective_request(REQUEST)
+ self.orchestrator.pending_question = "Which table?"
+ merged = self.orchestrator._effective_request(REPLY)
+
+ self.assertEqual(merged.count("users_replica"), 1)
+
+ def test_a_new_message_after_delivery_starts_over(self):
+ self.orchestrator._effective_request(REQUEST)
+ # Delivering a config clears the exchange.
+ self.orchestrator.pending_question = None
+ self.orchestrator._original_request = None
+
+ fresh = self.orchestrator._effective_request("Print the orders table.")
+
+ self.assertEqual(fresh, "Print the orders table.")
+ self.assertNotIn("users_replica", fresh)
+
+
+class EnrichedPromptAfterClarificationTest(unittest.TestCase):
+ """End to end over `process_user_input`, which is where the loss happened.
+
+ The planner is stubbed because the point under test is what the *generator*
+ receives, not how the plan is produced.
+ """
+
+ def _orchestrator_with_stubs(self, seen):
+ orchestrator = Orchestrator(client=mock.MagicMock())
+
+ skill = mock.MagicMock()
+ skill.fill_and_check.return_value = []
+ skill.fetch_all_metadata.return_value = "## Connector Option Rules\n"
+
+ def build_enriched_prompt(request, on_status, memory_store=None):
+ seen.append(request)
+ return f"## User Request:\n{request}"
+
+ skill.build_enriched_prompt.side_effect = build_enriched_prompt
+ return orchestrator, skill
+
+ def test_generator_still_sees_the_original_request(self):
+ seen: list[str] = []
+ orchestrator, skill = self._orchestrator_with_stubs(seen)
+
+ with mock.patch.object(
+ orchestrator, "_run_planner",
+ side_effect=[
+ {"type": "question", "content": "Which database?",
+ "config": None, "explanation": None},
+ {"type": "plan", "content": "source: Jdbc\nsink: Jdbc\nmode:
BATCH"},
+ ],
+ ), mock.patch("seatunnel_cli.skills.SkillExecutor",
return_value=skill), \
+ mock.patch("seatunnel_cli.skills.SkillRouter.match",
return_value=[]), \
+ mock.patch("seatunnel_cli.skills.parse_structured_plan") as
parse_plan, \
+ mock.patch.object(
+ orchestrator, "_run_config_generator",
+ return_value={"config": "env {}\nsource {}\nsink {}",
+ "explanation": ""}), \
+ mock.patch.object(orchestrator, "_run_validator",
return_value="PASS"), \
+ mock.patch("seatunnel_cli.agents.dry_run_config",
+ return_value={"valid": True, "phase1_local": "VALID",
+ "phase2_check": "PASS", "phase3_api":
None,
+ "summary": ""}):
+ parse_plan.return_value =
mock.MagicMock(pipelines=[mock.MagicMock()])
+
+ first = orchestrator.process_user_input(REQUEST)
+ self.assertEqual(first["type"], "question")
+
+ second = orchestrator.process_user_input(REPLY)
+ self.assertEqual(second["type"], "config")
+
+ self.assertEqual(len(seen), 1, "the generator should run once, after
the reply")
+ prompt_request = seen[0]
+ self.assertIn("users_replica", prompt_request)
+ self.assertIn("localhost:3306", prompt_request)
+
+ def test_a_question_records_the_pending_state(self):
+ seen: list[str] = []
+ orchestrator, skill = self._orchestrator_with_stubs(seen)
+
+ with mock.patch.object(
+ orchestrator, "_run_planner",
+ return_value={"type": "question", "content": "Which database?",
+ "config": None, "explanation": None},
+ ):
+ orchestrator.process_user_input(REQUEST)
+
+ self.assertEqual(orchestrator.pending_question, "Which database?")
+ self.assertEqual(orchestrator._original_request, REQUEST)
+
+
+class PendingStateIsClearedTest(unittest.TestCase):
+ """Every exit that serves or abandons the requirement must clear it.
+
+ Without these, removing any single reset would leave a stale requirement to
+ be merged into the next, unrelated, request and no test would go red.
+ """
+
+ def _orchestrator(self):
+ orchestrator = Orchestrator(client=mock.MagicMock())
+ orchestrator.pending_question = "Which database?"
+ orchestrator._original_request = REQUEST
+ return orchestrator
+
+ def _assert_cleared(self, orchestrator):
+ self.assertIsNone(orchestrator.pending_question)
+ self.assertIsNone(orchestrator._original_request)
+
+ def _stub_skill(self):
+ skill = mock.MagicMock()
+ skill.fill_and_check.return_value = []
+ skill.fetch_all_metadata.return_value = "## Connector Option Rules\n"
+ skill.build_enriched_prompt.return_value = "## User Request:\nx"
+ return skill
+
+ def test_a_delivered_config_clears_it(self):
+ orchestrator = self._orchestrator()
+
+ with mock.patch.object(
+ orchestrator, "_run_planner",
+ return_value={"type": "plan",
+ "content": "source: Jdbc\nsink: Console\nmode:
BATCH"},
+ ), mock.patch("seatunnel_cli.skills.SkillExecutor",
+ return_value=self._stub_skill()), \
+ mock.patch("seatunnel_cli.skills.SkillRouter.match",
return_value=[]), \
+ mock.patch.object(
+ orchestrator, "_run_config_generator",
+ return_value={"config": "env {}\nsource {}\nsink {}",
+ "explanation": ""}), \
+ mock.patch.object(orchestrator, "_run_validator",
return_value="PASS"), \
+ mock.patch("seatunnel_cli.agents.dry_run_config",
+ return_value={"valid": True, "phase1_local": "VALID",
+ "phase2_check": "PASS", "phase3_api":
None,
+ "summary": ""}):
+ result = orchestrator.process_user_input("Batch is fine.")
+
+ self.assertEqual(result["type"], "config")
+ self._assert_cleared(orchestrator)
+
+ def test_a_best_effort_config_after_three_rounds_clears_it(self):
+ orchestrator = self._orchestrator()
+
+ with mock.patch.object(
+ orchestrator, "_run_planner",
+ return_value={"type": "plan",
+ "content": "source: Jdbc\nsink: Console\nmode:
BATCH"},
+ ), mock.patch("seatunnel_cli.skills.SkillExecutor",
+ return_value=self._stub_skill()), \
+ mock.patch("seatunnel_cli.skills.SkillRouter.match",
return_value=[]), \
+ mock.patch.object(
+ orchestrator, "_run_config_generator",
+ return_value={"config": "env {}\nsource {}\nsink {}",
+ "explanation": ""}), \
+ mock.patch.object(orchestrator, "_run_validator",
+ return_value="FAIL: still broken"), \
+ mock.patch.object(orchestrator, "_run_fix",
+ return_value={"config": "env {}\nsource
{}\nsink {}",
+ "explanation": ""}):
+ result = orchestrator.process_user_input("Batch is fine.")
+
+ self.assertEqual(result["type"], "config")
+ self._assert_cleared(orchestrator)
+
+ def test_a_chat_reply_clears_it(self):
+ orchestrator = self._orchestrator()
+
+ with mock.patch.object(
+ orchestrator, "_run_planner",
+ return_value={"type": "chat", "content": "SeaTunnel is an ETL
tool."},
+ ):
+ result = orchestrator.process_user_input("What is SeaTunnel?")
+
+ self.assertEqual(result["type"], "chat")
+ self._assert_cleared(orchestrator)
+
+ def test_resuming_a_session_clears_it(self):
+ orchestrator = self._orchestrator()
+
+ orchestrator.load_history([
+ {"role": "user", "content": [{"text": "an unrelated earlier
session"}]},
+ ])
+
+ self._assert_cleared(orchestrator)
+
+
+if __name__ == "__main__":
+ unittest.main()