From 281c13fff08e5bc2712a2e7c185cb605fc573cfc Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 02:32:45 +0000 Subject: [PATCH 01/27] SQLite: synchronous=NORMAL under WAL; period_cache WITHOUT ROWID New databases get the compact table; tools/period_cache_without_rowid.py converts an existing one. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- CLAUDE.md | 2 +- lib/store.py | 3 +- tests/test_period_cache_tool.py | 25 ++++++++++++++++ tests/test_store.py | 6 ++++ tools/period_cache_without_rowid.py | 45 +++++++++++++++++++++++++++++ 5 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 tests/test_period_cache_tool.py create mode 100644 tools/period_cache_without_rowid.py diff --git a/CLAUDE.md b/CLAUDE.md index 2079044..5993fe7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,7 +27,7 @@ MCP data tools and web search. User-facing docs are in `README.md`; this file is drafts), `commands.py` (/credits), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop). - `app.py` wiring/startup; `context.py` `Ctx` (settings, store, backend, registry, bot). - `lib/` holds only what the bot needs to run. Developer helpers live in `tools/`: `report.py` (`make prompt-report`), - `capture.py` (real MCP output for fixtures); the bot never imports them. + `capture.py` (real MCP output for fixtures), `period_cache_without_rowid.py` (one-off DB conversion); the bot never imports them. ## Request flow `handlers.on_message` logs the message -> `_trigger` (mention / reply-to-bot / DM / movers list) -> diff --git a/lib/store.py b/lib/store.py index b58850d..08e01d8 100644 --- a/lib/store.py +++ b/lib/store.py @@ -38,7 +38,7 @@ -- for that window. movers = the holdings that moved most, as JSON [[code, gain, gain_pct], ...]. CREATE TABLE IF NOT EXISTS period_cache ( portfolio_id INTEGER, start TEXT, end TEXT, pct REAL, gain REAL, currency TEXT, movers TEXT, - v INTEGER, PRIMARY KEY (portfolio_id, start, end)); + v INTEGER, PRIMARY KEY (portfolio_id, start, end)) WITHOUT ROWID; """ PERIOD_CACHE_VERSION = 1 # bump when the stored shape changes: rows of another version are dropped @@ -77,6 +77,7 @@ def __init__(self, path: str): self._lock = threading.RLock() self.bot_id: int | None = None # set once the bot has started (DMs are stored per bot) self._run("PRAGMA journal_mode=WAL") + self._run("PRAGMA synchronous=NORMAL") # with WAL: no fsync per commit; a power cut may lose the last few with self._lock: self._conn.executescript(SCHEMA) # Drop "NOTHING" digests sent by an earlier version, so they aren't fed to the model. diff --git a/tests/test_period_cache_tool.py b/tests/test_period_cache_tool.py new file mode 100644 index 0000000..4102099 --- /dev/null +++ b/tests/test_period_cache_tool.py @@ -0,0 +1,25 @@ +import sqlite3 + +from lib.store import Store +from tools.period_cache_without_rowid import convert + +OLD = """CREATE TABLE period_cache ( + portfolio_id INTEGER, start TEXT, end TEXT, pct REAL, gain REAL, currency TEXT, movers TEXT, + v INTEGER, PRIMARY KEY (portfolio_id, start, end))""" + + +def test_an_old_period_cache_keeps_its_rows_and_becomes_without_rowid(tmp_path): + path = str(tmp_path / "x.db") + db = sqlite3.connect(path) + db.execute(OLD) + db.executemany("INSERT INTO period_cache VALUES (?,?,?,?,?,?,?,?)", + [(1, f"2026-01-0{d}", f"2026-01-0{d}", 1.5, 10.0, "AUD", "[]", 1) for d in range(1, 4)]) + db.commit() + db.close() + assert convert(path) == 3 + assert convert(path) is None # already converted + db = sqlite3.connect(path) + assert "WITHOUT ROWID" in db.execute("SELECT sql FROM sqlite_master WHERE name = 'period_cache'").fetchone()[0] + assert db.execute("SELECT COUNT(*) FROM period_cache").fetchone()[0] == 3 + db.close() + assert Store(path).period_count(1, "2026-01-01", "2026-01-31") == 3 # the bot opens it fine diff --git a/tests/test_store.py b/tests/test_store.py index 397dde5..658b5f2 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -109,3 +109,9 @@ def test_period_flush_by_date_range_removes_every_window_that_overlaps_it(tmp_pa row("2023-01-01", "2023-12-31"), row("2025-01-07", "2025-01-13")]) assert s.period_flush(7, "2024-01-01", "2024-12-31") == 3 # the week spanning New Year both ways, and June assert sorted(s.period_get(7, "2000-01-01", "2100-01-01")) == [("2023-01-01", "2023-12-31"), ("2025-01-07", "2025-01-13")] + + +def test_store_uses_wal_with_normal_sync_and_a_without_rowid_period_cache(tmp_path): + st = Store(str(tmp_path / "x.db")) + assert st._run("PRAGMA synchronous") == [(1,)] + assert "WITHOUT ROWID" in st._run("SELECT sql FROM sqlite_master WHERE name = 'period_cache'")[0][0] diff --git a/tools/period_cache_without_rowid.py b/tools/period_cache_without_rowid.py new file mode 100644 index 0000000..201b928 --- /dev/null +++ b/tools/period_cache_without_rowid.py @@ -0,0 +1,45 @@ +"""One-off: convert an existing database's period_cache to WITHOUT ROWID (stop the bot first).""" + +from __future__ import annotations + +import sqlite3 +import sys + +# Keep in step with the period_cache table in lib/store.py. +CREATE = """CREATE TABLE period_cache ( + portfolio_id INTEGER, start TEXT, end TEXT, pct REAL, gain REAL, currency TEXT, movers TEXT, + v INTEGER, PRIMARY KEY (portfolio_id, start, end)) WITHOUT ROWID""" + + +def convert(path: str) -> int | None: + """Rebuild period_cache keeping every row. Returns the row count, or None if there was nothing to do.""" + db = sqlite3.connect(path, isolation_level=None) + try: + row = db.execute("SELECT sql FROM sqlite_master WHERE name = 'period_cache'").fetchone() + if row is None or "WITHOUT ROWID" in row[0]: + return None + db.execute("BEGIN") + db.execute("ALTER TABLE period_cache RENAME TO period_cache_old") + db.execute(CREATE) + db.execute("INSERT INTO period_cache SELECT * FROM period_cache_old") + count = db.execute("SELECT COUNT(*) FROM period_cache").fetchone()[0] + db.execute("DROP TABLE period_cache_old") + db.execute("COMMIT") + db.execute("VACUUM") + return count + finally: + db.close() + + +def main(argv: list[str]) -> int: + if len(argv) != 2: + print("usage: python -m tools.period_cache_without_rowid (stop the bot first)") + return 2 + count = convert(argv[1]) + print("period_cache is already WITHOUT ROWID (or absent); nothing to do" if count is None + else f"converted period_cache: {count} rows kept") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) From d2a8d716ba087bcf15c692ed5c2f0e8925a885bd Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 01:42:07 +0000 Subject: [PATCH 02/27] /holdingnews says when the provider is busy; Sharesight MCP from robdevops fork Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/features/holding_news.py | 9 ++++++--- lib/llm/base.py | 1 + lib/telegram/handlers.py | 3 +-- mcp_servers.json | 2 +- tests/test_features.py | 8 ++++++++ 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/lib/features/holding_news.py b/lib/features/holding_news.py index fe561e0..3d9831f 100644 --- a/lib/features/holding_news.py +++ b/lib/features/holding_news.py @@ -18,7 +18,7 @@ from .. import config from ..ask import ask from ..context import Ctx -from ..llm.base import is_busy +from ..llm.base import BUSY_REPLY, is_busy from ..llm.gate import Route from ..mcp.results import table_records from ..prompts import HOLDING_NEWS_PROMPT @@ -233,11 +233,14 @@ async def on_command(ctx: Ctx, msg: Message) -> None: return draft = Draft(ctx.bot, msg.chat_id) await draft.start() + busy: list[str] = [] try: - result = (await run_holding_news(ctx, only_username=username)).get(username) + result = (await run_holding_news(ctx, only_username=username, busy=busy)).get(username) finally: draft.stop() - if result: + if busy: + await asyncio.to_thread(save, await msg.reply_text(BUSY_REPLY)) + elif result: await send_holding_news(ctx, msg.chat_id, *result) else: await asyncio.to_thread( diff --git a/lib/llm/base.py b/lib/llm/base.py index 15f72da..6272d0a 100644 --- a/lib/llm/base.py +++ b/lib/llm/base.py @@ -9,6 +9,7 @@ from ..mcp.schema import ToolDef BUSY_STATUSES = (429, 503, 529) +BUSY_REPLY = "The AI provider is at capacity right now. Please try again in a few minutes." class Overloaded(RuntimeError): diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index ad131b6..b4f669e 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -17,7 +17,7 @@ from ..context import Ctx from ..features import dm_buttons, holding_news, movers, periods, post from ..history import compact, format_rows -from ..llm.base import is_busy +from ..llm.base import BUSY_REPLY, is_busy from ..llm.gate import Route, is_portfolio_question, route from ..msgtext import describe, sender_name from ..prompts import MOVERS_EXPLAIN, PRESET_PROMPT, chat_prompt, down_note @@ -30,7 +30,6 @@ log = logging.getLogger("bot") PORTFOLIO_ADMINS_ONLY = "Portfolio data is for admins only." -BUSY_REPLY = "The AI provider is at capacity right now. Please try again in a few minutes." FOLLOW_UP_WORDS = 12 # a message this short right after the bot's reply may be a follow-up to it FOLLOW_UP_SECONDS = 30 * 60 FOLLOW_UP_CHARS = 400 # of the bot's reply that the tool gate also reads diff --git a/mcp_servers.json b/mcp_servers.json index da29fe0..c4124a7 100644 --- a/mcp_servers.json +++ b/mcp_servers.json @@ -17,7 +17,7 @@ "sharesight": { "description": "Sharesight portfolios. Never ask the user for portfolio IDs or holdings: call list_portfolios first (it names every portfolio and its owner), then get_performance_report for the ones involved. A report's holdings list is exactly their current positions, with quantity, value and gains. When comparing portfolios or explaining a result, write 2-3 short paragraphs: each portfolio's return, then its tailwinds (biggest contributors, with numbers) and headwinds (biggest detractors), and what explains the gap.", "command": "npx", - "args": ["-y", "github:sryburn/sharesight-mcp"], + "args": ["-y", "github:robdevops/sharesight-mcp"], "env": { "SHARESIGHT_CLIENT_ID": "${SHARESIGHT_CLIENT_ID}", "SHARESIGHT_CLIENT_SECRET": "${SHARESIGHT_CLIENT_SECRET}" diff --git a/tests/test_features.py b/tests/test_features.py index d7b5120..dbf78dd 100644 --- a/tests/test_features.py +++ b/tests/test_features.py @@ -261,3 +261,11 @@ async def test_flushcache_command_is_admin_only_and_flushes_by_portfolio_and_per every = user_msg(ctx.bot, "/flushcache", user_id=5, mid=5) await commands.flush_cache(ctx, update_for(every), N(args=[])) assert "Flushed 4 cached periods (all portfolios)" in every.replies[0]["text"] + + +async def test_holdingnews_command_says_when_the_provider_is_busy(env, store): + ctx = holding_ctx(env, store, [Overloaded("at capacity")]) + m = user_msg(ctx.bot, "/holdingnews", chat_id=5, user_id=5) + m.from_user.username = "bob_llama" + await holding_news.on_command(ctx, m) + assert "at capacity" in m.replies[-1]["text"] and "No major news" not in m.replies[-1]["text"] From d886028dc8767cc1086f56288d5c42682c4c4210 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 22:18:42 +0000 Subject: [PATCH 03/27] Route market-session questions to tools and strip tool remarks from replies "at close" matched no gate keyword, so the model got the no-tools prompt and answered from the transcript, opening with "(no tool calls needed)". Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/llm/gate.py | 3 ++- lib/llm/runner.py | 11 ++++++++--- lib/prompts.py | 4 ++-- lib/textfmt.py | 3 +++ tests/test_runner.py | 7 +++++++ tests/test_textfmt.py | 7 +++++++ 6 files changed, 29 insertions(+), 6 deletions(-) diff --git a/lib/llm/gate.py b/lib/llm/gate.py index b80dbc0..bd0a12f 100644 --- a/lib/llm/gate.py +++ b/lib/llm/gate.py @@ -33,7 +33,8 @@ r"\b(performance|gains?|positions?|winners?|losers?|p&l|pnl|(?:out|under)perform\w*|ahead of|behind)\b", re.I) LIVE_WORDS = re.compile( r"\b(news|today|latest|yesterday|tonight|this week|right now|breaking|announce\w*|who won|" - r"score|weather|happening|headlines?|current\w*|updates?|recent\w*|just (now|in))\b", re.I) + r"score|weather|happening|headlines?|current\w*|updates?|recent\w*|just (now|in)|" + r"(?:at|the|into|after) (?:the )?(?:close|open)|pre-?market|after[- ]hours|closing|opening)\b", re.I) @dataclass(frozen=True) diff --git a/lib/llm/runner.py b/lib/llm/runner.py index cfefe87..5fbf958 100644 --- a/lib/llm/runner.py +++ b/lib/llm/runner.py @@ -13,7 +13,7 @@ from .. import config from ..mcp.server import Registry -from ..textfmt import TOOL_SYNTAX_RE +from ..textfmt import LEADING_TOOL_REMARK_RE, TOOL_SYNTAX_RE from .base import Answer, Backend, Call, Request, Step, Usage log = logging.getLogger("bot") @@ -28,6 +28,11 @@ TOO_MANY_SEARCHES = "Skipped: too many searches at once. Use what you already have." +def clean_text(text: str) -> str: + """The model's answer without printed tool-call syntax or a leading remark about tools.""" + return LEADING_TOOL_REMARK_RE.sub("", TOOL_SYNTAX_RE.sub("", text).strip()).strip() + + def call_args(arguments: str) -> dict: """A tool call's JSON arguments as a dict ({} if they aren't a JSON object).""" try: @@ -201,7 +206,7 @@ async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: boo log.info("%.1fs, %d rounds, %d tools: in %d (%.0f%% cached) out %d, $%.4f", time.monotonic() - t0, rounds, tool_calls, total.tokens_in, total.cached_pct, total.tokens_out, total.cost) - text = TOOL_SYNTAX_RE.sub("", step.text).strip() + text = clean_text(step.text) if not text and step.finish == "length" and tool_calls: # The model spent the whole cap thinking about the tool results. Keep them (answering # from memory without them invents things): ask again with more room and less thinking. @@ -211,7 +216,7 @@ async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: boo step = await _step(backend, conv, more, "none") total.add(step.usage) log.info("Recovery round: in %d out %d, %s", step.usage.tokens_in, step.usage.tokens_out, step.finish) - text = TOOL_SYNTAX_RE.sub("", step.text).strip() + text = clean_text(step.text) if not text: if step.finish == "length": raise RuntimeError("hit the token cap before writing an answer") diff --git a/lib/prompts.py b/lib/prompts.py index d9cb3da..9d66c6d 100644 --- a/lib/prompts.py +++ b/lib/prompts.py @@ -24,12 +24,12 @@ NO_SEARCH_NOTE = "You have no web search: say plainly what you can't check instead of guessing.\n" SIMPLE_NOTE = ("You have no live data tools for this message. If answering needs live prices, " f"news or portfolio data, reply with exactly {NEEDS_TOOLS} and nothing else. Never claim " - "to have looked something up.\n") + "to have looked something up, and never remark on tools or what you did or didn't use.\n") PARTIAL_NOTE = ("If you need a data source or search you don't have this time, reply with exactly " f"{NEEDS_TOOLS} and nothing else.\n") NO_TOOLS_NOTE = ("You have no web access and no tools this time. Answer from the chat history and " "what you know, say plainly what you can't check, and never output tool-call " - "syntax or JSON as text.\n") + "syntax or JSON as text, or any remark about tools.\n") def tools_note(servers: list[MCPServer]) -> str: diff --git a/lib/textfmt.py b/lib/textfmt.py index fa1cffa..2bd0fae 100644 --- a/lib/textfmt.py +++ b/lib/textfmt.py @@ -35,6 +35,9 @@ re.DOTALL | re.IGNORECASE, ) +# A model told it has no tools sometimes opens with a remark about it: "(no tool calls needed)". +LEADING_TOOL_REMARK_RE = re.compile(r"^\s*\((?:no|without)\s+(?:tool|search|lookup)[^)\n]*\)\s*", re.I) + def strip_disclaimer(text: str) -> str: """Models add these despite the prompt, and copy their own earlier ones from the transcript.""" diff --git a/tests/test_runner.py b/tests/test_runner.py index 17b3d38..0c70d02 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -242,6 +242,13 @@ def test_gate_live_question_gets_search_only(settings): assert r.search and not r.servers +@pytest.mark.parametrize("text", ["what happened at close?", "how did it go at the open", "any pre-market moves"]) +def test_gate_market_session_words_are_live(text, settings): + y, s = servers() + r = route(text, settings, registry(y, s)) + assert r.search and not r.simple + + def test_gate_full_when_forced_or_token_saver_off(settings, env): y, s = servers() assert route("hello", settings, registry(y, s), force_full=True).servers == [y, s] diff --git a/tests/test_textfmt.py b/tests/test_textfmt.py index e7d8ba4..3cf3b66 100644 --- a/tests/test_textfmt.py +++ b/tests/test_textfmt.py @@ -19,6 +19,13 @@ def test_tool_syntax_does_not_eat_the_answer(): assert t.TOOL_SYNTAX_RE.sub("", "").strip() == "" +def test_leading_tool_remark_is_stripped(): + from lib.llm.runner import clean_text + assert clean_text(" (no tool calls needed)Wasion (3393) -6%") == "Wasion (3393) -6%" + assert clean_text("(No search needed) Fine.") == "Fine." + assert clean_text("Wasion (no tool calls needed) fell") == "Wasion (no tool calls needed) fell" + + def test_tg_tag_regex_keeps_bare_lt(): s = lambda x: t.TG_TAG_RE.sub("", x) # noqa: E731 assert s("NVDA <5% ok x") == "NVDA <5% ok x" From 95875cffceb504f70cdc0f9e78b7e67ad19d4547 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 22:21:11 +0000 Subject: [PATCH 04/27] Make trading-session questions fetch current prices and news Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/llm/gate.py | 12 +++++++++--- lib/prompts.py | 5 ++++- lib/telegram/handlers.py | 11 +++++++---- tests/test_runner.py | 9 +++++++++ 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/lib/llm/gate.py b/lib/llm/gate.py index bd0a12f..6f05d10 100644 --- a/lib/llm/gate.py +++ b/lib/llm/gate.py @@ -33,8 +33,10 @@ r"\b(performance|gains?|positions?|winners?|losers?|p&l|pnl|(?:out|under)perform\w*|ahead of|behind)\b", re.I) LIVE_WORDS = re.compile( r"\b(news|today|latest|yesterday|tonight|this week|right now|breaking|announce\w*|who won|" - r"score|weather|happening|headlines?|current\w*|updates?|recent\w*|just (now|in)|" - r"(?:at|the|into|after) (?:the )?(?:close|open)|pre-?market|after[- ]hours|closing|opening)\b", re.I) + r"score|weather|happening|headlines?|current\w*|updates?|recent\w*|just (now|in))\b", re.I) +# Asking about a trading session ("what happened at close"): answers must be fetched, not recalled. +SESSION_WORDS = re.compile( + r"\b(?:(?:at|the|into|after) (?:the )?(?:close|open)|pre-?market|after[- ]hours|closing|opening)\b", re.I) @dataclass(frozen=True) @@ -66,6 +68,10 @@ def is_portfolio_question(text: str, st: Settings) -> bool: return bool(PORTFOLIO_WEAK.search(text)) and not _has_ticker(text) +def is_session_question(text: str) -> bool: + return bool(SESSION_WORDS.search(text)) + + def route(text: str, st: Settings, registry: Registry, *, force_full: bool = False) -> Route: """Tools for this message. With TOKEN_SAVER off, or `force_full` (a search/data request such as a movers list, a preset button or a retry after NEEDS_TOOLS), everything is offered.""" @@ -74,7 +80,7 @@ def route(text: str, st: Settings, registry: Registry, *, force_full: bool = Fal return Route(up, st.search) portfolio = is_portfolio_question(text, st) market = portfolio or bool(MARKET_WORDS.search(text)) or _has_ticker(text) - live = bool(LIVE_WORDS.search(text)) + live = is_session_question(text) or bool(LIVE_WORDS.search(text)) servers = [s for s in up if (portfolio if s.cfg.get("gate") == "portfolio" else market)] search = st.search and (market or live) simple = not (servers or search) diff --git a/lib/prompts.py b/lib/prompts.py index 9d66c6d..9ac8369 100644 --- a/lib/prompts.py +++ b/lib/prompts.py @@ -69,7 +69,7 @@ def down_note(down: list[MCPServer]) -> str: def chat_prompt(*, transcript: str, sender: str, private: bool, reply_quote: str | None, msg_id: int, now: str, down: str = "", saver: bool = True, middle: str = "", - fresh: bool = False) -> str: + fresh: bool = False, session: bool = False) -> str: """The user turn: transcript, the replied-to message, and the volatile tail (kept last so the cached prefix before it is unchanged).""" head = (f"Private one-to-one chat with {sender}, not the group. Transcript:\n{transcript}" @@ -89,6 +89,9 @@ def chat_prompt(*, transcript: str, sender: str, private: bool, reply_quote: str "for anything else asked, such as prices or a comparison with market data); don't reuse " "numbers or refusals from earlier replies. When comparing portfolios, or asked \"why\", " "write 2-3 short paragraphs on the tailwinds and headwinds of each portfolio, not a summary.") + if session: # a trading-session question: earlier replies and movers lists are stale by now + tail += (" Fetch current prices and current news with your tools before answering, and say why " + "each stock moved. Don't reuse numbers or reasons from earlier messages in the transcript.") return head + tail + down diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index b4f669e..69356b4 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -18,7 +18,7 @@ from ..features import dm_buttons, holding_news, movers, periods, post from ..history import compact, format_rows from ..llm.base import BUSY_REPLY, is_busy -from ..llm.gate import Route, is_portfolio_question, route +from ..llm.gate import Route, is_portfolio_question, is_session_question, route from ..msgtext import describe, sender_name from ..prompts import MOVERS_EXPLAIN, PRESET_PROMPT, chat_prompt, down_note from ..store import is_dm @@ -179,7 +179,8 @@ async def _gate_text(self, msg: Message, trig: Trigger) -> str: text += " " + compact(prev.text)[:FOLLOW_UP_CHARS] return text - async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: bool = False) -> str: + async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: bool = False, + session: bool = False) -> str: ctx, st = self.ctx, self.ctx.st rows = await asyncio.to_thread(ctx.store.history, msg.chat_id, st.history_limit) transcript = format_rows(rows, ctx.self_name, st.tz, line_max=st.history_line_max, @@ -187,7 +188,8 @@ async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: return chat_prompt( transcript=transcript, sender=sender_name(msg), private=trig.private, reply_quote=self._quote(trig), msg_id=msg.message_id, now=ctx.now(), - down=down_note(ctx.registry.down()), saver=st.token_saver, middle=middle, fresh=fresh) + down=down_note(ctx.registry.down()), saver=st.token_saver, middle=middle, fresh=fresh, + session=session) # -- answering --------------------------------------------------------------------- async def _answer(self, msg: Message, trig: Trigger) -> None: @@ -257,7 +259,8 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s if chosen.simple: # the model must not be told it has no tools chosen = Route([], False) fresh = is_portfolio_question(context, st) and any(s.cfg.get("gate") == "portfolio" for s in chosen.servers) - parts = [{"type": "text", "text": await self._chat_text(msg, trig, fresh=fresh)}] + session = is_session_question(context) and not chosen.simple + parts = [{"type": "text", "text": await self._chat_text(msg, trig, fresh=fresh, session=session)}] parts += await self._images(msg, trig) answer = await ask(ctx, parts, chosen, cache_key=f"chat-{chat_id}", on_text=on_text, chat_id=chat_id, extra_tools=extra, diff --git a/tests/test_runner.py b/tests/test_runner.py index 0c70d02..0db51a6 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -249,6 +249,15 @@ def test_gate_market_session_words_are_live(text, settings): assert r.search and not r.simple +def test_session_question_prompt_demands_fresh_prices_and_news(): + from lib.llm.gate import is_session_question + from lib.prompts import chat_prompt + assert is_session_question("what happened at close?") and not is_session_question("hello") + kw = dict(transcript="t", sender="A", private=False, reply_quote=None, msg_id=1, now="now") + assert "current news" in chat_prompt(**kw, session=True) + assert "current news" not in chat_prompt(**kw) + + def test_gate_full_when_forced_or_token_saver_off(settings, env): y, s = servers() assert route("hello", settings, registry(y, s), force_full=True).servers == [y, s] From 573ff6101dab2a019df659b98a9536e17003b765 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 22:22:26 +0000 Subject: [PATCH 05/27] Force a search for trading-session questions Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/telegram/handlers.py | 1 + tests/test_handlers.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index 69356b4..8ab1550 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -263,6 +263,7 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s parts = [{"type": "text", "text": await self._chat_text(msg, trig, fresh=fresh, session=session)}] parts += await self._images(msg, trig) answer = await ask(ctx, parts, chosen, + must_search=session and chosen.search, # a session recap must not be recalled cache_key=f"chat-{chat_id}", on_text=on_text, chat_id=chat_id, extra_tools=extra, admin=admin) return format_answer(answer.text, movers_bots=st.movers_bots, summary=trig.movers) diff --git a/tests/test_handlers.py b/tests/test_handlers.py index ba7f5a5..4c01302 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -90,6 +90,14 @@ async def test_partial_route_that_needs_more_is_asked_again_with_everything(env, assert "NEEDS_TOOLS" in backend.seen[0]["system"] # the partial route told the model how to ask +async def test_session_question_forces_a_search(env, store): + ctx, backend, h = make_ctx(env, store, [step("fresh recap", searches=1)], servers=[FakeMcp("yahoo")]) + m = user_msg(ctx.bot, "@stockbot what happened at close?") + await run(h, m) + assert backend.seen[0]["choice"] == "required" and backend.seen[0]["search"] is True + assert "current news" in backend.seen[0]["prompt"] + + async def test_market_question_offers_tools_up_front(env, store): mcp = FakeMcp("yahoo") ctx, backend, h = make_ctx(env, store, [step("ok")], servers=[mcp]) From 7b22d532d8f121a1319c29717ab7a1809ac8075a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 22:24:35 +0000 Subject: [PATCH 06/27] Keep market data tools beside the forced search for session questions Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/ask.py | 8 ++++---- lib/llm/gate.py | 5 +++-- lib/telegram/handlers.py | 1 + tests/test_handlers.py | 1 + 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/ask.py b/lib/ask.py index 9c46368..c5b1b5e 100644 --- a/lib/ask.py +++ b/lib/ask.py @@ -44,17 +44,17 @@ def _public(servers: list) -> list: async def ask(ctx: Ctx, parts: list[dict], route: Route, *, must_search: bool = False, cache_key: str | None = None, on_text=None, kind: str = "chat", - chat_id: int = 0, extra_tools=(), admin: bool = True) -> Answer: + chat_id: int = 0, extra_tools=(), admin: bool = True, keep_tools: bool = False) -> Answer: """Ask the configured provider and record the usage. `route` says which tools to offer; - must_search forces a search (and offers nothing else). A message the gate gave fewer tools + must_search forces a tool call first (and offers only search, unless keep_tools). A message the gate gave fewer tools than exist that gets back NEEDS_TOOLS is asked again with everything on. A non-admin never gets the portfolio tools, in either request.""" st = ctx.st if must_search: if not st.search: raise RuntimeError("search is disabled, but this request needs it") - route = Route([], True) - elif not admin: + route = dataclasses.replace(route, search=True) if keep_tools else Route([], True) + if not admin: route = dataclasses.replace(route, servers=_public(route.servers)) saver = st.token_saver req = Request( diff --git a/lib/llm/gate.py b/lib/llm/gate.py index 6f05d10..43cf316 100644 --- a/lib/llm/gate.py +++ b/lib/llm/gate.py @@ -79,8 +79,9 @@ def route(text: str, st: Settings, registry: Registry, *, force_full: bool = Fal if force_full or not st.token_saver: return Route(up, st.search) portfolio = is_portfolio_question(text, st) - market = portfolio or bool(MARKET_WORDS.search(text)) or _has_ticker(text) - live = is_session_question(text) or bool(LIVE_WORDS.search(text)) + session = is_session_question(text) + market = portfolio or session or bool(MARKET_WORDS.search(text)) or _has_ticker(text) + live = session or bool(LIVE_WORDS.search(text)) servers = [s for s in up if (portfolio if s.cfg.get("gate") == "portfolio" else market)] search = st.search and (market or live) simple = not (servers or search) diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index 8ab1550..3b97a0a 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -264,6 +264,7 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s parts += await self._images(msg, trig) answer = await ask(ctx, parts, chosen, must_search=session and chosen.search, # a session recap must not be recalled + keep_tools=True, cache_key=f"chat-{chat_id}", on_text=on_text, chat_id=chat_id, extra_tools=extra, admin=admin) return format_answer(answer.text, movers_bots=st.movers_bots, summary=trig.movers) diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 4c01302..21a9ba6 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -95,6 +95,7 @@ async def test_session_question_forces_a_search(env, store): m = user_msg(ctx.bot, "@stockbot what happened at close?") await run(h, m) assert backend.seen[0]["choice"] == "required" and backend.seen[0]["search"] is True + assert backend.seen[0]["tools"] == ["yahoo__get_quote"] # quotes stay available beside the search assert "current news" in backend.seen[0]["prompt"] From 54d6f4c3f6f6e70ae38021f51c22bb6205ebf042 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 22:26:02 +0000 Subject: [PATCH 07/27] Revert "Keep market data tools beside the forced search for session questions" This reverts commit 7b22d532d8f121a1319c29717ab7a1809ac8075a. --- lib/ask.py | 8 ++++---- lib/llm/gate.py | 5 ++--- lib/telegram/handlers.py | 1 - tests/test_handlers.py | 1 - 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/ask.py b/lib/ask.py index c5b1b5e..9c46368 100644 --- a/lib/ask.py +++ b/lib/ask.py @@ -44,17 +44,17 @@ def _public(servers: list) -> list: async def ask(ctx: Ctx, parts: list[dict], route: Route, *, must_search: bool = False, cache_key: str | None = None, on_text=None, kind: str = "chat", - chat_id: int = 0, extra_tools=(), admin: bool = True, keep_tools: bool = False) -> Answer: + chat_id: int = 0, extra_tools=(), admin: bool = True) -> Answer: """Ask the configured provider and record the usage. `route` says which tools to offer; - must_search forces a tool call first (and offers only search, unless keep_tools). A message the gate gave fewer tools + must_search forces a search (and offers nothing else). A message the gate gave fewer tools than exist that gets back NEEDS_TOOLS is asked again with everything on. A non-admin never gets the portfolio tools, in either request.""" st = ctx.st if must_search: if not st.search: raise RuntimeError("search is disabled, but this request needs it") - route = dataclasses.replace(route, search=True) if keep_tools else Route([], True) - if not admin: + route = Route([], True) + elif not admin: route = dataclasses.replace(route, servers=_public(route.servers)) saver = st.token_saver req = Request( diff --git a/lib/llm/gate.py b/lib/llm/gate.py index 43cf316..6f05d10 100644 --- a/lib/llm/gate.py +++ b/lib/llm/gate.py @@ -79,9 +79,8 @@ def route(text: str, st: Settings, registry: Registry, *, force_full: bool = Fal if force_full or not st.token_saver: return Route(up, st.search) portfolio = is_portfolio_question(text, st) - session = is_session_question(text) - market = portfolio or session or bool(MARKET_WORDS.search(text)) or _has_ticker(text) - live = session or bool(LIVE_WORDS.search(text)) + market = portfolio or bool(MARKET_WORDS.search(text)) or _has_ticker(text) + live = is_session_question(text) or bool(LIVE_WORDS.search(text)) servers = [s for s in up if (portfolio if s.cfg.get("gate") == "portfolio" else market)] search = st.search and (market or live) simple = not (servers or search) diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index 3b97a0a..8ab1550 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -264,7 +264,6 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s parts += await self._images(msg, trig) answer = await ask(ctx, parts, chosen, must_search=session and chosen.search, # a session recap must not be recalled - keep_tools=True, cache_key=f"chat-{chat_id}", on_text=on_text, chat_id=chat_id, extra_tools=extra, admin=admin) return format_answer(answer.text, movers_bots=st.movers_bots, summary=trig.movers) diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 21a9ba6..4c01302 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -95,7 +95,6 @@ async def test_session_question_forces_a_search(env, store): m = user_msg(ctx.bot, "@stockbot what happened at close?") await run(h, m) assert backend.seen[0]["choice"] == "required" and backend.seen[0]["search"] is True - assert backend.seen[0]["tools"] == ["yahoo__get_quote"] # quotes stay available beside the search assert "current news" in backend.seen[0]["prompt"] From 34e7f0e420b43c175c086817acae63f384af7d4d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 22:26:24 +0000 Subject: [PATCH 08/27] Session questions: ask for current news only, not prices Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/prompts.py | 6 +++--- tests/test_handlers.py | 2 +- tests/test_runner.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/prompts.py b/lib/prompts.py index 9ac8369..d36f664 100644 --- a/lib/prompts.py +++ b/lib/prompts.py @@ -89,9 +89,9 @@ def chat_prompt(*, transcript: str, sender: str, private: bool, reply_quote: str "for anything else asked, such as prices or a comparison with market data); don't reuse " "numbers or refusals from earlier replies. When comparing portfolios, or asked \"why\", " "write 2-3 short paragraphs on the tailwinds and headwinds of each portfolio, not a summary.") - if session: # a trading-session question: earlier replies and movers lists are stale by now - tail += (" Fetch current prices and current news with your tools before answering, and say why " - "each stock moved. Don't reuse numbers or reasons from earlier messages in the transcript.") + if session: # a trading-session question: news and reasons in earlier replies are stale by now + tail += (" Search for current news before answering, and say why each stock moved. Don't reuse " + "news or reasons from earlier messages in the transcript.") return head + tail + down diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 4c01302..e31610c 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -95,7 +95,7 @@ async def test_session_question_forces_a_search(env, store): m = user_msg(ctx.bot, "@stockbot what happened at close?") await run(h, m) assert backend.seen[0]["choice"] == "required" and backend.seen[0]["search"] is True - assert "current news" in backend.seen[0]["prompt"] + assert "Search for current news" in backend.seen[0]["prompt"] async def test_market_question_offers_tools_up_front(env, store): diff --git a/tests/test_runner.py b/tests/test_runner.py index 0db51a6..cb24d2d 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -254,8 +254,8 @@ def test_session_question_prompt_demands_fresh_prices_and_news(): from lib.prompts import chat_prompt assert is_session_question("what happened at close?") and not is_session_question("hello") kw = dict(transcript="t", sender="A", private=False, reply_quote=None, msg_id=1, now="now") - assert "current news" in chat_prompt(**kw, session=True) - assert "current news" not in chat_prompt(**kw) + assert "Search for current news" in chat_prompt(**kw, session=True) + assert "Search for current news" not in chat_prompt(**kw) def test_gate_full_when_forced_or_token_saver_off(settings, env): From dd26bf84cdf9eee7e6bd43027aff963ddca15ffb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 22:27:46 +0000 Subject: [PATCH 09/27] Don't strip tool remarks from replies, so they stay visible Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/llm/runner.py | 11 +++-------- lib/textfmt.py | 3 --- tests/test_textfmt.py | 7 ------- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/lib/llm/runner.py b/lib/llm/runner.py index 5fbf958..cfefe87 100644 --- a/lib/llm/runner.py +++ b/lib/llm/runner.py @@ -13,7 +13,7 @@ from .. import config from ..mcp.server import Registry -from ..textfmt import LEADING_TOOL_REMARK_RE, TOOL_SYNTAX_RE +from ..textfmt import TOOL_SYNTAX_RE from .base import Answer, Backend, Call, Request, Step, Usage log = logging.getLogger("bot") @@ -28,11 +28,6 @@ TOO_MANY_SEARCHES = "Skipped: too many searches at once. Use what you already have." -def clean_text(text: str) -> str: - """The model's answer without printed tool-call syntax or a leading remark about tools.""" - return LEADING_TOOL_REMARK_RE.sub("", TOOL_SYNTAX_RE.sub("", text).strip()).strip() - - def call_args(arguments: str) -> dict: """A tool call's JSON arguments as a dict ({} if they aren't a JSON object).""" try: @@ -206,7 +201,7 @@ async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: boo log.info("%.1fs, %d rounds, %d tools: in %d (%.0f%% cached) out %d, $%.4f", time.monotonic() - t0, rounds, tool_calls, total.tokens_in, total.cached_pct, total.tokens_out, total.cost) - text = clean_text(step.text) + text = TOOL_SYNTAX_RE.sub("", step.text).strip() if not text and step.finish == "length" and tool_calls: # The model spent the whole cap thinking about the tool results. Keep them (answering # from memory without them invents things): ask again with more room and less thinking. @@ -216,7 +211,7 @@ async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: boo step = await _step(backend, conv, more, "none") total.add(step.usage) log.info("Recovery round: in %d out %d, %s", step.usage.tokens_in, step.usage.tokens_out, step.finish) - text = clean_text(step.text) + text = TOOL_SYNTAX_RE.sub("", step.text).strip() if not text: if step.finish == "length": raise RuntimeError("hit the token cap before writing an answer") diff --git a/lib/textfmt.py b/lib/textfmt.py index 2bd0fae..fa1cffa 100644 --- a/lib/textfmt.py +++ b/lib/textfmt.py @@ -35,9 +35,6 @@ re.DOTALL | re.IGNORECASE, ) -# A model told it has no tools sometimes opens with a remark about it: "(no tool calls needed)". -LEADING_TOOL_REMARK_RE = re.compile(r"^\s*\((?:no|without)\s+(?:tool|search|lookup)[^)\n]*\)\s*", re.I) - def strip_disclaimer(text: str) -> str: """Models add these despite the prompt, and copy their own earlier ones from the transcript.""" diff --git a/tests/test_textfmt.py b/tests/test_textfmt.py index 3cf3b66..e7d8ba4 100644 --- a/tests/test_textfmt.py +++ b/tests/test_textfmt.py @@ -19,13 +19,6 @@ def test_tool_syntax_does_not_eat_the_answer(): assert t.TOOL_SYNTAX_RE.sub("", "").strip() == "" -def test_leading_tool_remark_is_stripped(): - from lib.llm.runner import clean_text - assert clean_text(" (no tool calls needed)Wasion (3393) -6%") == "Wasion (3393) -6%" - assert clean_text("(No search needed) Fine.") == "Fine." - assert clean_text("Wasion (no tool calls needed) fell") == "Wasion (no tool calls needed) fell" - - def test_tg_tag_regex_keeps_bare_lt(): s = lambda x: t.TG_TAG_RE.sub("", x) # noqa: E731 assert s("NVDA <5% ok x") == "NVDA <5% ok x" From 9b01102474eb337f2062e7deb8e658a876152c2b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 22:31:12 +0000 Subject: [PATCH 10/27] Rename session prompt test Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- tests/test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_runner.py b/tests/test_runner.py index cb24d2d..5c6e16d 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -249,7 +249,7 @@ def test_gate_market_session_words_are_live(text, settings): assert r.search and not r.simple -def test_session_question_prompt_demands_fresh_prices_and_news(): +def test_session_question_prompt_demands_current_news(): from lib.llm.gate import is_session_question from lib.prompts import chat_prompt assert is_session_question("what happened at close?") and not is_session_question("hello") From 8e880086083bcc66900605231b7f6e60e8765c20 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 22:32:17 +0000 Subject: [PATCH 11/27] Don't tell the model to avoid remarks about tools Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/prompts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/prompts.py b/lib/prompts.py index d36f664..dcfda7a 100644 --- a/lib/prompts.py +++ b/lib/prompts.py @@ -24,12 +24,12 @@ NO_SEARCH_NOTE = "You have no web search: say plainly what you can't check instead of guessing.\n" SIMPLE_NOTE = ("You have no live data tools for this message. If answering needs live prices, " f"news or portfolio data, reply with exactly {NEEDS_TOOLS} and nothing else. Never claim " - "to have looked something up, and never remark on tools or what you did or didn't use.\n") + "to have looked something up.\n") PARTIAL_NOTE = ("If you need a data source or search you don't have this time, reply with exactly " f"{NEEDS_TOOLS} and nothing else.\n") NO_TOOLS_NOTE = ("You have no web access and no tools this time. Answer from the chat history and " "what you know, say plainly what you can't check, and never output tool-call " - "syntax or JSON as text, or any remark about tools.\n") + "syntax or JSON as text.\n") def tools_note(servers: list[MCPServer]) -> str: From 08f8695e6da1ed9821331d1f44cf4411a0cd1ed1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 00:02:30 +0000 Subject: [PATCH 12/27] xAI: keep only the last text part of a message, and log multi-part messages Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/llm/xai.py | 15 +++++++++++---- tests/test_llm_backends.py | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/lib/llm/xai.py b/lib/llm/xai.py index 7e5c6b9..51cf2c6 100644 --- a/lib/llm/xai.py +++ b/lib/llm/xai.py @@ -34,14 +34,19 @@ def _as_input(item) -> dict: def answer_text(resp) -> str: - """The response's last message. `output_text` joins every message, and with server-side search - the model can write a draft, search, then write the answer, all in one response.""" + """The response's last message, and in it the last text part. `output_text` joins everything, + and with server-side search the model can write a draft, search, then write the answer, as + separate messages or as separate parts of one.""" messages = [i for i in resp.output if i.type == "message"] - texts = ["".join(getattr(c, "text", "") or "" for c in m.content if c.type == "output_text") - for m in messages] + parts = [[getattr(c, "text", "") or "" for c in m.content if c.type == "output_text"] for m in messages] + texts = [next((t for t in reversed(p) if t.strip()), "") for p in parts] if len([t for t in texts if t.strip()]) > 1: log.info("xAI response had %d messages; kept the last (items: %s)", len(texts), " ".join(i.type for i in resp.output)) + for p in parts: + if len([t for t in p if t.strip()]) > 1: + log.info("xAI message had %d text parts (%s chars); kept the last", + len(p), "+".join(str(len(t)) for t in p)) return next((t for t in reversed(texts) if t.strip()), resp.output_text or "").strip() @@ -88,6 +93,8 @@ async def _create(self, req: Request, kw: dict, **inputs): async for event in stream: if event.type == "response.output_item.added" and event.item.type == "message": text = "" # a new message replaces the draft before it + elif event.type == "response.content_part.added" and event.part.type == "output_text": + text = "" # a new text part replaces the draft before it elif event.type == "response.output_text.delta": text += event.delta if req.on_text: diff --git a/tests/test_llm_backends.py b/tests/test_llm_backends.py index 79a25b2..b5c48a4 100644 --- a/tests/test_llm_backends.py +++ b/tests/test_llm_backends.py @@ -247,6 +247,25 @@ async def test_xai_keeps_only_the_last_message_when_it_drafts_before_searching() assert seen == ["draft", "answer"] +async def test_xai_keeps_only_the_last_text_part_of_one_message(): + from types import SimpleNamespace as N + st = config.load({"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "k"}) + message = N(type="message", content=[N(type="output_text", text="draft. One blank line after each."), + N(type="output_text", text="answer")]) + events = [N(type="response.output_item.added", item=message), + N(type="response.content_part.added", part=message.content[0]), + N(type="response.output_text.delta", delta="draft. One blank line after each."), + N(type="response.content_part.added", part=message.content[1]), + N(type="response.output_text.delta", delta="answer"), + xai_final([message], text="draft. One blank line after each.answer")] + seen = [] + b = XaiBackend(st, FakeXaiClient(events)) + r = req(on_text=seen.append) + step = await b.step(b.start(r), r, tool_choice=None) + assert step.text == "answer" + assert seen == ["draft. One blank line after each.", "answer"] + + async def test_xai_failed_response_at_capacity_is_overloaded_other_failures_are_not(): from types import SimpleNamespace as N st = config.load({"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "k"}) From b6fffa023337479e18310d39e9a821ec8901cfa7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 00:18:38 +0000 Subject: [PATCH 13/27] Revert session changes (built on stale feedback) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/llm/gate.py | 9 +-------- lib/llm/xai.py | 15 ++++----------- lib/prompts.py | 5 +---- lib/telegram/handlers.py | 12 ++++-------- tests/test_handlers.py | 8 -------- tests/test_llm_backends.py | 19 ------------------- tests/test_runner.py | 16 ---------------- 7 files changed, 10 insertions(+), 74 deletions(-) diff --git a/lib/llm/gate.py b/lib/llm/gate.py index 6f05d10..b80dbc0 100644 --- a/lib/llm/gate.py +++ b/lib/llm/gate.py @@ -34,9 +34,6 @@ LIVE_WORDS = re.compile( r"\b(news|today|latest|yesterday|tonight|this week|right now|breaking|announce\w*|who won|" r"score|weather|happening|headlines?|current\w*|updates?|recent\w*|just (now|in))\b", re.I) -# Asking about a trading session ("what happened at close"): answers must be fetched, not recalled. -SESSION_WORDS = re.compile( - r"\b(?:(?:at|the|into|after) (?:the )?(?:close|open)|pre-?market|after[- ]hours|closing|opening)\b", re.I) @dataclass(frozen=True) @@ -68,10 +65,6 @@ def is_portfolio_question(text: str, st: Settings) -> bool: return bool(PORTFOLIO_WEAK.search(text)) and not _has_ticker(text) -def is_session_question(text: str) -> bool: - return bool(SESSION_WORDS.search(text)) - - def route(text: str, st: Settings, registry: Registry, *, force_full: bool = False) -> Route: """Tools for this message. With TOKEN_SAVER off, or `force_full` (a search/data request such as a movers list, a preset button or a retry after NEEDS_TOOLS), everything is offered.""" @@ -80,7 +73,7 @@ def route(text: str, st: Settings, registry: Registry, *, force_full: bool = Fal return Route(up, st.search) portfolio = is_portfolio_question(text, st) market = portfolio or bool(MARKET_WORDS.search(text)) or _has_ticker(text) - live = is_session_question(text) or bool(LIVE_WORDS.search(text)) + live = bool(LIVE_WORDS.search(text)) servers = [s for s in up if (portfolio if s.cfg.get("gate") == "portfolio" else market)] search = st.search and (market or live) simple = not (servers or search) diff --git a/lib/llm/xai.py b/lib/llm/xai.py index 51cf2c6..7e5c6b9 100644 --- a/lib/llm/xai.py +++ b/lib/llm/xai.py @@ -34,19 +34,14 @@ def _as_input(item) -> dict: def answer_text(resp) -> str: - """The response's last message, and in it the last text part. `output_text` joins everything, - and with server-side search the model can write a draft, search, then write the answer, as - separate messages or as separate parts of one.""" + """The response's last message. `output_text` joins every message, and with server-side search + the model can write a draft, search, then write the answer, all in one response.""" messages = [i for i in resp.output if i.type == "message"] - parts = [[getattr(c, "text", "") or "" for c in m.content if c.type == "output_text"] for m in messages] - texts = [next((t for t in reversed(p) if t.strip()), "") for p in parts] + texts = ["".join(getattr(c, "text", "") or "" for c in m.content if c.type == "output_text") + for m in messages] if len([t for t in texts if t.strip()]) > 1: log.info("xAI response had %d messages; kept the last (items: %s)", len(texts), " ".join(i.type for i in resp.output)) - for p in parts: - if len([t for t in p if t.strip()]) > 1: - log.info("xAI message had %d text parts (%s chars); kept the last", - len(p), "+".join(str(len(t)) for t in p)) return next((t for t in reversed(texts) if t.strip()), resp.output_text or "").strip() @@ -93,8 +88,6 @@ async def _create(self, req: Request, kw: dict, **inputs): async for event in stream: if event.type == "response.output_item.added" and event.item.type == "message": text = "" # a new message replaces the draft before it - elif event.type == "response.content_part.added" and event.part.type == "output_text": - text = "" # a new text part replaces the draft before it elif event.type == "response.output_text.delta": text += event.delta if req.on_text: diff --git a/lib/prompts.py b/lib/prompts.py index dcfda7a..d9cb3da 100644 --- a/lib/prompts.py +++ b/lib/prompts.py @@ -69,7 +69,7 @@ def down_note(down: list[MCPServer]) -> str: def chat_prompt(*, transcript: str, sender: str, private: bool, reply_quote: str | None, msg_id: int, now: str, down: str = "", saver: bool = True, middle: str = "", - fresh: bool = False, session: bool = False) -> str: + fresh: bool = False) -> str: """The user turn: transcript, the replied-to message, and the volatile tail (kept last so the cached prefix before it is unchanged).""" head = (f"Private one-to-one chat with {sender}, not the group. Transcript:\n{transcript}" @@ -89,9 +89,6 @@ def chat_prompt(*, transcript: str, sender: str, private: bool, reply_quote: str "for anything else asked, such as prices or a comparison with market data); don't reuse " "numbers or refusals from earlier replies. When comparing portfolios, or asked \"why\", " "write 2-3 short paragraphs on the tailwinds and headwinds of each portfolio, not a summary.") - if session: # a trading-session question: news and reasons in earlier replies are stale by now - tail += (" Search for current news before answering, and say why each stock moved. Don't reuse " - "news or reasons from earlier messages in the transcript.") return head + tail + down diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index 8ab1550..b4f669e 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -18,7 +18,7 @@ from ..features import dm_buttons, holding_news, movers, periods, post from ..history import compact, format_rows from ..llm.base import BUSY_REPLY, is_busy -from ..llm.gate import Route, is_portfolio_question, is_session_question, route +from ..llm.gate import Route, is_portfolio_question, route from ..msgtext import describe, sender_name from ..prompts import MOVERS_EXPLAIN, PRESET_PROMPT, chat_prompt, down_note from ..store import is_dm @@ -179,8 +179,7 @@ async def _gate_text(self, msg: Message, trig: Trigger) -> str: text += " " + compact(prev.text)[:FOLLOW_UP_CHARS] return text - async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: bool = False, - session: bool = False) -> str: + async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: bool = False) -> str: ctx, st = self.ctx, self.ctx.st rows = await asyncio.to_thread(ctx.store.history, msg.chat_id, st.history_limit) transcript = format_rows(rows, ctx.self_name, st.tz, line_max=st.history_line_max, @@ -188,8 +187,7 @@ async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: return chat_prompt( transcript=transcript, sender=sender_name(msg), private=trig.private, reply_quote=self._quote(trig), msg_id=msg.message_id, now=ctx.now(), - down=down_note(ctx.registry.down()), saver=st.token_saver, middle=middle, fresh=fresh, - session=session) + down=down_note(ctx.registry.down()), saver=st.token_saver, middle=middle, fresh=fresh) # -- answering --------------------------------------------------------------------- async def _answer(self, msg: Message, trig: Trigger) -> None: @@ -259,11 +257,9 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s if chosen.simple: # the model must not be told it has no tools chosen = Route([], False) fresh = is_portfolio_question(context, st) and any(s.cfg.get("gate") == "portfolio" for s in chosen.servers) - session = is_session_question(context) and not chosen.simple - parts = [{"type": "text", "text": await self._chat_text(msg, trig, fresh=fresh, session=session)}] + parts = [{"type": "text", "text": await self._chat_text(msg, trig, fresh=fresh)}] parts += await self._images(msg, trig) answer = await ask(ctx, parts, chosen, - must_search=session and chosen.search, # a session recap must not be recalled cache_key=f"chat-{chat_id}", on_text=on_text, chat_id=chat_id, extra_tools=extra, admin=admin) return format_answer(answer.text, movers_bots=st.movers_bots, summary=trig.movers) diff --git a/tests/test_handlers.py b/tests/test_handlers.py index e31610c..ba7f5a5 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -90,14 +90,6 @@ async def test_partial_route_that_needs_more_is_asked_again_with_everything(env, assert "NEEDS_TOOLS" in backend.seen[0]["system"] # the partial route told the model how to ask -async def test_session_question_forces_a_search(env, store): - ctx, backend, h = make_ctx(env, store, [step("fresh recap", searches=1)], servers=[FakeMcp("yahoo")]) - m = user_msg(ctx.bot, "@stockbot what happened at close?") - await run(h, m) - assert backend.seen[0]["choice"] == "required" and backend.seen[0]["search"] is True - assert "Search for current news" in backend.seen[0]["prompt"] - - async def test_market_question_offers_tools_up_front(env, store): mcp = FakeMcp("yahoo") ctx, backend, h = make_ctx(env, store, [step("ok")], servers=[mcp]) diff --git a/tests/test_llm_backends.py b/tests/test_llm_backends.py index b5c48a4..79a25b2 100644 --- a/tests/test_llm_backends.py +++ b/tests/test_llm_backends.py @@ -247,25 +247,6 @@ async def test_xai_keeps_only_the_last_message_when_it_drafts_before_searching() assert seen == ["draft", "answer"] -async def test_xai_keeps_only_the_last_text_part_of_one_message(): - from types import SimpleNamespace as N - st = config.load({"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "k"}) - message = N(type="message", content=[N(type="output_text", text="draft. One blank line after each."), - N(type="output_text", text="answer")]) - events = [N(type="response.output_item.added", item=message), - N(type="response.content_part.added", part=message.content[0]), - N(type="response.output_text.delta", delta="draft. One blank line after each."), - N(type="response.content_part.added", part=message.content[1]), - N(type="response.output_text.delta", delta="answer"), - xai_final([message], text="draft. One blank line after each.answer")] - seen = [] - b = XaiBackend(st, FakeXaiClient(events)) - r = req(on_text=seen.append) - step = await b.step(b.start(r), r, tool_choice=None) - assert step.text == "answer" - assert seen == ["draft. One blank line after each.", "answer"] - - async def test_xai_failed_response_at_capacity_is_overloaded_other_failures_are_not(): from types import SimpleNamespace as N st = config.load({"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "k"}) diff --git a/tests/test_runner.py b/tests/test_runner.py index 5c6e16d..17b3d38 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -242,22 +242,6 @@ def test_gate_live_question_gets_search_only(settings): assert r.search and not r.servers -@pytest.mark.parametrize("text", ["what happened at close?", "how did it go at the open", "any pre-market moves"]) -def test_gate_market_session_words_are_live(text, settings): - y, s = servers() - r = route(text, settings, registry(y, s)) - assert r.search and not r.simple - - -def test_session_question_prompt_demands_current_news(): - from lib.llm.gate import is_session_question - from lib.prompts import chat_prompt - assert is_session_question("what happened at close?") and not is_session_question("hello") - kw = dict(transcript="t", sender="A", private=False, reply_quote=None, msg_id=1, now="now") - assert "Search for current news" in chat_prompt(**kw, session=True) - assert "Search for current news" not in chat_prompt(**kw) - - def test_gate_full_when_forced_or_token_saver_off(settings, env): y, s = servers() assert route("hello", settings, registry(y, s), force_full=True).servers == [y, s] From 12fe926abf09be431a0cb031ca9dda60852d99ea Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 00:19:01 +0000 Subject: [PATCH 14/27] xAI: log a message that has several text parts Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/llm/xai.py | 7 +++++-- tests/test_llm_backends.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/llm/xai.py b/lib/llm/xai.py index 7e5c6b9..477514d 100644 --- a/lib/llm/xai.py +++ b/lib/llm/xai.py @@ -37,8 +37,11 @@ def answer_text(resp) -> str: """The response's last message. `output_text` joins every message, and with server-side search the model can write a draft, search, then write the answer, all in one response.""" messages = [i for i in resp.output if i.type == "message"] - texts = ["".join(getattr(c, "text", "") or "" for c in m.content if c.type == "output_text") - for m in messages] + parts = [[getattr(c, "text", "") or "" for c in m.content if c.type == "output_text"] for m in messages] + texts = ["".join(p) for p in parts] + for p in parts: + if len([t for t in p if t.strip()]) > 1: # not handled yet: this line tells us if it happens + log.info("xAI message had %d text parts (%s chars)", len(p), "+".join(str(len(t)) for t in p)) if len([t for t in texts if t.strip()]) > 1: log.info("xAI response had %d messages; kept the last (items: %s)", len(texts), " ".join(i.type for i in resp.output)) diff --git a/tests/test_llm_backends.py b/tests/test_llm_backends.py index 79a25b2..f205dd5 100644 --- a/tests/test_llm_backends.py +++ b/tests/test_llm_backends.py @@ -247,6 +247,17 @@ async def test_xai_keeps_only_the_last_message_when_it_drafts_before_searching() assert seen == ["draft", "answer"] +async def test_xai_logs_a_message_with_several_text_parts_and_still_joins_them(caplog): + from types import SimpleNamespace as N + + from lib.llm.xai import answer_text + caplog.set_level("INFO", logger="bot") + message = N(type="message", content=[N(type="output_text", text="one"), N(type="output_text", text="two")]) + assert answer_text(N(output=[message], output_text="onetwo")) == "onetwo" + assert [r.getMessage() for r in caplog.records if "text parts" in r.getMessage()] == [ + "xAI message had 2 text parts (3+3 chars)"] + + async def test_xai_failed_response_at_capacity_is_overloaded_other_failures_are_not(): from types import SimpleNamespace as N st = config.load({"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "k"}) From 85db1b3330428108e3c68720eebb1573f03f6405 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 06:19:57 +0000 Subject: [PATCH 15/27] Open a PR right after pushing a claude/* branch, so the PR/CI button shows Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- CLAUDE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 5993fe7..2d5d2dd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,6 +7,8 @@ MCP data tools and web search. User-facing docs are in `README.md`; this file is - `make check`: ruff + pytest (no network, a few seconds). Run it before every commit, on Python 3.13.5 (production; `uv venv --python 3.13.5`). GitHub CI runs the latest stable Python and Node.js instead, to catch breakage from the next upgrade early. - `make prompt-report`: token breakdown of a sample request. `python bot.py [label]` runs the bot. +- Pull requests: right after pushing a `claude/*` branch, open its PR if it has none, so Claude Code shows the PR/CI + button. Never merge it unless asked. - Tests use fakes (`tests/conftest.py`: FakeBot, user_msg; `tests/fakes.py`: ScriptedBackend, FakeMcp, fake SDK clients; `tests/fixtures/` real anonymised MCP output). Add a test next to the module you change; don't write scratch scripts. From 17c9d79912e36a09342895472bc9083e8e3eb336 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 08:53:21 +0000 Subject: [PATCH 16/27] Drop the MCP library's 'Unknown SSE event: ping' keepalive warning A remote MCP server sends a ping event every 30 s, which the client logs as a warning. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/app.py | 8 +++++--- tests/test_app.py | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/app.py b/lib/app.py index 96f977d..92e0efe 100644 --- a/lib/app.py +++ b/lib/app.py @@ -59,9 +59,11 @@ def setup_logging() -> None: def _drop_protocol_line(record: logging.LogRecord) -> bool: - """The MCP library's own "Negotiated protocol version" line doesn't say which server it is about; the - bot logs "MCP : connected, protocol ..." instead.""" - return not record.getMessage().startswith("Negotiated protocol version") + """Library log lines that are noise here: the MCP library's own "Negotiated protocol version" line + doesn't say which server it is about (the bot logs "MCP : connected, protocol ..." instead), and + a remote server's 30 s keepalive arrives as an "Unknown SSE event: ping" warning.""" + msg = record.getMessage() + return not (msg.startswith("Negotiated protocol version") or msg == "Unknown SSE event: ping") async def alert_down(ctx: Ctx, server: MCPServer) -> None: diff --git a/tests/test_app.py b/tests/test_app.py index f8f84e7..3ea0c17 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -181,7 +181,7 @@ async def fake_loop(ctx): assert len(started) == expected -def test_the_mcp_librarys_unlabelled_protocol_line_is_dropped(monkeypatch): +def test_the_mcp_librarys_unlabelled_protocol_and_ping_lines_are_dropped(monkeypatch): import logging monkeypatch.delenv("JOURNAL_STREAM", raising=False) @@ -193,6 +193,8 @@ def test_the_mcp_librarys_unlabelled_protocol_line_is_dropped(monkeypatch): keep = [h for h in root.handlers] mk = lambda msg: logging.LogRecord("mcp.client", logging.INFO, "f", 1, msg, None, None) # noqa: E731 assert all(not h.filter(mk("Negotiated protocol version: 2025-11-25")) for h in keep) + assert all(not h.filter(mk("Unknown SSE event: ping")) for h in keep) + assert all(h.filter(mk("Unknown SSE event: other")) for h in keep) # other unknown events still show assert all(h.filter(mk("MCP yahoo: connected, protocol 2025-11-25")) for h in keep) finally: root.handlers[:] = saved[0] From 8ba97281438f6070c0f29cac9dc6592c0f4ac883 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 21:47:26 +0000 Subject: [PATCH 17/27] xAI: drop text written before a search call; log searches in the summary line With server-side search the model can write a draft, search, then keep writing into the same text, so the reply held the draft glued to the answer. Only the text after the last search call is kept. The per-request summary line now shows the provider's own searches next to the tool calls. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/llm/runner.py | 4 ++-- lib/llm/xai.py | 38 +++++++++++++++++++++++++------------- tests/test_llm_backends.py | 32 ++++++++++++++++++++++++++++++++ tests/test_runner.py | 6 ++++++ 4 files changed, 65 insertions(+), 15 deletions(-) diff --git a/lib/llm/runner.py b/lib/llm/runner.py index cfefe87..46be9b5 100644 --- a/lib/llm/runner.py +++ b/lib/llm/runner.py @@ -198,8 +198,8 @@ async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: boo break if req.must_search and not (tool_calls or searched): log.warning("This request needed a search, but the answer shows no sign of one") - log.info("%.1fs, %d rounds, %d tools: in %d (%.0f%% cached) out %d, $%.4f", - time.monotonic() - t0, rounds, tool_calls, total.tokens_in, total.cached_pct, + log.info("%.1fs, %d rounds, %d tools, %d searches: in %d (%.0f%% cached) out %d, $%.4f", + time.monotonic() - t0, rounds, tool_calls, searched, total.tokens_in, total.cached_pct, total.tokens_out, total.cost) text = TOOL_SYNTAX_RE.sub("", step.text).strip() if not text and step.finish == "length" and tool_calls: diff --git a/lib/llm/xai.py b/lib/llm/xai.py index 477514d..c78d313 100644 --- a/lib/llm/xai.py +++ b/lib/llm/xai.py @@ -33,19 +33,26 @@ def _as_input(item) -> dict: return item.model_dump(exclude_none=True) if hasattr(item, "model_dump") else dict(item) -def answer_text(resp) -> str: - """The response's last message. `output_text` joins every message, and with server-side search - the model can write a draft, search, then write the answer, all in one response.""" +def answer_text(resp, tail: str | None = None) -> str: + """The response's last message, and in it the last text part. `output_text` joins everything, and + with server-side search the model can write a draft, search, then write the answer, as separate + messages, as parts of one, or as one run of text with the searches in between. `tail` is the text + streamed after the last search call; when the chosen text ends with it, the draft before it goes.""" messages = [i for i in resp.output if i.type == "message"] parts = [[getattr(c, "text", "") or "" for c in m.content if c.type == "output_text"] for m in messages] texts = ["".join(p) for p in parts] - for p in parts: - if len([t for t in p if t.strip()]) > 1: # not handled yet: this line tells us if it happens - log.info("xAI message had %d text parts (%s chars)", len(p), "+".join(str(len(t)) for t in p)) if len([t for t in texts if t.strip()]) > 1: log.info("xAI response had %d messages; kept the last (items: %s)", len(texts), " ".join(i.type for i in resp.output)) - return next((t for t in reversed(texts) if t.strip()), resp.output_text or "").strip() + for p in parts: + if len([t for t in p if t.strip()]) > 1: # not handled yet: this line tells us if it happens + log.info("xAI message had %d text parts (%s chars)", len(p), "+".join(str(len(t)) for t in p)) + text = next((t for t in reversed(texts) if t.strip()), resp.output_text or "").strip() + if tail and tail.strip() and len(tail.strip()) < len(text) and text.endswith(tail.strip()): + log.info("xAI wrote %d chars before a search call; kept the %d after it (items: %s)", + len(text) - len(tail.strip()), len(tail.strip()), " ".join(i.type for i in resp.output)) + return tail.strip() + return text class XaiBackend(Backend): @@ -87,12 +94,17 @@ def _kwargs(self, req: Request, tool_choice: str | None) -> dict: async def _create(self, req: Request, kw: dict, **inputs): stream = await self.client.responses.create(**kw, **inputs) - text, final = "", None + text, final, tail = "", None, None async for event in stream: if event.type == "response.output_item.added" and event.item.type == "message": text = "" # a new message replaces the draft before it + elif event.type == "response.output_item.added" and event.item.type.endswith("_search_call"): + if text.strip(): # text written before a search is a draft; only what follows counts + text, tail = "", "" elif event.type == "response.output_text.delta": text += event.delta + if tail is not None: + tail = text if req.on_text: req.on_text(text) elif event.type in ("response.completed", "response.incomplete", "response.failed"): @@ -102,23 +114,23 @@ async def _create(self, req: Request, kw: dict, **inputs): if final.status == "failed": msg = f"xAI response failed: {final.error}" raise (Overloaded if getattr(final.error, "code", None) == "server_error" else RuntimeError)(msg) - return final + return final, tail async def step(self, conv: dict, req: Request, *, tool_choice: str | None) -> Step: kw = self._kwargs(req, tool_choice) prev = conv["resp"] try: if self.stateless or prev is None: - resp = await self._create(req, kw, input=conv["input"]) + resp, tail = await self._create(req, kw, input=conv["input"]) else: - resp = await self._create(req, kw, previous_response_id=prev.id, input=conv["pending"]) + resp, tail = await self._create(req, kw, previous_response_id=prev.id, input=conv["pending"]) except BadRequestError as e: if prev is None or not self.stateless: raise log.warning("xAI rejected the replayed conversation (%s); using previous_response_id " "from now on", e) self.stateless = False - resp = await self._create(req, kw, previous_response_id=prev.id, input=conv["last_results"]) + resp, tail = await self._create(req, kw, previous_response_id=prev.id, input=conv["last_results"]) conv["resp"] = resp calls = [Call(i.call_id, i.name, i.arguments) for i in resp.output if i.type == "function_call"] u = resp.usage @@ -130,7 +142,7 @@ async def step(self, conv: dict, req: Request, *, tool_choice: str | None) -> St cost=(getattr(u, "cost_in_usd_ticks", 0) or 0) / TICKS_PER_USD, ) searches = sum(1 for i in resp.output if i.type.endswith("_search_call")) - return Step(answer_text(resp), calls, usage, resp.status, searches) + return Step(answer_text(resp, tail), calls, usage, resp.status, searches) def add_results(self, conv: dict, step: Step, results: list[str]) -> None: items = [{"type": "function_call_output", "call_id": c.id, "output": out} diff --git a/tests/test_llm_backends.py b/tests/test_llm_backends.py index f205dd5..d9662da 100644 --- a/tests/test_llm_backends.py +++ b/tests/test_llm_backends.py @@ -258,6 +258,38 @@ async def test_xai_logs_a_message_with_several_text_parts_and_still_joins_them(c "xAI message had 2 text parts (3+3 chars)"] +async def test_xai_drops_text_written_before_a_search_inside_one_message(caplog): + from types import SimpleNamespace as N + caplog.set_level("INFO", logger="bot") + st = config.load({"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "k"}) + message = xai_message("generic draft.answer from the news") + items = [N(type="web_search_call"), message] + events = [N(type="response.output_item.added", item=message), + N(type="response.output_text.delta", delta="generic draft."), + N(type="response.output_item.added", item=items[0]), + N(type="response.output_text.delta", delta="answer from the news"), + xai_final(items, text="generic draft.answer from the news")] + seen = [] + b = XaiBackend(st, FakeXaiClient(events)) + r = req(on_text=seen.append) + step = await b.step(b.start(r), r, tool_choice=None) + assert step.text == "answer from the news" and step.searches == 1 + assert seen == ["generic draft.", "answer from the news"] + assert any("wrote 14 chars before a search call" in m.getMessage() for m in caplog.records) + + +async def test_xai_leaves_text_alone_when_no_search_interrupts_it(): + from types import SimpleNamespace as N + st = config.load({"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "k"}) + message = xai_message("one answer") + events = [N(type="response.output_item.added", item=message), + N(type="response.output_text.delta", delta="one answer"), + xai_final([message], text="one answer")] + b = XaiBackend(st, FakeXaiClient(events)) + r = req() + assert (await b.step(b.start(r), r, tool_choice=None)).text == "one answer" + + async def test_xai_failed_response_at_capacity_is_overloaded_other_failures_are_not(): from types import SimpleNamespace as N st = config.load({"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "k"}) diff --git a/tests/test_runner.py b/tests/test_runner.py index 17b3d38..ec3b3e9 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -133,6 +133,12 @@ async def test_forced_search_first_round_and_retry_without_required_on_400(): await run(b2, registry(), req(must_search=True)) +async def test_summary_line_counts_the_providers_own_searches(caplog): + caplog.set_level("INFO", logger="bot") + await run(ScriptedBackend([step("done", searches=3)]), registry(), req(must_search=True)) + assert any("0 tools, 3 searches" in r.getMessage() for r in caplog.records) + + async def test_fake_search_call_text_is_recovered_once_and_failures_reported(): async def search(q): raise RuntimeError("rate limited") From af307039a5bb2a13320296ba74a5e93ce67f99e2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 21:57:33 +0000 Subject: [PATCH 18/27] Log MCP tool lists on one line, cut at LOG_WIDTH (default 95) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- README.md | 1 + lib/app.py | 3 ++- lib/config.py | 3 +++ lib/mcp/server.py | 31 +++++++++++++++++++------------ tests/test_config.py | 7 +++++++ tests/test_mcp.py | 18 ++++++++++++++++++ 6 files changed, 50 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 9b1d57a..fdffd42 100644 --- a/README.md +++ b/README.md @@ -286,6 +286,7 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. | `DATA_DIR` | Directory for the bot's data (the database); a relative `DB_PATH` resolves inside it, an absolute one is used as given. Default: systemd's `STATE_DIRECTORY` (see [Running under systemd](#running-under-systemd)), else the working directory. In Docker, set `DATA_DIR=/data` and bind-mount a host directory there so history survives upgrades. | | `DB_PATH` | SQLite file for chat history (default chat_log.db). Instances may share it. | | `HISTORY_LIMIT` | Messages of chat history in the prompt; the window is HISTORY_LIMIT to 1.5x (default 20). | +| `LOG_WIDTH` | Longest line (characters) for the MCP tool lists in the log; a longer list is cut at a whole name and ends in `...` (default 95, 0 = no limit). | | `MAX_TOKENS` | Reply cap in tokens, reasoning included (default 3000). | | `MCP_CONFIG` | MCP server config file (default mcp_servers.json next to bot.py, whatever `DATA_DIR` is; missing = no MCP tools). | | `MOVERS_BOTS` | Usernames of bots whose end-of-day big-movers lists get explained. Empty (default) = feature off. | diff --git a/lib/app.py b/lib/app.py index 92e0efe..bf3310d 100644 --- a/lib/app.py +++ b/lib/app.py @@ -96,7 +96,8 @@ async def on_down(server: MCPServer) -> None: await alert_down(ctx, server) ctx.registry = Registry.load(st.mcp_config, timeout=config.MCP_TIMEOUT, - max_output=st.max_tool_output, on_down=on_down) + max_output=st.max_tool_output, on_down=on_down, + log_width=st.log_width) return ctx diff --git a/lib/config.py b/lib/config.py index a2e7ff1..a3de261 100644 --- a/lib/config.py +++ b/lib/config.py @@ -29,6 +29,7 @@ "DATA_DIR": "Directory for the bot's data (the database); a relative DB_PATH resolves inside it. Default: systemd's STATE_DIRECTORY, else the working directory.", "DB_PATH": "SQLite file for chat history (default chat_log.db). Instances may share it.", "HISTORY_LIMIT": "Messages of chat history in the prompt; the window is HISTORY_LIMIT to 1.5x (default 20).", + "LOG_WIDTH": "Longest line (characters) for the MCP tool lists in the log; longer lists end in \"...\" (default 95, 0 = no limit).", "MAX_TOKENS": "Reply cap in tokens, reasoning included (default 3000).", "MCP_CONFIG": "MCP server config file (default mcp_servers.json next to bot.py; missing = no MCP tools).", "MOVERS_BOTS": "Usernames of bots whose end-of-day big-movers lists get explained. Empty (default) = feature off.", @@ -75,6 +76,7 @@ class Settings: search_model: str max_tokens: int history_limit: int + log_width: int db_path: str tz: ZoneInfo mcp_config: str @@ -191,6 +193,7 @@ def load(env: Mapping[str, str] | None = None) -> Settings: search_model=search_model, max_tokens=int(env.get("MAX_TOKENS") or 3000), history_limit=int(env.get("HISTORY_LIMIT") or 20), + log_width=int(env.get("LOG_WIDTH") or 95), db_path=data_path(env, "DB_PATH", "chat_log.db"), tz=ZoneInfo(env.get("BOT_TZ", "UTC")), mcp_config=mcp_config_path(env), diff --git a/lib/mcp/server.py b/lib/mcp/server.py index 871cc9b..260cc79 100644 --- a/lib/mcp/server.py +++ b/lib/mcp/server.py @@ -9,7 +9,6 @@ import logging import re import tempfile -import textwrap import time from collections.abc import Awaitable, Callable from contextlib import AsyncExitStack @@ -44,11 +43,11 @@ class MCPServer: entered and exited in the same task.""" def __init__(self, label: str, cfg: dict, *, timeout: float, max_output: int, - on_down: Callable[[MCPServer], Awaitable[None]] | None = None): + on_down: Callable[[MCPServer], Awaitable[None]] | None = None, log_width: int = 95): self.label = re.sub(r"[^A-Za-z0-9_-]", "_", label) self.cfg = cfg self.description = cfg.get("description", "") - self.timeout, self.max_output, self.on_down = timeout, max_output, on_down + self.timeout, self.max_output, self.on_down, self.log_width = timeout, max_output, on_down, log_width self.session: ClientSession | None = None self.error: str | None = None # why the server is down, if it is self.tools: list[ToolDef] = [] @@ -148,9 +147,9 @@ async def _load_tools(self, session: ClientSession) -> None: self._warn_unknown_names(listed, allow | blocked | set(self._warm)) enabled = sorted(self.fn_names.values()) skipped = sorted(t.name for t in listed if t.name not in enabled) - log_names(f"MCP {self.label}: {len(enabled)} tools:", enabled) + log_names(f"MCP {self.label}: {len(enabled)} tools:", enabled, self.log_width) if skipped: - log_names(f"MCP {self.label}: {len(skipped)} skipped:", skipped) + log_names(f"MCP {self.label}: {len(skipped)} skipped:", skipped, self.log_width) def _warn_unknown_names(self, listed, configured: set[str]) -> None: """A blocked/allowed name that matches no tool does nothing; say so (and what was meant).""" @@ -290,7 +289,7 @@ def __init__(self, servers: list[MCPServer]): @classmethod def load(cls, path: str, *, timeout: float, max_output: int, - on_down: Callable[[MCPServer], Awaitable[None]] | None = None) -> Registry: + on_down: Callable[[MCPServer], Awaitable[None]] | None = None, log_width: int = 95) -> Registry: """Servers from a JSON file ({"mcpServers": {...}} or Claude Desktop style); a missing file means no servers.""" try: @@ -299,7 +298,8 @@ def load(cls, path: str, *, timeout: float, max_output: int, except FileNotFoundError: return cls([]) cfg = cfg.get("mcpServers", cfg) - return cls([MCPServer(label, c, timeout=timeout, max_output=max_output, on_down=on_down) + return cls([MCPServer(label, c, timeout=timeout, max_output=max_output, on_down=on_down, + log_width=log_width) for label, c in cfg.items() if not c.get("disabled")]) def start(self) -> None: @@ -330,11 +330,18 @@ def lookup(self, fn_name: str) -> tuple[MCPServer, str] | None: return None -def log_names(head: str, names: list[str], width: int = 80) -> None: - """Log a list of tool names on lines short enough not to wrap in a terminal or journal.""" - lines = textwrap.wrap(", ".join(names), width - len(head)) - for i, line in enumerate(lines): - log.info("%s %s", head if i == 0 else " " * len(head), line) +def log_names(head: str, names: list[str], width: int = 95) -> None: + """Log a list of tool names on one line of at most `width` characters (0 = no limit). A list that + doesn't fit is cut at a whole name and ends in "...".""" + line = f"{head} {', '.join(names)}" + if width and len(line) > width: + kept = f"{head} " + for i, name in enumerate(names): + if len(kept) + len(name) + len(", ...") > width and i: + break + kept += f"{name}, " + line = kept + "..." + log.info("%s", line) def _leaf_errors(e: BaseException) -> list[BaseException]: diff --git a/tests/test_config.py b/tests/test_config.py index d8cc75e..5ddfc3a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -74,6 +74,13 @@ def test_holding_news_and_movers_switch_on_by_listing_names(env): assert not config.load({**env, "MOVERS_BOTS": ","}).movers_explain +def test_log_width_defaults_to_95(env): + assert config.load(env).log_width == 95 + assert config.load({**env, "LOG_WIDTH": "120"}).log_width == 120 + assert config.load({**env, "LOG_WIDTH": "0"}).log_width == 0 # 0 = no limit + assert config.load({**env, "LOG_WIDTH": ""}).log_width == 95 + + def test_token_saver_switches_defaults(env): on, off = config.load(env), config.load({**env, "TOKEN_SAVER": "off"}) assert (on.max_tokens, off.max_tokens) == (3000, 3000) diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 24cc860..81a802a 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -105,6 +105,24 @@ def test_log_names_keeps_exact_names(caplog): assert "get_analyst_estimates" in caplog.text and "get_stock_quote" in caplog.text +def test_log_names_is_one_line_cut_at_a_whole_name(caplog): + import logging + + from lib.mcp.server import log_names + names = ["compare_companies", "get_company", "get_company_metrics", "get_data_notes", "get_events"] + head = "MCP t: 5 tools:" + with caplog.at_level(logging.INFO, logger="bot"): + log_names(head, names, 95) + log_names(head, names, 60) + log_names(head, names, 0) + log_names(head, names, 5) + short, cut, unlimited, tiny = (r.getMessage() for r in caplog.records) + assert short == f"{head} {', '.join(names)}" and len(short) <= 95 + assert cut == f"{head} compare_companies, get_company, ..." and len(cut) <= 60 + assert unlimited == short + assert tiny == f"{head} compare_companies, ..." # the first name is always shown + + async def test_call_slims_and_caches_identical_concurrent_calls(): s = make_server(cache_ttl=30) a, b = await asyncio.gather(s.call("get_a", {"t": "X"}), s.call("get_a", {"t": "X"})) From 78d69b8ff38e82f16fce50f7d7169c67b9ebc40e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 06:04:36 +0000 Subject: [PATCH 19/27] Movers explanations: leave the earlier chat out of the prompt Only the movers list itself goes in the transcript; the rest of the history has nothing to do with it. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/telegram/handlers.py | 7 +++++-- tests/test_handlers.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index b4f669e..848bdda 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -179,9 +179,12 @@ async def _gate_text(self, msg: Message, trig: Trigger) -> str: text += " " + compact(prev.text)[:FOLLOW_UP_CHARS] return text - async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: bool = False) -> str: + async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: bool = False, + only_this: bool = False) -> str: ctx, st = self.ctx, self.ctx.st rows = await asyncio.to_thread(ctx.store.history, msg.chat_id, st.history_limit) + if only_this: # the earlier chat has nothing to do with the request + rows = [r for r in rows if r.message_id == msg.message_id] transcript = format_rows(rows, ctx.self_name, st.tz, line_max=st.history_line_max, compact_text=st.token_saver, own_line_max=st.history_own_line_max) return chat_prompt( @@ -237,7 +240,7 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s kind="preset", chat_id=chat_id) elif trig.movers: listing = msg.text_html if msg.text else (msg.caption_html or "") - text = await self._chat_text(msg, trig, MOVERS_EXPLAIN.format(listing=listing)) + text = await self._chat_text(msg, trig, MOVERS_EXPLAIN.format(listing=listing), only_this=True) parts = [{"type": "text", "text": text}] answer = await ask(ctx, parts, route("", st, ctx.registry, force_full=True), must_search=True, cache_key=f"chat-{chat_id}", on_text=on_text, diff --git a/tests/test_handlers.py b/tests/test_handlers.py index ba7f5a5..9ef155c 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -188,6 +188,17 @@ async def test_busy_provider_gets_a_plain_reply_and_a_one_line_log(env, store, c assert "\n" not in line and not any(r.exc_info for r in caplog.records) +async def test_movers_prompt_leaves_out_the_earlier_chat(env, store): + ctx, backend, h = make_ctx(env, store, [step("NVDA up on news.")], MOVERS_BOTS="finbotibot") + await run(h, user_msg(ctx.bot, "did anyone watch the football last night")) + m = user_msg(ctx.bot, "≥ 5.0% at close (ASX):\nNVDA +6.1%\nAMD +5.5%", is_bot=True) + m.from_user.is_bot, m.from_user.username = True, "finbotibot" + m.parse_entities = lambda types=None: {"b": "≥ 5.0% at close (ASX):"} + await run(h, m) + prompt = backend.seen[0]["prompt"] + assert "football" not in prompt and "NVDA +6.1%" in prompt and "explain" in prompt + + async def test_movers_list_off_by_default_then_answered_once_without_tagging(env, store): text = "≥ 5.0% at close (ASX):\nNVDA +6.1%\nAMD +5.5%" for flag, expect in (({}, 0), ({"MOVERS_BOTS": "finbotibot"}, 1)): From 02c37271b71a50982723fbd6acccd192f3a10539 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 06:05:19 +0000 Subject: [PATCH 20/27] Fix the movers prompt test: give the two messages different ids Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- tests/test_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 9ef155c..9aeab67 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -191,7 +191,7 @@ async def test_busy_provider_gets_a_plain_reply_and_a_one_line_log(env, store, c async def test_movers_prompt_leaves_out_the_earlier_chat(env, store): ctx, backend, h = make_ctx(env, store, [step("NVDA up on news.")], MOVERS_BOTS="finbotibot") await run(h, user_msg(ctx.bot, "did anyone watch the football last night")) - m = user_msg(ctx.bot, "≥ 5.0% at close (ASX):\nNVDA +6.1%\nAMD +5.5%", is_bot=True) + m = user_msg(ctx.bot, "≥ 5.0% at close (ASX):\nNVDA +6.1%\nAMD +5.5%", is_bot=True, mid=2) m.from_user.is_bot, m.from_user.username = True, "finbotibot" m.parse_entities = lambda types=None: {"b": "≥ 5.0% at close (ASX):"} await run(h, m) From 22995d52096603b00908e1f69c24ad4a910a8787 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 06:07:59 +0000 Subject: [PATCH 21/27] Movers explanations: send the list once, with no transcript Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/prompts.py | 7 +++++-- lib/telegram/handlers.py | 8 +++----- tests/test_handlers.py | 3 ++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/prompts.py b/lib/prompts.py index d9cb3da..b4dc0f9 100644 --- a/lib/prompts.py +++ b/lib/prompts.py @@ -72,8 +72,11 @@ def chat_prompt(*, transcript: str, sender: str, private: bool, reply_quote: str fresh: bool = False) -> str: """The user turn: transcript, the replied-to message, and the volatile tail (kept last so the cached prefix before it is unchanged).""" - head = (f"Private one-to-one chat with {sender}, not the group. Transcript:\n{transcript}" - if private else f"Group chat transcript:\n{transcript}") + if transcript: + head = (f"Private one-to-one chat with {sender}, not the group. Transcript:\n{transcript}" + if private else f"Group chat transcript:\n{transcript}") + else: # a standalone request (a movers list): what follows is all there is + head = f"Private one-to-one chat with {sender}, not the group." if private else "Group chat." if reply_quote: head += f"\n\nThe last message (#{msg_id}) is a reply to this message:\n{reply_quote}" head += middle diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index 848bdda..4a9c691 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -180,11 +180,9 @@ async def _gate_text(self, msg: Message, trig: Trigger) -> str: return text async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: bool = False, - only_this: bool = False) -> str: + no_history: bool = False) -> str: ctx, st = self.ctx, self.ctx.st - rows = await asyncio.to_thread(ctx.store.history, msg.chat_id, st.history_limit) - if only_this: # the earlier chat has nothing to do with the request - rows = [r for r in rows if r.message_id == msg.message_id] + rows = [] if no_history else await asyncio.to_thread(ctx.store.history, msg.chat_id, st.history_limit) transcript = format_rows(rows, ctx.self_name, st.tz, line_max=st.history_line_max, compact_text=st.token_saver, own_line_max=st.history_own_line_max) return chat_prompt( @@ -240,7 +238,7 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s kind="preset", chat_id=chat_id) elif trig.movers: listing = msg.text_html if msg.text else (msg.caption_html or "") - text = await self._chat_text(msg, trig, MOVERS_EXPLAIN.format(listing=listing), only_this=True) + text = await self._chat_text(msg, trig, MOVERS_EXPLAIN.format(listing=listing), no_history=True) parts = [{"type": "text", "text": text}] answer = await ask(ctx, parts, route("", st, ctx.registry, force_full=True), must_search=True, cache_key=f"chat-{chat_id}", on_text=on_text, diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 9aeab67..66389b7 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -196,7 +196,8 @@ async def test_movers_prompt_leaves_out_the_earlier_chat(env, store): m.parse_entities = lambda types=None: {"b": "≥ 5.0% at close (ASX):"} await run(h, m) prompt = backend.seen[0]["prompt"] - assert "football" not in prompt and "NVDA +6.1%" in prompt and "explain" in prompt + assert "football" not in prompt and "explain" in prompt + assert prompt.count("NVDA +6.1%") == 1 # the list once, not also as the last transcript line async def test_movers_list_off_by_default_then_answered_once_without_tagging(env, store): From 70a71950723804d0e68a098d6e7af2c0924f1002 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 06:47:16 +0000 Subject: [PATCH 22/27] #nocontext: send the model only the message that tagged the bot The tag is removed before the model sees it. No chat history, no replied-to message or photo. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- CLAUDE.md | 2 +- README.md | 4 ++++ lib/telegram/handlers.py | 12 ++++++++++-- tests/test_handlers.py | 17 +++++++++++++++++ 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2d5d2dd..f3e8851 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -33,7 +33,7 @@ MCP data tools and web search. User-facing docs are in `README.md`; this file is ## Request flow `handlers.on_message` logs the message -> `_trigger` (mention / reply-to-bot / DM / movers list) -> -`_generate` builds the prompt (`history.format_rows` + `prompts.chat_prompt`) -> `gate.route` picks tools -> +`_generate` builds the prompt (`history.format_rows` + `prompts.chat_prompt`; `#nocontext` in the message keeps only that message) -> `gate.route` picks tools -> `ask.ask` builds a `Request` -> `policy.ask` -> `runner.run` loops `backend.step` / `run_calls` -> `format_answer` (md_to_html, link_tickers) -> `reply_chunks` (split_html, save each sent message). diff --git a/README.md b/README.md index fdffd42..30ab56d 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,10 @@ Restart=on-failure provider's prompt cache keeps working. The bot's own earlier replies appear as "You" (its latest reply in full, so follow-up questions about it work; older ones shortened); a replied-to message is quoted even if it is older than the window. +- **`#nocontext`:** put `#nocontext` anywhere in a message that tags the bot (or in a DM) and the model gets that + message alone: no chat history, and nothing quoted from the message it replies to (no photo from it either). The tag + itself is removed before the model sees it. Useful for a fresh question that earlier chat would only confuse + or make more expensive; the bot's own recent reply is not read either, so follow-ups need the full question. - **Shared database:** group history lives in `messages`, shared by every bot pointed at the same `DB_PATH` (so two bots in one group see each other's replies). Private chats live in `dm_messages`, keyed by bot, because each bot's DM with a person has its own message-ID sequence. diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index 4a9c691..583db15 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -33,6 +33,7 @@ FOLLOW_UP_WORDS = 12 # a message this short right after the bot's reply may be a follow-up to it FOLLOW_UP_SECONDS = 30 * 60 FOLLOW_UP_CHARS = 400 # of the bot's reply that the tool gate also reads +NO_CONTEXT = re.compile(r"[ \t]*(? Trigger | None: and reply_target.from_user.id == bot.id) if not (is_movers or private or mentions(text, bot.username) or replied_to_bot): return None - return Trigger(text, private, is_movers, reply_target, replied_to_bot) + nocontext = bool(NO_CONTEXT.search(text)) + if nocontext: # the model never sees the tag, or what the message replies to + text, reply_target = NO_CONTEXT.sub("", text).strip(), None + return Trigger(text, private, is_movers, reply_target, replied_to_bot, nocontext) async def _command(self, msg: Message, trig: Trigger) -> bool: """Private-chat commands handled here (/credits has its own handler).""" @@ -171,7 +176,7 @@ async def _gate_text(self, msg: Message, trig: Trigger) -> str: terms?") sent right after the bot's own reply also gets the start of that reply, so it counts as part of the same (portfolio, market...) conversation instead of reaching the model with no tools.""" text = trig.text + " " + (describe(trig.reply_target) if trig.reply_target else "") - if trig.reply_target is None and len(trig.text.split()) <= FOLLOW_UP_WORDS: + if trig.reply_target is None and not trig.nocontext and len(trig.text.split()) <= FOLLOW_UP_WORDS: rows = await asyncio.to_thread(self.ctx.store.history, msg.chat_id, 2) if len(rows) >= 2 and rows[-1].message_id == msg.message_id: prev = rows[-2] @@ -183,6 +188,9 @@ async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: no_history: bool = False) -> str: ctx, st = self.ctx, self.ctx.st rows = [] if no_history else await asyncio.to_thread(ctx.store.history, msg.chat_id, st.history_limit) + if trig.nocontext: # only the message that tagged the bot + rows = [r._replace(text=NO_CONTEXT.sub("", r.text).strip(), reply_to=None) + for r in rows if r.message_id == msg.message_id] transcript = format_rows(rows, ctx.self_name, st.tz, line_max=st.history_line_max, compact_text=st.token_saver, own_line_max=st.history_own_line_max) return chat_prompt( diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 66389b7..3247b60 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -188,6 +188,23 @@ async def test_busy_provider_gets_a_plain_reply_and_a_one_line_log(env, store, c assert "\n" not in line and not any(r.exc_info for r in caplog.records) +async def test_nocontext_sends_only_the_tagging_message_without_the_tag(env, store): + ctx, backend, h = make_ctx(env, store, [step("fine"), step("fine too")]) + await run(h, user_msg(ctx.bot, "did anyone watch the football last night", mid=1)) + target = user_msg(ctx.bot, "BHP looks cheap", mid=2) + await run(h, target) + m = user_msg(ctx.bot, "@stockbot #NoContext what do you make of NVDA?", mid=3) + m.reply_to_message = target + await run(h, m) + prompt = backend.seen[0]["prompt"] + assert "NVDA" in prompt and "nocontext" not in prompt.lower() + assert "football" not in prompt and "BHP" not in prompt # no history, no replied-to message + assert "is a reply to" not in prompt + assert len(m.replies) == 1 + await run(h, user_msg(ctx.bot, "@stockbot what about NVDA?", mid=4)) # without the tag: history is back + assert "football" in backend.seen[1]["prompt"] + + async def test_movers_prompt_leaves_out_the_earlier_chat(env, store): ctx, backend, h = make_ctx(env, store, [step("NVDA up on news.")], MOVERS_BOTS="finbotibot") await run(h, user_msg(ctx.bot, "did anyone watch the football last night")) From 943d91cfc7f2167ff29e46ad09dd67fd05b39233 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 14:46:47 +0000 Subject: [PATCH 23/27] Signal: answer DMs and group mentions through a signal-cli daemon the bot runs SIGNALCLI_PATH turns it on; SIGNAL_ALLOWED_SENDERS lists who may DM it (and get Sharesight data). Messages and replies travel over the daemon's JSON-RPC socket. History is kept per chat in signal_messages. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- CLAUDE.md | 2 + README.md | 31 +++++- lib/app.py | 4 +- lib/config.py | 6 ++ lib/signalbot/__init__.py | 34 ++++++ lib/signalbot/client.py | 144 ++++++++++++++++++++++++++ lib/signalbot/handlers.py | 136 ++++++++++++++++++++++++ lib/signalbot/parse.py | 65 ++++++++++++ lib/store.py | 23 +++-- lib/telegram/handlers.py | 3 +- lib/textfmt.py | 3 + tests/test_signal.py | 210 ++++++++++++++++++++++++++++++++++++++ 12 files changed, 651 insertions(+), 10 deletions(-) create mode 100644 lib/signalbot/__init__.py create mode 100644 lib/signalbot/client.py create mode 100644 lib/signalbot/handlers.py create mode 100644 lib/signalbot/parse.py create mode 100644 tests/test_signal.py diff --git a/CLAUDE.md b/CLAUDE.md index f3e8851..f3690f0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,6 +27,8 @@ MCP data tools and web search. User-facing docs are in `README.md`; this file is - `prompts.py` all prompts. `ask.py` one entry point: route -> Request -> policy.ask -> usage -> NEEDS_TOOLS rerun. - `telegram/`: `handlers.py` (trigger -> request -> reply), `send.py`, `draft.py` (typing + streaming drafts), `commands.py` (/credits), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop). +- `signalbot/` (SIGNALCLI_PATH): `client.py` runs `signal-cli daemon --socket` and speaks JSON-RPC to it, `parse.py` envelope -> `Incoming`, + `handlers.py` the Signal reply flow (own `signal_messages` history; no Telegram types). Reuses `ask`, `gate`, `prompts`, `history`. - `app.py` wiring/startup; `context.py` `Ctx` (settings, store, backend, registry, bot). - `lib/` holds only what the bot needs to run. Developer helpers live in `tools/`: `report.py` (`make prompt-report`), `capture.py` (real MCP output for fixtures), `period_cache_without_rowid.py` (one-off DB conversion); the bot never imports them. diff --git a/README.md b/README.md index 30ab56d..2933e51 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ from MCP servers (Yahoo Finance, Sharesight) and web search. It runs on **xAI (G ## Contents - [Quick start](#quick-start) · [Providers](#providers) · [How it answers](#how-it-answers) - [Search](#search) · [MCP data tools](#mcp-data-tools) · [Formatting and ticker links](#formatting-and-ticker-links) -- [Optional features](#optional-features-off-by-default) · [Commands](#commands) +- [Optional features](#optional-features-off-by-default) · [Signal](#signal) · [Commands](#commands) - [Caching and token saving](#caching-and-token-saving) · [Running several instances](#running-several-instances) - [Logs](#logs) · [Environment variables](#environment-variables) · [Choosing a model](#choosing-a-model) - [Development](#development) · [Troubleshooting](#troubleshooting) @@ -178,6 +178,33 @@ Each is off by default. The holding-news DM and the movers reply are switched on from being added: a group it joined earlier is unknown until someone posts there. If the name matches none of your groups the reply is the same whether or not the group exists. +## Signal +Off by default. Set `SIGNALCLI_PATH` to a [signal-cli](https://github.com/AsamK/signal-cli) executable that already has an +account (registered, or linked as a secondary device) and the bot answers on Signal as well as Telegram, with the same +model, tools, search and history. + +``` +SIGNALCLI_PATH=/opt/signal-cli/bin/signal-cli +SIGNAL_ALLOWED_SENDERS=aaaaaaaa-0000-4000-8000-000000000000,+61400000000 +``` +- **How it runs:** at startup the bot runs `signal-cli -o json listAccounts` to learn its own ID (the first account is + used), then starts `signal-cli -o json daemon --socket ` itself and keeps it running, restarting it + if it stops. Messages arrive, and replies go out, over the daemon's JSON-RPC socket. Don't run another signal-cli + daemon on the same account. +- **DMs:** answered only from the people in `SIGNAL_ALLOWED_SENDERS` (UUIDs or phone numbers, as signal-cli reports + them); anyone else is logged and ignored. Empty = nobody. These people are also the Signal "admins": only they get + Sharesight portfolio data. +- **Groups:** the bot answers when it is @mentioned or someone replies (quotes) one of its messages; every other group + message is logged as context. Group replies quote the message they answer. +- **History:** kept per chat in the `signal_messages` table, windowed like Telegram's. `#nocontext` works the same way + (see [How it answers](#how-it-answers)). +- **Replies are plain text:** Signal has no HTML, so bold and links are dropped (a link becomes `text (url)`) and + tickers are not linked. +- **Not on Signal:** images and other attachments (a message with no text is ignored), the typing indicator and + streamed drafts, movers lists, the daily holding-news DM, DM buttons, `/credits` and `/flushcache`. +- **Checking:** the log shows `Signal: connected to signal-cli` once the daemon is up, and each message as + `[Signal (group)] ...`. + ## Commands | Command | Who | What | |---|---|---| @@ -300,6 +327,8 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. | `SEARCH_MODEL` | OpenRouter only: model that runs the searches (default xiaomi/mimo-v2.6-flash:online). | | `SHARESIGHT_HOLDING_NEWS_RECIPIENTS` | Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off. | | `SHARESIGHT_HOLDING_NEWS_TIME` | HH:MM (BOT_TZ) for the daily holding-news check (default 08:00). | +| `SIGNALCLI_PATH` | Path to the `signal-cli` executable. Setting it turns [Signal](#signal) on: the bot starts `signal-cli daemon` itself. Empty (default) = Signal off. | +| `SIGNAL_ALLOWED_SENDERS` | Signal UUIDs or phone numbers (comma/space separated) that may DM the bot and get Sharesight data on Signal. Empty (default) = nobody. | | `TELEGRAM_DM_BUTTONS` | on|off. Preset-prompt buttons in private chats (default off). | | `TOKEN_SAVER` | on|off. Master switch for the token-saving heuristics (default on). | diff --git a/lib/app.py b/lib/app.py index bf3310d..e16e0ec 100644 --- a/lib/app.py +++ b/lib/app.py @@ -26,7 +26,7 @@ filters, ) -from . import config +from . import config, signalbot from .context import Ctx from .features import dm_buttons, holding_news, post from .features.periods import backfill @@ -116,6 +116,8 @@ async def post_init(app: Application) -> None: background.append(asyncio.create_task(dm_buttons.push_on_startup(ctx))) if st.period_backfill: background.append(asyncio.create_task(backfill.loop(ctx))) + if st.signal_cli: + background.append(asyncio.create_task(signalbot.run(ctx))) async def post_shutdown(app: Application) -> None: for task in background: diff --git a/lib/config.py b/lib/config.py index a3de261..e9c7dbe 100644 --- a/lib/config.py +++ b/lib/config.py @@ -39,6 +39,8 @@ "SEARCH_MODEL": "OpenRouter only: model that runs the searches (default xiaomi/mimo-v2.6-flash:online).", "SHARESIGHT_HOLDING_NEWS_RECIPIENTS": "Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off.", "SHARESIGHT_HOLDING_NEWS_TIME": "HH:MM (BOT_TZ) for the daily holding-news check (default 08:00).", + "SIGNALCLI_PATH": "Path to the signal-cli executable. Setting it turns Signal on: the bot starts `signal-cli daemon` itself, answers DMs from SIGNAL_ALLOWED_SENDERS and group @mentions or replies. Empty (default) = Signal off.", + "SIGNAL_ALLOWED_SENDERS": "Signal UUIDs or phone numbers (comma/space separated) that may DM the bot and get Sharesight data on Signal. Empty (default) = nobody.", "TELEGRAM_DM_BUTTONS": "on|off. Preset-prompt buttons in private chats (default off).", "TOKEN_SAVER": "on|off. Master switch for the token-saving heuristics (default on).", } @@ -77,6 +79,8 @@ class Settings: max_tokens: int history_limit: int log_width: int + signal_cli: str # path to signal-cli; empty = Signal off + signal_allowed: frozenset[str] # lowercase UUIDs/numbers that may DM the bot (and get Sharesight data) db_path: str tz: ZoneInfo mcp_config: str @@ -194,6 +198,8 @@ def load(env: Mapping[str, str] | None = None) -> Settings: max_tokens=int(env.get("MAX_TOKENS") or 3000), history_limit=int(env.get("HISTORY_LIMIT") or 20), log_width=int(env.get("LOG_WIDTH") or 95), + signal_cli=env.get("SIGNALCLI_PATH", "").strip(), + signal_allowed=frozenset(x.lower() for x in re.split(r"[,\s]+", env.get("SIGNAL_ALLOWED_SENDERS", "")) if x), db_path=data_path(env, "DB_PATH", "chat_log.db"), tz=ZoneInfo(env.get("BOT_TZ", "UTC")), mcp_config=mcp_config_path(env), diff --git a/lib/signalbot/__init__.py b/lib/signalbot/__init__.py new file mode 100644 index 0000000..11d92c4 --- /dev/null +++ b/lib/signalbot/__init__.py @@ -0,0 +1,34 @@ +"""SIGNALCLI_PATH: answer Signal DMs and group mentions through a signal-cli daemon the bot runs itself.""" + +from __future__ import annotations + +import logging + +from ..context import Ctx +from .client import SignalClient, list_accounts +from .handlers import SignalHandlers + +log = logging.getLogger("bot") + + +async def run(ctx: Ctx) -> None: + """Find the bot's Signal account, then keep the daemon running until cancelled.""" + path = ctx.st.signal_cli + try: + accounts = await list_accounts(path) + except Exception as e: + log.error("Signal is off: couldn't list accounts with %s: %s", path, e) + return + if not accounts: + log.error("Signal is off: signal-cli has no registered account") + return + if len(accounts) > 1: + log.warning("Signal: %d accounts registered; using the first", len(accounts)) + number, aci = accounts[0].get("number"), accounts[0].get("aci") or accounts[0].get("uuid") + if not aci: + log.error("Signal is off: the account has no ACI (listAccounts gave %s)", accounts[0]) + return + client = SignalClient(path) + client.on_message = SignalHandlers(ctx, client, number or aci, aci).on_message + log.info("Signal: starting for account %s", aci) + await client.run() diff --git a/lib/signalbot/client.py b/lib/signalbot/client.py new file mode 100644 index 0000000..7861c0d --- /dev/null +++ b/lib/signalbot/client.py @@ -0,0 +1,144 @@ +"""Runs `signal-cli daemon` and talks to it over its JSON-RPC socket: messages come in as `receive` +notifications, replies go out as `send` requests, all on one connection.""" + +from __future__ import annotations + +import asyncio +import contextlib +import itertools +import json +import logging +import shutil +import tempfile +from collections.abc import Awaitable, Callable +from pathlib import Path + +log = logging.getLogger("bot") +RPC_TIMEOUT = 60 # seconds to wait for signal-cli to answer a request +CONNECT_WAIT = 60 # seconds for a freshly started daemon to open its socket +LINE_LIMIT = 2**24 # an envelope with attachment details can be far longer than asyncio's 64 KiB default + + +class SignalError(RuntimeError): + """signal-cli answered a request with an error.""" + + +async def list_accounts(path: str) -> list[dict]: + """The registered accounts: `[{"number": null, "aci": ""}]`.""" + proc = await asyncio.create_subprocess_exec( + path, "-o", "json", "listAccounts", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) + out, err = await asyncio.wait_for(proc.communicate(), 60) + if proc.returncode: + raise RuntimeError(f"signal-cli listAccounts failed ({proc.returncode}): {err.decode(errors='replace').strip()}") + accounts = json.loads(out) + return accounts if isinstance(accounts, list) else [] + + +class SignalClient: + def __init__(self, path: str, on_message: Callable[[dict], Awaitable[None]] | None = None): + self.path = path + self.on_message = on_message + self._writer: asyncio.StreamWriter | None = None + self._pending: dict[int, asyncio.Future] = {} + self._ids = itertools.count(1) + self._tasks: set[asyncio.Task] = set() + + async def run(self) -> None: + """Keep a daemon running and connected, restarting it (with a growing pause) when it stops.""" + delay = 5 + while True: + try: + if await self._session(): + delay = 5 + except asyncio.CancelledError: + raise + except Exception as e: + log.warning("Signal: %s", e) + for fut in self._pending.values(): + if not fut.done(): + fut.set_exception(SignalError("signal-cli stopped")) + self._pending.clear() + log.warning("Signal: signal-cli stopped; starting it again in %d s", delay) + await asyncio.sleep(delay) + delay = min(delay * 2, 60) + + async def _session(self) -> bool: + """One daemon's lifetime. True if it got as far as connecting.""" + sock_dir = Path(tempfile.mkdtemp(prefix="signal-")) + sock = str(sock_dir / "socket") + proc = await asyncio.create_subprocess_exec( + self.path, "-o", "json", "daemon", "--socket", sock, + stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE) + pump = asyncio.create_task(self._pump_stderr(proc)) + connected = False + try: + reader, writer = await self._connect(sock, proc) + connected, self._writer = True, writer + log.info("Signal: connected to signal-cli") + async for line in reader: + self._dispatch(line) + return connected + finally: + self._writer = None + if proc.returncode is None: + proc.terminate() + with contextlib.suppress(Exception): + await asyncio.wait_for(proc.wait(), 10) + if proc.returncode is None: + proc.kill() + pump.cancel() + shutil.rmtree(sock_dir, ignore_errors=True) + + async def _connect(self, sock: str, proc) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]: + for _ in range(CONNECT_WAIT * 2): + if proc.returncode is not None: + raise RuntimeError(f"signal-cli exited with code {proc.returncode} before it was ready") + try: + return await asyncio.open_unix_connection(sock, limit=LINE_LIMIT) + except (FileNotFoundError, ConnectionRefusedError): + await asyncio.sleep(0.5) + raise RuntimeError(f"signal-cli didn't open its socket within {CONNECT_WAIT} s") + + async def _pump_stderr(self, proc) -> None: + async for line in proc.stderr: + text = line.decode(errors="replace").rstrip() + if text: + log.info("signal-cli: %s", text) + + def _dispatch(self, line: bytes) -> None: + try: + obj = json.loads(line) + except ValueError: + return + if not isinstance(obj, dict): + return + fut = self._pending.pop(obj["id"], None) if isinstance(obj.get("id"), int) else None + if fut is not None: + if "error" in obj: + err = obj["error"] + fut.set_exception(SignalError(str(err.get("message", err)) if isinstance(err, dict) else str(err))) + else: + fut.set_result(obj.get("result")) + elif self.on_message and (obj.get("method") == "receive" or "envelope" in obj): + task = asyncio.create_task(self._handle(obj)) + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + + async def _handle(self, obj: dict) -> None: + try: + await self.on_message(obj) + except Exception: + log.exception("Signal: couldn't handle a message") + + async def request(self, method: str, params: dict): + if self._writer is None: + raise SignalError("signal-cli isn't connected") + rid = next(self._ids) + fut = asyncio.get_running_loop().create_future() + self._pending[rid] = fut + try: + self._writer.write(json.dumps({"jsonrpc": "2.0", "method": method, "params": params, "id": rid}).encode() + b"\n") + await self._writer.drain() + return await asyncio.wait_for(fut, RPC_TIMEOUT) + finally: + self._pending.pop(rid, None) diff --git a/lib/signalbot/handlers.py b/lib/signalbot/handlers.py new file mode 100644 index 0000000..656fdf5 --- /dev/null +++ b/lib/signalbot/handlers.py @@ -0,0 +1,136 @@ +"""Signal message -> reply: the same history, tools and model as Telegram, with Signal's own plumbing.""" + +from __future__ import annotations + +import asyncio +import hashlib +import logging +import re +import time + +from ..ask import ask +from ..context import Ctx +from ..features import periods +from ..history import compact, format_rows +from ..llm.base import BUSY_REPLY, is_busy +from ..llm.gate import is_portfolio_question, route +from ..prompts import chat_prompt, down_note +from ..store import HistoryRow +from ..textfmt import NO_CONTEXT, md_to_html, plain_text, split_html, strip_disclaimer +from .client import SignalClient, SignalError +from .parse import Incoming, parse, unwrap + +log = logging.getLogger("bot") +PORTFOLIO_ADMINS_ONLY = "Portfolio data is for admins only." + + +def signal_chunks(model_text: str) -> list[str]: + """The model's Telegram-HTML answer as plain-text messages (Signal has no HTML), keeping its lines.""" + out = [] + for chunk in split_html(md_to_html(strip_disclaimer(model_text))): + lines = [plain_text(line) for line in chunk.split("\n")] + text = re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip() + if text: + out.append(text) + return out or ["(no response)"] + + +class SignalHandlers: + def __init__(self, ctx: Ctx, client: SignalClient, account: str, own_id: str): + self.ctx, self.client, self.account, self.own_id = ctx, client, account, own_id + + async def on_message(self, obj: dict) -> None: + unwrapped = unwrap(obj) + if unwrapped is None: + return + envelope, account = unwrapped + if account and account not in (self.account, self.own_id): + return # another account of the same daemon + inc = parse(envelope, self.own_id) + if inc is None: + return + await asyncio.to_thread(self.ctx.store.save_signal, self.own_id, inc.chat, + HistoryRow(inc.ts, inc.sender_name, inc.text, inc.ts // 1000, inc.quote_ts)) + if self._is_for_bot(inc): + await self._answer(inc) + + def _is_for_bot(self, inc: Incoming) -> bool: + """DMs from the allow-list; in groups an @mention or a reply to one of the bot's messages.""" + if inc.group: + return inc.mentioned_bot or inc.quote_author == self.own_id + if inc.sender_ids & self.ctx.st.signal_allowed: + return True + log.info("Ignored a Signal DM from %s: not in SIGNAL_ALLOWED_SENDERS", inc.sender) + return False + + async def _answer(self, inc: Incoming) -> None: + text = NO_CONTEXT.sub("", inc.text).strip() if NO_CONTEXT.search(inc.text) else inc.text + preview = " ".join(text.split()) + log.info("[Signal %s%s] %s", inc.sender_name, " (group)" if inc.group else "", + preview[:60].rstrip() + " ..." if len(preview) > 60 else preview) + try: + chunks = await self._generate(inc, text, nocontext=text != inc.text) + except Exception as e: + if is_busy(e): + log.warning("Reply failed, the provider is busy: %s", " ".join(str(e).split())) + chunks = [BUSY_REPLY] + else: + log.exception("Reply failed") + chunks = [f"Sorry, something went wrong:\n{type(e).__name__}: {e}"[:1500]] + await self._send(inc, chunks) + + async def _generate(self, inc: Incoming, text: str, *, nocontext: bool) -> list[str]: + ctx, st = self.ctx, self.ctx.st + admin = bool(inc.sender_ids & st.signal_allowed) # the people trusted with DMs also get portfolio data + if nocontext: # only the message that tagged the bot + rows = [HistoryRow(inc.ts, inc.sender_name, text, inc.ts // 1000, None)] + else: + rows = await asyncio.to_thread(ctx.store.signal_history, self.own_id, inc.chat, st.history_limit) + quote = None if nocontext else self._quote(inc, rows) + gate_text = text + ("" if nocontext else " " + inc.quote_text) + chosen = route(gate_text, st, ctx.registry) + portfolio_up = any(s.cfg.get("gate") == "portfolio" for s in ctx.registry.up()) + if not admin and portfolio_up and is_portfolio_question(gate_text, st): + return [PORTFOLIO_ADMINS_ONLY] + has_portfolio = any(s.cfg.get("gate") == "portfolio" for s in chosen.servers) + extra = [periods.tool(ctx)] if admin and has_portfolio else [] + fresh = has_portfolio and is_portfolio_question(gate_text, st) + prompt = chat_prompt( + transcript=format_rows(rows, ctx.self_name, st.tz, line_max=st.history_line_max, + compact_text=st.token_saver, own_line_max=st.history_own_line_max), + sender=inc.sender_name, private=not inc.group, reply_quote=quote, msg_id=inc.ts, + now=ctx.now(), down=down_note(ctx.registry.down()), saver=st.token_saver, fresh=fresh) + answer = await ask(ctx, [{"type": "text", "text": prompt}], chosen, + cache_key="signal-" + hashlib.sha1(inc.chat.encode()).hexdigest()[:12], + extra_tools=extra, admin=admin) + return signal_chunks(answer.text) + + def _quote(self, inc: Incoming, rows: list[HistoryRow]) -> str | None: + if inc.quote_ts is None: + return None + who = "You" if inc.quote_author == self.own_id else next( + (r.sender for r in rows if r.message_id == inc.quote_ts), "someone") + body = inc.quote_text + return f"[#{inc.quote_ts}] {who}: {compact(body) if self.ctx.st.token_saver else body}" + + async def _send(self, inc: Incoming, chunks: list[str]) -> None: + for i, chunk in enumerate(chunks): + params: dict = {"account": self.account, "message": chunk} + params.update({"groupId": inc.chat} if inc.group else {"recipient": [inc.sender]}) + if inc.group and i == 0: # a group reply quotes the message it answers, as on Telegram + params.update(quoteTimestamp=inc.ts, quoteAuthor=inc.sender, quoteMessage=inc.text[:200]) + try: + try: + result = await self.client.request("send", params) + except SignalError: + if "quoteTimestamp" not in params: + raise + for key in ("quoteTimestamp", "quoteAuthor", "quoteMessage"): + params.pop(key) + result = await self.client.request("send", params) + except SignalError as e: + log.error("Signal: couldn't send a reply: %s", e) + return + sent = int((result or {}).get("timestamp") or time.time() * 1000) + await asyncio.to_thread(self.ctx.store.save_signal, self.own_id, inc.chat, + HistoryRow(sent, self.ctx.self_name, chunk, sent // 1000, inc.ts)) diff --git a/lib/signalbot/parse.py b/lib/signalbot/parse.py new file mode 100644 index 0000000..1f74eab --- /dev/null +++ b/lib/signalbot/parse.py @@ -0,0 +1,65 @@ +"""signal-cli JSON envelopes -> the few facts the bot needs (None for typing, receipts, syncs...).""" + +from __future__ import annotations + +from dataclasses import dataclass + +OBJECT_REPLACEMENT = "" # what Signal puts in the text where a mention goes + + +@dataclass(frozen=True) +class Incoming: + chat: str # a group's ID, or "dm:" for a private chat + group: bool + sender: str # the sender's UUID (or number if there is no UUID) + sender_ids: frozenset[str] # every ID the sender is known by, lowercase (for the allow-list) + sender_name: str + text: str # mentions written out; the bot's own mention removed + ts: int # Signal's message ID: the sender's timestamp in ms + mentioned_bot: bool + quote_ts: int | None + quote_author: str | None + quote_text: str + + +def unwrap(obj: dict) -> tuple[dict, str] | None: + """(envelope, account) from a bare `{"envelope":..}` line or a JSON-RPC `receive` notification.""" + if obj.get("method") == "receive": + obj = obj.get("params") or {} + envelope = obj.get("envelope") + if not isinstance(envelope, dict): + return None + return envelope, str(obj.get("account") or "") + + +def _resolve_mentions(text: str, mentions: list[dict], own_id: str) -> tuple[str, bool]: + """Replace each mention placeholder (from the end, so offsets hold) with @name; the bot's own + mention is dropped.""" + mentioned = False + for m in sorted(mentions, key=lambda m: m.get("start", 0), reverse=True): + start, length = int(m.get("start", 0)), int(m.get("length", 1)) + is_bot = own_id and own_id in (m.get("uuid"), m.get("number")) + mentioned = mentioned or bool(is_bot) + who = "" if is_bot else "@" + str(m.get("name") or m.get("number") or str(m.get("uuid"))[:8]) + text = text[:start] + who + text[start + length:] + return " ".join(text.replace(OBJECT_REPLACEMENT, "").split()) if mentions else text.strip(), mentioned + + +def parse(envelope: dict, own_id: str) -> Incoming | None: + data = envelope.get("dataMessage") + if not isinstance(data, dict) or not data.get("message"): + return None + ids = [envelope.get("sourceUuid"), envelope.get("source"), envelope.get("sourceNumber")] + sender = next((i for i in ids if i), "") + if not sender or sender == own_id: + return None + group = (data.get("groupInfo") or {}).get("groupId") + text, mentioned = _resolve_mentions(data["message"], data.get("mentions") or [], own_id) + quote = data.get("quote") or {} + name = envelope.get("sourceName") or envelope.get("sourceNumber") or str(sender)[:8] + return Incoming( + chat=group or f"dm:{sender}", group=bool(group), sender=sender, + sender_ids=frozenset(str(i).lower() for i in ids if i), sender_name=str(name), + text=text, ts=int(data.get("timestamp") or envelope.get("timestamp") or 0), + mentioned_bot=mentioned, quote_ts=quote.get("id"), quote_author=quote.get("author") + or quote.get("authorUuid"), quote_text=str(quote.get("text") or "")) diff --git a/lib/store.py b/lib/store.py index 08e01d8..745498d 100644 --- a/lib/store.py +++ b/lib/store.py @@ -22,6 +22,11 @@ CREATE TABLE IF NOT EXISTS dm_messages ( bot_id INTEGER, chat_id INTEGER, message_id INTEGER, sender TEXT, text TEXT, ts INTEGER, reply_to INTEGER, PRIMARY KEY (bot_id, chat_id, message_id)); +-- Signal chats, keyed by the bot's account. chat = a group's ID, or "dm:"; message_id = the +-- sender's timestamp in ms (Signal's own message ID). +CREATE TABLE IF NOT EXISTS signal_messages ( + bot TEXT, chat TEXT, message_id INTEGER, sender TEXT, text TEXT, ts INTEGER, reply_to INTEGER, + PRIMARY KEY (bot, chat, message_id)); CREATE TABLE IF NOT EXISTS users (username TEXT PRIMARY KEY, user_id INTEGER); CREATE TABLE IF NOT EXISTS holding_news (username TEXT, ts INTEGER, text TEXT); CREATE TABLE IF NOT EXISTS holding_news_mutes ( @@ -118,16 +123,16 @@ def chats(self) -> list[tuple[int, str]]: return [(r[0], r[1]) for r in self._run("SELECT chat_id, title FROM chats")] def history(self, chat_id: int, limit: int) -> list[HistoryRow]: - """Recent messages, oldest first. + """Recent messages, oldest first. See `_window` for which ones.""" + if is_dm(chat_id): + return self._window("dm_messages", "bot_id = ? AND chat_id = ?", (self._own_id(), chat_id), limit) + return self._window("messages", "chat_id = ?", (chat_id,), limit) - A plain "last N" window drops its oldest line on every message, changing the start of + def _window(self, table: str, where: str, args: tuple, limit: int) -> list[HistoryRow]: + """A plain "last N" window drops its oldest line on every message, changing the start of the prompt and defeating prompt caching. Instead the window's start only moves every limit/2 messages: it holds between `limit` and 1.5 x `limit` messages and in between only grows at the end, so everything before the new messages stays cached.""" - if is_dm(chat_id): - table, where, args = "dm_messages", "bot_id = ? AND chat_id = ?", (self._own_id(), chat_id) - else: - table, where, args = "messages", "chat_id = ?", (chat_id,) total = self._run(f"SELECT COUNT(*) FROM {table} WHERE {where}", args)[0][0] step = max(1, limit // 2) size = total - max(0, (total - limit) // step * step) @@ -136,6 +141,12 @@ def history(self, chat_id: int, limit: int) -> list[HistoryRow]: f"WHERE {where} ORDER BY message_id DESC LIMIT ?", (*args, size)) return [HistoryRow(*r) for r in reversed(rows)] + def save_signal(self, bot: str, chat: str, row: HistoryRow) -> None: + self._run("INSERT OR REPLACE INTO signal_messages VALUES (?,?,?,?,?,?,?)", (bot, chat, *row)) + + def signal_history(self, bot: str, chat: str, limit: int) -> list[HistoryRow]: + return self._window("signal_messages", "bot = ? AND chat = ?", (bot, chat), limit) + def dm_chats(self) -> list[int]: """Everyone this bot has a private chat with.""" rows = self._run("SELECT DISTINCT chat_id FROM dm_messages WHERE bot_id = ?", (self._own_id(),)) diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index 583db15..ddb268b 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -22,7 +22,7 @@ from ..msgtext import describe, sender_name from ..prompts import MOVERS_EXPLAIN, PRESET_PROMPT, chat_prompt, down_note from ..store import is_dm -from ..textfmt import md_to_html, strip_disclaimer +from ..textfmt import NO_CONTEXT, md_to_html, strip_disclaimer from ..tickers import link_tickers from .access import is_admin from .draft import Draft, start_typing @@ -33,7 +33,6 @@ FOLLOW_UP_WORDS = 12 # a message this short right after the bot's reply may be a follow-up to it FOLLOW_UP_SECONDS = 30 * 60 FOLLOW_UP_CHARS = 400 # of the bot's reply that the tool gate also reads -NO_CONTEXT = re.compile(r"[ \t]*(?]*>|]*$", re.I) +# `#nocontext` in a message: the model gets that message alone (no history, nothing it replies to). +NO_CONTEXT = re.compile(r"[ \t]*(?NVDA up 3%\nAMD (src) & more\n\n\n\nbye") + assert out == ["NVDA up 3%\nAMD (src (https://x.test/a)) & more\n\nbye"] + + +# ---- the reply flow ------------------------------------------------------------------- +class FakeSignal: + def __init__(self): + self.requests = [] + self.fail_quote = False + + async def request(self, method, params): + self.requests.append((method, dict(params))) + if self.fail_quote and "quoteTimestamp" in params: + raise SignalError("bad quote") + return {"timestamp": 5_000_000 + len(self.requests)} + + +def make(env, store, script, **extra): + st = config.load({**env, "SIGNAL_ALLOWED_SENDERS": ROB.upper(), **extra}) + backend, client = ScriptedBackend(script), FakeSignal() + ctx = Ctx(st, store, backend, registry(), FakeBot()) + return ctx, backend, client, SignalHandlers(ctx, client, BOT, BOT) + + +async def test_group_is_answered_on_mention_or_reply_to_the_bot_only(env, store): + ctx, backend, client, h = make(env, store, [step("NVDA is up"), step("AMD too")]) + await h.on_message(envelope(group_data("lunch anyone?", mentions=[]), ts=1_000)) + assert client.requests == [] # logged as context, not answered + await h.on_message(envelope(group_data("\ufffc how is NVDA?", mentions=[mention()]), ts=2_000)) + method, params = client.requests[0] + assert method == "send" and params["groupId"] == GROUP and params["message"] == "NVDA is up" + assert params["account"] == BOT and params["quoteTimestamp"] == 2_000 and params["quoteAuthor"] == ROB + assert "lunch anyone?" in backend.seen[0]["prompt"] and "how is NVDA?" in backend.seen[0]["prompt"] + quote = {"id": 5_000_001, "author": BOT, "text": "NVDA is up"} + await h.on_message(envelope(group_data("and AMD?", quote=quote), ts=3_000)) + assert client.requests[1][1]["message"] == "AMD too" + rows = store.signal_history(BOT, GROUP, 20) + assert [r.sender for r in rows].count(ctx.self_name) == 2 # the bot's replies are in its history, as "You" + assert "[#5000001] You: NVDA is up" in backend.seen[1]["prompt"] + + +async def test_dm_needs_an_allowed_sender_and_goes_to_that_person(env, store): + ctx, backend, client, h = make(env, store, [step("hello Alex")]) + stranger = envelope({"message": "hi bot"}, source="dddddddd-0000-4000-8000-000000000004") + await h.on_message(stranger) + assert client.requests == [] and backend.seen == [] + await h.on_message(envelope({"message": "hi bot"})) + method, params = client.requests[0] + assert params["recipient"] == [ROB] and "groupId" not in params and "quoteTimestamp" not in params + assert "Private one-to-one chat with Alex" in backend.seen[0]["prompt"] + + +async def test_a_rejected_quote_is_retried_without_it(env, store): + ctx, backend, client, h = make(env, store, [step("ok")]) + client.fail_quote = True + await h.on_message(envelope(group_data("\ufffc hi", mentions=[mention()]))) + assert len(client.requests) == 2 and "quoteTimestamp" not in client.requests[1][1] + + +async def test_nocontext_sends_the_message_alone_without_the_tag(env, store): + ctx, backend, client, h = make(env, store, [step("fine")]) + await h.on_message(envelope(group_data("the football was great"), ts=1_000)) + await h.on_message(envelope(group_data("\ufffc #nocontext what about NVDA?", mentions=[mention()]), ts=2_000)) + prompt = backend.seen[0]["prompt"] + assert "NVDA" in prompt and "football" not in prompt and "nocontext" not in prompt.lower() + + +async def test_portfolio_questions_are_for_allowed_senders_only(env, store): + from .fakes import FakeMcp + ctx, backend, client, h = make(env, store, [step("never asked")]) + ctx.registry = registry(FakeMcp("sharesight", gate="portfolio", tools=("list_portfolios",))) + stranger = "dddddddd-0000-4000-8000-000000000004" # in the group, but not on the allow-list + await h.on_message(envelope(group_data("\ufffc how is my portfolio doing?", mentions=[mention()]), source=stranger)) + assert client.requests[0][1]["message"] == "Portfolio data is for admins only." and backend.seen == [] + + +# ---- the daemon ----------------------------------------------------------------------- +FAKE_DAEMON = """#!{python} +import asyncio, json, sys +args = sys.argv[1:] +if "listAccounts" in args: + print(json.dumps([{{"number": None, "aci": "{bot}"}}])) + sys.exit(0) +sock = args[args.index("--socket") + 1] + +async def handle(reader, writer): + note = {{"jsonrpc": "2.0", "method": "receive", "params": json.loads({envelope!r})}} + writer.write((json.dumps(note) + "\\n").encode()) + await writer.drain() + async for line in reader: + req = json.loads(line) + if req["method"] == "boom": + reply = {{"jsonrpc": "2.0", "id": req["id"], "error": {{"code": -1, "message": "no good"}}}} + else: + reply = {{"jsonrpc": "2.0", "id": req["id"], "result": {{"timestamp": 1234}}}} + writer.write((json.dumps(reply) + "\\n").encode()) + await writer.drain() + +async def main(): + server = await asyncio.start_unix_server(handle, sock) + async with server: + await server.serve_forever() + +asyncio.run(main()) +""" + + +@pytest.fixture +def fake_signal_cli(tmp_path): + path = tmp_path / "signal-cli" + path.write_text(FAKE_DAEMON.format(python=sys.executable, bot=BOT, envelope=json.dumps(envelope({"message": "hi"})))) + path.chmod(path.stat().st_mode | stat.S_IEXEC) + return str(path) + + +async def test_accounts_are_listed(fake_signal_cli): + assert await list_accounts(fake_signal_cli) == [{"number": None, "aci": BOT}] + + +async def test_client_receives_notifications_and_sends_requests(fake_signal_cli): + got = asyncio.Queue() + + async def on_message(obj): + await got.put(obj) + + client = SignalClient(fake_signal_cli, on_message) + task = asyncio.create_task(client.run()) + try: + obj = await asyncio.wait_for(got.get(), 20) + assert parse(unwrap(obj)[0], BOT).text == "hi" + assert await client.request("send", {"message": "x"}) == {"timestamp": 1234} + with pytest.raises(SignalError, match="no good"): + await client.request("boom", {}) + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + with pytest.raises(SignalError, match="isn't connected"): + await client.request("send", {}) From 15d2d632d2246f03bccc8da7b8e5937e5266e6bf Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 14:55:27 +0000 Subject: [PATCH 24/27] Rename SIGNALCLI_PATH to SIGNALCLI_BIN Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- CLAUDE.md | 2 +- README.md | 6 +++--- lib/config.py | 4 ++-- lib/signalbot/__init__.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f3690f0..3f98e3c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,7 +27,7 @@ MCP data tools and web search. User-facing docs are in `README.md`; this file is - `prompts.py` all prompts. `ask.py` one entry point: route -> Request -> policy.ask -> usage -> NEEDS_TOOLS rerun. - `telegram/`: `handlers.py` (trigger -> request -> reply), `send.py`, `draft.py` (typing + streaming drafts), `commands.py` (/credits), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop). -- `signalbot/` (SIGNALCLI_PATH): `client.py` runs `signal-cli daemon --socket` and speaks JSON-RPC to it, `parse.py` envelope -> `Incoming`, +- `signalbot/` (SIGNALCLI_BIN): `client.py` runs `signal-cli daemon --socket` and speaks JSON-RPC to it, `parse.py` envelope -> `Incoming`, `handlers.py` the Signal reply flow (own `signal_messages` history; no Telegram types). Reuses `ask`, `gate`, `prompts`, `history`. - `app.py` wiring/startup; `context.py` `Ctx` (settings, store, backend, registry, bot). - `lib/` holds only what the bot needs to run. Developer helpers live in `tools/`: `report.py` (`make prompt-report`), diff --git a/README.md b/README.md index 2933e51..d09c118 100644 --- a/README.md +++ b/README.md @@ -179,12 +179,12 @@ Each is off by default. The holding-news DM and the movers reply are switched on matches none of your groups the reply is the same whether or not the group exists. ## Signal -Off by default. Set `SIGNALCLI_PATH` to a [signal-cli](https://github.com/AsamK/signal-cli) executable that already has an +Off by default. Set `SIGNALCLI_BIN` to a [signal-cli](https://github.com/AsamK/signal-cli) executable that already has an account (registered, or linked as a secondary device) and the bot answers on Signal as well as Telegram, with the same model, tools, search and history. ``` -SIGNALCLI_PATH=/opt/signal-cli/bin/signal-cli +SIGNALCLI_BIN=/opt/signal-cli/bin/signal-cli SIGNAL_ALLOWED_SENDERS=aaaaaaaa-0000-4000-8000-000000000000,+61400000000 ``` - **How it runs:** at startup the bot runs `signal-cli -o json listAccounts` to learn its own ID (the first account is @@ -327,7 +327,7 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. | `SEARCH_MODEL` | OpenRouter only: model that runs the searches (default xiaomi/mimo-v2.6-flash:online). | | `SHARESIGHT_HOLDING_NEWS_RECIPIENTS` | Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off. | | `SHARESIGHT_HOLDING_NEWS_TIME` | HH:MM (BOT_TZ) for the daily holding-news check (default 08:00). | -| `SIGNALCLI_PATH` | Path to the `signal-cli` executable. Setting it turns [Signal](#signal) on: the bot starts `signal-cli daemon` itself. Empty (default) = Signal off. | +| `SIGNALCLI_BIN` | Path to the `signal-cli` executable. Setting it turns [Signal](#signal) on: the bot starts `signal-cli daemon` itself. Empty (default) = Signal off. | | `SIGNAL_ALLOWED_SENDERS` | Signal UUIDs or phone numbers (comma/space separated) that may DM the bot and get Sharesight data on Signal. Empty (default) = nobody. | | `TELEGRAM_DM_BUTTONS` | on|off. Preset-prompt buttons in private chats (default off). | | `TOKEN_SAVER` | on|off. Master switch for the token-saving heuristics (default on). | diff --git a/lib/config.py b/lib/config.py index e9c7dbe..11156ec 100644 --- a/lib/config.py +++ b/lib/config.py @@ -39,7 +39,7 @@ "SEARCH_MODEL": "OpenRouter only: model that runs the searches (default xiaomi/mimo-v2.6-flash:online).", "SHARESIGHT_HOLDING_NEWS_RECIPIENTS": "Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off.", "SHARESIGHT_HOLDING_NEWS_TIME": "HH:MM (BOT_TZ) for the daily holding-news check (default 08:00).", - "SIGNALCLI_PATH": "Path to the signal-cli executable. Setting it turns Signal on: the bot starts `signal-cli daemon` itself, answers DMs from SIGNAL_ALLOWED_SENDERS and group @mentions or replies. Empty (default) = Signal off.", + "SIGNALCLI_BIN": "Path to the signal-cli executable. Setting it turns Signal on: the bot starts `signal-cli daemon` itself, answers DMs from SIGNAL_ALLOWED_SENDERS and group @mentions or replies. Empty (default) = Signal off.", "SIGNAL_ALLOWED_SENDERS": "Signal UUIDs or phone numbers (comma/space separated) that may DM the bot and get Sharesight data on Signal. Empty (default) = nobody.", "TELEGRAM_DM_BUTTONS": "on|off. Preset-prompt buttons in private chats (default off).", "TOKEN_SAVER": "on|off. Master switch for the token-saving heuristics (default on).", @@ -198,7 +198,7 @@ def load(env: Mapping[str, str] | None = None) -> Settings: max_tokens=int(env.get("MAX_TOKENS") or 3000), history_limit=int(env.get("HISTORY_LIMIT") or 20), log_width=int(env.get("LOG_WIDTH") or 95), - signal_cli=env.get("SIGNALCLI_PATH", "").strip(), + signal_cli=env.get("SIGNALCLI_BIN", "").strip(), signal_allowed=frozenset(x.lower() for x in re.split(r"[,\s]+", env.get("SIGNAL_ALLOWED_SENDERS", "")) if x), db_path=data_path(env, "DB_PATH", "chat_log.db"), tz=ZoneInfo(env.get("BOT_TZ", "UTC")), diff --git a/lib/signalbot/__init__.py b/lib/signalbot/__init__.py index 11d92c4..82721fb 100644 --- a/lib/signalbot/__init__.py +++ b/lib/signalbot/__init__.py @@ -1,4 +1,4 @@ -"""SIGNALCLI_PATH: answer Signal DMs and group mentions through a signal-cli daemon the bot runs itself.""" +"""SIGNALCLI_BIN: answer Signal DMs and group mentions through a signal-cli daemon the bot runs itself.""" from __future__ import annotations From 08f9a76840046ed50dca0179bf2165ff4004ebd2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 15:18:00 +0000 Subject: [PATCH 25/27] Signal: connect to a daemon you run yourself (SIGNALCLI_SOCKET) The bot then never starts signal-cli: it connects to the socket, retrying with a growing pause, and asks the daemon for its account over JSON-RPC. SIGNALCLI_BIN keeps working and loses to the socket when both are set. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- CLAUDE.md | 2 +- README.md | 32 +++++++++++++++-------- lib/app.py | 2 +- lib/config.py | 5 +++- lib/signalbot/__init__.py | 47 +++++++++++++++++++--------------- lib/signalbot/client.py | 54 ++++++++++++++++++++++++++++++--------- lib/signalbot/handlers.py | 26 ++++++++++++++++++- tests/test_signal.py | 49 +++++++++++++++++++++++++++++++++++ 8 files changed, 171 insertions(+), 46 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3f98e3c..eb7fbe1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,7 +27,7 @@ MCP data tools and web search. User-facing docs are in `README.md`; this file is - `prompts.py` all prompts. `ask.py` one entry point: route -> Request -> policy.ask -> usage -> NEEDS_TOOLS rerun. - `telegram/`: `handlers.py` (trigger -> request -> reply), `send.py`, `draft.py` (typing + streaming drafts), `commands.py` (/credits), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop). -- `signalbot/` (SIGNALCLI_BIN): `client.py` runs `signal-cli daemon --socket` and speaks JSON-RPC to it, `parse.py` envelope -> `Incoming`, +- `signalbot/` (SIGNALCLI_BIN or SIGNALCLI_SOCKET): `client.py` runs `signal-cli daemon --socket` (or connects to one you run) and speaks JSON-RPC to it, `parse.py` envelope -> `Incoming`, `handlers.py` the Signal reply flow (own `signal_messages` history; no Telegram types). Reuses `ask`, `gate`, `prompts`, `history`. - `app.py` wiring/startup; `context.py` `Ctx` (settings, store, backend, registry, bot). - `lib/` holds only what the bot needs to run. Developer helpers live in `tools/`: `report.py` (`make prompt-report`), diff --git a/README.md b/README.md index d09c118..b999cac 100644 --- a/README.md +++ b/README.md @@ -179,18 +179,29 @@ Each is off by default. The holding-news DM and the movers reply are switched on matches none of your groups the reply is the same whether or not the group exists. ## Signal -Off by default. Set `SIGNALCLI_BIN` to a [signal-cli](https://github.com/AsamK/signal-cli) executable that already has an -account (registered, or linked as a secondary device) and the bot answers on Signal as well as Telegram, with the same -model, tools, search and history. +Off by default. The bot answers on Signal as well as Telegram, with the same model, tools, search and history, through a +[signal-cli](https://github.com/AsamK/signal-cli) account that is already registered (or linked as a secondary device). +There are two ways to run signal-cli; set one of: + +- **`SIGNALCLI_BIN=/opt/signal-cli/bin/signal-cli`: the bot runs the daemon.** At startup it runs + `signal-cli -o json listAccounts` to learn its own ID (the first account is used), then starts + `signal-cli -o json daemon --socket ` and keeps it running, restarting it if it stops. +- **`SIGNALCLI_SOCKET=/run/signal-cli/socket`: you run the daemon.** The bot only connects (retrying with a growing + pause if the daemon is down or restarts) and asks it for the account with a JSON-RPC `listAccounts`. Start the daemon + in multi-account mode, without `-a`: `signal-cli -o json daemon --socket /run/signal-cli/socket`. This wins over + `SIGNALCLI_BIN`. It suits a bot under systemd with `DynamicUser=`, `ProtectHome=` and tight limits, where it is easier + to run signal-cli as your own user in its own unit. ``` -SIGNALCLI_BIN=/opt/signal-cli/bin/signal-cli SIGNAL_ALLOWED_SENDERS=aaaaaaaa-0000-4000-8000-000000000000,+61400000000 ``` -- **How it runs:** at startup the bot runs `signal-cli -o json listAccounts` to learn its own ID (the first account is - used), then starts `signal-cli -o json daemon --socket ` itself and keeps it running, restarting it - if it stops. Messages arrive, and replies go out, over the daemon's JSON-RPC socket. Don't run another signal-cli - daemon on the same account. +Either way messages arrive, and replies go out, over the daemon's JSON-RPC socket, and nothing else may run a daemon +on the same account. + +- **Socket access:** anyone who can write to the socket can send as the account, so keep it in a directory only the + two services share. For example a group `signal-cli`; a directory made by `systemd-tmpfiles` + (`d /run/signal-cli 0750 signal-cli -`); the daemon started with `UMask=0007` (or `umask 007`) so the socket is + group-writable; and `SupplementaryGroups=signal-cli` in the bot's unit. - **DMs:** answered only from the people in `SIGNAL_ALLOWED_SENDERS` (UUIDs or phone numbers, as signal-cli reports them); anyone else is logged and ignored. Empty = nobody. These people are also the Signal "admins": only they get Sharesight portfolio data. @@ -202,8 +213,8 @@ SIGNAL_ALLOWED_SENDERS=aaaaaaaa-0000-4000-8000-000000000000,+61400000000 tickers are not linked. - **Not on Signal:** images and other attachments (a message with no text is ignored), the typing indicator and streamed drafts, movers lists, the daily holding-news DM, DM buttons, `/credits` and `/flushcache`. -- **Checking:** the log shows `Signal: connected to signal-cli` once the daemon is up, and each message as - `[Signal (group)] ...`. +- **Checking:** the log shows `Signal: connected to signal-cli` and `Signal: answering as account ...` once the + daemon is reachable, and each message as `[Signal (group)] ...`. ## Commands | Command | Who | What | @@ -328,6 +339,7 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. | `SHARESIGHT_HOLDING_NEWS_RECIPIENTS` | Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off. | | `SHARESIGHT_HOLDING_NEWS_TIME` | HH:MM (BOT_TZ) for the daily holding-news check (default 08:00). | | `SIGNALCLI_BIN` | Path to the `signal-cli` executable. Setting it turns [Signal](#signal) on: the bot starts `signal-cli daemon` itself. Empty (default) = Signal off. | +| `SIGNALCLI_SOCKET` | Path of the JSON-RPC socket of a `signal-cli daemon` you run yourself. Turns [Signal](#signal) on, and wins over `SIGNALCLI_BIN`: the bot connects instead of starting a daemon. | | `SIGNAL_ALLOWED_SENDERS` | Signal UUIDs or phone numbers (comma/space separated) that may DM the bot and get Sharesight data on Signal. Empty (default) = nobody. | | `TELEGRAM_DM_BUTTONS` | on|off. Preset-prompt buttons in private chats (default off). | | `TOKEN_SAVER` | on|off. Master switch for the token-saving heuristics (default on). | diff --git a/lib/app.py b/lib/app.py index e16e0ec..49f9885 100644 --- a/lib/app.py +++ b/lib/app.py @@ -116,7 +116,7 @@ async def post_init(app: Application) -> None: background.append(asyncio.create_task(dm_buttons.push_on_startup(ctx))) if st.period_backfill: background.append(asyncio.create_task(backfill.loop(ctx))) - if st.signal_cli: + if st.signal_cli or st.signal_socket: background.append(asyncio.create_task(signalbot.run(ctx))) async def post_shutdown(app: Application) -> None: diff --git a/lib/config.py b/lib/config.py index 11156ec..04b537c 100644 --- a/lib/config.py +++ b/lib/config.py @@ -40,6 +40,7 @@ "SHARESIGHT_HOLDING_NEWS_RECIPIENTS": "Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off.", "SHARESIGHT_HOLDING_NEWS_TIME": "HH:MM (BOT_TZ) for the daily holding-news check (default 08:00).", "SIGNALCLI_BIN": "Path to the signal-cli executable. Setting it turns Signal on: the bot starts `signal-cli daemon` itself, answers DMs from SIGNAL_ALLOWED_SENDERS and group @mentions or replies. Empty (default) = Signal off.", + "SIGNALCLI_SOCKET": "Path of the JSON-RPC socket of a signal-cli daemon you run yourself (`signal-cli -o json daemon --socket PATH`). Turns Signal on and takes priority over SIGNALCLI_BIN: the bot connects instead of starting a daemon.", "SIGNAL_ALLOWED_SENDERS": "Signal UUIDs or phone numbers (comma/space separated) that may DM the bot and get Sharesight data on Signal. Empty (default) = nobody.", "TELEGRAM_DM_BUTTONS": "on|off. Preset-prompt buttons in private chats (default off).", "TOKEN_SAVER": "on|off. Master switch for the token-saving heuristics (default on).", @@ -79,7 +80,8 @@ class Settings: max_tokens: int history_limit: int log_width: int - signal_cli: str # path to signal-cli; empty = Signal off + signal_cli: str # path to signal-cli, which the bot runs as a daemon; empty = not that way + signal_socket: str # socket of a daemon the bot only connects to; either one turns Signal on signal_allowed: frozenset[str] # lowercase UUIDs/numbers that may DM the bot (and get Sharesight data) db_path: str tz: ZoneInfo @@ -199,6 +201,7 @@ def load(env: Mapping[str, str] | None = None) -> Settings: history_limit=int(env.get("HISTORY_LIMIT") or 20), log_width=int(env.get("LOG_WIDTH") or 95), signal_cli=env.get("SIGNALCLI_BIN", "").strip(), + signal_socket=env.get("SIGNALCLI_SOCKET", "").strip(), signal_allowed=frozenset(x.lower() for x in re.split(r"[,\s]+", env.get("SIGNAL_ALLOWED_SENDERS", "")) if x), db_path=data_path(env, "DB_PATH", "chat_log.db"), tz=ZoneInfo(env.get("BOT_TZ", "UTC")), diff --git a/lib/signalbot/__init__.py b/lib/signalbot/__init__.py index 82721fb..c4f7d59 100644 --- a/lib/signalbot/__init__.py +++ b/lib/signalbot/__init__.py @@ -1,4 +1,4 @@ -"""SIGNALCLI_BIN: answer Signal DMs and group mentions through a signal-cli daemon the bot runs itself.""" +"""SIGNALCLI_BIN / SIGNALCLI_SOCKET: answer Signal DMs and group mentions through a signal-cli daemon.""" from __future__ import annotations @@ -12,23 +12,30 @@ async def run(ctx: Ctx) -> None: - """Find the bot's Signal account, then keep the daemon running until cancelled.""" - path = ctx.st.signal_cli - try: - accounts = await list_accounts(path) - except Exception as e: - log.error("Signal is off: couldn't list accounts with %s: %s", path, e) - return - if not accounts: - log.error("Signal is off: signal-cli has no registered account") - return - if len(accounts) > 1: - log.warning("Signal: %d accounts registered; using the first", len(accounts)) - number, aci = accounts[0].get("number"), accounts[0].get("aci") or accounts[0].get("uuid") - if not aci: - log.error("Signal is off: the account has no ACI (listAccounts gave %s)", accounts[0]) - return - client = SignalClient(path) - client.on_message = SignalHandlers(ctx, client, number or aci, aci).on_message - log.info("Signal: starting for account %s", aci) + """Connect to the daemon at SIGNALCLI_SOCKET, or start one from SIGNALCLI_BIN, until cancelled.""" + st = ctx.st + if st.signal_socket: + client = SignalClient(socket_path=st.signal_socket) + handlers = SignalHandlers(ctx, client) + client.on_connect = handlers.identify # asks the daemon whose account it is + log.info("Signal: connecting to the signal-cli daemon at %s", st.signal_socket) + else: + try: + accounts = await list_accounts(st.signal_cli) + except Exception as e: + log.error("Signal is off: couldn't list accounts with %s: %s", st.signal_cli, e) + return + if not accounts: + log.error("Signal is off: signal-cli has no registered account") + return + if len(accounts) > 1: + log.warning("Signal: %d accounts registered; using the first", len(accounts)) + number, aci = accounts[0].get("number"), accounts[0].get("aci") or accounts[0].get("uuid") + if not aci: + log.error("Signal is off: the account has no ACI (listAccounts gave %s)", accounts[0]) + return + client = SignalClient(st.signal_cli) + handlers = SignalHandlers(ctx, client, number or aci, aci) + log.info("Signal: starting for account %s", aci) + client.on_message = handlers.on_message await client.run() diff --git a/lib/signalbot/client.py b/lib/signalbot/client.py index 7861c0d..40c1904 100644 --- a/lib/signalbot/client.py +++ b/lib/signalbot/client.py @@ -35,9 +35,14 @@ async def list_accounts(path: str) -> list[dict]: class SignalClient: - def __init__(self, path: str, on_message: Callable[[dict], Awaitable[None]] | None = None): + """With `socket_path`, connects to a daemon someone else runs; otherwise starts `path daemon` itself.""" + + def __init__(self, path: str = "", on_message: Callable[[dict], Awaitable[None]] | None = None, *, + socket_path: str = ""): self.path = path + self.socket_path = socket_path self.on_message = on_message + self.on_connect: Callable[[], Awaitable[None]] | None = None # runs after every (re)connection self._writer: asyncio.StreamWriter | None = None self._pending: dict[int, asyncio.Future] = {} self._ids = itertools.count(1) @@ -58,12 +63,16 @@ async def run(self) -> None: if not fut.done(): fut.set_exception(SignalError("signal-cli stopped")) self._pending.clear() - log.warning("Signal: signal-cli stopped; starting it again in %d s", delay) + log.warning("Signal: lost signal-cli; trying again in %d s", delay) await asyncio.sleep(delay) delay = min(delay * 2, 60) async def _session(self) -> bool: - """One daemon's lifetime. True if it got as far as connecting.""" + """One connection's lifetime. True if it got as far as connecting.""" + if self.socket_path: + reader, writer = await self._connect(self.socket_path, None, wait=5) + await self._serve(reader, writer) + return True sock_dir = Path(tempfile.mkdtemp(prefix="signal-")) sock = str(sock_dir / "socket") proc = await asyncio.create_subprocess_exec( @@ -73,13 +82,10 @@ async def _session(self) -> bool: connected = False try: reader, writer = await self._connect(sock, proc) - connected, self._writer = True, writer - log.info("Signal: connected to signal-cli") - async for line in reader: - self._dispatch(line) + connected = True + await self._serve(reader, writer) return connected finally: - self._writer = None if proc.returncode is None: proc.terminate() with contextlib.suppress(Exception): @@ -89,15 +95,39 @@ async def _session(self) -> bool: pump.cancel() shutil.rmtree(sock_dir, ignore_errors=True) - async def _connect(self, sock: str, proc) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]: - for _ in range(CONNECT_WAIT * 2): - if proc.returncode is not None: + async def _serve(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + """Read the daemon's lines until it closes the connection.""" + self._writer = writer + log.info("Signal: connected to signal-cli") + hook = asyncio.create_task(self._on_connect()) if self.on_connect else None + try: + async for line in reader: + self._dispatch(line) + finally: + self._writer = None + if hook: + hook.cancel() + writer.close() + + async def _on_connect(self) -> None: + try: + await self.on_connect() + except Exception as e: + log.error("Signal: %s", e) + + async def _connect(self, sock: str, proc, *, wait: int = CONNECT_WAIT + ) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]: + for _ in range(wait * 2): + if proc is not None and proc.returncode is not None: raise RuntimeError(f"signal-cli exited with code {proc.returncode} before it was ready") try: return await asyncio.open_unix_connection(sock, limit=LINE_LIMIT) except (FileNotFoundError, ConnectionRefusedError): await asyncio.sleep(0.5) - raise RuntimeError(f"signal-cli didn't open its socket within {CONNECT_WAIT} s") + except PermissionError as e: + raise RuntimeError(f"can't open {sock}: {e}. The bot's user needs write access to the socket " + "and to the directory it is in") from e + raise RuntimeError(f"couldn't connect to signal-cli's socket {sock} within {wait} s") async def _pump_stderr(self, proc) -> None: async for line in proc.stderr: diff --git a/lib/signalbot/handlers.py b/lib/signalbot/handlers.py index 656fdf5..359124c 100644 --- a/lib/signalbot/handlers.py +++ b/lib/signalbot/handlers.py @@ -36,13 +36,37 @@ def signal_chunks(model_text: str) -> list[str]: class SignalHandlers: - def __init__(self, ctx: Ctx, client: SignalClient, account: str, own_id: str): + def __init__(self, ctx: Ctx, client: SignalClient, account: str = "", own_id: str = ""): + """`account`/`own_id` (what requests name, and the bot's own ACI) are known up front when the bot + started the daemon; with a daemon it only connects to, `identify` asks for them.""" self.ctx, self.client, self.account, self.own_id = ctx, client, account, own_id + self.ready = asyncio.Event() + if own_id: + self.ready.set() + + async def identify(self) -> None: + """Learn the bot's own account from the daemon (the first one, if it runs several).""" + accounts = await self.client.request("listAccounts", {}) + if not isinstance(accounts, list) or not accounts: + raise SignalError("the daemon has no registered account") + if len(accounts) > 1: + log.warning("Signal: %d accounts registered; using the first", len(accounts)) + number, aci = accounts[0].get("number"), accounts[0].get("aci") or accounts[0].get("uuid") + if not aci: + raise SignalError(f"the account has no ACI (listAccounts gave {accounts[0]})") + self.account, self.own_id = number or aci, aci + self.ready.set() + log.info("Signal: answering as account %s", aci) async def on_message(self, obj: dict) -> None: unwrapped = unwrap(obj) if unwrapped is None: return + try: + await asyncio.wait_for(self.ready.wait(), 30) + except TimeoutError: + log.warning("Signal: dropped a message, the account isn't known yet") + return envelope, account = unwrapped if account and account not in (self.account, self.own_id): return # another account of the same daemon diff --git a/tests/test_signal.py b/tests/test_signal.py index 6e529bd..855c0b8 100644 --- a/tests/test_signal.py +++ b/tests/test_signal.py @@ -79,8 +79,12 @@ def __init__(self): self.requests = [] self.fail_quote = False + accounts: list = [] + async def request(self, method, params): self.requests.append((method, dict(params))) + if method == "listAccounts": + return self.accounts if self.fail_quote and "quoteTimestamp" in params: raise SignalError("bad quote") return {"timestamp": 5_000_000 + len(self.requests)} @@ -162,6 +166,8 @@ async def handle(reader, writer): req = json.loads(line) if req["method"] == "boom": reply = {{"jsonrpc": "2.0", "id": req["id"], "error": {{"code": -1, "message": "no good"}}}} + elif req["method"] == "listAccounts": + reply = {{"jsonrpc": "2.0", "id": req["id"], "result": [{{"number": None, "aci": "{bot}"}}]}} else: reply = {{"jsonrpc": "2.0", "id": req["id"], "result": {{"timestamp": 1234}}}} writer.write((json.dumps(reply) + "\\n").encode()) @@ -184,6 +190,49 @@ def fake_signal_cli(tmp_path): return str(path) +async def test_a_daemon_you_run_yourself_is_connected_to_and_identified(fake_signal_cli, tmp_path, env, store): + sock = str(tmp_path / "ext.sock") + daemon = await asyncio.create_subprocess_exec(fake_signal_cli, "daemon", "--socket", sock) + got = asyncio.Queue() + client = SignalClient(socket_path=sock) + ctx = Ctx(config.load({**env, "SIGNALCLI_SOCKET": sock}), store, ScriptedBackend([]), registry(), FakeBot()) + assert ctx.st.signal_socket == sock and ctx.st.signal_cli == "" + handlers = SignalHandlers(ctx, client) # doesn't know its account yet + client.on_message = lambda obj: got.put(obj) + client.on_connect = handlers.identify + task = asyncio.create_task(client.run()) + try: + assert parse(unwrap(await asyncio.wait_for(got.get(), 20))[0], BOT).text == "hi" + await asyncio.wait_for(handlers.ready.wait(), 20) + assert (handlers.account, handlers.own_id) == (BOT, BOT) + assert await client.request("send", {"message": "x"}) == {"timestamp": 1234} + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + daemon.terminate() + await daemon.wait() + + +async def test_messages_wait_until_the_account_is_known(env, store): + ctx, backend, client, _ = make(env, store, [step("hello")]) + h = SignalHandlers(ctx, client) + task = asyncio.create_task(h.on_message(envelope({"message": "hi bot"}))) + await asyncio.sleep(0.05) + assert not task.done() and client.requests == [] + client.accounts = [{"number": "+61400000000", "aci": BOT}] + await h.identify() + await asyncio.wait_for(task, 5) + assert h.account == "+61400000000" and h.own_id == BOT and client.requests[-1][1]["message"] == "hello" + assert client.requests[0] == ("listAccounts", {}) and client.requests[-1][1]["account"] == "+61400000000" + + +async def test_a_missing_socket_is_reported_not_waited_on_forever(tmp_path): + client = SignalClient(socket_path=str(tmp_path / "nope")) + with pytest.raises(RuntimeError, match="couldn't connect to signal-cli's socket"): + await client._connect(client.socket_path, None, wait=1) + + async def test_accounts_are_listed(fake_signal_cli): assert await list_accounts(fake_signal_cli) == [{"number": None, "aci": BOT}] From af5333a7cab47d08f410fc69870e8210c6abe9df Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 15:22:58 +0000 Subject: [PATCH 26/27] Signal: remove the code that ran signal-cli; SIGNALCLI_SOCKET is the only setting The bot only connects to the socket of a daemon you run. SIGNALCLI_BIN, the daemon subprocess and the CLI listAccounts call are gone. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- CLAUDE.md | 2 +- README.md | 26 ++++++------- lib/app.py | 2 +- lib/config.py | 7 +--- lib/signalbot/__init__.py | 34 ++++------------- lib/signalbot/client.py | 77 +++++++-------------------------------- lib/signalbot/handlers.py | 9 ++--- tests/test_signal.py | 54 ++++++++------------------- 8 files changed, 55 insertions(+), 156 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index eb7fbe1..dfef60f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,7 +27,7 @@ MCP data tools and web search. User-facing docs are in `README.md`; this file is - `prompts.py` all prompts. `ask.py` one entry point: route -> Request -> policy.ask -> usage -> NEEDS_TOOLS rerun. - `telegram/`: `handlers.py` (trigger -> request -> reply), `send.py`, `draft.py` (typing + streaming drafts), `commands.py` (/credits), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop). -- `signalbot/` (SIGNALCLI_BIN or SIGNALCLI_SOCKET): `client.py` runs `signal-cli daemon --socket` (or connects to one you run) and speaks JSON-RPC to it, `parse.py` envelope -> `Incoming`, +- `signalbot/` (SIGNALCLI_SOCKET): `client.py` connects to the socket of a signal-cli daemon you run and speaks JSON-RPC to it, `parse.py` envelope -> `Incoming`, `handlers.py` the Signal reply flow (own `signal_messages` history; no Telegram types). Reuses `ask`, `gate`, `prompts`, `history`. - `app.py` wiring/startup; `context.py` `Ctx` (settings, store, backend, registry, bot). - `lib/` holds only what the bot needs to run. Developer helpers live in `tools/`: `report.py` (`make prompt-report`), diff --git a/README.md b/README.md index b999cac..5d03179 100644 --- a/README.md +++ b/README.md @@ -181,27 +181,24 @@ Each is off by default. The holding-news DM and the movers reply are switched on ## Signal Off by default. The bot answers on Signal as well as Telegram, with the same model, tools, search and history, through a [signal-cli](https://github.com/AsamK/signal-cli) account that is already registered (or linked as a secondary device). -There are two ways to run signal-cli; set one of: - -- **`SIGNALCLI_BIN=/opt/signal-cli/bin/signal-cli`: the bot runs the daemon.** At startup it runs - `signal-cli -o json listAccounts` to learn its own ID (the first account is used), then starts - `signal-cli -o json daemon --socket ` and keeps it running, restarting it if it stops. -- **`SIGNALCLI_SOCKET=/run/signal-cli/socket`: you run the daemon.** The bot only connects (retrying with a growing - pause if the daemon is down or restarts) and asks it for the account with a JSON-RPC `listAccounts`. Start the daemon - in multi-account mode, without `-a`: `signal-cli -o json daemon --socket /run/signal-cli/socket`. This wins over - `SIGNALCLI_BIN`. It suits a bot under systemd with `DynamicUser=`, `ProtectHome=` and tight limits, where it is easier - to run signal-cli as your own user in its own unit. +You run the signal-cli daemon yourself and the bot connects to its socket: ``` +signal-cli -o json daemon --socket /run/signal-cli/socket # multi-account mode: no -a +SIGNALCLI_SOCKET=/run/signal-cli/socket SIGNAL_ALLOWED_SENDERS=aaaaaaaa-0000-4000-8000-000000000000,+61400000000 ``` -Either way messages arrive, and replies go out, over the daemon's JSON-RPC socket, and nothing else may run a daemon -on the same account. +The bot never starts or executes signal-cli, so it needs no access to the binary or to your Signal data: only to the +socket. Messages arrive, and replies go out, over that one JSON-RPC connection. If the daemon is down or restarts the +bot retries with a growing pause (up to a minute) and picks up again. At each connection it asks the daemon which +account it is (JSON-RPC `listAccounts`; the first account is used). Nothing else may run a daemon on the same account. - **Socket access:** anyone who can write to the socket can send as the account, so keep it in a directory only the two services share. For example a group `signal-cli`; a directory made by `systemd-tmpfiles` (`d /run/signal-cli 0750 signal-cli -`); the daemon started with `UMask=0007` (or `umask 007`) so the socket is - group-writable; and `SupplementaryGroups=signal-cli` in the bot's unit. + group-writable; and `SupplementaryGroups=signal-cli` and `After=signal-cli.service` in the bot's unit. This suits a + bot under systemd with `DynamicUser=`, `ProtectHome=` and tight limits, with signal-cli running as your own user in + its own unit. - **DMs:** answered only from the people in `SIGNAL_ALLOWED_SENDERS` (UUIDs or phone numbers, as signal-cli reports them); anyone else is logged and ignored. Empty = nobody. These people are also the Signal "admins": only they get Sharesight portfolio data. @@ -338,8 +335,7 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. | `SEARCH_MODEL` | OpenRouter only: model that runs the searches (default xiaomi/mimo-v2.6-flash:online). | | `SHARESIGHT_HOLDING_NEWS_RECIPIENTS` | Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off. | | `SHARESIGHT_HOLDING_NEWS_TIME` | HH:MM (BOT_TZ) for the daily holding-news check (default 08:00). | -| `SIGNALCLI_BIN` | Path to the `signal-cli` executable. Setting it turns [Signal](#signal) on: the bot starts `signal-cli daemon` itself. Empty (default) = Signal off. | -| `SIGNALCLI_SOCKET` | Path of the JSON-RPC socket of a `signal-cli daemon` you run yourself. Turns [Signal](#signal) on, and wins over `SIGNALCLI_BIN`: the bot connects instead of starting a daemon. | +| `SIGNALCLI_SOCKET` | Path of the JSON-RPC socket of a `signal-cli daemon` you run yourself. Setting it turns [Signal](#signal) on. Empty (default) = Signal off. | | `SIGNAL_ALLOWED_SENDERS` | Signal UUIDs or phone numbers (comma/space separated) that may DM the bot and get Sharesight data on Signal. Empty (default) = nobody. | | `TELEGRAM_DM_BUTTONS` | on|off. Preset-prompt buttons in private chats (default off). | | `TOKEN_SAVER` | on|off. Master switch for the token-saving heuristics (default on). | diff --git a/lib/app.py b/lib/app.py index 49f9885..cc360ee 100644 --- a/lib/app.py +++ b/lib/app.py @@ -116,7 +116,7 @@ async def post_init(app: Application) -> None: background.append(asyncio.create_task(dm_buttons.push_on_startup(ctx))) if st.period_backfill: background.append(asyncio.create_task(backfill.loop(ctx))) - if st.signal_cli or st.signal_socket: + if st.signal_socket: background.append(asyncio.create_task(signalbot.run(ctx))) async def post_shutdown(app: Application) -> None: diff --git a/lib/config.py b/lib/config.py index 04b537c..eb7a277 100644 --- a/lib/config.py +++ b/lib/config.py @@ -39,8 +39,7 @@ "SEARCH_MODEL": "OpenRouter only: model that runs the searches (default xiaomi/mimo-v2.6-flash:online).", "SHARESIGHT_HOLDING_NEWS_RECIPIENTS": "Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off.", "SHARESIGHT_HOLDING_NEWS_TIME": "HH:MM (BOT_TZ) for the daily holding-news check (default 08:00).", - "SIGNALCLI_BIN": "Path to the signal-cli executable. Setting it turns Signal on: the bot starts `signal-cli daemon` itself, answers DMs from SIGNAL_ALLOWED_SENDERS and group @mentions or replies. Empty (default) = Signal off.", - "SIGNALCLI_SOCKET": "Path of the JSON-RPC socket of a signal-cli daemon you run yourself (`signal-cli -o json daemon --socket PATH`). Turns Signal on and takes priority over SIGNALCLI_BIN: the bot connects instead of starting a daemon.", + "SIGNALCLI_SOCKET": "Path of the JSON-RPC socket of a signal-cli daemon you run yourself (`signal-cli -o json daemon --socket PATH`). Setting it turns Signal on: the bot answers DMs from SIGNAL_ALLOWED_SENDERS and group @mentions or replies. Empty (default) = Signal off.", "SIGNAL_ALLOWED_SENDERS": "Signal UUIDs or phone numbers (comma/space separated) that may DM the bot and get Sharesight data on Signal. Empty (default) = nobody.", "TELEGRAM_DM_BUTTONS": "on|off. Preset-prompt buttons in private chats (default off).", "TOKEN_SAVER": "on|off. Master switch for the token-saving heuristics (default on).", @@ -80,8 +79,7 @@ class Settings: max_tokens: int history_limit: int log_width: int - signal_cli: str # path to signal-cli, which the bot runs as a daemon; empty = not that way - signal_socket: str # socket of a daemon the bot only connects to; either one turns Signal on + signal_socket: str # socket of a signal-cli daemon; empty = Signal off signal_allowed: frozenset[str] # lowercase UUIDs/numbers that may DM the bot (and get Sharesight data) db_path: str tz: ZoneInfo @@ -200,7 +198,6 @@ def load(env: Mapping[str, str] | None = None) -> Settings: max_tokens=int(env.get("MAX_TOKENS") or 3000), history_limit=int(env.get("HISTORY_LIMIT") or 20), log_width=int(env.get("LOG_WIDTH") or 95), - signal_cli=env.get("SIGNALCLI_BIN", "").strip(), signal_socket=env.get("SIGNALCLI_SOCKET", "").strip(), signal_allowed=frozenset(x.lower() for x in re.split(r"[,\s]+", env.get("SIGNAL_ALLOWED_SENDERS", "")) if x), db_path=data_path(env, "DB_PATH", "chat_log.db"), diff --git a/lib/signalbot/__init__.py b/lib/signalbot/__init__.py index c4f7d59..1312cd4 100644 --- a/lib/signalbot/__init__.py +++ b/lib/signalbot/__init__.py @@ -1,41 +1,21 @@ -"""SIGNALCLI_BIN / SIGNALCLI_SOCKET: answer Signal DMs and group mentions through a signal-cli daemon.""" +"""SIGNALCLI_SOCKET: answer Signal DMs and group mentions through a signal-cli daemon you run.""" from __future__ import annotations import logging from ..context import Ctx -from .client import SignalClient, list_accounts +from .client import SignalClient from .handlers import SignalHandlers log = logging.getLogger("bot") async def run(ctx: Ctx) -> None: - """Connect to the daemon at SIGNALCLI_SOCKET, or start one from SIGNALCLI_BIN, until cancelled.""" - st = ctx.st - if st.signal_socket: - client = SignalClient(socket_path=st.signal_socket) - handlers = SignalHandlers(ctx, client) - client.on_connect = handlers.identify # asks the daemon whose account it is - log.info("Signal: connecting to the signal-cli daemon at %s", st.signal_socket) - else: - try: - accounts = await list_accounts(st.signal_cli) - except Exception as e: - log.error("Signal is off: couldn't list accounts with %s: %s", st.signal_cli, e) - return - if not accounts: - log.error("Signal is off: signal-cli has no registered account") - return - if len(accounts) > 1: - log.warning("Signal: %d accounts registered; using the first", len(accounts)) - number, aci = accounts[0].get("number"), accounts[0].get("aci") or accounts[0].get("uuid") - if not aci: - log.error("Signal is off: the account has no ACI (listAccounts gave %s)", accounts[0]) - return - client = SignalClient(st.signal_cli) - handlers = SignalHandlers(ctx, client, number or aci, aci) - log.info("Signal: starting for account %s", aci) + """Stay connected to the daemon's socket until cancelled.""" + client = SignalClient(ctx.st.signal_socket) + handlers = SignalHandlers(ctx, client) client.on_message = handlers.on_message + client.on_connect = handlers.identify # asks the daemon whose account it is + log.info("Signal: connecting to the signal-cli daemon at %s", ctx.st.signal_socket) await client.run() diff --git a/lib/signalbot/client.py b/lib/signalbot/client.py index 40c1904..7320463 100644 --- a/lib/signalbot/client.py +++ b/lib/signalbot/client.py @@ -1,21 +1,18 @@ -"""Runs `signal-cli daemon` and talks to it over its JSON-RPC socket: messages come in as `receive` -notifications, replies go out as `send` requests, all on one connection.""" +"""Connects to a `signal-cli daemon` over its JSON-RPC socket: messages come in as `receive` +notifications, replies go out as `send` requests, all on one connection. The daemon is run by you, +not by the bot.""" from __future__ import annotations import asyncio -import contextlib import itertools import json import logging -import shutil -import tempfile from collections.abc import Awaitable, Callable -from pathlib import Path log = logging.getLogger("bot") RPC_TIMEOUT = 60 # seconds to wait for signal-cli to answer a request -CONNECT_WAIT = 60 # seconds for a freshly started daemon to open its socket +CONNECT_WAIT = 5 # seconds to keep trying to reach the socket before backing off LINE_LIMIT = 2**24 # an envelope with attachment details can be far longer than asyncio's 64 KiB default @@ -23,23 +20,8 @@ class SignalError(RuntimeError): """signal-cli answered a request with an error.""" -async def list_accounts(path: str) -> list[dict]: - """The registered accounts: `[{"number": null, "aci": ""}]`.""" - proc = await asyncio.create_subprocess_exec( - path, "-o", "json", "listAccounts", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) - out, err = await asyncio.wait_for(proc.communicate(), 60) - if proc.returncode: - raise RuntimeError(f"signal-cli listAccounts failed ({proc.returncode}): {err.decode(errors='replace').strip()}") - accounts = json.loads(out) - return accounts if isinstance(accounts, list) else [] - - class SignalClient: - """With `socket_path`, connects to a daemon someone else runs; otherwise starts `path daemon` itself.""" - - def __init__(self, path: str = "", on_message: Callable[[dict], Awaitable[None]] | None = None, *, - socket_path: str = ""): - self.path = path + def __init__(self, socket_path: str, on_message: Callable[[dict], Awaitable[None]] | None = None): self.socket_path = socket_path self.on_message = on_message self.on_connect: Callable[[], Awaitable[None]] | None = None # runs after every (re)connection @@ -49,7 +31,7 @@ def __init__(self, path: str = "", on_message: Callable[[dict], Awaitable[None]] self._tasks: set[asyncio.Task] = set() async def run(self) -> None: - """Keep a daemon running and connected, restarting it (with a growing pause) when it stops.""" + """Stay connected, reconnecting with a growing pause when the daemon is down or restarts.""" delay = 5 while True: try: @@ -69,31 +51,9 @@ async def run(self) -> None: async def _session(self) -> bool: """One connection's lifetime. True if it got as far as connecting.""" - if self.socket_path: - reader, writer = await self._connect(self.socket_path, None, wait=5) - await self._serve(reader, writer) - return True - sock_dir = Path(tempfile.mkdtemp(prefix="signal-")) - sock = str(sock_dir / "socket") - proc = await asyncio.create_subprocess_exec( - self.path, "-o", "json", "daemon", "--socket", sock, - stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE) - pump = asyncio.create_task(self._pump_stderr(proc)) - connected = False - try: - reader, writer = await self._connect(sock, proc) - connected = True - await self._serve(reader, writer) - return connected - finally: - if proc.returncode is None: - proc.terminate() - with contextlib.suppress(Exception): - await asyncio.wait_for(proc.wait(), 10) - if proc.returncode is None: - proc.kill() - pump.cancel() - shutil.rmtree(sock_dir, ignore_errors=True) + reader, writer = await self._connect() + await self._serve(reader, writer) + return True async def _serve(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: """Read the daemon's lines until it closes the connection.""" @@ -115,25 +75,16 @@ async def _on_connect(self) -> None: except Exception as e: log.error("Signal: %s", e) - async def _connect(self, sock: str, proc, *, wait: int = CONNECT_WAIT - ) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]: + async def _connect(self, wait: int = CONNECT_WAIT) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]: for _ in range(wait * 2): - if proc is not None and proc.returncode is not None: - raise RuntimeError(f"signal-cli exited with code {proc.returncode} before it was ready") try: - return await asyncio.open_unix_connection(sock, limit=LINE_LIMIT) + return await asyncio.open_unix_connection(self.socket_path, limit=LINE_LIMIT) except (FileNotFoundError, ConnectionRefusedError): await asyncio.sleep(0.5) except PermissionError as e: - raise RuntimeError(f"can't open {sock}: {e}. The bot's user needs write access to the socket " - "and to the directory it is in") from e - raise RuntimeError(f"couldn't connect to signal-cli's socket {sock} within {wait} s") - - async def _pump_stderr(self, proc) -> None: - async for line in proc.stderr: - text = line.decode(errors="replace").rstrip() - if text: - log.info("signal-cli: %s", text) + raise RuntimeError(f"can't open {self.socket_path}: {e}. The bot's user needs write access to " + "the socket and to the directory it is in") from e + raise RuntimeError(f"couldn't connect to signal-cli's socket {self.socket_path} within {wait} s") def _dispatch(self, line: bytes) -> None: try: diff --git a/lib/signalbot/handlers.py b/lib/signalbot/handlers.py index 359124c..30f326c 100644 --- a/lib/signalbot/handlers.py +++ b/lib/signalbot/handlers.py @@ -36,13 +36,10 @@ def signal_chunks(model_text: str) -> list[str]: class SignalHandlers: - def __init__(self, ctx: Ctx, client: SignalClient, account: str = "", own_id: str = ""): - """`account`/`own_id` (what requests name, and the bot's own ACI) are known up front when the bot - started the daemon; with a daemon it only connects to, `identify` asks for them.""" - self.ctx, self.client, self.account, self.own_id = ctx, client, account, own_id + def __init__(self, ctx: Ctx, client: SignalClient): + """`account` (what requests name) and `own_id` (the bot's own ACI) come from `identify`.""" + self.ctx, self.client, self.account, self.own_id = ctx, client, "", "" self.ready = asyncio.Event() - if own_id: - self.ready.set() async def identify(self) -> None: """Learn the bot's own account from the daemon (the first one, if it runs several).""" diff --git a/tests/test_signal.py b/tests/test_signal.py index 855c0b8..c5cb6db 100644 --- a/tests/test_signal.py +++ b/tests/test_signal.py @@ -7,7 +7,7 @@ from lib import config from lib.context import Ctx -from lib.signalbot.client import SignalClient, SignalError, list_accounts +from lib.signalbot.client import SignalClient, SignalError from lib.signalbot.handlers import SignalHandlers, signal_chunks from lib.signalbot.parse import parse, unwrap @@ -94,7 +94,10 @@ def make(env, store, script, **extra): st = config.load({**env, "SIGNAL_ALLOWED_SENDERS": ROB.upper(), **extra}) backend, client = ScriptedBackend(script), FakeSignal() ctx = Ctx(st, store, backend, registry(), FakeBot()) - return ctx, backend, client, SignalHandlers(ctx, client, BOT, BOT) + h = SignalHandlers(ctx, client) + h.account, h.own_id = BOT, BOT + h.ready.set() + return ctx, backend, client, h async def test_group_is_answered_on_mention_or_reply_to_the_bot_only(env, store): @@ -153,9 +156,6 @@ async def test_portfolio_questions_are_for_allowed_senders_only(env, store): FAKE_DAEMON = """#!{python} import asyncio, json, sys args = sys.argv[1:] -if "listAccounts" in args: - print(json.dumps([{{"number": None, "aci": "{bot}"}}])) - sys.exit(0) sock = args[args.index("--socket") + 1] async def handle(reader, writer): @@ -183,20 +183,20 @@ async def main(): @pytest.fixture -def fake_signal_cli(tmp_path): +def fake_daemon(tmp_path): path = tmp_path / "signal-cli" path.write_text(FAKE_DAEMON.format(python=sys.executable, bot=BOT, envelope=json.dumps(envelope({"message": "hi"})))) path.chmod(path.stat().st_mode | stat.S_IEXEC) return str(path) -async def test_a_daemon_you_run_yourself_is_connected_to_and_identified(fake_signal_cli, tmp_path, env, store): +async def test_a_daemon_you_run_yourself_is_connected_to_and_identified(fake_daemon, tmp_path, env, store): sock = str(tmp_path / "ext.sock") - daemon = await asyncio.create_subprocess_exec(fake_signal_cli, "daemon", "--socket", sock) + daemon = await asyncio.create_subprocess_exec(fake_daemon, "daemon", "--socket", sock) got = asyncio.Queue() - client = SignalClient(socket_path=sock) + client = SignalClient(sock) ctx = Ctx(config.load({**env, "SIGNALCLI_SOCKET": sock}), store, ScriptedBackend([]), registry(), FakeBot()) - assert ctx.st.signal_socket == sock and ctx.st.signal_cli == "" + assert ctx.st.signal_socket == sock handlers = SignalHandlers(ctx, client) # doesn't know its account yet client.on_message = lambda obj: got.put(obj) client.on_connect = handlers.identify @@ -206,12 +206,16 @@ async def test_a_daemon_you_run_yourself_is_connected_to_and_identified(fake_sig await asyncio.wait_for(handlers.ready.wait(), 20) assert (handlers.account, handlers.own_id) == (BOT, BOT) assert await client.request("send", {"message": "x"}) == {"timestamp": 1234} + with pytest.raises(SignalError, match="no good"): + await client.request("boom", {}) finally: task.cancel() with pytest.raises(asyncio.CancelledError): await task daemon.terminate() await daemon.wait() + with pytest.raises(SignalError, match="isn't connected"): + await client.request("send", {}) async def test_messages_wait_until_the_account_is_known(env, store): @@ -228,32 +232,6 @@ async def test_messages_wait_until_the_account_is_known(env, store): async def test_a_missing_socket_is_reported_not_waited_on_forever(tmp_path): - client = SignalClient(socket_path=str(tmp_path / "nope")) + client = SignalClient(str(tmp_path / "nope")) with pytest.raises(RuntimeError, match="couldn't connect to signal-cli's socket"): - await client._connect(client.socket_path, None, wait=1) - - -async def test_accounts_are_listed(fake_signal_cli): - assert await list_accounts(fake_signal_cli) == [{"number": None, "aci": BOT}] - - -async def test_client_receives_notifications_and_sends_requests(fake_signal_cli): - got = asyncio.Queue() - - async def on_message(obj): - await got.put(obj) - - client = SignalClient(fake_signal_cli, on_message) - task = asyncio.create_task(client.run()) - try: - obj = await asyncio.wait_for(got.get(), 20) - assert parse(unwrap(obj)[0], BOT).text == "hi" - assert await client.request("send", {"message": "x"}) == {"timestamp": 1234} - with pytest.raises(SignalError, match="no good"): - await client.request("boom", {}) - finally: - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - with pytest.raises(SignalError, match="isn't connected"): - await client.request("send", {}) + await client._connect(wait=1) From 7d244c999256cce7af800fe15ce08d72696d9057 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Oct 2026 00:48:53 +0000 Subject: [PATCH 27/27] Remove Signal: restore the tree to 70a7195 Signal support (SIGNALCLI_SOCKET, SIGNAL_ALLOWED_SENDERS, lib/signalbot, the signal_messages table) is taken out again. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- CLAUDE.md | 2 - README.md | 39 +------ lib/app.py | 4 +- lib/config.py | 6 - lib/signalbot/__init__.py | 21 ---- lib/signalbot/client.py | 125 -------------------- lib/signalbot/handlers.py | 157 ------------------------- lib/signalbot/parse.py | 65 ----------- lib/store.py | 23 +--- lib/telegram/handlers.py | 3 +- lib/textfmt.py | 3 - tests/test_signal.py | 237 -------------------------------------- 12 files changed, 10 insertions(+), 675 deletions(-) delete mode 100644 lib/signalbot/__init__.py delete mode 100644 lib/signalbot/client.py delete mode 100644 lib/signalbot/handlers.py delete mode 100644 lib/signalbot/parse.py delete mode 100644 tests/test_signal.py diff --git a/CLAUDE.md b/CLAUDE.md index dfef60f..f3e8851 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,8 +27,6 @@ MCP data tools and web search. User-facing docs are in `README.md`; this file is - `prompts.py` all prompts. `ask.py` one entry point: route -> Request -> policy.ask -> usage -> NEEDS_TOOLS rerun. - `telegram/`: `handlers.py` (trigger -> request -> reply), `send.py`, `draft.py` (typing + streaming drafts), `commands.py` (/credits), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop). -- `signalbot/` (SIGNALCLI_SOCKET): `client.py` connects to the socket of a signal-cli daemon you run and speaks JSON-RPC to it, `parse.py` envelope -> `Incoming`, - `handlers.py` the Signal reply flow (own `signal_messages` history; no Telegram types). Reuses `ask`, `gate`, `prompts`, `history`. - `app.py` wiring/startup; `context.py` `Ctx` (settings, store, backend, registry, bot). - `lib/` holds only what the bot needs to run. Developer helpers live in `tools/`: `report.py` (`make prompt-report`), `capture.py` (real MCP output for fixtures), `period_cache_without_rowid.py` (one-off DB conversion); the bot never imports them. diff --git a/README.md b/README.md index 5d03179..30ab56d 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ from MCP servers (Yahoo Finance, Sharesight) and web search. It runs on **xAI (G ## Contents - [Quick start](#quick-start) · [Providers](#providers) · [How it answers](#how-it-answers) - [Search](#search) · [MCP data tools](#mcp-data-tools) · [Formatting and ticker links](#formatting-and-ticker-links) -- [Optional features](#optional-features-off-by-default) · [Signal](#signal) · [Commands](#commands) +- [Optional features](#optional-features-off-by-default) · [Commands](#commands) - [Caching and token saving](#caching-and-token-saving) · [Running several instances](#running-several-instances) - [Logs](#logs) · [Environment variables](#environment-variables) · [Choosing a model](#choosing-a-model) - [Development](#development) · [Troubleshooting](#troubleshooting) @@ -178,41 +178,6 @@ Each is off by default. The holding-news DM and the movers reply are switched on from being added: a group it joined earlier is unknown until someone posts there. If the name matches none of your groups the reply is the same whether or not the group exists. -## Signal -Off by default. The bot answers on Signal as well as Telegram, with the same model, tools, search and history, through a -[signal-cli](https://github.com/AsamK/signal-cli) account that is already registered (or linked as a secondary device). -You run the signal-cli daemon yourself and the bot connects to its socket: - -``` -signal-cli -o json daemon --socket /run/signal-cli/socket # multi-account mode: no -a -SIGNALCLI_SOCKET=/run/signal-cli/socket -SIGNAL_ALLOWED_SENDERS=aaaaaaaa-0000-4000-8000-000000000000,+61400000000 -``` -The bot never starts or executes signal-cli, so it needs no access to the binary or to your Signal data: only to the -socket. Messages arrive, and replies go out, over that one JSON-RPC connection. If the daemon is down or restarts the -bot retries with a growing pause (up to a minute) and picks up again. At each connection it asks the daemon which -account it is (JSON-RPC `listAccounts`; the first account is used). Nothing else may run a daemon on the same account. - -- **Socket access:** anyone who can write to the socket can send as the account, so keep it in a directory only the - two services share. For example a group `signal-cli`; a directory made by `systemd-tmpfiles` - (`d /run/signal-cli 0750 signal-cli -`); the daemon started with `UMask=0007` (or `umask 007`) so the socket is - group-writable; and `SupplementaryGroups=signal-cli` and `After=signal-cli.service` in the bot's unit. This suits a - bot under systemd with `DynamicUser=`, `ProtectHome=` and tight limits, with signal-cli running as your own user in - its own unit. -- **DMs:** answered only from the people in `SIGNAL_ALLOWED_SENDERS` (UUIDs or phone numbers, as signal-cli reports - them); anyone else is logged and ignored. Empty = nobody. These people are also the Signal "admins": only they get - Sharesight portfolio data. -- **Groups:** the bot answers when it is @mentioned or someone replies (quotes) one of its messages; every other group - message is logged as context. Group replies quote the message they answer. -- **History:** kept per chat in the `signal_messages` table, windowed like Telegram's. `#nocontext` works the same way - (see [How it answers](#how-it-answers)). -- **Replies are plain text:** Signal has no HTML, so bold and links are dropped (a link becomes `text (url)`) and - tickers are not linked. -- **Not on Signal:** images and other attachments (a message with no text is ignored), the typing indicator and - streamed drafts, movers lists, the daily holding-news DM, DM buttons, `/credits` and `/flushcache`. -- **Checking:** the log shows `Signal: connected to signal-cli` and `Signal: answering as account ...` once the - daemon is reachable, and each message as `[Signal (group)] ...`. - ## Commands | Command | Who | What | |---|---|---| @@ -335,8 +300,6 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. | `SEARCH_MODEL` | OpenRouter only: model that runs the searches (default xiaomi/mimo-v2.6-flash:online). | | `SHARESIGHT_HOLDING_NEWS_RECIPIENTS` | Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off. | | `SHARESIGHT_HOLDING_NEWS_TIME` | HH:MM (BOT_TZ) for the daily holding-news check (default 08:00). | -| `SIGNALCLI_SOCKET` | Path of the JSON-RPC socket of a `signal-cli daemon` you run yourself. Setting it turns [Signal](#signal) on. Empty (default) = Signal off. | -| `SIGNAL_ALLOWED_SENDERS` | Signal UUIDs or phone numbers (comma/space separated) that may DM the bot and get Sharesight data on Signal. Empty (default) = nobody. | | `TELEGRAM_DM_BUTTONS` | on|off. Preset-prompt buttons in private chats (default off). | | `TOKEN_SAVER` | on|off. Master switch for the token-saving heuristics (default on). | diff --git a/lib/app.py b/lib/app.py index cc360ee..bf3310d 100644 --- a/lib/app.py +++ b/lib/app.py @@ -26,7 +26,7 @@ filters, ) -from . import config, signalbot +from . import config from .context import Ctx from .features import dm_buttons, holding_news, post from .features.periods import backfill @@ -116,8 +116,6 @@ async def post_init(app: Application) -> None: background.append(asyncio.create_task(dm_buttons.push_on_startup(ctx))) if st.period_backfill: background.append(asyncio.create_task(backfill.loop(ctx))) - if st.signal_socket: - background.append(asyncio.create_task(signalbot.run(ctx))) async def post_shutdown(app: Application) -> None: for task in background: diff --git a/lib/config.py b/lib/config.py index eb7a277..a3de261 100644 --- a/lib/config.py +++ b/lib/config.py @@ -39,8 +39,6 @@ "SEARCH_MODEL": "OpenRouter only: model that runs the searches (default xiaomi/mimo-v2.6-flash:online).", "SHARESIGHT_HOLDING_NEWS_RECIPIENTS": "Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off.", "SHARESIGHT_HOLDING_NEWS_TIME": "HH:MM (BOT_TZ) for the daily holding-news check (default 08:00).", - "SIGNALCLI_SOCKET": "Path of the JSON-RPC socket of a signal-cli daemon you run yourself (`signal-cli -o json daemon --socket PATH`). Setting it turns Signal on: the bot answers DMs from SIGNAL_ALLOWED_SENDERS and group @mentions or replies. Empty (default) = Signal off.", - "SIGNAL_ALLOWED_SENDERS": "Signal UUIDs or phone numbers (comma/space separated) that may DM the bot and get Sharesight data on Signal. Empty (default) = nobody.", "TELEGRAM_DM_BUTTONS": "on|off. Preset-prompt buttons in private chats (default off).", "TOKEN_SAVER": "on|off. Master switch for the token-saving heuristics (default on).", } @@ -79,8 +77,6 @@ class Settings: max_tokens: int history_limit: int log_width: int - signal_socket: str # socket of a signal-cli daemon; empty = Signal off - signal_allowed: frozenset[str] # lowercase UUIDs/numbers that may DM the bot (and get Sharesight data) db_path: str tz: ZoneInfo mcp_config: str @@ -198,8 +194,6 @@ def load(env: Mapping[str, str] | None = None) -> Settings: max_tokens=int(env.get("MAX_TOKENS") or 3000), history_limit=int(env.get("HISTORY_LIMIT") or 20), log_width=int(env.get("LOG_WIDTH") or 95), - signal_socket=env.get("SIGNALCLI_SOCKET", "").strip(), - signal_allowed=frozenset(x.lower() for x in re.split(r"[,\s]+", env.get("SIGNAL_ALLOWED_SENDERS", "")) if x), db_path=data_path(env, "DB_PATH", "chat_log.db"), tz=ZoneInfo(env.get("BOT_TZ", "UTC")), mcp_config=mcp_config_path(env), diff --git a/lib/signalbot/__init__.py b/lib/signalbot/__init__.py deleted file mode 100644 index 1312cd4..0000000 --- a/lib/signalbot/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -"""SIGNALCLI_SOCKET: answer Signal DMs and group mentions through a signal-cli daemon you run.""" - -from __future__ import annotations - -import logging - -from ..context import Ctx -from .client import SignalClient -from .handlers import SignalHandlers - -log = logging.getLogger("bot") - - -async def run(ctx: Ctx) -> None: - """Stay connected to the daemon's socket until cancelled.""" - client = SignalClient(ctx.st.signal_socket) - handlers = SignalHandlers(ctx, client) - client.on_message = handlers.on_message - client.on_connect = handlers.identify # asks the daemon whose account it is - log.info("Signal: connecting to the signal-cli daemon at %s", ctx.st.signal_socket) - await client.run() diff --git a/lib/signalbot/client.py b/lib/signalbot/client.py deleted file mode 100644 index 7320463..0000000 --- a/lib/signalbot/client.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Connects to a `signal-cli daemon` over its JSON-RPC socket: messages come in as `receive` -notifications, replies go out as `send` requests, all on one connection. The daemon is run by you, -not by the bot.""" - -from __future__ import annotations - -import asyncio -import itertools -import json -import logging -from collections.abc import Awaitable, Callable - -log = logging.getLogger("bot") -RPC_TIMEOUT = 60 # seconds to wait for signal-cli to answer a request -CONNECT_WAIT = 5 # seconds to keep trying to reach the socket before backing off -LINE_LIMIT = 2**24 # an envelope with attachment details can be far longer than asyncio's 64 KiB default - - -class SignalError(RuntimeError): - """signal-cli answered a request with an error.""" - - -class SignalClient: - def __init__(self, socket_path: str, on_message: Callable[[dict], Awaitable[None]] | None = None): - self.socket_path = socket_path - self.on_message = on_message - self.on_connect: Callable[[], Awaitable[None]] | None = None # runs after every (re)connection - self._writer: asyncio.StreamWriter | None = None - self._pending: dict[int, asyncio.Future] = {} - self._ids = itertools.count(1) - self._tasks: set[asyncio.Task] = set() - - async def run(self) -> None: - """Stay connected, reconnecting with a growing pause when the daemon is down or restarts.""" - delay = 5 - while True: - try: - if await self._session(): - delay = 5 - except asyncio.CancelledError: - raise - except Exception as e: - log.warning("Signal: %s", e) - for fut in self._pending.values(): - if not fut.done(): - fut.set_exception(SignalError("signal-cli stopped")) - self._pending.clear() - log.warning("Signal: lost signal-cli; trying again in %d s", delay) - await asyncio.sleep(delay) - delay = min(delay * 2, 60) - - async def _session(self) -> bool: - """One connection's lifetime. True if it got as far as connecting.""" - reader, writer = await self._connect() - await self._serve(reader, writer) - return True - - async def _serve(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - """Read the daemon's lines until it closes the connection.""" - self._writer = writer - log.info("Signal: connected to signal-cli") - hook = asyncio.create_task(self._on_connect()) if self.on_connect else None - try: - async for line in reader: - self._dispatch(line) - finally: - self._writer = None - if hook: - hook.cancel() - writer.close() - - async def _on_connect(self) -> None: - try: - await self.on_connect() - except Exception as e: - log.error("Signal: %s", e) - - async def _connect(self, wait: int = CONNECT_WAIT) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]: - for _ in range(wait * 2): - try: - return await asyncio.open_unix_connection(self.socket_path, limit=LINE_LIMIT) - except (FileNotFoundError, ConnectionRefusedError): - await asyncio.sleep(0.5) - except PermissionError as e: - raise RuntimeError(f"can't open {self.socket_path}: {e}. The bot's user needs write access to " - "the socket and to the directory it is in") from e - raise RuntimeError(f"couldn't connect to signal-cli's socket {self.socket_path} within {wait} s") - - def _dispatch(self, line: bytes) -> None: - try: - obj = json.loads(line) - except ValueError: - return - if not isinstance(obj, dict): - return - fut = self._pending.pop(obj["id"], None) if isinstance(obj.get("id"), int) else None - if fut is not None: - if "error" in obj: - err = obj["error"] - fut.set_exception(SignalError(str(err.get("message", err)) if isinstance(err, dict) else str(err))) - else: - fut.set_result(obj.get("result")) - elif self.on_message and (obj.get("method") == "receive" or "envelope" in obj): - task = asyncio.create_task(self._handle(obj)) - self._tasks.add(task) - task.add_done_callback(self._tasks.discard) - - async def _handle(self, obj: dict) -> None: - try: - await self.on_message(obj) - except Exception: - log.exception("Signal: couldn't handle a message") - - async def request(self, method: str, params: dict): - if self._writer is None: - raise SignalError("signal-cli isn't connected") - rid = next(self._ids) - fut = asyncio.get_running_loop().create_future() - self._pending[rid] = fut - try: - self._writer.write(json.dumps({"jsonrpc": "2.0", "method": method, "params": params, "id": rid}).encode() + b"\n") - await self._writer.drain() - return await asyncio.wait_for(fut, RPC_TIMEOUT) - finally: - self._pending.pop(rid, None) diff --git a/lib/signalbot/handlers.py b/lib/signalbot/handlers.py deleted file mode 100644 index 30f326c..0000000 --- a/lib/signalbot/handlers.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Signal message -> reply: the same history, tools and model as Telegram, with Signal's own plumbing.""" - -from __future__ import annotations - -import asyncio -import hashlib -import logging -import re -import time - -from ..ask import ask -from ..context import Ctx -from ..features import periods -from ..history import compact, format_rows -from ..llm.base import BUSY_REPLY, is_busy -from ..llm.gate import is_portfolio_question, route -from ..prompts import chat_prompt, down_note -from ..store import HistoryRow -from ..textfmt import NO_CONTEXT, md_to_html, plain_text, split_html, strip_disclaimer -from .client import SignalClient, SignalError -from .parse import Incoming, parse, unwrap - -log = logging.getLogger("bot") -PORTFOLIO_ADMINS_ONLY = "Portfolio data is for admins only." - - -def signal_chunks(model_text: str) -> list[str]: - """The model's Telegram-HTML answer as plain-text messages (Signal has no HTML), keeping its lines.""" - out = [] - for chunk in split_html(md_to_html(strip_disclaimer(model_text))): - lines = [plain_text(line) for line in chunk.split("\n")] - text = re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip() - if text: - out.append(text) - return out or ["(no response)"] - - -class SignalHandlers: - def __init__(self, ctx: Ctx, client: SignalClient): - """`account` (what requests name) and `own_id` (the bot's own ACI) come from `identify`.""" - self.ctx, self.client, self.account, self.own_id = ctx, client, "", "" - self.ready = asyncio.Event() - - async def identify(self) -> None: - """Learn the bot's own account from the daemon (the first one, if it runs several).""" - accounts = await self.client.request("listAccounts", {}) - if not isinstance(accounts, list) or not accounts: - raise SignalError("the daemon has no registered account") - if len(accounts) > 1: - log.warning("Signal: %d accounts registered; using the first", len(accounts)) - number, aci = accounts[0].get("number"), accounts[0].get("aci") or accounts[0].get("uuid") - if not aci: - raise SignalError(f"the account has no ACI (listAccounts gave {accounts[0]})") - self.account, self.own_id = number or aci, aci - self.ready.set() - log.info("Signal: answering as account %s", aci) - - async def on_message(self, obj: dict) -> None: - unwrapped = unwrap(obj) - if unwrapped is None: - return - try: - await asyncio.wait_for(self.ready.wait(), 30) - except TimeoutError: - log.warning("Signal: dropped a message, the account isn't known yet") - return - envelope, account = unwrapped - if account and account not in (self.account, self.own_id): - return # another account of the same daemon - inc = parse(envelope, self.own_id) - if inc is None: - return - await asyncio.to_thread(self.ctx.store.save_signal, self.own_id, inc.chat, - HistoryRow(inc.ts, inc.sender_name, inc.text, inc.ts // 1000, inc.quote_ts)) - if self._is_for_bot(inc): - await self._answer(inc) - - def _is_for_bot(self, inc: Incoming) -> bool: - """DMs from the allow-list; in groups an @mention or a reply to one of the bot's messages.""" - if inc.group: - return inc.mentioned_bot or inc.quote_author == self.own_id - if inc.sender_ids & self.ctx.st.signal_allowed: - return True - log.info("Ignored a Signal DM from %s: not in SIGNAL_ALLOWED_SENDERS", inc.sender) - return False - - async def _answer(self, inc: Incoming) -> None: - text = NO_CONTEXT.sub("", inc.text).strip() if NO_CONTEXT.search(inc.text) else inc.text - preview = " ".join(text.split()) - log.info("[Signal %s%s] %s", inc.sender_name, " (group)" if inc.group else "", - preview[:60].rstrip() + " ..." if len(preview) > 60 else preview) - try: - chunks = await self._generate(inc, text, nocontext=text != inc.text) - except Exception as e: - if is_busy(e): - log.warning("Reply failed, the provider is busy: %s", " ".join(str(e).split())) - chunks = [BUSY_REPLY] - else: - log.exception("Reply failed") - chunks = [f"Sorry, something went wrong:\n{type(e).__name__}: {e}"[:1500]] - await self._send(inc, chunks) - - async def _generate(self, inc: Incoming, text: str, *, nocontext: bool) -> list[str]: - ctx, st = self.ctx, self.ctx.st - admin = bool(inc.sender_ids & st.signal_allowed) # the people trusted with DMs also get portfolio data - if nocontext: # only the message that tagged the bot - rows = [HistoryRow(inc.ts, inc.sender_name, text, inc.ts // 1000, None)] - else: - rows = await asyncio.to_thread(ctx.store.signal_history, self.own_id, inc.chat, st.history_limit) - quote = None if nocontext else self._quote(inc, rows) - gate_text = text + ("" if nocontext else " " + inc.quote_text) - chosen = route(gate_text, st, ctx.registry) - portfolio_up = any(s.cfg.get("gate") == "portfolio" for s in ctx.registry.up()) - if not admin and portfolio_up and is_portfolio_question(gate_text, st): - return [PORTFOLIO_ADMINS_ONLY] - has_portfolio = any(s.cfg.get("gate") == "portfolio" for s in chosen.servers) - extra = [periods.tool(ctx)] if admin and has_portfolio else [] - fresh = has_portfolio and is_portfolio_question(gate_text, st) - prompt = chat_prompt( - transcript=format_rows(rows, ctx.self_name, st.tz, line_max=st.history_line_max, - compact_text=st.token_saver, own_line_max=st.history_own_line_max), - sender=inc.sender_name, private=not inc.group, reply_quote=quote, msg_id=inc.ts, - now=ctx.now(), down=down_note(ctx.registry.down()), saver=st.token_saver, fresh=fresh) - answer = await ask(ctx, [{"type": "text", "text": prompt}], chosen, - cache_key="signal-" + hashlib.sha1(inc.chat.encode()).hexdigest()[:12], - extra_tools=extra, admin=admin) - return signal_chunks(answer.text) - - def _quote(self, inc: Incoming, rows: list[HistoryRow]) -> str | None: - if inc.quote_ts is None: - return None - who = "You" if inc.quote_author == self.own_id else next( - (r.sender for r in rows if r.message_id == inc.quote_ts), "someone") - body = inc.quote_text - return f"[#{inc.quote_ts}] {who}: {compact(body) if self.ctx.st.token_saver else body}" - - async def _send(self, inc: Incoming, chunks: list[str]) -> None: - for i, chunk in enumerate(chunks): - params: dict = {"account": self.account, "message": chunk} - params.update({"groupId": inc.chat} if inc.group else {"recipient": [inc.sender]}) - if inc.group and i == 0: # a group reply quotes the message it answers, as on Telegram - params.update(quoteTimestamp=inc.ts, quoteAuthor=inc.sender, quoteMessage=inc.text[:200]) - try: - try: - result = await self.client.request("send", params) - except SignalError: - if "quoteTimestamp" not in params: - raise - for key in ("quoteTimestamp", "quoteAuthor", "quoteMessage"): - params.pop(key) - result = await self.client.request("send", params) - except SignalError as e: - log.error("Signal: couldn't send a reply: %s", e) - return - sent = int((result or {}).get("timestamp") or time.time() * 1000) - await asyncio.to_thread(self.ctx.store.save_signal, self.own_id, inc.chat, - HistoryRow(sent, self.ctx.self_name, chunk, sent // 1000, inc.ts)) diff --git a/lib/signalbot/parse.py b/lib/signalbot/parse.py deleted file mode 100644 index 1f74eab..0000000 --- a/lib/signalbot/parse.py +++ /dev/null @@ -1,65 +0,0 @@ -"""signal-cli JSON envelopes -> the few facts the bot needs (None for typing, receipts, syncs...).""" - -from __future__ import annotations - -from dataclasses import dataclass - -OBJECT_REPLACEMENT = "" # what Signal puts in the text where a mention goes - - -@dataclass(frozen=True) -class Incoming: - chat: str # a group's ID, or "dm:" for a private chat - group: bool - sender: str # the sender's UUID (or number if there is no UUID) - sender_ids: frozenset[str] # every ID the sender is known by, lowercase (for the allow-list) - sender_name: str - text: str # mentions written out; the bot's own mention removed - ts: int # Signal's message ID: the sender's timestamp in ms - mentioned_bot: bool - quote_ts: int | None - quote_author: str | None - quote_text: str - - -def unwrap(obj: dict) -> tuple[dict, str] | None: - """(envelope, account) from a bare `{"envelope":..}` line or a JSON-RPC `receive` notification.""" - if obj.get("method") == "receive": - obj = obj.get("params") or {} - envelope = obj.get("envelope") - if not isinstance(envelope, dict): - return None - return envelope, str(obj.get("account") or "") - - -def _resolve_mentions(text: str, mentions: list[dict], own_id: str) -> tuple[str, bool]: - """Replace each mention placeholder (from the end, so offsets hold) with @name; the bot's own - mention is dropped.""" - mentioned = False - for m in sorted(mentions, key=lambda m: m.get("start", 0), reverse=True): - start, length = int(m.get("start", 0)), int(m.get("length", 1)) - is_bot = own_id and own_id in (m.get("uuid"), m.get("number")) - mentioned = mentioned or bool(is_bot) - who = "" if is_bot else "@" + str(m.get("name") or m.get("number") or str(m.get("uuid"))[:8]) - text = text[:start] + who + text[start + length:] - return " ".join(text.replace(OBJECT_REPLACEMENT, "").split()) if mentions else text.strip(), mentioned - - -def parse(envelope: dict, own_id: str) -> Incoming | None: - data = envelope.get("dataMessage") - if not isinstance(data, dict) or not data.get("message"): - return None - ids = [envelope.get("sourceUuid"), envelope.get("source"), envelope.get("sourceNumber")] - sender = next((i for i in ids if i), "") - if not sender or sender == own_id: - return None - group = (data.get("groupInfo") or {}).get("groupId") - text, mentioned = _resolve_mentions(data["message"], data.get("mentions") or [], own_id) - quote = data.get("quote") or {} - name = envelope.get("sourceName") or envelope.get("sourceNumber") or str(sender)[:8] - return Incoming( - chat=group or f"dm:{sender}", group=bool(group), sender=sender, - sender_ids=frozenset(str(i).lower() for i in ids if i), sender_name=str(name), - text=text, ts=int(data.get("timestamp") or envelope.get("timestamp") or 0), - mentioned_bot=mentioned, quote_ts=quote.get("id"), quote_author=quote.get("author") - or quote.get("authorUuid"), quote_text=str(quote.get("text") or "")) diff --git a/lib/store.py b/lib/store.py index 745498d..08e01d8 100644 --- a/lib/store.py +++ b/lib/store.py @@ -22,11 +22,6 @@ CREATE TABLE IF NOT EXISTS dm_messages ( bot_id INTEGER, chat_id INTEGER, message_id INTEGER, sender TEXT, text TEXT, ts INTEGER, reply_to INTEGER, PRIMARY KEY (bot_id, chat_id, message_id)); --- Signal chats, keyed by the bot's account. chat = a group's ID, or "dm:"; message_id = the --- sender's timestamp in ms (Signal's own message ID). -CREATE TABLE IF NOT EXISTS signal_messages ( - bot TEXT, chat TEXT, message_id INTEGER, sender TEXT, text TEXT, ts INTEGER, reply_to INTEGER, - PRIMARY KEY (bot, chat, message_id)); CREATE TABLE IF NOT EXISTS users (username TEXT PRIMARY KEY, user_id INTEGER); CREATE TABLE IF NOT EXISTS holding_news (username TEXT, ts INTEGER, text TEXT); CREATE TABLE IF NOT EXISTS holding_news_mutes ( @@ -123,16 +118,16 @@ def chats(self) -> list[tuple[int, str]]: return [(r[0], r[1]) for r in self._run("SELECT chat_id, title FROM chats")] def history(self, chat_id: int, limit: int) -> list[HistoryRow]: - """Recent messages, oldest first. See `_window` for which ones.""" - if is_dm(chat_id): - return self._window("dm_messages", "bot_id = ? AND chat_id = ?", (self._own_id(), chat_id), limit) - return self._window("messages", "chat_id = ?", (chat_id,), limit) + """Recent messages, oldest first. - def _window(self, table: str, where: str, args: tuple, limit: int) -> list[HistoryRow]: - """A plain "last N" window drops its oldest line on every message, changing the start of + A plain "last N" window drops its oldest line on every message, changing the start of the prompt and defeating prompt caching. Instead the window's start only moves every limit/2 messages: it holds between `limit` and 1.5 x `limit` messages and in between only grows at the end, so everything before the new messages stays cached.""" + if is_dm(chat_id): + table, where, args = "dm_messages", "bot_id = ? AND chat_id = ?", (self._own_id(), chat_id) + else: + table, where, args = "messages", "chat_id = ?", (chat_id,) total = self._run(f"SELECT COUNT(*) FROM {table} WHERE {where}", args)[0][0] step = max(1, limit // 2) size = total - max(0, (total - limit) // step * step) @@ -141,12 +136,6 @@ def _window(self, table: str, where: str, args: tuple, limit: int) -> list[Histo f"WHERE {where} ORDER BY message_id DESC LIMIT ?", (*args, size)) return [HistoryRow(*r) for r in reversed(rows)] - def save_signal(self, bot: str, chat: str, row: HistoryRow) -> None: - self._run("INSERT OR REPLACE INTO signal_messages VALUES (?,?,?,?,?,?,?)", (bot, chat, *row)) - - def signal_history(self, bot: str, chat: str, limit: int) -> list[HistoryRow]: - return self._window("signal_messages", "bot = ? AND chat = ?", (bot, chat), limit) - def dm_chats(self) -> list[int]: """Everyone this bot has a private chat with.""" rows = self._run("SELECT DISTINCT chat_id FROM dm_messages WHERE bot_id = ?", (self._own_id(),)) diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index ddb268b..583db15 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -22,7 +22,7 @@ from ..msgtext import describe, sender_name from ..prompts import MOVERS_EXPLAIN, PRESET_PROMPT, chat_prompt, down_note from ..store import is_dm -from ..textfmt import NO_CONTEXT, md_to_html, strip_disclaimer +from ..textfmt import md_to_html, strip_disclaimer from ..tickers import link_tickers from .access import is_admin from .draft import Draft, start_typing @@ -33,6 +33,7 @@ FOLLOW_UP_WORDS = 12 # a message this short right after the bot's reply may be a follow-up to it FOLLOW_UP_SECONDS = 30 * 60 FOLLOW_UP_CHARS = 400 # of the bot's reply that the tool gate also reads +NO_CONTEXT = re.compile(r"[ \t]*(?]*>|]*$", re.I) -# `#nocontext` in a message: the model gets that message alone (no history, nothing it replies to). -NO_CONTEXT = re.compile(r"[ \t]*(?NVDA up 3%\nAMD (src) & more\n\n\n\nbye") - assert out == ["NVDA up 3%\nAMD (src (https://x.test/a)) & more\n\nbye"] - - -# ---- the reply flow ------------------------------------------------------------------- -class FakeSignal: - def __init__(self): - self.requests = [] - self.fail_quote = False - - accounts: list = [] - - async def request(self, method, params): - self.requests.append((method, dict(params))) - if method == "listAccounts": - return self.accounts - if self.fail_quote and "quoteTimestamp" in params: - raise SignalError("bad quote") - return {"timestamp": 5_000_000 + len(self.requests)} - - -def make(env, store, script, **extra): - st = config.load({**env, "SIGNAL_ALLOWED_SENDERS": ROB.upper(), **extra}) - backend, client = ScriptedBackend(script), FakeSignal() - ctx = Ctx(st, store, backend, registry(), FakeBot()) - h = SignalHandlers(ctx, client) - h.account, h.own_id = BOT, BOT - h.ready.set() - return ctx, backend, client, h - - -async def test_group_is_answered_on_mention_or_reply_to_the_bot_only(env, store): - ctx, backend, client, h = make(env, store, [step("NVDA is up"), step("AMD too")]) - await h.on_message(envelope(group_data("lunch anyone?", mentions=[]), ts=1_000)) - assert client.requests == [] # logged as context, not answered - await h.on_message(envelope(group_data("\ufffc how is NVDA?", mentions=[mention()]), ts=2_000)) - method, params = client.requests[0] - assert method == "send" and params["groupId"] == GROUP and params["message"] == "NVDA is up" - assert params["account"] == BOT and params["quoteTimestamp"] == 2_000 and params["quoteAuthor"] == ROB - assert "lunch anyone?" in backend.seen[0]["prompt"] and "how is NVDA?" in backend.seen[0]["prompt"] - quote = {"id": 5_000_001, "author": BOT, "text": "NVDA is up"} - await h.on_message(envelope(group_data("and AMD?", quote=quote), ts=3_000)) - assert client.requests[1][1]["message"] == "AMD too" - rows = store.signal_history(BOT, GROUP, 20) - assert [r.sender for r in rows].count(ctx.self_name) == 2 # the bot's replies are in its history, as "You" - assert "[#5000001] You: NVDA is up" in backend.seen[1]["prompt"] - - -async def test_dm_needs_an_allowed_sender_and_goes_to_that_person(env, store): - ctx, backend, client, h = make(env, store, [step("hello Alex")]) - stranger = envelope({"message": "hi bot"}, source="dddddddd-0000-4000-8000-000000000004") - await h.on_message(stranger) - assert client.requests == [] and backend.seen == [] - await h.on_message(envelope({"message": "hi bot"})) - method, params = client.requests[0] - assert params["recipient"] == [ROB] and "groupId" not in params and "quoteTimestamp" not in params - assert "Private one-to-one chat with Alex" in backend.seen[0]["prompt"] - - -async def test_a_rejected_quote_is_retried_without_it(env, store): - ctx, backend, client, h = make(env, store, [step("ok")]) - client.fail_quote = True - await h.on_message(envelope(group_data("\ufffc hi", mentions=[mention()]))) - assert len(client.requests) == 2 and "quoteTimestamp" not in client.requests[1][1] - - -async def test_nocontext_sends_the_message_alone_without_the_tag(env, store): - ctx, backend, client, h = make(env, store, [step("fine")]) - await h.on_message(envelope(group_data("the football was great"), ts=1_000)) - await h.on_message(envelope(group_data("\ufffc #nocontext what about NVDA?", mentions=[mention()]), ts=2_000)) - prompt = backend.seen[0]["prompt"] - assert "NVDA" in prompt and "football" not in prompt and "nocontext" not in prompt.lower() - - -async def test_portfolio_questions_are_for_allowed_senders_only(env, store): - from .fakes import FakeMcp - ctx, backend, client, h = make(env, store, [step("never asked")]) - ctx.registry = registry(FakeMcp("sharesight", gate="portfolio", tools=("list_portfolios",))) - stranger = "dddddddd-0000-4000-8000-000000000004" # in the group, but not on the allow-list - await h.on_message(envelope(group_data("\ufffc how is my portfolio doing?", mentions=[mention()]), source=stranger)) - assert client.requests[0][1]["message"] == "Portfolio data is for admins only." and backend.seen == [] - - -# ---- the daemon ----------------------------------------------------------------------- -FAKE_DAEMON = """#!{python} -import asyncio, json, sys -args = sys.argv[1:] -sock = args[args.index("--socket") + 1] - -async def handle(reader, writer): - note = {{"jsonrpc": "2.0", "method": "receive", "params": json.loads({envelope!r})}} - writer.write((json.dumps(note) + "\\n").encode()) - await writer.drain() - async for line in reader: - req = json.loads(line) - if req["method"] == "boom": - reply = {{"jsonrpc": "2.0", "id": req["id"], "error": {{"code": -1, "message": "no good"}}}} - elif req["method"] == "listAccounts": - reply = {{"jsonrpc": "2.0", "id": req["id"], "result": [{{"number": None, "aci": "{bot}"}}]}} - else: - reply = {{"jsonrpc": "2.0", "id": req["id"], "result": {{"timestamp": 1234}}}} - writer.write((json.dumps(reply) + "\\n").encode()) - await writer.drain() - -async def main(): - server = await asyncio.start_unix_server(handle, sock) - async with server: - await server.serve_forever() - -asyncio.run(main()) -""" - - -@pytest.fixture -def fake_daemon(tmp_path): - path = tmp_path / "signal-cli" - path.write_text(FAKE_DAEMON.format(python=sys.executable, bot=BOT, envelope=json.dumps(envelope({"message": "hi"})))) - path.chmod(path.stat().st_mode | stat.S_IEXEC) - return str(path) - - -async def test_a_daemon_you_run_yourself_is_connected_to_and_identified(fake_daemon, tmp_path, env, store): - sock = str(tmp_path / "ext.sock") - daemon = await asyncio.create_subprocess_exec(fake_daemon, "daemon", "--socket", sock) - got = asyncio.Queue() - client = SignalClient(sock) - ctx = Ctx(config.load({**env, "SIGNALCLI_SOCKET": sock}), store, ScriptedBackend([]), registry(), FakeBot()) - assert ctx.st.signal_socket == sock - handlers = SignalHandlers(ctx, client) # doesn't know its account yet - client.on_message = lambda obj: got.put(obj) - client.on_connect = handlers.identify - task = asyncio.create_task(client.run()) - try: - assert parse(unwrap(await asyncio.wait_for(got.get(), 20))[0], BOT).text == "hi" - await asyncio.wait_for(handlers.ready.wait(), 20) - assert (handlers.account, handlers.own_id) == (BOT, BOT) - assert await client.request("send", {"message": "x"}) == {"timestamp": 1234} - with pytest.raises(SignalError, match="no good"): - await client.request("boom", {}) - finally: - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - daemon.terminate() - await daemon.wait() - with pytest.raises(SignalError, match="isn't connected"): - await client.request("send", {}) - - -async def test_messages_wait_until_the_account_is_known(env, store): - ctx, backend, client, _ = make(env, store, [step("hello")]) - h = SignalHandlers(ctx, client) - task = asyncio.create_task(h.on_message(envelope({"message": "hi bot"}))) - await asyncio.sleep(0.05) - assert not task.done() and client.requests == [] - client.accounts = [{"number": "+61400000000", "aci": BOT}] - await h.identify() - await asyncio.wait_for(task, 5) - assert h.account == "+61400000000" and h.own_id == BOT and client.requests[-1][1]["message"] == "hello" - assert client.requests[0] == ("listAccounts", {}) and client.requests[-1][1]["account"] == "+61400000000" - - -async def test_a_missing_socket_is_reported_not_waited_on_forever(tmp_path): - client = SignalClient(str(tmp_path / "nope")) - with pytest.raises(RuntimeError, match="couldn't connect to signal-cli's socket"): - await client._connect(wait=1)