From 26aeca716b3d2a4a7102d819f559f8baa6b06ba4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 11:36:23 +0000 Subject: [PATCH 01/34] Add DATA_DIR: the database and MCP config live there when set Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- README.md | 1 + lib/config.py | 13 +++++++++++-- tests/test_config.py | 10 ++++++++++ tools/capture.py | 2 +- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d637524..c8130f3 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,7 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. |---|---| | `ADMIN_CHAT_IDS` | Telegram IDs of the bot's admins (comma/space separated). Your user ID gets a message when an MCP server goes down and may use `/credits`; a group's chat ID gets the down-server messages only. Admins of the groups the bot is in count as admins too. | | `ADMIN_ONLY` | on|off. Only admins may DM the bot or get Sharesight data (default on). | +| `DATA_DIR` | Directory for the bot's data files (the database, `mcp_servers.json`); a relative `DB_PATH` / `MCP_CONFIG` resolves inside it, an absolute one is used as given. Default: the working directory. In Docker, set `DATA_DIR=/data` and bind-mount a host directory there so history survives upgrades. | | `DB_PATH` | SQLite file for chat history (default chat_log.db). Instances may share it. | | `HISTORY_LIMIT` | Messages of chat history in the prompt; the window is HISTORY_LIMIT to 1.5x (default 20). | | `MAX_TOKENS` | Reply cap in tokens, reasoning included (default 3000). | diff --git a/lib/config.py b/lib/config.py index 3368092..9569fbe 100644 --- a/lib/config.py +++ b/lib/config.py @@ -25,6 +25,7 @@ # Advanced and optional "ADMIN_CHAT_IDS": "Telegram IDs of the bot's admins (comma/space separated). Your user ID gets MCP-down alerts and may use /credits; a group's chat ID gets the alerts only. Admins of the groups the bot is in count as admins too.", "ADMIN_ONLY": "on|off. Only admins may DM the bot or get Sharesight data (default on).", + "DATA_DIR": "Directory for the bot's data files (the database, mcp_servers.json); relative DB_PATH / MCP_CONFIG resolve inside it. Default: the working directory.", "DB_PATH": "SQLite file for chat history (default chat_log.db). Instances may share it.", "HISTORY_LIMIT": "Messages of chat history in the prompt; the window is HISTORY_LIMIT to 1.5x (default 20).", "MAX_TOKENS": "Reply cap in tokens, reasoning included (default 3000).", @@ -108,6 +109,14 @@ def history_own_line_max(self) -> int | None: return 160 if self.token_saver else None +def data_path(env: Mapping[str, str], var: str, default: str) -> str: + """A data file named by env var `var` (or `default`), inside DATA_DIR when that is set. + An absolute path is used as given.""" + name = env.get(var, "").strip() or default + data_dir = env.get("DATA_DIR", "").strip() + return os.path.join(data_dir, name) if data_dir else name # join keeps an absolute `name` as is + + def _flag(env: Mapping[str, str], name: str, default: bool = False) -> bool: value = env.get(name, "").strip().lower() if not value: @@ -164,9 +173,9 @@ def load(env: Mapping[str, str] | None = None) -> Settings: search_model=search_model, max_tokens=int(env.get("MAX_TOKENS") or 3000), history_limit=int(env.get("HISTORY_LIMIT") or 20), - db_path=env.get("DB_PATH", "chat_log.db"), + db_path=data_path(env, "DB_PATH", "chat_log.db"), tz=ZoneInfo(env.get("BOT_TZ", "UTC")), - mcp_config=env.get("MCP_CONFIG", "mcp_servers.json"), + mcp_config=data_path(env, "MCP_CONFIG", "mcp_servers.json"), admin_chats=_ids(env.get("ADMIN_CHAT_IDS", "")), admin_only=_flag(env, "ADMIN_ONLY", default=True), movers_bots=frozenset( diff --git a/tests/test_config.py b/tests/test_config.py index 2f975d4..87d9264 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -6,6 +6,16 @@ from lib import config +def test_data_dir_holds_the_data_files(): + base = {"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "a"} + plain = config.load(base) + assert (plain.db_path, plain.mcp_config) == ("chat_log.db", "mcp_servers.json") + s = config.load({**base, "DATA_DIR": "/data"}) + assert (s.db_path, s.mcp_config) == ("/data/chat_log.db", "/data/mcp_servers.json") + s = config.load({**base, "DATA_DIR": "/data", "DB_PATH": "other.db", "MCP_CONFIG": "/etc/mcp.json"}) + assert (s.db_path, s.mcp_config) == ("/data/other.db", "/etc/mcp.json") # relative joins, absolute stays + + def test_both_keys_refused(): with pytest.raises(config.ConfigError, match="only one"): config.load({"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "a", "OPENROUTER_API_KEY": "b"}) diff --git a/tools/capture.py b/tools/capture.py index 60137b0..164b69f 100644 --- a/tools/capture.py +++ b/tools/capture.py @@ -106,7 +106,7 @@ async def capture(server: MCPServer, days: int) -> None: async def run(label: str, days: int) -> int: - registry = Registry.load(os.environ.get("MCP_CONFIG", "mcp_servers.json"), timeout=config.MCP_TIMEOUT, + registry = Registry.load(config.data_path(os.environ, "MCP_CONFIG", "mcp_servers.json"), timeout=config.MCP_TIMEOUT, max_output=10**7) server = registry.servers.get(label) if server is None: From 219b73da9887e53404bb206f771679b5ddeaeb4c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 06:06:45 +0000 Subject: [PATCH 02/34] DATA_DIR defaults to systemd STATE_DIRECTORY; MCP_CONFIG defaults next to bot.py Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- README.md | 21 +++++++++++++++++++-- lib/config.py | 21 +++++++++++++++------ tests/test_config.py | 13 +++++++++++-- tools/capture.py | 2 +- 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c8130f3..9f59b76 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,23 @@ The optional `label` is ignored; it only appears in `ps`/`top` so you can tell i A systemd unit just needs `ExecStart=/usr/bin/python3 /path/to/bot.py mimo` and an `Environment=` or `EnvironmentFile=` for the variables below. +### Running under systemd +With `StateDirectory=`, systemd sets `STATE_DIRECTORY` and the bot keeps its database there (`DATA_DIR` +defaults to it), so the code directory can stay read-only. `mcp_servers.json` is read from the checkout +unless `MCP_CONFIG` says otherwise. Copy an existing `chat_log.db` (and `-wal`/`-shm`) into the state +directory before the first start. + +```ini +[Service] +DynamicUser=yes +WorkingDirectory=/opt/grokbot +EnvironmentFile=/opt/grokbot.env +StateDirectory=grokbot +Environment=PYTHONUNBUFFERED=1 +ExecStart=/opt/grokbot/.venv/bin/python bot.py +Restart=on-failure +``` + ## Providers - `OPENROUTER_API_KEY` set -> OpenRouter (default model `z-ai/glm-5.3-flash`). - `XAI_API_KEY` set -> xAI (default model `grok-4.3`). @@ -246,11 +263,11 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. |---|---| | `ADMIN_CHAT_IDS` | Telegram IDs of the bot's admins (comma/space separated). Your user ID gets a message when an MCP server goes down and may use `/credits`; a group's chat ID gets the down-server messages only. Admins of the groups the bot is in count as admins too. | | `ADMIN_ONLY` | on|off. Only admins may DM the bot or get Sharesight data (default on). | -| `DATA_DIR` | Directory for the bot's data files (the database, `mcp_servers.json`); a relative `DB_PATH` / `MCP_CONFIG` resolves inside it, an absolute one is used as given. Default: the working directory. In Docker, set `DATA_DIR=/data` and bind-mount a host directory there so history survives upgrades. | +| `DATA_DIR` | Directory for the bot's data (the database); a relative `DB_PATH` resolves inside it, an absolute one is used as given. Default: systemd's `STATE_DIRECTORY` (see [Running under systemd](#running-under-systemd)), else the working directory. In Docker, set `DATA_DIR=/data` and bind-mount a host directory there so history survives upgrades. | | `DB_PATH` | SQLite file for chat history (default chat_log.db). Instances may share it. | | `HISTORY_LIMIT` | Messages of chat history in the prompt; the window is HISTORY_LIMIT to 1.5x (default 20). | | `MAX_TOKENS` | Reply cap in tokens, reasoning included (default 3000). | -| `MCP_CONFIG` | MCP server config file (default mcp_servers.json; missing = no MCP tools). | +| `MCP_CONFIG` | MCP server config file (default mcp_servers.json next to bot.py, whatever `DATA_DIR` is; missing = no MCP tools). | | `MOVERS_BOTS` | Usernames of bots whose end-of-day big-movers lists get explained. Empty (default) = feature off. | | `PORTFOLIO_NAMES` | Comma list of Sharesight portfolio names; a message naming one is treated as a portfolio question (default: the recipients' portfolio names). | | `POST_TO_GROUPS_FROM_DM` | on|off. A group admin can have the bot post in that group from a DM (default off). | diff --git a/lib/config.py b/lib/config.py index 9569fbe..dbbbcd1 100644 --- a/lib/config.py +++ b/lib/config.py @@ -6,6 +6,7 @@ import re from collections.abc import Mapping from dataclasses import dataclass +from pathlib import Path from zoneinfo import ZoneInfo # Every environment variable the bot reads, with a one-line description. README.md must @@ -25,11 +26,11 @@ # Advanced and optional "ADMIN_CHAT_IDS": "Telegram IDs of the bot's admins (comma/space separated). Your user ID gets MCP-down alerts and may use /credits; a group's chat ID gets the alerts only. Admins of the groups the bot is in count as admins too.", "ADMIN_ONLY": "on|off. Only admins may DM the bot or get Sharesight data (default on).", - "DATA_DIR": "Directory for the bot's data files (the database, mcp_servers.json); relative DB_PATH / MCP_CONFIG resolve inside it. Default: the working directory.", + "DATA_DIR": "Directory for the bot's data (the database); a relative DB_PATH resolves inside it. Default: systemd's STATE_DIRECTORY, else the working directory.", "DB_PATH": "SQLite file for chat history (default chat_log.db). Instances may share it.", "HISTORY_LIMIT": "Messages of chat history in the prompt; the window is HISTORY_LIMIT to 1.5x (default 20).", "MAX_TOKENS": "Reply cap in tokens, reasoning included (default 3000).", - "MCP_CONFIG": "MCP server config file (default mcp_servers.json; missing = no MCP tools).", + "MCP_CONFIG": "MCP server config file (default mcp_servers.json next to bot.py; missing = no MCP tools).", "MOVERS_BOTS": "Usernames of bots whose end-of-day big-movers lists get explained. Empty (default) = feature off.", "PORTFOLIO_NAMES": "Comma list of Sharesight portfolio names; a message naming one is treated as a portfolio question (default: the recipients' portfolio names).", "POST_TO_GROUPS_FROM_DM": "on|off. Lets an admin of a group the bot is in make it post there from a DM (default off).", @@ -109,14 +110,22 @@ def history_own_line_max(self) -> int | None: return 160 if self.token_saver else None +ROOT = Path(__file__).resolve().parent.parent # the checkout, where bot.py is + + def data_path(env: Mapping[str, str], var: str, default: str) -> str: - """A data file named by env var `var` (or `default`), inside DATA_DIR when that is set. - An absolute path is used as given.""" + """A data file named by env var `var` (or `default`), inside DATA_DIR, which defaults to systemd's + STATE_DIRECTORY (first entry if it lists several). An absolute path is used as given.""" name = env.get(var, "").strip() or default - data_dir = env.get("DATA_DIR", "").strip() + data_dir = env.get("DATA_DIR", "").strip() or env.get("STATE_DIRECTORY", "").split(":")[0].strip() return os.path.join(data_dir, name) if data_dir else name # join keeps an absolute `name` as is +def mcp_config_path(env: Mapping[str, str]) -> str: + """MCP_CONFIG, default mcp_servers.json next to bot.py. Deployment config, so DATA_DIR doesn't apply.""" + return env.get("MCP_CONFIG", "").strip() or str(ROOT / "mcp_servers.json") + + def _flag(env: Mapping[str, str], name: str, default: bool = False) -> bool: value = env.get(name, "").strip().lower() if not value: @@ -175,7 +184,7 @@ def load(env: Mapping[str, str] | None = None) -> Settings: history_limit=int(env.get("HISTORY_LIMIT") or 20), db_path=data_path(env, "DB_PATH", "chat_log.db"), tz=ZoneInfo(env.get("BOT_TZ", "UTC")), - mcp_config=data_path(env, "MCP_CONFIG", "mcp_servers.json"), + mcp_config=mcp_config_path(env), admin_chats=_ids(env.get("ADMIN_CHAT_IDS", "")), admin_only=_flag(env, "ADMIN_ONLY", default=True), movers_bots=frozenset( diff --git a/tests/test_config.py b/tests/test_config.py index 87d9264..e19faaf 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -8,14 +8,23 @@ def test_data_dir_holds_the_data_files(): base = {"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "a"} + mcp = str(config.ROOT / "mcp_servers.json") # next to bot.py, whatever DATA_DIR is plain = config.load(base) - assert (plain.db_path, plain.mcp_config) == ("chat_log.db", "mcp_servers.json") + assert (plain.db_path, plain.mcp_config) == ("chat_log.db", mcp) s = config.load({**base, "DATA_DIR": "/data"}) - assert (s.db_path, s.mcp_config) == ("/data/chat_log.db", "/data/mcp_servers.json") + assert (s.db_path, s.mcp_config) == ("/data/chat_log.db", mcp) s = config.load({**base, "DATA_DIR": "/data", "DB_PATH": "other.db", "MCP_CONFIG": "/etc/mcp.json"}) assert (s.db_path, s.mcp_config) == ("/data/other.db", "/etc/mcp.json") # relative joins, absolute stays +def test_data_dir_defaults_to_systemd_state_directory(): + base = {"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "a"} + assert config.load({**base, "STATE_DIRECTORY": "/var/lib/x"}).db_path == "/var/lib/x/chat_log.db" + assert config.load({**base, "STATE_DIRECTORY": "/a:/b"}).db_path == "/a/chat_log.db" # first of several + assert config.load({**base, "STATE_DIRECTORY": "/a", "DATA_DIR": "/data"}).db_path == "/data/chat_log.db" + assert config.load({**base, "STATE_DIRECTORY": "/a", "DB_PATH": "/x/y.db"}).db_path == "/x/y.db" + + def test_both_keys_refused(): with pytest.raises(config.ConfigError, match="only one"): config.load({"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "a", "OPENROUTER_API_KEY": "b"}) diff --git a/tools/capture.py b/tools/capture.py index 164b69f..6f9a916 100644 --- a/tools/capture.py +++ b/tools/capture.py @@ -106,7 +106,7 @@ async def capture(server: MCPServer, days: int) -> None: async def run(label: str, days: int) -> int: - registry = Registry.load(config.data_path(os.environ, "MCP_CONFIG", "mcp_servers.json"), timeout=config.MCP_TIMEOUT, + registry = Registry.load(config.mcp_config_path(os.environ), timeout=config.MCP_TIMEOUT, max_output=10**7) server = registry.servers.get(label) if server is None: From 2857faee85b500878bfa8730c011ad7c2c56cc94 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 06:23:21 +0000 Subject: [PATCH 03/34] Portfolio prompt: also use market tools when asked; paragraph template only for portfolio comparisons Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/prompts.py | 7 ++++--- tests/test_handlers.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/prompts.py b/lib/prompts.py index 3c58121..d9cb3da 100644 --- a/lib/prompts.py +++ b/lib/prompts.py @@ -85,9 +85,10 @@ def chat_prompt(*, transcript: str, sender: str, private: bool, reply_quote: str "earlier replies, and follow the formatting rules in your instructions even " "where your earlier replies didn't.") if fresh: # a portfolio question: stale numbers and refusals in the history must not be reused - tail += (" Get current figures with your portfolio tools before answering; don't reuse " - "numbers or refusals from earlier replies. For a comparison or a \"why\", write 2-3 " - "short paragraphs on the tailwinds and headwinds of each portfolio, not a summary.") + tail += (" Get current figures with your portfolio tools before answering (and your market tools " + "for anything else asked, such as prices or a comparison with market data); don't reuse " + "numbers or refusals from earlier replies. When comparing portfolios, or asked \"why\", " + "write 2-3 short paragraphs on the tailwinds and headwinds of each portfolio, not a summary.") return head + tail + down diff --git a/tests/test_handlers.py b/tests/test_handlers.py index f3ae829..b46dc08 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -104,6 +104,7 @@ async def test_portfolio_question_reminds_the_model_to_fetch_fresh_figures(env, await run(h, user_msg(ctx.bot, "@stockbot how is NVDA looking?", mid=2)) assert "portfolio tools before answering" in backend.seen[0]["prompt"] assert "tailwinds and headwinds" in backend.seen[0]["prompt"] + assert "your market tools" in backend.seen[0]["prompt"] # not portfolio tools only (Yahoo comparisons) assert "portfolio tools before answering" not in backend.seen[1]["prompt"] From 4cea09b8c29252dbd948a66b16578388ff3cd7ab Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 06:27:13 +0000 Subject: [PATCH 04/34] Nudge once when the user names a data source (e.g. Yahoo) the model never called; log tools offered Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/ask.py | 4 +++- lib/llm/base.py | 1 + lib/llm/runner.py | 25 ++++++++++++++++++++++++- lib/telegram/handlers.py | 3 ++- tests/test_handlers.py | 10 ++++++++++ tests/test_runner.py | 20 ++++++++++++++++++++ 6 files changed, 60 insertions(+), 3 deletions(-) diff --git a/lib/ask.py b/lib/ask.py index 9c46368..f050229 100644 --- a/lib/ask.py +++ b/lib/ask.py @@ -44,7 +44,8 @@ def _public(servers: list) -> list: async def ask(ctx: Ctx, parts: list[dict], route: Route, *, must_search: bool = False, cache_key: str | None = None, on_text=None, kind: str = "chat", - chat_id: int = 0, extra_tools=(), admin: bool = True) -> Answer: + chat_id: int = 0, extra_tools=(), admin: bool = True, + named_sources: tuple[str, ...] = ()) -> Answer: """Ask the configured provider and record the usage. `route` says which tools to offer; must_search forces a search (and offers nothing else). A message the gate gave fewer tools than exist that gets back NEEDS_TOOLS is asked again with everything on. A non-admin never @@ -68,6 +69,7 @@ async def ask(ctx: Ctx, parts: list[dict], route: Route, *, must_search: bool = on_text=_hide_needs_tools(on_text) if (route.simple or route.partial) else on_text, no_tools_system=no_tools_system(ctx.first_name, saver=saver), max_tokens=st.max_tokens, + named_sources=tuple(s.label for s in route.servers if s.label in named_sources), ) answer = await policy.ask(ctx.backend, ctx.registry, req) await _record(ctx, req.model, kind, chat_id, answer) diff --git a/lib/llm/base.py b/lib/llm/base.py index 0ad329d..57a8894 100644 --- a/lib/llm/base.py +++ b/lib/llm/base.py @@ -65,6 +65,7 @@ class Request: no_tools_system: str = "" # system prompt for the retry without tools local: dict[str, Callable[[dict], Awaitable[str]]] = field(default_factory=dict) # tools run in the bot max_tokens: int | None = None # reply cap incl. reasoning; None = the provider's setting + named_sources: tuple[str, ...] = () # MCP server labels the user named; one never called gets a nudge @dataclass diff --git a/lib/llm/runner.py b/lib/llm/runner.py index 9a3670e..ab803e1 100644 --- a/lib/llm/runner.py +++ b/lib/llm/runner.py @@ -25,6 +25,8 @@ "include_sales", "response_format"} SAME_AS_EARLIER = "Same call as one already answered in this conversation; use that result." WRITE_NOW = "Write the answer now, from the tool results above. Keep any thinking brief." +USE_NAMED = ("You were asked to use {names} and haven't called it yet. Call its tools now, using what " + "you've found so far, then answer.") TOO_MANY_SEARCHES = "Skipped: too many searches at once. Use what you already have." @@ -163,12 +165,26 @@ async def _recover_fake_calls(step: Step, backend: Backend, usage: Usage) -> str return "\n\n".join(parts) + "\n\nAnswer the question now using these results." +def _offered(req: Request, registry: Registry) -> dict[str, int]: + """Tools offered per MCP server label, for the log.""" + counts: dict[str, int] = {} + for t in req.tools: + found = registry.lookup(t.name) + label = found[0].label if found else "local" + counts[label] = counts.get(label, 0) + 1 + return counts + + async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: bool = True) -> Answer: t0 = time.monotonic() conv = backend.start(req) total, rounds, tool_calls = Usage(), 0, 0 seen: dict[tuple[str, str], int] = {} searched = 0 + used: set[str] = set() # labels of the MCP servers called so far + nudged = False + log.info("Tools offered: %s%s", ", ".join(f"{label} {n}" for label, n in _offered(req, registry).items()) + or "none", " + search" if req.search else "") while True: final = rounds >= config.MAX_TOOL_ROUNDS choice = "none" if final else ("required" if req.must_search and rounds == 0 else None) @@ -185,7 +201,8 @@ async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: boo break if step.calls: tool_calls += len(step.calls) - results = await run_calls(step.calls, backend, registry, req, seen, dedupe, total) + used |= {found[0].label for c in step.calls if (found := registry.lookup(c.name))} + results =await run_calls(step.calls, backend, registry, req, seen, dedupe, total) backend.add_results(conv, step, results) continue nudge = await _recover_fake_calls(step, backend, total) if req.search else None @@ -193,6 +210,12 @@ async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: boo backend.add_user_message(conv, nudge) req = dataclasses.replace(req, search=False) # recovery happens once continue + missing = [n for n in req.named_sources if n not in used] + if missing and not nudged: # the user named a source the model skipped: ask once more + nudged = True + log.info("Nudging: %s named but not used", ", ".join(missing)) + backend.add_user_message(conv, USE_NAMED.format(names=" and ".join(missing))) + continue break if req.must_search and not (tool_calls or searched): log.warning("This request needed a search, but the answer shows no sign of one") diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index d7d8009..c26836d 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -236,9 +236,10 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s fresh = is_portfolio_question(context, st) and any(s.cfg.get("gate") == "portfolio" for s in chosen.servers) parts = [{"type": "text", "text": await self._chat_text(msg, trig, fresh=fresh)}] parts += await self._images(msg, trig) + named = tuple(s.label for s in chosen.servers if re.search(rf"\b{re.escape(s.label)}\b", trig.text, re.I)) answer = await ask(ctx, parts, chosen, cache_key=f"chat-{chat_id}", on_text=on_text, chat_id=chat_id, extra_tools=extra, - admin=admin) + admin=admin, named_sources=named) return format_answer(answer.text, movers_bots=st.movers_bots, summary=trig.movers) diff --git a/tests/test_handlers.py b/tests/test_handlers.py index b46dc08..340ac0f 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -97,6 +97,16 @@ async def test_market_question_offers_tools_up_front(env, store): assert backend.seen[0]["tools"] == ["yahoo__get_quote"] and backend.seen[0]["search"] is True +async def test_a_source_named_in_the_message_is_nudged_when_the_model_skips_it(env, store): + yahoo = FakeMcp("yahoo") + ctx, backend, h = make_ctx(env, store, [step("from memory"), step(calls=[("1", "yahoo__get_quote", "{}")], + finish="tool_calls"), step("quoted")], + servers=[yahoo]) + m = user_msg(ctx.bot, "@stockbot what does yahoo say about NVDA?") + await run(h, m) + assert yahoo.calls and "quoted" in m.replies[0]["text"] + + async def test_portfolio_question_reminds_the_model_to_fetch_fresh_figures(env, store): sharesight = FakeMcp("sharesight", gate="portfolio", tools=("list_portfolios",)) ctx, backend, h = make_ctx(env, store, [step("a"), step("b")], servers=[sharesight]) diff --git a/tests/test_runner.py b/tests/test_runner.py index 6ba5e2d..62ea753 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -35,6 +35,26 @@ async def test_tool_round_trip_runs_mcp_and_returns_answer(): assert ans.usage.tokens_in == 200 and ans.usage.cached == 80 +async def test_named_source_that_was_skipped_gets_one_nudge(caplog): + caplog.set_level("INFO", logger="bot") + sharesight, yahoo = FakeMcp("sharesight", tools=("list",)), FakeMcp("yahoo") + reg = registry(sharesight, yahoo) + tools = [ToolDef(n, "d") for m in (sharesight, yahoo) for n in m.fn_names] + r = req(tools=tools, named_sources=("yahoo",)) + b = ScriptedBackend([step(calls=[("1", "sharesight__list", "{}")], finish="tool_calls"), step("no yahoo"), + step(calls=[("2", "yahoo__get_quote", "{}")], finish="tool_calls"), step("compared")]) + ans = await run(b, reg, r) + assert ans.text == "compared" and ans.rounds == 4 and yahoo.calls == [("get_quote", {})] + assert "yahoo named but not used" in caplog.text and "Tools offered: sharesight 1, yahoo 1" in caplog.text + # the model ignores the nudge: asked once, then the answer stands + b2 = ScriptedBackend([step("no yahoo"), step("still none")]) + assert (await run(b2, reg, req(tools=tools, named_sources=("yahoo",)))).text == "still none" + # a source that was used, or none named, is never nudged + b3 = ScriptedBackend([step(calls=[("1", "yahoo__get_quote", "{}")], finish="tool_calls"), step("done")]) + assert (await run(b3, reg, req(tools=tools, named_sources=("yahoo",)))).rounds == 2 + assert (await run(ScriptedBackend([step("hi")]), reg, req(tools=tools))).rounds == 1 + + async def test_final_round_forces_answer_without_tools(): mcp = FakeMcp() loop = [step(calls=[(str(i), "yahoo__get_quote", f'{{"n":{i}}}')], finish="tool_calls") From 7a85a4c7e4566a421689252d0ef8b04dc0415586 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 06:33:15 +0000 Subject: [PATCH 05/34] Revert "Nudge once when the user names a data source (e.g. Yahoo) the model never called; log tools offered" This reverts commit 4cea09b. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/ask.py | 4 +--- lib/llm/base.py | 1 - lib/llm/runner.py | 25 +------------------------ lib/telegram/handlers.py | 3 +-- tests/test_handlers.py | 10 ---------- tests/test_runner.py | 20 -------------------- 6 files changed, 3 insertions(+), 60 deletions(-) diff --git a/lib/ask.py b/lib/ask.py index f050229..9c46368 100644 --- a/lib/ask.py +++ b/lib/ask.py @@ -44,8 +44,7 @@ def _public(servers: list) -> list: async def ask(ctx: Ctx, parts: list[dict], route: Route, *, must_search: bool = False, cache_key: str | None = None, on_text=None, kind: str = "chat", - chat_id: int = 0, extra_tools=(), admin: bool = True, - named_sources: tuple[str, ...] = ()) -> Answer: + chat_id: int = 0, extra_tools=(), admin: bool = True) -> Answer: """Ask the configured provider and record the usage. `route` says which tools to offer; must_search forces a search (and offers nothing else). A message the gate gave fewer tools than exist that gets back NEEDS_TOOLS is asked again with everything on. A non-admin never @@ -69,7 +68,6 @@ async def ask(ctx: Ctx, parts: list[dict], route: Route, *, must_search: bool = on_text=_hide_needs_tools(on_text) if (route.simple or route.partial) else on_text, no_tools_system=no_tools_system(ctx.first_name, saver=saver), max_tokens=st.max_tokens, - named_sources=tuple(s.label for s in route.servers if s.label in named_sources), ) answer = await policy.ask(ctx.backend, ctx.registry, req) await _record(ctx, req.model, kind, chat_id, answer) diff --git a/lib/llm/base.py b/lib/llm/base.py index 57a8894..0ad329d 100644 --- a/lib/llm/base.py +++ b/lib/llm/base.py @@ -65,7 +65,6 @@ class Request: no_tools_system: str = "" # system prompt for the retry without tools local: dict[str, Callable[[dict], Awaitable[str]]] = field(default_factory=dict) # tools run in the bot max_tokens: int | None = None # reply cap incl. reasoning; None = the provider's setting - named_sources: tuple[str, ...] = () # MCP server labels the user named; one never called gets a nudge @dataclass diff --git a/lib/llm/runner.py b/lib/llm/runner.py index ab803e1..9a3670e 100644 --- a/lib/llm/runner.py +++ b/lib/llm/runner.py @@ -25,8 +25,6 @@ "include_sales", "response_format"} SAME_AS_EARLIER = "Same call as one already answered in this conversation; use that result." WRITE_NOW = "Write the answer now, from the tool results above. Keep any thinking brief." -USE_NAMED = ("You were asked to use {names} and haven't called it yet. Call its tools now, using what " - "you've found so far, then answer.") TOO_MANY_SEARCHES = "Skipped: too many searches at once. Use what you already have." @@ -165,26 +163,12 @@ async def _recover_fake_calls(step: Step, backend: Backend, usage: Usage) -> str return "\n\n".join(parts) + "\n\nAnswer the question now using these results." -def _offered(req: Request, registry: Registry) -> dict[str, int]: - """Tools offered per MCP server label, for the log.""" - counts: dict[str, int] = {} - for t in req.tools: - found = registry.lookup(t.name) - label = found[0].label if found else "local" - counts[label] = counts.get(label, 0) + 1 - return counts - - async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: bool = True) -> Answer: t0 = time.monotonic() conv = backend.start(req) total, rounds, tool_calls = Usage(), 0, 0 seen: dict[tuple[str, str], int] = {} searched = 0 - used: set[str] = set() # labels of the MCP servers called so far - nudged = False - log.info("Tools offered: %s%s", ", ".join(f"{label} {n}" for label, n in _offered(req, registry).items()) - or "none", " + search" if req.search else "") while True: final = rounds >= config.MAX_TOOL_ROUNDS choice = "none" if final else ("required" if req.must_search and rounds == 0 else None) @@ -201,8 +185,7 @@ async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: boo break if step.calls: tool_calls += len(step.calls) - used |= {found[0].label for c in step.calls if (found := registry.lookup(c.name))} - results =await run_calls(step.calls, backend, registry, req, seen, dedupe, total) + results = await run_calls(step.calls, backend, registry, req, seen, dedupe, total) backend.add_results(conv, step, results) continue nudge = await _recover_fake_calls(step, backend, total) if req.search else None @@ -210,12 +193,6 @@ async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: boo backend.add_user_message(conv, nudge) req = dataclasses.replace(req, search=False) # recovery happens once continue - missing = [n for n in req.named_sources if n not in used] - if missing and not nudged: # the user named a source the model skipped: ask once more - nudged = True - log.info("Nudging: %s named but not used", ", ".join(missing)) - backend.add_user_message(conv, USE_NAMED.format(names=" and ".join(missing))) - continue break if req.must_search and not (tool_calls or searched): log.warning("This request needed a search, but the answer shows no sign of one") diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index c26836d..d7d8009 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -236,10 +236,9 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s fresh = is_portfolio_question(context, st) and any(s.cfg.get("gate") == "portfolio" for s in chosen.servers) parts = [{"type": "text", "text": await self._chat_text(msg, trig, fresh=fresh)}] parts += await self._images(msg, trig) - named = tuple(s.label for s in chosen.servers if re.search(rf"\b{re.escape(s.label)}\b", trig.text, re.I)) answer = await ask(ctx, parts, chosen, cache_key=f"chat-{chat_id}", on_text=on_text, chat_id=chat_id, extra_tools=extra, - admin=admin, named_sources=named) + admin=admin) return format_answer(answer.text, movers_bots=st.movers_bots, summary=trig.movers) diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 340ac0f..b46dc08 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -97,16 +97,6 @@ async def test_market_question_offers_tools_up_front(env, store): assert backend.seen[0]["tools"] == ["yahoo__get_quote"] and backend.seen[0]["search"] is True -async def test_a_source_named_in_the_message_is_nudged_when_the_model_skips_it(env, store): - yahoo = FakeMcp("yahoo") - ctx, backend, h = make_ctx(env, store, [step("from memory"), step(calls=[("1", "yahoo__get_quote", "{}")], - finish="tool_calls"), step("quoted")], - servers=[yahoo]) - m = user_msg(ctx.bot, "@stockbot what does yahoo say about NVDA?") - await run(h, m) - assert yahoo.calls and "quoted" in m.replies[0]["text"] - - async def test_portfolio_question_reminds_the_model_to_fetch_fresh_figures(env, store): sharesight = FakeMcp("sharesight", gate="portfolio", tools=("list_portfolios",)) ctx, backend, h = make_ctx(env, store, [step("a"), step("b")], servers=[sharesight]) diff --git a/tests/test_runner.py b/tests/test_runner.py index 62ea753..6ba5e2d 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -35,26 +35,6 @@ async def test_tool_round_trip_runs_mcp_and_returns_answer(): assert ans.usage.tokens_in == 200 and ans.usage.cached == 80 -async def test_named_source_that_was_skipped_gets_one_nudge(caplog): - caplog.set_level("INFO", logger="bot") - sharesight, yahoo = FakeMcp("sharesight", tools=("list",)), FakeMcp("yahoo") - reg = registry(sharesight, yahoo) - tools = [ToolDef(n, "d") for m in (sharesight, yahoo) for n in m.fn_names] - r = req(tools=tools, named_sources=("yahoo",)) - b = ScriptedBackend([step(calls=[("1", "sharesight__list", "{}")], finish="tool_calls"), step("no yahoo"), - step(calls=[("2", "yahoo__get_quote", "{}")], finish="tool_calls"), step("compared")]) - ans = await run(b, reg, r) - assert ans.text == "compared" and ans.rounds == 4 and yahoo.calls == [("get_quote", {})] - assert "yahoo named but not used" in caplog.text and "Tools offered: sharesight 1, yahoo 1" in caplog.text - # the model ignores the nudge: asked once, then the answer stands - b2 = ScriptedBackend([step("no yahoo"), step("still none")]) - assert (await run(b2, reg, req(tools=tools, named_sources=("yahoo",)))).text == "still none" - # a source that was used, or none named, is never nudged - b3 = ScriptedBackend([step(calls=[("1", "yahoo__get_quote", "{}")], finish="tool_calls"), step("done")]) - assert (await run(b3, reg, req(tools=tools, named_sources=("yahoo",)))).rounds == 2 - assert (await run(ScriptedBackend([step("hi")]), reg, req(tools=tools))).rounds == 1 - - async def test_final_round_forces_answer_without_tools(): mcp = FakeMcp() loop = [step(calls=[(str(i), "yahoo__get_quote", f'{{"n":{i}}}')], finish="tool_calls") From 6f23490f4b60e221872904941539edb5a29d9ac8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 07:47:02 +0000 Subject: [PATCH 06/34] monthly_returns tool: month-by-month Sharesight returns in one call, sold holdings included Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- CLAUDE.md | 2 +- lib/features/monthly.py | 107 +++++++++++++++++++++++++++++++++++++++ lib/llm/runner.py | 1 + lib/mcp/server.py | 6 ++- lib/telegram/handlers.py | 4 +- tests/test_handlers.py | 2 +- tests/test_mcp.py | 8 +++ tests/test_monthly.py | 56 ++++++++++++++++++++ tests/test_runner.py | 8 +++ 9 files changed, 190 insertions(+), 4 deletions(-) create mode 100644 lib/features/monthly.py create mode 100644 tests/test_monthly.py diff --git a/CLAUDE.md b/CLAUDE.md index fac1272..e6178f9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ MCP data tools and web search. User-facing docs are in `README.md`; this file is `openrouter.py` + `zai.py` (the only provider-specific code), `runner.py` (THE tool loop), `policy.py` (THE retry ladder), `gate.py` (which tools a message needs). - `prompts.py` all prompts. `ask.py` one entry point: route -> Request -> policy.ask -> usage -> NEEDS_TOOLS rerun. - `telegram/`: `handlers.py` (trigger -> request -> reply), `send.py`, `draft.py` (typing + streaming - drafts), `commands.py` (/credits), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `post.py`. + drafts), `commands.py` (/credits), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `post.py`, `monthly.py` (local `monthly_returns` tool: one call instead of a report per month). - `app.py` wiring/startup; `context.py` `Ctx` (settings, store, backend, registry, bot). - `lib/` holds only what the bot needs to run. Developer helpers live in `tools/`: `report.py` (`make prompt-report`), `capture.py` (real MCP output for fixtures); the bot never imports them. diff --git a/lib/features/monthly.py b/lib/features/monthly.py new file mode 100644 index 0000000..ecaba45 --- /dev/null +++ b/lib/features/monthly.py @@ -0,0 +1,107 @@ +"""monthly_returns: a Sharesight portfolio's return per month, fetched by the bot in one tool call +instead of one model-driven report per month (64 calls and 290k tokens for "best month ever").""" + +from __future__ import annotations + +import asyncio +import calendar +import json +import logging +from datetime import date, datetime + +from ..context import Ctx +from ..mcp.schema import ToolDef + +log = logging.getLogger("bot") + +MAX_MONTHS = 120 +TOOL = ToolDef( + "monthly_returns", + "Total return of a Sharesight portfolio for each month, with the best and worst month. Use it for " + "best/worst month, month-by-month or year-by-year questions, instead of fetching one " + "get_performance_report per period. Sold holdings are included. Omit start for 'since inception'.", + {"type": "object", "required": ["portfolio_id"], "properties": { + "portfolio_id": {"type": "number", "description": "From list_portfolios"}, + "start": {"type": "string", "description": "First month, YYYY-MM (default: the portfolio's inception)"}, + "end": {"type": "string", "description": "Last month, YYYY-MM (default: this month)"}}}) + + +def _month(text: str) -> tuple[int, int] | None: + try: + d = datetime.strptime(str(text).strip()[:7], "%Y-%m") + except ValueError: + return None + return d.year, d.month + + +def month_windows(first: tuple[int, int], last: tuple[int, int], today: date) -> list[tuple[str, date, date]]: + """(label, first day, last day) per month from `first` to `last`; the current month ends today.""" + out, (y, m) = [], first + while (y, m) <= last and len(out) < MAX_MONTHS: + end = min(date(y, m, calendar.monthrange(y, m)[1]), today) + out.append((f"{y}-{m:02d}", date(y, m, 1), end)) + y, m = (y + 1, 1) if m == 12 else (y, m + 1) + return out + + +def _report(text: str) -> dict | None: + try: + report = json.loads(text).get("report") + except (ValueError, AttributeError): + return None + return report if isinstance(report, dict) and report.get("total_gain_percent") is not None else None + + +async def _inception(server, portfolio_id: int) -> tuple[int, int] | None: + try: + portfolios = json.loads(await server.call("list_portfolios", {})).get("portfolios") or [] + except (ValueError, AttributeError): + return None + for p in portfolios: + if isinstance(p, dict) and p.get("id") == portfolio_id: + return _month(p.get("inception_date") or "") + return None + + +def tool(ctx: Ctx): + """(definition, function) of monthly_returns, backed by the Sharesight server (the one with a + get_performance_report tool).""" + + async def monthly(args: dict) -> str: + server = next((s for s in ctx.registry.up() if "get_performance_report" in s.fn_names.values()), None) + if server is None: + return "Error: the portfolio data source isn't connected." + try: + pid = int(args.get("portfolio_id")) + except (TypeError, ValueError): + return "Error: give the portfolio_id (call list_portfolios first)." + today = datetime.now(ctx.st.tz).date() + first = _month(args["start"]) if args.get("start") else await _inception(server, pid) + last = (_month(args["end"]) if args.get("end") else None) or (today.year, today.month) + if first is None: + return "Error: couldn't tell when the portfolio started; pass start as YYYY-MM." + windows = month_windows(first, last, today) + if not windows: + return "Error: no months in that range." + results = await asyncio.gather(*(server.call("get_performance_report", { + "portfolio_id": pid, "start_date": a.isoformat(), "end_date": b.isoformat(), + "_keep_sales": True}) for _, a, b in windows)) + rows, missing = [], 0 + for (label, _, _), text in zip(windows, results, strict=True): + report = _report(text) + if report is None: + missing += 1 + continue + rows.append((label, float(report["total_gain_percent"]), float(report.get("total_gain") or 0), + report.get("currency") or "")) + if not rows: + return "Error: no monthly reports came back." + best, worst = max(rows, key=lambda r: r[1]), min(rows, key=lambda r: r[1]) + lines = [f"Total return per month ({rows[0][3]}), {rows[0][0]} to {rows[-1][0]}, sold holdings included:"] + lines += [f"{label} {pct:+.2f}% {gain:+,.0f}" for label, pct, gain, _ in rows] + lines.append(f"Best: {best[0]} {best[1]:+.2f}%. Worst: {worst[0]} {worst[1]:+.2f}%." + + (f" {missing} months had no data." if missing else "")) + log.info("monthly_returns: %d months for portfolio %s", len(rows), pid) + return "\n".join(lines) + + return TOOL, monthly diff --git a/lib/llm/runner.py b/lib/llm/runner.py index 9a3670e..450e169 100644 --- a/lib/llm/runner.py +++ b/lib/llm/runner.py @@ -71,6 +71,7 @@ async def _run_mcp(call: Call, registry: Registry) -> str: args = json.loads(call.arguments or "{}") if not isinstance(args, dict): return f"Error: arguments for {tool} must be a JSON object." + args.pop("_keep_sales", None) # reserved for the bot's own tools log.info("MCP %s.%s %s", server.label, tool, brief_args(args)) out = await server.call(tool, args) # A tiny result is usually an empty list or an error: show it, so "why did it say diff --git a/lib/mcp/server.py b/lib/mcp/server.py index 9707020..cad028b 100644 --- a/lib/mcp/server.py +++ b/lib/mcp/server.py @@ -232,8 +232,12 @@ async def _warm_once(self) -> None: async def _call(self, tool: str, args: dict) -> str: holdings_only = bool(self.cfg.get("current_holdings_only")) + keep_sales = args.get("_keep_sales") # set by the bot's own tools (history), never by the model + args = {k: v for k, v in args.items() if k != "_keep_sales"} if holdings_only and tool == "get_performance_report": - args = {**args, "include_sales": False} # Sharesight's own switch for sold holdings + args = {**args, "include_sales": bool(keep_sales)} # Sharesight's own switch for sold holdings + if keep_sales: + holdings_only = False # closed holdings stay in the report too async with self._sem: result = await asyncio.wait_for(self.session.call_tool(tool, args), self.timeout) parts = [c.text if c.type == "text" else f"[{c.type} content omitted]" for c in result.content] diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index d7d8009..71d8916 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -15,7 +15,7 @@ from .. import config from ..ask import ask from ..context import Ctx -from ..features import dm_buttons, holding_news, movers, post +from ..features import dm_buttons, holding_news, monthly, movers, post from ..history import compact, format_rows from ..llm.gate import Route, is_portfolio_question, route from ..msgtext import describe, sender_name @@ -229,6 +229,8 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s s.cfg.get("gate") == "portfolio" for s in ctx.registry.up()): return PORTFOLIO_ADMINS_ONLY extra = [] + if admin and any(s.cfg.get("gate") == "portfolio" for s in chosen.servers): + extra.append(monthly.tool(ctx)) if st.post_to_groups and trig.private and post.wants_post(trig.text): extra = [post.tool(ctx, msg.from_user.id, msg.from_user.full_name or "")] if chosen.simple: # the model must not be told it has no tools diff --git a/tests/test_handlers.py b/tests/test_handlers.py index b46dc08..ecd0d22 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -253,7 +253,7 @@ async def test_non_admin_gets_no_portfolio_data_in_a_group(env, store): assert asked.replies[0]["text"] == "Portfolio data is for admins only." and backend.seen == [] admin = user_msg(ctx.bot, "@stockbot how is my portfolio doing?", user_id=5, mid=2) await run(h, admin) - assert backend.seen[0]["tools"] == ["sharesight__list_portfolios"] + assert backend.seen[0]["tools"] == ["sharesight__list_portfolios", "monthly_returns"] # admins also get the monthly tool async def test_non_admin_never_gets_portfolio_tools_even_on_the_rerun(env, store): diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 42ec112..1f073a8 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -224,6 +224,14 @@ async def test_sharesight_holdings_only_flag_sets_include_sales(): assert s.session.calls[0][1]["include_sales"] is False +async def test_bots_own_tools_can_keep_sold_holdings(): + s = MCPServer("sharesight", {"current_holdings_only": True}, timeout=5, max_output=10_000) + s.session = FakeSession() + await s.call("get_performance_report", {"portfolio_id": 1, "_keep_sales": True}) + sent = s.session.calls[0][1] + assert sent["include_sales"] is True and "_keep_sales" not in sent + + async def test_output_cap(): s = MCPServer("y", {}, timeout=5, max_output=20) s.session = FakeSession() diff --git a/tests/test_monthly.py b/tests/test_monthly.py new file mode 100644 index 0000000..df3acdb --- /dev/null +++ b/tests/test_monthly.py @@ -0,0 +1,56 @@ +import json +from datetime import date + +from lib import config +from lib.context import Ctx +from lib.features import monthly + +from .conftest import FakeBot +from .fakes import FakeMcp, ScriptedBackend, registry + + +class Sharesight(FakeMcp): + """Returns a report per month: +1% in 2026-01, +5% in 2026-02, nothing before 2026.""" + + async def call(self, tool, args): + self.calls.append((tool, args)) + if tool == "list_portfolios": + return json.dumps({"portfolios": [{"id": 7, "inception_date": "2025-12-20"}]}) + start = args["start_date"] + pct = {"2025-12": -2.0, "2026-01": 1.0, "2026-02": 5.0}.get(start[:7]) + if pct is None: + return "Tool error: no data" + return json.dumps({"report": {"total_gain_percent": pct, "total_gain": pct * 100, "currency": "AUD"}}) + + +def make(env, ss): + return Ctx(config.load(env), None, ScriptedBackend([]), registry(ss), FakeBot()) + + +async def test_months_run_from_inception_with_sold_holdings_and_best_worst(env): + ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) + defn, fn = monthly.tool(make(env, ss)) + out = await fn({"portfolio_id": 7, "end": "2026-03"}) + assert defn.name == "monthly_returns" + assert "2025-12 -2.00% -200" in out and "2026-02 +5.00% +500" in out + assert "Best: 2026-02 +5.00%. Worst: 2025-12 -2.00%. 1 months had no data." in out # March: no data + reports = [a for t, a in ss.calls if t == "get_performance_report"] + assert [r["start_date"] for r in reports][:2] == ["2025-12-01", "2026-01-01"] + assert reports[0]["end_date"] == "2025-12-31" and all(r["_keep_sales"] is True for r in reports) + + +async def test_start_given_skips_the_inception_lookup_and_bad_input_is_reported(env): + ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) + _, fn = monthly.tool(make(env, ss)) + out = await fn({"portfolio_id": 7, "start": "2026-01", "end": "2026-02"}) + assert out.splitlines()[1].startswith("2026-01 +1.00%") and not any(t == "list_portfolios" for t, _ in ss.calls) + assert "portfolio_id" in await fn({}) + assert "start as YYYY-MM" in await fn({"portfolio_id": 99}) # unknown portfolio: no inception date + _, none = monthly.tool(Ctx(config.load(env), None, ScriptedBackend([]), registry(), FakeBot())) + assert "isn't connected" in await none({"portfolio_id": 7}) + + +def test_month_windows_end_today_in_the_current_month(): + w = monthly.month_windows((2025, 11), (2026, 2), date(2026, 2, 10)) + assert [x[0] for x in w] == ["2025-11", "2025-12", "2026-01", "2026-02"] + assert w[0][2] == date(2025, 11, 30) and w[-1][2] == date(2026, 2, 10) diff --git a/tests/test_runner.py b/tests/test_runner.py index 6ba5e2d..f8f8fa9 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -35,6 +35,14 @@ async def test_tool_round_trip_runs_mcp_and_returns_answer(): assert ans.usage.tokens_in == 200 and ans.usage.cached == 80 +async def test_the_model_cannot_ask_for_sold_holdings_to_be_kept(): + from lib.llm.base import Call + mcp = FakeMcp("sharesight", tools=("get_performance_report",)) + await run_calls([Call("1", "sharesight__get_performance_report", '{"portfolio_id":1,"_keep_sales":true}')], + ScriptedBackend([]), registry(mcp), with_tools(mcp), {}, True, Usage()) + assert mcp.calls == [("get_performance_report", {"portfolio_id": 1})] + + async def test_final_round_forces_answer_without_tools(): mcp = FakeMcp() loop = [step(calls=[(str(i), "yahoo__get_quote", f'{{"n":{i}}}')], finish="tool_calls") From ae80cb860cc165455fc926b76777fd8fde6f8730 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 07:50:13 +0000 Subject: [PATCH 07/34] Gate: any word ending in smsf is a portfolio name Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/llm/gate.py | 2 +- tests/test_runner.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/llm/gate.py b/lib/llm/gate.py index 53c7432..b80dbc0 100644 --- a/lib/llm/gate.py +++ b/lib/llm/gate.py @@ -20,7 +20,7 @@ # A portfolio question: strong phrases always count; "my " phrases count; weak words # count only when no ticker is named ("performance of NVDA" is a market question). PORTFOLIO_STRONG = re.compile( - r"\b(portfolios?|holdings?|sharesight|smsf|superannuation|super fund|net worth|" + r"\b(portfolios?|holdings?|sharesight|\w*smsf|superannuation|super fund|net worth|" r"what do i (?:own|hold)|how am i doing|am i (?:up|down)|" r"what (?:did|have) i (?:make|made|lose|lost)|how did i do|" r"(?:i|we) (?:out|under)perform\w*|(?:out|under)perform\w* (?:me|my|us|our)|" diff --git a/tests/test_runner.py b/tests/test_runner.py index f8f8fa9..fc4707d 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -254,6 +254,7 @@ def test_gate_full_when_forced_or_token_saver_off(settings, env): "my position in CBA", "am I up or down today", "show performance this year", "bob smsf vs bob personal", "are my dividends coming", "why did I outperform Alice over the past month?", "how did I do last month", "did I beat Alice this year", "why did Alice underperform me", "who's ahead of who this quarter, me or Alice", + "what was the best month for robsmsf since inception?", "how is BobSMSF doing", ] From 1d07b50fbedee136da0fa216a12feff517b6e627 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 07:59:48 +0000 Subject: [PATCH 08/34] period_returns: day/week/month/quarter/year returns in one call (replaces monthly_returns); day and week limited to recent windows Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- CLAUDE.md | 2 +- lib/features/monthly.py | 107 ------------------------- lib/features/periods.py | 169 +++++++++++++++++++++++++++++++++++++++ lib/telegram/handlers.py | 4 +- tests/test_handlers.py | 2 +- tests/test_monthly.py | 56 ------------- tests/test_periods.py | 115 ++++++++++++++++++++++++++ 7 files changed, 288 insertions(+), 167 deletions(-) delete mode 100644 lib/features/monthly.py create mode 100644 lib/features/periods.py delete mode 100644 tests/test_monthly.py create mode 100644 tests/test_periods.py diff --git a/CLAUDE.md b/CLAUDE.md index e6178f9..d9b5c34 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ MCP data tools and web search. User-facing docs are in `README.md`; this file is `openrouter.py` + `zai.py` (the only provider-specific code), `runner.py` (THE tool loop), `policy.py` (THE retry ladder), `gate.py` (which tools a message needs). - `prompts.py` all prompts. `ask.py` one entry point: route -> Request -> policy.ask -> usage -> NEEDS_TOOLS rerun. - `telegram/`: `handlers.py` (trigger -> request -> reply), `send.py`, `draft.py` (typing + streaming - drafts), `commands.py` (/credits), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `post.py`, `monthly.py` (local `monthly_returns` tool: one call instead of a report per month). + drafts), `commands.py` (/credits), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `post.py`, `periods.py` (local `period_returns` tool: one call instead of a report per day/week/month/quarter/year). - `app.py` wiring/startup; `context.py` `Ctx` (settings, store, backend, registry, bot). - `lib/` holds only what the bot needs to run. Developer helpers live in `tools/`: `report.py` (`make prompt-report`), `capture.py` (real MCP output for fixtures); the bot never imports them. diff --git a/lib/features/monthly.py b/lib/features/monthly.py deleted file mode 100644 index ecaba45..0000000 --- a/lib/features/monthly.py +++ /dev/null @@ -1,107 +0,0 @@ -"""monthly_returns: a Sharesight portfolio's return per month, fetched by the bot in one tool call -instead of one model-driven report per month (64 calls and 290k tokens for "best month ever").""" - -from __future__ import annotations - -import asyncio -import calendar -import json -import logging -from datetime import date, datetime - -from ..context import Ctx -from ..mcp.schema import ToolDef - -log = logging.getLogger("bot") - -MAX_MONTHS = 120 -TOOL = ToolDef( - "monthly_returns", - "Total return of a Sharesight portfolio for each month, with the best and worst month. Use it for " - "best/worst month, month-by-month or year-by-year questions, instead of fetching one " - "get_performance_report per period. Sold holdings are included. Omit start for 'since inception'.", - {"type": "object", "required": ["portfolio_id"], "properties": { - "portfolio_id": {"type": "number", "description": "From list_portfolios"}, - "start": {"type": "string", "description": "First month, YYYY-MM (default: the portfolio's inception)"}, - "end": {"type": "string", "description": "Last month, YYYY-MM (default: this month)"}}}) - - -def _month(text: str) -> tuple[int, int] | None: - try: - d = datetime.strptime(str(text).strip()[:7], "%Y-%m") - except ValueError: - return None - return d.year, d.month - - -def month_windows(first: tuple[int, int], last: tuple[int, int], today: date) -> list[tuple[str, date, date]]: - """(label, first day, last day) per month from `first` to `last`; the current month ends today.""" - out, (y, m) = [], first - while (y, m) <= last and len(out) < MAX_MONTHS: - end = min(date(y, m, calendar.monthrange(y, m)[1]), today) - out.append((f"{y}-{m:02d}", date(y, m, 1), end)) - y, m = (y + 1, 1) if m == 12 else (y, m + 1) - return out - - -def _report(text: str) -> dict | None: - try: - report = json.loads(text).get("report") - except (ValueError, AttributeError): - return None - return report if isinstance(report, dict) and report.get("total_gain_percent") is not None else None - - -async def _inception(server, portfolio_id: int) -> tuple[int, int] | None: - try: - portfolios = json.loads(await server.call("list_portfolios", {})).get("portfolios") or [] - except (ValueError, AttributeError): - return None - for p in portfolios: - if isinstance(p, dict) and p.get("id") == portfolio_id: - return _month(p.get("inception_date") or "") - return None - - -def tool(ctx: Ctx): - """(definition, function) of monthly_returns, backed by the Sharesight server (the one with a - get_performance_report tool).""" - - async def monthly(args: dict) -> str: - server = next((s for s in ctx.registry.up() if "get_performance_report" in s.fn_names.values()), None) - if server is None: - return "Error: the portfolio data source isn't connected." - try: - pid = int(args.get("portfolio_id")) - except (TypeError, ValueError): - return "Error: give the portfolio_id (call list_portfolios first)." - today = datetime.now(ctx.st.tz).date() - first = _month(args["start"]) if args.get("start") else await _inception(server, pid) - last = (_month(args["end"]) if args.get("end") else None) or (today.year, today.month) - if first is None: - return "Error: couldn't tell when the portfolio started; pass start as YYYY-MM." - windows = month_windows(first, last, today) - if not windows: - return "Error: no months in that range." - results = await asyncio.gather(*(server.call("get_performance_report", { - "portfolio_id": pid, "start_date": a.isoformat(), "end_date": b.isoformat(), - "_keep_sales": True}) for _, a, b in windows)) - rows, missing = [], 0 - for (label, _, _), text in zip(windows, results, strict=True): - report = _report(text) - if report is None: - missing += 1 - continue - rows.append((label, float(report["total_gain_percent"]), float(report.get("total_gain") or 0), - report.get("currency") or "")) - if not rows: - return "Error: no monthly reports came back." - best, worst = max(rows, key=lambda r: r[1]), min(rows, key=lambda r: r[1]) - lines = [f"Total return per month ({rows[0][3]}), {rows[0][0]} to {rows[-1][0]}, sold holdings included:"] - lines += [f"{label} {pct:+.2f}% {gain:+,.0f}" for label, pct, gain, _ in rows] - lines.append(f"Best: {best[0]} {best[1]:+.2f}%. Worst: {worst[0]} {worst[1]:+.2f}%." - + (f" {missing} months had no data." if missing else "")) - log.info("monthly_returns: %d months for portfolio %s", len(rows), pid) - return "\n".join(lines) - - return TOOL, monthly diff --git a/lib/features/periods.py b/lib/features/periods.py new file mode 100644 index 0000000..61ea09d --- /dev/null +++ b/lib/features/periods.py @@ -0,0 +1,169 @@ +"""period_returns: a Sharesight portfolio's return per day, week, month, quarter or year, fetched by the bot +in one tool call instead of one model-driven report per period (64 calls and 290k tokens for "best month ever").""" + +from __future__ import annotations + +import asyncio +import json +import logging +from datetime import date, datetime, timedelta + +from ..context import Ctx +from ..mcp.schema import ToolDef + +log = logging.getLogger("bot") + +PERIODS = ("day", "week", "month", "quarter", "year") +MAX_PERIODS = 120 # reports one call may fetch (about 2.3 s each) +MAX_WEEKS = 26 +MAX_DAYS = 90 +TOOL = ToolDef( + "period_returns", + "Total return of a Sharesight portfolio for each day, week, month, quarter or year, with the best and " + "worst one. Use it for best/worst day, week, month, quarter or year, or any period-by-period question, " + "instead of fetching one get_performance_report per period. Sold holdings are included. Month, quarter " + "and year can cover the whole history (omit start for 'since inception'); day and week only recent " + f"windows (last {MAX_DAYS} days, {MAX_WEEKS} weeks): if the tool refuses a longer range, tell the user the limit.", + {"type": "object", "required": ["portfolio_id"], "properties": { + "portfolio_id": {"type": "number", "description": "From list_portfolios"}, + "period": {"type": "string", "enum": list(PERIODS), "description": "Default month"}, + "start": {"type": "string", "description": "First day: YYYY-MM-DD, YYYY-MM or YYYY (default: inception)"}, + "end": {"type": "string", "description": "Last day, same formats (default: today)"}}}) + + +def _date(text: str, *, end: bool = False) -> date | None: + """A date from YYYY-MM-DD, YYYY-MM or YYYY; a partial one means its first day, or its last for `end`.""" + text = str(text).strip() + for fmt, size in (("%Y-%m-%d", 10), ("%Y-%m", 7), ("%Y", 4)): + try: + d = datetime.strptime(text, fmt).date() + except ValueError: + continue + if end and size < 10: + d = (date(d.year + 1, 1, 1) if size == 4 else date(d.year + (d.month == 12), d.month % 12 + 1, 1)) \ + - timedelta(days=1) + return d + return None + + +def _month_add(d: date, months: int) -> date: + n = d.year * 12 + d.month - 1 + months + return date(n // 12, n % 12 + 1, 1) + + +def period_windows(period: str, first: date, last: date) -> list[tuple[str, date, date]]: + """(label, first day, last day) of each period between `first` and `last`; the ends are clipped to them.""" + out: list[tuple[str, date, date]] = [] + if period == "day": + d = first + while d <= last: + if d.weekday() < 5: # no trading at weekends + out.append((d.isoformat(), d, d)) + d += timedelta(days=1) + elif period == "week": + monday = first - timedelta(days=first.weekday()) + while monday <= last: + iso = monday.isocalendar() + out.append((f"{iso[0]}-W{iso[1]:02d}", max(monday, first), min(monday + timedelta(days=6), last))) + monday += timedelta(days=7) + else: + step = {"month": 1, "quarter": 3, "year": 12}[period] + cur = date(first.year, ((first.month - 1) // step) * step + 1, 1) + while cur <= last: + nxt = _month_add(cur, step) + label = {"month": f"{cur.year}-{cur.month:02d}", "quarter": f"{cur.year}-Q{(cur.month - 1) // 3 + 1}", + "year": str(cur.year)}[period] + out.append((label, max(cur, first), min(nxt - timedelta(days=1), last))) + cur = nxt + return out + + +def _report(text: str) -> dict | None: + try: + report = json.loads(text).get("report") + except (ValueError, AttributeError): + return None + return report if isinstance(report, dict) and report.get("total_gain_percent") is not None else None + + +async def _inception(server, portfolio_id: int) -> date | None: + try: + portfolios = json.loads(await server.call("list_portfolios", {})).get("portfolios") or [] + except (ValueError, AttributeError): + return None + for p in portfolios: + if isinstance(p, dict) and p.get("id") == portfolio_id: + return _date(p.get("inception_date") or "") + return None + + +def _too_long(period: str, first: date, last: date, count: int) -> str | None: + """Why this range is refused (with the earliest start that is allowed), or None.""" + if period == "day" and (last - first).days + 1 > MAX_DAYS: + return (f"daily returns are limited to the last {MAX_DAYS} days; " + f"ask again with start={last - timedelta(days=MAX_DAYS - 1)} or later") + if period == "week" and count > MAX_WEEKS: + monday = last - timedelta(days=last.weekday()) + return (f"weekly returns are limited to {MAX_WEEKS} weeks; " + f"ask again with start={monday - timedelta(weeks=MAX_WEEKS - 1)} or later") + if count > MAX_PERIODS: + return f"that is {count} {period}s; the limit is {MAX_PERIODS}. Use a shorter range or a longer period" + return None + + +def tool(ctx: Ctx): + """(definition, function) of period_returns, backed by the Sharesight server (the one with a + get_performance_report tool).""" + + async def periods(args: dict) -> str: + server = next((s for s in ctx.registry.up() if "get_performance_report" in s.fn_names.values()), None) + if server is None: + return "Error: the portfolio data source isn't connected." + try: + pid = int(args.get("portfolio_id")) + except (TypeError, ValueError): + return "Error: give the portfolio_id (call list_portfolios first)." + period = str(args.get("period") or "month").lower() + if period not in PERIODS: + return f"Error: period must be one of {', '.join(PERIODS)}." + today = datetime.now(ctx.st.tz).date() + last = min(_date(args["end"], end=True) or today, today) if args.get("end") else today + if args.get("start"): + first = _date(args["start"]) + elif period == "day": + first = last - timedelta(days=MAX_DAYS - 1) + elif period == "week": + first = last - timedelta(days=last.weekday()) - timedelta(weeks=MAX_WEEKS - 1) + else: + first = await _inception(server, pid) + if first is None: + return "Error: couldn't tell when to start; pass start as YYYY-MM-DD." + if first > last: + return "Error: start is after end." + windows = period_windows(period, first, last) + if refused := _too_long(period, first, last, len(windows)): + return "Error: " + refused + if not windows: + return "Error: no trading days in that range." + results = await asyncio.gather(*(server.call("get_performance_report", { + "portfolio_id": pid, "start_date": a.isoformat(), "end_date": b.isoformat(), + "_keep_sales": True}) for _, a, b in windows)) + rows, missing = [], 0 + for (label, _, _), text in zip(windows, results, strict=True): + report = _report(text) + if report is None: + missing += 1 + continue + rows.append((label, float(report["total_gain_percent"]), float(report.get("total_gain") or 0), + report.get("currency") or "")) + if not rows: + return "Error: no reports came back." + best, worst = max(rows, key=lambda r: r[1]), min(rows, key=lambda r: r[1]) + lines = [f"Total return per {period} ({rows[0][3]}), {rows[0][0]} to {rows[-1][0]}, sold holdings included:"] + lines += [f"{label} {pct:+.2f}% {gain:+,.0f}" for label, pct, gain, _ in rows] + lines.append(f"Best: {best[0]} {best[1]:+.2f}%. Worst: {worst[0]} {worst[1]:+.2f}%." + + (f" {missing} {period}s had no data." if missing else "")) + log.info("period_returns: %d %ss for portfolio %s", len(rows), period, pid) + return "\n".join(lines) + + return TOOL, periods diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index 71d8916..c48d4a3 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -15,7 +15,7 @@ from .. import config from ..ask import ask from ..context import Ctx -from ..features import dm_buttons, holding_news, monthly, movers, post +from ..features import dm_buttons, holding_news, movers, periods, post from ..history import compact, format_rows from ..llm.gate import Route, is_portfolio_question, route from ..msgtext import describe, sender_name @@ -230,7 +230,7 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s return PORTFOLIO_ADMINS_ONLY extra = [] if admin and any(s.cfg.get("gate") == "portfolio" for s in chosen.servers): - extra.append(monthly.tool(ctx)) + extra.append(periods.tool(ctx)) if st.post_to_groups and trig.private and post.wants_post(trig.text): extra = [post.tool(ctx, msg.from_user.id, msg.from_user.full_name or "")] if chosen.simple: # the model must not be told it has no tools diff --git a/tests/test_handlers.py b/tests/test_handlers.py index ecd0d22..4c93c98 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -253,7 +253,7 @@ async def test_non_admin_gets_no_portfolio_data_in_a_group(env, store): assert asked.replies[0]["text"] == "Portfolio data is for admins only." and backend.seen == [] admin = user_msg(ctx.bot, "@stockbot how is my portfolio doing?", user_id=5, mid=2) await run(h, admin) - assert backend.seen[0]["tools"] == ["sharesight__list_portfolios", "monthly_returns"] # admins also get the monthly tool + assert backend.seen[0]["tools"] == ["sharesight__list_portfolios", "period_returns"] # admins also get the period tool async def test_non_admin_never_gets_portfolio_tools_even_on_the_rerun(env, store): diff --git a/tests/test_monthly.py b/tests/test_monthly.py deleted file mode 100644 index df3acdb..0000000 --- a/tests/test_monthly.py +++ /dev/null @@ -1,56 +0,0 @@ -import json -from datetime import date - -from lib import config -from lib.context import Ctx -from lib.features import monthly - -from .conftest import FakeBot -from .fakes import FakeMcp, ScriptedBackend, registry - - -class Sharesight(FakeMcp): - """Returns a report per month: +1% in 2026-01, +5% in 2026-02, nothing before 2026.""" - - async def call(self, tool, args): - self.calls.append((tool, args)) - if tool == "list_portfolios": - return json.dumps({"portfolios": [{"id": 7, "inception_date": "2025-12-20"}]}) - start = args["start_date"] - pct = {"2025-12": -2.0, "2026-01": 1.0, "2026-02": 5.0}.get(start[:7]) - if pct is None: - return "Tool error: no data" - return json.dumps({"report": {"total_gain_percent": pct, "total_gain": pct * 100, "currency": "AUD"}}) - - -def make(env, ss): - return Ctx(config.load(env), None, ScriptedBackend([]), registry(ss), FakeBot()) - - -async def test_months_run_from_inception_with_sold_holdings_and_best_worst(env): - ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) - defn, fn = monthly.tool(make(env, ss)) - out = await fn({"portfolio_id": 7, "end": "2026-03"}) - assert defn.name == "monthly_returns" - assert "2025-12 -2.00% -200" in out and "2026-02 +5.00% +500" in out - assert "Best: 2026-02 +5.00%. Worst: 2025-12 -2.00%. 1 months had no data." in out # March: no data - reports = [a for t, a in ss.calls if t == "get_performance_report"] - assert [r["start_date"] for r in reports][:2] == ["2025-12-01", "2026-01-01"] - assert reports[0]["end_date"] == "2025-12-31" and all(r["_keep_sales"] is True for r in reports) - - -async def test_start_given_skips_the_inception_lookup_and_bad_input_is_reported(env): - ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) - _, fn = monthly.tool(make(env, ss)) - out = await fn({"portfolio_id": 7, "start": "2026-01", "end": "2026-02"}) - assert out.splitlines()[1].startswith("2026-01 +1.00%") and not any(t == "list_portfolios" for t, _ in ss.calls) - assert "portfolio_id" in await fn({}) - assert "start as YYYY-MM" in await fn({"portfolio_id": 99}) # unknown portfolio: no inception date - _, none = monthly.tool(Ctx(config.load(env), None, ScriptedBackend([]), registry(), FakeBot())) - assert "isn't connected" in await none({"portfolio_id": 7}) - - -def test_month_windows_end_today_in_the_current_month(): - w = monthly.month_windows((2025, 11), (2026, 2), date(2026, 2, 10)) - assert [x[0] for x in w] == ["2025-11", "2025-12", "2026-01", "2026-02"] - assert w[0][2] == date(2025, 11, 30) and w[-1][2] == date(2026, 2, 10) diff --git a/tests/test_periods.py b/tests/test_periods.py new file mode 100644 index 0000000..afb81d3 --- /dev/null +++ b/tests/test_periods.py @@ -0,0 +1,115 @@ +import json +from datetime import date, datetime + +import pytest + +from lib import config +from lib.context import Ctx +from lib.features import periods + +from .conftest import FakeBot +from .fakes import FakeMcp, ScriptedBackend, registry + + +class Sharesight(FakeMcp): + """Reports by start date: +1% in 2026-01, +5% in 2026-02, -2% in 2025-12; no data otherwise.""" + + async def call(self, tool, args): + self.calls.append((tool, args)) + if tool == "list_portfolios": + return json.dumps({"portfolios": [{"id": 7, "inception_date": "2025-12-20"}]}) + pct = {"2025-12": -2.0, "2026-01": 1.0, "2026-02": 5.0}.get(args["start_date"][:7]) + if pct is None: + return "Tool error: no data" + return json.dumps({"report": {"total_gain_percent": pct, "total_gain": pct * 100, "currency": "AUD"}}) + + +def make(env, ss=None): + ss = ss or Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) + return Ctx(config.load(env), None, ScriptedBackend([]), registry(ss), FakeBot()), ss + + +def reports(ss): + return [a for t, a in ss.calls if t == "get_performance_report"] + + +async def test_months_run_from_inception_with_sold_holdings_and_best_worst(env): + ctx, ss = make(env) + defn, fn = periods.tool(ctx) + out = await fn({"portfolio_id": 7, "end": "2026-03"}) + assert defn.name == "period_returns" and "Total return per month (AUD)" in out + assert "2025-12 -2.00% -200" in out and "2026-02 +5.00% +500" in out + assert "Best: 2026-02 +5.00%. Worst: 2025-12 -2.00%. 1 months had no data." in out # March: no data + first = reports(ss)[0] + assert (first["start_date"], first["end_date"]) == ("2025-12-20", "2025-12-31") # clipped to inception + assert all(r["_keep_sales"] is True for r in reports(ss)) + + +async def test_quarters_and_years_are_asked_for_whole_windows(env): + ctx, ss = make(env) + _, fn = periods.tool(ctx) + out = await fn({"portfolio_id": 7, "period": "quarter", "start": "2025-10", "end": "2026-06"}) + assert [(r["start_date"], r["end_date"]) for r in reports(ss)] == [ + ("2025-10-01", "2025-12-31"), ("2026-01-01", "2026-03-31"), ("2026-04-01", "2026-06-30")] + assert "2026-Q1 +1.00%" in out and "2 quarters had no data" in out + ctx, ss = make(env) + _, fn = periods.tool(ctx) + await fn({"portfolio_id": 7, "period": "year", "start": "2024", "end": "2025-06"}) + assert [(r["start_date"], r["end_date"]) for r in reports(ss)] == [ + ("2024-01-01", "2024-12-31"), ("2025-01-01", "2025-06-30")] + + +async def test_start_given_skips_the_inception_lookup_and_bad_input_is_reported(env): + ctx, ss = make(env) + _, fn = periods.tool(ctx) + out = await fn({"portfolio_id": 7, "start": "2026-01", "end": "2026-02"}) + assert out.splitlines()[1].startswith("2026-01 +1.00%") and not any(t == "list_portfolios" for t, _ in ss.calls) + assert "portfolio_id" in await fn({}) + assert "period must be" in await fn({"portfolio_id": 7, "period": "decade"}) + assert "start as YYYY-MM-DD" in await fn({"portfolio_id": 99}) # unknown portfolio: no inception date + assert "after end" in await fn({"portfolio_id": 7, "start": "2026-03", "end": "2026-01"}) + none_ctx = Ctx(config.load(env), None, ScriptedBackend([]), registry(), FakeBot()) + assert "isn't connected" in await periods.tool(none_ctx)[1]({"portfolio_id": 7}) + + +async def test_day_and_week_are_limited_to_recent_windows_and_refuse_without_calling(env): + ctx, ss = make(env) + _, fn = periods.tool(ctx) + day = await fn({"portfolio_id": 7, "period": "day", "start": "2020-01-01"}) + week = await fn({"portfolio_id": 7, "period": "week", "start": "2020-01-01"}) + assert "limited to the last 90 days" in day and "start=" in day + assert "limited to 26 weeks" in week and "start=" in week + assert "limit is 120" in await fn({"portfolio_id": 7, "period": "month", "start": "2000-01"}) + assert reports(ss) == [] + + +async def test_day_and_week_default_to_the_allowed_window(env): + ctx, ss = make(env) + _, fn = periods.tool(ctx) + today = datetime.now(ctx.st.tz).date() + await fn({"portfolio_id": 7, "period": "day"}) + days = [r["start_date"] for r in reports(ss)] + assert days and all(date.fromisoformat(d).weekday() < 5 for d in days) and date.fromisoformat(days[-1]) <= today + ss.calls.clear() + await fn({"portfolio_id": 7, "period": "week"}) + assert len(reports(ss)) == periods.MAX_WEEKS + + +@pytest.mark.parametrize("period, first, last, labels", [ + ("month", date(2025, 11, 15), date(2026, 2, 10), ["2025-11", "2025-12", "2026-01", "2026-02"]), + ("quarter", date(2025, 8, 1), date(2026, 2, 10), ["2025-Q3", "2025-Q4", "2026-Q1"]), + ("year", date(2024, 6, 1), date(2026, 2, 10), ["2024", "2025", "2026"]), + ("week", date(2026, 3, 4), date(2026, 3, 17), ["2026-W10", "2026-W11", "2026-W12"]), + ("day", date(2026, 3, 6), date(2026, 3, 10), ["2026-03-06", "2026-03-09", "2026-03-10"]), # Fri, Mon, Tue +]) +def test_period_windows_labels_and_weekends(period, first, last, labels): + w = periods.period_windows(period, first, last) + assert [x[0] for x in w] == labels + assert w[0][1] == first and w[-1][2] == last # clipped to the requested range + + +def test_dates_accept_day_month_or_year_and_end_means_the_period_end(): + assert periods._date("2026-02-03") == date(2026, 2, 3) + assert periods._date("2026-02") == date(2026, 2, 1) and periods._date("2026-02", end=True) == date(2026, 2, 28) + assert periods._date("2025-12", end=True) == date(2025, 12, 31) and periods._date("2025", end=True) == date(2025, 12, 31) + assert periods._date("nope") is None From b6f54abe080ef3107f2b1674ec49ed5f74ceca1e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 08:01:52 +0000 Subject: [PATCH 09/34] period_returns: name the holdings behind the best and worst periods Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/features/periods.py | 25 +++++++++++++++++++++++-- tests/test_periods.py | 13 ++++++++++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/lib/features/periods.py b/lib/features/periods.py index 61ea09d..49530a8 100644 --- a/lib/features/periods.py +++ b/lib/features/periods.py @@ -9,6 +9,7 @@ from datetime import date, datetime, timedelta from ..context import Ctx +from ..mcp.results import table_records from ..mcp.schema import ToolDef log = logging.getLogger("bot") @@ -78,6 +79,21 @@ def period_windows(period: str, first: date, last: date) -> list[tuple[str, date return out +def _movers(report: dict) -> list[tuple[str, float, object]]: + """(code, total gain, gain %) per holding of a report, sold ones included.""" + return [(h["code"], float(h["total_gain"]), h.get("total_gain_percent")) for h in table_records(report.get("holdings")) + if h.get("code") and isinstance(h.get("total_gain"), (int, float))] + + +def _drivers(movers: list[tuple[str, float, object]], n: int = 3) -> str: + """The holdings that added most and lost most in a period: "up: CRWD +9,975 (+64.5%); down: ...".""" + def show(m): + return f"{m[0]} {m[1]:+,.0f}" + (f" ({m[2]:+.1f}%)" if isinstance(m[2], (int, float)) else "") + ups = sorted((m for m in movers if m[1] > 0), key=lambda m: -m[1])[:n] + downs = sorted((m for m in movers if m[1] < 0), key=lambda m: m[1])[:n] + return "; ".join(f"{name}: {', '.join(map(show, group))}" for name, group in (("up", ups), ("down", downs)) if group) + + def _report(text: str) -> dict | None: try: report = json.loads(text).get("report") @@ -155,14 +171,19 @@ async def periods(args: dict) -> str: missing += 1 continue rows.append((label, float(report["total_gain_percent"]), float(report.get("total_gain") or 0), - report.get("currency") or "")) + report.get("currency") or "", _movers(report))) if not rows: return "Error: no reports came back." best, worst = max(rows, key=lambda r: r[1]), min(rows, key=lambda r: r[1]) lines = [f"Total return per {period} ({rows[0][3]}), {rows[0][0]} to {rows[-1][0]}, sold holdings included:"] - lines += [f"{label} {pct:+.2f}% {gain:+,.0f}" for label, pct, gain, _ in rows] + lines += [f"{label} {pct:+.2f}% {gain:+,.0f}" for label, pct, gain, *_ in rows] lines.append(f"Best: {best[0]} {best[1]:+.2f}%. Worst: {worst[0]} {worst[1]:+.2f}%." + (f" {missing} {period}s had no data." if missing else "")) + # What moved the extremes (best/worst by percent and by money), so the answer can say why. + extremes = dict.fromkeys(r[0] for r in (best, worst, max(rows, key=lambda r: r[2]), min(rows, key=lambda r: r[2]))) + for r in rows: + if r[0] in extremes and (drivers := _drivers(r[4])): + lines.append(f"Holdings behind {r[0]}: {drivers}") log.info("period_returns: %d %ss for portfolio %s", len(rows), period, pid) return "\n".join(lines) diff --git a/tests/test_periods.py b/tests/test_periods.py index afb81d3..91b6db3 100644 --- a/tests/test_periods.py +++ b/tests/test_periods.py @@ -21,7 +21,9 @@ async def call(self, tool, args): pct = {"2025-12": -2.0, "2026-01": 1.0, "2026-02": 5.0}.get(args["start_date"][:7]) if pct is None: return "Tool error: no data" - return json.dumps({"report": {"total_gain_percent": pct, "total_gain": pct * 100, "currency": "AUD"}}) + rows = [["AAA", pct * 150, 40.0], ["BBB", -pct * 50, -10.0], ["CCC", 0, 0]] # a sold holding could be among them + return json.dumps({"report": {"total_gain_percent": pct, "total_gain": pct * 100, "currency": "AUD", + "holdings": {"columns": ["code", "total_gain", "total_gain_percent"], "rows": rows}}}) def make(env, ss=None): @@ -45,6 +47,15 @@ async def test_months_run_from_inception_with_sold_holdings_and_best_worst(env): assert all(r["_keep_sales"] is True for r in reports(ss)) +async def test_best_and_worst_periods_say_which_holdings_drove_them(env): + ctx, ss = make(env) + _, fn = periods.tool(ctx) + out = await fn({"portfolio_id": 7, "end": "2026-03"}) + assert "Holdings behind 2026-02: up: AAA +750 (+40.0%); down: BBB -250 (-10.0%)" in out + assert "Holdings behind 2025-12: up: BBB +100 (-10.0%); down: AAA -300 (+40.0%)" in out # worst month + assert "Holdings behind 2026-01" not in out and "CCC" not in out # only the extremes, only movers + + async def test_quarters_and_years_are_asked_for_whole_windows(env): ctx, ss = make(env) _, fn = periods.tool(ctx) From 3e87f42d6e79867cda440de8c42cc0fc9d166734 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 08:16:48 +0000 Subject: [PATCH 10/34] period_returns: cache finished periods in SQLite (cache-first, live limit counts only uncached windows) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/features/periods.py | 190 ------------------------------- lib/features/periods/__init__.py | 102 +++++++++++++++++ lib/features/periods/rows.py | 49 ++++++++ lib/features/periods/windows.py | 80 +++++++++++++ lib/store.py | 44 +++++++ tests/test_periods.py | 100 ++++++++++------ tests/test_store.py | 18 +++ 7 files changed, 356 insertions(+), 227 deletions(-) delete mode 100644 lib/features/periods.py create mode 100644 lib/features/periods/__init__.py create mode 100644 lib/features/periods/rows.py create mode 100644 lib/features/periods/windows.py diff --git a/lib/features/periods.py b/lib/features/periods.py deleted file mode 100644 index 49530a8..0000000 --- a/lib/features/periods.py +++ /dev/null @@ -1,190 +0,0 @@ -"""period_returns: a Sharesight portfolio's return per day, week, month, quarter or year, fetched by the bot -in one tool call instead of one model-driven report per period (64 calls and 290k tokens for "best month ever").""" - -from __future__ import annotations - -import asyncio -import json -import logging -from datetime import date, datetime, timedelta - -from ..context import Ctx -from ..mcp.results import table_records -from ..mcp.schema import ToolDef - -log = logging.getLogger("bot") - -PERIODS = ("day", "week", "month", "quarter", "year") -MAX_PERIODS = 120 # reports one call may fetch (about 2.3 s each) -MAX_WEEKS = 26 -MAX_DAYS = 90 -TOOL = ToolDef( - "period_returns", - "Total return of a Sharesight portfolio for each day, week, month, quarter or year, with the best and " - "worst one. Use it for best/worst day, week, month, quarter or year, or any period-by-period question, " - "instead of fetching one get_performance_report per period. Sold holdings are included. Month, quarter " - "and year can cover the whole history (omit start for 'since inception'); day and week only recent " - f"windows (last {MAX_DAYS} days, {MAX_WEEKS} weeks): if the tool refuses a longer range, tell the user the limit.", - {"type": "object", "required": ["portfolio_id"], "properties": { - "portfolio_id": {"type": "number", "description": "From list_portfolios"}, - "period": {"type": "string", "enum": list(PERIODS), "description": "Default month"}, - "start": {"type": "string", "description": "First day: YYYY-MM-DD, YYYY-MM or YYYY (default: inception)"}, - "end": {"type": "string", "description": "Last day, same formats (default: today)"}}}) - - -def _date(text: str, *, end: bool = False) -> date | None: - """A date from YYYY-MM-DD, YYYY-MM or YYYY; a partial one means its first day, or its last for `end`.""" - text = str(text).strip() - for fmt, size in (("%Y-%m-%d", 10), ("%Y-%m", 7), ("%Y", 4)): - try: - d = datetime.strptime(text, fmt).date() - except ValueError: - continue - if end and size < 10: - d = (date(d.year + 1, 1, 1) if size == 4 else date(d.year + (d.month == 12), d.month % 12 + 1, 1)) \ - - timedelta(days=1) - return d - return None - - -def _month_add(d: date, months: int) -> date: - n = d.year * 12 + d.month - 1 + months - return date(n // 12, n % 12 + 1, 1) - - -def period_windows(period: str, first: date, last: date) -> list[tuple[str, date, date]]: - """(label, first day, last day) of each period between `first` and `last`; the ends are clipped to them.""" - out: list[tuple[str, date, date]] = [] - if period == "day": - d = first - while d <= last: - if d.weekday() < 5: # no trading at weekends - out.append((d.isoformat(), d, d)) - d += timedelta(days=1) - elif period == "week": - monday = first - timedelta(days=first.weekday()) - while monday <= last: - iso = monday.isocalendar() - out.append((f"{iso[0]}-W{iso[1]:02d}", max(monday, first), min(monday + timedelta(days=6), last))) - monday += timedelta(days=7) - else: - step = {"month": 1, "quarter": 3, "year": 12}[period] - cur = date(first.year, ((first.month - 1) // step) * step + 1, 1) - while cur <= last: - nxt = _month_add(cur, step) - label = {"month": f"{cur.year}-{cur.month:02d}", "quarter": f"{cur.year}-Q{(cur.month - 1) // 3 + 1}", - "year": str(cur.year)}[period] - out.append((label, max(cur, first), min(nxt - timedelta(days=1), last))) - cur = nxt - return out - - -def _movers(report: dict) -> list[tuple[str, float, object]]: - """(code, total gain, gain %) per holding of a report, sold ones included.""" - return [(h["code"], float(h["total_gain"]), h.get("total_gain_percent")) for h in table_records(report.get("holdings")) - if h.get("code") and isinstance(h.get("total_gain"), (int, float))] - - -def _drivers(movers: list[tuple[str, float, object]], n: int = 3) -> str: - """The holdings that added most and lost most in a period: "up: CRWD +9,975 (+64.5%); down: ...".""" - def show(m): - return f"{m[0]} {m[1]:+,.0f}" + (f" ({m[2]:+.1f}%)" if isinstance(m[2], (int, float)) else "") - ups = sorted((m for m in movers if m[1] > 0), key=lambda m: -m[1])[:n] - downs = sorted((m for m in movers if m[1] < 0), key=lambda m: m[1])[:n] - return "; ".join(f"{name}: {', '.join(map(show, group))}" for name, group in (("up", ups), ("down", downs)) if group) - - -def _report(text: str) -> dict | None: - try: - report = json.loads(text).get("report") - except (ValueError, AttributeError): - return None - return report if isinstance(report, dict) and report.get("total_gain_percent") is not None else None - - -async def _inception(server, portfolio_id: int) -> date | None: - try: - portfolios = json.loads(await server.call("list_portfolios", {})).get("portfolios") or [] - except (ValueError, AttributeError): - return None - for p in portfolios: - if isinstance(p, dict) and p.get("id") == portfolio_id: - return _date(p.get("inception_date") or "") - return None - - -def _too_long(period: str, first: date, last: date, count: int) -> str | None: - """Why this range is refused (with the earliest start that is allowed), or None.""" - if period == "day" and (last - first).days + 1 > MAX_DAYS: - return (f"daily returns are limited to the last {MAX_DAYS} days; " - f"ask again with start={last - timedelta(days=MAX_DAYS - 1)} or later") - if period == "week" and count > MAX_WEEKS: - monday = last - timedelta(days=last.weekday()) - return (f"weekly returns are limited to {MAX_WEEKS} weeks; " - f"ask again with start={monday - timedelta(weeks=MAX_WEEKS - 1)} or later") - if count > MAX_PERIODS: - return f"that is {count} {period}s; the limit is {MAX_PERIODS}. Use a shorter range or a longer period" - return None - - -def tool(ctx: Ctx): - """(definition, function) of period_returns, backed by the Sharesight server (the one with a - get_performance_report tool).""" - - async def periods(args: dict) -> str: - server = next((s for s in ctx.registry.up() if "get_performance_report" in s.fn_names.values()), None) - if server is None: - return "Error: the portfolio data source isn't connected." - try: - pid = int(args.get("portfolio_id")) - except (TypeError, ValueError): - return "Error: give the portfolio_id (call list_portfolios first)." - period = str(args.get("period") or "month").lower() - if period not in PERIODS: - return f"Error: period must be one of {', '.join(PERIODS)}." - today = datetime.now(ctx.st.tz).date() - last = min(_date(args["end"], end=True) or today, today) if args.get("end") else today - if args.get("start"): - first = _date(args["start"]) - elif period == "day": - first = last - timedelta(days=MAX_DAYS - 1) - elif period == "week": - first = last - timedelta(days=last.weekday()) - timedelta(weeks=MAX_WEEKS - 1) - else: - first = await _inception(server, pid) - if first is None: - return "Error: couldn't tell when to start; pass start as YYYY-MM-DD." - if first > last: - return "Error: start is after end." - windows = period_windows(period, first, last) - if refused := _too_long(period, first, last, len(windows)): - return "Error: " + refused - if not windows: - return "Error: no trading days in that range." - results = await asyncio.gather(*(server.call("get_performance_report", { - "portfolio_id": pid, "start_date": a.isoformat(), "end_date": b.isoformat(), - "_keep_sales": True}) for _, a, b in windows)) - rows, missing = [], 0 - for (label, _, _), text in zip(windows, results, strict=True): - report = _report(text) - if report is None: - missing += 1 - continue - rows.append((label, float(report["total_gain_percent"]), float(report.get("total_gain") or 0), - report.get("currency") or "", _movers(report))) - if not rows: - return "Error: no reports came back." - best, worst = max(rows, key=lambda r: r[1]), min(rows, key=lambda r: r[1]) - lines = [f"Total return per {period} ({rows[0][3]}), {rows[0][0]} to {rows[-1][0]}, sold holdings included:"] - lines += [f"{label} {pct:+.2f}% {gain:+,.0f}" for label, pct, gain, *_ in rows] - lines.append(f"Best: {best[0]} {best[1]:+.2f}%. Worst: {worst[0]} {worst[1]:+.2f}%." - + (f" {missing} {period}s had no data." if missing else "")) - # What moved the extremes (best/worst by percent and by money), so the answer can say why. - extremes = dict.fromkeys(r[0] for r in (best, worst, max(rows, key=lambda r: r[2]), min(rows, key=lambda r: r[2]))) - for r in rows: - if r[0] in extremes and (drivers := _drivers(r[4])): - lines.append(f"Holdings behind {r[0]}: {drivers}") - log.info("period_returns: %d %ss for portfolio %s", len(rows), period, pid) - return "\n".join(lines) - - return TOOL, periods diff --git a/lib/features/periods/__init__.py b/lib/features/periods/__init__.py new file mode 100644 index 0000000..7df2f85 --- /dev/null +++ b/lib/features/periods/__init__.py @@ -0,0 +1,102 @@ +"""period_returns: a Sharesight portfolio's return per day, week, month, quarter or year, in one tool call +instead of one model-driven report per period. Finished periods are cached in SQLite (see backfill.py).""" + +from __future__ import annotations + +import asyncio +import json +import logging +from datetime import date, datetime + +from ...context import Ctx +from ...mcp.schema import ToolDef +from .rows import drivers, fetch_row +from .windows import PERIODS, key, parse_date, period_windows, refusal + +log = logging.getLogger("bot") + +TOOL = ToolDef( + "period_returns", + "Total return of a Sharesight portfolio for each day, week, month, quarter or year, with the best and " + "worst one and the holdings behind them. Use it for best/worst day, week, month, quarter or year, or any " + "period-by-period question, instead of fetching one get_performance_report per period. Sold holdings are " + "included. Omit start for 'since inception'. History that isn't cached yet is fetched live, which is slow " + "for days and weeks: if the tool refuses a range, tell the user why and offer the start it suggests.", + {"type": "object", "required": ["portfolio_id"], "properties": { + "portfolio_id": {"type": "number", "description": "From list_portfolios"}, + "period": {"type": "string", "enum": list(PERIODS), "description": "Default month"}, + "start": {"type": "string", "description": "First day: YYYY-MM-DD, YYYY-MM or YYYY (default: inception)"}, + "end": {"type": "string", "description": "Last day, same formats (default: today)"}}}) + + +async def inception(server, portfolio_id: int) -> date | None: + try: + portfolios = json.loads(await server.call("list_portfolios", {})).get("portfolios") or [] + except (ValueError, AttributeError): + return None + for p in portfolios: + if isinstance(p, dict) and p.get("id") == portfolio_id: + return parse_date(p.get("inception_date") or "") + return None + + +def sharesight(ctx: Ctx): + """The MCP server that has a get_performance_report tool, if it is up.""" + return next((s for s in ctx.registry.up() if "get_performance_report" in s.fn_names.values()), None) + + +def tool(ctx: Ctx): + """(definition, function) of period_returns.""" + + async def periods(args: dict) -> str: + server = sharesight(ctx) + if server is None: + return "Error: the portfolio data source isn't connected." + try: + pid = int(args.get("portfolio_id")) + except (TypeError, ValueError): + return "Error: give the portfolio_id (call list_portfolios first)." + period = str(args.get("period") or "month").lower() + if period not in PERIODS: + return f"Error: period must be one of {', '.join(PERIODS)}." + today = datetime.now(ctx.st.tz).date() + last = min(parse_date(args["end"], end=True) or today, today) if args.get("end") else today + first = parse_date(args["start"]) if args.get("start") else await inception(server, pid) + if first is None: + return "Error: couldn't tell when to start; pass start as YYYY-MM-DD." + if first > last: + return "Error: start is after end." + windows = period_windows(period, first, last) + if not windows: + return "Error: no trading days in that range." + cached = await asyncio.to_thread(ctx.store.period_get, pid, first.isoformat(), last.isoformat()) + have = {key(w): cached[key(w)] for w in windows if key(w) in cached} + if refused := refusal(period, windows, set(have)): + return "Error: " + refused + todo = [w for w in windows if key(w) not in have] + fetched = await asyncio.gather(*(fetch_row(server, pid, a, b) for _, a, b in todo)) + fresh = [r for r in fetched if r is not None] + have.update({(r.start, r.end): r for r in fresh}) + if finished := [r for r in fresh if date.fromisoformat(r.end) < today]: # never the unfinished period + await asyncio.to_thread(ctx.store.period_put, finished) + rows = [(label, have[key((label, a, b))]) for label, a, b in windows if key((label, a, b)) in have] + missing = len(windows) - len(rows) + if not rows: + return "Error: no reports came back." + best = max(rows, key=lambda r: r[1].pct) + worst = min(rows, key=lambda r: r[1].pct) + lines = [f"Total return per {period} ({rows[0][1].currency}), {rows[0][0]} to {rows[-1][0]}, " + "sold holdings included:"] + lines += [f"{label} {r.pct:+.2f}% {r.gain:+,.0f}" for label, r in rows] + lines.append(f"Best: {best[0]} {best[1].pct:+.2f}%. Worst: {worst[0]} {worst[1].pct:+.2f}%." + + (f" {missing} {period}s had no data." if missing else "")) + # What moved the extremes (best/worst by percent and by money), so the answer can say why. + extremes = dict.fromkeys(label for label, _ in (best, worst, max(rows, key=lambda r: r[1].gain), + min(rows, key=lambda r: r[1].gain))) + for label, r in rows: + if label in extremes and (text := drivers(r.movers)): + lines.append(f"Holdings behind {label}: {text}") + log.info("period_returns: %d %ss for portfolio %s (%d fetched)", len(rows), period, pid, len(todo)) + return "\n".join(lines) + + return TOOL, periods diff --git a/lib/features/periods/rows.py b/lib/features/periods/rows.py new file mode 100644 index 0000000..fe709de --- /dev/null +++ b/lib/features/periods/rows.py @@ -0,0 +1,49 @@ +"""Turning Sharesight performance reports into cached period rows, and the "what drove it" text.""" + +from __future__ import annotations + +import json +from datetime import date + +from ...mcp.results import table_records +from ...store import PeriodRow + +MAX_MOVERS = 12 # holdings kept per period +Mover = tuple[str, float, object] # code, total gain, gain % + + +def parse_report(text: str) -> dict | None: + try: + report = json.loads(text).get("report") + except (ValueError, AttributeError): + return None + return report if isinstance(report, dict) and report.get("total_gain_percent") is not None else None + + +def movers(report: dict) -> list[list]: + """The holdings with the largest gain or loss in a report, sold ones included: [[code, gain, gain %], ...].""" + rows = [[h["code"], float(h["total_gain"]), h.get("total_gain_percent")] + for h in table_records(report.get("holdings")) + if h.get("code") and isinstance(h.get("total_gain"), (int, float)) and h["total_gain"]] + return sorted(rows, key=lambda m: -abs(m[1]))[:MAX_MOVERS] + + +def drivers(rows: list, n: int = 3) -> str: + """The holdings that added most and lost most: "up: CRWD +9,975 (+64.5%); down: ...".""" + def show(m): + return f"{m[0]} {m[1]:+,.0f}" + (f" ({m[2]:+.1f}%)" if isinstance(m[2], (int, float)) else "") + ups = sorted((m for m in rows if m[1] > 0), key=lambda m: -m[1])[:n] + downs = sorted((m for m in rows if m[1] < 0), key=lambda m: m[1])[:n] + return "; ".join(f"{name}: {', '.join(map(show, group))}" for name, group in (("up", ups), ("down", downs)) if group) + + +async def fetch_row(server, portfolio_id: int, first: date, last: date) -> PeriodRow | None: + """One window's return from Sharesight (sold holdings included), or None if no report came back.""" + text = await server.call("get_performance_report", { + "portfolio_id": portfolio_id, "start_date": first.isoformat(), "end_date": last.isoformat(), + "_keep_sales": True}) + report = parse_report(text) + if report is None: + return None + return PeriodRow(portfolio_id, first.isoformat(), last.isoformat(), float(report["total_gain_percent"]), + float(report.get("total_gain") or 0), report.get("currency") or "", movers(report)) diff --git a/lib/features/periods/windows.py b/lib/features/periods/windows.py new file mode 100644 index 0000000..fcb21f9 --- /dev/null +++ b/lib/features/periods/windows.py @@ -0,0 +1,80 @@ +"""Period windows for period_returns: parsing dates, cutting a date range into days/weeks/months/quarters/years, +and the limits on how many uncached ones one request may fetch.""" + +from __future__ import annotations + +from datetime import date, datetime, timedelta + +PERIODS = ("day", "week", "month", "quarter", "year") +# Uncached reports one call may fetch (about 2.4 s each); cached ones are free. +LIVE_LIMIT = {"day": 64, "week": 26, "month": 120, "quarter": 120, "year": 120} + +Window = tuple[str, date, date] # label, first day, last day + + +def parse_date(text: str, *, end: bool = False) -> date | None: + """A date from YYYY-MM-DD, YYYY-MM or YYYY; a partial one means its first day, or its last for `end`.""" + text = str(text).strip() + for fmt, size in (("%Y-%m-%d", 10), ("%Y-%m", 7), ("%Y", 4)): + try: + d = datetime.strptime(text, fmt).date() + except ValueError: + continue + if end and size < 10: + d = (date(d.year + 1, 1, 1) if size == 4 else month_add(d, 1)) - timedelta(days=1) + return d + return None + + +def month_add(d: date, months: int) -> date: + n = d.year * 12 + d.month - 1 + months + return date(n // 12, n % 12 + 1, 1) + + +def period_windows(period: str, first: date, last: date) -> list[Window]: + """(label, first day, last day) of each period between `first` and `last`; the ends are clipped to them.""" + out: list[Window] = [] + if period == "day": + d = first + while d <= last: + if d.weekday() < 5: # no trading at weekends + out.append((d.isoformat(), d, d)) + d += timedelta(days=1) + elif period == "week": + monday = first - timedelta(days=first.weekday()) + while monday <= last: + iso = monday.isocalendar() + out.append((f"{iso[0]}-W{iso[1]:02d}", max(monday, first), min(monday + timedelta(days=6), last))) + monday += timedelta(days=7) + else: + step = {"month": 1, "quarter": 3, "year": 12}[period] + cur = date(first.year, ((first.month - 1) // step) * step + 1, 1) + while cur <= last: + nxt = month_add(cur, step) + label = {"month": f"{cur.year}-{cur.month:02d}", "quarter": f"{cur.year}-Q{(cur.month - 1) // 3 + 1}", + "year": str(cur.year)}[period] + out.append((label, max(cur, first), min(nxt - timedelta(days=1), last))) + cur = nxt + return out + + +def key(w: Window) -> tuple[str, str]: + return w[1].isoformat(), w[2].isoformat() + + +def refusal(period: str, windows: list[Window], cached: set[tuple[str, str]]) -> str | None: + """Why a request needs too many uncached reports (with the earliest start that fits), or None.""" + limit = LIVE_LIMIT[period] + missing = [w for w in windows if key(w) not in cached] + if len(missing) <= limit: + return None + seen, start = 0, windows[-1][1] + for w in reversed(windows): # walk back from the end until the budget of uncached reports is spent + if key(w) not in cached: + seen += 1 + if seen > limit: + break + start = w[1] + return (f"{len(windows) - len(missing)} of {len(windows)} {period}s are cached and the other {len(missing)} " + f"would take about {len(missing) * 2.4 / 60:.0f} minutes to fetch (the background fill may still be " + f"running or off). Ask again with start={start} or later, or try again later") diff --git a/lib/store.py b/lib/store.py index ea39c34..bfddd9c 100644 --- a/lib/store.py +++ b/lib/store.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import sqlite3 import threading import time @@ -33,7 +34,24 @@ CREATE TABLE IF NOT EXISTS usage ( ts INTEGER, bot_id INTEGER, chat_id INTEGER, model TEXT, kind TEXT, rounds INTEGER, tokens_in INTEGER, tokens_cached INTEGER, tokens_out INTEGER, cost REAL); +-- Finished-period Sharesight returns (sold holdings included), kept until flushed. pct NULL = no data +-- for that window. movers = the holdings that moved most, as JSON [[code, gain, gain_pct], ...]. +CREATE TABLE IF NOT EXISTS period_cache ( + portfolio_id INTEGER, start TEXT, end TEXT, pct REAL, gain REAL, currency TEXT, movers TEXT, + v INTEGER, PRIMARY KEY (portfolio_id, start, end)); """ +PERIOD_CACHE_VERSION = 1 # bump when the stored shape changes: rows of another version are dropped + + +class PeriodRow(NamedTuple): + """One cached window of a portfolio's return (start/end are ISO dates).""" + portfolio_id: int + start: str + end: str + pct: float | None + gain: float | None + currency: str + movers: list class HistoryRow(NamedTuple): @@ -65,6 +83,7 @@ def __init__(self, path: str): self._conn.execute( "DELETE FROM holding_news WHERE length(text) < 40 AND upper(text) LIKE '%NOTHING%'" ) + self._conn.execute("DELETE FROM period_cache WHERE v != ?", (PERIOD_CACHE_VERSION,)) def _run(self, sql: str, params=()) -> list[tuple]: with self._lock: @@ -181,3 +200,28 @@ def usage_summary(self, since_ts: int) -> list[tuple]: "SELECT model, COUNT(*), SUM(tokens_in), SUM(tokens_cached), SUM(tokens_out), SUM(cost) " "FROM usage WHERE bot_id = ? AND ts >= ? GROUP BY model ORDER BY SUM(cost) DESC", (self.bot_id or 0, since_ts)) + + # -- Sharesight period cache ------------------------------------------------------- + def period_get(self, portfolio_id: int, first: str, last: str) -> dict[tuple[str, str], PeriodRow]: + """Cached windows of a portfolio lying within first..last (ISO dates), by (start, end).""" + rows = self._run("SELECT start, end, pct, gain, currency, movers FROM period_cache " + "WHERE portfolio_id = ? AND start >= ? AND end <= ?", (portfolio_id, first, last)) + return {(r[0], r[1]): PeriodRow(portfolio_id, r[0], r[1], r[2], r[3], r[4] or "", json.loads(r[5] or "[]")) + for r in rows} + + def period_put(self, rows: list[PeriodRow]) -> None: + with self._lock: + self._conn.executemany( + "INSERT OR REPLACE INTO period_cache VALUES (?,?,?,?,?,?,?,?)", + [(r.portfolio_id, r.start, r.end, r.pct, r.gain, r.currency, json.dumps(r.movers), + PERIOD_CACHE_VERSION) for r in rows]) + + def period_count(self, portfolio_id: int, first: str, last: str) -> int: + return self._run("SELECT COUNT(*) FROM period_cache WHERE portfolio_id = ? AND start >= ? AND end <= ?", + (portfolio_id, first, last))[0][0] + + def period_flush(self, portfolio_id: int | None = None) -> int: + """Delete the cached windows of one portfolio (or all); returns how many went.""" + with self._lock: + where, args = ("WHERE portfolio_id = ?", (portfolio_id,)) if portfolio_id is not None else ("", ()) + return self._conn.execute(f"DELETE FROM period_cache {where}", args).rowcount diff --git a/tests/test_periods.py b/tests/test_periods.py index 91b6db3..c98a726 100644 --- a/tests/test_periods.py +++ b/tests/test_periods.py @@ -6,6 +6,8 @@ from lib import config from lib.context import Ctx from lib.features import periods +from lib.features.periods import windows +from lib.store import PeriodRow from .conftest import FakeBot from .fakes import FakeMcp, ScriptedBackend, registry @@ -26,17 +28,23 @@ async def call(self, tool, args): "holdings": {"columns": ["code", "total_gain", "total_gain_percent"], "rows": rows}}}) -def make(env, ss=None): +class Always(Sharesight): + async def call(self, tool, args): + self.calls.append((tool, args)) + return json.dumps({"report": {"total_gain_percent": 1.0, "total_gain": 10.0, "currency": "AUD"}}) + + +def make(env, store, ss=None): ss = ss or Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) - return Ctx(config.load(env), None, ScriptedBackend([]), registry(ss), FakeBot()), ss + return Ctx(config.load(env), store, ScriptedBackend([]), registry(ss), FakeBot()), ss def reports(ss): return [a for t, a in ss.calls if t == "get_performance_report"] -async def test_months_run_from_inception_with_sold_holdings_and_best_worst(env): - ctx, ss = make(env) +async def test_months_run_from_inception_with_sold_holdings_and_best_worst(env, store): + ctx, ss = make(env, store) defn, fn = periods.tool(ctx) out = await fn({"portfolio_id": 7, "end": "2026-03"}) assert defn.name == "period_returns" and "Total return per month (AUD)" in out @@ -47,8 +55,8 @@ async def test_months_run_from_inception_with_sold_holdings_and_best_worst(env): assert all(r["_keep_sales"] is True for r in reports(ss)) -async def test_best_and_worst_periods_say_which_holdings_drove_them(env): - ctx, ss = make(env) +async def test_best_and_worst_periods_say_which_holdings_drove_them(env, store): + ctx, ss = make(env, store) _, fn = periods.tool(ctx) out = await fn({"portfolio_id": 7, "end": "2026-03"}) assert "Holdings behind 2026-02: up: AAA +750 (+40.0%); down: BBB -250 (-10.0%)" in out @@ -56,22 +64,43 @@ async def test_best_and_worst_periods_say_which_holdings_drove_them(env): assert "Holdings behind 2026-01" not in out and "CCC" not in out # only the extremes, only movers -async def test_quarters_and_years_are_asked_for_whole_windows(env): - ctx, ss = make(env) +async def test_finished_periods_are_cached_forever_and_served_without_sharesight(env, store): + ctx, ss = make(env, store) + _, fn = periods.tool(ctx) + first = await fn({"portfolio_id": 7, "start": "2025-12", "end": "2026-02"}) + n = len(reports(ss)) + assert n == 3 and store.period_count(7, "2000-01-01", "2100-01-01") == 3 + assert await fn({"portfolio_id": 7, "start": "2025-12", "end": "2026-02"}) == first # same text, from the cache + assert len(reports(ss)) == n + assert store.period_flush(7) == 3 and store.period_flush() == 0 + await fn({"portfolio_id": 7, "start": "2025-12", "end": "2026-02"}) + assert len(reports(ss)) == 2 * n # fetched again after the flush + + +async def test_the_unfinished_period_is_never_cached(env, store): + ctx, ss = make(env, store, Always("sharesight", tools=("list_portfolios", "get_performance_report"))) + _, fn = periods.tool(ctx) + today = datetime.now(ctx.st.tz).date() + await fn({"portfolio_id": 7, "start": today.strftime("%Y-%m")}) # this month, ends today + assert len(reports(ss)) == 1 and store.period_count(7, "2000-01-01", "2100-01-01") == 0 + + +async def test_quarters_and_years_are_asked_for_whole_windows(env, store): + ctx, ss = make(env, store) _, fn = periods.tool(ctx) out = await fn({"portfolio_id": 7, "period": "quarter", "start": "2025-10", "end": "2026-06"}) assert [(r["start_date"], r["end_date"]) for r in reports(ss)] == [ ("2025-10-01", "2025-12-31"), ("2026-01-01", "2026-03-31"), ("2026-04-01", "2026-06-30")] assert "2026-Q1 +1.00%" in out and "2 quarters had no data" in out - ctx, ss = make(env) + ctx, ss = make(env, store) _, fn = periods.tool(ctx) await fn({"portfolio_id": 7, "period": "year", "start": "2024", "end": "2025-06"}) assert [(r["start_date"], r["end_date"]) for r in reports(ss)] == [ ("2024-01-01", "2024-12-31"), ("2025-01-01", "2025-06-30")] -async def test_start_given_skips_the_inception_lookup_and_bad_input_is_reported(env): - ctx, ss = make(env) +async def test_start_given_skips_the_inception_lookup_and_bad_input_is_reported(env, store): + ctx, ss = make(env, store) _, fn = periods.tool(ctx) out = await fn({"portfolio_id": 7, "start": "2026-01", "end": "2026-02"}) assert out.splitlines()[1].startswith("2026-01 +1.00%") and not any(t == "list_portfolios" for t, _ in ss.calls) @@ -79,31 +108,20 @@ async def test_start_given_skips_the_inception_lookup_and_bad_input_is_reported( assert "period must be" in await fn({"portfolio_id": 7, "period": "decade"}) assert "start as YYYY-MM-DD" in await fn({"portfolio_id": 99}) # unknown portfolio: no inception date assert "after end" in await fn({"portfolio_id": 7, "start": "2026-03", "end": "2026-01"}) - none_ctx = Ctx(config.load(env), None, ScriptedBackend([]), registry(), FakeBot()) + none_ctx = Ctx(config.load(env), store, ScriptedBackend([]), registry(), FakeBot()) assert "isn't connected" in await periods.tool(none_ctx)[1]({"portfolio_id": 7}) -async def test_day_and_week_are_limited_to_recent_windows_and_refuse_without_calling(env): - ctx, ss = make(env) - _, fn = periods.tool(ctx) - day = await fn({"portfolio_id": 7, "period": "day", "start": "2020-01-01"}) - week = await fn({"portfolio_id": 7, "period": "week", "start": "2020-01-01"}) - assert "limited to the last 90 days" in day and "start=" in day - assert "limited to 26 weeks" in week and "start=" in week - assert "limit is 120" in await fn({"portfolio_id": 7, "period": "month", "start": "2000-01"}) - assert reports(ss) == [] - - -async def test_day_and_week_default_to_the_allowed_window(env): - ctx, ss = make(env) +async def test_only_uncached_windows_count_towards_the_live_limit(env, store): + ctx, ss = make(env, store) _, fn = periods.tool(ctx) - today = datetime.now(ctx.st.tz).date() - await fn({"portfolio_id": 7, "period": "day"}) - days = [r["start_date"] for r in reports(ss)] - assert days and all(date.fromisoformat(d).weekday() < 5 for d in days) and date.fromisoformat(days[-1]) <= today - ss.calls.clear() - await fn({"portfolio_id": 7, "period": "week"}) - assert len(reports(ss)) == periods.MAX_WEEKS + refused = await fn({"portfolio_id": 7, "period": "day", "start": "2020-01-01"}) + assert "0 of " in refused and "days are cached" in refused and "start=" in refused and reports(ss) == [] + # Cache every weekday from 2025-01-01 to 2026-03-31 except the last ten: the rest fits the live budget. + days = windows.period_windows("day", date(2025, 1, 1), date(2026, 3, 31)) + store.period_put([PeriodRow(7, a.isoformat(), b.isoformat(), 0.5, 5.0, "AUD", []) for _, a, b in days[:-10]]) + out = await fn({"portfolio_id": 7, "period": "day", "start": "2025-01-01", "end": "2026-03-31"}) + assert not out.startswith("Error") and len(reports(ss)) == 10 # only the ten uncached days were fetched @pytest.mark.parametrize("period, first, last, labels", [ @@ -114,13 +132,21 @@ async def test_day_and_week_default_to_the_allowed_window(env): ("day", date(2026, 3, 6), date(2026, 3, 10), ["2026-03-06", "2026-03-09", "2026-03-10"]), # Fri, Mon, Tue ]) def test_period_windows_labels_and_weekends(period, first, last, labels): - w = periods.period_windows(period, first, last) + w = windows.period_windows(period, first, last) assert [x[0] for x in w] == labels assert w[0][1] == first and w[-1][2] == last # clipped to the requested range def test_dates_accept_day_month_or_year_and_end_means_the_period_end(): - assert periods._date("2026-02-03") == date(2026, 2, 3) - assert periods._date("2026-02") == date(2026, 2, 1) and periods._date("2026-02", end=True) == date(2026, 2, 28) - assert periods._date("2025-12", end=True) == date(2025, 12, 31) and periods._date("2025", end=True) == date(2025, 12, 31) - assert periods._date("nope") is None + d = windows.parse_date + assert d("2026-02-03") == date(2026, 2, 3) + assert d("2026-02") == date(2026, 2, 1) and d("2026-02", end=True) == date(2026, 2, 28) + assert d("2025-12", end=True) == date(2025, 12, 31) and d("2025", end=True) == date(2025, 12, 31) + assert d("nope") is None + + +def test_refusal_suggests_the_earliest_start_that_fits_and_ignores_cached_windows(): + w = windows.period_windows("week", date(2026, 1, 5), date(2027, 1, 3)) # 52 weeks + msg = windows.refusal("week", w, set()) + assert "0 of 52 weeks are cached" in msg and f"start={w[-26][1]}" in msg + assert windows.refusal("week", w, {windows.key(x) for x in w[:30]}) is None # 22 uncached: fits diff --git a/tests/test_store.py b/tests/test_store.py index f04f7a1..7aa8b9d 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -80,3 +80,21 @@ def test_writes_are_committed_immediately(tmp_path): second = Store(path) # another process sharing the file sees the writes without any explicit commit second.bot_id = 7 assert second.kv_get("k") == "v" and second.chats() == [(-1, "Group")] + + +def test_period_cache_round_trip_flush_and_version(tmp_path): + from lib.store import PeriodRow + + path = str(tmp_path / "p.db") + s = Store(path) + s.period_put([PeriodRow(7, "2026-01-01", "2026-01-31", 1.5, 150.0, "AUD", [["AAA", 90.0, 12.0]]), + PeriodRow(7, "2026-02-01", "2026-02-28", -2.0, -200.0, "AUD", []), + PeriodRow(8, "2026-01-01", "2026-01-31", 3.0, 30.0, "USD", [])]) + got = s.period_get(7, "2026-01-01", "2026-02-15") # only windows lying within the range + assert list(got) == [("2026-01-01", "2026-01-31")] and got[("2026-01-01", "2026-01-31")].movers == [["AAA", 90.0, 12.0]] + assert s.period_count(7, "2000-01-01", "2100-01-01") == 2 + s.period_put([PeriodRow(7, "2026-01-01", "2026-01-31", 9.0, 1.0, "AUD", [])]) # replaces + assert s.period_get(7, "2026-01-01", "2026-01-31")[("2026-01-01", "2026-01-31")].pct == 9.0 + s._conn.execute("UPDATE period_cache SET v = 0 WHERE portfolio_id = 8") # an older format + assert Store(path).period_count(8, "2000-01-01", "2100-01-01") == 0 # dropped on open + assert s.period_flush(7) == 2 and s.period_flush() == 0 From 8236a47df1ce5a989e92fb0cc0396a1d071b4073 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 08:20:19 +0000 Subject: [PATCH 11/34] PERIOD_BACKFILL: background fill of the period cache; /flushcache and --flush-cache; docs Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- CLAUDE.md | 5 +- README.md | 17 ++++ lib/app.py | 18 ++++ lib/config.py | 3 + lib/features/periods/backfill.py | 126 ++++++++++++++++++++++++++++ lib/mcp/server.py | 5 ++ lib/telegram/commands.py | 16 ++++ tests/test_app.py | 47 ++++++++++- tests/test_backfill.py | 136 +++++++++++++++++++++++++++++++ tests/test_config.py | 7 ++ tests/test_features.py | 22 +++++ 11 files changed, 397 insertions(+), 5 deletions(-) create mode 100644 lib/features/periods/backfill.py create mode 100644 tests/test_backfill.py diff --git a/CLAUDE.md b/CLAUDE.md index d9b5c34..54f5e57 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,7 +15,7 @@ MCP data tools and web search. User-facing docs are in `README.md`; this file is - `config.py`: `Settings` from env (`load()`), `ENV_VARS` registry, tuning constants. The only place that knows the provider. Every env var must appear in README.md (a test enforces it). - `store.py`: SQLite `Store`: history (`messages` shared by groups, `dm_messages` per bot), users, kv, - holding-news state, `usage` ledger. Thread-safe; call through `asyncio.to_thread`. + holding-news state, `usage` ledger, `period_cache` (finished Sharesight periods, kept until flushed). Thread-safe; call through `asyncio.to_thread`. - `msgtext.py` sender_name/describe. `history.py` transcript lines. `textfmt.py` HTML helpers (md_to_html, split_html, plain_text, TG_TAG_RE). `tickers.py` Yahoo links (+ `tickers_data.py`). - `mcp/`: `server.py` (MCPServer lifecycle, TTL cache, `Registry`), `schema.py` (neutral `ToolDef`, @@ -24,7 +24,7 @@ MCP data tools and web search. User-facing docs are in `README.md`; this file is `openrouter.py` + `zai.py` (the only provider-specific code), `runner.py` (THE tool loop), `policy.py` (THE retry ladder), `gate.py` (which tools a message needs). - `prompts.py` all prompts. `ask.py` one entry point: route -> Request -> policy.ask -> usage -> NEEDS_TOOLS rerun. - `telegram/`: `handlers.py` (trigger -> request -> reply), `send.py`, `draft.py` (typing + streaming - drafts), `commands.py` (/credits), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `post.py`, `periods.py` (local `period_returns` tool: one call instead of a report per day/week/month/quarter/year). + drafts), `commands.py` (/credits), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop). - `app.py` wiring/startup; `context.py` `Ctx` (settings, store, backend, registry, bot). - `lib/` holds only what the bot needs to run. Developer helpers live in `tools/`: `report.py` (`make prompt-report`), `capture.py` (real MCP output for fixtures); the bot never imports them. @@ -57,6 +57,7 @@ MCP data tools and web search. User-facing docs are in `README.md`; this file is - z.ai: `tool_choice` is ignored (even "none", so the backend omits tools instead), its built-in `web_search` is ignored whenever function tools are present (so search is a function tool run through its search API), thinking can't be disabled (default `reasoning_effort` low), an invalid search recency is accepted silently (so the tool schema fixes the list), and usage has no cost (price table). +- Finished Sharesight periods are cached forever (`period_cache`); after a backdated change run `/flushcache` or `python bot.py --flush-cache`. The unfinished period is never cached. `_keep_sales` in tool args is reserved for the bot's own tools (the runner strips it from the model's). - `must_search` requests are never retried without tools (answering from memory would invent news). - Telegram HTML: only b/i/u/s/code/pre/a/blockquote/tg-spoiler. `Draft` strips tags (half-written HTML is rejected). - No live Telegram/xAI/OpenRouter/MCP access in tests or in this sandbox; say so when behaviour can't be checked. diff --git a/README.md b/README.md index 9f59b76..b0e13e1 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,7 @@ Each is off by default. The holding-news DM and the movers reply are switched on | `/start` | DM | Greeting (and the buttons if enabled) | | `/holdingnews` | DM, if holding news is on | Run the holding-news check now | | `/credits` | an admin (see below), in a group or a DM | Provider balance (OpenRouter; xAI and z.ai have none to show), then requests, tokens (and % cached) and cost per model for the last 24 h and 7 days | +| `/flushcache [portfolio_id]` | an admin | Forget the cached Sharesight period returns (of one portfolio, or all); see [Caching Sharesight history](#caching-sharesight-history) | **Admins** are the user IDs in `ADMIN_CHAT_IDS` plus anyone who administers a group the bot is in (checked with Telegram, cached for five minutes). The bot only knows a group once someone has written in it, or it was added, since @@ -187,6 +188,21 @@ gets "Portfolio data is for admins only." Holding-news DMs, buttons and down-ser unaffected. With nobody in `ADMIN_CHAT_IDS` and no known group, nobody can DM the bot or reach Sharesight: set `ADMIN_ONLY=off` to allow everyone. +## Caching Sharesight history +Questions like "best day / week / month / quarter / year since inception" are answered by the `period_returns` tool +(admins only), which asks Sharesight for one report per period (about 2.4 s each). Every **finished** period is +stored in the SQLite database (next to the chat history, so it survives upgrades with `DATA_DIR`) and **never +expires**: the current, unfinished period is always fetched fresh. Past history rarely changes; after a backdated +trade or correction in Sharesight, flush the cache with `/flushcache [portfolio_id]` (an admin, in Telegram) or +`python bot.py --flush-cache [portfolio_id]` (needs only `DB_PATH`/`DATA_DIR`, not the bot's keys). + +Without the cache, a request may fetch at most 64 days, 26 weeks or 120 months/quarters/years that aren't cached; a +longer range is refused with the earliest start that fits. Set `PERIOD_BACKFILL=[,...]` (or `all`) to +fill the cache in the background: months first, then quarters, years, weeks and days, one Sharesight call at a time, +never while a user's request is using the server, with a pause between calls (about 1,650 calls per portfolio, a few +hours the first time; progress is logged and it resumes after a restart). Once filled, any range is instant and the +limits above no longer matter. Every six hours it fetches whatever newly finished. + ## Caching and token saving **Provider prompt caching** (xAI and OpenRouter keep a conversation on the server holding its cached prompt; z.ai caches automatically and gets no key): xAI gets `prompt_cache_key` and the `x-grok-conv-id` header; OpenRouter gets `session_id`, @@ -270,6 +286,7 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. | `MCP_CONFIG` | MCP server config file (default mcp_servers.json next to bot.py, whatever `DATA_DIR` is; missing = no MCP tools). | | `MOVERS_BOTS` | Usernames of bots whose end-of-day big-movers lists get explained. Empty (default) = feature off. | | `PORTFOLIO_NAMES` | Comma list of Sharesight portfolio names; a message naming one is treated as a portfolio question (default: the recipients' portfolio names). | +| `PERIOD_BACKFILL` | Comma list of Sharesight portfolio names (or `all`) whose returns per day, week, month, quarter and year are fetched into the local cache in the background (about 1,650 Sharesight calls per portfolio the first time, a few hours), so "best day since inception" answers instantly. Empty (default) = off. See [Caching Sharesight history](#caching-sharesight-history). | | `POST_TO_GROUPS_FROM_DM` | on|off. A group admin can have the bot post in that group from a DM (default off). | | `SEARCH_MODEL` | OpenRouter only: model that runs the searches (default xiaomi/mimo-v2.6-flash:online). | | `SHARESIGHT_HOLDING_NEWS_RECIPIENTS` | Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off. | diff --git a/lib/app.py b/lib/app.py index d8b6bd5..b246e3a 100644 --- a/lib/app.py +++ b/lib/app.py @@ -29,6 +29,7 @@ from . import config from .context import Ctx from .features import dm_buttons, holding_news, post +from .features.periods import backfill from .llm.base import Backend from .llm.openrouter import OpenRouterBackend from .llm.xai import XaiBackend @@ -101,6 +102,8 @@ async def post_init(app: Application) -> None: background.append(asyncio.create_task(holding_news.daily_loop(ctx))) if st.dm_buttons: background.append(asyncio.create_task(dm_buttons.push_on_startup(ctx))) + if st.period_backfill: + background.append(asyncio.create_task(backfill.loop(ctx))) async def post_shutdown(app: Application) -> None: for task in background: @@ -124,6 +127,7 @@ async def wrapper(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: # Commands first: the first matching handler wins, and the catch-all would swallow them. app.add_handler(CommandHandler("credits", command(commands.credits))) + app.add_handler(CommandHandler("flushcache", command(commands.flush_cache))) app.add_handler(MessageHandler( (filters.ChatType.GROUPS | filters.ChatType.PRIVATE) & (filters.UpdateType.MESSAGE | filters.UpdateType.EDITED_MESSAGE) @@ -151,12 +155,26 @@ def git_hash() -> str: return done.stdout.strip() if done.returncode == 0 else "" +def flush_cache(which: str) -> str: + """--flush-cache: needs only the database path, not the bot's keys. Returns the message to print.""" + if which != "all" and not which.isdigit(): + return f"Give a portfolio ID (a number) or nothing for all, not '{which}'." + n = Store(config.data_path(os.environ, "DB_PATH", "chat_log.db")).period_flush(None if which == "all" else int(which)) + return f"Flushed {n} cached periods." + + def main(argv: Sequence[str] | None = None) -> None: # The label does nothing: it only shows in ps/top, to tell instances apart when several run # side by side with different environments (python bot.py mimo). parser = argparse.ArgumentParser(description="Telegram group bot that answers @mentions with an LLM.") parser.add_argument("label", nargs="?", help="ignored; identifies this instance in ps/top") + parser.add_argument("--flush-cache", nargs="?", const="all", metavar="PORTFOLIO_ID", + help="delete the cached Sharesight period returns (of one portfolio, or all) and exit") args, _ = parser.parse_known_args(argv) + if args.flush_cache: + message = flush_cache(args.flush_cache) + print(message) + sys.exit(0 if message.startswith("Flushed") else 1) try: st = config.load() except config.ConfigError as e: diff --git a/lib/config.py b/lib/config.py index dbbbcd1..cee90e5 100644 --- a/lib/config.py +++ b/lib/config.py @@ -33,6 +33,7 @@ "MCP_CONFIG": "MCP server config file (default mcp_servers.json next to bot.py; missing = no MCP tools).", "MOVERS_BOTS": "Usernames of bots whose end-of-day big-movers lists get explained. Empty (default) = feature off.", "PORTFOLIO_NAMES": "Comma list of Sharesight portfolio names; a message naming one is treated as a portfolio question (default: the recipients' portfolio names).", + "PERIOD_BACKFILL": "Comma list of Sharesight portfolio names (or all) whose daily/weekly/monthly/quarterly/yearly returns are fetched into the local cache in the background, so period questions answer instantly. Empty (default) = off.", "POST_TO_GROUPS_FROM_DM": "on|off. Lets an admin of a group the bot is in make it post there from a DM (default off).", "SEARCH_MODEL": "OpenRouter only: model that runs the searches (default xiaomi/mimo-v2.6-flash:online).", "SHARESIGHT_HOLDING_NEWS_RECIPIENTS": "Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off.", @@ -83,6 +84,7 @@ class Settings: holding_news_time: str holding_news_recipients: dict[str, str] portfolio_names: frozenset[str] + period_backfill: frozenset[str] token_saver: bool @property @@ -196,4 +198,5 @@ def load(env: Mapping[str, str] | None = None) -> Settings: holding_news_recipients=recipients, portfolio_names=_names(env.get("PORTFOLIO_NAMES", "")) or frozenset(recipients), token_saver=saver, + period_backfill=_names(env.get("PERIOD_BACKFILL", "")), ) diff --git a/lib/features/periods/backfill.py b/lib/features/periods/backfill.py new file mode 100644 index 0000000..5cadcde --- /dev/null +++ b/lib/features/periods/backfill.py @@ -0,0 +1,126 @@ +"""PERIOD_BACKFILL: fetch finished days, weeks, months, quarters and years of selected Sharesight portfolios into +the period cache in the background, politely (one call at a time, never while a user's request is using the server).""" + +from __future__ import annotations + +import asyncio +import json +import logging +from datetime import date, datetime, timedelta + +from ... import config +from ...context import Ctx +from . import sharesight +from .rows import fetch_row +from .windows import Window, key, parse_date, period_windows + +log = logging.getLogger("bot") + +ORDER = ("month", "quarter", "year", "week", "day") # what pays off first: few calls, most questions +PAUSE = 3.0 # seconds between calls +BUSY_POLL = 2.0 +RECHECK = 6 * 3600 # seconds between passes (a pass fetches only what newly finished) +MAX_FAILS = 5 # consecutive failed fetches that end a pass (Sharesight down or throttling) +PROGRESS_EVERY = 100 + +_wake: asyncio.Event | None = None # set to start a pass now (after a flush) +BAD: set[tuple[int, str, str]] = set() # windows Sharesight had nothing for; not retried until reset() + + +def wake_event() -> asyncio.Event: + global _wake + if _wake is None: + _wake = asyncio.Event() + return _wake + + +def reset() -> None: + """After a cache flush: forget what failed and run a pass now.""" + BAD.clear() + wake_event().set() + + +def natural_end(period: str, w: Window) -> bool: + """Does the window end where its period ends (so it is a finished, canonical window)?""" + end, nxt = w[2], w[2] + timedelta(days=1) + return {"day": True, "week": end.weekday() == 6, "month": nxt.day == 1, + "quarter": nxt.day == 1 and nxt.month in (1, 4, 7, 10), + "year": end.month == 12 and end.day == 31}[period] + + +def wanted(first: date, today: date) -> list[Window]: + """Finished windows from `first` up to yesterday, in the order worth fetching them.""" + yesterday = today - timedelta(days=1) + return [w for p in ORDER for w in period_windows(p, first, yesterday) if natural_end(p, w)] \ + if first <= yesterday else [] + + +def selected(st: config.Settings, listing: str) -> list[tuple[int, str, date]]: + """(id, name, inception) of the portfolios PERIOD_BACKFILL names (or all of them).""" + try: + portfolios = json.loads(listing).get("portfolios") or [] + except (ValueError, AttributeError): + return [] + out = [] + for p in portfolios: + if not isinstance(p, dict) or not isinstance(p.get("id"), int): + continue + name = str(p.get("name") or "") + start = parse_date(p.get("inception_date") or "") + if start and ("all" in st.period_backfill or name.lower() in st.period_backfill): + out.append((p["id"], name, start)) + return out + + +async def fill_once(ctx: Ctx, server, portfolios: list[tuple[int, str, date]], today: date) -> int: + """One pass: fetch every wanted window that isn't cached yet. Returns how many were stored.""" + stored = 0 + for pid, name, first in portfolios: + windows = wanted(first, today) + have = await asyncio.to_thread(ctx.store.period_get, pid, first.isoformat(), today.isoformat()) + todo = [w for w in windows if key(w) not in have and (pid, *key(w)) not in BAD] + if todo: + log.info("Period backfill: %d windows to fetch for %s", len(todo), name) + fails = 0 + for i, w in enumerate(todo, 1): + while server.busy: # a user's request comes first + await asyncio.sleep(BUSY_POLL) + row = await fetch_row(server, pid, w[1], w[2]) + if row is None: + BAD.add((pid, *key(w))) + fails += 1 + if fails >= MAX_FAILS: + log.warning("Period backfill: %d fetches in a row failed; stopping this pass", fails) + return stored + else: + fails = 0 + await asyncio.to_thread(ctx.store.period_put, [row]) + stored += 1 + if i % PROGRESS_EVERY == 0: + log.info("Period backfill: %s %d/%d", name, i, len(todo)) + await asyncio.sleep(PAUSE) + return stored + + +async def loop(ctx: Ctx) -> None: + await ctx.registry.wait_started(config.MCP_STARTUP_WAIT) + while True: + wait = RECHECK + server = sharesight(ctx) + if server is None: + log.warning("Period backfill: Sharesight isn't connected; trying again in 10 minutes") + wait = 600 + else: + try: + portfolios = selected(ctx.st, await server.call("list_portfolios", {})) + if not portfolios: + log.warning("Period backfill: no Sharesight portfolio matches PERIOD_BACKFILL") + n = await fill_once(ctx, server, portfolios, datetime.now(ctx.st.tz).date()) + log.info("Period backfill: pass done, %d new windows cached", n) + except Exception: + log.exception("Period backfill pass failed") + try: + await asyncio.wait_for(wake_event().wait(), wait) + except TimeoutError: + pass + wake_event().clear() diff --git a/lib/mcp/server.py b/lib/mcp/server.py index cad028b..13da544 100644 --- a/lib/mcp/server.py +++ b/lib/mcp/server.py @@ -158,6 +158,11 @@ def _warn_unknown_names(self, listed, configured: set[str]) -> None: log.warning("MCP %s: blocked_tools/allowed_tools entry '%s' matches no tool%s", self.label, name, hint) + @property + def busy(self) -> bool: + """Is every call slot taken right now? (Background work waits for this to clear.)""" + return self._sem.locked() + async def call(self, tool: str, args: dict) -> str: """Run one tool and return its (slimmed, size-capped) text. Identical concurrent or recent calls share one result when cache_ttl is set.""" diff --git a/lib/telegram/commands.py b/lib/telegram/commands.py index f2ac424..76d11fc 100644 --- a/lib/telegram/commands.py +++ b/lib/telegram/commands.py @@ -10,6 +10,7 @@ from telegram.ext import ContextTypes from ..context import Ctx +from ..features.periods import backfill from .access import is_admin log = logging.getLogger("bot") @@ -58,3 +59,18 @@ async def credits(ctx: Ctx, update: Update, context: ContextTypes.DEFAULT_TYPE) rows = await asyncio.to_thread(ctx.store.usage_summary, now - secs) parts.append(usage_text(rows, label)) await _reply(ctx, update, "\n\n".join(parts)) + + +async def flush_cache(ctx: Ctx, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: + """/flushcache [portfolio_id] - admins only: forget the cached Sharesight period returns (of one + portfolio, or all), e.g. after a backdated change. The background fill, if on, re-fetches them.""" + if not await _gate(ctx, update): + return + which = (context.args[0] if context.args else "all").strip() + if which != "all" and not which.isdigit(): + await _reply(ctx, update, "Usage: /flushcache [portfolio_id]") + return + n = await asyncio.to_thread(ctx.store.period_flush, None if which == "all" else int(which)) + backfill.reset() + log.info("Flushed %d cached period returns (%s)", n, which) + await _reply(ctx, update, f"Flushed {n} cached periods." + (" Refilling in the background." if ctx.st.period_backfill else "")) diff --git a/tests/test_app.py b/tests/test_app.py index 1152f29..caab91b 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,3 +1,6 @@ +import asyncio +from types import SimpleNamespace + import pytest from telegram.ext import CallbackQueryHandler, CommandHandler, MessageHandler @@ -30,12 +33,12 @@ def build(**extra): return app.build_app(Ctx(st, store, ScriptedBackend([]), registry())), st plain, st = build() - assert handler_kinds(plain) == ["CommandHandler", "MessageHandler"] and app.allowed_updates(st) == ["message", "edited_message"] + assert handler_kinds(plain) == ["CommandHandler", "CommandHandler", "MessageHandler"] and app.allowed_updates(st) == ["message", "edited_message"] full, st = build(ADMIN_CHAT_IDS="5", SHARESIGHT_HOLDING_NEWS_RECIPIENTS="Pf:alice") kinds = handler_kinds(full) - assert kinds == ["CommandHandler", "MessageHandler", "CallbackQueryHandler"] + assert kinds == ["CommandHandler", "CommandHandler", "MessageHandler", "CallbackQueryHandler"] assert "callback_query" in app.allowed_updates(st) - assert {c for h in full.handlers[0] if isinstance(h, CommandHandler) for c in h.commands} == {"credits"} + assert {c for h in full.handlers[0] if isinstance(h, CommandHandler) for c in h.commands} == {"credits", "flushcache"} assert any(isinstance(h, MessageHandler) for h in full.handlers[0]) locked, st = build(ADMIN_ONLY="on") # remembers the groups it is in, to know who their admins are assert "ChatMemberHandler" in handler_kinds(locked) and "my_chat_member" in app.allowed_updates(st) @@ -104,3 +107,41 @@ async def test_alert_down_notifies_admin_chats_only(env, store): quiet = Ctx(config.load(env), store, ScriptedBackend([]), registry(), FakeBot()) await app.alert_down(quiet, server) assert quiet.bot.sent == [] + + +def test_flush_cache_cli_needs_no_keys_and_flushes_one_or_all(monkeypatch, tmp_path, capsys): + from lib.store import PeriodRow, Store + + db = str(tmp_path / "x.db") + for var in ("TELEGRAM_BOT_TOKEN", "OPENROUTER_API_KEY", "XAI_API_KEY", "ZAI_API_KEY"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("DB_PATH", db) + s = Store(db) + s.period_put([PeriodRow(7, "2026-01-01", "2026-01-31", 1.0, 1.0, "AUD", []), + PeriodRow(7, "2026-02-01", "2026-02-28", 1.0, 1.0, "AUD", []), + PeriodRow(8, "2026-01-01", "2026-01-31", 1.0, 1.0, "AUD", [])]) + with pytest.raises(SystemExit) as e: + app.main(["--flush-cache", "7"]) + assert e.value.code == 0 and "Flushed 2 cached periods." in capsys.readouterr().out + with pytest.raises(SystemExit) as e: + app.main(["--flush-cache"]) + assert e.value.code == 0 and "Flushed 1 cached periods." in capsys.readouterr().out + with pytest.raises(SystemExit) as e: + app.main(["--flush-cache", "abc"]) + assert e.value.code == 1 and "portfolio ID" in capsys.readouterr().out + + +async def test_backfill_task_starts_only_when_period_backfill_is_set(env, store, monkeypatch): + started = [] + + async def fake_loop(ctx): + started.append(ctx) + + monkeypatch.setattr(app.backfill, "loop", fake_loop) + for extra, expected in (({}, 0), ({"PERIOD_BACKFILL": "x"}, 1)): + st = config.load({**env, **extra}) + ctx = Ctx(st, store, ScriptedBackend([]), registry(), FakeBot()) + built = app.build_app(ctx) + await built.post_init(SimpleNamespace(bot=SimpleNamespace(id=1))) + await asyncio.sleep(0) + assert len(started) == expected diff --git a/tests/test_backfill.py b/tests/test_backfill.py new file mode 100644 index 0000000..eff51fc --- /dev/null +++ b/tests/test_backfill.py @@ -0,0 +1,136 @@ +import asyncio +import json +from datetime import date, timedelta + +import pytest + +from lib import config +from lib.context import Ctx +from lib.features.periods import backfill +from lib.features.periods.windows import key, period_windows + +from .conftest import FakeBot +from .fakes import FakeMcp, ScriptedBackend, registry + + +class Sharesight(FakeMcp): + def __init__(self, *a, fail=False, inception="2026-01-15", **kw): + super().__init__(*a, **kw) + self.fail = fail + self.inception = inception + self.busy_for = 0 # polls during which the server counts as busy + self.polls = 0 + self.order: list[tuple[str, str]] = [] + + @property + def busy(self): + self.polls += 1 + return self.polls <= self.busy_for + + async def call(self, tool, args): + self.calls.append((tool, args)) + if tool == "list_portfolios": + return json.dumps({"portfolios": [{"id": 7, "name": "RobSMSF", "inception_date": self.inception}, + {"id": 8, "name": "Other", "inception_date": "2026-01-15"}]}) + self.order.append((args["start_date"], args["end_date"])) + if self.fail: + return "Error: timed out" + return json.dumps({"report": {"total_gain_percent": 1.0, "total_gain": 10.0, "currency": "AUD"}}) + + +@pytest.fixture(autouse=True) +def fast(monkeypatch): + monkeypatch.setattr(backfill, "PAUSE", 0) + monkeypatch.setattr(backfill, "BUSY_POLL", 0) + monkeypatch.setattr(backfill, "_wake", None) + backfill.BAD.clear() + + +def make(env, store, ss, **extra): + st = config.load({**env, "PERIOD_BACKFILL": "robsmsf", **extra}) + return Ctx(st, store, ScriptedBackend([]), registry(ss), FakeBot()), st + + +def test_only_finished_canonical_windows_are_wanted_months_first(): + w = backfill.wanted(date(2026, 1, 15), date(2026, 3, 3)) # yesterday = 2026-03-02 + labels = [x[0] for x in w] + assert labels[:2] == ["2026-01", "2026-02"] and "2026-03" not in labels # March isn't over + assert labels.index("2026-W03") < labels.index("2026-01-15") # weeks before days + assert [x[0] for x in w if x[0].startswith("2026-W")][-1] == "2026-W09" # week ending Sun 1 Mar; not the next + assert w[0][1] == date(2026, 1, 15) # the first window starts at inception + assert not any(x[0] == "2026-Q1" or x[0] == "2026" for x in w) # quarter and year not finished + assert backfill.wanted(date(2026, 3, 5), date(2026, 3, 3)) == [] + + +def test_portfolios_are_selected_by_name_or_all(env): + listing = json.dumps({"portfolios": [{"id": 7, "name": "RobSMSF", "inception_date": "2026-01-15"}, + {"id": 8, "name": "Other", "inception_date": "2026-01-15"}, + {"id": 9, "name": "NoDate"}]}) + pick = lambda **e: [p[1] for p in backfill.selected(config.load({**env, "TELEGRAM_BOT_TOKEN": "1:x", **e}), listing)] # noqa: E731 + assert pick(PERIOD_BACKFILL="robsmsf") == ["RobSMSF"] + assert pick(PERIOD_BACKFILL="all") == ["RobSMSF", "Other"] # a portfolio without an inception date is skipped + assert pick() == [] + + +async def test_a_pass_fetches_what_is_missing_then_only_what_is_new(env, store): + ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) + ctx, st = make(env, store, ss) + portfolios = backfill.selected(st, await ss.call("list_portfolios", {})) + want = backfill.wanted(date(2026, 1, 15), date(2026, 3, 3)) + assert await backfill.fill_once(ctx, ss, portfolios, date(2026, 3, 3)) == len(want) + assert ss.order[0] == (want[0][1].isoformat(), want[0][2].isoformat()) # the first month, from inception + assert store.period_count(7, "2026-01-01", "2026-03-31") == len(want) and store.period_count(8, "2000-01-01", "2100-01-01") == 0 + ss.order.clear() + assert await backfill.fill_once(ctx, ss, portfolios, date(2026, 3, 3)) == 0 and ss.order == [] + assert await backfill.fill_once(ctx, ss, portfolios, date(2026, 3, 4)) == 1 # yesterday, Tue 3 Mar, newly finished + assert ss.order == [("2026-03-03", "2026-03-03")] + + +async def test_it_waits_while_a_user_request_is_using_the_server(env, store): + ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) + ss.busy_for = 3 + ctx, st = make(env, store, ss) + await backfill.fill_once(ctx, ss, [(7, "RobSMSF", date(2026, 3, 1))], date(2026, 3, 3)) + assert ss.polls > 3 and ss.order # polled until free, then fetched + + +async def test_failures_end_the_pass_and_failed_windows_are_not_retried_until_reset(env, store): + ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report"), fail=True) + ctx, _ = make(env, store, ss) + args = ctx, ss, [(7, "RobSMSF", date(2026, 1, 15))], date(2026, 3, 3) + assert await backfill.fill_once(*args) == 0 and len(ss.order) == backfill.MAX_FAILS # gave up after five + first_five = list(ss.order) + ss.order.clear() + await backfill.fill_once(*args) + assert len(ss.order) == backfill.MAX_FAILS and not set(ss.order) & set(first_five) # skipped those five + backfill.reset() + assert not backfill.BAD and backfill.wake_event().is_set() + + +async def test_the_loop_runs_a_pass_then_sleeps_until_woken(env, store, monkeypatch): + recent = (date.today() - timedelta(days=4)).isoformat() + ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report"), inception=recent) + ctx, _ = make(env, store, ss) + monkeypatch.setattr(backfill, "RECHECK", 3600) + task = asyncio.create_task(backfill.loop(ctx)) + expected = len(backfill.wanted(date.today() - timedelta(days=4), date.today())) + for _ in range(500): # the store runs in a thread, so give it real time + await asyncio.sleep(0.01) + if store.period_count(7, recent, "2100-01-01") == expected: + break + assert expected and store.period_count(7, recent, "2100-01-01") == expected + assert store.period_count(8, "2000-01-01", "2100-01-01") == 0 + n = len(ss.order) + backfill.reset() # as after a flush: runs again straight away (nothing new, so nothing fetched) + for _ in range(50): + await asyncio.sleep(0.01) + if not backfill.wake_event().is_set(): + break + await asyncio.sleep(0.05) + assert len(ss.order) == n and not backfill.wake_event().is_set() + task.cancel() + + +def test_keys_match_the_tool_windows(): + w = period_windows("month", date(2026, 1, 15), date(2026, 3, 2))[0] + assert key(w) == ("2026-01-15", "2026-01-31") diff --git a/tests/test_config.py b/tests/test_config.py index e19faaf..98c259c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -25,6 +25,13 @@ def test_data_dir_defaults_to_systemd_state_directory(): assert config.load({**base, "STATE_DIRECTORY": "/a", "DB_PATH": "/x/y.db"}).db_path == "/x/y.db" +def test_period_backfill_is_off_by_default_and_lists_names(): + base = {"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "a"} + assert config.load(base).period_backfill == frozenset() + assert config.load({**base, "PERIOD_BACKFILL": "RobSMSF, bobsmsf"}).period_backfill == {"robsmsf", "bobsmsf"} + assert "all" in config.load({**base, "PERIOD_BACKFILL": "all"}).period_backfill + + def test_both_keys_refused(): with pytest.raises(config.ConfigError, match="only one"): config.load({"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "a", "OPENROUTER_API_KEY": "b"}) diff --git a/tests/test_features.py b/tests/test_features.py index 7d3f2e9..a9bab9b 100644 --- a/tests/test_features.py +++ b/tests/test_features.py @@ -207,3 +207,25 @@ async def test_group_admin_can_run_credits_in_the_group_and_in_a_dm(env, store): @pytest.mark.parametrize("cached,tin,pct", [(0, 0, 0), (50, 100, 50)]) def test_usage_text_percentages(cached, tin, pct): assert f"({pct}% cached)" in commands.usage_text([("m", 1, tin, cached, 1, 0.0)], "x") + + +async def test_flushcache_command_is_admin_only_and_flushes_one_portfolio_or_all(env, store): + from lib.features.periods import backfill + from lib.store import PeriodRow + + st = config.load({**env, "ADMIN_CHAT_IDS": "5", "PERIOD_BACKFILL": "x"}) + ctx = Ctx(st, store, ScriptedBackend([]), registry(), FakeBot()) + store.period_put([PeriodRow(p, f"2026-0{m}-01", f"2026-0{m}-28", 1.0, 1.0, "AUD", []) for p in (7, 8) for m in (1, 2)]) + stranger = user_msg(ctx.bot, "/flushcache", user_id=9, mid=1) + await commands.flush_cache(ctx, update_for(stranger), N(args=[])) + assert stranger.replies == [] and store.period_count(7, "2000-01-01", "2100-01-01") == 2 + one = user_msg(ctx.bot, "/flushcache 7", user_id=5, mid=2) + await commands.flush_cache(ctx, update_for(one), N(args=["7"])) + assert "Flushed 2 cached periods. Refilling" in one.replies[0]["text"] and store.period_count(8, "2000-01-01", "2100-01-01") == 2 + assert backfill.wake_event().is_set() # the background fill starts again + bad = user_msg(ctx.bot, "/flushcache x", user_id=5, mid=3) + await commands.flush_cache(ctx, update_for(bad), N(args=["x"])) + assert "Usage" in bad.replies[0]["text"] + every = user_msg(ctx.bot, "/flushcache", user_id=5, mid=4) + await commands.flush_cache(ctx, update_for(every), N(args=[])) + assert "Flushed 2 cached periods" in every.replies[0]["text"] From aa70d68684f6817889d5689c61b618e805ec9f21 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 08:32:28 +0000 Subject: [PATCH 12/34] Read the git hash from .git directly (works under DynamicUser); log when the period backfill is on Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/app.py | 28 +++++++++++++++++++++------- tests/test_app.py | 47 ++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/lib/app.py b/lib/app.py index b246e3a..3dc47db 100644 --- a/lib/app.py +++ b/lib/app.py @@ -8,7 +8,7 @@ import json import logging import os -import subprocess +import re import sys from collections.abc import Sequence from pathlib import Path @@ -145,14 +145,26 @@ def allowed_updates(st: config.Settings) -> list[str]: + (["my_chat_member"] if st.post_to_groups or st.admin_only else [])) -def git_hash() -> str: - """Short hash of the commit this checkout is on, or "" if it can't be read (no git, no checkout).""" +def git_hash(root: Path | None = None) -> str: + """Short hash of the commit this checkout is on, or "" if it can't be read. Read from .git directly: the + git command refuses a checkout owned by another user (a systemd DynamicUser runs as one).""" + root = root or Path(__file__).resolve().parent.parent try: - done = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=Path(__file__).resolve().parent.parent, - capture_output=True, text=True, timeout=2) - except (OSError, subprocess.SubprocessError): + git = root / ".git" + if git.is_file(): # a linked worktree: ".git" holds "gitdir: " + git = (root / git.read_text().split(":", 1)[1].strip()).resolve() + common = git / (git / "commondir").read_text().strip() if (git / "commondir").exists() else git + head = (git / "HEAD").read_text().strip() + if head.startswith("ref:"): + ref = head.split(None, 1)[1] + try: + head = (common / ref).read_text().strip() + except FileNotFoundError: # packed by `git gc` + packed = (common / "packed-refs").read_text().splitlines() + head = next((line.split()[0] for line in packed if line.endswith(" " + ref)), "") + return head[:7] if re.fullmatch(r"[0-9a-f]{40,64}", head) else "" + except (OSError, IndexError): return "" - return done.stdout.strip() if done.returncode == 0 else "" def flush_cache(which: str) -> str: @@ -186,4 +198,6 @@ def main(argv: Sequence[str] | None = None) -> None: log.info("Starting%s%s: %s on %s, reasoning %s, MCP servers: %s", f" instance {args.label}" if args.label else "", f" ({commit})" if commit else "", st.model, st.provider, st.reasoning or "default", ", ".join(ctx.registry.servers) or "none") + if st.period_backfill: + log.info("Period backfill: on for %s", ", ".join(sorted(st.period_backfill))) app.run_polling(allowed_updates=allowed_updates(st)) diff --git a/tests/test_app.py b/tests/test_app.py index caab91b..7b11cfb 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -73,7 +73,7 @@ def test_main_accepts_and_ignores_a_label_and_unknown_args(monkeypatch, tmp_path assert started -def test_startup_logs_the_git_hash_and_survives_without_git(monkeypatch, tmp_path, caplog): +def test_startup_logs_the_git_hash_and_whether_the_backfill_is_on(monkeypatch, tmp_path, caplog): monkeypatch.setenv("TELEGRAM_BOT_TOKEN", "1:x") monkeypatch.setenv("OPENROUTER_API_KEY", "b") monkeypatch.delenv("XAI_API_KEY", raising=False) @@ -81,18 +81,47 @@ def test_startup_logs_the_git_hash_and_survives_without_git(monkeypatch, tmp_pat monkeypatch.setenv("MCP_CONFIG", str(tmp_path / "none.json")) monkeypatch.setattr(app.Application, "run_polling", lambda self, **kw: None) caplog.set_level("INFO", logger="bot") - monkeypatch.setattr(app.subprocess, "run", lambda *a, **kw: type("R", (), {"returncode": 0, "stdout": "abc1234\n"})()) + monkeypatch.setattr(app, "git_hash", lambda: "abc1234") app.main(["grokbot"]) - assert "Starting instance grokbot (abc1234): " in caplog.text + assert "Starting instance grokbot (abc1234): " in caplog.text and "Period backfill" not in caplog.text caplog.clear() - - def no_git(*a, **kw): - raise FileNotFoundError("git") - - monkeypatch.setattr(app.subprocess, "run", no_git) - assert app.git_hash() == "" + monkeypatch.setattr(app, "git_hash", lambda: "") + monkeypatch.setenv("PERIOD_BACKFILL", "RobSMSF, other") app.main(["grokbot"]) assert "Starting instance grokbot: " in caplog.text + assert "Period backfill: on for other, robsmsf" in caplog.text + + +def test_git_hash_is_read_from_the_git_directory_without_running_git(tmp_path): + sha = "0123456789abcdef0123456789abcdef01234567" + assert app.git_hash(tmp_path) == "" # no checkout + git = tmp_path / ".git" + (git / "refs" / "heads").mkdir(parents=True) + (git / "HEAD").write_text("ref: refs/heads/main\n") + assert app.git_hash(tmp_path) == "" # the branch has no commit yet + (git / "refs" / "heads" / "main").write_text(sha + "\n") + assert app.git_hash(tmp_path) == "0123456" + (git / "refs" / "heads" / "main").unlink() # packed by gc + (git / "packed-refs").write_text(f"# pack-refs\n{sha} refs/heads/main\n") + assert app.git_hash(tmp_path) == "0123456" + (git / "HEAD").write_text(sha + "\n") # detached + assert app.git_hash(tmp_path) == "0123456" + (git / "HEAD").write_text("garbage") + assert app.git_hash(tmp_path) == "" + + +def test_git_hash_follows_a_linked_worktree(tmp_path): + sha = "fedcba9876543210fedcba9876543210fedcba98" + main, tree = tmp_path / "main" / ".git", tmp_path / "wt" + (main / "refs" / "heads").mkdir(parents=True) + (main / "refs" / "heads" / "feat").write_text(sha + "\n") + admin = main / "worktrees" / "wt" + admin.mkdir(parents=True) + (admin / "HEAD").write_text("ref: refs/heads/feat\n") + (admin / "commondir").write_text("../..\n") + tree.mkdir() + (tree / ".git").write_text(f"gitdir: {admin}\n") + assert app.git_hash(tree) == "fedcba9" async def test_alert_down_notifies_admin_chats_only(env, store): From 21c7110e795cf6cc273a0e8fc4d41b9da17d894b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 08:40:53 +0000 Subject: [PATCH 13/34] Period backfill: wait 1 second between fetches Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/features/periods/backfill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/features/periods/backfill.py b/lib/features/periods/backfill.py index 5cadcde..31938b6 100644 --- a/lib/features/periods/backfill.py +++ b/lib/features/periods/backfill.py @@ -17,7 +17,7 @@ log = logging.getLogger("bot") ORDER = ("month", "quarter", "year", "week", "day") # what pays off first: few calls, most questions -PAUSE = 3.0 # seconds between calls +PAUSE = 1.0 # seconds between calls BUSY_POLL = 2.0 RECHECK = 6 * 3600 # seconds between passes (a pass fetches only what newly finished) MAX_FAILS = 5 # consecutive failed fetches that end a pass (Sharesight down or throttling) From 76ff2a14966dc5544c7e25a8caa0a0ecd5f63a72 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 08:41:26 +0000 Subject: [PATCH 14/34] Period backfill: no pause between fetches (still one at a time) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- README.md | 4 ++-- lib/features/periods/backfill.py | 2 -- tests/test_backfill.py | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b0e13e1..ca85e36 100644 --- a/README.md +++ b/README.md @@ -198,8 +198,8 @@ trade or correction in Sharesight, flush the cache with `/flushcache [portfolio_ Without the cache, a request may fetch at most 64 days, 26 weeks or 120 months/quarters/years that aren't cached; a longer range is refused with the earliest start that fits. Set `PERIOD_BACKFILL=[,...]` (or `all`) to -fill the cache in the background: months first, then quarters, years, weeks and days, one Sharesight call at a time, -never while a user's request is using the server, with a pause between calls (about 1,650 calls per portfolio, a few +fill the cache in the background: months first, then quarters, years, weeks and days, one Sharesight call at a time (no concurrency), +never while a user's request is using the server (about 1,650 calls per portfolio, a few hours the first time; progress is logged and it resumes after a restart). Once filled, any range is instant and the limits above no longer matter. Every six hours it fetches whatever newly finished. diff --git a/lib/features/periods/backfill.py b/lib/features/periods/backfill.py index 31938b6..e603f12 100644 --- a/lib/features/periods/backfill.py +++ b/lib/features/periods/backfill.py @@ -17,7 +17,6 @@ log = logging.getLogger("bot") ORDER = ("month", "quarter", "year", "week", "day") # what pays off first: few calls, most questions -PAUSE = 1.0 # seconds between calls BUSY_POLL = 2.0 RECHECK = 6 * 3600 # seconds between passes (a pass fetches only what newly finished) MAX_FAILS = 5 # consecutive failed fetches that end a pass (Sharesight down or throttling) @@ -98,7 +97,6 @@ async def fill_once(ctx: Ctx, server, portfolios: list[tuple[int, str, date]], t stored += 1 if i % PROGRESS_EVERY == 0: log.info("Period backfill: %s %d/%d", name, i, len(todo)) - await asyncio.sleep(PAUSE) return stored diff --git a/tests/test_backfill.py b/tests/test_backfill.py index eff51fc..97cbf7d 100644 --- a/tests/test_backfill.py +++ b/tests/test_backfill.py @@ -40,7 +40,6 @@ async def call(self, tool, args): @pytest.fixture(autouse=True) def fast(monkeypatch): - monkeypatch.setattr(backfill, "PAUSE", 0) monkeypatch.setattr(backfill, "BUSY_POLL", 0) monkeypatch.setattr(backfill, "_wake", None) backfill.BAD.clear() From 782c0acf6b99a72c74bd7116e82dd079c5da29ea Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 08:48:59 +0000 Subject: [PATCH 15/34] Don't truncate results of the bot's own Sharesight calls (cut-off JSON made every backfill fetch fail) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/mcp/server.py | 2 +- tests/test_mcp.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/mcp/server.py b/lib/mcp/server.py index 13da544..88704c2 100644 --- a/lib/mcp/server.py +++ b/lib/mcp/server.py @@ -253,7 +253,7 @@ async def _call(self, tool: str, args: dict) -> str: out = "Tool error: " + out else: out = slim_result(out, self.cfg.get("slim", self.label), drop_closed=holdings_only) - if len(out) > self.max_output: + if len(out) > self.max_output and not keep_sales: # the cap protects the model's context; the bot's own tools aren't sent to it log.warning("MCP %s.%s result truncated (%d chars)", self.label, tool, len(out)) out = out[:self.max_output] + f"\n...[truncated; {len(out)} chars total]" return out diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 1f073a8..7f22e56 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -232,6 +232,14 @@ async def test_bots_own_tools_can_keep_sold_holdings(): assert sent["include_sales"] is True and "_keep_sales" not in sent +async def test_bots_own_tools_are_not_truncated_but_the_models_are(): + s = MCPServer("sharesight", {}, timeout=5, max_output=20) + s.session = FakeSession() + assert "truncated" in await s.call("get_performance_report", {"portfolio_id": 1}) + whole = await s.call("get_performance_report", {"portfolio_id": 1, "_keep_sales": True}) + assert "truncated" not in whole and len(whole) > 20 + + async def test_output_cap(): s = MCPServer("y", {}, timeout=5, max_output=20) s.session = FakeSession() From ee4c288b8e703af70ae46d405e26c768881912ad Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 08:58:49 +0000 Subject: [PATCH 16/34] Period backfill: two fetches at a time (Sharesight max_concurrent 2); servers track in-flight calls Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- README.md | 5 ++- lib/features/periods/backfill.py | 62 ++++++++++++++++++++++---------- lib/mcp/server.py | 14 ++++---- mcp_servers.json | 2 +- tests/test_backfill.py | 30 +++++++++++++--- tests/test_mcp.py | 24 +++++++++++++ 6 files changed, 104 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index ca85e36..61e1fd0 100644 --- a/README.md +++ b/README.md @@ -198,9 +198,8 @@ trade or correction in Sharesight, flush the cache with `/flushcache [portfolio_ Without the cache, a request may fetch at most 64 days, 26 weeks or 120 months/quarters/years that aren't cached; a longer range is refused with the earliest start that fits. Set `PERIOD_BACKFILL=[,...]` (or `all`) to -fill the cache in the background: months first, then quarters, years, weeks and days, one Sharesight call at a time (no concurrency), -never while a user's request is using the server (about 1,650 calls per portfolio, a few -hours the first time; progress is logged and it resumes after a restart). Once filled, any range is instant and the +fill the cache in the background: months first, then quarters, years, weeks and days, up to two Sharesight calls at a time, +never while a user's request is using the server (about 1,650 calls per portfolio, an hour or two the first time; progress is logged and it resumes after a restart). Once filled, any range is instant and the limits above no longer matter. Every six hours it fetches whatever newly finished. ## Caching and token saving diff --git a/lib/features/periods/backfill.py b/lib/features/periods/backfill.py index e603f12..1f3a570 100644 --- a/lib/features/periods/backfill.py +++ b/lib/features/periods/backfill.py @@ -1,5 +1,5 @@ """PERIOD_BACKFILL: fetch finished days, weeks, months, quarters and years of selected Sharesight portfolios into -the period cache in the background, politely (one call at a time, never while a user's request is using the server).""" +the period cache in the background, politely (two calls at a time, never while a user's request is using the server).""" from __future__ import annotations @@ -19,6 +19,7 @@ ORDER = ("month", "quarter", "year", "week", "day") # what pays off first: few calls, most questions BUSY_POLL = 2.0 RECHECK = 6 * 3600 # seconds between passes (a pass fetches only what newly finished) +WORKERS = 2 # fetches at a time (needs max_concurrent >= 2 on the Sharesight server in mcp_servers.json) MAX_FAILS = 5 # consecutive failed fetches that end a pass (Sharesight down or throttling) PROGRESS_EVERY = 100 @@ -71,32 +72,57 @@ def selected(st: config.Settings, listing: str) -> list[tuple[int, str, date]]: return out -async def fill_once(ctx: Ctx, server, portfolios: list[tuple[int, str, date]], today: date) -> int: - """One pass: fetch every wanted window that isn't cached yet. Returns how many were stored.""" - stored = 0 - for pid, name, first in portfolios: - windows = wanted(first, today) - have = await asyncio.to_thread(ctx.store.period_get, pid, first.isoformat(), today.isoformat()) - todo = [w for w in windows if key(w) not in have and (pid, *key(w)) not in BAD] - if todo: - log.info("Period backfill: %d windows to fetch for %s", len(todo), name) - fails = 0 - for i, w in enumerate(todo, 1): - while server.busy: # a user's request comes first +async def fill_portfolio(ctx: Ctx, server, pid: int, name: str, todo: list[Window]) -> tuple[int, bool]: + """Fetch these windows with WORKERS fetches at a time. Returns (how many were stored, whether the pass + was stopped by MAX_FAILS failures in a row).""" + pending = iter(todo) # shared: each window goes to exactly one worker + stored = done = fails = mine = 0 + stopped = False + + async def worker() -> None: + nonlocal stored, done, fails, mine, stopped + for w in pending: + while server.in_flight > mine and not stopped: # a user's request comes first await asyncio.sleep(BUSY_POLL) - row = await fetch_row(server, pid, w[1], w[2]) + if stopped: + return + mine += 1 + try: + row = await fetch_row(server, pid, w[1], w[2]) + finally: + mine -= 1 if row is None: BAD.add((pid, *key(w))) fails += 1 - if fails >= MAX_FAILS: + if fails >= MAX_FAILS and not stopped: + stopped = True log.warning("Period backfill: %d fetches in a row failed; stopping this pass", fails) - return stored else: fails = 0 await asyncio.to_thread(ctx.store.period_put, [row]) stored += 1 - if i % PROGRESS_EVERY == 0: - log.info("Period backfill: %s %d/%d", name, i, len(todo)) + done += 1 + if done % PROGRESS_EVERY == 0: + log.info("Period backfill: %s %d/%d", name, done, len(todo)) + + await asyncio.gather(*(worker() for _ in range(WORKERS))) + return stored, stopped + + +async def fill_once(ctx: Ctx, server, portfolios: list[tuple[int, str, date]], today: date) -> int: + """One pass: fetch every wanted window that isn't cached yet. Returns how many were stored.""" + stored = 0 + for pid, name, first in portfolios: + windows = wanted(first, today) + have = await asyncio.to_thread(ctx.store.period_get, pid, first.isoformat(), today.isoformat()) + todo = [w for w in windows if key(w) not in have and (pid, *key(w)) not in BAD] + if not todo: + continue + log.info("Period backfill: %d windows to fetch for %s", len(todo), name) + n, stopped = await fill_portfolio(ctx, server, pid, name, todo) + stored += n + if stopped: + break return stored diff --git a/lib/mcp/server.py b/lib/mcp/server.py index 88704c2..4b5dd53 100644 --- a/lib/mcp/server.py +++ b/lib/mcp/server.py @@ -54,6 +54,7 @@ def __init__(self, label: str, cfg: dict, *, timeout: float, max_output: int, self.fn_names: dict[str, str] = {} # function name sent to the model -> MCP tool name # Cap simultaneous calls: a 20-stock request fired all at once gets rate-limited. self._sem = asyncio.Semaphore(int(cfg.get("max_concurrent", 4))) + self.in_flight = 0 # calls running or waiting for a slot (background work yields to these) self._ready = asyncio.Event() self._stop = asyncio.Event() self._task: asyncio.Task | None = None @@ -158,11 +159,6 @@ def _warn_unknown_names(self, listed, configured: set[str]) -> None: log.warning("MCP %s: blocked_tools/allowed_tools entry '%s' matches no tool%s", self.label, name, hint) - @property - def busy(self) -> bool: - """Is every call slot taken right now? (Background work waits for this to clear.)""" - return self._sem.locked() - async def call(self, tool: str, args: dict) -> str: """Run one tool and return its (slimmed, size-capped) text. Identical concurrent or recent calls share one result when cache_ttl is set.""" @@ -243,8 +239,12 @@ async def _call(self, tool: str, args: dict) -> str: args = {**args, "include_sales": bool(keep_sales)} # Sharesight's own switch for sold holdings if keep_sales: holdings_only = False # closed holdings stay in the report too - async with self._sem: - result = await asyncio.wait_for(self.session.call_tool(tool, args), self.timeout) + self.in_flight += 1 + try: + async with self._sem: + result = await asyncio.wait_for(self.session.call_tool(tool, args), self.timeout) + finally: + self.in_flight -= 1 parts = [c.text if c.type == "text" else f"[{c.type} content omitted]" for c in result.content] if not parts and getattr(result, "structuredContent", None): parts.append(json.dumps(result.structuredContent)) diff --git a/mcp_servers.json b/mcp_servers.json index e34c573..8ae5a81 100644 --- a/mcp_servers.json +++ b/mcp_servers.json @@ -39,7 +39,7 @@ ], "slim": "sharesight", "gate": "portfolio", - "max_concurrent": 1, + "max_concurrent": 2, "cache_ttl": 1800, "keep_warm": {"list_portfolios": null, "get_performance_report": 0} } diff --git a/tests/test_backfill.py b/tests/test_backfill.py index 97cbf7d..1573f8c 100644 --- a/tests/test_backfill.py +++ b/tests/test_backfill.py @@ -20,12 +20,18 @@ def __init__(self, *a, fail=False, inception="2026-01-15", **kw): self.inception = inception self.busy_for = 0 # polls during which the server counts as busy self.polls = 0 + self.delay = 0 + self.running = self.peak = 0 self.order: list[tuple[str, str]] = [] @property - def busy(self): + def in_flight(self): # a user's request is running for the first few polls self.polls += 1 - return self.polls <= self.busy_for + return 1 if self.polls <= self.busy_for else 0 + + @in_flight.setter + def in_flight(self, value): + pass async def call(self, tool, args): self.calls.append((tool, args)) @@ -33,6 +39,10 @@ async def call(self, tool, args): return json.dumps({"portfolios": [{"id": 7, "name": "RobSMSF", "inception_date": self.inception}, {"id": 8, "name": "Other", "inception_date": "2026-01-15"}]}) self.order.append((args["start_date"], args["end_date"])) + self.running += 1 + self.peak = max(self.peak, self.running) + await asyncio.sleep(self.delay) + self.running -= 1 if self.fail: return "Error: timed out" return json.dumps({"report": {"total_gain_percent": 1.0, "total_gain": 10.0, "currency": "AUD"}}) @@ -93,15 +103,27 @@ async def test_it_waits_while_a_user_request_is_using_the_server(env, store): assert ss.polls > 3 and ss.order # polled until free, then fetched +async def test_two_fetches_run_at_a_time_and_each_window_is_fetched_once(env, store): + ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) + ss.delay = 0.01 + ctx, _ = make(env, store, ss) + want = backfill.wanted(date(2026, 1, 15), date(2026, 3, 3)) + assert await backfill.fill_once(ctx, ss, [(7, "RobSMSF", date(2026, 1, 15))], date(2026, 3, 3)) == len(want) + assert ss.peak == backfill.WORKERS == 2 + assert len(ss.order) == len(set(ss.order)) == len(want) # none twice + assert store.period_count(7, "2026-01-01", "2026-03-31") == len(want) + + async def test_failures_end_the_pass_and_failed_windows_are_not_retried_until_reset(env, store): ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report"), fail=True) ctx, _ = make(env, store, ss) args = ctx, ss, [(7, "RobSMSF", date(2026, 1, 15))], date(2026, 3, 3) - assert await backfill.fill_once(*args) == 0 and len(ss.order) == backfill.MAX_FAILS # gave up after five + assert await backfill.fill_once(*args) == 0 + assert backfill.MAX_FAILS <= len(ss.order) <= backfill.MAX_FAILS + backfill.WORKERS - 1 # gave up after five first_five = list(ss.order) ss.order.clear() await backfill.fill_once(*args) - assert len(ss.order) == backfill.MAX_FAILS and not set(ss.order) & set(first_five) # skipped those five + assert len(ss.order) >= backfill.MAX_FAILS and not set(ss.order) & set(first_five) # skipped the failed ones backfill.reset() assert not backfill.BAD and backfill.wake_event().is_set() diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 7f22e56..168d12b 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -240,6 +240,30 @@ async def test_bots_own_tools_are_not_truncated_but_the_models_are(): assert "truncated" not in whole and len(whole) > 20 +async def test_in_flight_counts_running_calls_and_returns_to_zero(): + s = MCPServer("y", {"max_concurrent": 1}, timeout=0.05, max_output=1000) + gate = asyncio.Event() + + class Slow(FakeSession): + async def call_tool(self, name, args): + if name == "hang": + await asyncio.sleep(1) # longer than the timeout + await gate.wait() + return await super().call_tool(name, args) + + s.session = Slow() + first = asyncio.create_task(s.call("get_a", {})) + second = asyncio.create_task(s.call("get_b", {})) # waits for the only slot, still counted + await asyncio.sleep(0.01) + assert s.in_flight == 2 + gate.set() + await asyncio.gather(first, second) + assert s.in_flight == 0 + with pytest.raises(TimeoutError): + await s.call("hang", {}) + assert s.in_flight == 0 # also after a timeout + + async def test_output_cap(): s = MCPServer("y", {}, timeout=5, max_output=20) s.session = FakeSession() From 68c441ab3be27022960f36be983b712a3ecacfce Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 09:06:40 +0000 Subject: [PATCH 17/34] Log MCP connects with the server name; drop the library's unlabelled protocol line Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/app.py | 8 ++++++++ lib/mcp/server.py | 3 ++- tests/test_app.py | 18 ++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/lib/app.py b/lib/app.py index 3dc47db..5c33bda 100644 --- a/lib/app.py +++ b/lib/app.py @@ -53,6 +53,14 @@ def setup_logging() -> None: format="%(levelname)s %(message)s" if os.getenv("JOURNAL_STREAM") else "%(asctime)s %(levelname)s %(message)s") logging.getLogger("httpx").setLevel(logging.WARNING) + for handler in logging.getLogger().handlers: + handler.addFilter(_drop_protocol_line) + + +def _drop_protocol_line(record: logging.LogRecord) -> bool: + """The MCP library's own "Negotiated protocol version" line doesn't say which server it is about; the + bot logs "MCP : connected, protocol ..." instead.""" + return not record.getMessage().startswith("Negotiated protocol version") async def alert_down(ctx: Ctx, server: MCPServer) -> None: diff --git a/lib/mcp/server.py b/lib/mcp/server.py index 4b5dd53..ec46101 100644 --- a/lib/mcp/server.py +++ b/lib/mcp/server.py @@ -108,7 +108,8 @@ async def _run(self) -> None: env=expand_env(self.cfg.get("env"), self.label)) read, write = await stack.enter_async_context(stdio_client(params, errlog=errlog)) session = await stack.enter_async_context(ClientSession(read, write)) - await session.initialize() + init = await session.initialize() + log.info("MCP %s: connected, protocol %s", self.label, getattr(init, "protocolVersion", "?")) await self._load_tools(session) self.session, self.error = session, None self._ready.set() diff --git a/tests/test_app.py b/tests/test_app.py index 7b11cfb..8405971 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -174,3 +174,21 @@ async def fake_loop(ctx): await built.post_init(SimpleNamespace(bot=SimpleNamespace(id=1))) await asyncio.sleep(0) assert len(started) == expected + + +def test_the_mcp_librarys_unlabelled_protocol_line_is_dropped(monkeypatch): + import logging + + monkeypatch.delenv("JOURNAL_STREAM", raising=False) + root = logging.getLogger() + saved = list(root.handlers), root.level + root.handlers.clear() + try: + app.setup_logging() + keep = [h for h in root.handlers] + mk = lambda msg: logging.LogRecord("mcp.client", logging.INFO, "f", 1, msg, None, None) # noqa: E731 + assert all(not h.filter(mk("Negotiated protocol version: 2025-11-25")) for h in keep) + assert all(h.filter(mk("MCP yahoo: connected, protocol 2025-11-25")) for h in keep) + finally: + root.handlers[:] = saved[0] + root.setLevel(saved[1]) From a9525df86bf02b665895d9de2c75b606e0e8bb84 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 09:10:17 +0000 Subject: [PATCH 18/34] Flush the period cache for one year, month or day (/flushcache, --flush-cache) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- CLAUDE.md | 2 +- README.md | 8 +++++--- lib/app.py | 24 +++++++++++++++--------- lib/features/periods/windows.py | 25 +++++++++++++++++++++++++ lib/store.py | 16 ++++++++++++---- lib/telegram/commands.py | 21 +++++++++++++-------- tests/test_app.py | 30 +++++++++++++++++------------- tests/test_features.py | 25 +++++++++++++++---------- tests/test_periods.py | 11 +++++++++++ tests/test_store.py | 11 +++++++++++ 10 files changed, 125 insertions(+), 48 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 54f5e57..2079044 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,7 +57,7 @@ MCP data tools and web search. User-facing docs are in `README.md`; this file is - z.ai: `tool_choice` is ignored (even "none", so the backend omits tools instead), its built-in `web_search` is ignored whenever function tools are present (so search is a function tool run through its search API), thinking can't be disabled (default `reasoning_effort` low), an invalid search recency is accepted silently (so the tool schema fixes the list), and usage has no cost (price table). -- Finished Sharesight periods are cached forever (`period_cache`); after a backdated change run `/flushcache` or `python bot.py --flush-cache`. The unfinished period is never cached. `_keep_sales` in tool args is reserved for the bot's own tools (the runner strips it from the model's). +- Finished Sharesight periods are cached forever (`period_cache`); after a backdated change run `/flushcache [portfolio] [year|month|day]` or `python bot.py --flush-cache ...`. The unfinished period is never cached. `_keep_sales` in tool args is reserved for the bot's own tools (the runner strips it from the model's). - `must_search` requests are never retried without tools (answering from memory would invent news). - Telegram HTML: only b/i/u/s/code/pre/a/blockquote/tg-spoiler. `Draft` strips tags (half-written HTML is rejected). - No live Telegram/xAI/OpenRouter/MCP access in tests or in this sandbox; say so when behaviour can't be checked. diff --git a/README.md b/README.md index 61e1fd0..def73c5 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ Each is off by default. The holding-news DM and the movers reply are switched on | `/start` | DM | Greeting (and the buttons if enabled) | | `/holdingnews` | DM, if holding news is on | Run the holding-news check now | | `/credits` | an admin (see below), in a group or a DM | Provider balance (OpenRouter; xAI and z.ai have none to show), then requests, tokens (and % cached) and cost per model for the last 24 h and 7 days | -| `/flushcache [portfolio_id]` | an admin | Forget the cached Sharesight period returns (of one portfolio, or all); see [Caching Sharesight history](#caching-sharesight-history) | +| `/flushcache [portfolio_id] [year or month or day]` | an admin | Forget cached Sharesight period returns: all, or of one portfolio, or those overlapping a year (`2024`), month (`2024-05`) or day (`2024-05-17`), e.g. `/flushcache 684141 2024`; see [Caching Sharesight history](#caching-sharesight-history) | **Admins** are the user IDs in `ADMIN_CHAT_IDS` plus anyone who administers a group the bot is in (checked with Telegram, cached for five minutes). The bot only knows a group once someone has written in it, or it was added, since @@ -193,8 +193,10 @@ Questions like "best day / week / month / quarter / year since inception" are an (admins only), which asks Sharesight for one report per period (about 2.4 s each). Every **finished** period is stored in the SQLite database (next to the chat history, so it survives upgrades with `DATA_DIR`) and **never expires**: the current, unfinished period is always fetched fresh. Past history rarely changes; after a backdated -trade or correction in Sharesight, flush the cache with `/flushcache [portfolio_id]` (an admin, in Telegram) or -`python bot.py --flush-cache [portfolio_id]` (needs only `DB_PATH`/`DATA_DIR`, not the bot's keys). +trade or correction in Sharesight, flush the cache with `/flushcache [portfolio_id] [year|month|day]` (an admin, in +Telegram) or `python bot.py --flush-cache [portfolio_id] [year|month|day]` (needs only `DB_PATH`/`DATA_DIR`, not the +bot's keys). With a year, month or day, only the periods overlapping it go (every day, week, month, quarter and year +that contains it), and the background fill re-fetches just those. Without the cache, a request may fetch at most 64 days, 26 weeks or 120 months/quarters/years that aren't cached; a longer range is refused with the earliest start that fits. Set `PERIOD_BACKFILL=[,...]` (or `all`) to diff --git a/lib/app.py b/lib/app.py index 5c33bda..96f977d 100644 --- a/lib/app.py +++ b/lib/app.py @@ -30,6 +30,7 @@ from .context import Ctx from .features import dm_buttons, holding_news, post from .features.periods import backfill +from .features.periods.windows import describe_scope, flush_scope from .llm.base import Backend from .llm.openrouter import OpenRouterBackend from .llm.xai import XaiBackend @@ -175,12 +176,16 @@ def git_hash(root: Path | None = None) -> str: return "" -def flush_cache(which: str) -> str: - """--flush-cache: needs only the database path, not the bot's keys. Returns the message to print.""" - if which != "all" and not which.isdigit(): - return f"Give a portfolio ID (a number) or nothing for all, not '{which}'." - n = Store(config.data_path(os.environ, "DB_PATH", "chat_log.db")).period_flush(None if which == "all" else int(which)) - return f"Flushed {n} cached periods." +def flush_cache(words: list[str]) -> str: + """--flush-cache [portfolio ID] [year|month|day]: needs only the database path, not the bot's keys. + Returns the message to print.""" + scope = flush_scope(words) + if scope is None: + return "Usage: --flush-cache [portfolio ID] [2024 | 2024-05 | 2024-05-17]" + pid, first, last = scope + path = config.data_path(os.environ, "DB_PATH", "chat_log.db") + n = Store(path).period_flush(pid, first and first.isoformat(), last and last.isoformat()) + return f"Flushed {n} cached periods ({describe_scope(pid, first, last)})." def main(argv: Sequence[str] | None = None) -> None: @@ -188,10 +193,11 @@ def main(argv: Sequence[str] | None = None) -> None: # side by side with different environments (python bot.py mimo). parser = argparse.ArgumentParser(description="Telegram group bot that answers @mentions with an LLM.") parser.add_argument("label", nargs="?", help="ignored; identifies this instance in ps/top") - parser.add_argument("--flush-cache", nargs="?", const="all", metavar="PORTFOLIO_ID", - help="delete the cached Sharesight period returns (of one portfolio, or all) and exit") + parser.add_argument("--flush-cache", nargs="*", metavar="WHAT", + help="delete the cached Sharesight period returns and exit: all, or of a portfolio ID " + "and/or a year, month or day (2024, 2024-05, 2024-05-17)") args, _ = parser.parse_known_args(argv) - if args.flush_cache: + if args.flush_cache is not None: message = flush_cache(args.flush_cache) print(message) sys.exit(0 if message.startswith("Flushed") else 1) diff --git a/lib/features/periods/windows.py b/lib/features/periods/windows.py index fcb21f9..c0fa320 100644 --- a/lib/features/periods/windows.py +++ b/lib/features/periods/windows.py @@ -78,3 +78,28 @@ def refusal(period: str, windows: list[Window], cached: set[tuple[str, str]]) -> return (f"{len(windows) - len(missing)} of {len(windows)} {period}s are cached and the other {len(missing)} " f"would take about {len(missing) * 2.4 / 60:.0f} minutes to fetch (the background fill may still be " f"running or off). Ask again with start={start} or later, or try again later") + + +def flush_scope(words: list[str]) -> tuple[int | None, date | None, date | None] | None: + """What /flushcache and --flush-cache were asked to forget: any of a portfolio ID (5+ digits), a year + (2024), a month (2024-05) or a day (2024-05-17), in any order, plus an optional "all". Returns + (portfolio ID, first day, last day), each None when not given, or None if a word isn't understood.""" + pid, first, last = None, None, None + for word in words: + word = word.strip().lower() + if word in ("", "all"): + continue + if word.isdigit() and len(word) >= 5 and pid is None: + pid = int(word) + elif first is None and (d := parse_date(word)) and 1990 <= d.year <= 2100 and word[:4].isdigit(): + first, last = d, parse_date(word, end=True) + else: + return None + return pid, first, last + + +def describe_scope(pid: int | None, first: date | None, last: date | None) -> str: + parts = ["all portfolios" if pid is None else f"portfolio {pid}"] + if first: + parts.append(f"{first}" if first == last else f"{first} to {last}") + return ", ".join(parts) diff --git a/lib/store.py b/lib/store.py index bfddd9c..b58850d 100644 --- a/lib/store.py +++ b/lib/store.py @@ -220,8 +220,16 @@ def period_count(self, portfolio_id: int, first: str, last: str) -> int: return self._run("SELECT COUNT(*) FROM period_cache WHERE portfolio_id = ? AND start >= ? AND end <= ?", (portfolio_id, first, last))[0][0] - def period_flush(self, portfolio_id: int | None = None) -> int: - """Delete the cached windows of one portfolio (or all); returns how many went.""" + def period_flush(self, portfolio_id: int | None = None, first: str | None = None, last: str | None = None) -> int: + """Delete cached windows, of one portfolio or all, and (if first/last ISO dates are given) only those + overlapping that date range. Returns how many went.""" + where, args = [], [] + if portfolio_id is not None: + where.append("portfolio_id = ?") + args.append(portfolio_id) + if first and last: + where.append("start <= ? AND end >= ?") + args += [last, first] with self._lock: - where, args = ("WHERE portfolio_id = ?", (portfolio_id,)) if portfolio_id is not None else ("", ()) - return self._conn.execute(f"DELETE FROM period_cache {where}", args).rowcount + sql = "DELETE FROM period_cache" + (" WHERE " + " AND ".join(where) if where else "") + return self._conn.execute(sql, args).rowcount diff --git a/lib/telegram/commands.py b/lib/telegram/commands.py index 76d11fc..a4130a2 100644 --- a/lib/telegram/commands.py +++ b/lib/telegram/commands.py @@ -11,6 +11,7 @@ from ..context import Ctx from ..features.periods import backfill +from ..features.periods.windows import describe_scope, flush_scope from .access import is_admin log = logging.getLogger("bot") @@ -62,15 +63,19 @@ async def credits(ctx: Ctx, update: Update, context: ContextTypes.DEFAULT_TYPE) async def flush_cache(ctx: Ctx, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: - """/flushcache [portfolio_id] - admins only: forget the cached Sharesight period returns (of one - portfolio, or all), e.g. after a backdated change. The background fill, if on, re-fetches them.""" + """/flushcache [portfolio_id] [year | month | day] - admins only: forget the cached Sharesight period + returns (all, or of one portfolio, or the ones overlapping a year/month/day), e.g. after a backdated + change. The background fill, if on, re-fetches them.""" if not await _gate(ctx, update): return - which = (context.args[0] if context.args else "all").strip() - if which != "all" and not which.isdigit(): - await _reply(ctx, update, "Usage: /flushcache [portfolio_id]") + scope = flush_scope(context.args or []) + if scope is None: + await _reply(ctx, update, "Usage: /flushcache [portfolio_id] [2024 | 2024-05 | 2024-05-17]") return - n = await asyncio.to_thread(ctx.store.period_flush, None if which == "all" else int(which)) + pid, first, last = scope + n = await asyncio.to_thread(ctx.store.period_flush, pid, first and first.isoformat(), last and last.isoformat()) backfill.reset() - log.info("Flushed %d cached period returns (%s)", n, which) - await _reply(ctx, update, f"Flushed {n} cached periods." + (" Refilling in the background." if ctx.st.period_backfill else "")) + what = describe_scope(pid, first, last) + log.info("Flushed %d cached period returns (%s)", n, what) + await _reply(ctx, update, f"Flushed {n} cached periods ({what})." + + (" Refilling in the background." if ctx.st.period_backfill else "")) diff --git a/tests/test_app.py b/tests/test_app.py index 8405971..91db5d9 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -138,7 +138,7 @@ async def test_alert_down_notifies_admin_chats_only(env, store): assert quiet.bot.sent == [] -def test_flush_cache_cli_needs_no_keys_and_flushes_one_or_all(monkeypatch, tmp_path, capsys): +def test_flush_cache_cli_needs_no_keys_and_flushes_by_portfolio_and_period(monkeypatch, tmp_path, capsys): from lib.store import PeriodRow, Store db = str(tmp_path / "x.db") @@ -146,18 +146,22 @@ def test_flush_cache_cli_needs_no_keys_and_flushes_one_or_all(monkeypatch, tmp_p monkeypatch.delenv(var, raising=False) monkeypatch.setenv("DB_PATH", db) s = Store(db) - s.period_put([PeriodRow(7, "2026-01-01", "2026-01-31", 1.0, 1.0, "AUD", []), - PeriodRow(7, "2026-02-01", "2026-02-28", 1.0, 1.0, "AUD", []), - PeriodRow(8, "2026-01-01", "2026-01-31", 1.0, 1.0, "AUD", [])]) - with pytest.raises(SystemExit) as e: - app.main(["--flush-cache", "7"]) - assert e.value.code == 0 and "Flushed 2 cached periods." in capsys.readouterr().out - with pytest.raises(SystemExit) as e: - app.main(["--flush-cache"]) - assert e.value.code == 0 and "Flushed 1 cached periods." in capsys.readouterr().out - with pytest.raises(SystemExit) as e: - app.main(["--flush-cache", "abc"]) - assert e.value.code == 1 and "portfolio ID" in capsys.readouterr().out + row = lambda pid, a, b: PeriodRow(pid, a, b, 1.0, 1.0, "AUD", []) # noqa: E731 + s.period_put([row(70007, "2023-12-26", "2024-01-01"), row(70007, "2024-03-01", "2024-03-31"), + row(70007, "2025-01-01", "2025-12-31"), row(80008, "2024-03-01", "2024-03-31")]) + + def run(*words): + with pytest.raises(SystemExit) as e: + app.main(["--flush-cache", *words]) + return e.value.code, capsys.readouterr().out + + assert run("70007", "2024") == (0, "Flushed 2 cached periods (portfolio 70007, 2024-01-01 to 2024-12-31).\n") + assert run("2025-06") == (0, "Flushed 1 cached periods (all portfolios, 2025-06-01 to 2025-06-30).\n") + assert run("80008") == (0, "Flushed 1 cached periods (portfolio 80008).\n") + assert run() == (0, "Flushed 0 cached periods (all portfolios).\n") + code, out = run("abc") + assert code == 1 and "Usage" in out + assert run("70007", "70008")[0] == 1 # one portfolio at a time async def test_backfill_task_starts_only_when_period_backfill_is_set(env, store, monkeypatch): diff --git a/tests/test_features.py b/tests/test_features.py index a9bab9b..afc607a 100644 --- a/tests/test_features.py +++ b/tests/test_features.py @@ -209,23 +209,28 @@ def test_usage_text_percentages(cached, tin, pct): assert f"({pct}% cached)" in commands.usage_text([("m", 1, tin, cached, 1, 0.0)], "x") -async def test_flushcache_command_is_admin_only_and_flushes_one_portfolio_or_all(env, store): +async def test_flushcache_command_is_admin_only_and_flushes_by_portfolio_and_period(env, store): from lib.features.periods import backfill from lib.store import PeriodRow st = config.load({**env, "ADMIN_CHAT_IDS": "5", "PERIOD_BACKFILL": "x"}) ctx = Ctx(st, store, ScriptedBackend([]), registry(), FakeBot()) - store.period_put([PeriodRow(p, f"2026-0{m}-01", f"2026-0{m}-28", 1.0, 1.0, "AUD", []) for p in (7, 8) for m in (1, 2)]) + store.period_put([PeriodRow(p, f"{y}-0{m}-01", f"{y}-0{m}-28", 1.0, 1.0, "AUD", []) + for p in (70007, 80008) for y in (2024, 2025) for m in (1, 2)]) + count = lambda p: store.period_count(p, "2000-01-01", "2100-01-01") # noqa: E731 stranger = user_msg(ctx.bot, "/flushcache", user_id=9, mid=1) await commands.flush_cache(ctx, update_for(stranger), N(args=[])) - assert stranger.replies == [] and store.period_count(7, "2000-01-01", "2100-01-01") == 2 - one = user_msg(ctx.bot, "/flushcache 7", user_id=5, mid=2) - await commands.flush_cache(ctx, update_for(one), N(args=["7"])) - assert "Flushed 2 cached periods. Refilling" in one.replies[0]["text"] and store.period_count(8, "2000-01-01", "2100-01-01") == 2 - assert backfill.wake_event().is_set() # the background fill starts again - bad = user_msg(ctx.bot, "/flushcache x", user_id=5, mid=3) + assert stranger.replies == [] and count(70007) == 4 + year = user_msg(ctx.bot, "/flushcache 70007 2024", user_id=5, mid=2) + await commands.flush_cache(ctx, update_for(year), N(args=["70007", "2024"])) + assert "Flushed 2 cached periods (portfolio 70007, 2024-01-01 to 2024-12-31). Refilling" in year.replies[0]["text"] + assert count(70007) == 2 and count(80008) == 4 and backfill.wake_event().is_set() + month = user_msg(ctx.bot, "/flushcache 2025-01", user_id=5, mid=3) + await commands.flush_cache(ctx, update_for(month), N(args=["2025-01"])) + assert "Flushed 2 cached periods (all portfolios, 2025-01-01 to 2025-01-31)" in month.replies[0]["text"] + bad = user_msg(ctx.bot, "/flushcache x", user_id=5, mid=4) await commands.flush_cache(ctx, update_for(bad), N(args=["x"])) assert "Usage" in bad.replies[0]["text"] - every = user_msg(ctx.bot, "/flushcache", user_id=5, mid=4) + every = user_msg(ctx.bot, "/flushcache", user_id=5, mid=5) await commands.flush_cache(ctx, update_for(every), N(args=[])) - assert "Flushed 2 cached periods" in every.replies[0]["text"] + assert "Flushed 4 cached periods (all portfolios)" in every.replies[0]["text"] diff --git a/tests/test_periods.py b/tests/test_periods.py index c98a726..c03c199 100644 --- a/tests/test_periods.py +++ b/tests/test_periods.py @@ -150,3 +150,14 @@ def test_refusal_suggests_the_earliest_start_that_fits_and_ignores_cached_window msg = windows.refusal("week", w, set()) assert "0 of 52 weeks are cached" in msg and f"start={w[-26][1]}" in msg assert windows.refusal("week", w, {windows.key(x) for x in w[:30]}) is None # 22 uncached: fits + + +def test_flush_scope_understands_portfolio_year_month_day_and_all(): + f = windows.flush_scope + assert f([]) == (None, None, None) and f(["all"]) == (None, None, None) + assert f(["684141"]) == (684141, None, None) + assert f(["2024"]) == (None, date(2024, 1, 1), date(2024, 12, 31)) + assert f(["2024-05", "684141"]) == (684141, date(2024, 5, 1), date(2024, 5, 31)) + assert f(["2024-05-17"]) == (None, date(2024, 5, 17), date(2024, 5, 17)) + assert f(["nope"]) is None and f(["684141", "702349"]) is None and f(["2024", "2025"]) is None + assert f(["1234"]) is None # 4 digits that aren't a year in range diff --git a/tests/test_store.py b/tests/test_store.py index 7aa8b9d..397dde5 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -98,3 +98,14 @@ def test_period_cache_round_trip_flush_and_version(tmp_path): s._conn.execute("UPDATE period_cache SET v = 0 WHERE portfolio_id = 8") # an older format assert Store(path).period_count(8, "2000-01-01", "2100-01-01") == 0 # dropped on open assert s.period_flush(7) == 2 and s.period_flush() == 0 + + +def test_period_flush_by_date_range_removes_every_window_that_overlaps_it(tmp_path): + from lib.store import PeriodRow + + s = Store(str(tmp_path / "p.db")) + row = lambda a, b: PeriodRow(7, a, b, 1.0, 1.0, "AUD", []) # noqa: E731 + s.period_put([row("2023-12-26", "2024-01-01"), row("2024-12-31", "2025-01-06"), row("2024-06-01", "2024-06-30"), + row("2023-01-01", "2023-12-31"), row("2025-01-07", "2025-01-13")]) + assert s.period_flush(7, "2024-01-01", "2024-12-31") == 3 # the week spanning New Year both ways, and June + assert sorted(s.period_get(7, "2000-01-01", "2100-01-01")) == [("2023-01-01", "2023-12-31"), ("2025-01-07", "2025-01-13")] From 7ce0f2a50dbb3ef470b631b55a34b25df9d683e4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 09:37:15 +0000 Subject: [PATCH 19/34] period_returns: best/worst by money as well as by percent; short follow-ups to the bot's reply keep their tools Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/features/periods/__init__.py | 12 +++++++----- lib/telegram/handlers.py | 18 +++++++++++++++++- tests/test_handlers.py | 16 ++++++++++++++++ tests/test_periods.py | 17 ++++++++++++++++- 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/lib/features/periods/__init__.py b/lib/features/periods/__init__.py index 7df2f85..0bde18c 100644 --- a/lib/features/periods/__init__.py +++ b/lib/features/periods/__init__.py @@ -17,8 +17,8 @@ TOOL = ToolDef( "period_returns", - "Total return of a Sharesight portfolio for each day, week, month, quarter or year, with the best and " - "worst one and the holdings behind them. Use it for best/worst day, week, month, quarter or year, or any " + "Total return of a Sharesight portfolio for each day, week, month, quarter or year, as a percentage and a " + "money gain, with the best and worst one by each measure and the holdings behind them. Use it for best/worst day, week, month, quarter or year, or any " "period-by-period question, instead of fetching one get_performance_report per period. Sold holdings are " "included. Omit start for 'since inception'. History that isn't cached yet is fetched live, which is slow " "for days and weeks: if the tool refuses a range, tell the user why and offer the start it suggests.", @@ -88,11 +88,13 @@ async def periods(args: dict) -> str: lines = [f"Total return per {period} ({rows[0][1].currency}), {rows[0][0]} to {rows[-1][0]}, " "sold holdings included:"] lines += [f"{label} {r.pct:+.2f}% {r.gain:+,.0f}" for label, r in rows] - lines.append(f"Best: {best[0]} {best[1].pct:+.2f}%. Worst: {worst[0]} {worst[1].pct:+.2f}%." + rich, poor = max(rows, key=lambda r: r[1].gain), min(rows, key=lambda r: r[1].gain) + both = lambda row: f"{row[0]} {row[1].pct:+.2f}% ({row[1].gain:+,.0f})" # noqa: E731 + lines.append(f"Best by %: {both(best)}. Worst by %: {both(worst)}. " + f"Best by money: {both(rich)}. Worst by money: {both(poor)}." + (f" {missing} {period}s had no data." if missing else "")) # What moved the extremes (best/worst by percent and by money), so the answer can say why. - extremes = dict.fromkeys(label for label, _ in (best, worst, max(rows, key=lambda r: r[1].gain), - min(rows, key=lambda r: r[1].gain))) + extremes = dict.fromkeys(label for label, _ in (best, worst, rich, poor)) for label, r in rows: if label in extremes and (text := drivers(r.movers)): lines.append(f"Holdings behind {label}: {text}") diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index c48d4a3..fd6ba8a 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -29,6 +29,9 @@ log = logging.getLogger("bot") PORTFOLIO_ADMINS_ONLY = "Portfolio data is for admins only." +FOLLOW_UP_WORDS = 12 # a message this short right after the bot's reply may be a follow-up to it +FOLLOW_UP_SECONDS = 30 * 60 +FOLLOW_UP_CHARS = 400 # of the bot's reply that the tool gate also reads READ_IMAGE_WORDS = re.compile(r"\b(read|text|says?|chart|table|number|screenshot|ocr|zoom|detail|label)\b", re.I) @@ -162,6 +165,19 @@ def _quote(self, trig: Trigger) -> str | None: body = describe(t) return f"[#{t.message_id}] {who}: {compact(body) if self.ctx.st.token_saver else body}" + async def _gate_text(self, msg: Message, trig: Trigger) -> str: + """What the tool gate reads: the message and what it replies to. A short follow-up ("what about in dollar + terms?") sent right after the bot's own reply also gets the start of that reply, so it counts as part of + the same (portfolio, market...) conversation instead of reaching the model with no tools.""" + text = trig.text + " " + (describe(trig.reply_target) if trig.reply_target else "") + if trig.reply_target is None and len(trig.text.split()) <= FOLLOW_UP_WORDS: + rows = await asyncio.to_thread(self.ctx.store.history, msg.chat_id, 2) + if len(rows) >= 2 and rows[-1].message_id == msg.message_id: + prev = rows[-2] + if prev.sender == self.ctx.self_name and int(msg.date.timestamp()) - prev.ts <= FOLLOW_UP_SECONDS: + text += " " + compact(prev.text)[:FOLLOW_UP_CHARS] + return text + async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: bool = False) -> str: ctx, st = self.ctx, self.ctx.st rows = await asyncio.to_thread(ctx.store.history, msg.chat_id, st.history_limit) @@ -222,7 +238,7 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s must_search=True, cache_key=f"chat-{chat_id}", on_text=on_text, kind="movers", chat_id=chat_id) else: - context = trig.text + " " + (describe(trig.reply_target) if trig.reply_target else "") + context = await self._gate_text(msg, trig) chosen = route(context, st, ctx.registry) admin = not st.admin_only or await self._is_admin(msg) if not admin and is_portfolio_question(context, st) and any( diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 4c93c98..1b2c487 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -97,6 +97,22 @@ async def test_market_question_offers_tools_up_front(env, store): assert backend.seen[0]["tools"] == ["yahoo__get_quote"] and backend.seen[0]["search"] is True +async def test_a_short_follow_up_to_a_portfolio_reply_still_gets_the_portfolio_tools(env, store): + sharesight = FakeMcp("sharesight", gate="portfolio", tools=("list_portfolios",)) + ctx, backend, h = make_ctx(env, store, [step("Your portfolio best month was May"), step("In dollars: ..."), step("c"), step("d")], + servers=[sharesight]) + await run(h, user_msg(ctx.bot, "@stockbot how is my portfolio doing?", mid=1)) + await run(h, user_msg(ctx.bot, "@stockbot what about in dollar terms?", mid=2000)) + assert "sharesight__list_portfolios" in backend.seen[1]["tools"] and "period_returns" in backend.seen[1]["tools"] + # too long after the reply: a plain message again + await run(h, user_msg(ctx.bot, "@stockbot and what about the other one?", mid=3000, ts=100_000)) + assert backend.seen[2]["tools"] == [] + # a long unrelated message right after a reply is not read as a follow-up + long = "@stockbot " + " ".join(["tell me a long story about the history of the word mortgage"] * 3) + await run(h, user_msg(ctx.bot, long, mid=4000)) + assert backend.seen[3]["tools"] == [] + + async def test_portfolio_question_reminds_the_model_to_fetch_fresh_figures(env, store): sharesight = FakeMcp("sharesight", gate="portfolio", tools=("list_portfolios",)) ctx, backend, h = make_ctx(env, store, [step("a"), step("b")], servers=[sharesight]) diff --git a/tests/test_periods.py b/tests/test_periods.py index c03c199..47edcbe 100644 --- a/tests/test_periods.py +++ b/tests/test_periods.py @@ -49,7 +49,8 @@ async def test_months_run_from_inception_with_sold_holdings_and_best_worst(env, out = await fn({"portfolio_id": 7, "end": "2026-03"}) assert defn.name == "period_returns" and "Total return per month (AUD)" in out assert "2025-12 -2.00% -200" in out and "2026-02 +5.00% +500" in out - assert "Best: 2026-02 +5.00%. Worst: 2025-12 -2.00%. 1 months had no data." in out # March: no data + assert "Best by %: 2026-02 +5.00% (+500). Worst by %: 2025-12 -2.00% (-200)." in out + assert "Best by money: 2026-02 +5.00% (+500). Worst by money: 2025-12 -2.00% (-200). 1 months had no data." in out # March: no data first = reports(ss)[0] assert (first["start_date"], first["end_date"]) == ("2025-12-20", "2025-12-31") # clipped to inception assert all(r["_keep_sales"] is True for r in reports(ss)) @@ -161,3 +162,17 @@ def test_flush_scope_understands_portfolio_year_month_day_and_all(): assert f(["2024-05-17"]) == (None, date(2024, 5, 17), date(2024, 5, 17)) assert f(["nope"]) is None and f(["684141", "702349"]) is None and f(["2024", "2025"]) is None assert f(["1234"]) is None # 4 digits that aren't a year in range + + +async def test_best_by_money_can_differ_from_best_by_percent(env, store): + class Mixed(Sharesight): + async def call(self, tool, args): + self.calls.append((tool, args)) + pct, gain = {"2026-01": (10.0, 100.0), "2026-02": (2.0, 900.0)}[args["start_date"][:7]] + return json.dumps({"report": {"total_gain_percent": pct, "total_gain": gain, "currency": "AUD"}}) + + ctx, _ = make(env, store, Mixed("sharesight", tools=("list_portfolios", "get_performance_report"))) + _, fn = periods.tool(ctx) + out = await fn({"portfolio_id": 7, "start": "2026-01", "end": "2026-02"}) + assert "Best by %: 2026-01 +10.00% (+100)" in out and "Best by money: 2026-02 +2.00% (+900)" in out + assert "Worst by money: 2026-01 +10.00% (+100)" in out From 4cd3cfb324f2b6bdfe663861edad87fdf850064c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 09:45:44 +0000 Subject: [PATCH 20/34] Period backfill: windows with no data no longer end the pass; only throttling/outages count; log why fetches fail Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/features/periods/backfill.py | 27 ++++++++++++++-------- lib/features/periods/rows.py | 39 ++++++++++++++++++++++++-------- tests/test_backfill.py | 27 +++++++++++++++++++++- 3 files changed, 73 insertions(+), 20 deletions(-) diff --git a/lib/features/periods/backfill.py b/lib/features/periods/backfill.py index 1f3a570..afd7d64 100644 --- a/lib/features/periods/backfill.py +++ b/lib/features/periods/backfill.py @@ -11,7 +11,7 @@ from ... import config from ...context import Ctx from . import sharesight -from .rows import fetch_row +from .rows import fetch from .windows import Window, key, parse_date, period_windows log = logging.getLogger("bot") @@ -74,13 +74,14 @@ def selected(st: config.Settings, listing: str) -> list[tuple[int, str, date]]: async def fill_portfolio(ctx: Ctx, server, pid: int, name: str, todo: list[Window]) -> tuple[int, bool]: """Fetch these windows with WORKERS fetches at a time. Returns (how many were stored, whether the pass - was stopped by MAX_FAILS failures in a row).""" + was stopped by MAX_FAILS transient failures in a row). A window Sharesight has no data for is skipped + (until the next restart or flush) without counting as a failure: early days before any trade are normal.""" pending = iter(todo) # shared: each window goes to exactly one worker - stored = done = fails = mine = 0 + stored = done = fails = nodata = transient = mine = 0 stopped = False async def worker() -> None: - nonlocal stored, done, fails, mine, stopped + nonlocal stored, done, fails, nodata, transient, mine, stopped for w in pending: while server.in_flight > mine and not stopped: # a user's request comes first await asyncio.sleep(BUSY_POLL) @@ -88,24 +89,30 @@ async def worker() -> None: return mine += 1 try: - row = await fetch_row(server, pid, w[1], w[2]) + got = await fetch(server, pid, w[1], w[2]) finally: mine -= 1 - if row is None: + if got.row is not None: + fails = 0 + await asyncio.to_thread(ctx.store.period_put, [got.row]) + stored += 1 + elif got.kind == "nodata": + BAD.add((pid, *key(w))) + nodata += 1 + else: BAD.add((pid, *key(w))) fails += 1 + transient += 1 + log.warning("Period backfill: %s %s..%s failed: %s", name, w[1], w[2], got.text[:150]) if fails >= MAX_FAILS and not stopped: stopped = True log.warning("Period backfill: %d fetches in a row failed; stopping this pass", fails) - else: - fails = 0 - await asyncio.to_thread(ctx.store.period_put, [row]) - stored += 1 done += 1 if done % PROGRESS_EVERY == 0: log.info("Period backfill: %s %d/%d", name, done, len(todo)) await asyncio.gather(*(worker() for _ in range(WORKERS))) + log.info("Period backfill: %s: %d stored, %d had no data, %d failed", name, stored, nodata, transient) return stored, stopped diff --git a/lib/features/periods/rows.py b/lib/features/periods/rows.py index fe709de..91fd64d 100644 --- a/lib/features/periods/rows.py +++ b/lib/features/periods/rows.py @@ -3,7 +3,9 @@ from __future__ import annotations import json +import re from datetime import date +from typing import NamedTuple from ...mcp.results import table_records from ...store import PeriodRow @@ -37,13 +39,32 @@ def show(m): return "; ".join(f"{name}: {', '.join(map(show, group))}" for name, group in (("up", ups), ("down", downs)) if group) -async def fetch_row(server, portfolio_id: int, first: date, last: date) -> PeriodRow | None: - """One window's return from Sharesight (sold holdings included), or None if no report came back.""" - text = await server.call("get_performance_report", { - "portfolio_id": portfolio_id, "start_date": first.isoformat(), "end_date": last.isoformat(), - "_keep_sales": True}) +THROTTLE = re.compile(r"429|rate.?limit|too many|throttl|timed? ?out|unavailable|temporar|overload|\b5\d\d\b", re.I) + + +class Fetched(NamedTuple): + row: PeriodRow | None + kind: str # "ok", "nodata" (Sharesight has nothing for that window) or "transient" (outage, timeout, throttling) + text: str # what came back, for the log + + +async def fetch(server, portfolio_id: int, first: date, last: date) -> Fetched: + """One window's return from Sharesight (sold holdings included).""" + try: + text = await server.call("get_performance_report", { + "portfolio_id": portfolio_id, "start_date": first.isoformat(), "end_date": last.isoformat(), + "_keep_sales": True}) + except Exception as e: # a timeout or a dropped connection + return Fetched(None, "transient", f"{type(e).__name__}: {e}") report = parse_report(text) - if report is None: - return None - return PeriodRow(portfolio_id, first.isoformat(), last.isoformat(), float(report["total_gain_percent"]), - float(report.get("total_gain") or 0), report.get("currency") or "", movers(report)) + if report is not None: + row = PeriodRow(portfolio_id, first.isoformat(), last.isoformat(), float(report["total_gain_percent"]), + float(report.get("total_gain") or 0), report.get("currency") or "", movers(report)) + return Fetched(row, "ok", "") + transient = text.startswith("Error:") or bool(THROTTLE.search(text[:300])) + return Fetched(None, "transient" if transient else "nodata", text) + + +async def fetch_row(server, portfolio_id: int, first: date, last: date) -> PeriodRow | None: + """fetch(), keeping only the row (None if no report came back).""" + return (await fetch(server, portfolio_id, first, last)).row diff --git a/tests/test_backfill.py b/tests/test_backfill.py index 1573f8c..969bb24 100644 --- a/tests/test_backfill.py +++ b/tests/test_backfill.py @@ -17,6 +17,8 @@ class Sharesight(FakeMcp): def __init__(self, *a, fail=False, inception="2026-01-15", **kw): super().__init__(*a, **kw) self.fail = fail + self.fail_text = "Error: timed out" + self.data_from = "" self.inception = inception self.busy_for = 0 # polls during which the server counts as busy self.polls = 0 @@ -44,7 +46,9 @@ async def call(self, tool, args): await asyncio.sleep(self.delay) self.running -= 1 if self.fail: - return "Error: timed out" + return self.fail_text + if args["end_date"] < self.data_from: # nothing in Sharesight before this date + return "Tool error: no data for this period" return json.dumps({"report": {"total_gain_percent": 1.0, "total_gain": 10.0, "currency": "AUD"}}) @@ -128,6 +132,27 @@ async def test_failures_end_the_pass_and_failed_windows_are_not_retried_until_re assert not backfill.BAD and backfill.wake_event().is_set() +async def test_windows_with_no_data_do_not_stop_the_pass_and_are_not_retried(env, store, caplog): + caplog.set_level("INFO", logger="bot") + ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) + ss.data_from = "2026-02-15" # everything ending before this has no data: dozens of empty windows in a row + ctx, _ = make(env, store, ss) + args = ctx, ss, [(7, "RobSMSF", date(2026, 1, 15))], date(2026, 3, 3) + stored = await backfill.fill_once(*args) + assert stored > 0 and store.period_count(7, "2026-01-01", "2026-03-31") == stored # later windows still got stored + assert "had no data" in caplog.text and "fetches in a row failed" not in caplog.text + ss.order.clear() + assert await backfill.fill_once(*args) == 0 and ss.order == [] # the empty ones are skipped until reset() + + +async def test_throttling_text_counts_as_a_transient_failure_and_is_logged(env, store, caplog): + ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report"), fail=True) + ss.fail_text = "Tool error: HTTP 429 Too Many Requests" + ctx, _ = make(env, store, ss) + assert await backfill.fill_once(ctx, ss, [(7, "RobSMSF", date(2026, 1, 15))], date(2026, 3, 3)) == 0 + assert "fetches in a row failed" in caplog.text and "429 Too Many Requests" in caplog.text + + async def test_the_loop_runs_a_pass_then_sleeps_until_woken(env, store, monkeypatch): recent = (date.today() - timedelta(days=4)).isoformat() ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report"), inception=recent) From 24d45b57f4814adb8f96fc4e196b54ccecd8b73b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 09:49:21 +0000 Subject: [PATCH 21/34] Period backfill: retry in ten minutes after a pass stopped by failures; blips don't mark windows as skipped Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- README.md | 3 ++- lib/features/periods/backfill.py | 26 +++++++++++++++++--------- tests/test_backfill.py | 21 ++++++++++++++++++--- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index def73c5..ec28d31 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,8 @@ Without the cache, a request may fetch at most 64 days, 26 weeks or 120 months/q longer range is refused with the earliest start that fits. Set `PERIOD_BACKFILL=[,...]` (or `all`) to fill the cache in the background: months first, then quarters, years, weeks and days, up to two Sharesight calls at a time, never while a user's request is using the server (about 1,650 calls per portfolio, an hour or two the first time; progress is logged and it resumes after a restart). Once filled, any range is instant and the -limits above no longer matter. Every six hours it fetches whatever newly finished. +limits above no longer matter. Every six hours it fetches whatever newly finished. If five fetches in a row fail (Sharesight down or throttling), +the pass stops and tries again after ten minutes; windows Sharesight simply has no data for are skipped until a restart. ## Caching and token saving **Provider prompt caching** (xAI and OpenRouter keep a conversation on the server holding its cached diff --git a/lib/features/periods/backfill.py b/lib/features/periods/backfill.py index afd7d64..fc5392c 100644 --- a/lib/features/periods/backfill.py +++ b/lib/features/periods/backfill.py @@ -19,12 +19,13 @@ ORDER = ("month", "quarter", "year", "week", "day") # what pays off first: few calls, most questions BUSY_POLL = 2.0 RECHECK = 6 * 3600 # seconds between passes (a pass fetches only what newly finished) +RETRY_AFTER_FAILURE = 600 # seconds before trying again after a pass stopped by MAX_FAILS failures in a row WORKERS = 2 # fetches at a time (needs max_concurrent >= 2 on the Sharesight server in mcp_servers.json) MAX_FAILS = 5 # consecutive failed fetches that end a pass (Sharesight down or throttling) PROGRESS_EVERY = 100 _wake: asyncio.Event | None = None # set to start a pass now (after a flush) -BAD: set[tuple[int, str, str]] = set() # windows Sharesight had nothing for; not retried until reset() +BAD: set[tuple[int, str, str]] = set() # windows Sharesight had no data for; not retried until restart or reset() def wake_event() -> asyncio.Event: @@ -99,8 +100,7 @@ async def worker() -> None: elif got.kind == "nodata": BAD.add((pid, *key(w))) nodata += 1 - else: - BAD.add((pid, *key(w))) + else: # a blip says nothing about the window itself, so it is tried again next pass fails += 1 transient += 1 log.warning("Period backfill: %s %s..%s failed: %s", name, w[1], w[2], got.text[:150]) @@ -116,8 +116,9 @@ async def worker() -> None: return stored, stopped -async def fill_once(ctx: Ctx, server, portfolios: list[tuple[int, str, date]], today: date) -> int: - """One pass: fetch every wanted window that isn't cached yet. Returns how many were stored.""" +async def fill_pass(ctx: Ctx, server, portfolios: list[tuple[int, str, date]], today: date) -> tuple[int, bool]: + """One pass: fetch every wanted window that isn't cached yet. Returns (how many were stored, whether + failures in a row cut it short).""" stored = 0 for pid, name, first in portfolios: windows = wanted(first, today) @@ -129,8 +130,12 @@ async def fill_once(ctx: Ctx, server, portfolios: list[tuple[int, str, date]], t n, stopped = await fill_portfolio(ctx, server, pid, name, todo) stored += n if stopped: - break - return stored + return stored, True + return stored, False + + +async def fill_once(ctx: Ctx, server, portfolios: list[tuple[int, str, date]], today: date) -> int: + return (await fill_pass(ctx, server, portfolios, today))[0] async def loop(ctx: Ctx) -> None: @@ -146,8 +151,11 @@ async def loop(ctx: Ctx) -> None: portfolios = selected(ctx.st, await server.call("list_portfolios", {})) if not portfolios: log.warning("Period backfill: no Sharesight portfolio matches PERIOD_BACKFILL") - n = await fill_once(ctx, server, portfolios, datetime.now(ctx.st.tz).date()) - log.info("Period backfill: pass done, %d new windows cached", n) + n, stopped = await fill_pass(ctx, server, portfolios, datetime.now(ctx.st.tz).date()) + log.info("Period backfill: pass %s, %d new windows cached%s", "stopped" if stopped else "done", n, + f"; trying again in {RETRY_AFTER_FAILURE // 60} minutes" if stopped else "") + if stopped: + wait = RETRY_AFTER_FAILURE except Exception: log.exception("Period backfill pass failed") try: diff --git a/tests/test_backfill.py b/tests/test_backfill.py index 969bb24..2352abd 100644 --- a/tests/test_backfill.py +++ b/tests/test_backfill.py @@ -118,20 +118,35 @@ async def test_two_fetches_run_at_a_time_and_each_window_is_fetched_once(env, st assert store.period_count(7, "2026-01-01", "2026-03-31") == len(want) -async def test_failures_end_the_pass_and_failed_windows_are_not_retried_until_reset(env, store): +async def test_failures_end_the_pass_and_the_same_windows_are_tried_again_next_time(env, store): ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report"), fail=True) ctx, _ = make(env, store, ss) args = ctx, ss, [(7, "RobSMSF", date(2026, 1, 15))], date(2026, 3, 3) assert await backfill.fill_once(*args) == 0 assert backfill.MAX_FAILS <= len(ss.order) <= backfill.MAX_FAILS + backfill.WORKERS - 1 # gave up after five - first_five = list(ss.order) + assert not backfill.BAD # a blip doesn't mark the windows + first_pass = list(ss.order) ss.order.clear() await backfill.fill_once(*args) - assert len(ss.order) >= backfill.MAX_FAILS and not set(ss.order) & set(first_five) # skipped the failed ones + assert ss.order[:backfill.MAX_FAILS] == first_pass[:backfill.MAX_FAILS] # same windows again backfill.reset() assert not backfill.BAD and backfill.wake_event().is_set() +async def test_a_stopped_pass_is_retried_after_ten_minutes_not_six_hours(env, store, monkeypatch): + ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report"), fail=True) + ctx, _ = make(env, store, ss) + monkeypatch.setattr(backfill, "RECHECK", 3600) + monkeypatch.setattr(backfill, "RETRY_AFTER_FAILURE", 0.05) + task = asyncio.create_task(backfill.loop(ctx)) + for _ in range(200): + await asyncio.sleep(0.02) + if len(ss.order) >= 3 * backfill.MAX_FAILS: # several passes, each stopped by failures + break + task.cancel() + assert len(ss.order) >= 3 * backfill.MAX_FAILS + + async def test_windows_with_no_data_do_not_stop_the_pass_and_are_not_retried(env, store, caplog): caplog.set_level("INFO", logger="bot") ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) From 6e6ad5c4c01af74aa6cbce0dbc61e079f86b5adc Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 10:04:54 +0000 Subject: [PATCH 22/34] Period backfill: every portfolio's months/quarters/years/weeks before anyone's days Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- README.md | 2 +- lib/features/periods/backfill.py | 54 ++++++++++++++++++-------------- tests/test_backfill.py | 24 ++++++++++++++ 3 files changed, 55 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index ec28d31..f89fb58 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,7 @@ that contains it), and the background fill re-fetches just those. Without the cache, a request may fetch at most 64 days, 26 weeks or 120 months/quarters/years that aren't cached; a longer range is refused with the earliest start that fits. Set `PERIOD_BACKFILL=[,...]` (or `all`) to -fill the cache in the background: months first, then quarters, years, weeks and days, up to two Sharesight calls at a time, +fill the cache in the background: months first, then quarters, years, weeks and days (each type for every portfolio before the next), up to two Sharesight calls at a time, never while a user's request is using the server (about 1,650 calls per portfolio, an hour or two the first time; progress is logged and it resumes after a restart). Once filled, any range is instant and the limits above no longer matter. Every six hours it fetches whatever newly finished. If five fetches in a row fail (Sharesight down or throttling), the pass stops and tries again after ten minutes; windows Sharesight simply has no data for are skipped until a restart. diff --git a/lib/features/periods/backfill.py b/lib/features/periods/backfill.py index fc5392c..7224042 100644 --- a/lib/features/periods/backfill.py +++ b/lib/features/periods/backfill.py @@ -49,10 +49,10 @@ def natural_end(period: str, w: Window) -> bool: "year": end.month == 12 and end.day == 31}[period] -def wanted(first: date, today: date) -> list[Window]: - """Finished windows from `first` up to yesterday, in the order worth fetching them.""" +def wanted(first: date, today: date, periods: tuple[str, ...] = ORDER) -> list[Window]: + """Finished windows of these period types from `first` up to yesterday, in the order worth fetching them.""" yesterday = today - timedelta(days=1) - return [w for p in ORDER for w in period_windows(p, first, yesterday) if natural_end(p, w)] \ + return [w for p in periods for w in period_windows(p, first, yesterday) if natural_end(p, w)] \ if first <= yesterday else [] @@ -73,16 +73,18 @@ def selected(st: config.Settings, listing: str) -> list[tuple[int, str, date]]: return out -async def fill_portfolio(ctx: Ctx, server, pid: int, name: str, todo: list[Window]) -> tuple[int, bool]: +async def fill_portfolio(ctx: Ctx, server, pid: int, name: str, todo: list[Window], + streak: list[int] | None = None) -> tuple[int, bool]: """Fetch these windows with WORKERS fetches at a time. Returns (how many were stored, whether the pass was stopped by MAX_FAILS transient failures in a row). A window Sharesight has no data for is skipped (until the next restart or flush) without counting as a failure: early days before any trade are normal.""" pending = iter(todo) # shared: each window goes to exactly one worker - stored = done = fails = nodata = transient = mine = 0 + streak = streak if streak is not None else [0] # failures in a row; shared across a pass's steps + stored = done = nodata = transient = mine = 0 stopped = False async def worker() -> None: - nonlocal stored, done, fails, nodata, transient, mine, stopped + nonlocal stored, done, nodata, transient, mine, stopped for w in pending: while server.in_flight > mine and not stopped: # a user's request comes first await asyncio.sleep(BUSY_POLL) @@ -94,19 +96,19 @@ async def worker() -> None: finally: mine -= 1 if got.row is not None: - fails = 0 + streak[0] = 0 await asyncio.to_thread(ctx.store.period_put, [got.row]) stored += 1 elif got.kind == "nodata": BAD.add((pid, *key(w))) nodata += 1 else: # a blip says nothing about the window itself, so it is tried again next pass - fails += 1 + streak[0] += 1 transient += 1 log.warning("Period backfill: %s %s..%s failed: %s", name, w[1], w[2], got.text[:150]) - if fails >= MAX_FAILS and not stopped: + if streak[0] >= MAX_FAILS and not stopped: stopped = True - log.warning("Period backfill: %d fetches in a row failed; stopping this pass", fails) + log.warning("Period backfill: %d fetches in a row failed; stopping this pass", streak[0]) done += 1 if done % PROGRESS_EVERY == 0: log.info("Period backfill: %s %d/%d", name, done, len(todo)) @@ -117,20 +119,24 @@ async def worker() -> None: async def fill_pass(ctx: Ctx, server, portfolios: list[tuple[int, str, date]], today: date) -> tuple[int, bool]: - """One pass: fetch every wanted window that isn't cached yet. Returns (how many were stored, whether - failures in a row cut it short).""" - stored = 0 - for pid, name, first in portfolios: - windows = wanted(first, today) - have = await asyncio.to_thread(ctx.store.period_get, pid, first.isoformat(), today.isoformat()) - todo = [w for w in windows if key(w) not in have and (pid, *key(w)) not in BAD] - if not todo: - continue - log.info("Period backfill: %d windows to fetch for %s", len(todo), name) - n, stopped = await fill_portfolio(ctx, server, pid, name, todo) - stored += n - if stopped: - return stored, True + """One pass: fetch every wanted window that isn't cached yet, one period type at a time across all the + portfolios (everyone's months, then quarters, years, weeks, and the many days last). Returns (how many + were stored, whether failures in a row cut it short).""" + cached = {} + for pid, _, first in portfolios: + cached[pid] = set(await asyncio.to_thread(ctx.store.period_get, pid, first.isoformat(), today.isoformat())) + stored, streak = 0, [0] + for period in ORDER: + for pid, name, first in portfolios: + todo = [w for w in wanted(first, today, (period,)) + if key(w) not in cached[pid] and (pid, *key(w)) not in BAD] + if not todo: + continue + log.info("Period backfill: %d %ss to fetch for %s", len(todo), period, name) + n, stopped = await fill_portfolio(ctx, server, pid, f"{name} {period}s", todo, streak) + stored += n + if stopped: + return stored, True return stored, False diff --git a/tests/test_backfill.py b/tests/test_backfill.py index 2352abd..1b9ab77 100644 --- a/tests/test_backfill.py +++ b/tests/test_backfill.py @@ -99,6 +99,30 @@ async def test_a_pass_fetches_what_is_missing_then_only_what_is_new(env, store): assert ss.order == [("2026-03-03", "2026-03-03")] +async def test_every_portfolios_cheap_periods_come_before_anyones_days(env, store): + ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) + ctx, _ = make(env, store, ss) + seen = [] + real = ss.call + + async def spy(tool, args): + if tool == "get_performance_report": + seen.append((args["portfolio_id"], args["start_date"], args["end_date"])) + return await real(tool, args) + + ss.call = spy + folios = [(7, "RobSMSF", date(2026, 1, 15)), (8, "Other", date(2026, 1, 15))] + today = date(2026, 3, 3) + total = await backfill.fill_once(ctx, ss, folios, today) + assert total == 2 * len(backfill.wanted(date(2026, 1, 15), today)) and len(set(seen)) == len(seen) # once each + is_day = lambda x: x[1] == x[2] # noqa: E731 + last_cheap = max(i for i, x in enumerate(seen) if not is_day(x)) + first_day = min(i for i, x in enumerate(seen) if is_day(x)) + assert last_cheap < first_day # all weeks/months of both portfolios, then days + assert {x[0] for x in seen[:last_cheap + 1]} == {7, 8} + assert await backfill.fill_once(ctx, ss, folios, today) == 0 + + async def test_it_waits_while_a_user_request_is_using_the_server(env, store): ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) ss.busy_for = 3 From 5006c0e89e9bd202441c3d8e06f2c95921d0b083 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 11:39:19 +0000 Subject: [PATCH 23/34] Period backfill: progress every 50 fetches; log when held back by a user's request or a fetch is slow Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/features/periods/backfill.py | 12 +++++++++++- tests/test_backfill.py | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/lib/features/periods/backfill.py b/lib/features/periods/backfill.py index 7224042..8cedc38 100644 --- a/lib/features/periods/backfill.py +++ b/lib/features/periods/backfill.py @@ -6,6 +6,7 @@ import asyncio import json import logging +import time from datetime import date, datetime, timedelta from ... import config @@ -22,7 +23,9 @@ RETRY_AFTER_FAILURE = 600 # seconds before trying again after a pass stopped by MAX_FAILS failures in a row WORKERS = 2 # fetches at a time (needs max_concurrent >= 2 on the Sharesight server in mcp_servers.json) MAX_FAILS = 5 # consecutive failed fetches that end a pass (Sharesight down or throttling) -PROGRESS_EVERY = 100 +PROGRESS_EVERY = 50 +WAIT_LOG = 30.0 # seconds held back by a user's request before the journal says so +SLOW_FETCH = 20.0 # seconds one Sharesight fetch may take before the journal says so _wake: asyncio.Event | None = None # set to start a pass now (after a flush) BAD: set[tuple[int, str, str]] = set() # windows Sharesight had no data for; not retried until restart or reset() @@ -86,15 +89,22 @@ async def fill_portfolio(ctx: Ctx, server, pid: int, name: str, todo: list[Windo async def worker() -> None: nonlocal stored, done, nodata, transient, mine, stopped for w in pending: + waited, said = time.monotonic(), False while server.in_flight > mine and not stopped: # a user's request comes first await asyncio.sleep(BUSY_POLL) + if not said and time.monotonic() - waited >= WAIT_LOG: + said = True + log.info("Period backfill: %s held back by a user's request for %.0f s", name, WAIT_LOG) if stopped: return mine += 1 + began = time.monotonic() try: got = await fetch(server, pid, w[1], w[2]) finally: mine -= 1 + if time.monotonic() - began >= SLOW_FETCH: + log.warning("Period backfill: %s %s..%s was slow: %.0f s", name, w[1], w[2], time.monotonic() - began) if got.row is not None: streak[0] = 0 await asyncio.to_thread(ctx.store.period_put, [got.row]) diff --git a/tests/test_backfill.py b/tests/test_backfill.py index 1b9ab77..3a8ad07 100644 --- a/tests/test_backfill.py +++ b/tests/test_backfill.py @@ -192,6 +192,27 @@ async def test_throttling_text_counts_as_a_transient_failure_and_is_logged(env, assert "fetches in a row failed" in caplog.text and "429 Too Many Requests" in caplog.text +async def test_the_journal_says_when_it_is_held_back_or_a_fetch_is_slow(env, store, monkeypatch, caplog): + caplog.set_level("INFO", logger="bot") + monkeypatch.setattr(backfill, "WAIT_LOG", 0) + monkeypatch.setattr(backfill, "SLOW_FETCH", 0.005) + ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) + ss.busy_for = 3 + ss.delay = 0.01 + ctx, _ = make(env, store, ss) + await backfill.fill_once(ctx, ss, [(7, "RobSMSF", date(2026, 3, 1))], date(2026, 3, 3)) + assert "held back by a user's request" in caplog.text and "was slow" in caplog.text + + +async def test_progress_is_logged_every_n_fetches(env, store, monkeypatch, caplog): + caplog.set_level("INFO", logger="bot") + monkeypatch.setattr(backfill, "PROGRESS_EVERY", 5) + ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) + ctx, _ = make(env, store, ss) + await backfill.fill_once(ctx, ss, [(7, "RobSMSF", date(2026, 1, 15))], date(2026, 3, 3)) + assert "RobSMSF days 5/" in caplog.text and "RobSMSF days 10/" in caplog.text + + async def test_the_loop_runs_a_pass_then_sleeps_until_woken(env, store, monkeypatch): recent = (date.today() - timedelta(days=4)).isoformat() ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report"), inception=recent) From 67adfece900629ad088f813505c03a3078eb50a7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 11:43:39 +0000 Subject: [PATCH 24/34] Period backfill: progress every 50; log when held back, when a fetch is slow, and a few Sharesight no-data messages Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/features/periods/backfill.py | 3 +++ tests/test_backfill.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/lib/features/periods/backfill.py b/lib/features/periods/backfill.py index 8cedc38..4732c3c 100644 --- a/lib/features/periods/backfill.py +++ b/lib/features/periods/backfill.py @@ -25,6 +25,7 @@ MAX_FAILS = 5 # consecutive failed fetches that end a pass (Sharesight down or throttling) PROGRESS_EVERY = 50 WAIT_LOG = 30.0 # seconds held back by a user's request before the journal says so +SAMPLE_NODATA = 3 # "no data" windows logged individually per step SLOW_FETCH = 20.0 # seconds one Sharesight fetch may take before the journal says so _wake: asyncio.Event | None = None # set to start a pass now (after a flush) @@ -112,6 +113,8 @@ async def worker() -> None: elif got.kind == "nodata": BAD.add((pid, *key(w))) nodata += 1 + if nodata <= SAMPLE_NODATA: # enough of them to see what Sharesight says, not one line per empty day + log.info("Period backfill: %s %s..%s has no data: %s", name, w[1], w[2], got.text[:150]) else: # a blip says nothing about the window itself, so it is tried again next pass streak[0] += 1 transient += 1 diff --git a/tests/test_backfill.py b/tests/test_backfill.py index 3a8ad07..403abb5 100644 --- a/tests/test_backfill.py +++ b/tests/test_backfill.py @@ -180,6 +180,8 @@ async def test_windows_with_no_data_do_not_stop_the_pass_and_are_not_retried(env stored = await backfill.fill_once(*args) assert stored > 0 and store.period_count(7, "2026-01-01", "2026-03-31") == stored # later windows still got stored assert "had no data" in caplog.text and "fetches in a row failed" not in caplog.text + shown = [r.message for r in caplog.records if "has no data:" in r.message] + assert 0 < len(shown) <= 3 * 5 and "no data for this period" in shown[0] # a few samples per step, with Sharesight's words ss.order.clear() assert await backfill.fill_once(*args) == 0 and ss.order == [] # the empty ones are skipped until reset() From 0a72f05c9f89d80038df752a546184ffd88c0612 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 12:49:54 +0000 Subject: [PATCH 25/34] Period backfill: say which user calls hold it back; background fetches are tagged so only real user calls count Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/features/periods/backfill.py | 14 +++++--------- lib/features/periods/rows.py | 7 ++++--- lib/llm/runner.py | 1 + lib/mcp/server.py | 30 ++++++++++++++++++++++++++---- tests/test_backfill.py | 11 +++-------- tests/test_mcp.py | 22 ++++++++++++++++++++++ tests/test_runner.py | 2 +- 7 files changed, 62 insertions(+), 25 deletions(-) diff --git a/lib/features/periods/backfill.py b/lib/features/periods/backfill.py index 4732c3c..1e7ed55 100644 --- a/lib/features/periods/backfill.py +++ b/lib/features/periods/backfill.py @@ -84,26 +84,22 @@ async def fill_portfolio(ctx: Ctx, server, pid: int, name: str, todo: list[Windo (until the next restart or flush) without counting as a failure: early days before any trade are normal.""" pending = iter(todo) # shared: each window goes to exactly one worker streak = streak if streak is not None else [0] # failures in a row; shared across a pass's steps - stored = done = nodata = transient = mine = 0 + stored = done = nodata = transient = 0 stopped = False async def worker() -> None: - nonlocal stored, done, nodata, transient, mine, stopped + nonlocal stored, done, nodata, transient, stopped for w in pending: waited, said = time.monotonic(), False - while server.in_flight > mine and not stopped: # a user's request comes first + while (users := server.user_calls()) and not stopped: # a user's request comes first await asyncio.sleep(BUSY_POLL) if not said and time.monotonic() - waited >= WAIT_LOG: said = True - log.info("Period backfill: %s held back by a user's request for %.0f s", name, WAIT_LOG) + log.info("Period backfill: %s held back for %.0f s by: %s", name, WAIT_LOG, "; ".join(users[:3])) if stopped: return - mine += 1 began = time.monotonic() - try: - got = await fetch(server, pid, w[1], w[2]) - finally: - mine -= 1 + got = await fetch(server, pid, w[1], w[2], background=True) if time.monotonic() - began >= SLOW_FETCH: log.warning("Period backfill: %s %s..%s was slow: %.0f s", name, w[1], w[2], time.monotonic() - began) if got.row is not None: diff --git a/lib/features/periods/rows.py b/lib/features/periods/rows.py index 91fd64d..3edea76 100644 --- a/lib/features/periods/rows.py +++ b/lib/features/periods/rows.py @@ -48,12 +48,13 @@ class Fetched(NamedTuple): text: str # what came back, for the log -async def fetch(server, portfolio_id: int, first: date, last: date) -> Fetched: - """One window's return from Sharesight (sold holdings included).""" +async def fetch(server, portfolio_id: int, first: date, last: date, background: bool = False) -> Fetched: + """One window's return from Sharesight (sold holdings included). `background`: the backfill, which yields to + users' calls.""" try: text = await server.call("get_performance_report", { "portfolio_id": portfolio_id, "start_date": first.isoformat(), "end_date": last.isoformat(), - "_keep_sales": True}) + "_keep_sales": True, **({"_bg": True} if background else {})}) except Exception as e: # a timeout or a dropped connection return Fetched(None, "transient", f"{type(e).__name__}: {e}") report = parse_report(text) diff --git a/lib/llm/runner.py b/lib/llm/runner.py index 450e169..cfefe87 100644 --- a/lib/llm/runner.py +++ b/lib/llm/runner.py @@ -72,6 +72,7 @@ async def _run_mcp(call: Call, registry: Registry) -> str: if not isinstance(args, dict): return f"Error: arguments for {tool} must be a JSON object." args.pop("_keep_sales", None) # reserved for the bot's own tools + args.pop("_bg", None) log.info("MCP %s.%s %s", server.label, tool, brief_args(args)) out = await server.call(tool, args) # A tiny result is usually an empty list or an error: show it, so "why did it say diff --git a/lib/mcp/server.py b/lib/mcp/server.py index ec46101..871cc9b 100644 --- a/lib/mcp/server.py +++ b/lib/mcp/server.py @@ -4,6 +4,7 @@ import asyncio import difflib +import itertools import json import logging import re @@ -54,7 +55,8 @@ def __init__(self, label: str, cfg: dict, *, timeout: float, max_output: int, self.fn_names: dict[str, str] = {} # function name sent to the model -> MCP tool name # Cap simultaneous calls: a 20-stock request fired all at once gets rate-limited. self._sem = asyncio.Semaphore(int(cfg.get("max_concurrent", 4))) - self.in_flight = 0 # calls running or waiting for a slot (background work yields to these) + self._active: dict[int, tuple[str, bool]] = {} # calls running or waiting for a slot: text, background? + self._call_ids = itertools.count() self._ready = asyncio.Event() self._stop = asyncio.Event() self._task: asyncio.Task | None = None @@ -232,20 +234,32 @@ async def _warm_once(self) -> None: else: fresh.used = False + @property + def in_flight(self) -> int: + """Calls running or waiting for a slot.""" + return len(self._active) + + def user_calls(self) -> list[str]: + """What the calls that are not background work (a chat request, a period_returns question) are doing. + Background work (the period backfill) waits while there are any.""" + return [text for text, background in self._active.values() if not background] + async def _call(self, tool: str, args: dict) -> str: holdings_only = bool(self.cfg.get("current_holdings_only")) keep_sales = args.get("_keep_sales") # set by the bot's own tools (history), never by the model - args = {k: v for k, v in args.items() if k != "_keep_sales"} + background = bool(args.get("_bg")) # the period backfill, which must not hold up a user's request + args = {k: v for k, v in args.items() if k not in ("_keep_sales", "_bg")} if holdings_only and tool == "get_performance_report": args = {**args, "include_sales": bool(keep_sales)} # Sharesight's own switch for sold holdings if keep_sales: holdings_only = False # closed holdings stay in the report too - self.in_flight += 1 + token = next(self._call_ids) + self._active[token] = (describe_call(tool, args, bool(keep_sales)), background) try: async with self._sem: result = await asyncio.wait_for(self.session.call_tool(tool, args), self.timeout) finally: - self.in_flight -= 1 + del self._active[token] parts = [c.text if c.type == "text" else f"[{c.type} content omitted]" for c in result.content] if not parts and getattr(result, "structuredContent", None): parts.append(json.dumps(result.structuredContent)) @@ -260,6 +274,14 @@ async def _call(self, tool: str, args: dict) -> str: return out +def describe_call(tool: str, args: dict, bot_own: bool = False) -> str: + """A call in a few words, for the log: "get_performance_report portfolio_id=7 2024-01-01..2024-01-31".""" + shown = [f"{k}={v}" for k, v in args.items() if k in ("portfolio_id", "symbol", "ticker", "query")] + if args.get("start_date") or args.get("end_date"): + shown.append(f"{args.get('start_date', '')}..{args.get('end_date', '')}") + return " ".join([tool, *shown]) + (" (period_returns)" if bot_own else "") + + class Registry: """All configured MCP servers.""" diff --git a/tests/test_backfill.py b/tests/test_backfill.py index 403abb5..355d529 100644 --- a/tests/test_backfill.py +++ b/tests/test_backfill.py @@ -26,14 +26,9 @@ def __init__(self, *a, fail=False, inception="2026-01-15", **kw): self.running = self.peak = 0 self.order: list[tuple[str, str]] = [] - @property - def in_flight(self): # a user's request is running for the first few polls + def user_calls(self): # a user's request is running for the first few polls self.polls += 1 - return 1 if self.polls <= self.busy_for else 0 - - @in_flight.setter - def in_flight(self, value): - pass + return ["get_performance_report portfolio_id=7 2024-01-01..2024-12-31"] if self.polls <= self.busy_for else [] async def call(self, tool, args): self.calls.append((tool, args)) @@ -203,7 +198,7 @@ async def test_the_journal_says_when_it_is_held_back_or_a_fetch_is_slow(env, sto ss.delay = 0.01 ctx, _ = make(env, store, ss) await backfill.fill_once(ctx, ss, [(7, "RobSMSF", date(2026, 3, 1))], date(2026, 3, 3)) - assert "held back by a user's request" in caplog.text and "was slow" in caplog.text + assert "held back for 0 s by: get_performance_report portfolio_id=7" in caplog.text and "was slow" in caplog.text async def test_progress_is_logged_every_n_fetches(env, store, monkeypatch, caplog): diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 168d12b..24cc860 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -240,6 +240,28 @@ async def test_bots_own_tools_are_not_truncated_but_the_models_are(): assert "truncated" not in whole and len(whole) > 20 +async def test_user_calls_describe_what_is_running_but_not_background_work(): + s = MCPServer("sharesight", {"max_concurrent": 2}, timeout=5, max_output=1000) + gate = asyncio.Event() + + class Held(FakeSession): + async def call_tool(self, name, args): + await gate.wait() + return await super().call_tool(name, args) + + s.session = Held() + chat = asyncio.create_task(s.call("get_performance_report", {"portfolio_id": 7, "start_date": "2024-01-01", "end_date": "2024-01-31"})) + own = asyncio.create_task(s.call("get_performance_report", {"portfolio_id": 7, "start_date": "2024-02-01", "end_date": "2024-02-29", "_keep_sales": True})) + bg = asyncio.create_task(s.call("get_performance_report", {"portfolio_id": 7, "start_date": "2024-03-01", "end_date": "2024-03-31", "_keep_sales": True, "_bg": True})) + await asyncio.sleep(0.01) + assert s.in_flight == 3 + assert s.user_calls() == ["get_performance_report portfolio_id=7 2024-01-01..2024-01-31", + "get_performance_report portfolio_id=7 2024-02-01..2024-02-29 (period_returns)"] + gate.set() + await asyncio.gather(chat, own, bg) + assert s.user_calls() == [] and s.in_flight == 0 + + async def test_in_flight_counts_running_calls_and_returns_to_zero(): s = MCPServer("y", {"max_concurrent": 1}, timeout=0.05, max_output=1000) gate = asyncio.Event() diff --git a/tests/test_runner.py b/tests/test_runner.py index fc4707d..9e775b5 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -38,7 +38,7 @@ async def test_tool_round_trip_runs_mcp_and_returns_answer(): async def test_the_model_cannot_ask_for_sold_holdings_to_be_kept(): from lib.llm.base import Call mcp = FakeMcp("sharesight", tools=("get_performance_report",)) - await run_calls([Call("1", "sharesight__get_performance_report", '{"portfolio_id":1,"_keep_sales":true}')], + await run_calls([Call("1", "sharesight__get_performance_report", '{"portfolio_id":1,"_keep_sales":true,"_bg":true}')], ScriptedBackend([]), registry(mcp), with_tools(mcp), {}, True, Usage()) assert mcp.calls == [("get_performance_report", {"portfolio_id": 1})] From 440515bd9e900b68a1310a35a4648ffe71c31f3d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 20:43:50 +0000 Subject: [PATCH 26/34] period_returns: long ranges list only the top and bottom five; tool asks for one call per portfolio Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/features/periods/__init__.py | 25 +++++++++++++++++++++++-- tests/test_periods.py | 19 +++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/lib/features/periods/__init__.py b/lib/features/periods/__init__.py index 0bde18c..b93afcf 100644 --- a/lib/features/periods/__init__.py +++ b/lib/features/periods/__init__.py @@ -18,7 +18,9 @@ TOOL = ToolDef( "period_returns", "Total return of a Sharesight portfolio for each day, week, month, quarter or year, as a percentage and a " - "money gain, with the best and worst one by each measure and the holdings behind them. Use it for best/worst day, week, month, quarter or year, or any " + "money gain, with the best and worst one by each measure and the holdings behind them (a long range lists " + "only the top and bottom five, so one call answers 'best day since inception'). Call it once per portfolio " + "and cover every portfolio asked about. Use it for best/worst day, week, month, quarter or year, or any " "period-by-period question, instead of fetching one get_performance_report per period. Sold holdings are " "included. Omit start for 'since inception'. History that isn't cached yet is fetched live, which is slow " "for days and weeks: if the tool refuses a range, tell the user why and offer the start it suggests.", @@ -45,6 +47,18 @@ def sharesight(ctx: Ctx): return next((s for s in ctx.registry.up() if "get_performance_report" in s.fn_names.values()), None) +MAX_TABLE_ROWS = 40 # longer results list only the extremes +TOP_N = 5 + + +def pct_of(row) -> float: + return row[1].pct + + +def gain_of(row) -> float: + return row[1].gain + + def tool(ctx: Ctx): """(definition, function) of period_returns.""" @@ -87,9 +101,16 @@ async def periods(args: dict) -> str: worst = min(rows, key=lambda r: r[1].pct) lines = [f"Total return per {period} ({rows[0][1].currency}), {rows[0][0]} to {rows[-1][0]}, " "sold holdings included:"] - lines += [f"{label} {r.pct:+.2f}% {r.gain:+,.0f}" for label, r in rows] rich, poor = max(rows, key=lambda r: r[1].gain), min(rows, key=lambda r: r[1].gain) both = lambda row: f"{row[0]} {row[1].pct:+.2f}% ({row[1].gain:+,.0f})" # noqa: E731 + if len(rows) <= MAX_TABLE_ROWS: + lines += [f"{label} {r.pct:+.2f}% {r.gain:+,.0f}" for label, r in rows] + else: # thousands of rows would cost tens of thousands of tokens; the extremes are what gets asked + lines[0] = lines[0][:-1] + f" ({len(rows)} {period}s; the full list is left out, ask for a shorter range to see it):" + for title, key_fn, newest_first in (("Top", pct_of, True), ("Bottom", pct_of, False), + ("Top", gain_of, True), ("Bottom", gain_of, False)): + picked = sorted(rows, key=key_fn, reverse=newest_first)[:TOP_N] + lines.append(f"{title} {TOP_N} by {'%' if key_fn is pct_of else 'money'}: " + ", ".join(both(r) for r in picked)) lines.append(f"Best by %: {both(best)}. Worst by %: {both(worst)}. " f"Best by money: {both(rich)}. Worst by money: {both(poor)}." + (f" {missing} {period}s had no data." if missing else "")) diff --git a/tests/test_periods.py b/tests/test_periods.py index 47edcbe..84e8899 100644 --- a/tests/test_periods.py +++ b/tests/test_periods.py @@ -176,3 +176,22 @@ async def call(self, tool, args): out = await fn({"portfolio_id": 7, "start": "2026-01", "end": "2026-02"}) assert "Best by %: 2026-01 +10.00% (+100)" in out and "Best by money: 2026-02 +2.00% (+900)" in out assert "Worst by money: 2026-01 +10.00% (+100)" in out + + +async def test_long_results_list_only_the_extremes_not_every_row(env, store): + class Spread(Sharesight): + async def call(self, tool, args): + self.calls.append((tool, args)) + day = int(args["start_date"][-2:]) + 31 * int(args["start_date"][5:7]) # a different number every day + return json.dumps({"report": {"total_gain_percent": day / 10, "total_gain": -day * 10.0, "currency": "AUD"}}) + + ctx, _ = make(env, store, Spread("sharesight", tools=("list_portfolios", "get_performance_report"))) + _, fn = periods.tool(ctx) + out = await fn({"portfolio_id": 7, "period": "day", "start": "2026-01-01", "end": "2026-03-31"}) + lines = out.splitlines() + assert len(lines) < 15 and "full list is left out" in lines[0] and "64 days" in lines[0] + assert lines[1].startswith("Top 5 by %: 2026-03-31 ") and lines[2].startswith("Bottom 5 by %: 2026-01-01 ") + assert lines[3].startswith("Top 5 by money: 2026-01-01 ") and lines[4].startswith("Bottom 5 by money: 2026-03-31 ") + assert "Best by %:" in out + short = await fn({"portfolio_id": 7, "period": "day", "start": "2026-03-02", "end": "2026-03-13"}) + assert "Top 5" not in short and "2026-03-02 " in short # a short range still lists every row From 7c5ef929fa3ae5b0ee566b6758e1490239ff3979 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 00:51:27 +0000 Subject: [PATCH 27/34] Pie and bar charts: send_chart tool draws a PNG with Pillow and posts it, offered only for chart requests Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- CLAUDE.md | 2 +- README.md | 7 ++ lib/app.py | 4 +- lib/features/charts/__init__.py | 62 +++++++++++++ lib/features/charts/render.py | 158 ++++++++++++++++++++++++++++++++ lib/llm/gate.py | 1 + lib/telegram/handlers.py | 8 +- requirements.txt | 2 +- tests/conftest.py | 5 + tests/test_app.py | 6 +- tests/test_charts.py | 95 +++++++++++++++++++ tests/test_handlers.py | 14 +++ 12 files changed, 358 insertions(+), 6 deletions(-) create mode 100644 lib/features/charts/__init__.py create mode 100644 lib/features/charts/render.py create mode 100644 tests/test_charts.py diff --git a/CLAUDE.md b/CLAUDE.md index 2079044..69c5b9c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ MCP data tools and web search. User-facing docs are in `README.md`; this file is `openrouter.py` + `zai.py` (the only provider-specific code), `runner.py` (THE tool loop), `policy.py` (THE retry ladder), `gate.py` (which tools a message needs). - `prompts.py` all prompts. `ask.py` one entry point: route -> Request -> policy.ask -> usage -> NEEDS_TOOLS rerun. - `telegram/`: `handlers.py` (trigger -> request -> reply), `send.py`, `draft.py` (typing + streaming - drafts), `commands.py` (/credits), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop). + drafts), `commands.py` (/credits), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `post.py`, `charts/` (local `send_chart` tool: Pillow pie/bar PNGs, offered only for chart requests), `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop). - `app.py` wiring/startup; `context.py` `Ctx` (settings, store, backend, registry, bot). - `lib/` holds only what the bot needs to run. Developer helpers live in `tools/`: `report.py` (`make prompt-report`), `capture.py` (real MCP output for fixtures); the bot never imports them. diff --git a/README.md b/README.md index f89fb58..08c34ad 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,13 @@ gets "Portfolio data is for admins only." Holding-news DMs, buttons and down-ser unaffected. With nobody in `ADMIN_CHAT_IDS` and no known group, nobody can DM the bot or reach Sharesight: set `ADMIN_ONLY=off` to allow everyone. +## Charts +Ask for a "pie chart", "bar chart", "chart" or "graph" and the bot draws a real picture (a PNG) and posts it as a +reply, using numbers it got from its data tools: a pie for shares of a whole (portfolio sectors, holdings), a +horizontal bar chart for values per category or period (monthly gains, negatives in red). It is drawn with Pillow +(`pip install -r requirements.txt`); without it the bot says so at start-up and answers in text instead. There is +nothing to configure: the tool is only offered to the model when the message asks for a chart. + ## Caching Sharesight history Questions like "best day / week / month / quarter / year since inception" are answered by the `period_returns` tool (admins only), which asks Sharesight for one report per period (about 2.4 s each). Every **finished** period is diff --git a/lib/app.py b/lib/app.py index 96f977d..6407b11 100644 --- a/lib/app.py +++ b/lib/app.py @@ -28,7 +28,7 @@ from . import config from .context import Ctx -from .features import dm_buttons, holding_news, post +from .features import charts, dm_buttons, holding_news, post from .features.periods import backfill from .features.periods.windows import describe_scope, flush_scope from .llm.base import Backend @@ -212,6 +212,8 @@ def main(argv: Sequence[str] | None = None) -> None: log.info("Starting%s%s: %s on %s, reasoning %s, MCP servers: %s", f" instance {args.label}" if args.label else "", f" ({commit})" if commit else "", st.model, st.provider, st.reasoning or "default", ", ".join(ctx.registry.servers) or "none") + if not charts.available(): + log.warning("Pillow isn't installed, so pie and bar charts are off (pip install -r requirements.txt)") if st.period_backfill: log.info("Period backfill: on for %s", ", ".join(sorted(st.period_backfill))) app.run_polling(allowed_updates=allowed_updates(st)) diff --git a/lib/features/charts/__init__.py b/lib/features/charts/__init__.py new file mode 100644 index 0000000..bdbcd13 --- /dev/null +++ b/lib/features/charts/__init__.py @@ -0,0 +1,62 @@ +"""send_chart: draw a pie or bar chart from numbers the model got from its tools and post it in the chat.""" + +from __future__ import annotations + +import asyncio +import html +import logging + +from telegram import Message +from telegram.constants import ParseMode +from telegram.error import TelegramError + +from ...context import Ctx +from ...mcp.schema import ToolDef +from . import render + +log = logging.getLogger("bot") + +CAPTION_MAX = 900 +TOOL = ToolDef( + "send_chart", + "Draw a pie chart (shares of a whole, e.g. sectors) or a horizontal bar chart (values per category or per " + "period, negatives allowed) and post the picture in this chat. Use it whenever someone asks for a pie chart, " + "bar chart, chart or graph. Every value must come from tool results, never invented; group categories " + "yourself if you want different slices. Call it once per chart, then write one or two sentences, not the " + "numbers again.", + {"type": "object", "required": ["kind", "title", "items"], "properties": { + "kind": {"type": "string", "enum": ["pie", "bar"]}, + "title": {"type": "string", "description": "Short title shown on the picture"}, + "items": {"type": "array", "maxItems": 200, "items": {"type": "object", "required": ["label", "value"], "properties": { + "label": {"type": "string"}, "value": {"type": "number"}}}}, + "subtitle": {"type": "string", "description": "Optional line under the title, e.g. the total"}, + "caption": {"type": "string", "description": "Optional caption under the picture"}}}) + + +def available() -> bool: + return render.available() + + +def tool(ctx: Ctx, msg: Message): + """(definition, function) of send_chart, posting as a reply to `msg`.""" + + async def send(args: dict) -> str: + kind, title = str(args.get("kind", "")).lower(), str(args.get("title", "")).strip() + try: + png = await asyncio.to_thread(render.draw, kind, title, args.get("items"), str(args.get("subtitle") or "")) + except render.ChartError as e: + return f"Error: {e}." + except Exception as e: # a drawing bug must not sink the answer: the model falls back to text + log.exception("Chart drawing failed") + return f"Error: couldn't draw the chart ({type(e).__name__}); answer in text instead." + caption = html.escape(str(args.get("caption") or "").strip())[:CAPTION_MAX] + try: + sent = await ctx.bot.send_photo(msg.chat_id, photo=png, caption=caption or None, parse_mode=ParseMode.HTML, + reply_to_message_id=msg.message_id, disable_notification=True) + except TelegramError as e: + return f"Error: couldn't send the chart: {e}." + await asyncio.to_thread(ctx.store.save, sent) + log.info("Sent a %s chart (%d bytes): %s", kind, len(png), title[:60]) + return "Chart sent to the chat. Add one or two sentences at most; don't repeat the numbers." + + return TOOL, send diff --git a/lib/features/charts/render.py b/lib/features/charts/render.py new file mode 100644 index 0000000..987618e --- /dev/null +++ b/lib/features/charts/render.py @@ -0,0 +1,158 @@ +"""Chart images (pie and horizontal bar) drawn with Pillow: no network, no config or cache directories.""" + +from __future__ import annotations + +import io +import math + +WIDTH = 1000 +SCALE = 2 # drawn at 2x and shrunk, which smooths the edges +PALETTE = ["#4E79A7", "#F28E2B", "#59A14F", "#E15759", "#76B7B2", "#EDC948", "#B07AA1", "#FF9DA7", "#9C755F", "#86BCB6"] +OTHER_COLOUR, INK, MUTED, BACKGROUND, NEGATIVE = "#A0A0A0", "#222222", "#666666", "#FFFFFF", "#C8553D" +MAX_ITEMS = 30 +MAX_SLICES = 10 # named slices before the rest become "Other" +OTHER_BELOW = 0.02 # slices under this share merge into "Other" +LABEL_CHARS = 28 + + +class ChartError(ValueError): + """The data can't be drawn as asked.""" + + +def available() -> bool: + try: + import PIL # noqa: F401 + except ImportError: + return False + return True + + +def short(value: float) -> str: + """12345 -> 12.3k, 2500000 -> 2.5M, 3.2 -> 3.2.""" + mag = abs(value) + if mag >= 1e6: + return f"{value / 1e6:.1f}M" + if mag >= 1e3: + return f"{value / 1e3:.1f}k" + return f"{value:.1f}" if mag < 10 and value != int(value) else f"{value:.0f}" + + +def clean(kind: str, items: object) -> list[tuple[str, float]]: + """Validated (label, value) pairs; a pie keeps only positive values.""" + if kind not in ("pie", "bar"): + raise ChartError("kind must be 'pie' or 'bar'") + if not isinstance(items, list) or not items: + raise ChartError("give a list of items with a label and a value each") + out = [] + for item in items: + try: + label, value = str(item["label"]).strip(), float(item["value"]) + except (TypeError, KeyError, ValueError): + raise ChartError("every item needs a label and a numeric value") from None + if not math.isfinite(value): + raise ChartError(f"the value for {label!r} isn't a number") + out.append((label if len(label) <= LABEL_CHARS else label[:LABEL_CHARS - 1] + "…", value)) + if kind == "pie": + out = [(label, value) for label, value in out if value > 0] + if not out: + raise ChartError("a pie chart needs at least one positive value") + if len(out) > (200 if kind == "pie" else MAX_ITEMS): + raise ChartError(f"too many items ({len(out)}); group them into at most {MAX_ITEMS}") + return out + + +def pie_slices(items: list[tuple[str, float]]) -> list[tuple[str, float, float, bool]]: + """(label, value, share, is_other) largest first; small slices and everything past MAX_SLICES merge.""" + total = sum(value for _, value in items) + ranked = sorted(items, key=lambda i: -i[1]) + named = [i for i in ranked[:MAX_SLICES] if i[1] / total >= OTHER_BELOW] + rest = sum(value for _, value in ranked[len(named):]) + slices = [(label, value, value / total, False) for label, value in named] + if rest > 0: + slices.append(("Other", rest, rest / total, True)) + return slices + + +def _font(size: int): + from PIL import ImageFont + try: + return ImageFont.load_default(size=size * SCALE) + except TypeError: # Pillow before 10.1 has only the small bitmap font + return ImageFont.load_default() + + +def _fit(draw, text: str, font, width: float) -> str: + """Text cut with an ellipsis to fit `width` (1x pixels).""" + if draw.textlength(text, font=font) <= width * SCALE: + return text + while text and draw.textlength(text + "…", font=font) > width * SCALE: + text = text[:-1] + return text.rstrip() + "…" + + +def draw(kind: str, title: str, items: list, subtitle: str = "") -> bytes: + """A PNG of the chart. Raises ChartError for data that can't be drawn.""" + pairs = clean(kind, items) + from PIL import Image, ImageDraw + + height = _pie_height(pairs) if kind == "pie" else 130 + 38 * len(pairs) + img = Image.new("RGB", (WIDTH * SCALE, height * SCALE), BACKGROUND) + d = ImageDraw.Draw(img) + s = SCALE + d.text((40 * s, 28 * s), _fit(d, title.strip() or "Chart", _font(36), WIDTH - 80), fill=INK, font=_font(36)) + if subtitle: + d.text((40 * s, 76 * s), _fit(d, subtitle, _font(24), WIDTH - 80), fill=MUTED, font=_font(24)) + (_draw_pie if kind == "pie" else _draw_bar)(d, pairs, height) + img = img.resize((WIDTH, height), Image.LANCZOS) + out = io.BytesIO() + img.save(out, "PNG", optimize=True) + return out.getvalue() + + +def _pie_height(pairs: list[tuple[str, float]]) -> int: + return max(600, 150 + 62 * len(pie_slices(pairs))) + + +def _draw_pie(d, pairs: list[tuple[str, float]], height: int) -> None: + s = SCALE + slices = pie_slices(pairs) + left, top, size = 40, 130, 420 + box = [left * s, top * s, (left + size) * s, (top + size) * s] + angle = -90.0 + for i, (_, _, share, is_other) in enumerate(slices): + colour = OTHER_COLOUR if is_other else PALETTE[i % len(PALETTE)] + end = angle + 360 * share + if share > 0.9999: + d.ellipse(box, fill=colour) + else: + d.pieslice(box, angle, end, fill=colour, outline=BACKGROUND, width=2 * s) + angle = end + label_font, detail_font = _font(26), _font(22) + for i, (label, value, share, is_other) in enumerate(slices): + y = 135 + 62 * i + colour = OTHER_COLOUR if is_other else PALETTE[i % len(PALETTE)] + d.rectangle([500 * s, (y + 4) * s, 524 * s, (y + 28) * s], fill=colour) + d.text((540 * s, y * s), _fit(d, label, label_font, 420), fill=INK, font=label_font) + d.text((540 * s, (y + 31) * s), f"{share * 100:.1f}% {short(value)}", fill=MUTED, font=detail_font) + + +def _draw_bar(d, pairs: list[tuple[str, float]], height: int) -> None: + s = SCALE + x_left, x_right = 300, 860 + top_max = max(0.0, max(v for _, v in pairs)) + low_max = max(0.0, -min(v for _, v in pairs)) + span = (top_max + low_max) or 1.0 + zero = x_left + (x_right - x_left) * low_max / span + label_font, value_font = _font(24), _font(22) + for i, (label, value) in enumerate(pairs): + y = 115 + 38 * i + d.text((40 * s, (y + 2) * s), _fit(d, label, label_font, 245), fill=INK, font=label_font) + length = (x_right - x_left) * abs(value) / span + x0, x1 = (zero, zero + length) if value >= 0 else (zero - length, zero) + d.rectangle([x0 * s, y * s, max(x1, x0 + 1) * s, (y + 28) * s], fill=PALETTE[0] if value >= 0 else NEGATIVE) + text = short(value) + if value >= 0: + d.text(((x1 + 8) * s, (y + 2) * s), text, fill=MUTED, font=value_font) + else: + d.text((x0 * s - 8 * s - d.textlength(text, font=value_font), (y + 2) * s), text, fill=MUTED, font=value_font) + d.line([zero * s, 108 * s, zero * s, (115 + 38 * len(pairs)) * s], fill=MUTED, width=s) diff --git a/lib/llm/gate.py b/lib/llm/gate.py index b80dbc0..a8b49bd 100644 --- a/lib/llm/gate.py +++ b/lib/llm/gate.py @@ -31,6 +31,7 @@ r"p&l|pnl|investments?|funds?|etfs?|trades?|watchlist)\b", re.I) PORTFOLIO_WEAK = re.compile( r"\b(performance|gains?|positions?|winners?|losers?|p&l|pnl|(?:out|under)perform\w*|ahead of|behind)\b", re.I) +CHART_WORDS = re.compile(r"\b(pie|bar chart|chart|graph|plot|diagram)\b", re.I) # a request for a picture of the data LIVE_WORDS = re.compile( r"\b(news|today|latest|yesterday|tonight|this week|right now|breaking|announce\w*|who won|" r"score|weather|happening|headlines?|current\w*|updates?|recent\w*|just (now|in))\b", re.I) diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index fd6ba8a..d797c9d 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -15,9 +15,9 @@ from .. import config from ..ask import ask from ..context import Ctx -from ..features import dm_buttons, holding_news, movers, periods, post +from ..features import charts, dm_buttons, holding_news, movers, periods, post from ..history import compact, format_rows -from ..llm.gate import Route, is_portfolio_question, route +from ..llm.gate import CHART_WORDS, Route, is_portfolio_question, route from ..msgtext import describe, sender_name from ..prompts import MOVERS_EXPLAIN, PRESET_PROMPT, chat_prompt, down_note from ..store import is_dm @@ -251,6 +251,10 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s extra = [post.tool(ctx, msg.from_user.id, msg.from_user.full_name or "")] if chosen.simple: # the model must not be told it has no tools chosen = Route([], False) + if CHART_WORDS.search(trig.text) and charts.available(): + extra.append(charts.tool(ctx, msg)) + if chosen.simple: # the model must not be told it has no tools + chosen = Route([], False) fresh = is_portfolio_question(context, st) and any(s.cfg.get("gate") == "portfolio" for s in chosen.servers) parts = [{"type": "text", "text": await self._chat_text(msg, trig, fresh=fresh)}] parts += await self._images(msg, trig) diff --git a/requirements.txt b/requirements.txt index cfb6bcd..1c36dc9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ python-telegram-bot>=21.0 openai>=1.40 httpx>=0.27 mcp>=1.9,<2 - +Pillow>=10.1 diff --git a/tests/conftest.py b/tests/conftest.py index 573ea0a..46d9cc9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -53,6 +53,7 @@ class FakeBot: def __init__(self, bot_id=111, username="stockbot", first_name="Stock"): self.id, self.username, self.first_name = bot_id, username, first_name self.sent: list[dict] = [] + self.photos: list[dict] = [] self.drafts: list[str] = [] self.actions: list[str] = [] self.fail_send: dict[int, Exception] = {} @@ -72,6 +73,10 @@ async def send_message(self, chat_id, text, **kw): self.sent.append({"chat_id": chat_id, "text": text, **kw}) return self._msg(chat_id, text) + async def send_photo(self, chat_id, photo=None, caption=None, **kw): + self.photos.append({"chat_id": chat_id, "bytes": photo, "caption": caption, **kw}) + return self._msg(chat_id, caption or "[photo]") + async def get_chat_administrators(self, chat_id): if chat_id not in self.admins: raise TelegramError("Bad Request: chat not found") diff --git a/tests/test_app.py b/tests/test_app.py index 91db5d9..80f3046 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -89,7 +89,11 @@ def test_startup_logs_the_git_hash_and_whether_the_backfill_is_on(monkeypatch, t monkeypatch.setenv("PERIOD_BACKFILL", "RobSMSF, other") app.main(["grokbot"]) assert "Starting instance grokbot: " in caplog.text - assert "Period backfill: on for other, robsmsf" in caplog.text + assert "Period backfill: on for other, robsmsf" in caplog.text and "Pillow isn't installed" not in caplog.text + caplog.clear() + monkeypatch.setattr(app.charts, "available", lambda: False) + app.main(["grokbot"]) + assert "Pillow isn't installed, so pie and bar charts are off" in caplog.text def test_git_hash_is_read_from_the_git_directory_without_running_git(tmp_path): diff --git a/tests/test_charts.py b/tests/test_charts.py new file mode 100644 index 0000000..d55772b --- /dev/null +++ b/tests/test_charts.py @@ -0,0 +1,95 @@ +import io + +import pytest +from telegram.error import TelegramError + +from lib import config +from lib.context import Ctx +from lib.features import charts +from lib.features.charts import render + +from .conftest import FakeBot, user_msg +from .fakes import ScriptedBackend, registry + +PNG = b"\x89PNG\r\n\x1a\n" +SECTORS = [{"label": "Semis", "value": 520.0}, {"label": "ETFs", "value": 250.0}, {"label": "Software", "value": 150.0}, + {"label": "Health", "value": 12.0}, {"label": "REA", "value": 10.0}] + + +def size(png: bytes) -> tuple[int, int]: + from PIL import Image + return Image.open(io.BytesIO(png)).size + + +def test_a_pie_is_a_png_and_small_slices_merge_into_other(): + png = render.draw("pie", "Sectors", SECTORS, "total 942") + assert png.startswith(PNG) and size(png)[0] == render.WIDTH and len(png) < 150_000 + slices = render.pie_slices([(i["label"], i["value"]) for i in SECTORS]) + assert [s[0] for s in slices] == ["Semis", "ETFs", "Software", "Other"] # Health and REA are under 2% + assert slices[-1][1] == 22.0 and slices[-1][3] and abs(sum(s[2] for s in slices) - 1) < 1e-9 + + +def test_more_than_ten_slices_fold_the_rest_into_other(): + items = [(f"s{i}", 100 - i) for i in range(15)] + slices = render.pie_slices(items) + assert len(slices) == render.MAX_SLICES + 1 and slices[-1][0] == "Other" + + +def test_a_bar_chart_takes_negatives_and_grows_with_the_rows(): + few = render.draw("bar", "Gain", [{"label": "Jan", "value": -5}, {"label": "Feb", "value": 12}]) + many = render.draw("bar", "Gain", [{"label": str(i), "value": i - 10} for i in range(25)]) + assert few.startswith(PNG) and many.startswith(PNG) and size(many)[1] > size(few)[1] + + +@pytest.mark.parametrize("kind, items, message", [ + ("donut", SECTORS, "kind must be"), + ("pie", [], "list of items"), + ("pie", "x", "list of items"), + ("pie", [{"label": "a"}], "label and a numeric value"), + ("pie", [{"label": "a", "value": "x"}], "numeric value"), + ("pie", [{"label": "a", "value": float("nan")}], "isn't a number"), + ("pie", [{"label": "a", "value": -1}, {"label": "b", "value": 0}], "positive value"), + ("bar", [{"label": str(i), "value": 1} for i in range(31)], "too many"), +]) +def test_bad_data_is_refused_with_a_clear_reason(kind, items, message): + with pytest.raises(render.ChartError, match=message): + render.draw(kind, "t", items) + + +def test_short_values_and_long_labels(): + assert [render.short(v) for v in (2_500_000, 173_600, 12, 3.2, -4_000)] == ["2.5M", "173.6k", "12", "3.2", "-4.0k"] + label = render.clean("pie", [{"label": "x" * 60, "value": 1}])[0][0] + assert len(label) == render.LABEL_CHARS and label.endswith("…") + + +async def test_the_tool_sends_one_photo_as_a_reply_and_saves_it(env, store): + bot = FakeBot() + ctx = Ctx(config.load(env), store, ScriptedBackend([]), registry(), bot) + msg = user_msg(bot, "@stockbot pie chart of my sectors", mid=7) + defn, send = charts.tool(ctx, msg) + out = await send({"kind": "pie", "title": "Sectors", "items": SECTORS, "caption": "A test & more"}) + assert defn.name == "send_chart" and out.startswith("Chart sent") + photo = bot.photos[0] + assert photo["bytes"].startswith(PNG) and photo["reply_to_message_id"] == 7 and photo["chat_id"] == -100 + assert photo["caption"] == "A <b>test</b> & more" and photo["disable_notification"] is True + assert any("A <b>" in r.text for r in store.history(-100, 20)) # the sent message is in history + + +async def test_the_tool_returns_errors_as_text_so_the_model_can_fall_back(env, store, monkeypatch): + bot = FakeBot() + ctx = Ctx(config.load(env), store, ScriptedBackend([]), registry(), bot) + _, send = charts.tool(ctx, user_msg(bot, "chart", mid=1)) + assert (await send({"kind": "pie", "title": "t", "items": []})).startswith("Error: give a list of items") + assert bot.photos == [] + + async def broken(*a, **kw): + raise TelegramError("Bad Request: not enough rights") + + monkeypatch.setattr(bot, "send_photo", broken) + assert "couldn't send the chart" in await send({"kind": "bar", "title": "t", "items": SECTORS}) + + def explode(*a, **kw): + raise RuntimeError("font missing") + + monkeypatch.setattr(render, "draw", explode) + assert "answer in text instead" in await send({"kind": "pie", "title": "t", "items": SECTORS}) diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 1b2c487..9a89aa4 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -113,6 +113,20 @@ async def test_a_short_follow_up_to_a_portfolio_reply_still_gets_the_portfolio_t assert backend.seen[3]["tools"] == [] +async def test_the_chart_tool_is_offered_only_when_asked_for_a_chart(env, store, monkeypatch): + from lib.features import charts + + ctx, backend, h = make_ctx(env, store, [step("a"), step("b"), step("c"), step("d")]) + await run(h, user_msg(ctx.bot, "@stockbot how is NVDA looking?", mid=1)) + await run(h, user_msg(ctx.bot, "@stockbot make me a pie chart of the semis sector", mid=2000)) + await run(h, user_msg(ctx.bot, "@stockbot draw a graph of chips", mid=3000)) + assert "send_chart" not in backend.seen[0]["tools"] and "send_chart" in backend.seen[1]["tools"] + assert "send_chart" in backend.seen[2]["tools"] # also on the route that would otherwise have no tools + monkeypatch.setattr(charts, "available", lambda: False) # Pillow not installed: the bot just doesn't offer it + await run(h, user_msg(ctx.bot, "@stockbot pie chart please", mid=4000)) + assert "send_chart" not in backend.seen[3]["tools"] + + async def test_portfolio_question_reminds_the_model_to_fetch_fresh_figures(env, store): sharesight = FakeMcp("sharesight", gate="portfolio", tools=("list_portfolios",)) ctx, backend, h = make_ctx(env, store, [step("a"), step("b")], servers=[sharesight]) From 2fae5de9f43627fbe2a8c3486d55090e5a1ed0fd Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 00:54:26 +0000 Subject: [PATCH 28/34] Chart triggers: visualise, breakdown, allocation, and 'by sector/industry/...' Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- README.md | 2 +- lib/llm/gate.py | 5 ++++- tests/test_runner.py | 11 +++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 08c34ad..e39be45 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ unaffected. With nobody in `ADMIN_CHAT_IDS` and no known group, nobody can DM th `ADMIN_ONLY=off` to allow everyone. ## Charts -Ask for a "pie chart", "bar chart", "chart" or "graph" and the bot draws a real picture (a PNG) and posts it as a +Ask for a "pie chart", "bar chart", "chart" or "graph", to "visualise" something, or for a breakdown "by sector" or "by industry", and the bot draws a real picture (a PNG) and posts it as a reply, using numbers it got from its data tools: a pie for shares of a whole (portfolio sectors, holdings), a horizontal bar chart for values per category or period (monthly gains, negatives in red). It is drawn with Pillow (`pip install -r requirements.txt`); without it the bot says so at start-up and answers in text instead. There is diff --git a/lib/llm/gate.py b/lib/llm/gate.py index a8b49bd..4b9d7ab 100644 --- a/lib/llm/gate.py +++ b/lib/llm/gate.py @@ -31,7 +31,10 @@ r"p&l|pnl|investments?|funds?|etfs?|trades?|watchlist)\b", re.I) PORTFOLIO_WEAK = re.compile( r"\b(performance|gains?|positions?|winners?|losers?|p&l|pnl|(?:out|under)perform\w*|ahead of|behind)\b", re.I) -CHART_WORDS = re.compile(r"\b(pie|bar chart|chart|graph|plot|diagram)\b", re.I) # a request for a picture of the data +CHART_WORDS = re.compile( # a request for a picture of the data + r"\b(pie|bar chart|chart|graph|plot|diagram|visuali[sz]e|visuali[sz]ation|breakdown|allocation)\b" + r"|\bby\s+(?:sectors?|industr(?:y|ies)|asset class(?:es)?|markets?|countr(?:y|ies)|currenc(?:y|ies)|regions?|holdings?|stocks?)\b", + re.I) LIVE_WORDS = re.compile( r"\b(news|today|latest|yesterday|tonight|this week|right now|breaking|announce\w*|who won|" r"score|weather|happening|headlines?|current\w*|updates?|recent\w*|just (now|in))\b", re.I) diff --git a/tests/test_runner.py b/tests/test_runner.py index 9e775b5..a54c94d 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -293,3 +293,14 @@ def test_gate_marks_routes_that_offer_fewer_tools(settings): def test_wants_tools(): assert wants_tools("NEEDS_TOOLS") and wants_tools(" needs_tools.") and not wants_tools("Sure thing") + + +@pytest.mark.parametrize("text, wanted", [ + ("can you do a pie chart of my sectors?", True), ("visualise my portfolio", True), ("Visualize my portfolio", True), + ("my portfolio by sector", True), ("robsmsf by industry", True), ("robsmsf by industries", True), + ("show holdings by country", True), ("what's my allocation", True), ("sector breakdown please", True), + ("graph NVDA this year", True), ("how is NVDA looking?", False), ("what did I make this month", False), + ("which sector is strongest", False), ("buy or sell?", False)]) +def test_chart_words_catch_requests_for_a_picture_or_a_grouping(text, wanted): + from lib.llm.gate import CHART_WORDS + assert bool(CHART_WORDS.search(text)) is wanted From 2086e8e67c1cab142a88f1b37a4a38400b04449a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 01:02:19 +0000 Subject: [PATCH 29/34] Log which local tools (send_chart, period_returns...) were offered to the model Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/telegram/handlers.py | 2 ++ tests/test_handlers.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index d797c9d..9d30ad9 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -255,6 +255,8 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s extra.append(charts.tool(ctx, msg)) if chosen.simple: # the model must not be told it has no tools chosen = Route([], False) + if extra: + log.info("Local tools offered: %s", ", ".join(t.name for t, _ in extra)) fresh = is_portfolio_question(context, st) and any(s.cfg.get("gate") == "portfolio" for s in chosen.servers) parts = [{"type": "text", "text": await self._chat_text(msg, trig, fresh=fresh)}] parts += await self._images(msg, trig) diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 9a89aa4..98f2ec1 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -113,15 +113,18 @@ async def test_a_short_follow_up_to_a_portfolio_reply_still_gets_the_portfolio_t assert backend.seen[3]["tools"] == [] -async def test_the_chart_tool_is_offered_only_when_asked_for_a_chart(env, store, monkeypatch): +async def test_the_chart_tool_is_offered_only_when_asked_for_a_chart(env, store, monkeypatch, caplog): from lib.features import charts + caplog.set_level("INFO", logger="bot") + ctx, backend, h = make_ctx(env, store, [step("a"), step("b"), step("c"), step("d")]) await run(h, user_msg(ctx.bot, "@stockbot how is NVDA looking?", mid=1)) await run(h, user_msg(ctx.bot, "@stockbot make me a pie chart of the semis sector", mid=2000)) await run(h, user_msg(ctx.bot, "@stockbot draw a graph of chips", mid=3000)) assert "send_chart" not in backend.seen[0]["tools"] and "send_chart" in backend.seen[1]["tools"] assert "send_chart" in backend.seen[2]["tools"] # also on the route that would otherwise have no tools + assert "Local tools offered: send_chart" in caplog.text monkeypatch.setattr(charts, "available", lambda: False) # Pillow not installed: the bot just doesn't offer it await run(h, user_msg(ctx.bot, "@stockbot pie chart please", mid=4000)) assert "send_chart" not in backend.seen[3]["tools"] From d54f1500be975f87956598ca972c990bb71ed74a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 01:15:31 +0000 Subject: [PATCH 30/34] Charts: tell the model to draw when send_chart is offered; keep the industry of each holding Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/mcp/results.py | 1 + lib/prompts.py | 8 +++++++- lib/telegram/handlers.py | 10 ++++++---- tests/test_fixtures.py | 2 +- tests/test_handlers.py | 1 + tests/test_history.py | 8 ++++++++ tests/test_mcp.py | 9 +++++++++ 7 files changed, 33 insertions(+), 6 deletions(-) diff --git a/lib/mcp/results.py b/lib/mcp/results.py index e129382..b3d73be 100644 --- a/lib/mcp/results.py +++ b/lib/mcp/results.py @@ -78,6 +78,7 @@ def slim_holding(h): "currency": inst.get("currency_code"), "type": inst.get("friendly_instrument_description"), "sector": inst.get("sector_classification_name"), + "industry": inst.get("industry_classification_name"), } out.update({k: h.get(k) for k in HOLDING_KEEP}) if out.get("type") == "Ordinary Shares": # the default; only say when it's something else diff --git a/lib/prompts.py b/lib/prompts.py index d9cb3da..f96ec63 100644 --- a/lib/prompts.py +++ b/lib/prompts.py @@ -69,7 +69,7 @@ def down_note(down: list[MCPServer]) -> str: def chat_prompt(*, transcript: str, sender: str, private: bool, reply_quote: str | None, msg_id: int, now: str, down: str = "", saver: bool = True, middle: str = "", - fresh: bool = False) -> str: + fresh: bool = False, chart: bool = False) -> str: """The user turn: transcript, the replied-to message, and the volatile tail (kept last so the cached prefix before it is unchanged).""" head = (f"Private one-to-one chat with {sender}, not the group. Transcript:\n{transcript}" @@ -89,9 +89,15 @@ def chat_prompt(*, transcript: str, sender: str, private: bool, reply_quote: str "for anything else asked, such as prices or a comparison with market data); don't reuse " "numbers or refusals from earlier replies. When comparing portfolios, or asked \"why\", " "write 2-3 short paragraphs on the tailwinds and headwinds of each portfolio, not a summary.") + if chart: # send_chart is on offer: the phrasing ("by sector", "visualise") asks for a picture + tail += CHART_TAIL return head + tail + down +CHART_TAIL = (" They want a picture, not a list: once you have the numbers, draw it with send_chart (a pie for " + "shares of a whole such as sectors, industries or holdings, a bar chart for values per category or " + "period; add up the holdings' value by the field asked for) and reply with one or two sentences.") + PRESET_PROMPT = ("This is a private chat with {sender}. They tapped the \"{label}\" button, which " "asks for: {ask}. It's now {now}. Search {what} for current information, and do " "the whole job before replying.") diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index 9d30ad9..90eecfb 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -178,7 +178,8 @@ async def _gate_text(self, msg: Message, trig: Trigger) -> str: text += " " + compact(prev.text)[:FOLLOW_UP_CHARS] return text - async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: bool = False) -> str: + async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: bool = False, + chart: bool = False) -> str: ctx, st = self.ctx, self.ctx.st rows = await asyncio.to_thread(ctx.store.history, msg.chat_id, st.history_limit) transcript = format_rows(rows, ctx.self_name, st.tz, line_max=st.history_line_max, @@ -186,7 +187,7 @@ async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: return chat_prompt( transcript=transcript, sender=sender_name(msg), private=trig.private, reply_quote=self._quote(trig), msg_id=msg.message_id, now=ctx.now(), - down=down_note(ctx.registry.down()), saver=st.token_saver, middle=middle, fresh=fresh) + down=down_note(ctx.registry.down()), saver=st.token_saver, middle=middle, fresh=fresh, chart=chart) # -- answering --------------------------------------------------------------------- async def _answer(self, msg: Message, trig: Trigger) -> None: @@ -251,14 +252,15 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s extra = [post.tool(ctx, msg.from_user.id, msg.from_user.full_name or "")] if chosen.simple: # the model must not be told it has no tools chosen = Route([], False) - if CHART_WORDS.search(trig.text) and charts.available(): + chart = bool(CHART_WORDS.search(trig.text)) and charts.available() + if chart: extra.append(charts.tool(ctx, msg)) if chosen.simple: # the model must not be told it has no tools chosen = Route([], False) if extra: log.info("Local tools offered: %s", ", ".join(t.name for t, _ in extra)) fresh = is_portfolio_question(context, st) and any(s.cfg.get("gate") == "portfolio" for s in chosen.servers) - parts = [{"type": "text", "text": await self._chat_text(msg, trig, fresh=fresh)}] + parts = [{"type": "text", "text": await self._chat_text(msg, trig, fresh=fresh, chart=chart)}] parts += await self._images(msg, trig) answer = await ask(ctx, parts, chosen, cache_key=f"chat-{chat_id}", on_text=on_text, chat_id=chat_id, extra_tools=extra, diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py index a211e22..7d0631c 100644 --- a/tests/test_fixtures.py +++ b/tests/test_fixtures.py @@ -41,7 +41,7 @@ def test_real_performance_report_is_flattened_to_a_fraction_of_its_size(key): text = (FIX / "sharesight" / f"performance_report_{key}.json").read_text() slim = slim_result(text, "sharesight", drop_closed=True) report, original = json.loads(slim)["report"], json.loads(text)["report"] - assert len(slim) < len(text) / 8 + assert len(slim) < len(text) / 7.5 # was 8 before each holding also kept its industry (for "by industry" charts) rows = table_records(report["holdings"]) assert [(r["code"], r["quantity"], r["value"]) for r in rows] == [ (h["instrument"]["code"], h["quantity"], h["value"]) for h in original["holdings"]] diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 98f2ec1..a4d3914 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -125,6 +125,7 @@ async def test_the_chart_tool_is_offered_only_when_asked_for_a_chart(env, store, assert "send_chart" not in backend.seen[0]["tools"] and "send_chart" in backend.seen[1]["tools"] assert "send_chart" in backend.seen[2]["tools"] # also on the route that would otherwise have no tools assert "Local tools offered: send_chart" in caplog.text + assert "draw it with send_chart" not in backend.seen[0]["prompt"] and "draw it with send_chart" in backend.seen[1]["prompt"] monkeypatch.setattr(charts, "available", lambda: False) # Pillow not installed: the bot just doesn't offer it await run(h, user_msg(ctx.bot, "@stockbot pie chart please", mid=4000)) assert "send_chart" not in backend.seen[3]["tools"] diff --git a/tests/test_history.py b/tests/test_history.py index 0ef6d92..08facc9 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -44,3 +44,11 @@ def test_latest_own_reply_has_a_generous_cap_and_no_own_reply_is_fine(): def test_raw_mode_keeps_html(): rows = [HistoryRow(1, "Bob", "x", 0, None)] assert format_rows(rows, "z", UTC, line_max=100, compact_text=False).endswith("x") + + +def test_chat_prompt_asks_for_a_picture_only_when_the_chart_tool_is_offered(): + from lib.prompts import chat_prompt + base = dict(transcript="t", sender="Bob", private=False, reply_quote=None, msg_id=3, now="now") + plain, chart = chat_prompt(**base), chat_prompt(**base, chart=True, fresh=True) + assert "send_chart" not in plain and "draw it with send_chart" in chart + assert chart.index("Get current figures") < chart.index("They want a picture") # after the portfolio tail diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 24cc860..b76ecf5 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -374,3 +374,12 @@ def test_compact_schema_hides_listed_params_but_never_required_ones_and_drops_sc "required": ["a"]} out = compact_schema(schema, hide=frozenset({"a", "b"})) assert set(out["properties"]) == {"a", "c"} and "$schema" not in out and "additionalProperties" not in out + + +def test_slimmed_holdings_keep_both_sector_and_industry(): + from lib.mcp.results import slim_holding + h = {"quantity": 5, "value": 10.0, "instrument": {"code": "CRWD", "market_code": "NASDAQ", "name": "Crowdstrike", + "sector_classification_name": "Technology Services", "industry_classification_name": "Packaged Software"}} + out = slim_holding(h) + assert out["sector"] == "Technology Services" and out["industry"] == "Packaged Software" + assert "industry" not in slim_holding({"instrument": {"code": "X"}}) # missing fields are dropped as before From 2d6bbfe1fd1b1fae9d7ff504f6f5d681d7c9450c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 01:22:07 +0000 Subject: [PATCH 31/34] portfolio_chart: the bot groups a portfolio's holdings by sector/industry/holding itself, so the two fields stay distinct Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- README.md | 7 ++- lib/features/charts/__init__.py | 106 +++++++++++++++++++++++++++----- lib/prompts.py | 8 ++- lib/telegram/handlers.py | 2 + tests/test_charts.py | 67 +++++++++++++++++++- tests/test_handlers.py | 11 +++- tests/test_history.py | 3 +- 7 files changed, 180 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index e39be45..5b18f8a 100644 --- a/README.md +++ b/README.md @@ -190,10 +190,13 @@ unaffected. With nobody in `ADMIN_CHAT_IDS` and no known group, nobody can DM th ## Charts Ask for a "pie chart", "bar chart", "chart" or "graph", to "visualise" something, or for a breakdown "by sector" or "by industry", and the bot draws a real picture (a PNG) and posts it as a -reply, using numbers it got from its data tools: a pie for shares of a whole (portfolio sectors, holdings), a +reply, using numbers it got from its data tools: a pie for shares of a whole (a portfolio's sectors, industries, holdings or markets), a horizontal bar chart for values per category or period (monthly gains, negatives in red). It is drawn with Pillow (`pip install -r requirements.txt`); without it the bot says so at start-up and answers in text instead. There is -nothing to configure: the tool is only offered to the model when the message asks for a chart. +nothing to configure: the tools are only offered to the model when the message asks for a chart. For a +portfolio grouped by sector, industry, holding, market or type (admins only) the bot fetches and adds up the +holdings itself (`portfolio_chart`), so sector and industry stay distinct and the totals don't depend on the +model's arithmetic. ## Caching Sharesight history Questions like "best day / week / month / quarter / year since inception" are answered by the `period_returns` tool diff --git a/lib/features/charts/__init__.py b/lib/features/charts/__init__.py index bdbcd13..7e3790e 100644 --- a/lib/features/charts/__init__.py +++ b/lib/features/charts/__init__.py @@ -4,13 +4,16 @@ import asyncio import html +import json import logging +from datetime import datetime from telegram import Message from telegram.constants import ParseMode from telegram.error import TelegramError from ...context import Ctx +from ...mcp.results import table_records from ...mcp.schema import ToolDef from . import render @@ -37,26 +40,97 @@ def available() -> bool: return render.available() +async def _send(ctx: Ctx, msg: Message, kind: str, title: str, items: object, subtitle: str, caption: str) -> str: + """Draw the chart and post it as a reply to `msg`; "Error: ..." text if that fails, else "sent".""" + try: + png = await asyncio.to_thread(render.draw, kind, title, items, subtitle) + except render.ChartError as e: + return f"Error: {e}." + except Exception as e: # a drawing bug must not sink the answer: the model falls back to text + log.exception("Chart drawing failed") + return f"Error: couldn't draw the chart ({type(e).__name__}); answer in text instead." + caption = html.escape(caption.strip())[:CAPTION_MAX] + try: + sent = await ctx.bot.send_photo(msg.chat_id, photo=png, caption=caption or None, parse_mode=ParseMode.HTML, + reply_to_message_id=msg.message_id, disable_notification=True) + except TelegramError as e: + return f"Error: couldn't send the chart: {e}." + await asyncio.to_thread(ctx.store.save, sent) + log.info("Sent a %s chart (%d bytes): %s", kind, len(png), title[:60]) + return "sent" + + def tool(ctx: Ctx, msg: Message): """(definition, function) of send_chart, posting as a reply to `msg`.""" async def send(args: dict) -> str: - kind, title = str(args.get("kind", "")).lower(), str(args.get("title", "")).strip() + out = await _send(ctx, msg, str(args.get("kind", "")).lower(), str(args.get("title", "")).strip(), + args.get("items"), str(args.get("subtitle") or ""), str(args.get("caption") or "")) + return out if out.startswith("Error") else "Chart sent to the chat. Add one or two sentences at most; don't repeat the numbers." + + return TOOL, send + + +# -- a portfolio grouped by sector, industry, holding...: the bot adds it up, not the model --------------- +GROUPS = {"sector": "sector", "industry": "industry", "holding": "code", "market": "market", "type": "type"} +PORTFOLIO_TOOL = ToolDef( + "portfolio_chart", + "Draw a chart of one Sharesight portfolio's current holdings grouped by sector, industry, holding, market or " + "type, and post it in this chat. The bot fetches and adds up the holdings itself, so use this (not " + "send_chart) for 'my portfolio by sector' and the like. Sector and industry are different Sharesight " + "fields (Technology Services is a sector, Packaged Software an industry): pass exactly the one asked for. " + "Call it once per portfolio asked about; portfolio_id comes from list_portfolios.", + {"type": "object", "required": ["portfolio_id", "by"], "properties": { + "portfolio_id": {"type": "number"}, + "by": {"type": "string", "enum": list(GROUPS)}, + "kind": {"type": "string", "enum": ["pie", "bar"], "description": "Default pie"}}}) + + +def group_holdings(holdings: list[dict], by: str) -> list[tuple[str, float]]: + """Value per group (largest first), from slimmed Sharesight holdings; a missing field is "Unclassified".""" + field, totals = GROUPS[by], {} + for h in holdings: + value = h.get("value") + if not isinstance(value, (int, float)) or value <= 0: + continue + name = str(h.get(field) or "Unclassified") + totals[name] = totals.get(name, 0.0) + float(value) + return sorted(totals.items(), key=lambda kv: -kv[1]) + + +def portfolio_tool(ctx: Ctx, msg: Message): + """(definition, function) of portfolio_chart, posting as a reply to `msg`.""" + + async def send(args: dict) -> str: + server = next((s for s in ctx.registry.up() if "get_performance_report" in s.fn_names.values()), None) + if server is None: + return "Error: the portfolio data source isn't connected." + by, kind = str(args.get("by", "")).lower(), str(args.get("kind") or "pie").lower() + if by not in GROUPS: + return f"Error: by must be one of {', '.join(GROUPS)}." try: - png = await asyncio.to_thread(render.draw, kind, title, args.get("items"), str(args.get("subtitle") or "")) - except render.ChartError as e: - return f"Error: {e}." - except Exception as e: # a drawing bug must not sink the answer: the model falls back to text - log.exception("Chart drawing failed") - return f"Error: couldn't draw the chart ({type(e).__name__}); answer in text instead." - caption = html.escape(str(args.get("caption") or "").strip())[:CAPTION_MAX] + pid = int(args.get("portfolio_id")) + except (TypeError, ValueError): + return "Error: give the portfolio_id (call list_portfolios first)." + today = datetime.now(ctx.st.tz).date().isoformat() try: - sent = await ctx.bot.send_photo(msg.chat_id, photo=png, caption=caption or None, parse_mode=ParseMode.HTML, - reply_to_message_id=msg.message_id, disable_notification=True) - except TelegramError as e: - return f"Error: couldn't send the chart: {e}." - await asyncio.to_thread(ctx.store.save, sent) - log.info("Sent a %s chart (%d bytes): %s", kind, len(png), title[:60]) - return "Chart sent to the chat. Add one or two sentences at most; don't repeat the numbers." + report = json.loads(await server.call("get_performance_report", { + "portfolio_id": pid, "start_date": today, "end_date": today}))["report"] + names = {p.get("id"): p.get("name") for p in + json.loads(await server.call("list_portfolios", {})).get("portfolios", []) if isinstance(p, dict)} + except (ValueError, KeyError, TypeError, AttributeError): + return "Error: couldn't read that portfolio's holdings." + groups = group_holdings(table_records(report.get("holdings")), by) + if not groups: + return "Error: that portfolio has no holdings with a value." + currency, total = report.get("currency") or "", sum(v for _, v in groups) + cash = sum(c.get("value") or 0 for c in report.get("cash_accounts") or [] if isinstance(c, dict)) + subtitle = f"{currency} {total:,.0f} in holdings" + (f", {cash:,.0f} cash" if cash > 0 else "") + title = f"{names.get(pid) or f'Portfolio {pid}'} by {by}" + sent = await _send(ctx, msg, kind, title, [{"label": k, "value": v} for k, v in groups], subtitle, "") + if sent.startswith("Error"): + return sent + top = ", ".join(f"{k} {v / total * 100:.1f}% ({render.short(v)})" for k, v in groups[:8]) + return f"Chart sent to the chat ({by}: {top}). Add one or two sentences at most; don't repeat the numbers." - return TOOL, send + return PORTFOLIO_TOOL, send diff --git a/lib/prompts.py b/lib/prompts.py index f96ec63..e5f3174 100644 --- a/lib/prompts.py +++ b/lib/prompts.py @@ -94,9 +94,11 @@ def chat_prompt(*, transcript: str, sender: str, private: bool, reply_quote: str return head + tail + down -CHART_TAIL = (" They want a picture, not a list: once you have the numbers, draw it with send_chart (a pie for " - "shares of a whole such as sectors, industries or holdings, a bar chart for values per category or " - "period; add up the holdings' value by the field asked for) and reply with one or two sentences.") +CHART_TAIL = (" They want a picture, not a list: once you have the numbers, draw it. For a portfolio's sectors, " + "industries, holdings or markets call portfolio_chart (it fetches and adds up the holdings itself; " + "sector and industry are different fields, so pass exactly the one asked for), once per portfolio " + "asked about. For anything else use send_chart (a pie for shares of a whole, a bar chart for values " + "per category or period). Then reply with one or two sentences.") PRESET_PROMPT = ("This is a private chat with {sender}. They tapped the \"{label}\" button, which " "asks for: {ask}. It's now {now}. Search {what} for current information, and do " diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index 90eecfb..a494b3a 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -255,6 +255,8 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s chart = bool(CHART_WORDS.search(trig.text)) and charts.available() if chart: extra.append(charts.tool(ctx, msg)) + if admin and any(s.cfg.get("gate") == "portfolio" for s in chosen.servers): + extra.append(charts.portfolio_tool(ctx, msg)) if chosen.simple: # the model must not be told it has no tools chosen = Route([], False) if extra: diff --git a/tests/test_charts.py b/tests/test_charts.py index d55772b..a75604b 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -1,4 +1,5 @@ import io +import json import pytest from telegram.error import TelegramError @@ -9,7 +10,7 @@ from lib.features.charts import render from .conftest import FakeBot, user_msg -from .fakes import ScriptedBackend, registry +from .fakes import FakeMcp, ScriptedBackend, registry PNG = b"\x89PNG\r\n\x1a\n" SECTORS = [{"label": "Semis", "value": 520.0}, {"label": "ETFs", "value": 250.0}, {"label": "Software", "value": 150.0}, @@ -93,3 +94,67 @@ def explode(*a, **kw): monkeypatch.setattr(render, "draw", explode) assert "answer in text instead" in await send({"kind": "pie", "title": "t", "items": SECTORS}) + + +HOLDINGS = {"columns": ["code", "market", "name", "sector", "industry", "value", "quantity"], "rows": [ + ["CRWD", "NASDAQ", "Crowdstrike", "Technology Services", "Packaged Software", 90.0, 1], + ["S", "NYSE", "SentinelOne", "Technology Services", "Packaged Software", 10.0, 1], + ["NVDA", "NASDAQ", "Nvidia", "Electronic Technology", "Semiconductors", 170.0, 1], + ["TSM", "NYSE", "TSMC", "Electronic Technology", "Semiconductors", 50.0, 1], + ["SMH", "NASDAQ", "VanEck Semi", None, "Investment Trusts/Mutual Funds", 80.0, 1], + ["OLD", "ASX", "Sold", "Health", "Biotech", 0.0, 0]]} + + +class Sharesight(FakeMcp): + async def call(self, tool, args): + self.calls.append((tool, args)) + if tool == "list_portfolios": + return json.dumps({"portfolios": [{"id": 7, "name": "Main"}]}) + return json.dumps({"report": {"currency": "AUD", "holdings": HOLDINGS, + "cash_accounts": [{"name": "Cash", "value": 25.0, "currency": "AUD"}]}}) + + +def portfolio_ctx(env, store): + bot = FakeBot() + ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) + return Ctx(config.load(env), store, ScriptedBackend([]), registry(ss), bot), bot, ss + + +def test_holdings_are_grouped_by_exactly_the_field_asked_for(): + rows = [dict(zip(HOLDINGS["columns"], r, strict=True)) for r in HOLDINGS["rows"]] + assert charts.group_holdings(rows, "sector") == [("Electronic Technology", 220.0), ("Technology Services", 100.0), + ("Unclassified", 80.0)] # the sold holding (value 0) is skipped + assert charts.group_holdings(rows, "industry")[0] == ("Semiconductors", 220.0) + assert charts.group_holdings(rows, "industry")[1] == ("Packaged Software", 100.0) + assert charts.group_holdings(rows, "holding")[0] == ("NVDA", 170.0) + + +async def test_portfolio_chart_groups_by_sector_not_industry_and_names_the_portfolio(env, store, monkeypatch): + ctx, bot, ss = portfolio_ctx(env, store) + drawn = {} + real = render.draw + + def spy(kind, title, items, subtitle=""): + drawn.update(kind=kind, title=title, items=items, subtitle=subtitle) + return real(kind, title, items, subtitle) + + monkeypatch.setattr(render, "draw", spy) + msg = user_msg(bot, "@stockbot my portfolios by sector", mid=5) + defn, send = charts.portfolio_tool(ctx, msg) + out = await send({"portfolio_id": 7, "by": "sector"}) + assert defn.name == "portfolio_chart" and out.startswith("Chart sent to the chat (sector: Electronic Technology 55.0%") + assert drawn["title"] == "Main by sector" and drawn["kind"] == "pie" + assert [i["label"] for i in drawn["items"]] == ["Electronic Technology", "Technology Services", "Unclassified"] + assert "AUD 400 in holdings, 25 cash" in drawn["subtitle"] + assert len(bot.photos) == 1 and bot.photos[0]["reply_to_message_id"] == 5 + assert ss.calls[0][1]["start_date"] == ss.calls[0][1]["end_date"] and "_keep_sales" not in ss.calls[0][1] # current holdings + + +async def test_portfolio_chart_refuses_bad_input_as_text(env, store): + ctx, bot, _ = portfolio_ctx(env, store) + _, send = charts.portfolio_tool(ctx, user_msg(bot, "chart", mid=1)) + assert "by must be one of" in await send({"portfolio_id": 7, "by": "colour"}) + assert "portfolio_id" in await send({"by": "sector"}) + none_ctx = Ctx(config.load(env), store, ScriptedBackend([]), registry(), bot) + assert "isn't connected" in await charts.portfolio_tool(none_ctx, user_msg(bot, "chart", mid=2))[1]({"portfolio_id": 7, "by": "sector"}) + assert bot.photos == [] diff --git a/tests/test_handlers.py b/tests/test_handlers.py index a4d3914..af872b2 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -125,12 +125,21 @@ async def test_the_chart_tool_is_offered_only_when_asked_for_a_chart(env, store, assert "send_chart" not in backend.seen[0]["tools"] and "send_chart" in backend.seen[1]["tools"] assert "send_chart" in backend.seen[2]["tools"] # also on the route that would otherwise have no tools assert "Local tools offered: send_chart" in caplog.text - assert "draw it with send_chart" not in backend.seen[0]["prompt"] and "draw it with send_chart" in backend.seen[1]["prompt"] + assert "They want a picture" not in backend.seen[0]["prompt"] and "They want a picture" in backend.seen[1]["prompt"] monkeypatch.setattr(charts, "available", lambda: False) # Pillow not installed: the bot just doesn't offer it await run(h, user_msg(ctx.bot, "@stockbot pie chart please", mid=4000)) assert "send_chart" not in backend.seen[3]["tools"] +async def test_portfolio_chart_is_offered_for_a_portfolio_chart_request_to_admins_only(env, store): + sharesight = FakeMcp("sharesight", gate="portfolio", tools=("list_portfolios", "get_performance_report")) + ctx, backend, h = make_ctx(env, store, [step("a"), step("b")], servers=[sharesight], ADMIN_ONLY="on", ADMIN_CHAT_IDS="5") + await run(h, user_msg(ctx.bot, "@stockbot my portfolios by sector", mid=1, user_id=5)) + assert {"send_chart", "portfolio_chart", "period_returns"} <= set(backend.seen[0]["tools"]) + await run(h, user_msg(ctx.bot, "@stockbot my portfolios by sector", mid=2000, user_id=9)) + assert len(backend.seen) == 1 # a non-admin gets the "admins only" message, not a chart + + async def test_portfolio_question_reminds_the_model_to_fetch_fresh_figures(env, store): sharesight = FakeMcp("sharesight", gate="portfolio", tools=("list_portfolios",)) ctx, backend, h = make_ctx(env, store, [step("a"), step("b")], servers=[sharesight]) diff --git a/tests/test_history.py b/tests/test_history.py index 08facc9..2b49c16 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -50,5 +50,6 @@ def test_chat_prompt_asks_for_a_picture_only_when_the_chart_tool_is_offered(): from lib.prompts import chat_prompt base = dict(transcript="t", sender="Bob", private=False, reply_quote=None, msg_id=3, now="now") plain, chart = chat_prompt(**base), chat_prompt(**base, chart=True, fresh=True) - assert "send_chart" not in plain and "draw it with send_chart" in chart + assert "send_chart" not in plain and "portfolio_chart" not in plain + assert "They want a picture" in chart and "portfolio_chart" in chart and "send_chart" in chart assert chart.index("Get current figures") < chart.index("They want a picture") # after the portfolio tail From f3327abde05efeed0c12290d83960abafc88e1b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 01:30:45 +0000 Subject: [PATCH 32/34] Charts: the answer is the photo's caption (one message or one album) instead of a separate message Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- README.md | 3 +- lib/features/charts/__init__.py | 67 ++++++++++------------- lib/prompts.py | 3 +- lib/telegram/handlers.py | 16 +++--- lib/telegram/send.py | 44 ++++++++++++++- tests/conftest.py | 11 ++++ tests/test_charts.py | 45 +++++++--------- tests/test_handlers.py | 15 ++++++ tests/test_send.py | 96 +++++++++++++++++++++++++++++++++ 9 files changed, 224 insertions(+), 76 deletions(-) create mode 100644 tests/test_send.py diff --git a/README.md b/README.md index 5b18f8a..ec87514 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,8 @@ unaffected. With nobody in `ADMIN_CHAT_IDS` and no known group, nobody can DM th ## Charts Ask for a "pie chart", "bar chart", "chart" or "graph", to "visualise" something, or for a breakdown "by sector" or "by industry", and the bot draws a real picture (a PNG) and posts it as a -reply, using numbers it got from its data tools: a pie for shares of a whole (a portfolio's sectors, industries, holdings or markets), a +reply, with its answer as the picture's caption (one message; several charts go as one album; an answer +over Telegram's 1,024-character caption limit follows as a second message), using numbers it got from its data tools: a pie for shares of a whole (a portfolio's sectors, industries, holdings or markets), a horizontal bar chart for values per category or period (monthly gains, negatives in red). It is drawn with Pillow (`pip install -r requirements.txt`); without it the bot says so at start-up and answers in text instead. There is nothing to configure: the tools are only offered to the model when the message asks for a chart. For a diff --git a/lib/features/charts/__init__.py b/lib/features/charts/__init__.py index 7e3790e..5dbb5fc 100644 --- a/lib/features/charts/__init__.py +++ b/lib/features/charts/__init__.py @@ -1,17 +1,12 @@ -"""send_chart: draw a pie or bar chart from numbers the model got from its tools and post it in the chat.""" +"""send_chart / portfolio_chart: draw a pie or bar chart; the picture goes out with the answer as its caption.""" from __future__ import annotations import asyncio -import html import json import logging from datetime import datetime -from telegram import Message -from telegram.constants import ParseMode -from telegram.error import TelegramError - from ...context import Ctx from ...mcp.results import table_records from ...mcp.schema import ToolDef @@ -19,29 +14,29 @@ log = logging.getLogger("bot") -CAPTION_MAX = 900 +READY = ("Chart ready: it will be attached to your reply, and your reply becomes its caption. Write that reply now " + "in a few sentences at most, without repeating the numbers.") TOOL = ToolDef( "send_chart", "Draw a pie chart (shares of a whole, e.g. sectors) or a horizontal bar chart (values per category or per " - "period, negatives allowed) and post the picture in this chat. Use it whenever someone asks for a pie chart, " - "bar chart, chart or graph. Every value must come from tool results, never invented; group categories " - "yourself if you want different slices. Call it once per chart, then write one or two sentences, not the " - "numbers again.", + "period, negatives allowed). The picture is posted together with your reply, which becomes its caption. Use it " + "whenever someone asks for a pie chart, bar chart, chart or graph. Every value must come from tool results, " + "never invented; group categories yourself if you want different slices. Call it once per chart, then write " + "one or two sentences, not the numbers again.", {"type": "object", "required": ["kind", "title", "items"], "properties": { "kind": {"type": "string", "enum": ["pie", "bar"]}, "title": {"type": "string", "description": "Short title shown on the picture"}, "items": {"type": "array", "maxItems": 200, "items": {"type": "object", "required": ["label", "value"], "properties": { "label": {"type": "string"}, "value": {"type": "number"}}}}, - "subtitle": {"type": "string", "description": "Optional line under the title, e.g. the total"}, - "caption": {"type": "string", "description": "Optional caption under the picture"}}}) + "subtitle": {"type": "string", "description": "Optional line under the title, e.g. the total"}}}) def available() -> bool: return render.available() -async def _send(ctx: Ctx, msg: Message, kind: str, title: str, items: object, subtitle: str, caption: str) -> str: - """Draw the chart and post it as a reply to `msg`; "Error: ..." text if that fails, else "sent".""" +async def _draw(images: list[bytes], kind: str, title: str, items: object, subtitle: str) -> str: + """Draw the chart and queue it for the reply; "Error: ..." text if that fails, else the READY note.""" try: png = await asyncio.to_thread(render.draw, kind, title, items, subtitle) except render.ChartError as e: @@ -49,24 +44,17 @@ async def _send(ctx: Ctx, msg: Message, kind: str, title: str, items: object, su except Exception as e: # a drawing bug must not sink the answer: the model falls back to text log.exception("Chart drawing failed") return f"Error: couldn't draw the chart ({type(e).__name__}); answer in text instead." - caption = html.escape(caption.strip())[:CAPTION_MAX] - try: - sent = await ctx.bot.send_photo(msg.chat_id, photo=png, caption=caption or None, parse_mode=ParseMode.HTML, - reply_to_message_id=msg.message_id, disable_notification=True) - except TelegramError as e: - return f"Error: couldn't send the chart: {e}." - await asyncio.to_thread(ctx.store.save, sent) - log.info("Sent a %s chart (%d bytes): %s", kind, len(png), title[:60]) - return "sent" + images.append(png) + log.info("Drew a %s chart (%d bytes): %s", kind, len(png), title[:60]) + return READY -def tool(ctx: Ctx, msg: Message): - """(definition, function) of send_chart, posting as a reply to `msg`.""" +def tool(ctx: Ctx, images: list[bytes]): + """(definition, function) of send_chart; the picture is appended to `images` for the reply to carry.""" async def send(args: dict) -> str: - out = await _send(ctx, msg, str(args.get("kind", "")).lower(), str(args.get("title", "")).strip(), - args.get("items"), str(args.get("subtitle") or ""), str(args.get("caption") or "")) - return out if out.startswith("Error") else "Chart sent to the chat. Add one or two sentences at most; don't repeat the numbers." + return await _draw(images, str(args.get("kind", "")).lower(), str(args.get("title", "")).strip(), + args.get("items"), str(args.get("subtitle") or "")) return TOOL, send @@ -76,10 +64,11 @@ async def send(args: dict) -> str: PORTFOLIO_TOOL = ToolDef( "portfolio_chart", "Draw a chart of one Sharesight portfolio's current holdings grouped by sector, industry, holding, market or " - "type, and post it in this chat. The bot fetches and adds up the holdings itself, so use this (not " - "send_chart) for 'my portfolio by sector' and the like. Sector and industry are different Sharesight " - "fields (Technology Services is a sector, Packaged Software an industry): pass exactly the one asked for. " - "Call it once per portfolio asked about; portfolio_id comes from list_portfolios.", + "type. The picture is posted together with your reply, which becomes its caption. The bot fetches and adds up " + "the holdings itself, so use this (not send_chart) for 'my portfolio by sector' and the like. Sector and " + "industry are different Sharesight fields (Technology Services is a sector, Packaged Software an industry): " + "pass exactly the one asked for. Call it once per portfolio asked about; portfolio_id comes from " + "list_portfolios.", {"type": "object", "required": ["portfolio_id", "by"], "properties": { "portfolio_id": {"type": "number"}, "by": {"type": "string", "enum": list(GROUPS)}, @@ -98,8 +87,8 @@ def group_holdings(holdings: list[dict], by: str) -> list[tuple[str, float]]: return sorted(totals.items(), key=lambda kv: -kv[1]) -def portfolio_tool(ctx: Ctx, msg: Message): - """(definition, function) of portfolio_chart, posting as a reply to `msg`.""" +def portfolio_tool(ctx: Ctx, images: list[bytes]): + """(definition, function) of portfolio_chart; the picture is appended to `images` for the reply to carry.""" async def send(args: dict) -> str: server = next((s for s in ctx.registry.up() if "get_performance_report" in s.fn_names.values()), None) @@ -127,10 +116,10 @@ async def send(args: dict) -> str: cash = sum(c.get("value") or 0 for c in report.get("cash_accounts") or [] if isinstance(c, dict)) subtitle = f"{currency} {total:,.0f} in holdings" + (f", {cash:,.0f} cash" if cash > 0 else "") title = f"{names.get(pid) or f'Portfolio {pid}'} by {by}" - sent = await _send(ctx, msg, kind, title, [{"label": k, "value": v} for k, v in groups], subtitle, "") - if sent.startswith("Error"): - return sent + out = await _draw(images, kind, title, [{"label": k, "value": v} for k, v in groups], subtitle) + if out != READY: + return out top = ", ".join(f"{k} {v / total * 100:.1f}% ({render.short(v)})" for k, v in groups[:8]) - return f"Chart sent to the chat ({by}: {top}). Add one or two sentences at most; don't repeat the numbers." + return f"{out} ({by}: {top})" return PORTFOLIO_TOOL, send diff --git a/lib/prompts.py b/lib/prompts.py index e5f3174..f93dfe9 100644 --- a/lib/prompts.py +++ b/lib/prompts.py @@ -98,7 +98,8 @@ def chat_prompt(*, transcript: str, sender: str, private: bool, reply_quote: str "industries, holdings or markets call portfolio_chart (it fetches and adds up the holdings itself; " "sector and industry are different fields, so pass exactly the one asked for), once per portfolio " "asked about. For anything else use send_chart (a pie for shares of a whole, a bar chart for values " - "per category or period). Then reply with one or two sentences.") + "per category or period). Then reply with one or two sentences: the picture is posted with your reply, " + "which becomes its caption.") PRESET_PROMPT = ("This is a private chat with {sender}. They tapped the \"{label}\" button, which " "asks for: {ask}. It's now {now}. Search {what} for current information, and do " diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index a494b3a..d5d79bb 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -25,7 +25,7 @@ from ..tickers import link_tickers from .access import is_admin from .draft import Draft, start_typing -from .send import reply_chunks, reply_html +from .send import reply_chunks, reply_html, reply_with_photos log = logging.getLogger("bot") PORTFOLIO_ADMINS_ONLY = "Portfolio data is for admins only." @@ -204,8 +204,9 @@ async def _answer(self, msg: Message, trig: Trigger) -> None: await draft.start() else: typing = await start_typing(ctx.bot, msg.chat_id) # a Telegram hiccup is not fatal + images: list[bytes] = [] # charts drawn while answering; they go out with the reply try: - html_text = await self._generate(msg, trig, draft) + html_text = await self._generate(msg, trig, draft, images) except Exception as e: log.exception("Reply failed") html_text = ("Sorry, something went wrong:\n" @@ -216,9 +217,12 @@ async def _answer(self, msg: Message, trig: Trigger) -> None: else: typing.cancel() markup = dm_buttons.keyboard() if trig.private and st.dm_buttons else None - await reply_chunks(msg, html_text, ctx.store, quote=not trig.private, last_markup=markup) + if images: + await reply_with_photos(msg, images, html_text, ctx.store, quote=not trig.private, last_markup=markup) + else: + await reply_chunks(msg, html_text, ctx.store, quote=not trig.private, last_markup=markup) - async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> str: + async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None, images: list[bytes]) -> str: ctx, st = self.ctx, self.ctx.st await ctx.registry.wait_started(config.MCP_STARTUP_WAIT) preset = dm_buttons.preset_for(trig.text) if trig.private and st.dm_buttons else None @@ -254,9 +258,9 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> s chosen = Route([], False) chart = bool(CHART_WORDS.search(trig.text)) and charts.available() if chart: - extra.append(charts.tool(ctx, msg)) + extra.append(charts.tool(ctx, images)) if admin and any(s.cfg.get("gate") == "portfolio" for s in chosen.servers): - extra.append(charts.portfolio_tool(ctx, msg)) + extra.append(charts.portfolio_tool(ctx, images)) if chosen.simple: # the model must not be told it has no tools chosen = Route([], False) if extra: diff --git a/lib/telegram/send.py b/lib/telegram/send.py index d7fa227..f3f173d 100644 --- a/lib/telegram/send.py +++ b/lib/telegram/send.py @@ -5,9 +5,9 @@ import asyncio import logging -from telegram import LinkPreviewOptions, Message +from telegram import InputMediaPhoto, LinkPreviewOptions, Message from telegram.constants import ParseMode -from telegram.error import BadRequest +from telegram.error import BadRequest, TelegramError from ..store import Store from ..textfmt import is_parse_error, plain_text, split_html @@ -15,6 +15,8 @@ log = logging.getLogger("bot") # Otherwise a list of Yahoo-linked tickers gets a big preview card under it. NO_PREVIEW = LinkPreviewOptions(is_disabled=True) +CAPTION_MAX = 1024 # Telegram's limit on a photo's caption (characters of text, not markup) +ALBUM_MAX = 10 # photos in one album async def send_html(bot, chat_id: int, text: str, reply_markup=None): @@ -55,3 +57,41 @@ async def reply_chunks(msg: Message, html_text: str, store: Store, *, quote: boo markup = last_markup if i == len(chunks) - 1 else None sent = await reply_html(msg, chunk, quote=quote, reply_markup=markup) await asyncio.to_thread(store.save, sent) + + +async def reply_with_photos(msg: Message, images: list[bytes], html_text: str, store: Store, *, quote: bool = True, + last_markup=None) -> None: + """Reply with pictures and the answer as the caption of the first one (one message, or one album). An answer + too long for a caption goes out as normal text after the pictures. If Telegram refuses the pictures the + answer is still sent as text.""" + caption = html_text if len(plain_text(html_text)) <= CAPTION_MAX else None + try: + for start in range(0, len(images), ALBUM_MAX): + batch = images[start:start + ALBUM_MAX] + text = caption if start == 0 else None + for sent in await _send_photos(msg, batch, text, quote and start == 0, + last_markup if caption and start + ALBUM_MAX >= len(images) else None): + await asyncio.to_thread(store.save, sent) + except TelegramError as e: + log.warning("Couldn't send the chart(s), sending the answer as text: %s", e) + await reply_chunks(msg, html_text, store, quote=quote, last_markup=last_markup) + return + if caption is None: + await reply_chunks(msg, html_text, store, quote=False, last_markup=last_markup) + + +async def _send_photos(msg: Message, batch: list[bytes], caption: str | None, quote: bool, markup) -> list[Message]: + """One photo, or an album of 2-10; the caption (Telegram HTML, plain text if it is rejected) rides on the first.""" + for text, mode in ((caption, ParseMode.HTML), (plain_text(caption) if caption else None, None)): + try: + if len(batch) == 1: + return [await msg.reply_photo(batch[0], caption=text, parse_mode=mode, do_quote=quote, + disable_notification=True, reply_markup=markup)] + media = [InputMediaPhoto(img, caption=text if i == 0 else None, parse_mode=mode if i == 0 else None) + for i, img in enumerate(batch)] + return list(await msg.reply_media_group(media, do_quote=quote, disable_notification=True)) + except BadRequest as e: + if not is_parse_error(e) or caption is None or mode is None: + raise + log.warning("Bad HTML from the model, sending the caption as plain text: %s", e) + raise AssertionError("unreachable") diff --git a/tests/conftest.py b/tests/conftest.py index 46d9cc9..bbe452d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -100,6 +100,17 @@ async def reply_text(text, **rkw): return bot._msg(chat_id, text) m.reply_text = reply_text + m.photo_replies = [] + + async def reply_photo(photo, **rkw): + m.photo_replies.append({"photo": photo, **rkw}) + return bot._msg(chat_id, rkw.get("caption") or "[photo]") + + async def reply_media_group(media, **rkw): + m.photo_replies.append({"album": media, **rkw}) + return [bot._msg(chat_id, (media[0].caption or "[photo]") if i == 0 else "[photo]") for i, _ in enumerate(media)] + + m.reply_photo, m.reply_media_group = reply_photo, reply_media_group m.parse_entities = lambda types=None: {} m.parse_caption_entities = lambda types=None: {} return m diff --git a/tests/test_charts.py b/tests/test_charts.py index a75604b..a7f0a9b 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -2,14 +2,13 @@ import json import pytest -from telegram.error import TelegramError from lib import config from lib.context import Ctx from lib.features import charts from lib.features.charts import render -from .conftest import FakeBot, user_msg +from .conftest import FakeBot from .fakes import FakeMcp, ScriptedBackend, registry PNG = b"\x89PNG\r\n\x1a\n" @@ -63,37 +62,29 @@ def test_short_values_and_long_labels(): assert len(label) == render.LABEL_CHARS and label.endswith("…") -async def test_the_tool_sends_one_photo_as_a_reply_and_saves_it(env, store): - bot = FakeBot() +async def test_the_tool_queues_the_picture_for_the_reply_and_sends_nothing_itself(env, store): + bot, images = FakeBot(), [] ctx = Ctx(config.load(env), store, ScriptedBackend([]), registry(), bot) - msg = user_msg(bot, "@stockbot pie chart of my sectors", mid=7) - defn, send = charts.tool(ctx, msg) - out = await send({"kind": "pie", "title": "Sectors", "items": SECTORS, "caption": "A test & more"}) - assert defn.name == "send_chart" and out.startswith("Chart sent") - photo = bot.photos[0] - assert photo["bytes"].startswith(PNG) and photo["reply_to_message_id"] == 7 and photo["chat_id"] == -100 - assert photo["caption"] == "A <b>test</b> & more" and photo["disable_notification"] is True - assert any("A <b>" in r.text for r in store.history(-100, 20)) # the sent message is in history + defn, send = charts.tool(ctx, images) + out = await send({"kind": "pie", "title": "Sectors", "items": SECTORS}) + assert defn.name == "send_chart" and out == charts.READY and "becomes its caption" in out + assert len(images) == 1 and images[0].startswith(PNG) + assert bot.photos == [] and bot.sent == [] # the handler sends it, with the answer as the caption + assert "caption" not in defn.parameters["properties"] async def test_the_tool_returns_errors_as_text_so_the_model_can_fall_back(env, store, monkeypatch): - bot = FakeBot() + bot, images = FakeBot(), [] ctx = Ctx(config.load(env), store, ScriptedBackend([]), registry(), bot) - _, send = charts.tool(ctx, user_msg(bot, "chart", mid=1)) + _, send = charts.tool(ctx, images) assert (await send({"kind": "pie", "title": "t", "items": []})).startswith("Error: give a list of items") - assert bot.photos == [] - - async def broken(*a, **kw): - raise TelegramError("Bad Request: not enough rights") - - monkeypatch.setattr(bot, "send_photo", broken) - assert "couldn't send the chart" in await send({"kind": "bar", "title": "t", "items": SECTORS}) def explode(*a, **kw): raise RuntimeError("font missing") monkeypatch.setattr(render, "draw", explode) assert "answer in text instead" in await send({"kind": "pie", "title": "t", "items": SECTORS}) + assert images == [] HOLDINGS = {"columns": ["code", "market", "name", "sector", "industry", "value", "quantity"], "rows": [ @@ -139,22 +130,22 @@ def spy(kind, title, items, subtitle=""): return real(kind, title, items, subtitle) monkeypatch.setattr(render, "draw", spy) - msg = user_msg(bot, "@stockbot my portfolios by sector", mid=5) - defn, send = charts.portfolio_tool(ctx, msg) + images = [] + defn, send = charts.portfolio_tool(ctx, images) out = await send({"portfolio_id": 7, "by": "sector"}) - assert defn.name == "portfolio_chart" and out.startswith("Chart sent to the chat (sector: Electronic Technology 55.0%") + assert defn.name == "portfolio_chart" and "(sector: Electronic Technology 55.0%" in out and out.startswith(charts.READY) assert drawn["title"] == "Main by sector" and drawn["kind"] == "pie" assert [i["label"] for i in drawn["items"]] == ["Electronic Technology", "Technology Services", "Unclassified"] assert "AUD 400 in holdings, 25 cash" in drawn["subtitle"] - assert len(bot.photos) == 1 and bot.photos[0]["reply_to_message_id"] == 5 + assert len(images) == 1 and bot.photos == [] assert ss.calls[0][1]["start_date"] == ss.calls[0][1]["end_date"] and "_keep_sales" not in ss.calls[0][1] # current holdings async def test_portfolio_chart_refuses_bad_input_as_text(env, store): ctx, bot, _ = portfolio_ctx(env, store) - _, send = charts.portfolio_tool(ctx, user_msg(bot, "chart", mid=1)) + _, send = charts.portfolio_tool(ctx, []) assert "by must be one of" in await send({"portfolio_id": 7, "by": "colour"}) assert "portfolio_id" in await send({"by": "sector"}) none_ctx = Ctx(config.load(env), store, ScriptedBackend([]), registry(), bot) - assert "isn't connected" in await charts.portfolio_tool(none_ctx, user_msg(bot, "chart", mid=2))[1]({"portfolio_id": 7, "by": "sector"}) + assert "isn't connected" in await charts.portfolio_tool(none_ctx, [])[1]({"portfolio_id": 7, "by": "sector"}) assert bot.photos == [] diff --git a/tests/test_handlers.py b/tests/test_handlers.py index af872b2..03a5c7a 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -131,6 +131,21 @@ async def test_the_chart_tool_is_offered_only_when_asked_for_a_chart(env, store, assert "send_chart" not in backend.seen[3]["tools"] +async def test_a_chart_request_is_answered_with_one_photo_whose_caption_is_the_answer(env, store): + args = '{"kind":"pie","title":"Sectors","items":[{"label":"Semis","value":3},{"label":"Software","value":1}]}' + ctx, backend, h = make_ctx(env, store, [step(calls=[("1", "send_chart", args)], finish="tool_calls"), + step("Semis are 75% of it.")]) + m = user_msg(ctx.bot, "@stockbot pie chart of my sectors", mid=1) + await run(h, m) + assert len(m.photo_replies) == 1 and m.replies == [] # one message: the picture, with the answer as its caption + assert m.photo_replies[0]["caption"] == "Semis are 75% of it." and m.photo_replies[0]["photo"].startswith(b"\x89PNG") + assert any("Semis are" in r.text for r in store.history(-100, 20)) + plain = user_msg(ctx.bot, "@stockbot how is NVDA?", mid=2000) + ctx.backend.script.append(step("NVDA is fine")) + await run(h, plain) + assert plain.photo_replies == [] and "is fine" in plain.replies[0]["text"] + + async def test_portfolio_chart_is_offered_for_a_portfolio_chart_request_to_admins_only(env, store): sharesight = FakeMcp("sharesight", gate="portfolio", tools=("list_portfolios", "get_performance_report")) ctx, backend, h = make_ctx(env, store, [step("a"), step("b")], servers=[sharesight], ADMIN_ONLY="on", ADMIN_CHAT_IDS="5") diff --git a/tests/test_send.py b/tests/test_send.py new file mode 100644 index 0000000..920d782 --- /dev/null +++ b/tests/test_send.py @@ -0,0 +1,96 @@ +import pytest +from telegram.error import BadRequest, TelegramError + +from lib.telegram.send import CAPTION_MAX, reply_with_photos + +from .conftest import FakeBot, user_msg + +PNG = b"\x89PNG\r\n\x1a\nfake" + + +async def reply(bot, store, text, images, **kw): + msg = user_msg(bot, "chart please", mid=5) + await reply_with_photos(msg, images, text, store, **kw) + return msg + + +async def test_a_short_answer_is_the_caption_of_one_photo(store): + bot = FakeBot() + msg = await reply(bot, store, "Semis are 52%.", [PNG]) + assert len(msg.photo_replies) == 1 and msg.replies == [] + sent = msg.photo_replies[0] + assert sent["photo"] == PNG and sent["caption"] == "Semis are 52%." and sent["do_quote"] is True + assert sent["disable_notification"] is True + assert [r.text for r in store.history(-100, 20)] == ["Semis are 52%."] # saved, so follow-ups see it + + +async def test_several_charts_go_as_one_album_with_the_caption_on_the_first(store): + bot = FakeBot() + msg = await reply(bot, store, "Three portfolios.", [PNG, PNG, PNG]) + assert len(msg.photo_replies) == 1 and msg.replies == [] + album = msg.photo_replies[0]["album"] + assert [m.caption for m in album] == ["Three portfolios.", None, None] + + +async def test_more_than_ten_charts_split_into_albums_and_only_the_first_has_the_caption(store): + bot = FakeBot() + msg = await reply(bot, store, "Many.", [PNG] * 12) + first, second = msg.photo_replies + assert len(first["album"]) == 10 and first["album"][0].caption == "Many." and len(second["album"]) == 2 + assert second["album"][0].caption is None and second["do_quote"] is False + + +async def test_an_answer_too_long_for_a_caption_follows_as_text(store): + bot = FakeBot() + long = "word " * (CAPTION_MAX // 5 + 20) + msg = await reply(bot, store, long, [PNG]) + assert msg.photo_replies[0]["caption"] is None # the picture alone + assert msg.replies and "word" in msg.replies[0]["text"] # then the answer as a normal message + + +async def test_a_rejected_caption_is_sent_as_plain_text(store): + bot = FakeBot() + msg = user_msg(bot, "chart", mid=5) + real = msg.reply_photo + calls = [] + + async def picky(photo, **kw): + calls.append(kw.get("parse_mode")) + if kw.get("parse_mode"): + raise BadRequest("Can't parse entities: bad tag") + return await real(photo, **kw) + + msg.reply_photo = picky + await reply_with_photos(msg, [PNG], "Bad ", store) + assert len(calls) == 2 and calls[1] is None and msg.photo_replies[0]["caption"] == "Bad " + + +async def test_if_telegram_refuses_the_picture_the_answer_is_still_sent(store): + bot = FakeBot() + msg = user_msg(bot, "chart", mid=5) + + async def refuse(photo, **kw): + raise TelegramError("Bad Request: not enough rights to send photos") + + msg.reply_photo = refuse + await reply_with_photos(msg, [PNG], "The answer.", store) + assert msg.photo_replies == [] and [r["text"] for r in msg.replies] == ["The answer."] + + +async def test_other_bad_requests_are_not_swallowed_as_parse_errors(store): + bot = FakeBot() + msg = user_msg(bot, "chart", mid=5) + + async def broken(photo, **kw): + raise BadRequest("Wrong file identifier") + + msg.reply_photo = broken + await reply_with_photos(msg, [PNG], "Answer.", store) # falls back to text instead of crashing + assert [r["text"] for r in msg.replies] == ["Answer."] + + +@pytest.mark.parametrize("n", [1, 3]) +async def test_the_markup_rides_on_a_single_photo_only(store, n): + bot = FakeBot() + msg = await reply(bot, store, "Hi.", [PNG] * n, last_markup="KB") + assert (msg.photo_replies[0].get("reply_markup") == "KB") is (n == 1) From e9beafdf98729bda743125af20f5a692b1a11fa6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 01:55:13 +0000 Subject: [PATCH 33/34] Revert the pie/bar chart work (7c5ef92..f3327ab): back to the tree of 440515b Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- CLAUDE.md | 2 +- README.md | 11 --- lib/app.py | 4 +- lib/features/charts/__init__.py | 125 ------------------------- lib/features/charts/render.py | 158 -------------------------------- lib/llm/gate.py | 4 - lib/mcp/results.py | 1 - lib/prompts.py | 11 +-- lib/telegram/handlers.py | 32 ++----- lib/telegram/send.py | 44 +-------- requirements.txt | 2 +- tests/conftest.py | 16 ---- tests/test_app.py | 6 +- tests/test_charts.py | 151 ------------------------------ tests/test_fixtures.py | 2 +- tests/test_handlers.py | 42 --------- tests/test_history.py | 9 -- tests/test_mcp.py | 9 -- tests/test_runner.py | 11 --- tests/test_send.py | 96 ------------------- 20 files changed, 17 insertions(+), 719 deletions(-) delete mode 100644 lib/features/charts/__init__.py delete mode 100644 lib/features/charts/render.py delete mode 100644 tests/test_charts.py delete mode 100644 tests/test_send.py diff --git a/CLAUDE.md b/CLAUDE.md index 69c5b9c..2079044 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ MCP data tools and web search. User-facing docs are in `README.md`; this file is `openrouter.py` + `zai.py` (the only provider-specific code), `runner.py` (THE tool loop), `policy.py` (THE retry ladder), `gate.py` (which tools a message needs). - `prompts.py` all prompts. `ask.py` one entry point: route -> Request -> policy.ask -> usage -> NEEDS_TOOLS rerun. - `telegram/`: `handlers.py` (trigger -> request -> reply), `send.py`, `draft.py` (typing + streaming - drafts), `commands.py` (/credits), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `post.py`, `charts/` (local `send_chart` tool: Pillow pie/bar PNGs, offered only for chart requests), `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop). + drafts), `commands.py` (/credits), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop). - `app.py` wiring/startup; `context.py` `Ctx` (settings, store, backend, registry, bot). - `lib/` holds only what the bot needs to run. Developer helpers live in `tools/`: `report.py` (`make prompt-report`), `capture.py` (real MCP output for fixtures); the bot never imports them. diff --git a/README.md b/README.md index ec87514..f89fb58 100644 --- a/README.md +++ b/README.md @@ -188,17 +188,6 @@ gets "Portfolio data is for admins only." Holding-news DMs, buttons and down-ser unaffected. With nobody in `ADMIN_CHAT_IDS` and no known group, nobody can DM the bot or reach Sharesight: set `ADMIN_ONLY=off` to allow everyone. -## Charts -Ask for a "pie chart", "bar chart", "chart" or "graph", to "visualise" something, or for a breakdown "by sector" or "by industry", and the bot draws a real picture (a PNG) and posts it as a -reply, with its answer as the picture's caption (one message; several charts go as one album; an answer -over Telegram's 1,024-character caption limit follows as a second message), using numbers it got from its data tools: a pie for shares of a whole (a portfolio's sectors, industries, holdings or markets), a -horizontal bar chart for values per category or period (monthly gains, negatives in red). It is drawn with Pillow -(`pip install -r requirements.txt`); without it the bot says so at start-up and answers in text instead. There is -nothing to configure: the tools are only offered to the model when the message asks for a chart. For a -portfolio grouped by sector, industry, holding, market or type (admins only) the bot fetches and adds up the -holdings itself (`portfolio_chart`), so sector and industry stay distinct and the totals don't depend on the -model's arithmetic. - ## Caching Sharesight history Questions like "best day / week / month / quarter / year since inception" are answered by the `period_returns` tool (admins only), which asks Sharesight for one report per period (about 2.4 s each). Every **finished** period is diff --git a/lib/app.py b/lib/app.py index 6407b11..96f977d 100644 --- a/lib/app.py +++ b/lib/app.py @@ -28,7 +28,7 @@ from . import config from .context import Ctx -from .features import charts, dm_buttons, holding_news, post +from .features import dm_buttons, holding_news, post from .features.periods import backfill from .features.periods.windows import describe_scope, flush_scope from .llm.base import Backend @@ -212,8 +212,6 @@ def main(argv: Sequence[str] | None = None) -> None: log.info("Starting%s%s: %s on %s, reasoning %s, MCP servers: %s", f" instance {args.label}" if args.label else "", f" ({commit})" if commit else "", st.model, st.provider, st.reasoning or "default", ", ".join(ctx.registry.servers) or "none") - if not charts.available(): - log.warning("Pillow isn't installed, so pie and bar charts are off (pip install -r requirements.txt)") if st.period_backfill: log.info("Period backfill: on for %s", ", ".join(sorted(st.period_backfill))) app.run_polling(allowed_updates=allowed_updates(st)) diff --git a/lib/features/charts/__init__.py b/lib/features/charts/__init__.py deleted file mode 100644 index 5dbb5fc..0000000 --- a/lib/features/charts/__init__.py +++ /dev/null @@ -1,125 +0,0 @@ -"""send_chart / portfolio_chart: draw a pie or bar chart; the picture goes out with the answer as its caption.""" - -from __future__ import annotations - -import asyncio -import json -import logging -from datetime import datetime - -from ...context import Ctx -from ...mcp.results import table_records -from ...mcp.schema import ToolDef -from . import render - -log = logging.getLogger("bot") - -READY = ("Chart ready: it will be attached to your reply, and your reply becomes its caption. Write that reply now " - "in a few sentences at most, without repeating the numbers.") -TOOL = ToolDef( - "send_chart", - "Draw a pie chart (shares of a whole, e.g. sectors) or a horizontal bar chart (values per category or per " - "period, negatives allowed). The picture is posted together with your reply, which becomes its caption. Use it " - "whenever someone asks for a pie chart, bar chart, chart or graph. Every value must come from tool results, " - "never invented; group categories yourself if you want different slices. Call it once per chart, then write " - "one or two sentences, not the numbers again.", - {"type": "object", "required": ["kind", "title", "items"], "properties": { - "kind": {"type": "string", "enum": ["pie", "bar"]}, - "title": {"type": "string", "description": "Short title shown on the picture"}, - "items": {"type": "array", "maxItems": 200, "items": {"type": "object", "required": ["label", "value"], "properties": { - "label": {"type": "string"}, "value": {"type": "number"}}}}, - "subtitle": {"type": "string", "description": "Optional line under the title, e.g. the total"}}}) - - -def available() -> bool: - return render.available() - - -async def _draw(images: list[bytes], kind: str, title: str, items: object, subtitle: str) -> str: - """Draw the chart and queue it for the reply; "Error: ..." text if that fails, else the READY note.""" - try: - png = await asyncio.to_thread(render.draw, kind, title, items, subtitle) - except render.ChartError as e: - return f"Error: {e}." - except Exception as e: # a drawing bug must not sink the answer: the model falls back to text - log.exception("Chart drawing failed") - return f"Error: couldn't draw the chart ({type(e).__name__}); answer in text instead." - images.append(png) - log.info("Drew a %s chart (%d bytes): %s", kind, len(png), title[:60]) - return READY - - -def tool(ctx: Ctx, images: list[bytes]): - """(definition, function) of send_chart; the picture is appended to `images` for the reply to carry.""" - - async def send(args: dict) -> str: - return await _draw(images, str(args.get("kind", "")).lower(), str(args.get("title", "")).strip(), - args.get("items"), str(args.get("subtitle") or "")) - - return TOOL, send - - -# -- a portfolio grouped by sector, industry, holding...: the bot adds it up, not the model --------------- -GROUPS = {"sector": "sector", "industry": "industry", "holding": "code", "market": "market", "type": "type"} -PORTFOLIO_TOOL = ToolDef( - "portfolio_chart", - "Draw a chart of one Sharesight portfolio's current holdings grouped by sector, industry, holding, market or " - "type. The picture is posted together with your reply, which becomes its caption. The bot fetches and adds up " - "the holdings itself, so use this (not send_chart) for 'my portfolio by sector' and the like. Sector and " - "industry are different Sharesight fields (Technology Services is a sector, Packaged Software an industry): " - "pass exactly the one asked for. Call it once per portfolio asked about; portfolio_id comes from " - "list_portfolios.", - {"type": "object", "required": ["portfolio_id", "by"], "properties": { - "portfolio_id": {"type": "number"}, - "by": {"type": "string", "enum": list(GROUPS)}, - "kind": {"type": "string", "enum": ["pie", "bar"], "description": "Default pie"}}}) - - -def group_holdings(holdings: list[dict], by: str) -> list[tuple[str, float]]: - """Value per group (largest first), from slimmed Sharesight holdings; a missing field is "Unclassified".""" - field, totals = GROUPS[by], {} - for h in holdings: - value = h.get("value") - if not isinstance(value, (int, float)) or value <= 0: - continue - name = str(h.get(field) or "Unclassified") - totals[name] = totals.get(name, 0.0) + float(value) - return sorted(totals.items(), key=lambda kv: -kv[1]) - - -def portfolio_tool(ctx: Ctx, images: list[bytes]): - """(definition, function) of portfolio_chart; the picture is appended to `images` for the reply to carry.""" - - async def send(args: dict) -> str: - server = next((s for s in ctx.registry.up() if "get_performance_report" in s.fn_names.values()), None) - if server is None: - return "Error: the portfolio data source isn't connected." - by, kind = str(args.get("by", "")).lower(), str(args.get("kind") or "pie").lower() - if by not in GROUPS: - return f"Error: by must be one of {', '.join(GROUPS)}." - try: - pid = int(args.get("portfolio_id")) - except (TypeError, ValueError): - return "Error: give the portfolio_id (call list_portfolios first)." - today = datetime.now(ctx.st.tz).date().isoformat() - try: - report = json.loads(await server.call("get_performance_report", { - "portfolio_id": pid, "start_date": today, "end_date": today}))["report"] - names = {p.get("id"): p.get("name") for p in - json.loads(await server.call("list_portfolios", {})).get("portfolios", []) if isinstance(p, dict)} - except (ValueError, KeyError, TypeError, AttributeError): - return "Error: couldn't read that portfolio's holdings." - groups = group_holdings(table_records(report.get("holdings")), by) - if not groups: - return "Error: that portfolio has no holdings with a value." - currency, total = report.get("currency") or "", sum(v for _, v in groups) - cash = sum(c.get("value") or 0 for c in report.get("cash_accounts") or [] if isinstance(c, dict)) - subtitle = f"{currency} {total:,.0f} in holdings" + (f", {cash:,.0f} cash" if cash > 0 else "") - title = f"{names.get(pid) or f'Portfolio {pid}'} by {by}" - out = await _draw(images, kind, title, [{"label": k, "value": v} for k, v in groups], subtitle) - if out != READY: - return out - top = ", ".join(f"{k} {v / total * 100:.1f}% ({render.short(v)})" for k, v in groups[:8]) - return f"{out} ({by}: {top})" - - return PORTFOLIO_TOOL, send diff --git a/lib/features/charts/render.py b/lib/features/charts/render.py deleted file mode 100644 index 987618e..0000000 --- a/lib/features/charts/render.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Chart images (pie and horizontal bar) drawn with Pillow: no network, no config or cache directories.""" - -from __future__ import annotations - -import io -import math - -WIDTH = 1000 -SCALE = 2 # drawn at 2x and shrunk, which smooths the edges -PALETTE = ["#4E79A7", "#F28E2B", "#59A14F", "#E15759", "#76B7B2", "#EDC948", "#B07AA1", "#FF9DA7", "#9C755F", "#86BCB6"] -OTHER_COLOUR, INK, MUTED, BACKGROUND, NEGATIVE = "#A0A0A0", "#222222", "#666666", "#FFFFFF", "#C8553D" -MAX_ITEMS = 30 -MAX_SLICES = 10 # named slices before the rest become "Other" -OTHER_BELOW = 0.02 # slices under this share merge into "Other" -LABEL_CHARS = 28 - - -class ChartError(ValueError): - """The data can't be drawn as asked.""" - - -def available() -> bool: - try: - import PIL # noqa: F401 - except ImportError: - return False - return True - - -def short(value: float) -> str: - """12345 -> 12.3k, 2500000 -> 2.5M, 3.2 -> 3.2.""" - mag = abs(value) - if mag >= 1e6: - return f"{value / 1e6:.1f}M" - if mag >= 1e3: - return f"{value / 1e3:.1f}k" - return f"{value:.1f}" if mag < 10 and value != int(value) else f"{value:.0f}" - - -def clean(kind: str, items: object) -> list[tuple[str, float]]: - """Validated (label, value) pairs; a pie keeps only positive values.""" - if kind not in ("pie", "bar"): - raise ChartError("kind must be 'pie' or 'bar'") - if not isinstance(items, list) or not items: - raise ChartError("give a list of items with a label and a value each") - out = [] - for item in items: - try: - label, value = str(item["label"]).strip(), float(item["value"]) - except (TypeError, KeyError, ValueError): - raise ChartError("every item needs a label and a numeric value") from None - if not math.isfinite(value): - raise ChartError(f"the value for {label!r} isn't a number") - out.append((label if len(label) <= LABEL_CHARS else label[:LABEL_CHARS - 1] + "…", value)) - if kind == "pie": - out = [(label, value) for label, value in out if value > 0] - if not out: - raise ChartError("a pie chart needs at least one positive value") - if len(out) > (200 if kind == "pie" else MAX_ITEMS): - raise ChartError(f"too many items ({len(out)}); group them into at most {MAX_ITEMS}") - return out - - -def pie_slices(items: list[tuple[str, float]]) -> list[tuple[str, float, float, bool]]: - """(label, value, share, is_other) largest first; small slices and everything past MAX_SLICES merge.""" - total = sum(value for _, value in items) - ranked = sorted(items, key=lambda i: -i[1]) - named = [i for i in ranked[:MAX_SLICES] if i[1] / total >= OTHER_BELOW] - rest = sum(value for _, value in ranked[len(named):]) - slices = [(label, value, value / total, False) for label, value in named] - if rest > 0: - slices.append(("Other", rest, rest / total, True)) - return slices - - -def _font(size: int): - from PIL import ImageFont - try: - return ImageFont.load_default(size=size * SCALE) - except TypeError: # Pillow before 10.1 has only the small bitmap font - return ImageFont.load_default() - - -def _fit(draw, text: str, font, width: float) -> str: - """Text cut with an ellipsis to fit `width` (1x pixels).""" - if draw.textlength(text, font=font) <= width * SCALE: - return text - while text and draw.textlength(text + "…", font=font) > width * SCALE: - text = text[:-1] - return text.rstrip() + "…" - - -def draw(kind: str, title: str, items: list, subtitle: str = "") -> bytes: - """A PNG of the chart. Raises ChartError for data that can't be drawn.""" - pairs = clean(kind, items) - from PIL import Image, ImageDraw - - height = _pie_height(pairs) if kind == "pie" else 130 + 38 * len(pairs) - img = Image.new("RGB", (WIDTH * SCALE, height * SCALE), BACKGROUND) - d = ImageDraw.Draw(img) - s = SCALE - d.text((40 * s, 28 * s), _fit(d, title.strip() or "Chart", _font(36), WIDTH - 80), fill=INK, font=_font(36)) - if subtitle: - d.text((40 * s, 76 * s), _fit(d, subtitle, _font(24), WIDTH - 80), fill=MUTED, font=_font(24)) - (_draw_pie if kind == "pie" else _draw_bar)(d, pairs, height) - img = img.resize((WIDTH, height), Image.LANCZOS) - out = io.BytesIO() - img.save(out, "PNG", optimize=True) - return out.getvalue() - - -def _pie_height(pairs: list[tuple[str, float]]) -> int: - return max(600, 150 + 62 * len(pie_slices(pairs))) - - -def _draw_pie(d, pairs: list[tuple[str, float]], height: int) -> None: - s = SCALE - slices = pie_slices(pairs) - left, top, size = 40, 130, 420 - box = [left * s, top * s, (left + size) * s, (top + size) * s] - angle = -90.0 - for i, (_, _, share, is_other) in enumerate(slices): - colour = OTHER_COLOUR if is_other else PALETTE[i % len(PALETTE)] - end = angle + 360 * share - if share > 0.9999: - d.ellipse(box, fill=colour) - else: - d.pieslice(box, angle, end, fill=colour, outline=BACKGROUND, width=2 * s) - angle = end - label_font, detail_font = _font(26), _font(22) - for i, (label, value, share, is_other) in enumerate(slices): - y = 135 + 62 * i - colour = OTHER_COLOUR if is_other else PALETTE[i % len(PALETTE)] - d.rectangle([500 * s, (y + 4) * s, 524 * s, (y + 28) * s], fill=colour) - d.text((540 * s, y * s), _fit(d, label, label_font, 420), fill=INK, font=label_font) - d.text((540 * s, (y + 31) * s), f"{share * 100:.1f}% {short(value)}", fill=MUTED, font=detail_font) - - -def _draw_bar(d, pairs: list[tuple[str, float]], height: int) -> None: - s = SCALE - x_left, x_right = 300, 860 - top_max = max(0.0, max(v for _, v in pairs)) - low_max = max(0.0, -min(v for _, v in pairs)) - span = (top_max + low_max) or 1.0 - zero = x_left + (x_right - x_left) * low_max / span - label_font, value_font = _font(24), _font(22) - for i, (label, value) in enumerate(pairs): - y = 115 + 38 * i - d.text((40 * s, (y + 2) * s), _fit(d, label, label_font, 245), fill=INK, font=label_font) - length = (x_right - x_left) * abs(value) / span - x0, x1 = (zero, zero + length) if value >= 0 else (zero - length, zero) - d.rectangle([x0 * s, y * s, max(x1, x0 + 1) * s, (y + 28) * s], fill=PALETTE[0] if value >= 0 else NEGATIVE) - text = short(value) - if value >= 0: - d.text(((x1 + 8) * s, (y + 2) * s), text, fill=MUTED, font=value_font) - else: - d.text((x0 * s - 8 * s - d.textlength(text, font=value_font), (y + 2) * s), text, fill=MUTED, font=value_font) - d.line([zero * s, 108 * s, zero * s, (115 + 38 * len(pairs)) * s], fill=MUTED, width=s) diff --git a/lib/llm/gate.py b/lib/llm/gate.py index 4b9d7ab..b80dbc0 100644 --- a/lib/llm/gate.py +++ b/lib/llm/gate.py @@ -31,10 +31,6 @@ r"p&l|pnl|investments?|funds?|etfs?|trades?|watchlist)\b", re.I) PORTFOLIO_WEAK = re.compile( r"\b(performance|gains?|positions?|winners?|losers?|p&l|pnl|(?:out|under)perform\w*|ahead of|behind)\b", re.I) -CHART_WORDS = re.compile( # a request for a picture of the data - r"\b(pie|bar chart|chart|graph|plot|diagram|visuali[sz]e|visuali[sz]ation|breakdown|allocation)\b" - r"|\bby\s+(?:sectors?|industr(?:y|ies)|asset class(?:es)?|markets?|countr(?:y|ies)|currenc(?:y|ies)|regions?|holdings?|stocks?)\b", - re.I) LIVE_WORDS = re.compile( r"\b(news|today|latest|yesterday|tonight|this week|right now|breaking|announce\w*|who won|" r"score|weather|happening|headlines?|current\w*|updates?|recent\w*|just (now|in))\b", re.I) diff --git a/lib/mcp/results.py b/lib/mcp/results.py index b3d73be..e129382 100644 --- a/lib/mcp/results.py +++ b/lib/mcp/results.py @@ -78,7 +78,6 @@ def slim_holding(h): "currency": inst.get("currency_code"), "type": inst.get("friendly_instrument_description"), "sector": inst.get("sector_classification_name"), - "industry": inst.get("industry_classification_name"), } out.update({k: h.get(k) for k in HOLDING_KEEP}) if out.get("type") == "Ordinary Shares": # the default; only say when it's something else diff --git a/lib/prompts.py b/lib/prompts.py index f93dfe9..d9cb3da 100644 --- a/lib/prompts.py +++ b/lib/prompts.py @@ -69,7 +69,7 @@ def down_note(down: list[MCPServer]) -> str: def chat_prompt(*, transcript: str, sender: str, private: bool, reply_quote: str | None, msg_id: int, now: str, down: str = "", saver: bool = True, middle: str = "", - fresh: bool = False, chart: bool = False) -> str: + fresh: bool = False) -> str: """The user turn: transcript, the replied-to message, and the volatile tail (kept last so the cached prefix before it is unchanged).""" head = (f"Private one-to-one chat with {sender}, not the group. Transcript:\n{transcript}" @@ -89,18 +89,9 @@ def chat_prompt(*, transcript: str, sender: str, private: bool, reply_quote: str "for anything else asked, such as prices or a comparison with market data); don't reuse " "numbers or refusals from earlier replies. When comparing portfolios, or asked \"why\", " "write 2-3 short paragraphs on the tailwinds and headwinds of each portfolio, not a summary.") - if chart: # send_chart is on offer: the phrasing ("by sector", "visualise") asks for a picture - tail += CHART_TAIL return head + tail + down -CHART_TAIL = (" They want a picture, not a list: once you have the numbers, draw it. For a portfolio's sectors, " - "industries, holdings or markets call portfolio_chart (it fetches and adds up the holdings itself; " - "sector and industry are different fields, so pass exactly the one asked for), once per portfolio " - "asked about. For anything else use send_chart (a pie for shares of a whole, a bar chart for values " - "per category or period). Then reply with one or two sentences: the picture is posted with your reply, " - "which becomes its caption.") - PRESET_PROMPT = ("This is a private chat with {sender}. They tapped the \"{label}\" button, which " "asks for: {ask}. It's now {now}. Search {what} for current information, and do " "the whole job before replying.") diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index d5d79bb..fd6ba8a 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -15,9 +15,9 @@ from .. import config from ..ask import ask from ..context import Ctx -from ..features import charts, dm_buttons, holding_news, movers, periods, post +from ..features import dm_buttons, holding_news, movers, periods, post from ..history import compact, format_rows -from ..llm.gate import CHART_WORDS, Route, is_portfolio_question, route +from ..llm.gate import Route, is_portfolio_question, route from ..msgtext import describe, sender_name from ..prompts import MOVERS_EXPLAIN, PRESET_PROMPT, chat_prompt, down_note from ..store import is_dm @@ -25,7 +25,7 @@ from ..tickers import link_tickers from .access import is_admin from .draft import Draft, start_typing -from .send import reply_chunks, reply_html, reply_with_photos +from .send import reply_chunks, reply_html log = logging.getLogger("bot") PORTFOLIO_ADMINS_ONLY = "Portfolio data is for admins only." @@ -178,8 +178,7 @@ async def _gate_text(self, msg: Message, trig: Trigger) -> str: text += " " + compact(prev.text)[:FOLLOW_UP_CHARS] return text - async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: bool = False, - chart: bool = False) -> str: + async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: bool = False) -> str: ctx, st = self.ctx, self.ctx.st rows = await asyncio.to_thread(ctx.store.history, msg.chat_id, st.history_limit) transcript = format_rows(rows, ctx.self_name, st.tz, line_max=st.history_line_max, @@ -187,7 +186,7 @@ async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: return chat_prompt( transcript=transcript, sender=sender_name(msg), private=trig.private, reply_quote=self._quote(trig), msg_id=msg.message_id, now=ctx.now(), - down=down_note(ctx.registry.down()), saver=st.token_saver, middle=middle, fresh=fresh, chart=chart) + down=down_note(ctx.registry.down()), saver=st.token_saver, middle=middle, fresh=fresh) # -- answering --------------------------------------------------------------------- async def _answer(self, msg: Message, trig: Trigger) -> None: @@ -204,9 +203,8 @@ async def _answer(self, msg: Message, trig: Trigger) -> None: await draft.start() else: typing = await start_typing(ctx.bot, msg.chat_id) # a Telegram hiccup is not fatal - images: list[bytes] = [] # charts drawn while answering; they go out with the reply try: - html_text = await self._generate(msg, trig, draft, images) + html_text = await self._generate(msg, trig, draft) except Exception as e: log.exception("Reply failed") html_text = ("Sorry, something went wrong:\n" @@ -217,12 +215,9 @@ async def _answer(self, msg: Message, trig: Trigger) -> None: else: typing.cancel() markup = dm_buttons.keyboard() if trig.private and st.dm_buttons else None - if images: - await reply_with_photos(msg, images, html_text, ctx.store, quote=not trig.private, last_markup=markup) - else: - await reply_chunks(msg, html_text, ctx.store, quote=not trig.private, last_markup=markup) + await reply_chunks(msg, html_text, ctx.store, quote=not trig.private, last_markup=markup) - async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None, images: list[bytes]) -> str: + async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None) -> str: ctx, st = self.ctx, self.ctx.st await ctx.registry.wait_started(config.MCP_STARTUP_WAIT) preset = dm_buttons.preset_for(trig.text) if trig.private and st.dm_buttons else None @@ -256,17 +251,8 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None, imag extra = [post.tool(ctx, msg.from_user.id, msg.from_user.full_name or "")] if chosen.simple: # the model must not be told it has no tools chosen = Route([], False) - chart = bool(CHART_WORDS.search(trig.text)) and charts.available() - if chart: - extra.append(charts.tool(ctx, images)) - if admin and any(s.cfg.get("gate") == "portfolio" for s in chosen.servers): - extra.append(charts.portfolio_tool(ctx, images)) - if chosen.simple: # the model must not be told it has no tools - chosen = Route([], False) - if extra: - log.info("Local tools offered: %s", ", ".join(t.name for t, _ in extra)) fresh = is_portfolio_question(context, st) and any(s.cfg.get("gate") == "portfolio" for s in chosen.servers) - parts = [{"type": "text", "text": await self._chat_text(msg, trig, fresh=fresh, chart=chart)}] + parts = [{"type": "text", "text": await self._chat_text(msg, trig, fresh=fresh)}] parts += await self._images(msg, trig) answer = await ask(ctx, parts, chosen, cache_key=f"chat-{chat_id}", on_text=on_text, chat_id=chat_id, extra_tools=extra, diff --git a/lib/telegram/send.py b/lib/telegram/send.py index f3f173d..d7fa227 100644 --- a/lib/telegram/send.py +++ b/lib/telegram/send.py @@ -5,9 +5,9 @@ import asyncio import logging -from telegram import InputMediaPhoto, LinkPreviewOptions, Message +from telegram import LinkPreviewOptions, Message from telegram.constants import ParseMode -from telegram.error import BadRequest, TelegramError +from telegram.error import BadRequest from ..store import Store from ..textfmt import is_parse_error, plain_text, split_html @@ -15,8 +15,6 @@ log = logging.getLogger("bot") # Otherwise a list of Yahoo-linked tickers gets a big preview card under it. NO_PREVIEW = LinkPreviewOptions(is_disabled=True) -CAPTION_MAX = 1024 # Telegram's limit on a photo's caption (characters of text, not markup) -ALBUM_MAX = 10 # photos in one album async def send_html(bot, chat_id: int, text: str, reply_markup=None): @@ -57,41 +55,3 @@ async def reply_chunks(msg: Message, html_text: str, store: Store, *, quote: boo markup = last_markup if i == len(chunks) - 1 else None sent = await reply_html(msg, chunk, quote=quote, reply_markup=markup) await asyncio.to_thread(store.save, sent) - - -async def reply_with_photos(msg: Message, images: list[bytes], html_text: str, store: Store, *, quote: bool = True, - last_markup=None) -> None: - """Reply with pictures and the answer as the caption of the first one (one message, or one album). An answer - too long for a caption goes out as normal text after the pictures. If Telegram refuses the pictures the - answer is still sent as text.""" - caption = html_text if len(plain_text(html_text)) <= CAPTION_MAX else None - try: - for start in range(0, len(images), ALBUM_MAX): - batch = images[start:start + ALBUM_MAX] - text = caption if start == 0 else None - for sent in await _send_photos(msg, batch, text, quote and start == 0, - last_markup if caption and start + ALBUM_MAX >= len(images) else None): - await asyncio.to_thread(store.save, sent) - except TelegramError as e: - log.warning("Couldn't send the chart(s), sending the answer as text: %s", e) - await reply_chunks(msg, html_text, store, quote=quote, last_markup=last_markup) - return - if caption is None: - await reply_chunks(msg, html_text, store, quote=False, last_markup=last_markup) - - -async def _send_photos(msg: Message, batch: list[bytes], caption: str | None, quote: bool, markup) -> list[Message]: - """One photo, or an album of 2-10; the caption (Telegram HTML, plain text if it is rejected) rides on the first.""" - for text, mode in ((caption, ParseMode.HTML), (plain_text(caption) if caption else None, None)): - try: - if len(batch) == 1: - return [await msg.reply_photo(batch[0], caption=text, parse_mode=mode, do_quote=quote, - disable_notification=True, reply_markup=markup)] - media = [InputMediaPhoto(img, caption=text if i == 0 else None, parse_mode=mode if i == 0 else None) - for i, img in enumerate(batch)] - return list(await msg.reply_media_group(media, do_quote=quote, disable_notification=True)) - except BadRequest as e: - if not is_parse_error(e) or caption is None or mode is None: - raise - log.warning("Bad HTML from the model, sending the caption as plain text: %s", e) - raise AssertionError("unreachable") diff --git a/requirements.txt b/requirements.txt index 1c36dc9..cfb6bcd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ python-telegram-bot>=21.0 openai>=1.40 httpx>=0.27 mcp>=1.9,<2 -Pillow>=10.1 + diff --git a/tests/conftest.py b/tests/conftest.py index bbe452d..573ea0a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -53,7 +53,6 @@ class FakeBot: def __init__(self, bot_id=111, username="stockbot", first_name="Stock"): self.id, self.username, self.first_name = bot_id, username, first_name self.sent: list[dict] = [] - self.photos: list[dict] = [] self.drafts: list[str] = [] self.actions: list[str] = [] self.fail_send: dict[int, Exception] = {} @@ -73,10 +72,6 @@ async def send_message(self, chat_id, text, **kw): self.sent.append({"chat_id": chat_id, "text": text, **kw}) return self._msg(chat_id, text) - async def send_photo(self, chat_id, photo=None, caption=None, **kw): - self.photos.append({"chat_id": chat_id, "bytes": photo, "caption": caption, **kw}) - return self._msg(chat_id, caption or "[photo]") - async def get_chat_administrators(self, chat_id): if chat_id not in self.admins: raise TelegramError("Bad Request: chat not found") @@ -100,17 +95,6 @@ async def reply_text(text, **rkw): return bot._msg(chat_id, text) m.reply_text = reply_text - m.photo_replies = [] - - async def reply_photo(photo, **rkw): - m.photo_replies.append({"photo": photo, **rkw}) - return bot._msg(chat_id, rkw.get("caption") or "[photo]") - - async def reply_media_group(media, **rkw): - m.photo_replies.append({"album": media, **rkw}) - return [bot._msg(chat_id, (media[0].caption or "[photo]") if i == 0 else "[photo]") for i, _ in enumerate(media)] - - m.reply_photo, m.reply_media_group = reply_photo, reply_media_group m.parse_entities = lambda types=None: {} m.parse_caption_entities = lambda types=None: {} return m diff --git a/tests/test_app.py b/tests/test_app.py index 80f3046..91db5d9 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -89,11 +89,7 @@ def test_startup_logs_the_git_hash_and_whether_the_backfill_is_on(monkeypatch, t monkeypatch.setenv("PERIOD_BACKFILL", "RobSMSF, other") app.main(["grokbot"]) assert "Starting instance grokbot: " in caplog.text - assert "Period backfill: on for other, robsmsf" in caplog.text and "Pillow isn't installed" not in caplog.text - caplog.clear() - monkeypatch.setattr(app.charts, "available", lambda: False) - app.main(["grokbot"]) - assert "Pillow isn't installed, so pie and bar charts are off" in caplog.text + assert "Period backfill: on for other, robsmsf" in caplog.text def test_git_hash_is_read_from_the_git_directory_without_running_git(tmp_path): diff --git a/tests/test_charts.py b/tests/test_charts.py deleted file mode 100644 index a7f0a9b..0000000 --- a/tests/test_charts.py +++ /dev/null @@ -1,151 +0,0 @@ -import io -import json - -import pytest - -from lib import config -from lib.context import Ctx -from lib.features import charts -from lib.features.charts import render - -from .conftest import FakeBot -from .fakes import FakeMcp, ScriptedBackend, registry - -PNG = b"\x89PNG\r\n\x1a\n" -SECTORS = [{"label": "Semis", "value": 520.0}, {"label": "ETFs", "value": 250.0}, {"label": "Software", "value": 150.0}, - {"label": "Health", "value": 12.0}, {"label": "REA", "value": 10.0}] - - -def size(png: bytes) -> tuple[int, int]: - from PIL import Image - return Image.open(io.BytesIO(png)).size - - -def test_a_pie_is_a_png_and_small_slices_merge_into_other(): - png = render.draw("pie", "Sectors", SECTORS, "total 942") - assert png.startswith(PNG) and size(png)[0] == render.WIDTH and len(png) < 150_000 - slices = render.pie_slices([(i["label"], i["value"]) for i in SECTORS]) - assert [s[0] for s in slices] == ["Semis", "ETFs", "Software", "Other"] # Health and REA are under 2% - assert slices[-1][1] == 22.0 and slices[-1][3] and abs(sum(s[2] for s in slices) - 1) < 1e-9 - - -def test_more_than_ten_slices_fold_the_rest_into_other(): - items = [(f"s{i}", 100 - i) for i in range(15)] - slices = render.pie_slices(items) - assert len(slices) == render.MAX_SLICES + 1 and slices[-1][0] == "Other" - - -def test_a_bar_chart_takes_negatives_and_grows_with_the_rows(): - few = render.draw("bar", "Gain", [{"label": "Jan", "value": -5}, {"label": "Feb", "value": 12}]) - many = render.draw("bar", "Gain", [{"label": str(i), "value": i - 10} for i in range(25)]) - assert few.startswith(PNG) and many.startswith(PNG) and size(many)[1] > size(few)[1] - - -@pytest.mark.parametrize("kind, items, message", [ - ("donut", SECTORS, "kind must be"), - ("pie", [], "list of items"), - ("pie", "x", "list of items"), - ("pie", [{"label": "a"}], "label and a numeric value"), - ("pie", [{"label": "a", "value": "x"}], "numeric value"), - ("pie", [{"label": "a", "value": float("nan")}], "isn't a number"), - ("pie", [{"label": "a", "value": -1}, {"label": "b", "value": 0}], "positive value"), - ("bar", [{"label": str(i), "value": 1} for i in range(31)], "too many"), -]) -def test_bad_data_is_refused_with_a_clear_reason(kind, items, message): - with pytest.raises(render.ChartError, match=message): - render.draw(kind, "t", items) - - -def test_short_values_and_long_labels(): - assert [render.short(v) for v in (2_500_000, 173_600, 12, 3.2, -4_000)] == ["2.5M", "173.6k", "12", "3.2", "-4.0k"] - label = render.clean("pie", [{"label": "x" * 60, "value": 1}])[0][0] - assert len(label) == render.LABEL_CHARS and label.endswith("…") - - -async def test_the_tool_queues_the_picture_for_the_reply_and_sends_nothing_itself(env, store): - bot, images = FakeBot(), [] - ctx = Ctx(config.load(env), store, ScriptedBackend([]), registry(), bot) - defn, send = charts.tool(ctx, images) - out = await send({"kind": "pie", "title": "Sectors", "items": SECTORS}) - assert defn.name == "send_chart" and out == charts.READY and "becomes its caption" in out - assert len(images) == 1 and images[0].startswith(PNG) - assert bot.photos == [] and bot.sent == [] # the handler sends it, with the answer as the caption - assert "caption" not in defn.parameters["properties"] - - -async def test_the_tool_returns_errors_as_text_so_the_model_can_fall_back(env, store, monkeypatch): - bot, images = FakeBot(), [] - ctx = Ctx(config.load(env), store, ScriptedBackend([]), registry(), bot) - _, send = charts.tool(ctx, images) - assert (await send({"kind": "pie", "title": "t", "items": []})).startswith("Error: give a list of items") - - def explode(*a, **kw): - raise RuntimeError("font missing") - - monkeypatch.setattr(render, "draw", explode) - assert "answer in text instead" in await send({"kind": "pie", "title": "t", "items": SECTORS}) - assert images == [] - - -HOLDINGS = {"columns": ["code", "market", "name", "sector", "industry", "value", "quantity"], "rows": [ - ["CRWD", "NASDAQ", "Crowdstrike", "Technology Services", "Packaged Software", 90.0, 1], - ["S", "NYSE", "SentinelOne", "Technology Services", "Packaged Software", 10.0, 1], - ["NVDA", "NASDAQ", "Nvidia", "Electronic Technology", "Semiconductors", 170.0, 1], - ["TSM", "NYSE", "TSMC", "Electronic Technology", "Semiconductors", 50.0, 1], - ["SMH", "NASDAQ", "VanEck Semi", None, "Investment Trusts/Mutual Funds", 80.0, 1], - ["OLD", "ASX", "Sold", "Health", "Biotech", 0.0, 0]]} - - -class Sharesight(FakeMcp): - async def call(self, tool, args): - self.calls.append((tool, args)) - if tool == "list_portfolios": - return json.dumps({"portfolios": [{"id": 7, "name": "Main"}]}) - return json.dumps({"report": {"currency": "AUD", "holdings": HOLDINGS, - "cash_accounts": [{"name": "Cash", "value": 25.0, "currency": "AUD"}]}}) - - -def portfolio_ctx(env, store): - bot = FakeBot() - ss = Sharesight("sharesight", tools=("list_portfolios", "get_performance_report")) - return Ctx(config.load(env), store, ScriptedBackend([]), registry(ss), bot), bot, ss - - -def test_holdings_are_grouped_by_exactly_the_field_asked_for(): - rows = [dict(zip(HOLDINGS["columns"], r, strict=True)) for r in HOLDINGS["rows"]] - assert charts.group_holdings(rows, "sector") == [("Electronic Technology", 220.0), ("Technology Services", 100.0), - ("Unclassified", 80.0)] # the sold holding (value 0) is skipped - assert charts.group_holdings(rows, "industry")[0] == ("Semiconductors", 220.0) - assert charts.group_holdings(rows, "industry")[1] == ("Packaged Software", 100.0) - assert charts.group_holdings(rows, "holding")[0] == ("NVDA", 170.0) - - -async def test_portfolio_chart_groups_by_sector_not_industry_and_names_the_portfolio(env, store, monkeypatch): - ctx, bot, ss = portfolio_ctx(env, store) - drawn = {} - real = render.draw - - def spy(kind, title, items, subtitle=""): - drawn.update(kind=kind, title=title, items=items, subtitle=subtitle) - return real(kind, title, items, subtitle) - - monkeypatch.setattr(render, "draw", spy) - images = [] - defn, send = charts.portfolio_tool(ctx, images) - out = await send({"portfolio_id": 7, "by": "sector"}) - assert defn.name == "portfolio_chart" and "(sector: Electronic Technology 55.0%" in out and out.startswith(charts.READY) - assert drawn["title"] == "Main by sector" and drawn["kind"] == "pie" - assert [i["label"] for i in drawn["items"]] == ["Electronic Technology", "Technology Services", "Unclassified"] - assert "AUD 400 in holdings, 25 cash" in drawn["subtitle"] - assert len(images) == 1 and bot.photos == [] - assert ss.calls[0][1]["start_date"] == ss.calls[0][1]["end_date"] and "_keep_sales" not in ss.calls[0][1] # current holdings - - -async def test_portfolio_chart_refuses_bad_input_as_text(env, store): - ctx, bot, _ = portfolio_ctx(env, store) - _, send = charts.portfolio_tool(ctx, []) - assert "by must be one of" in await send({"portfolio_id": 7, "by": "colour"}) - assert "portfolio_id" in await send({"by": "sector"}) - none_ctx = Ctx(config.load(env), store, ScriptedBackend([]), registry(), bot) - assert "isn't connected" in await charts.portfolio_tool(none_ctx, [])[1]({"portfolio_id": 7, "by": "sector"}) - assert bot.photos == [] diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py index 7d0631c..a211e22 100644 --- a/tests/test_fixtures.py +++ b/tests/test_fixtures.py @@ -41,7 +41,7 @@ def test_real_performance_report_is_flattened_to_a_fraction_of_its_size(key): text = (FIX / "sharesight" / f"performance_report_{key}.json").read_text() slim = slim_result(text, "sharesight", drop_closed=True) report, original = json.loads(slim)["report"], json.loads(text)["report"] - assert len(slim) < len(text) / 7.5 # was 8 before each holding also kept its industry (for "by industry" charts) + assert len(slim) < len(text) / 8 rows = table_records(report["holdings"]) assert [(r["code"], r["quantity"], r["value"]) for r in rows] == [ (h["instrument"]["code"], h["quantity"], h["value"]) for h in original["holdings"]] diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 03a5c7a..1b2c487 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -113,48 +113,6 @@ async def test_a_short_follow_up_to_a_portfolio_reply_still_gets_the_portfolio_t assert backend.seen[3]["tools"] == [] -async def test_the_chart_tool_is_offered_only_when_asked_for_a_chart(env, store, monkeypatch, caplog): - from lib.features import charts - - caplog.set_level("INFO", logger="bot") - - ctx, backend, h = make_ctx(env, store, [step("a"), step("b"), step("c"), step("d")]) - await run(h, user_msg(ctx.bot, "@stockbot how is NVDA looking?", mid=1)) - await run(h, user_msg(ctx.bot, "@stockbot make me a pie chart of the semis sector", mid=2000)) - await run(h, user_msg(ctx.bot, "@stockbot draw a graph of chips", mid=3000)) - assert "send_chart" not in backend.seen[0]["tools"] and "send_chart" in backend.seen[1]["tools"] - assert "send_chart" in backend.seen[2]["tools"] # also on the route that would otherwise have no tools - assert "Local tools offered: send_chart" in caplog.text - assert "They want a picture" not in backend.seen[0]["prompt"] and "They want a picture" in backend.seen[1]["prompt"] - monkeypatch.setattr(charts, "available", lambda: False) # Pillow not installed: the bot just doesn't offer it - await run(h, user_msg(ctx.bot, "@stockbot pie chart please", mid=4000)) - assert "send_chart" not in backend.seen[3]["tools"] - - -async def test_a_chart_request_is_answered_with_one_photo_whose_caption_is_the_answer(env, store): - args = '{"kind":"pie","title":"Sectors","items":[{"label":"Semis","value":3},{"label":"Software","value":1}]}' - ctx, backend, h = make_ctx(env, store, [step(calls=[("1", "send_chart", args)], finish="tool_calls"), - step("Semis are 75% of it.")]) - m = user_msg(ctx.bot, "@stockbot pie chart of my sectors", mid=1) - await run(h, m) - assert len(m.photo_replies) == 1 and m.replies == [] # one message: the picture, with the answer as its caption - assert m.photo_replies[0]["caption"] == "Semis are 75% of it." and m.photo_replies[0]["photo"].startswith(b"\x89PNG") - assert any("Semis are" in r.text for r in store.history(-100, 20)) - plain = user_msg(ctx.bot, "@stockbot how is NVDA?", mid=2000) - ctx.backend.script.append(step("NVDA is fine")) - await run(h, plain) - assert plain.photo_replies == [] and "is fine" in plain.replies[0]["text"] - - -async def test_portfolio_chart_is_offered_for_a_portfolio_chart_request_to_admins_only(env, store): - sharesight = FakeMcp("sharesight", gate="portfolio", tools=("list_portfolios", "get_performance_report")) - ctx, backend, h = make_ctx(env, store, [step("a"), step("b")], servers=[sharesight], ADMIN_ONLY="on", ADMIN_CHAT_IDS="5") - await run(h, user_msg(ctx.bot, "@stockbot my portfolios by sector", mid=1, user_id=5)) - assert {"send_chart", "portfolio_chart", "period_returns"} <= set(backend.seen[0]["tools"]) - await run(h, user_msg(ctx.bot, "@stockbot my portfolios by sector", mid=2000, user_id=9)) - assert len(backend.seen) == 1 # a non-admin gets the "admins only" message, not a chart - - async def test_portfolio_question_reminds_the_model_to_fetch_fresh_figures(env, store): sharesight = FakeMcp("sharesight", gate="portfolio", tools=("list_portfolios",)) ctx, backend, h = make_ctx(env, store, [step("a"), step("b")], servers=[sharesight]) diff --git a/tests/test_history.py b/tests/test_history.py index 2b49c16..0ef6d92 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -44,12 +44,3 @@ def test_latest_own_reply_has_a_generous_cap_and_no_own_reply_is_fine(): def test_raw_mode_keeps_html(): rows = [HistoryRow(1, "Bob", "x", 0, None)] assert format_rows(rows, "z", UTC, line_max=100, compact_text=False).endswith("x") - - -def test_chat_prompt_asks_for_a_picture_only_when_the_chart_tool_is_offered(): - from lib.prompts import chat_prompt - base = dict(transcript="t", sender="Bob", private=False, reply_quote=None, msg_id=3, now="now") - plain, chart = chat_prompt(**base), chat_prompt(**base, chart=True, fresh=True) - assert "send_chart" not in plain and "portfolio_chart" not in plain - assert "They want a picture" in chart and "portfolio_chart" in chart and "send_chart" in chart - assert chart.index("Get current figures") < chart.index("They want a picture") # after the portfolio tail diff --git a/tests/test_mcp.py b/tests/test_mcp.py index b76ecf5..24cc860 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -374,12 +374,3 @@ def test_compact_schema_hides_listed_params_but_never_required_ones_and_drops_sc "required": ["a"]} out = compact_schema(schema, hide=frozenset({"a", "b"})) assert set(out["properties"]) == {"a", "c"} and "$schema" not in out and "additionalProperties" not in out - - -def test_slimmed_holdings_keep_both_sector_and_industry(): - from lib.mcp.results import slim_holding - h = {"quantity": 5, "value": 10.0, "instrument": {"code": "CRWD", "market_code": "NASDAQ", "name": "Crowdstrike", - "sector_classification_name": "Technology Services", "industry_classification_name": "Packaged Software"}} - out = slim_holding(h) - assert out["sector"] == "Technology Services" and out["industry"] == "Packaged Software" - assert "industry" not in slim_holding({"instrument": {"code": "X"}}) # missing fields are dropped as before diff --git a/tests/test_runner.py b/tests/test_runner.py index a54c94d..9e775b5 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -293,14 +293,3 @@ def test_gate_marks_routes_that_offer_fewer_tools(settings): def test_wants_tools(): assert wants_tools("NEEDS_TOOLS") and wants_tools(" needs_tools.") and not wants_tools("Sure thing") - - -@pytest.mark.parametrize("text, wanted", [ - ("can you do a pie chart of my sectors?", True), ("visualise my portfolio", True), ("Visualize my portfolio", True), - ("my portfolio by sector", True), ("robsmsf by industry", True), ("robsmsf by industries", True), - ("show holdings by country", True), ("what's my allocation", True), ("sector breakdown please", True), - ("graph NVDA this year", True), ("how is NVDA looking?", False), ("what did I make this month", False), - ("which sector is strongest", False), ("buy or sell?", False)]) -def test_chart_words_catch_requests_for_a_picture_or_a_grouping(text, wanted): - from lib.llm.gate import CHART_WORDS - assert bool(CHART_WORDS.search(text)) is wanted diff --git a/tests/test_send.py b/tests/test_send.py deleted file mode 100644 index 920d782..0000000 --- a/tests/test_send.py +++ /dev/null @@ -1,96 +0,0 @@ -import pytest -from telegram.error import BadRequest, TelegramError - -from lib.telegram.send import CAPTION_MAX, reply_with_photos - -from .conftest import FakeBot, user_msg - -PNG = b"\x89PNG\r\n\x1a\nfake" - - -async def reply(bot, store, text, images, **kw): - msg = user_msg(bot, "chart please", mid=5) - await reply_with_photos(msg, images, text, store, **kw) - return msg - - -async def test_a_short_answer_is_the_caption_of_one_photo(store): - bot = FakeBot() - msg = await reply(bot, store, "Semis are 52%.", [PNG]) - assert len(msg.photo_replies) == 1 and msg.replies == [] - sent = msg.photo_replies[0] - assert sent["photo"] == PNG and sent["caption"] == "Semis are 52%." and sent["do_quote"] is True - assert sent["disable_notification"] is True - assert [r.text for r in store.history(-100, 20)] == ["Semis are 52%."] # saved, so follow-ups see it - - -async def test_several_charts_go_as_one_album_with_the_caption_on_the_first(store): - bot = FakeBot() - msg = await reply(bot, store, "Three portfolios.", [PNG, PNG, PNG]) - assert len(msg.photo_replies) == 1 and msg.replies == [] - album = msg.photo_replies[0]["album"] - assert [m.caption for m in album] == ["Three portfolios.", None, None] - - -async def test_more_than_ten_charts_split_into_albums_and_only_the_first_has_the_caption(store): - bot = FakeBot() - msg = await reply(bot, store, "Many.", [PNG] * 12) - first, second = msg.photo_replies - assert len(first["album"]) == 10 and first["album"][0].caption == "Many." and len(second["album"]) == 2 - assert second["album"][0].caption is None and second["do_quote"] is False - - -async def test_an_answer_too_long_for_a_caption_follows_as_text(store): - bot = FakeBot() - long = "word " * (CAPTION_MAX // 5 + 20) - msg = await reply(bot, store, long, [PNG]) - assert msg.photo_replies[0]["caption"] is None # the picture alone - assert msg.replies and "word" in msg.replies[0]["text"] # then the answer as a normal message - - -async def test_a_rejected_caption_is_sent_as_plain_text(store): - bot = FakeBot() - msg = user_msg(bot, "chart", mid=5) - real = msg.reply_photo - calls = [] - - async def picky(photo, **kw): - calls.append(kw.get("parse_mode")) - if kw.get("parse_mode"): - raise BadRequest("Can't parse entities: bad tag") - return await real(photo, **kw) - - msg.reply_photo = picky - await reply_with_photos(msg, [PNG], "Bad ", store) - assert len(calls) == 2 and calls[1] is None and msg.photo_replies[0]["caption"] == "Bad " - - -async def test_if_telegram_refuses_the_picture_the_answer_is_still_sent(store): - bot = FakeBot() - msg = user_msg(bot, "chart", mid=5) - - async def refuse(photo, **kw): - raise TelegramError("Bad Request: not enough rights to send photos") - - msg.reply_photo = refuse - await reply_with_photos(msg, [PNG], "The answer.", store) - assert msg.photo_replies == [] and [r["text"] for r in msg.replies] == ["The answer."] - - -async def test_other_bad_requests_are_not_swallowed_as_parse_errors(store): - bot = FakeBot() - msg = user_msg(bot, "chart", mid=5) - - async def broken(photo, **kw): - raise BadRequest("Wrong file identifier") - - msg.reply_photo = broken - await reply_with_photos(msg, [PNG], "Answer.", store) # falls back to text instead of crashing - assert [r["text"] for r in msg.replies] == ["Answer."] - - -@pytest.mark.parametrize("n", [1, 3]) -async def test_the_markup_rides_on_a_single_photo_only(store, n): - bot = FakeBot() - msg = await reply(bot, store, "Hi.", [PNG] * n, last_markup="KB") - assert (msg.photo_replies[0].get("reply_markup") == "KB") is (n == 1) From 43e4a0ed15097708064d49d765862ec45d4b253f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 02:14:04 +0000 Subject: [PATCH 34/34] Period backfill is on by default (PERIOD_BACKFILL=all; off disables) A bot without Sharesight warns once, then quietly retries. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- README.md | 2 +- lib/config.py | 9 +++++++-- lib/features/periods/backfill.py | 6 ++++-- tests/test_app.py | 5 +++-- tests/test_backfill.py | 3 ++- tests/test_config.py | 6 ++++-- 6 files changed, 21 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f89fb58..d8cceb2 100644 --- a/README.md +++ b/README.md @@ -288,7 +288,7 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. | `MCP_CONFIG` | MCP server config file (default mcp_servers.json next to bot.py, whatever `DATA_DIR` is; missing = no MCP tools). | | `MOVERS_BOTS` | Usernames of bots whose end-of-day big-movers lists get explained. Empty (default) = feature off. | | `PORTFOLIO_NAMES` | Comma list of Sharesight portfolio names; a message naming one is treated as a portfolio question (default: the recipients' portfolio names). | -| `PERIOD_BACKFILL` | Comma list of Sharesight portfolio names (or `all`) whose returns per day, week, month, quarter and year are fetched into the local cache in the background (about 1,650 Sharesight calls per portfolio the first time, a few hours), so "best day since inception" answers instantly. Empty (default) = off. See [Caching Sharesight history](#caching-sharesight-history). | +| `PERIOD_BACKFILL` | Comma list of Sharesight portfolio names (or `all`) whose returns per day, week, month, quarter and year are fetched into the local cache in the background (about 1,650 Sharesight calls per portfolio the first time, a few hours), so "best day since inception" answers instantly. Default `all`; list names to limit it, or `off` to disable. See [Caching Sharesight history](#caching-sharesight-history). | | `POST_TO_GROUPS_FROM_DM` | on|off. A group admin can have the bot post in that group from a DM (default off). | | `SEARCH_MODEL` | OpenRouter only: model that runs the searches (default xiaomi/mimo-v2.6-flash:online). | | `SHARESIGHT_HOLDING_NEWS_RECIPIENTS` | Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off. | diff --git a/lib/config.py b/lib/config.py index cee90e5..fdf6509 100644 --- a/lib/config.py +++ b/lib/config.py @@ -33,7 +33,7 @@ "MCP_CONFIG": "MCP server config file (default mcp_servers.json next to bot.py; missing = no MCP tools).", "MOVERS_BOTS": "Usernames of bots whose end-of-day big-movers lists get explained. Empty (default) = feature off.", "PORTFOLIO_NAMES": "Comma list of Sharesight portfolio names; a message naming one is treated as a portfolio question (default: the recipients' portfolio names).", - "PERIOD_BACKFILL": "Comma list of Sharesight portfolio names (or all) whose daily/weekly/monthly/quarterly/yearly returns are fetched into the local cache in the background, so period questions answer instantly. Empty (default) = off.", + "PERIOD_BACKFILL": "Comma list of Sharesight portfolio names (or all) whose daily/weekly/monthly/quarterly/yearly returns are fetched into the local cache in the background, so period questions answer instantly. Default all; list names to limit it, or off to disable.", "POST_TO_GROUPS_FROM_DM": "on|off. Lets an admin of a group the bot is in make it post there from a DM (default off).", "SEARCH_MODEL": "OpenRouter only: model that runs the searches (default xiaomi/mimo-v2.6-flash:online).", "SHARESIGHT_HOLDING_NEWS_RECIPIENTS": "Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off.", @@ -150,6 +150,11 @@ def _names(text: str) -> frozenset[str]: return frozenset(n.strip().lower() for n in re.split(r"[,\n]", text) if n.strip()) +def _backfill(text: str) -> frozenset[str]: + names = _names(text) + return frozenset() if names & {"off", "none", "false", "0"} else names + + def load(env: Mapping[str, str] | None = None) -> Settings: """Build Settings from `env` (default: os.environ). Raises ConfigError if unusable.""" env = os.environ if env is None else env @@ -198,5 +203,5 @@ def load(env: Mapping[str, str] | None = None) -> Settings: holding_news_recipients=recipients, portfolio_names=_names(env.get("PORTFOLIO_NAMES", "")) or frozenset(recipients), token_saver=saver, - period_backfill=_names(env.get("PERIOD_BACKFILL", "")), + period_backfill=_backfill(env.get("PERIOD_BACKFILL", "all")), ) diff --git a/lib/features/periods/backfill.py b/lib/features/periods/backfill.py index 1e7ed55..c9e3c50 100644 --- a/lib/features/periods/backfill.py +++ b/lib/features/periods/backfill.py @@ -155,11 +155,13 @@ async def fill_once(ctx: Ctx, server, portfolios: list[tuple[int, str, date]], t async def loop(ctx: Ctx) -> None: await ctx.registry.wait_started(config.MCP_STARTUP_WAIT) + warned = False while True: wait = RECHECK server = sharesight(ctx) - if server is None: - log.warning("Period backfill: Sharesight isn't connected; trying again in 10 minutes") + if server is None: # on by default, so a bot without Sharesight must not nag every 10 minutes + (log.debug if warned else log.warning)("Period backfill: Sharesight isn't connected; trying again in 10 minutes") + warned = True wait = 600 else: try: diff --git a/tests/test_app.py b/tests/test_app.py index 91db5d9..f8f84e7 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -82,6 +82,7 @@ def test_startup_logs_the_git_hash_and_whether_the_backfill_is_on(monkeypatch, t monkeypatch.setattr(app.Application, "run_polling", lambda self, **kw: None) caplog.set_level("INFO", logger="bot") monkeypatch.setattr(app, "git_hash", lambda: "abc1234") + monkeypatch.setenv("PERIOD_BACKFILL", "off") app.main(["grokbot"]) assert "Starting instance grokbot (abc1234): " in caplog.text and "Period backfill" not in caplog.text caplog.clear() @@ -164,14 +165,14 @@ def run(*words): assert run("70007", "70008")[0] == 1 # one portfolio at a time -async def test_backfill_task_starts_only_when_period_backfill_is_set(env, store, monkeypatch): +async def test_backfill_task_starts_unless_period_backfill_is_off(env, store, monkeypatch): started = [] async def fake_loop(ctx): started.append(ctx) monkeypatch.setattr(app.backfill, "loop", fake_loop) - for extra, expected in (({}, 0), ({"PERIOD_BACKFILL": "x"}, 1)): + for extra, expected in (({"PERIOD_BACKFILL": "off"}, 0), ({}, 1), ({"PERIOD_BACKFILL": "x"}, 2)): st = config.load({**env, **extra}) ctx = Ctx(st, store, ScriptedBackend([]), registry(), FakeBot()) built = app.build_app(ctx) diff --git a/tests/test_backfill.py b/tests/test_backfill.py index 355d529..279f872 100644 --- a/tests/test_backfill.py +++ b/tests/test_backfill.py @@ -77,7 +77,8 @@ def test_portfolios_are_selected_by_name_or_all(env): pick = lambda **e: [p[1] for p in backfill.selected(config.load({**env, "TELEGRAM_BOT_TOKEN": "1:x", **e}), listing)] # noqa: E731 assert pick(PERIOD_BACKFILL="robsmsf") == ["RobSMSF"] assert pick(PERIOD_BACKFILL="all") == ["RobSMSF", "Other"] # a portfolio without an inception date is skipped - assert pick() == [] + assert pick() == ["RobSMSF", "Other"] # on by default + assert pick(PERIOD_BACKFILL="off") == [] async def test_a_pass_fetches_what_is_missing_then_only_what_is_new(env, store): diff --git a/tests/test_config.py b/tests/test_config.py index 98c259c..d8cc75e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -25,9 +25,11 @@ def test_data_dir_defaults_to_systemd_state_directory(): assert config.load({**base, "STATE_DIRECTORY": "/a", "DB_PATH": "/x/y.db"}).db_path == "/x/y.db" -def test_period_backfill_is_off_by_default_and_lists_names(): +def test_period_backfill_is_all_by_default_and_can_be_listed_or_off(): base = {"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "a"} - assert config.load(base).period_backfill == frozenset() + assert config.load(base).period_backfill == {"all"} + assert config.load({**base, "PERIOD_BACKFILL": "off"}).period_backfill == frozenset() + assert config.load({**base, "PERIOD_BACKFILL": ""}).period_backfill == frozenset() assert config.load({**base, "PERIOD_BACKFILL": "RobSMSF, bobsmsf"}).period_backfill == {"robsmsf", "bobsmsf"} assert "all" in config.load({**base, "PERIOD_BACKFILL": "all"}).period_backfill