Merge pull request #2610 from DerekMelchin/bug-split-equity-fundamental-data-skill

Split equity-fundamental-data SKILL into per-family SKILLs

Author: DerekMelchin

skill-templates/bundle-skills.py

@@ -236,12 +236,44 @@ def _table_cell(text: str) -> str:
     return " ".join(html.unescape(text).split()).replace("|", "\\|")
 
 
-def format_fundamental_lookup(lookup: dict) -> str:
+def _norm_path(path: str) -> str:
+    """Language-neutral path form: strip the `f.` root, drop underscores, lowercase —
+    so a template scope written as `financial_statements.income_statement` matches both
+    `f.financial_statements.income_statement...` and `f.FinancialStatements.IncomeStatement...`."""
+    p = path[2:] if path[:2] in ("f.", "F.") else path
+    return p.replace("_", "").lower()
+
+
+def format_fundamental_lookup(lookup: dict, scopes: list[str] | None = None) -> str:
     """Render the data-point table (path + description), then each enum's
     constants — as a Value/Description table where the inspector documents them,
-    or a plain list where it doesn't (e.g. MorningstarIndustryGroupCode)."""
+    or a plain list where it doesn't (e.g. MorningstarIndustryGroupCode).
+
+    `scopes` filters rows by path prefix (comma tokens from the template marker):
+    plain tokens include matching subtrees; `!token` excludes them (no plain
+    tokens = start from everything). The Classification-code-constants section is
+    emitted only when the filtered rows still contain `asset_classification` paths,
+    since only those compare against the constants. No scopes = the full table
+    (backward compatible)."""
+    rows = lookup["rows"]
+    if scopes:
+        pos = [_norm_path(s) for s in scopes if not s.startswith("!")]
+        neg = [_norm_path(s[1:]) for s in scopes if s.startswith("!")]
+
+        def keep(path: str) -> bool:
+            p = _norm_path(path)
+            ok = any(p.startswith(x) for x in pos) if pos else True
+            return ok and not any(p.startswith(x) for x in neg)
+
+        rows = [(p, d) for p, d in rows if keep(p)]
+        if not rows:
+            raise SystemExit(f"fundamental-lookup scopes matched no rows: {scopes}")
+    include_enums = any(_norm_path(p).startswith("assetclassification") for p, _ in rows)
     table = ["| Data point | Description |", "|---|---|"]
-    table += [f"| `{path}` | {_table_cell(desc)} |" for path, desc in lookup["rows"]]
+    table += [f"| `{path}` | {_table_cell(desc)} |" for path, desc in rows]
+
+    if not include_enums:
+        return "\n".join(table)
 
     sections = []
     for type_name, values in lookup["enums"].items():
@@ -370,21 +402,28 @@ def main() -> int:
     # Wipe the build dir so deletions in the source propagate.
     remove_tree(skills_root, dry_run=args.dry_run)
 
-    fund_lookup_marker = "<!-- fundamental-lookup -->"
-    # Lazily fetched per language; populated only when a skill needs the lookup.
-    fundamental_lookup: dict[str, str] = {}
+    # `<!-- fundamental-lookup -->` = full table; `<!-- fundamental-lookup: a, b, !c -->`
+    # = rows scoped by path prefix (see format_fundamental_lookup). The raw inspector
+    # tree is fetched lazily, once per language, and filtered per marker.
+    fund_marker_re = re.compile(r"<!--\s*fundamental-lookup(?::([^>]*?))?\s*-->")
+    fundamental_lookup_raw: dict[str, dict] = {}
 
     # Build: split each skill into a Python and a C# tree.
     for skill in skills:
         for lang in LANGS:
             content = split_for_language(skill.content, lang)
-            if fund_lookup_marker in content:
-                if lang not in fundamental_lookup:
+            if fund_marker_re.search(content):
+                if lang not in fundamental_lookup_raw:
                     print(f"Fetching Fundamental property tree ({lang}) from inspector...")
-                    fundamental_lookup[lang] = format_fundamental_lookup(
-                        fetch_fundamental_lookup(lang)
+                    fundamental_lookup_raw[lang] = fetch_fundamental_lookup(lang)
+
+                def _expand(m: "re.Match[str]") -> str:
+                    scopes = [t.strip() for t in (m.group(1) or "").split(",") if t.strip()]
+                    return format_fundamental_lookup(
+                        fundamental_lookup_raw[lang], scopes or None
                     )
-                content = content.replace(fund_lookup_marker, fundamental_lookup[lang])
+
+                content = fund_marker_re.sub(_expand, content)
             write_file(
                 skills_root / lang / skill.rel_dir / "SKILL.md",
                 content,

skill-templates/equity-fundamental-data/SKILL.md

@@ -1,18 +1,31 @@
 ---
 name: equity-fundamental-data
-description: Use to look up the exact path or spelling of any Morningstar fundamental data point on a QuantConnect/LEAN `Fundamental` object `f` — every field under py`f.financial_statements.*`cs`f.FinancialStatements.*` (income statement, balance sheet, cash flow statement), the operation / valuation / earning ratios, earning reports, company profile, company & security reference, and asset classification — plus the Morningstar sector and industry classification code constants. Triggers — a missing-attribute / compile error on a Fundamental property path; questions like "what's the path to net income / operating cash flow / shares outstanding / PE ratio / sector code". Skip when — you need how to build or screen a universe (see the fundamental-universes skill).
+description: START HERE to look up the exact path or spelling of any Morningstar fundamental data point on a QuantConnect/LEAN `Fundamental` object `f`. This skill holds the path-reading rules, the top-level and filing-metadata fields (market cap, `period_ending_date`, `file_date`, ...), and the index of the six field-family skills that hold the full tables — fundamental-income-statement, fundamental-balance-sheet, fundamental-cash-flow-statement, fundamental-ratios, fundamental-company-data, fundamental-classification. Triggers — a missing-attribute / compile error on a Fundamental property path; questions like "what's the path to net income / operating cash flow / shares outstanding / PE ratio / sector code". Skip when — you need how to build or screen a universe (see the fundamental-universes skill).
 ---
 
 # Fundamental data-point attributes — QuantConnect / LEAN
 
-Every readable Morningstar data point on a `Fundamental` object, written as a full path from the snapshot `f` with its description. Copy the path you need rather than guessing from English names — a wrong path wastes a backtest run. Get `f` from an py`add_universe(...)`cs`AddUniverse(...)` selection callback (each element is a `Fundamental`), from py`self.securities["SPY"].fundamentals`cs`Securities["SPY"].Fundamentals`, or from a history request.
+Morningstar data points are read as a full path from the snapshot `f` — copy the path you need rather than guessing from English names; a wrong path wastes a backtest run. Get `f` from an py`add_universe(...)`cs`AddUniverse(...)` selection callback (each element is a `Fundamental`), from py`self.securities["SPY"].fundamentals`cs`Securities["SPY"].Fundamentals`, or from a history request. The field tables are split across skills by family: THIS skill carries the top-level and filing-metadata fields plus the index below — load the family skill that holds your field's table.
+
+## Where every field lives — load the matching skill
+
+| Field family | Load this skill | Contents |
+|---|---|---|
+| py`f.financial_statements.income_statement.*`cs`f.FinancialStatements.IncomeStatement.*` | `fundamental-income-statement` | revenue, cost/expense lines, operating & net income, EBIT/EBITDA, interest, tax, dividends paid |
+| py`f.financial_statements.balance_sheet.*`cs`f.FinancialStatements.BalanceSheet.*` | `fundamental-balance-sheet` | assets, liabilities, equity, debt, working-capital components, share counts |
+| py`f.financial_statements.cash_flow_statement.*`cs`f.FinancialStatements.CashFlowStatement.*` | `fundamental-cash-flow-statement` | operating / investing / financing cash flows, capex, issuance & repurchase, dividends |
+| py`f.operation_ratios.*`cs`f.OperationRatios.*`, py`f.valuation_ratios.*`cs`f.ValuationRatios.*`, py`f.earning_ratios.*`cs`f.EarningRatios.*` | `fundamental-ratios` | ROA/ROE/margins/turnover, PE/PB/PS/EV multiples & yields, EPS/DPS growth rates |
+| py`f.earning_reports.*`cs`f.EarningReports.*`, py`f.company_reference.*`cs`f.CompanyReference.*`, py`f.security_reference.*`cs`f.SecurityReference.*`, py`f.company_profile.*`cs`f.CompanyProfile.*` | `fundamental-company-data` | EPS & report dates, listing/exchange/share-class reference, company profile basics |
+| py`f.asset_classification.*`cs`f.AssetClassification.*` + code constants | `fundamental-classification` | sector / industry-group / industry codes and the `MorningstarSectorCode`-style constants they compare against |
 
 ## Reading the paths
 
 - A path ending in `.[value 1M 2M 3M 6M 9M 12M]` is a `MultiPeriodField` — append **one** period accessor to read the number. py`.value`cs`.Value` is the most recent reported period; the `1M`–`12M` tokens are py`.one_month .two_months .three_months .six_months .nine_months .twelve_months`cs`.OneMonth .TwoMonths .ThreeMonths .SixMonths .NineMonths .TwelveMonths` respectively (trailing-twelve-month at `12M`). e.g. py`f.financial_statements.income_statement.net_income.twelve_months`cs`f.FinancialStatements.IncomeStatement.NetIncome.TwelveMonths`. Forgetting the accessor is silent — the wrapper compares as truthy and numeric inequalities give nonsense.
 - A path with **no** bracket is read directly. e.g. py`f.valuation_ratios.pe_ratio`cs`f.ValuationRatios.PERatio`.
-- The integer `*_code` fields under `asset_classification` compare against the named constants in the **Classification code constants** section at the end, e.g. py`f.asset_classification.morningstar_sector_code == MorningstarSectorCode.TECHNOLOGY`cs`f.AssetClassification.MorningstarSectorCode == MorningstarSectorCode.Technology`.
+- The integer `*_code` fields under `asset_classification` compare against the named constants in the **fundamental-classification** skill, e.g. py`f.asset_classification.morningstar_sector_code == MorningstarSectorCode.TECHNOLOGY`cs`f.AssetClassification.MorningstarSectorCode == MorningstarSectorCode.Technology`.
+
+## Top-level and filing-metadata data points
 
-## Data points
+The snapshot's own attributes and the filing/timing fields under py`f.financial_statements`cs`f.FinancialStatements` (period end, file date, period type, ...) — the fields every point-in-time strategy needs:
 
-<!-- fundamental-lookup -->
+<!-- fundamental-lookup: !financial_statements.income_statement, !financial_statements.balance_sheet, !financial_statements.cash_flow_statement, !operation_ratios, !valuation_ratios, !earning_ratios, !earning_reports, !company_reference, !security_reference, !company_profile, !asset_classification -->

skill-templates/equity-fundamental-data/fundamental-balance-sheet/SKILL.md

@@ -0,0 +1,18 @@
+---
+name: fundamental-balance-sheet
+description: Use to look up the exact path or spelling of any Morningstar BALANCE SHEET field on a QuantConnect/LEAN `Fundamental` object — everything under py`f.financial_statements.balance_sheet.*`cs`f.FinancialStatements.BalanceSheet.*` — assets, liabilities, stockholders' equity, debt, working-capital components, and share counts. Triggers — "path to total assets / total equity / long-term debt / cash and equivalents / inventory / shares issued". For other field families start at the equity-fundamental-data skill.
+---
+
+# Balance-sheet fields — `Fundamental` data points
+
+Full path from the snapshot `f` with the field's description — copy the path rather than guessing from English names; a wrong path wastes a backtest run. The path-reading rules and the index of all field-family skills are in the **equity-fundamental-data** skill.
+
+## Reading the paths
+
+- A path ending in `.[value 1M 2M 3M 6M 9M 12M]` is a `MultiPeriodField` — append **one** period accessor to read the number. py`.value`cs`.Value` is the most recent reported period; the `1M`–`12M` tokens are py`.one_month .two_months .three_months .six_months .nine_months .twelve_months`cs`.OneMonth .TwoMonths .ThreeMonths .SixMonths .NineMonths .TwelveMonths` respectively (trailing-twelve-month at `12M`). e.g. py`f.financial_statements.income_statement.net_income.twelve_months`cs`f.FinancialStatements.IncomeStatement.NetIncome.TwelveMonths`. Forgetting the accessor is silent — the wrapper compares as truthy and numeric inequalities give nonsense.
+- A path with **no** bracket is read directly. e.g. py`f.valuation_ratios.pe_ratio`cs`f.ValuationRatios.PERatio`.
+- The integer `*_code` fields under `asset_classification` compare against the named constants in the **fundamental-classification** skill.
+
+## Data points
+
+<!-- fundamental-lookup: financial_statements.balance_sheet -->

skill-templates/equity-fundamental-data/fundamental-cash-flow-statement/SKILL.md

@@ -0,0 +1,18 @@
+---
+name: fundamental-cash-flow-statement
+description: Use to look up the exact path or spelling of any Morningstar CASH FLOW STATEMENT field on a QuantConnect/LEAN `Fundamental` object — everything under py`f.financial_statements.cash_flow_statement.*`cs`f.FinancialStatements.CashFlowStatement.*` — operating / investing / financing cash flows, capital expenditure, stock issuance and repurchase, and cash dividends paid. Triggers — "path to operating cash flow / free cash flow / capex / cash dividends paid / stock repurchase". For other field families start at the equity-fundamental-data skill.
+---
+
+# Cash-flow-statement fields — `Fundamental` data points
+
+Full path from the snapshot `f` with the field's description — copy the path rather than guessing from English names; a wrong path wastes a backtest run. The path-reading rules and the index of all field-family skills are in the **equity-fundamental-data** skill.
+
+## Reading the paths
+
+- A path ending in `.[value 1M 2M 3M 6M 9M 12M]` is a `MultiPeriodField` — append **one** period accessor to read the number. py`.value`cs`.Value` is the most recent reported period; the `1M`–`12M` tokens are py`.one_month .two_months .three_months .six_months .nine_months .twelve_months`cs`.OneMonth .TwoMonths .ThreeMonths .SixMonths .NineMonths .TwelveMonths` respectively (trailing-twelve-month at `12M`). e.g. py`f.financial_statements.income_statement.net_income.twelve_months`cs`f.FinancialStatements.IncomeStatement.NetIncome.TwelveMonths`. Forgetting the accessor is silent — the wrapper compares as truthy and numeric inequalities give nonsense.
+- A path with **no** bracket is read directly. e.g. py`f.valuation_ratios.pe_ratio`cs`f.ValuationRatios.PERatio`.
+- The integer `*_code` fields under `asset_classification` compare against the named constants in the **fundamental-classification** skill.
+
+## Data points
+
+<!-- fundamental-lookup: financial_statements.cash_flow_statement -->

skill-templates/equity-fundamental-data/fundamental-classification/SKILL.md

@@ -0,0 +1,18 @@
+---
+name: fundamental-classification
+description: Use to look up the exact path or spelling of any Morningstar ASSET CLASSIFICATION field on a QuantConnect/LEAN `Fundamental` object — everything under py`f.asset_classification.*`cs`f.AssetClassification.*` (Morningstar sector / industry-group / industry codes, style box, financial health grade) — plus the named code constants (`MorningstarSectorCode`, `MorningstarIndustryGroupCode`, `MorningstarIndustryCode`, ...) the integer `*_code` fields compare against. Triggers — "path to sector code / industry group code / how do I filter to Technology / SIC or NAICS code". For other field families start at the equity-fundamental-data skill.
+---
+
+# Asset-classification fields and code constants — `Fundamental` data points
+
+Full path from the snapshot `f` with the field's description — copy the path rather than guessing from English names; a wrong path wastes a backtest run. The path-reading rules and the index of all field-family skills are in the **equity-fundamental-data** skill.
+
+## Reading the paths
+
+- A path ending in `.[value 1M 2M 3M 6M 9M 12M]` is a `MultiPeriodField` — append **one** period accessor to read the number. py`.value`cs`.Value` is the most recent reported period; the `1M`–`12M` tokens are py`.one_month .two_months .three_months .six_months .nine_months .twelve_months`cs`.OneMonth .TwoMonths .ThreeMonths .SixMonths .NineMonths .TwelveMonths` respectively (trailing-twelve-month at `12M`). e.g. py`f.financial_statements.income_statement.net_income.twelve_months`cs`f.FinancialStatements.IncomeStatement.NetIncome.TwelveMonths`. Forgetting the accessor is silent — the wrapper compares as truthy and numeric inequalities give nonsense.
+- A path with **no** bracket is read directly. e.g. py`f.valuation_ratios.pe_ratio`cs`f.ValuationRatios.PERatio`.
+- The integer `*_code` fields compare against the named constants in the **Classification code constants** section at the end, e.g. py`f.asset_classification.morningstar_sector_code == MorningstarSectorCode.TECHNOLOGY`cs`f.AssetClassification.MorningstarSectorCode == MorningstarSectorCode.Technology`.
+
+## Data points
+
+<!-- fundamental-lookup: asset_classification -->

skill-templates/equity-fundamental-data/fundamental-company-data/SKILL.md

@@ -0,0 +1,18 @@
+---
+name: fundamental-company-data
+description: Use to look up the exact path or spelling of any Morningstar EARNING REPORT, COMPANY/SECURITY REFERENCE, or COMPANY PROFILE field on a QuantConnect/LEAN `Fundamental` object — everything under py`f.earning_reports.*`cs`f.EarningReports.*` (EPS, DPS, report/file dates, shares), py`f.company_reference.*`cs`f.CompanyReference.*` (country, exchange, industry template), py`f.security_reference.*`cs`f.SecurityReference.*` (security type, primary share, listing status, IPO date), and py`f.company_profile.*`cs`f.CompanyProfile.*`. Triggers — "path to file date of the earning report / basic EPS / primary exchange / share class / is primary share / IPO date". For other field families start at the equity-fundamental-data skill.
+---
+
+# Company data fields — `Fundamental` data points
+
+Full path from the snapshot `f` with the field's description — copy the path rather than guessing from English names; a wrong path wastes a backtest run. The path-reading rules and the index of all field-family skills are in the **equity-fundamental-data** skill.
+
+## Reading the paths
+
+- A path ending in `.[value 1M 2M 3M 6M 9M 12M]` is a `MultiPeriodField` — append **one** period accessor to read the number. py`.value`cs`.Value` is the most recent reported period; the `1M`–`12M` tokens are py`.one_month .two_months .three_months .six_months .nine_months .twelve_months`cs`.OneMonth .TwoMonths .ThreeMonths .SixMonths .NineMonths .TwelveMonths` respectively (trailing-twelve-month at `12M`). e.g. py`f.financial_statements.income_statement.net_income.twelve_months`cs`f.FinancialStatements.IncomeStatement.NetIncome.TwelveMonths`. Forgetting the accessor is silent — the wrapper compares as truthy and numeric inequalities give nonsense.
+- A path with **no** bracket is read directly. e.g. py`f.valuation_ratios.pe_ratio`cs`f.ValuationRatios.PERatio`.
+- The integer `*_code` fields under `asset_classification` compare against the named constants in the **fundamental-classification** skill.
+
+## Data points
+
+<!-- fundamental-lookup: earning_reports, company_reference, security_reference, company_profile -->