Support Markdown with Gherkin feature files

CHANGES.rst

@@ -36,6 +36,7 @@ Unreleased
 
 Added
 +++++
+* Support `Markdown with Gherkin <https://github.com/cucumber/gherkin/blob/main/MARKDOWN_WITH_GHERKIN.md>`_ feature files. Files ending in ``.md`` are parsed with the official Markdown token matcher, and ``scenarios()`` collects ``*.feature.md`` files from directories. Requires ``gherkin-official>=32.2.0``; older releases raise ``FeatureError`` for Markdown files. `#780 <https://github.com/pytest-dev/pytest-bdd/issues/780>`_
 
 Changed
 +++++++

README.rst

@@ -1212,6 +1212,33 @@ tests/test_publish_article.py:
 The `features_base_dir` parameter can also be passed to the `@scenario` decorator.
 
 
+Markdown with Gherkin
+---------------------
+
+Feature files can also be written in `Markdown with Gherkin <https://github.com/cucumber/gherkin/blob/main/MARKDOWN_WITH_GHERKIN.md>`__ (MDG), so that prose documentation and scenarios live in the same document.
+A feature file whose name ends with ``.md`` is parsed as MDG, and ``scenarios()`` collects ``*.feature.md`` files from directories next to ``*.feature`` files. Other Markdown files, such as ``README.md``, are not collected.
+MDG requires ``gherkin-official>=32.2.0``.
+
+features/publish_article.feature.md:
+
+.. code-block:: markdown
+
+    `@article`
+    # Feature: Blog
+
+    Prose that documents the feature stays out of the scenarios.
+
+    ## Scenario: Publishing the article
+
+    * Given I'm an author user
+    * And I have an article
+    * When I go to the article page
+    * Then I should see the published article
+
+Headings carry the keywords, steps are list items, tags are inline code on the line above a heading, doc strings are fenced code blocks right after a step, and data tables and examples tables must be indented by two to five spaces.
+Because of these rules, a fenced code block placed directly after a step becomes that step's doc string, and inline code that looks like a tag (for example `` `@slow` ``) in a prose line is read as a tag line.
+
+
 Avoid retyping the feature file name
 ------------------------------------
 

src/pytest_bdd/feature.py

@@ -35,6 +35,9 @@
 # Global features dictionary
 features: dict[str, Feature] = {}
 
+# Gherkin files and Markdown with Gherkin files collected from directories
+FEATURE_FILE_PATTERNS = ("*.feature", "*.feature.md")
+
 
 def get_feature(base_path: str, filename: str, encoding: str = "utf-8") -> Feature:
     """Get a feature by the filename.
@@ -71,7 +74,11 @@ def get_features(paths: Iterable[str], encoding: str = "utf-8") -> list[Feature]
         if path not in seen_names:
             seen_names.add(path)
             if os.path.isdir(path):
-                file_paths = list(glob.iglob(os.path.join(path, "**", "*.feature"), recursive=True))
+                file_paths = [
+                    file_path
+                    for pattern in FEATURE_FILE_PATTERNS
+                    for file_path in glob.iglob(os.path.join(path, "**", pattern), recursive=True)
+                ]
                 _features.extend(get_features(file_paths, encoding=encoding))
             else:
                 base, name = os.path.split(path)

src/pytest_bdd/gherkin_parser.py

@@ -6,16 +6,23 @@
 import typing
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field
+from importlib.metadata import version
 from typing import Any
 
 from gherkin.errors import CompositeParserException  # type: ignore
 from gherkin.parser import Parser  # type: ignore
+from gherkin.token_matcher_markdown import GherkinInMarkdownTokenMatcher  # type: ignore
+from gherkin.token_scanner import TokenScanner  # type: ignore
+from packaging.version import parse as parse_version
 
 from . import exceptions
 
 if typing.TYPE_CHECKING:
     from typing_extensions import Self
 
+MARKDOWN_SUFFIX = ".md"
+# Earlier releases crash on any Markdown with Gherkin document in the Python parser.
+MARKDOWN_MINIMUM_GHERKIN_VERSION = parse_version("32.2.0")
 
 ERROR_PATTERNS = [
     (
@@ -304,12 +311,28 @@ def from_dict(cls, data: Mapping[str, Any]) -> Self:
         )
 
 
+def parse_gherkin(feature_file_text: str, abs_filename: str) -> Mapping[str, Any]:
+    """Parse Gherkin, or Markdown with Gherkin when the file name ends with ``.md``."""
+    if not abs_filename.endswith(MARKDOWN_SUFFIX):
+        return Parser().parse(feature_file_text)
+    gherkin_version = parse_version(version("gherkin-official"))
+    if gherkin_version < MARKDOWN_MINIMUM_GHERKIN_VERSION:
+        raise exceptions.FeatureError(
+            f"Markdown with Gherkin requires gherkin-official>={MARKDOWN_MINIMUM_GHERKIN_VERSION}, "
+            f"found {gherkin_version}",
+            0,
+            "",
+            abs_filename,
+        )
+    return Parser().parse(TokenScanner(feature_file_text), GherkinInMarkdownTokenMatcher())
+
+
 def get_gherkin_document(abs_filename: str, encoding: str = "utf-8") -> GherkinDocument:
     with open(abs_filename, encoding=encoding) as f:
         feature_file_text = f.read()
 
     try:
-        gherkin_data = Parser().parse(feature_file_text)
+        gherkin_data = parse_gherkin(feature_file_text, abs_filename)
     except CompositeParserException as e:
         message = e.args[0]
         line = e.errors[0].location["line"]

tests/feature/test_markdown.py

@@ -0,0 +1,237 @@
+"""Test Markdown with Gherkin (MDG) feature files."""
+
+from __future__ import annotations
+
+import textwrap
+from importlib.metadata import version
+
+import pytest
+from packaging.version import parse as parse_version
+
+MARKDOWN_SUPPORTED = parse_version(version("gherkin-official")) >= parse_version("32.2.0")
+
+requires_markdown = pytest.mark.skipif(
+    not MARKDOWN_SUPPORTED, reason="Markdown with Gherkin requires gherkin-official>=32.2.0"
+)
+
+STEPS = """\
+import json
+
+from pytest_bdd import given, parsers, then
+
+
+@given("I have a bar", target_fixture="bar")
+def _():
+    return "bar"
+
+
+@given("the payload", target_fixture="payload")
+def _(docstring):
+    return json.loads(docstring)
+
+
+@given("the cheeses", target_fixture="cheeses")
+def _(datatable):
+    return [row[0] for row in datatable[1:]]
+
+
+@given(parsers.parse("I have {count:d} cucumbers"), target_fixture="count")
+def _(count):
+    return count
+
+
+@then("I have a bar")
+def _(bar):
+    assert bar == "bar"
+
+
+@then(parsers.parse('the payload name is "{name}"'))
+def _(payload, name):
+    assert payload["name"] == name
+
+
+@then("I have gouda and edam")
+def _(cheeses):
+    assert cheeses == ["gouda", "edam"]
+
+
+@then(parsers.parse("I have {count:d} cucumbers left"))
+def _(count):
+    assert count > 0
+"""
+
+MARKDOWN_FEATURE = """\
+`@bdd`
+# Feature: Markdown with Gherkin
+
+This prose documents the behaviour and is not part of any scenario.
+
+| term | meaning          |
+| ---- | ---------------- |
+| bar  | something to own |
+
+```mermaid
+flowchart LR
+  given --> then
+```
+
+## Background:
+
+* Given I have a bar
+
+`@slow`
+## Scenario: Tagged scenario
+
+Prose between the heading and the first step.
+
+* Then I have a bar
+
+## Scenario: Doc string argument
+
+* Given the payload
+  ```json
+  {"name": "cucumber"}
+  ```
+* Then the payload name is "cucumber"
+
+## Scenario: Data table argument
+
+* Given the cheeses
+  | name  |
+  | gouda |
+  | edam  |
+* Then I have gouda and edam
+
+## Scenario Outline: Examples table
+
+* Given I have <count> cucumbers
+* Then I have <count> cucumbers left
+
+### Examples:
+
+  | count |
+  | ----- |
+  | 1     |
+  | 2     |
+"""
+
+
+@requires_markdown
+def test_scenarios_collect_markdown_feature_files(pytester):
+    """`scenarios()` collects `.feature.md` files beside `.feature` files and ignores other Markdown."""
+    pytester.makeconftest(STEPS)
+    features = pytester.mkdir("features")
+    features.joinpath("plain.feature").write_text(
+        textwrap.dedent(
+            """\
+            Feature: Plain Gherkin
+                Scenario: Plain scenario
+                    Given I have a bar
+                    Then I have a bar
+            """
+        ),
+        "utf-8",
+    )
+    features.joinpath("markdown.feature.md").write_text(MARKDOWN_FEATURE, "utf-8")
+    features.joinpath("README.md").write_text(
+        "# Feature: Not a feature file\n\n## Scenario: Must not be collected\n\n* Given I have a bar\n",
+        "utf-8",
+    )
+    pytester.makepyfile(
+        """
+        from pytest_bdd import scenarios
+
+        scenarios("features")
+        """
+    )
+
+    result = pytester.runpytest("-v")
+
+    result.assert_outcomes(passed=6)
+    result.stdout.no_fnmatch_line("*must_not_be_collected*")
+
+
+@requires_markdown
+def test_markdown_tags_become_markers(pytester):
+    """Inline code tags above a heading are applied to the scenario as markers."""
+    pytester.makeini(
+        """
+        [pytest]
+        markers =
+            bdd
+            slow
+        """
+    )
+    pytester.makeconftest(STEPS)
+    pytester.path.joinpath("markdown.feature.md").write_text(MARKDOWN_FEATURE, "utf-8")
+    pytester.makepyfile(
+        """
+        from pytest_bdd import scenarios
+
+        scenarios("markdown.feature.md")
+        """
+    )
+
+    result = pytester.runpytest("-v", "-m", "slow")
+
+    result.assert_outcomes(passed=1, deselected=4)
+    result.stdout.fnmatch_lines(["*test_tagged_scenario PASSED*"])
+
+
+@requires_markdown
+def test_scenario_decorator_binds_markdown_scenario(pytester):
+    """`@scenario` binds a scenario from a `.feature.md` file by name."""
+    pytester.makeconftest(STEPS)
+    pytester.path.joinpath("markdown.feature.md").write_text(MARKDOWN_FEATURE, "utf-8")
+    pytester.makepyfile(
+        """
+        from pytest_bdd import scenario
+
+        @scenario("markdown.feature.md", "Doc string argument")
+        def test_doc_string():
+            pass
+        """
+    )
+
+    result = pytester.runpytest()
+
+    result.assert_outcomes(passed=1)
+
+
+@requires_markdown
+def test_markdown_parse_error_reports_line(pytester):
+    """A Markdown with Gherkin syntax error reports the offending line."""
+    pytester.path.joinpath("broken.feature.md").write_text(
+        "# Feature: Broken\n\n`@orphan`\n* Given I have a bar\n",
+        "utf-8",
+    )
+    pytester.makepyfile(
+        """
+        from pytest_bdd import scenarios
+
+        scenarios("broken.feature.md")
+        """
+    )
+
+    result = pytester.runpytest()
+
+    result.assert_outcomes(errors=1)
+    result.stdout.fnmatch_lines(["*Line number: 4*", "*Line: * Given I have a bar*"])
+
+
+@pytest.mark.skipif(MARKDOWN_SUPPORTED, reason="gherkin-official supports Markdown with Gherkin")
+def test_markdown_requires_recent_gherkin(pytester):
+    """Older gherkin-official releases are rejected with an actionable error."""
+    pytester.path.joinpath("markdown.feature.md").write_text(MARKDOWN_FEATURE, "utf-8")
+    pytester.makepyfile(
+        """
+        from pytest_bdd import scenarios
+
+        scenarios("markdown.feature.md")
+        """
+    )
+
+    result = pytester.runpytest()
+
+    result.assert_outcomes(errors=1)
+    result.stdout.fnmatch_lines(["*Markdown with Gherkin requires gherkin-official>=32.2.0*"])